Add Glob module and portable FileSystem.glob

.changeset/portable-filesystem-glob.md

@@ -0,0 +1,11 @@
+---
+"effect": minor
+"@effect/platform-node-shared": minor
+"@effect/platform-deno": minor
+---
+
+Add experimental `Glob` path matching and a shared `FileSystem.glob` implementation for Node, Bun, and Deno.
+
+- Breaking: custom `FileSystem` implementations, including inputs to `FileSystem.make`, must provide `readDirectoryEntries`; mapping `readDirectory` names to `{ name, type: "Unknown" }` is supported. Wrappers must apply directory-listing changes to both methods.
+- `FileSystem.make` now derives `glob` instead of accepting it. Apply custom overrides afterward with `FileSystem.of({ ...fs, glob: customGlob })`.
+- Glob patterns use `/` separators and backslash escapes; Node extglobs no longer expand. Results use `/` separators and are relative to `root` unless the pattern is absolute. Deno wildcards now skip dotfiles unless explicitly named or `dot: true`; unreadable wildcard-reached directories are skipped, while read failures at the root or a leading literal directory propagate.

packages/effect/src/FileSystem.ts

@@ -17,6 +17,7 @@ import * as Cause from "./Cause.ts"
 import * as Context from "./Context.ts"
 import * as Effect from "./Effect.ts"
 import { pipe } from "./Function.ts"
+import * as Glob from "./internal/glob.ts"
 import * as Layer from "./Layer.ts"
 import * as Option from "./Option.ts"
 import { badArgument, type PlatformError, systemError } from "./PlatformError.ts"
@@ -130,13 +131,36 @@ export interface FileSystem {
     gid: number
   ) => Effect.Effect<void, PlatformError>
   /**
-   * Glob a directory.
+   * Finds paths under `root` that match a glob pattern.
+   *
+   * **Details**
+   *
+   * Patterns follow `Glob.make`: wildcards skip names that start with `.`
+   * unless the pattern segment starts with `.` or `dot` is `true`, and
+   * `dir/**` includes `dir` itself. Results are `/`-separated paths relative
+   * to `root` (or absolute for an absolute pattern), visited depth-first with
+   * each directory's entries sorted by name. `**` does not traverse symbolic
+   * links to directories, but a link named by another segment is followed.
+   * Paths that match an `exclude` pattern are dropped, and excluded
+   * directories are not read. Entries whose type cannot be determined, such
+   * as broken links, are treated as files.
+   *
+   * Read failures at the root or a leading literal directory propagate. Missing
+   * paths return an empty result; unreadable wildcard-reached directories are
+   * skipped.
+   *
+   * **Gotchas**
+   *
+   * Wildcard matching is case-sensitive. Leading literal segments use `stat`,
+   * so their case sensitivity follows the file system and results retain the
+   * pattern's spelling.
    */
   readonly glob: (
     pattern: string,
     options?: {
       readonly root?: string | undefined
       readonly exclude?: ReadonlyArray<string> | undefined
+      readonly dot?: boolean | undefined
     }
   ) => Effect.Effect<Array<string>, PlatformError>
   /**
@@ -247,6 +271,22 @@ export interface FileSystem {
       readonly recursive?: boolean | undefined
     }
   ) => Effect.Effect<Array<string>, PlatformError>
+  /**
+   * Lists a directory with the type of each entry.
+   *
+   * **Details**
+   *
+   * Returns the same bare names as `readDirectory(path)`, excluding `.` and
+   * `..`, in no particular order. Types describe entries, not symlink targets.
+   * `"Unknown"` is allowed when the listing cannot determine the type, and
+   * `glob` then falls back to `stat`.
+   *
+   * **Gotchas**
+   *
+   * Wrappers that change `readDirectory`, such as by prefixing paths, must
+   * apply the same change here for `glob` to list the correct directory.
+   */
+  readonly readDirectoryEntries: (path: string) => Effect.Effect<ReadonlyArray<DirectoryEntry>, PlatformError>
   /**
    * Read the contents of a file.
    */
@@ -478,6 +518,21 @@ export type OpenFlag =
  */
 export const FileSystem: Context.Service<FileSystem, FileSystem> = Context.Service("effect/FileSystem")
 
+/**
+ * A directory entry with its type, independent of any symlink target.
+ * `"Unknown"` means the listing could not determine the type.
+ *
+ * @see {@link FileSystem} `readDirectoryEntries`, which returns these entries
+ *
+ * @stability unstable
+ * @category models
+ * @since 4.0.0
+ */
+export interface DirectoryEntry {
+  readonly name: string
+  readonly type: File.Type
+}
+
 /**
  * Creates a FileSystem implementation from a partial implementation.
  *
@@ -490,8 +545,10 @@ export const FileSystem: Context.Service<FileSystem, FileSystem> = Context.Servi
  * **Details**
  *
  * This function takes a partial FileSystem implementation and automatically provides
- * default implementations for `exists`, `readFileString`, `stream`, `sink`, and
- * `writeFileString` methods based on the provided core methods.
+ * default implementations for `exists`, `glob`, `readFileString`, `stream`, `sink`,
+ * and `writeFileString` methods based on the provided core methods. `glob` walks
+ * the tree with `readDirectoryEntries`, using `stat` and `readLink` for entries
+ * whose type is `"Unknown"`.
  *
  * @see {@link makeNoop} for a testing stub that accepts method overrides without requiring a complete implementation
  * @see {@link layerNoop} for providing a no-op `FileSystem` as a `Layer` in tests
@@ -501,11 +558,12 @@ export const FileSystem: Context.Service<FileSystem, FileSystem> = Context.Servi
  * @since 4.0.0
  */
 export const make = (
-  impl: Omit<FileSystem, typeof TypeId | "exists" | "readFileString" | "stream" | "sink" | "writeFileString">
+  impl: Omit<FileSystem, typeof TypeId | "exists" | "glob" | "readFileString" | "stream" | "sink" | "writeFileString">
 ): FileSystem =>
   FileSystem.of({
     ...impl,
     [TypeId]: TypeId,
+    glob: Glob.make(impl),
     exists: (path) =>
       pipe(
         impl.access(path),
@@ -692,6 +750,9 @@ export const makeNoop = (fileSystem: Partial<FileSystem>): FileSystem =>
     readDirectory(path) {
       return Effect.fail(notFound("readDirectory", path))
     },
+    readDirectoryEntries(path) {
+      return Effect.fail(notFound("readDirectoryEntries", path))
+    },
     readFile(path) {
       return Effect.fail(notFound("readFile", path))
     },

packages/effect/src/Glob.ts

@@ -0,0 +1,157 @@
+/**
+ * Matches `/`-separated paths without accessing the file system, using the
+ * same pattern syntax as `FileSystem.glob`.
+ *
+ * @stability experimental
+ * @since 4.0.0
+ */
+import { format } from "./Formatter.ts"
+import { dual } from "./Function.ts"
+import { type Inspectable, NodeInspectSymbol } from "./Inspectable.ts"
+import * as internal from "./internal/glob.ts"
+import { type Pipeable, pipeArguments } from "./Pipeable.ts"
+
+const TypeId = "~effect/Glob"
+
+/**
+ * A compiled glob pattern.
+ *
+ * **Details**
+ *
+ * `pattern` is the source text and `dot` records whether wildcards match names
+ * that start with `.`. Match paths against it with {@link match}.
+ *
+ * @see {@link make} for compiling a pattern
+ *
+ * @stability experimental
+ * @category models
+ * @since 4.0.0
+ */
+export interface Glob extends Pipeable, Inspectable {
+  readonly [TypeId]: typeof TypeId
+  readonly pattern: string
+  readonly dot: boolean
+}
+
+interface GlobImpl extends Glob {
+  readonly compiled: internal.Compiled
+}
+
+const GlobProto = {
+  [TypeId]: TypeId,
+  toJSON(this: Glob) {
+    return { _id: "Glob", pattern: this.pattern, dot: this.dot }
+  },
+  toString(this: Glob) {
+    return `Glob(${format(this.toJSON())})`
+  },
+  [NodeInspectSymbol](this: Glob) {
+    return this.toJSON()
+  },
+  pipe() {
+    return pipeArguments(this, arguments)
+  }
+}
+
+/**
+ * Compiles a glob pattern into a reusable `Glob`.
+ *
+ * **Details**
+ *
+ * Patterns use `/` as the separator. `*` matches any run of characters within
+ * one path segment, `?` matches one character, and `[...]` matches one
+ * character from a set, with `!` or `^` negating it and POSIX classes such as
+ * `[[:digit:]]` allowed. `{a,b}` expands to each alternative and may nest. A
+ * segment that is exactly `**` matches zero or more whole segments. A trailing
+ * `/` matches only directories, and `\` makes the next character literal.
+ *
+ * **Gotchas**
+ *
+ * Wildcards and `**` skip names that start with `.` unless the pattern segment
+ * itself starts with `.` or `dot` is `true`; they never match `.` or `..`.
+ * `x/.` is the same as `x/`, and a `.` or `..` segment after `**` makes the
+ * pattern match nothing, as does the empty pattern. UNC patterns and paths
+ * such as `//server/share` are not supported and never match. Braces and
+ * commas inside `[...]` are class members. Extended glob syntax and leading
+ * `!` negation are not interpreted, but ordinary wildcards still apply within
+ * them (for example, `?(a)` matches `x(a)`). Malformed syntax, such as an
+ * unclosed `[`, matches literally. Braces also stay literal if expansion would
+ * exceed 10,000 alternatives.
+ *
+ * **Example** (Matching source files)
+ *
+ * ```ts import.meta.vitest
+ * import { Glob } from "effect"
+ *
+ * const sources = Glob.make("src/**\/*.{ts,tsx}")
+ *
+ * Glob.match(sources, "src/index.ts") // => true
+ * Glob.match(sources, "src/ui/Button.tsx") // => true
+ * Glob.match(sources, "src/.cache/index.ts") // => false
+ * Glob.match(sources, "test/index.ts") // => false
+ * ```
+ *
+ * @stability experimental
+ * @category constructors
+ * @since 4.0.0
+ */
+export const make = (pattern: string, options?: {
+  /** Whether wildcards and `**` match names that start with `.`. */
+  readonly dot?: boolean | undefined
+}): Glob => {
+  const dot = options?.dot ?? false
+  const self = Object.create(GlobProto)
+  self.pattern = pattern
+  self.dot = dot
+  self.compiled = internal.compile([pattern], dot)
+  return self
+}
+
+/**
+ * Tests whether a `/`-separated path matches a `Glob`.
+ *
+ * **Details**
+ *
+ * Empty segments and `./` segments in the path are ignored. A trailing `/`
+ * marks the path as a directory, which a pattern ending in `/` requires.
+ *
+ * **Example** (Filtering paths)
+ *
+ * ```ts import.meta.vitest
+ * import { Glob, pipe } from "effect"
+ *
+ * const tests = Glob.make("**\/*.test.ts")
+ * const paths = ["src/a.ts", "src/a.test.ts", "test/b.test.ts"]
+ *
+ * paths.filter((path) => Glob.match(tests, path)) // => ["src/a.test.ts", "test/b.test.ts"]
+ * pipe(tests, Glob.match("./src/a.test.ts")) // => true
+ * ```
+ *
+ * @stability experimental
+ * @category predicates
+ * @since 4.0.0
+ */
+export const match: {
+  (path: string): (self: Glob) => boolean
+  (self: Glob, path: string): boolean
+} = dual(2, (self: Glob, path: string): boolean => internal.match((self as GlobImpl).compiled, path))
+
+/**
+ * Escapes glob syntax so a string matches only itself.
+ *
+ * **Example** (Matching a literal file name)
+ *
+ * ```ts import.meta.vitest
+ * import { Glob } from "effect"
+ *
+ * const name = "report[1].txt"
+ *
+ * Glob.match(Glob.make(name), name) // => false
+ * Glob.match(Glob.make(Glob.escape(name)), name) // => true
+ * ```
+ *
+ * @stability experimental
+ * @category utils
+ * @since 4.0.0
+ */
+export const escape = (input: string): string => input.replace(/[\\*?[\]{}]/g, "\\$&")

packages/effect/src/index.ts

@@ -284,6 +284,12 @@ export * as Formatter from "./Formatter.ts"
  */
 export * as Function from "./Function.ts"
 
+/**
+ * @stability experimental
+ * @since 4.0.0
+ */
+export * as Glob from "./Glob.ts"
+
 /**
  * @stability unstable
  * @since 4.0.0