Merge pull request #41 from sqlcj-dev/feature/schema-files-and-migration-directories

feat: read schema from file lists and migration directories

Author: omer-cengel

docs/configuration.md

@@ -56,7 +56,7 @@ java:
 | `version` | string | Configuration contract version. Must be `"1"`. |
 | `sql` | list | Non-empty, ordered list of source entries. |
 | `sql[].name` | string | Identity of the query group. Names the generated repository. |
-| `sql[].schema` | string | Path to a file containing `CREATE TABLE` statements. |
+| `sql[].schema` | string or list | One path, or a non-empty ordered list of paths, to a file containing `CREATE TABLE` statements or to a directory of `.sql` migration files. |
 | `sql[].queries` | string | Path to a file containing named queries. |
 | `java` | mapping | Java generation settings. |
 | `java.package` | string | Package of the generated Java classes. |
@@ -70,7 +70,7 @@ against that entry's own schema, and query order inside a query file is
 preserved.
 
 Schemas are not shared or merged between entries. A query can only use tables
-declared in the schema file of its own entry.
+declared in the schema sources of its own entry.
 
 One entry generates exactly one repository containing every query of its query
 source, in declared query order. Query names are therefore scoped to their
@@ -97,6 +97,64 @@ sqlcj: Invalid configuration in /home/dev/project/sqlcj.yaml: 'sql[0].name' valu
 sqlcj does not derive the name from a table or a file name. A query file may
 join or write several tables, so the group boundary is declared, not guessed.
 
+### `sql[].schema`
+
+`sql[].schema` is either one path or a non-empty ordered list of paths:
+
+```yaml
+sql:
+  - name: Author
+    schema: sql/authors/schema.sql
+    queries: sql/authors/queries.sql
+  - name: Order
+    schema:
+      - sql/orders/baseline.sql
+      - sql/orders/migrations
+    queries: sql/orders/queries.sql
+```
+
+A listed path is either a file, which contributes itself, or a directory, which
+contributes the regular files in it whose name ends in `.sql`. A directory is
+read non-recursively, and the name match is case-sensitive: a subdirectory and a
+file named anything else, including `schema.SQL`, are ignored. The listed paths
+keep their declared order and each directory is expanded in place.
+
+A directory is ordered like a Flyway migration directory:
+
+1. A versioned migration file, named `V<version>__<description>.sql` with
+   version parts separated by `.` or `_`, comes first. Two versions are compared
+   part by part as numbers, counting a missing trailing part as zero, so
+   `V1__init.sql`, `V1_1__add_index.sql`, `V2__add_orders.sql`, and
+   `V10__add_totals.sql` are ordered exactly that way.
+2. Every other `.sql` file follows, ordered by file name.
+3. An undo file, named `U<version>__<description>.sql`, is ignored.
+
+The order never depends on how the filesystem lists the directory. sqlcj only
+reads the files, so a Flyway placeholder, a configured file-name prefix, and a
+repeatable migration have no meaning of their own.
+
+Each schema file is parsed on its own, and the tables of the entry follow file
+order, so a schema failure names the file that contains it:
+
+```text
+sqlcj: Invalid schema source /home/dev/project/sql/orders/migrations/V2__add_orders.sql: Unsupported schema statement: Alter
+```
+
+A directory that contributes no file, and two versioned files of one directory
+that declare the same version, are invalid schema input:
+
+```text
+sqlcj: Invalid schema source /home/dev/project/sql/orders/migrations: directory contains no .sql files
+sqlcj: Invalid schema source /home/dev/project/sql/orders/migrations: duplicate migration version in V1_0__add_index.sql and V1__init.sql
+```
+
+A configured path that cannot be read, whether it is a file or a directory,
+ends the run the same way:
+
+```text
+sqlcj: Cannot read schema source: /home/dev/project/sql/orders/migrations: Permission denied
+```
+
 ### `java.package`
 
 `java.package` must be a dot-separated sequence of valid, non-keyword Java
@@ -115,14 +173,14 @@ generated/dev/example/generated/AuthorRepository.java
 - String values must not be blank.
 - Unknown fields are rejected.
 - Wrong-typed fields are rejected, including an unquoted numeric `version`.
-- A missing or unsupported `version`, a missing section, an empty `sql` list, a
-  null `sql` entry, a blank value, an invalid `sql[].name`, and an invalid
-  `java.package` are all invalid configuration.
+- A missing or unsupported `version`, a missing section, an empty `sql` list, an
+  empty `sql[].schema` list, a null `sql` entry, a blank value, an invalid
+  `sql[].name`, and an invalid `java.package` are all invalid configuration.
 
 ## Path Resolution
 
-A relative `schema`, `queries`, or `java.out` path is resolved against the
-directory that contains the configuration file, whether that file is the
+A relative `queries`, `java.out`, or listed `schema` path is resolved against
+the directory that contains the configuration file, whether that file is the
 default `sqlcj.yaml` or one named by `--config`, and not against the process
 working directory at a later point in time. An absolute path is used as-is.
 

docs/postgresql.md

@@ -87,12 +87,15 @@ test-scoped dependencies of the build.
 
 ## Schema Snapshot Input
 
-A `sql[].schema` file is an explicit snapshot of the tables a query may use.
+`sql[].schema` is an explicit snapshot of the tables a query may use. It is one
+file, a list of files, or a directory of `.sql` migration files; see
+[`sql[].schema`](configuration.md#sqlschema) for the accepted forms and the
+order the files are read in.
 
-- Only `CREATE TABLE` statements are accepted. Any other statement in the file
-  is rejected.
+- Only `CREATE TABLE` statements are accepted. Any other statement in a schema
+  file is rejected.
 - A file may contain several `CREATE TABLE` statements, and table order is
-  preserved.
+  preserved across the schema files of one entry.
 - SQL identifier delimiters are removed for the parsed model, so the table
   `"user data"` is modeled as `user data` and the column `"user id"` is modeled
   as `user id`.

sqlcj-cli/src/main/java/dev/sqlcj/compiler/DefaultSourceLoader.java

@@ -10,12 +10,29 @@
 import dev.sqlcj.parser.QueryParser;
 
 import java.io.IOException;
+import java.math.BigInteger;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
 
 public final class DefaultSourceLoader implements SourceLoader {
 
+    private static final String SQL_SUFFIX = ".sql";
+
+    /** A Flyway versioned migration file, such as {@code V1_1__add_index.sql}. */
+    private static final Pattern VERSIONED = Pattern.compile("V(\\d+([._]\\d+)*)__.*\\.sql");
+
+    /** A Flyway undo migration file, such as {@code U1_1__add_index.sql}. */
+    private static final Pattern UNDO = Pattern.compile("U(\\d+([._]\\d+)*)__.*\\.sql");
+
+    /** One part of a migration version, separated by {@code .} or {@code _}. */
+    private static final Pattern VERSION_SEPARATOR = Pattern.compile("[._]");
+
     private final FileLoader fileLoader = new DefaultFileLoader();
     private final QueryParser queryParser = new DefaultQueryParser();
 
@@ -29,17 +46,15 @@ public List<Source> load(Config config) {
         List<Source> sources = new ArrayList<>();
 
         for (SqlConfig sqlConfig : config.sql()) {
-            Path schemaPath = Path.of(sqlConfig.schema());
             Path queriesPath = Path.of(sqlConfig.queries());
 
-            String schema = read(schemaPath, "schema");
+            List<SchemaFile> schemaFiles = loadSchema(sqlConfig.schema());
             List<Query> queries = parse(read(queriesPath, "queries"), queriesPath);
 
             sources.add(
                 new Source(
                     sqlConfig.name(),
-                    schemaPath,
-                    schema,
+                    schemaFiles,
                     queriesPath,
                     queries
                 )
@@ -49,6 +64,130 @@ public List<Source> load(Config config) {
         return sources;
     }
 
+    /**
+     * Loads the schema files of one entry in the order they apply: the
+     * configured paths in declared order, each directory expanded in place into
+     * its own migration files.
+     */
+    private List<SchemaFile> loadSchema(List<String> configuredPaths) {
+        List<SchemaFile> schemaFiles = new ArrayList<>();
+
+        for (String configuredPath : configuredPaths) {
+            Path path = Path.of(configuredPath);
+
+            if (Files.isDirectory(path)) {
+                for (Path file : migrationFiles(path)) {
+                    schemaFiles.add(new SchemaFile(file, read(file, "schema")));
+                }
+            } else {
+                schemaFiles.add(new SchemaFile(path, read(path, "schema")));
+            }
+        }
+
+        return List.copyOf(schemaFiles);
+    }
+
+    /**
+     * The {@code .sql} files one directory contributes, in Flyway migration
+     * order: versioned files by version, then every other file by name. The
+     * order never depends on how the filesystem lists the directory.
+     */
+    private List<Path> migrationFiles(Path directory) {
+        List<Path> candidates = list(directory).stream()
+            .filter(Files::isRegularFile)
+            .filter(path -> fileName(path).endsWith(SQL_SUFFIX))
+            .filter(path -> !UNDO.matcher(fileName(path)).matches())
+            .sorted(Comparator.comparing(this::fileName))
+            .toList();
+
+        if (candidates.isEmpty()) {
+            throw new CompilationException(
+                "Invalid schema source %s: directory contains no .sql files"
+                    .formatted(directory)
+            );
+        }
+
+        List<Path> versioned = candidates.stream()
+            .filter(path -> version(path) != null)
+            .sorted(Comparator.comparing(this::version, this::compareVersions))
+            .toList();
+
+        checkDuplicateVersions(directory, versioned);
+
+        List<Path> files = new ArrayList<>(versioned);
+
+        candidates.stream()
+            .filter(path -> version(path) == null)
+            .forEach(files::add);
+
+        return files;
+    }
+
+    private List<Path> list(Path directory) {
+        try (Stream<Path> entries = Files.list(directory)) {
+            return entries.toList();
+        } catch (IOException e) {
+            throw new CompilationException(
+                "Cannot read schema source: %s: %s"
+                    .formatted(directory, FileSystemReason.of(e)),
+                e
+            );
+        }
+    }
+
+    /** The declared version of a versioned migration file, or {@code null}. */
+    private String version(Path path) {
+        Matcher matcher = VERSIONED.matcher(fileName(path));
+
+        return matcher.matches() ? matcher.group(1) : null;
+    }
+
+    /**
+     * Compares two migration versions part by part as numbers, counting a
+     * missing trailing part as zero, so that {@code 1} precedes {@code 1.1} and
+     * {@code 2} precedes {@code 10}.
+     */
+    private int compareVersions(String left, String right) {
+        String[] leftParts = VERSION_SEPARATOR.split(left);
+        String[] rightParts = VERSION_SEPARATOR.split(right);
+
+        for (int i = 0; i < Math.max(leftParts.length, rightParts.length); i++) {
+            int comparison = part(leftParts, i).compareTo(part(rightParts, i));
+
+            if (comparison != 0) {
+                return comparison;
+            }
+        }
+
+        return 0;
+    }
+
+    private BigInteger part(String[] parts, int index) {
+        return index < parts.length ? new BigInteger(parts[index]) : BigInteger.ZERO;
+    }
+
+    /**
+     * Rejects two versioned files of one directory that declare the same
+     * version, because neither order of the two is the applied order.
+     */
+    private void checkDuplicateVersions(Path directory, List<Path> versioned) {
+        for (int i = 1; i < versioned.size(); i++) {
+            Path previous = versioned.get(i - 1);
+            Path current = versioned.get(i);
+
+            if (compareVersions(version(previous), version(current)) == 0) {
+                throw new CompilationException(
+                    "Invalid schema source %s: duplicate migration version in %s and %s"
+                        .formatted(directory, fileName(previous), fileName(current))
+                );
+            }
+        }
+    }
+
+    private String fileName(Path path) {
+        return path.getFileName().toString();
+    }
+
     private String read(Path path, String kind) {
         try {
             return fileLoader.read(path);

sqlcj-cli/src/main/java/dev/sqlcj/compiler/SchemaFile.java

@@ -0,0 +1,13 @@
+package dev.sqlcj.compiler;
+
+import java.nio.file.Path;
+
+/**
+ * One loaded schema file and the path it was loaded from, which identifies it
+ * in a compilation diagnostic.
+ */
+public record SchemaFile(
+    Path path,
+    String sql
+) {
+}

sqlcj-cli/src/main/java/dev/sqlcj/compiler/Source.java

@@ -10,11 +10,11 @@
  * identify a source in a compilation diagnostic.
  *
  * @param name the configured group identity that names the generated repository
+ * @param schemaFiles the schema files of the entry, in the order they apply
  */
 public record Source(
     String name,
-    Path schemaPath,
-    String schema,
+    List<SchemaFile> schemaFiles,
     Path queriesPath,
     List<Query> queries
 ) {

sqlcj-cli/src/main/java/dev/sqlcj/compiler/SqlcjCompiler.java

@@ -11,6 +11,7 @@
 import dev.sqlcj.io.GeneratedOutputManifest;
 import dev.sqlcj.parser.Query;
 import dev.sqlcj.schema.Schema;
+import dev.sqlcj.schema.Table;
 import dev.sqlcj.schema.parser.DefaultSchemaParser;
 import dev.sqlcj.schema.parser.SchemaParser;
 import dev.sqlcj.sql.ParsedSql;
@@ -89,12 +90,26 @@ private List<GeneratedFile> generate(List<Source> sources, CodeGenerator codeGen
         return files;
     }
 
+    /**
+     * Parses the schema files of one entry in order into that entry's schema,
+     * so a table keeps the position of the file that declares it.
+     */
     private Schema parseSchema(Source source) {
+        List<Table> tables = new ArrayList<>();
+
+        for (SchemaFile schemaFile : source.schemaFiles()) {
+            tables.addAll(parseSchemaFile(schemaFile).tables());
+        }
+
+        return new Schema(tables);
+    }
+
+    private Schema parseSchemaFile(SchemaFile schemaFile) {
         try {
-            return schemaParser.parse(source.schema());
+            return schemaParser.parse(schemaFile.sql());
         } catch (RuntimeException e) {
             throw new CompilationException(
-                "Invalid schema source %s: %s".formatted(source.schemaPath(), reason(e)),
+                "Invalid schema source %s: %s".formatted(schemaFile.path(), reason(e)),
                 e
             );
         }