feat: delete stale generated files listed by an output manifest

README.md

@@ -84,7 +84,8 @@ A relative `--config` value is resolved against the directory the command is
 run in, while the relative paths inside the file stay relative to the file's own
 directory. `generate` exits with status `0` on success, and with status `1`
 after printing one diagnostic for invalid configuration, an unreadable source, a
-query it cannot compile, or an output directory or file it cannot write.
+query it cannot compile, an output directory or file it cannot write, a stale
+generated file it cannot delete, or an output manifest it cannot read or write.
 
 ## Getting started
 

docs/configuration.md

@@ -288,15 +288,18 @@ sqlcj: Generated file paths for repositories 'UserData' and 'Userdata' differ on
 ## Diagnostics
 
 Invalid configuration, unreadable SQL sources, and output that cannot be written
-make `sqlcj generate` print a single message on standard error and exit with a
-non-zero status. The message names the configuration file and, when available,
-the offending field, value, source path, or output path:
+or cleaned up make `sqlcj generate` print a single message on standard error and
+exit with a non-zero status. The message names the configuration file and, when
+available, the offending field, value, source path, or output path:
 
 ```text
 sqlcj: Invalid configuration in /home/dev/project/sqlcj.yaml: 'java.package' value 'dev.class.generated' is not a valid Java package name
 sqlcj: Cannot read queries source: /home/dev/project/sql/missing.sql
 sqlcj: Cannot create output directory: /home/dev/project/target/generated-sources/sqlcj/dev/example/generated
 sqlcj: Cannot write generated file: /home/dev/project/target/generated-sources/sqlcj/dev/example/generated/UsersRepository.java
+sqlcj: Cannot read output manifest: /home/dev/project/target/generated-sources/sqlcj/sqlcj-manifest.txt
+sqlcj: Cannot delete stale generated file: /home/dev/project/target/generated-sources/sqlcj/dev/example/generated/OrdersRepository.java
+sqlcj: Cannot write output manifest: /home/dev/project/target/generated-sources/sqlcj/sqlcj-manifest.txt
 ```
 
 ## Generated Output and Failures
@@ -316,8 +319,22 @@ A filesystem failure while writing ends the run with one diagnostic naming the
 output directory or generated file it could not write and exit status `1`. The
 files the run had already written stay in place.
 
-sqlcj also never deletes a generated file that the current run did not produce,
-so a renamed or removed configuration entry leaves its previous repository
-behind. Generate into a
-build-owned directory and let the build's clean step remove stale output, as the
-[Quickstart](quickstart.md) does.
+A successful run records what it generated in `sqlcj-manifest.txt` inside
+`java.out`. The manifest is a UTF-8 text file listing every generated file as a
+path relative to `java.out`, with `/` between its name elements, sorted, one per
+line. The next successful run deletes the files the previous manifest listed
+that it did not generate itself, so a renamed or removed configuration entry
+leaves no stale repository behind.
+
+Cleanup is deliberately narrow:
+
+- A file no previous manifest listed is never deleted, so a hand-written file in
+  the generated package survives, and so does output written before the first
+  manifest existed.
+- No directory is deleted, so a package directory stays in place after the
+  repository inside it is removed.
+- Changing `java.out` leaves the previous directory and its files untouched,
+  because cleanup only reads the manifest of the directory it generates into.
+- A run that fails while writing a generated file, deleting a stale one, or
+  writing the manifest leaves the previous manifest in place, so the next
+  successful run cleans up the files it still lists.

docs/quickstart.md

@@ -410,10 +410,11 @@ mvn compile
 mvn exec:java
 ```
 
-- `mvn clean` deletes `target`, including previously generated sources. sqlcj
-  writes and overwrites its own files but never deletes a file it did not just
-  generate, so a query that was renamed or removed would otherwise leave a stale
-  class behind that still compiles.
+- `mvn clean` deletes `target`, including previously generated sources and the
+  classes compiled from them. sqlcj writes and overwrites its own files and
+  deletes a repository its previous run recorded that the current run no longer
+  generates, but it never deletes a compiled class, so a query that was renamed
+  or removed would otherwise leave a stale class behind that still compiles.
 - `sqlcj generate` is a separate command. It is not bound to the Maven
   lifecycle, so it must run after `clean` and before `compile`. The commands
   above run from the project root, because `generate` reads `sqlcj.yaml` from

sqlcj-cli/src/main/java/dev/sqlcj/compiler/SqlcjCompiler.java

@@ -8,6 +8,7 @@
 import dev.sqlcj.generator.GeneratedFile;
 import dev.sqlcj.generator.JavaCodeGenerator;
 import dev.sqlcj.io.GeneratedFileWriter;
+import dev.sqlcj.io.GeneratedOutputManifest;
 import dev.sqlcj.parser.Query;
 import dev.sqlcj.schema.Schema;
 import dev.sqlcj.schema.parser.DefaultSchemaParser;
@@ -29,12 +30,18 @@ public final class SqlcjCompiler {
     private final SqlParser sqlParser = new SqlParser();
     private final QueryAnalyzer queryAnalyzer = new QueryAnalyzer();
     private final GeneratedFileWriter generatedFileWriter = new GeneratedFileWriter();
+    private final GeneratedOutputManifest generatedOutputManifest = new GeneratedOutputManifest();
     private final SchemaParser schemaParser = new DefaultSchemaParser();
 
     /**
      * Compiles every configured entry before any generated file is written, so
      * a failure in a later source cannot leave a mixture of previously
      * generated and newly generated output.
+     *
+     * <p>The manifest of the previous run is read before the first write, and
+     * the files it lists that this run did not produce are deleted before the
+     * new manifest replaces it, so a failed write or cleanup keeps the previous
+     * manifest for the next run.
      */
     public void compile(Config config) {
         List<Source> sources = sourceLoader.load(config);
@@ -45,9 +52,19 @@ public void compile(Config config) {
 
         Path outputDirectory = Path.of(config.java().out());
 
+        List<String> previousEntries = readManifest(outputDirectory);
+
         for (GeneratedFile file : files) {
             write(file, outputDirectory);
         }
+
+        List<Path> generatedPaths = files.stream()
+            .map(GeneratedFile::path)
+            .toList();
+
+        deleteStale(outputDirectory, previousEntries, generatedPaths);
+
+        writeManifest(outputDirectory, generatedPaths);
     }
 
     /** Generates one repository per configured query group. */
@@ -201,6 +218,34 @@ private void write(GeneratedFile file, Path outputDirectory) {
         }
     }
 
+    private List<String> readManifest(Path outputDirectory) {
+        try {
+            return generatedOutputManifest.read(outputDirectory);
+        } catch (IOException e) {
+            throw new CompilationException(e.getMessage(), e);
+        }
+    }
+
+    private void deleteStale(
+        Path outputDirectory,
+        List<String> previousEntries,
+        List<Path> generatedPaths
+    ) {
+        try {
+            generatedOutputManifest.deleteStale(outputDirectory, previousEntries, generatedPaths);
+        } catch (IOException e) {
+            throw new CompilationException(e.getMessage(), e);
+        }
+    }
+
+    private void writeManifest(Path outputDirectory, List<Path> generatedPaths) {
+        try {
+            generatedOutputManifest.write(outputDirectory, generatedPaths);
+        } catch (IOException e) {
+            throw new CompilationException(e.getMessage(), e);
+        }
+    }
+
     private record GeneratedRepository(String groupName, Path path) {
     }
 }

sqlcj-cli/src/main/java/dev/sqlcj/io/GeneratedOutputManifest.java

@@ -0,0 +1,200 @@
+package dev.sqlcj.io;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.InvalidPathException;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Records the generated files of one successful run in {@code java.out} and
+ * deletes the files a previous run recorded that this run did not produce.
+ *
+ * <p>Every entry is one generated file path relative to the output directory,
+ * with {@code /} between its name elements, so a manifest written on one
+ * filesystem reads the same on another.
+ */
+public final class GeneratedOutputManifest {
+
+    /** The manifest file name, inside the output directory. */
+    public static final String FILE_NAME = "sqlcj-manifest.txt";
+
+    /**
+     * The entries of the manifest a previous run wrote, or no entry when the
+     * output directory holds no manifest.
+     */
+    public List<String> read(Path outputDirectory) throws IOException {
+        Path manifestPath = manifestPath(outputDirectory);
+
+        if (!Files.exists(manifestPath)) {
+            return List.of();
+        }
+
+        try {
+            return List.copyOf(
+                Files.readAllLines(manifestPath, StandardCharsets.UTF_8)
+            );
+        } catch (IOException e) {
+            throw new IOException("Cannot read output manifest: " + manifestPath, e);
+        }
+    }
+
+    /**
+     * Deletes every generated file the previous manifest listed that this run
+     * did not produce, and nothing else. An entry that does not name a regular
+     * file inside the output directory is left alone.
+     */
+    public void deleteStale(
+        Path outputDirectory,
+        List<String> previousEntries,
+        List<Path> generatedPaths
+    ) throws IOException {
+        List<String> producedEntries = entries(generatedPaths);
+
+        for (String previousEntry : previousEntries) {
+            if (producedEntries.contains(previousEntry)) {
+                continue;
+            }
+
+            deleteStaleEntry(outputDirectory, previousEntry, generatedPaths);
+        }
+    }
+
+    /** Records the generated paths of one successful run, one entry per line. */
+    public void write(Path outputDirectory, List<Path> generatedPaths) throws IOException {
+        Path manifestPath = manifestPath(outputDirectory);
+
+        StringBuilder content = new StringBuilder();
+
+        entries(generatedPaths)
+            .stream()
+            .sorted()
+            .forEach(entry -> content.append(entry).append('\n'));
+
+        try {
+            Files.writeString(
+                manifestPath,
+                content,
+                StandardCharsets.UTF_8,
+                StandardOpenOption.CREATE,
+                StandardOpenOption.TRUNCATE_EXISTING
+            );
+        } catch (IOException e) {
+            throw new IOException("Cannot write output manifest: " + manifestPath, e);
+        }
+    }
+
+    /**
+     * Deletes one stale entry, skipping an entry that does not name a regular
+     * file inside the output directory and an entry that names a file this run
+     * wrote, which a rename differing only by case produces on a
+     * case-insensitive filesystem.
+     */
+    private void deleteStaleEntry(
+        Path outputDirectory,
+        String entry,
+        List<Path> generatedPaths
+    ) throws IOException {
+        Path relativePath = relativePath(entry);
+
+        if (relativePath == null) {
+            return;
+        }
+
+        Path file = outputDirectory.resolve(relativePath);
+
+        try {
+            if (!Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) {
+                return;
+            }
+
+            if (isGenerated(file, outputDirectory, generatedPaths)) {
+                return;
+            }
+
+            Files.delete(file);
+        } catch (IOException e) {
+            throw new IOException("Cannot delete stale generated file: " + file, e);
+        }
+    }
+
+    /**
+     * The relative path of one manifest entry, or {@code null} when the entry
+     * is not a normalized relative path that stays inside the output
+     * directory.
+     */
+    private Path relativePath(String entry) {
+        if (entry.isEmpty()) {
+            return null;
+        }
+
+        Path path;
+
+        try {
+            path = Path.of(entry);
+        } catch (InvalidPathException e) {
+            return null;
+        }
+
+        if (path.getRoot() != null) {
+            return null;
+        }
+
+        if (path.getName(0).toString().equals("..")) {
+            return null;
+        }
+
+        if (!entry.equals(entry(path.normalize()))) {
+            return null;
+        }
+
+        return path;
+    }
+
+    private boolean isGenerated(
+        Path file,
+        Path outputDirectory,
+        List<Path> generatedPaths
+    ) throws IOException {
+        for (Path generatedPath : generatedPaths) {
+            if (Files.isSameFile(file, outputDirectory.resolve(generatedPath))) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private List<String> entries(List<Path> generatedPaths) {
+        List<String> entries = new ArrayList<>(generatedPaths.size());
+
+        for (Path generatedPath : generatedPaths) {
+            entries.add(entry(generatedPath));
+        }
+
+        return entries;
+    }
+
+    /** One entry, joining the path's name elements with {@code /}. */
+    private String entry(Path path) {
+        StringBuilder entry = new StringBuilder();
+
+        for (Path name : path) {
+            if (!entry.isEmpty()) {
+                entry.append('/');
+            }
+
+            entry.append(name);
+        }
+
+        return entry.toString();
+    }
+
+    private Path manifestPath(Path outputDirectory) {
+        return outputDirectory.resolve(FILE_NAME);
+    }
+}