perf: skip discarded syntax AST rebuild

ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java

@@ -17,6 +17,7 @@
 import org.suikasoft.jOptions.Datakey.KeyFactory;
 import org.suikasoft.jOptions.Interfaces.DataStore;
 import pt.up.fe.specs.clang.ClangAstKeys;
+import pt.up.fe.specs.clang.ClangFiles;
 import pt.up.fe.specs.clang.ClangResources;
 import pt.up.fe.specs.clang.dumper.ClangAstData;
 import pt.up.fe.specs.clang.dumper.ClangAstDumper;
@@ -70,6 +71,9 @@ public class ParallelCodeParser extends CodeParser {
     public static final DataKey<Boolean> CONTINUE_ON_PARSING_ERRORS = KeyFactory.bool("continueOnParsingErrors")
             .setLabel("Ignores parsing errors in C/C++ source code");
 
+    public static final DataKey<Boolean> SYNTAX_ONLY = KeyFactory.bool("syntaxOnly")
+            .setLabel("Runs the compiler/dumper pipeline only to validate syntax, without decoding the AST");
+
     // public static final DataKey<Integer> SYSTEM_INCLUDES_THRESHOLD = KeyFactory.integer("systemIncludesThreshold", 1)
     // .setLabel("Number of threads to use for parallel parsing");
 
@@ -86,6 +90,9 @@ public App parse(List<File> inputSources, List<String> compilerOptions, ClavaCon
         Map<String, File> allSources = SpecsIo.getFileMap(allSourceFolders, SourceType.getPermittedExtensions());
 
         ConcurrentLinkedQueue<String> clangDump = new ConcurrentLinkedQueue<>();
+        ConcurrentLinkedQueue<String> syntaxErrors = new ConcurrentLinkedQueue<>();
+
+        boolean syntaxOnly = get(SYNTAX_ONLY);
 
         DataStore options = ClangAstKeys.toDataStore(compilerOptions);
 
@@ -139,8 +146,7 @@ public App parse(List<File> inputSources, List<String> compilerOptions, ClavaCon
 
             Future<ClangAstData> tUnit = executor
                     .submit(() -> parseSource(source, id, standard, options, clangDump,
-                            counter, parsingFolder, clangFiles.clangExecutable(), clangFiles.builtinIncludes(),
-                            clangFiles.systemResourceDir()));
+                            counter, parsingFolder, clangFiles, syntaxErrors));
 
             futureTUnits.add(tUnit);
 
@@ -159,6 +165,10 @@ public App parse(List<File> inputSources, List<String> compilerOptions, ClavaCon
                 var parserData = SpecsSystem.get(future);
                 clangParserResults.add(parserData);
             } catch (Exception e) {
+                if (syntaxOnly) {
+                    throw new RuntimeException("Error while validating syntax of file '" + sources.get(i) + "'", e);
+                }
+
                 SpecsLogs.warn("Could not parse file '" + sources.get(i) + "', will be ignored", e);
                 ignoredFiles.add(sources.get(i));
                 continue;
@@ -177,6 +187,16 @@ public App parse(List<File> inputSources, List<String> compilerOptions, ClavaCon
         // Delete temporary folder
         SpecsIo.deleteFolder(parsingFolder);
 
+        // No AST was decoded, just report syntax validation errors
+        if (syntaxOnly) {
+            List<String> validationErrors = new ArrayList<>(syntaxErrors);
+            if (!validationErrors.isEmpty() && !get(CONTINUE_ON_PARSING_ERRORS)) {
+                throw new ClavaParserException(validationErrors, clangFiles);
+            }
+
+            return null;
+        }
+
         // List<TranslationUnit> tUnits = SpecsCollections.getStream(allSources.keySet(), get(PARALLEL_PARSING))
         // .map(sourceFile -> parseSource(new File(sourceFile), standard, options, clangDump,
         // counter, parsingFolder))
@@ -359,7 +379,7 @@ private Standard getStandard(Collection<File> sources, DataStore options) {
 
     private ClangAstData parseSource(File sourceFile, String id, Standard standard, DataStore options,
                                      ConcurrentLinkedQueue<String> clangDump, ParallelProgressCounter counter, File parsingFolder,
-                                     File clangExecutable, List<String> builtinIncludes, File systemResourceDir) {
+                                     ClangFiles clangFiles, ConcurrentLinkedQueue<String> syntaxErrors) {
 
         // ConcurrentLinkedQueue<String> clangDump, ConcurrentLinkedQueue<File> workingFolders) {
 
@@ -370,15 +390,25 @@ private ClangAstData parseSource(File sourceFile, String id, Standard standard,
         // Only show output of console after parsing is done, when using parallel parsing
         boolean streamConsoleOutput = !get(PARALLEL_PARSING);
 
-        ClangAstDumper clangParser = new ClangAstDumper(streamConsoleOutput, clangExecutable, builtinIncludes,
-                systemResourceDir, this)
+        ClangAstDumper clangParser = new ClangAstDumper(streamConsoleOutput, clangFiles.clangExecutable(),
+                clangFiles.builtinIncludes(), clangFiles.systemResourceDir(), this)
                 .setBaseFolder(parsingFolder)
                 .setSystemIncludesThreshold(get(SYSTEM_INCLUDES_THRESHOLD));
 
         // .setUsePlatformLibc(get(ClangAstKeys.USE_PLATFORM_INCLUDES));
 
         counter.print(sourceFile);
-        // ClavaLog.info("Parsing '" + sourceFile.getAbsolutePath() + "'");
+
+        // Run the same clang invocation, discard dumper output
+        if (get(SYNTAX_ONLY)) {
+            String error = clangParser.validateSyntax(sourceFile, id, standard, options);
+            if (error != null) {
+                syntaxErrors.add(error);
+            }
+
+            return null;
+        }
+
         ClangAstData clangParserData = clangParser.parse(sourceFile, id, standard, options);
 
         if (get(SHOW_CLANG_DUMP)) {

ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java

@@ -41,7 +41,6 @@
 import java.io.File;
 import java.io.InputStream;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 
@@ -80,6 +79,8 @@ public static boolean usePlugin() {
     private File systemResourceDir;
     private int systemIncludesThreshold;
     private final ClangResources clangResources;
+    private boolean validationOnly;
+    private String lastValidationError;
 
     private final CodeParser parserConfig;
 
@@ -138,6 +139,25 @@ public ClangAstData parse(File sourceFile, String id, Standard standard, DataSto
         return parsePrivate(sourceFile, id, standard, config);
     }
 
+    /**
+     * Invokes Clang with the same arguments as parsing, while discarding dumper output.
+     *
+     * @return null if the syntax is valid, otherwise an error message
+     */
+    public String validateSyntax(File sourceFile, String id, Standard standard, DataStore config) {
+        if (config.get(ClangAstKeys.USES_CILK)) {
+            sourceFile = new CilkParser().prepareCilkFile(sourceFile);
+        }
+
+        validationOnly = true;
+        try {
+            parsePrivate(sourceFile, id, standard, config);
+            return lastValidationError;
+        } finally {
+            validationOnly = false;
+        }
+    }
+
     private ClangAstData parsePrivate(File sourceFile, String id, Standard standard, DataStore config) {
         ClavaLog.debug(() -> "Data store config for single file parser: " + config);
 
@@ -277,6 +297,11 @@ else if (SourceType.isHeader(sourceFile)) {
 
         ClavaLog.debug(() -> "Calling Clang AST Dumper: " + arguments);
 
+        if (validationOnly) {
+            lastValidationError = validateSyntax(arguments, sourceFile, id);
+            return null;
+        }
+
         ClangAstData parsedData = null;
         ProcessOutput<String, ClangAstData> output = null;
 
@@ -350,6 +375,34 @@ private void addCudaPathArgument(List<String> arguments, String cudaPath) {
         }
     }
 
+    private String validateSyntax(List<String> arguments, File sourceFile, String id) {
+        lastWorkingFolder = SpecsIo.mkdir(baseFolder, sourceFile.getName() + "_" + id);
+
+        var output = SpecsSystem.runProcess(arguments, lastWorkingFolder,
+                this::discardOutput,
+                inputStream -> processOutput(inputStream));
+
+        output.getOutputException().ifPresent(exception -> {
+            throw new RuntimeException("Exception while validating syntax", exception);
+        });
+
+        if (output.isError()) {
+            return "Syntax validation failed for '" + sourceFile.getAbsolutePath() + "':\n" + output.getStdErr();
+        }
+
+        return null;
+    }
+
+    private String discardOutput(InputStream inputStream) {
+        try (LineStream lines = LineStream.newInstance(inputStream, null)) {
+            while (lines.hasNextLine()) {
+                lines.nextLine();
+            }
+        }
+
+        return "";
+    }
+
     private String processOutput(InputStream inputStream) {
         StringBuilder output = new StringBuilder();
         try (LineStream lines = LineStream.newInstance(inputStream, null)) {

ClavaWeaver/src/pt/up/fe/specs/clava/weaver/CxxWeaver.java

@@ -575,26 +575,26 @@ public App createApp(List<File> sources, List<String> parserOptions, List<String
         ClavaLog.debug(() -> "Creating App using the following options: " + parserOptions);
         ClavaLog.debug(() -> "Creating App using the following extra options: " + extraOptions);
 
-        // Collect additional include folders
-        Set<String> sourceIncludeFolders = getSourceIncludes(sources);
-        ClavaLog.debug(() -> "Source include folders: " + sourceIncludeFolders);
+        CodeParser codeParser = newCodeParser();
 
-        // Add include folders to extra options
-        List<String> adaptedExtraOptions = new ArrayList<>(sourceIncludeFolders.size() + extraOptions.size());
-        adaptedExtraOptions.addAll(extraOptions);
-        sourceIncludeFolders.stream().map(includeFolder -> "-I" + includeFolder).forEach(adaptedExtraOptions::add);
+        List<String> allParserOptions = addSourceIncludes(sources, parserOptions, extraOptions);
+        App app = codeParser.parse(sources, allParserOptions, context);
 
-        List<String> allFiles = sources.stream().map(File::toString).collect(Collectors.toList());
+        // Set source paths of each TranslationUnit
+        app.setSources(currentBases);
+        app.setSourceFoldernames(sourceFoldernames);
 
-        // Sort filenames so that select order of files is consistent between OSes
-        Collections.sort(allFiles);
+        // Set external dependencies
+        app.getExternalDependencies()
+                .setDisableRemoteDependencies(this.dataStore.get(ClavaOptions.DISABLE_REMOTE_DEPENDENCIES));
 
-        boolean useCustomResources = this.dataStore.get(ClavaOptions.CUSTOM_RESOURCES);
+        return app;
+    }
 
+    private CodeParser newCodeParser() {
         CodeParser codeParser = CodeParser.newInstance();
 
-        // Setup code parser
-        codeParser.set(CodeParser.USE_CUSTOM_RESOURCES, useCustomResources);
+        codeParser.set(CodeParser.USE_CUSTOM_RESOURCES, this.dataStore.get(ClavaOptions.CUSTOM_RESOURCES));
         codeParser.set(CodeParser.CUDA_GPU_ARCH, this.dataStore.get(CodeParser.CUDA_GPU_ARCH));
         codeParser.set(CodeParser.CUDA_PATH, this.dataStore.get(CodeParser.CUDA_PATH));
         codeParser.set(ParallelCodeParser.PARALLEL_PARSING, this.dataStore.get(ParallelCodeParser.PARALLEL_PARSING));
@@ -606,20 +606,18 @@ public App createApp(List<File> sources, List<String> parserOptions, List<String
         codeParser.set(ClangAstKeys.LIBC_CXX_MODE, this.dataStore.get(ClangAstKeys.LIBC_CXX_MODE));
         codeParser.set(CodeParser.DUMPER_FOLDER, this.dataStore.get(CodeParser.DUMPER_FOLDER));
 
-        List<String> allParserOptions = new ArrayList<>(parserOptions.size() + adaptedExtraOptions.size());
-        allParserOptions.addAll(parserOptions);
-        allParserOptions.addAll(adaptedExtraOptions);
-        App app = codeParser.parse(SpecsCollections.map(allFiles, File::new), allParserOptions, context);
-
-        // Set source paths of each TranslationUnit
-        app.setSources(currentBases);
-        app.setSourceFoldernames(sourceFoldernames);
+        return codeParser;
+    }
 
-        // Set external dependencies
-        app.getExternalDependencies()
-                .setDisableRemoteDependencies(this.dataStore.get(ClavaOptions.DISABLE_REMOTE_DEPENDENCIES));
+    private List<String> addSourceIncludes(List<File> sources, List<String> parserOptions, List<String> extraOptions) {
+        Set<String> sourceIncludeFolders = getSourceIncludes(sources);
+        ClavaLog.debug(() -> "Source include folders: " + sourceIncludeFolders);
 
-        return app;
+        List<String> allParserOptions = new ArrayList<>(parserOptions.size() + sourceIncludeFolders.size() + extraOptions.size());
+        allParserOptions.addAll(parserOptions);
+        allParserOptions.addAll(extraOptions);
+        sourceIncludeFolders.stream().map(includeFolder -> "-I" + includeFolder).forEach(allParserOptions::add);
+        return allParserOptions;
     }
 
     private Set<String> getSourceIncludes(List<File> sources) {
@@ -1233,6 +1231,14 @@ public boolean rebuildAst(boolean update) {
                 .forEach(writtenFile -> rebuildBases.put(SpecsIo.getCanonicalFile(writtenFile), tempFolder));
 
         currentBases = rebuildBases;
+        if (!update) {
+            CodeParser codeParser = newCodeParser();
+            codeParser.set(ParallelCodeParser.SYNTAX_ONLY, true);
+            codeParser.parse(writtenFiles, addSourceIncludes(writtenFiles, rebuildOptions, extraOptions), context);
+            currentBases = previousBases;
+            return true;
+        }
+
         App rebuiltApp = createApp(writtenFiles, rebuildOptions, extraOptions);
 
         // Restore current bases