diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java index 91b7ca5b9..2118f8447 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/codeparser/ParallelCodeParser.java @@ -17,6 +17,7 @@ import org.suikasoft.jOptions.Datakey.KeyFactory; import org.suikasoft.jOptions.Interfaces.DataStore; import pt.up.fe.specs.clang.ClangAstKeys; +import pt.up.fe.specs.clang.ClangFiles; import pt.up.fe.specs.clang.ClangResources; import pt.up.fe.specs.clang.dumper.ClangAstData; import pt.up.fe.specs.clang.dumper.ClangAstDumper; @@ -70,6 +71,9 @@ public class ParallelCodeParser extends CodeParser { public static final DataKey CONTINUE_ON_PARSING_ERRORS = KeyFactory.bool("continueOnParsingErrors") .setLabel("Ignores parsing errors in C/C++ source code"); + public static final DataKey SYNTAX_ONLY = KeyFactory.bool("syntaxOnly") + .setLabel("Runs the compiler/dumper pipeline only to validate syntax, without decoding the AST"); + // public static final DataKey SYSTEM_INCLUDES_THRESHOLD = KeyFactory.integer("systemIncludesThreshold", 1) // .setLabel("Number of threads to use for parallel parsing"); @@ -86,6 +90,9 @@ public App parse(List inputSources, List compilerOptions, ClavaCon Map allSources = SpecsIo.getFileMap(allSourceFolders, SourceType.getPermittedExtensions()); ConcurrentLinkedQueue clangDump = new ConcurrentLinkedQueue<>(); + ConcurrentLinkedQueue syntaxErrors = new ConcurrentLinkedQueue<>(); + + boolean syntaxOnly = get(SYNTAX_ONLY); DataStore options = ClangAstKeys.toDataStore(compilerOptions); @@ -139,8 +146,7 @@ public App parse(List inputSources, List compilerOptions, ClavaCon Future tUnit = executor .submit(() -> parseSource(source, id, standard, options, clangDump, - counter, parsingFolder, clangFiles.clangExecutable(), clangFiles.builtinIncludes(), - clangFiles.systemResourceDir())); + counter, parsingFolder, clangFiles, syntaxErrors)); futureTUnits.add(tUnit); @@ -159,6 +165,10 @@ public App parse(List inputSources, List compilerOptions, ClavaCon var parserData = SpecsSystem.get(future); clangParserResults.add(parserData); } catch (Exception e) { + if (syntaxOnly) { + throw new RuntimeException("Error while validating syntax of file '" + sources.get(i) + "'", e); + } + SpecsLogs.warn("Could not parse file '" + sources.get(i) + "', will be ignored", e); ignoredFiles.add(sources.get(i)); continue; @@ -177,6 +187,16 @@ public App parse(List inputSources, List compilerOptions, ClavaCon // Delete temporary folder SpecsIo.deleteFolder(parsingFolder); + // No AST was decoded, just report syntax validation errors + if (syntaxOnly) { + List validationErrors = new ArrayList<>(syntaxErrors); + if (!validationErrors.isEmpty() && !get(CONTINUE_ON_PARSING_ERRORS)) { + throw new ClavaParserException(validationErrors, clangFiles); + } + + return null; + } + // List tUnits = SpecsCollections.getStream(allSources.keySet(), get(PARALLEL_PARSING)) // .map(sourceFile -> parseSource(new File(sourceFile), standard, options, clangDump, // counter, parsingFolder)) @@ -359,7 +379,7 @@ private Standard getStandard(Collection sources, DataStore options) { private ClangAstData parseSource(File sourceFile, String id, Standard standard, DataStore options, ConcurrentLinkedQueue clangDump, ParallelProgressCounter counter, File parsingFolder, - File clangExecutable, List builtinIncludes, File systemResourceDir) { + ClangFiles clangFiles, ConcurrentLinkedQueue syntaxErrors) { // ConcurrentLinkedQueue clangDump, ConcurrentLinkedQueue workingFolders) { @@ -370,15 +390,25 @@ private ClangAstData parseSource(File sourceFile, String id, Standard standard, // Only show output of console after parsing is done, when using parallel parsing boolean streamConsoleOutput = !get(PARALLEL_PARSING); - ClangAstDumper clangParser = new ClangAstDumper(streamConsoleOutput, clangExecutable, builtinIncludes, - systemResourceDir, this) + ClangAstDumper clangParser = new ClangAstDumper(streamConsoleOutput, clangFiles.clangExecutable(), + clangFiles.builtinIncludes(), clangFiles.systemResourceDir(), this) .setBaseFolder(parsingFolder) .setSystemIncludesThreshold(get(SYSTEM_INCLUDES_THRESHOLD)); // .setUsePlatformLibc(get(ClangAstKeys.USE_PLATFORM_INCLUDES)); counter.print(sourceFile); - // ClavaLog.info("Parsing '" + sourceFile.getAbsolutePath() + "'"); + + // Run the same clang invocation, discard dumper output + if (get(SYNTAX_ONLY)) { + String error = clangParser.validateSyntax(sourceFile, id, standard, options); + if (error != null) { + syntaxErrors.add(error); + } + + return null; + } + ClangAstData clangParserData = clangParser.parse(sourceFile, id, standard, options); if (get(SHOW_CLANG_DUMP)) { diff --git a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java index de800f155..fd2d6f4ef 100644 --- a/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java +++ b/ClangAstParser/src/pt/up/fe/specs/clang/dumper/ClangAstDumper.java @@ -41,7 +41,6 @@ import java.io.File; import java.io.InputStream; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -80,6 +79,8 @@ public static boolean usePlugin() { private File systemResourceDir; private int systemIncludesThreshold; private final ClangResources clangResources; + private boolean validationOnly; + private String lastValidationError; private final CodeParser parserConfig; @@ -138,6 +139,25 @@ public ClangAstData parse(File sourceFile, String id, Standard standard, DataSto return parsePrivate(sourceFile, id, standard, config); } + /** + * Invokes Clang with the same arguments as parsing, while discarding dumper output. + * + * @return null if the syntax is valid, otherwise an error message + */ + public String validateSyntax(File sourceFile, String id, Standard standard, DataStore config) { + if (config.get(ClangAstKeys.USES_CILK)) { + sourceFile = new CilkParser().prepareCilkFile(sourceFile); + } + + validationOnly = true; + try { + parsePrivate(sourceFile, id, standard, config); + return lastValidationError; + } finally { + validationOnly = false; + } + } + private ClangAstData parsePrivate(File sourceFile, String id, Standard standard, DataStore config) { ClavaLog.debug(() -> "Data store config for single file parser: " + config); @@ -277,6 +297,11 @@ else if (SourceType.isHeader(sourceFile)) { ClavaLog.debug(() -> "Calling Clang AST Dumper: " + arguments); + if (validationOnly) { + lastValidationError = validateSyntax(arguments, sourceFile, id); + return null; + } + ClangAstData parsedData = null; ProcessOutput output = null; @@ -350,6 +375,34 @@ private void addCudaPathArgument(List arguments, String cudaPath) { } } + private String validateSyntax(List arguments, File sourceFile, String id) { + lastWorkingFolder = SpecsIo.mkdir(baseFolder, sourceFile.getName() + "_" + id); + + var output = SpecsSystem.runProcess(arguments, lastWorkingFolder, + this::discardOutput, + inputStream -> processOutput(inputStream)); + + output.getOutputException().ifPresent(exception -> { + throw new RuntimeException("Exception while validating syntax", exception); + }); + + if (output.isError()) { + return "Syntax validation failed for '" + sourceFile.getAbsolutePath() + "':\n" + output.getStdErr(); + } + + return null; + } + + private String discardOutput(InputStream inputStream) { + try (LineStream lines = LineStream.newInstance(inputStream, null)) { + while (lines.hasNextLine()) { + lines.nextLine(); + } + } + + return ""; + } + private String processOutput(InputStream inputStream) { StringBuilder output = new StringBuilder(); try (LineStream lines = LineStream.newInstance(inputStream, null)) { diff --git a/ClavaWeaver/src/pt/up/fe/specs/clava/weaver/CxxWeaver.java b/ClavaWeaver/src/pt/up/fe/specs/clava/weaver/CxxWeaver.java index e76857769..210f92029 100644 --- a/ClavaWeaver/src/pt/up/fe/specs/clava/weaver/CxxWeaver.java +++ b/ClavaWeaver/src/pt/up/fe/specs/clava/weaver/CxxWeaver.java @@ -575,26 +575,26 @@ public App createApp(List sources, List parserOptions, List "Creating App using the following options: " + parserOptions); ClavaLog.debug(() -> "Creating App using the following extra options: " + extraOptions); - // Collect additional include folders - Set sourceIncludeFolders = getSourceIncludes(sources); - ClavaLog.debug(() -> "Source include folders: " + sourceIncludeFolders); + CodeParser codeParser = newCodeParser(); - // Add include folders to extra options - List adaptedExtraOptions = new ArrayList<>(sourceIncludeFolders.size() + extraOptions.size()); - adaptedExtraOptions.addAll(extraOptions); - sourceIncludeFolders.stream().map(includeFolder -> "-I" + includeFolder).forEach(adaptedExtraOptions::add); + List allParserOptions = addSourceIncludes(sources, parserOptions, extraOptions); + App app = codeParser.parse(sources, allParserOptions, context); - List allFiles = sources.stream().map(File::toString).collect(Collectors.toList()); + // Set source paths of each TranslationUnit + app.setSources(currentBases); + app.setSourceFoldernames(sourceFoldernames); - // Sort filenames so that select order of files is consistent between OSes - Collections.sort(allFiles); + // Set external dependencies + app.getExternalDependencies() + .setDisableRemoteDependencies(this.dataStore.get(ClavaOptions.DISABLE_REMOTE_DEPENDENCIES)); - boolean useCustomResources = this.dataStore.get(ClavaOptions.CUSTOM_RESOURCES); + return app; + } + private CodeParser newCodeParser() { CodeParser codeParser = CodeParser.newInstance(); - // Setup code parser - codeParser.set(CodeParser.USE_CUSTOM_RESOURCES, useCustomResources); + codeParser.set(CodeParser.USE_CUSTOM_RESOURCES, this.dataStore.get(ClavaOptions.CUSTOM_RESOURCES)); codeParser.set(CodeParser.CUDA_GPU_ARCH, this.dataStore.get(CodeParser.CUDA_GPU_ARCH)); codeParser.set(CodeParser.CUDA_PATH, this.dataStore.get(CodeParser.CUDA_PATH)); codeParser.set(ParallelCodeParser.PARALLEL_PARSING, this.dataStore.get(ParallelCodeParser.PARALLEL_PARSING)); @@ -606,20 +606,18 @@ public App createApp(List sources, List parserOptions, List allParserOptions = new ArrayList<>(parserOptions.size() + adaptedExtraOptions.size()); - allParserOptions.addAll(parserOptions); - allParserOptions.addAll(adaptedExtraOptions); - App app = codeParser.parse(SpecsCollections.map(allFiles, File::new), allParserOptions, context); - - // Set source paths of each TranslationUnit - app.setSources(currentBases); - app.setSourceFoldernames(sourceFoldernames); + return codeParser; + } - // Set external dependencies - app.getExternalDependencies() - .setDisableRemoteDependencies(this.dataStore.get(ClavaOptions.DISABLE_REMOTE_DEPENDENCIES)); + private List addSourceIncludes(List sources, List parserOptions, List extraOptions) { + Set sourceIncludeFolders = getSourceIncludes(sources); + ClavaLog.debug(() -> "Source include folders: " + sourceIncludeFolders); - return app; + List allParserOptions = new ArrayList<>(parserOptions.size() + sourceIncludeFolders.size() + extraOptions.size()); + allParserOptions.addAll(parserOptions); + allParserOptions.addAll(extraOptions); + sourceIncludeFolders.stream().map(includeFolder -> "-I" + includeFolder).forEach(allParserOptions::add); + return allParserOptions; } private Set getSourceIncludes(List sources) { @@ -1233,6 +1231,14 @@ public boolean rebuildAst(boolean update) { .forEach(writtenFile -> rebuildBases.put(SpecsIo.getCanonicalFile(writtenFile), tempFolder)); currentBases = rebuildBases; + if (!update) { + CodeParser codeParser = newCodeParser(); + codeParser.set(ParallelCodeParser.SYNTAX_ONLY, true); + codeParser.parse(writtenFiles, addSourceIncludes(writtenFiles, rebuildOptions, extraOptions), context); + currentBases = previousBases; + return true; + } + App rebuiltApp = createApp(writtenFiles, rebuildOptions, extraOptions); // Restore current bases