Better cuda cache

ClangAstParser/build.gradle

@@ -24,6 +24,8 @@ dependencies {
 
     implementation 'com.google.code.gson:gson:2.12.1'
     implementation 'com.google.guava:guava:33.4.0-jre'
+    implementation 'org.apache.commons:commons-compress:1.27.1'
+    implementation 'org.tukaani:xz:1.9'
 
     testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0'
     testRuntimeOnly 'org.junit.platform:junit-platform-launcher:1.10.0'
@@ -52,6 +54,7 @@ sourceSets {
 
 processResources {
     from('clang-dumper-release.tag')
+    from('cuda-release.tag')
 }
 
 // Test coverage configuration

ClangAstParser/clang-dumper-release.tag

@@ -1 +1 @@
-v18.1.8_1
+v18.1.8_2

ClangAstParser/cuda-release.tag

@@ -0,0 +1 @@
+12.3.2

ClangAstParser/src/pt/up/fe/specs/clang/CacheFiles.java

@@ -154,6 +154,11 @@ public void close() {
 
     static File installFile(Path cacheRoot, File destination, FileResourceProvider resource, String expectedSha256,
                             String description) {
+        return installFile(cacheRoot, destination, resource, expectedSha256, -1, description);
+    }
+
+    static File installFile(Path cacheRoot, File destination, FileResourceProvider resource, String expectedSha256,
+                            long expectedSize, String description) {
         if (destination.isFile()) {
             return destination;
         }
@@ -166,6 +171,11 @@ static File installFile(Path cacheRoot, File destination, FileResourceProvider r
                 throw new RuntimeException("Could not download " + description);
             }
 
+            if (expectedSize >= 0 && stagedFile.length() != expectedSize) {
+                throw new RuntimeException("Downloaded " + description + " does not match expected size '"
+                        + expectedSize + "' (actual: " + stagedFile.length() + ")");
+            }
+
             if (expectedSha256 != null && !hasExpectedSha256(stagedFile, expectedSha256)) {
                 throw new RuntimeException("Downloaded " + description + " does not match expected SHA-256 '"
                         + expectedSha256 + "'");

ClangAstParser/src/pt/up/fe/specs/clang/ClangAstWebResource.java

@@ -28,12 +28,8 @@
 public final class ClangAstWebResource {
 
     private static final String RELEASE_ROOT = "https://github.com/specs-feup/clang-dumper/releases/download/";
-    private static final String CUDA_RELEASE_ROOT =
-            "https://github.com/specs-feup/clava/releases/download/clang_ast_dumper_v12.0.7.1/";
-    public static final String CUDA_LIB_FILENAME = "cudalib.zip";
-    static final WebResourceProvider CUDA_LIB =
-            WebResourceProvider.newInstance(CUDA_RELEASE_ROOT, CUDA_LIB_FILENAME, "v11.3.0");
     private static final String RELEASE_TAG_RESOURCE = "clang-dumper-release.tag";
+    private static final String CUDA_RELEASE_TAG_RESOURCE = "cuda-release.tag";
     public static final String MANIFEST_FILENAME = "clang-dumper-release-manifest.json";
 
     private static final Gson GSON = new Gson();
@@ -47,24 +43,28 @@ public static DumperSource getDumperSource() {
     }
 
     private static DumperSource readDumperSource() {
-        var inputStream = ClangAstWebResource.class.getClassLoader().getResourceAsStream(RELEASE_TAG_RESOURCE);
+        return parseDumperSource(readReleaseTag(RELEASE_TAG_RESOURCE));
+    }
+
+    private static String readReleaseTag(String resourceName) {
+        var inputStream = ClangAstWebResource.class.getClassLoader().getResourceAsStream(resourceName);
 
         if (inputStream == null) {
-            throw new RuntimeException("Could not find resource '" + RELEASE_TAG_RESOURCE + "'");
+            throw new RuntimeException("Could not find resource '" + resourceName + "'");
         }
 
         String value;
         try (inputStream) {
             value = SpecsIo.read(inputStream).trim();
         } catch (IOException e) {
-            throw new UncheckedIOException("Could not read resource '" + RELEASE_TAG_RESOURCE + "'", e);
+            throw new UncheckedIOException("Could not read resource '" + resourceName + "'", e);
         }
 
         if (value.isBlank()) {
-            throw new RuntimeException("Resource '" + RELEASE_TAG_RESOURCE + "' is empty");
+            throw new RuntimeException("Resource '" + resourceName + "' is empty");
         }
 
-        return parseDumperSource(value);
+        return value;
     }
 
     static DumperSource parseDumperSource(String value) {
@@ -90,6 +90,17 @@ public static String getReleaseTag() {
         throw new IllegalStateException("The clang-dumper resource points to a local build");
     }
 
+    public static String getCudaReleaseTag() {
+        var releaseTag = readReleaseTag(CUDA_RELEASE_TAG_RESOURCE);
+        if (releaseTag.equals(".") || releaseTag.equals("..")
+                || releaseTag.contains("/") || releaseTag.contains("\\")) {
+            throw new RuntimeException("Release resource '" + CUDA_RELEASE_TAG_RESOURCE
+                    + "' must contain a single path component: '" + releaseTag + "'");
+        }
+
+        return releaseTag;
+    }
+
     public static ClangDumperManifest getManifest(File resourceFolder) {
         var releaseTag = getReleaseTag();
         var manifestResource = WebResourceProvider.newInstance(getReleaseBaseUrl(releaseTag), MANIFEST_FILENAME,

ClangAstParser/src/pt/up/fe/specs/clang/ClangFiles.java

@@ -16,6 +16,6 @@
 import java.io.File;
 import java.util.List;
 
-public record ClangFiles(File clangExecutable, List<String> builtinIncludes) {
+public record ClangFiles(File clangExecutable, List<String> builtinIncludes, File systemResourceDir) {
 
 }

ClangAstParser/src/pt/up/fe/specs/clang/ClangResources.java

@@ -43,6 +43,7 @@ public class ClangResources {
 
     private static final Map<String, CachedClangFiles> CLANG_FILES_CACHE = new ConcurrentHashMap<>();
     private static final String CLANG_FOLDERNAME = "clang_ast_exe";
+    private static final String CLANG_CACHE_FOLDERNAME = "clang-dumper";
     private static final String RELEASES_FOLDERNAME = "releases";
     private static final String INCLUDES_FOLDERNAME = "includes";
     private static final Duration STALE_CACHE_MAX_AGE = Duration.ofDays(60);
@@ -55,32 +56,27 @@ public ClangResources(CodeParser options) {
         this.options = options;
     }
 
-    public File getBuiltinCudaLib() {
-        var cudaResourceFolder = SpecsIo.mkdir(options.get(CodeParser.DUMPER_FOLDER), "cuda");
-        var cudaFolder = SpecsIo.mkdir(cudaResourceFolder, "cudalib");
-        var zipFile = ClangAstWebResource.CUDA_LIB.writeVersioned(cudaResourceFolder, ClangResources.class);
-
-        if (zipFile.isNewFile() || !isCudaInstallation(cudaFolder)) {
-            SpecsIo.deleteFolderContents(cudaFolder);
-            SpecsIo.extractZip(zipFile.getFile(), cudaFolder);
-        }
-
-        return cudaFolder;
+    public static boolean isBuiltinCudaSupported() {
+        return CudaResources.isSupportedPlatform();
     }
 
-    private static boolean isCudaInstallation(File folder) {
-        return folder.isDirectory() && new File(folder, "include/cuda_runtime.h").isFile();
+    public File getBuiltinCudaLib() {
+        return CudaResources.getBuiltinCudaLib(options.get(CodeParser.DUMPER_FOLDER).toPath());
     }
 
     public ClangFiles getClangFiles(LibcMode libcMode) {
 
         var source = ClangAstWebResource.getDumperSource();
+        var useBuiltinCuda = options.get(CodeParser.CUDA_PATH).equalsIgnoreCase(CodeParser.getBuiltinOption());
 
         if (source instanceof LocalBuild localBuild) {
-            return new ClangFiles(getLocalExecutable(localBuild.folder()), List.of());
+            var clangExecutable = getLocalExecutable(localBuild.folder());
+            var systemResourceDir = libcMode == LibcMode.SYSTEM && useBuiltinCuda
+                    ? findSystemClangResourceDir(null)
+                    : null;
+            return new ClangFiles(clangExecutable, List.of(), systemResourceDir);
         }
 
-        var useBuiltinCuda = options.get(CodeParser.CUDA_PATH).equalsIgnoreCase(CodeParser.getBuiltinOption());
         var resourceFolder = getClangResourceFolder();
         var key = libcMode.name() + "_" + useBuiltinCuda + "_" + source + "_"
                 + resourceFolder.getAbsolutePath();
@@ -97,6 +93,9 @@ public ClangFiles getClangFiles(LibcMode libcMode) {
         var manifest = ClangAstWebResource.getManifest(resourceFolder);
         File clangExecutable = prepareResources(manifest, resourceFolder);
         var includes = prepareIncludes(manifest, clangExecutable, libcMode);
+        var systemResourceDir = libcMode == LibcMode.SYSTEM && useBuiltinCuda
+                ? prepareSystemClangResourceDir(manifest)
+                : null;
 
         if (useBuiltinCuda) {
             getBuiltinCudaLib();
@@ -105,7 +104,7 @@ public ClangFiles getClangFiles(LibcMode libcMode) {
         touchUse(resourceFolder, includes.extractedFolder());
         updateLastUsedAndCleanupStaleVersions(resourceFolder, includes.extractedFolder());
 
-        var newFiles = new CachedClangFiles(new ClangFiles(clangExecutable, includes.folders()),
+        var newFiles = new CachedClangFiles(new ClangFiles(clangExecutable, includes.folders(), systemResourceDir),
                 includes.extractedFolder());
         var existingFiles = CLANG_FILES_CACHE.putIfAbsent(key, newFiles);
         var selectedFiles = existingFiles == null ? newFiles : existingFiles;
@@ -119,11 +118,16 @@ private boolean isUsable(CachedClangFiles cached) {
             return false;
         }
 
-        return CacheFiles.withMaintenanceLock(options.get(CodeParser.DUMPER_FOLDER).toPath(), () -> {
+        return CacheFiles.withMaintenanceLock(getClangCacheRoot().toPath(), () -> {
             if (!cached.files().clangExecutable().isFile()) {
                 return false;
             }
 
+            if (cached.files().systemResourceDir() != null
+                    && !cached.files().systemResourceDir().isDirectory()) {
+                return false;
+            }
+
             var includesFolder = cached.includesFolder();
             if (includesFolder == null) {
                 return true;
@@ -143,7 +147,7 @@ private boolean isUsable(CachedClangFiles cached) {
     }
 
     private void touchUse(File resourceFolder, File includesFolder) {
-        CacheFiles.withMaintenanceLock(options.get(CodeParser.DUMPER_FOLDER).toPath(), () -> {
+        CacheFiles.withMaintenanceLock(getClangCacheRoot().toPath(), () -> {
             CacheFiles.touch(resourceFolder.toPath());
             if (includesFolder != null) {
                 CacheFiles.touch(includesFolder.toPath());
@@ -178,7 +182,7 @@ private File prepareResources(ClangDumperManifest manifest, File resourceFolder)
 
         var executableKind = ClangAstDumper.usePlugin() ? "plugin" : "tool";
         var asset = getCurrentAsset(manifest, executableKind);
-        File executable = CacheFiles.installFile(options.get(CodeParser.DUMPER_FOLDER).toPath(),
+        File executable = CacheFiles.installFile(getClangCacheRoot().toPath(),
                 new File(resourceFolder, asset.filename()),
                 ClangAstWebResource.getAssetResource(asset), asset.sha256(),
                 "clang-dumper asset '" + asset.filename() + "'");
@@ -214,7 +218,7 @@ private void unblockWindowsFile(File executable) {
     }
 
     public File getClangResourceFolder() {
-        var cacheFolder = options.get(CodeParser.DUMPER_FOLDER);
+        var cacheFolder = getClangCacheRoot();
         return CacheFiles.withMaintenanceLock(cacheFolder.toPath(), () -> {
             var releaseFolder = SpecsIo.mkdir(getReleasesFolder(), ClangAstWebResource.getReleaseTag());
             CacheFiles.touch(releaseFolder.toPath());
@@ -227,11 +231,15 @@ public static File getDefaultTempFolder() {
     }
 
     private File getReleasesFolder() {
-        return SpecsIo.mkdir(options.get(CodeParser.DUMPER_FOLDER), RELEASES_FOLDERNAME);
+        return SpecsIo.mkdir(getClangCacheRoot(), RELEASES_FOLDERNAME);
     }
 
     private File getIncludesRoot() {
-        return new File(options.get(CodeParser.DUMPER_FOLDER), INCLUDES_FOLDERNAME);
+        return new File(getClangCacheRoot(), INCLUDES_FOLDERNAME);
+    }
+
+    private File getClangCacheRoot() {
+        return new File(options.get(CodeParser.DUMPER_FOLDER), CLANG_CACHE_FOLDERNAME);
     }
 
     static File getSharedIncludesFolder(File cacheFolder, String sha256) {
@@ -296,9 +304,8 @@ private static ProcessOutputAsString runClangAstDumper(File clangExecutable, Fil
     private PreparedIncludes prepareIncludes(ClangDumperManifest manifest, File clangExecutable,
                                              LibcMode libcMode) {
         var useBuiltinLibc = useBuiltinLibc(clangExecutable, libcMode);
-        var useBuiltinCuda = options.get(CodeParser.CUDA_PATH).equalsIgnoreCase(CodeParser.getBuiltinOption());
 
-        if (!useBuiltinLibc && !useBuiltinCuda) {
+        if (!useBuiltinLibc) {
             return new PreparedIncludes(List.of(), null);
         }
 
@@ -309,9 +316,61 @@ private PreparedIncludes prepareIncludes(ClangDumperManifest manifest, File clan
         return new PreparedIncludes(includeFolders.stream().map(File::getAbsolutePath).toList(), extractedFolder);
     }
 
+    private File prepareSystemClangResourceDir(ClangDumperManifest manifest) {
+        var executableKind = ClangAstDumper.usePlugin() ? "plugin" : "tool";
+        var llvmMajor = getCurrentAsset(manifest, executableKind).llvm_major();
+        return findSystemClangResourceDir(llvmMajor);
+    }
+
+    private File findSystemClangResourceDir(Integer llvmMajor) {
+        var commandNames = getSystemClangCommandNames(llvmMajor);
+        for (var commandName : commandNames) {
+            final ProcessOutputAsString output;
+            try {
+                output = SpecsSystem.runProcess(List.of(commandName, "-print-resource-dir"), true, false);
+            } catch (RuntimeException e) {
+                continue;
+            }
+
+            if (output.getReturnValue() != 0 || output.getStdOut() == null) {
+                continue;
+            }
+
+            var resourceDir = new File(output.getStdOut().trim());
+            if (isSystemClangResourceDir(resourceDir, llvmMajor)) {
+                SpecsLogs.debug(() -> "Using system Clang resource directory '"
+                        + resourceDir.getAbsolutePath() + "'");
+                return resourceDir;
+            }
+        }
+
+        var expectedVersion = llvmMajor == null ? "the local clang-dumper build's version"
+                : "LLVM " + llvmMajor;
+        throw new RuntimeException("Could not find a system Clang resource directory for SYSTEM mode with built-in CUDA"
+                + " on host '" + SupportedPlatform.getCurrentPlatform() + "' (expected " + expectedVersion
+                + "). Tried: " + commandNames);
+    }
+
+    private static List<String> getSystemClangCommandNames(Integer llvmMajor) {
+        var suffix = SupportedPlatform.getCurrentPlatform().isWindows() ? ".exe" : "";
+        if (llvmMajor == null) {
+            return List.of("clang++" + suffix);
+        }
+
+        return List.of("clang++-" + llvmMajor + suffix, "clang++" + suffix);
+    }
+
+    private static boolean isSystemClangResourceDir(File resourceDir, Integer llvmMajor) {
+        if (!resourceDir.isDirectory() || !new File(resourceDir, "include").isDirectory()) {
+            return false;
+        }
+
+        return llvmMajor == null || resourceDir.getName().equals(Integer.toString(llvmMajor));
+    }
+
     private File prepareIncludesFolder(ClangDumperManifest manifest) {
         var includesAsset = getCurrentAsset(manifest, "includes");
-        return resolveIncludes(options.get(CodeParser.DUMPER_FOLDER), includesAsset,
+        return resolveIncludes(getClangCacheRoot(), includesAsset,
                 ClangAstWebResource.getAssetResource(includesAsset));
     }
 
@@ -453,7 +512,7 @@ void deleteStaleVersions(Instant now, File currentVersionFolder) {
 
     private void deleteStaleVersions(Instant now, File currentVersionFolder, File currentIncludesFolder) {
         var cutoff = now.minus(STALE_CACHE_MAX_AGE);
-        var cacheRoot = options.get(CodeParser.DUMPER_FOLDER).toPath();
+        var cacheRoot = getClangCacheRoot().toPath();
         try {
             CacheFiles.deleteStaleDirectories(cacheRoot, getReleasesFolder().toPath(), cutoff,
                     currentVersionFolder.toPath());