From 466eece8c8454c3e39d665381536d8af0f27e8e4 Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Thu, 11 Dec 2025 11:45:03 +0100 Subject: [PATCH 01/18] Jsonize results --- Makefile | 6 ++-- parse_results.py | 81 ++++++++++++++++++++++++++++++++++++------------ 2 files changed, 65 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index abede70..299dc67 100644 --- a/Makefile +++ b/Makefile @@ -116,7 +116,7 @@ run_envs: clone_yamls knit_report: clone_reports - R -e 'rmarkdown::render("$(REPORTS_DIR)/07_metrics_across_backends.Rmd", params = list(performance_bn = "performance-results.rds", metrics_bn = "metrics-results.rds", clustering_dir = "."))' - R -e 'rmarkdown::render("$(REPORTS_DIR)/08_performances_across_backends.Rmd", params = list(performance_bn = "performance-results.rds", metrics_bn = "metrics-results.rds", clustering_dir = "."))' - python parse_results.py + # R -e 'rmarkdown::render("$(REPORTS_DIR)/07_metrics_across_backends.Rmd", params = list(performance_bn = "performance-results.rds", metrics_bn = "metrics-results.rds", clustering_dir = "."))' + # R -e 'rmarkdown::render("$(REPORTS_DIR)/08_performances_across_backends.Rmd", params = list(performance_bn = "performance-results.rds", metrics_bn = "metrics-results.rds", clustering_dir = "."))' + python parse_results.py > aggregated_results.json R -e 'rmarkdown::render("analyze_results.Rmd")' diff --git a/parse_results.py b/parse_results.py index a3e60a9..27f2132 100755 --- a/parse_results.py +++ b/parse_results.py @@ -13,40 +13,83 @@ from typing import Dict, List, Optional +# def parse_result_path(path: Path) -> Dict[str, str]: +# """ +# Parse a result path and extract components. + +# """ +# parts = path.parts + +# # print(pars) + +# result = {} + +# # Parse out-{backend}-{rep} +# out_match = re.match( r"out_(?P[a-zA-Z0-9]+)_seed_(?P\d+)_run_(?P\d+)", parts) +# if out_match: +# result['backend'] = out_match.group(1) +# result['seed'] = out_match.group(2) +# result['rep'] = out_match.group(3) + +# # Find dataset_generator part +# for part in parts: +# if part.startswith('dataset_generator-'): +# # Parse dataset_generator-{generator}_dataset_name-{name} +# dataset_match = re.match(r'dataset_generator-([^_]+)_dataset_name-(.+)', part) +# if dataset_match: +# result['generator'] = dataset_match.group(1) +# result['dataset_name'] = dataset_match.group(2) +# break + +# # The method is the last part (after clustering/) +# if 'clustering' in parts: +# clustering_idx = parts.index('clustering') +# if clustering_idx + 1 < len(parts): +# result['method'] = parts[clustering_idx + 1] + +# result['path'] = str(path) + +# return result + + def parse_result_path(path: Path) -> Dict[str, str]: """ - Parse a result path and extract components. - - Pattern: out-{backend}-{rep}/data/clustbench/dataset_generator-{generator}_dataset_name-{name}/clustering/{method} + Parse a result path and extract components: + - backend, seed, run (from out_* directories) + - dataset generator and dataset name + - method (after clustering/) """ parts = path.parts + result: Dict[str, str] = {} - result = {} + # Parse out_{backend}_seed_{seed}_run_{run} + out_match = re.match( + r"out_(?P[a-zA-Z0-9]+)_seed_(?P\d+)_run_(?P\d+)", + parts[0]) - # Parse out-{backend}-{rep} - out_match = re.match(r'out-([^-]+)-(\d+)', parts[0]) if out_match: - result['backend'] = out_match.group(1) - result['rep'] = out_match.group(2) + result["backend"] = out_match.group("backend") + result["seed"] = out_match.group("seed") + result["run"] = out_match.group("run") # Find dataset_generator part for part in parts: - if part.startswith('dataset_generator-'): - # Parse dataset_generator-{generator}_dataset_name-{name} - dataset_match = re.match(r'dataset_generator-([^_]+)_dataset_name-(.+)', part) + if part.startswith("dataset_generator-"): + dataset_match = re.match( + r"dataset_generator-([^_]+)_dataset_name-(.+)", part + ) if dataset_match: - result['generator'] = dataset_match.group(1) - result['dataset_name'] = dataset_match.group(2) + result["generator"] = dataset_match.group(1) + result["dataset_name"] = dataset_match.group(2) break # The method is the last part (after clustering/) - if 'clustering' in parts: - clustering_idx = parts.index('clustering') + if "clustering" in parts: + clustering_idx = parts.index("clustering") if clustering_idx + 1 < len(parts): - result['method'] = parts[clustering_idx + 1] - - result['path'] = str(path) + result["method"] = parts[clustering_idx + 1] + result["path"] = str(path) return result @@ -185,7 +228,7 @@ def parse_metrics(param_dir: Path) -> Dict[str, Dict[str, Dict[str, float]]]: return metrics -def find_results(base_dir: str = '.', pattern: str = 'out-*/data/clustbench/dataset_generator-*/clustering/*') -> List[Dict[str, str]]: +def find_results(base_dir: str = '.', pattern: str = 'out_*/data/clustbench/dataset_generator-*/clustering/*') -> List[Dict[str, str]]: """ Find all result directories matching the pattern. From a8e617db2cb1cdee711f533e2c9e188166b5ca32 Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Thu, 11 Dec 2025 17:06:30 +0100 Subject: [PATCH 02/18] Generate tabular aggregated results, long format --- Makefile | 2 +- analyze_results_izaskun.Rmd | 119 ++++++++++++++++++++++++++++++++++++ parse_results.py | 4 +- 3 files changed, 122 insertions(+), 3 deletions(-) create mode 100644 analyze_results_izaskun.Rmd diff --git a/Makefile b/Makefile index 299dc67..21b4c90 100644 --- a/Makefile +++ b/Makefile @@ -119,4 +119,4 @@ knit_report: clone_reports # R -e 'rmarkdown::render("$(REPORTS_DIR)/07_metrics_across_backends.Rmd", params = list(performance_bn = "performance-results.rds", metrics_bn = "metrics-results.rds", clustering_dir = "."))' # R -e 'rmarkdown::render("$(REPORTS_DIR)/08_performances_across_backends.Rmd", params = list(performance_bn = "performance-results.rds", metrics_bn = "metrics-results.rds", clustering_dir = "."))' python parse_results.py > aggregated_results.json - R -e 'rmarkdown::render("analyze_results.Rmd")' + R -e 'rmarkdown::render("analyze_results_izaskun.Rmd")' diff --git a/analyze_results_izaskun.Rmd b/analyze_results_izaskun.Rmd new file mode 100644 index 0000000..4464484 --- /dev/null +++ b/analyze_results_izaskun.Rmd @@ -0,0 +1,119 @@ +--- +title: "Clustbench Performance Analysis" +output: + html_document: default +date: "`r Sys.Date()`" +--- + +```{r setup, include=TRUE} +library(knitr) +library(tidyverse) +library(jsonlite) + + +knitr::opts_chunk$set( + echo = TRUE, + warning = TRUE, + message = TRUE, + fig.path = "plots/", + dev = c("png", "svg"), + fig.width = 6, + fig.height = 6) + + +``` + +## Load Data + +```{r load-data} +# Run the Python script and capture JSON output +json_output <- system("python3 parse_results.py 2>/dev/null", intern = TRUE) +json_text <- paste(json_output, collapse = "\n") +``` + + +```{r} +# flatten configurations into long rows of param_name/param_value +flatten_configurations <- function(cfgs) { + if (is.null(cfgs) || length(cfgs) == 0) { + return(data.frame(parameter_dir = NA_character_, + param_name = NA_character_, + param_value = NA_character_, + stringsAsFactors = FALSE)) + } + out <- data.frame() + for (cfg in cfgs) { + parameter_dir <- cfg$parameter_dir + params <- cfg$parameters + if (is.data.frame(params)) params <- as.list(params) + flat <- unlist(params, use.names = TRUE) + for (p in names(flat)) { + out <- rbind(out, data.frame( + parameter_dir = parameter_dir, + param_name = p, + param_value = as.character(flat[[p]]), + stringsAsFactors = FALSE + )) + } + } + out +} + +flatten_record <- function(rec) { + pm <- rec$metrics$partition_metrics + perf <- rec$performance + cfg_df <- flatten_configurations(rec$configurations) + + rows <- list() + idx <- 0 + + for (metric_name in names(pm)) { + metric_vals <- pm[[metric_name]] + for (k in names(metric_vals)) { + for (i in seq_len(nrow(cfg_df))) { + idx <- idx + 1 + rows[[idx]] <- data.frame( + backend = rec$backend, + seed = rec$seed, + run = rec$run, + generator = rec$generator, + dataset_name = rec$dataset_name, + method = rec$method, + path = rec$path, + method_params= rec$method_params, + method_full = rec$method_full, + parameter_dir= cfg_df$parameter_dir[i], + param_name = cfg_df$param_name[i], + param_value = cfg_df$param_value[i], + k = as.integer(k), + metric_name = metric_name, + metric_value = metric_vals[[k]], + # performance metrics + s = perf$s, + h_m_s = perf[["h:m:s"]], + max_rss = perf$max_rss, + max_vms = perf$max_vms, + max_uss = perf$max_uss, + max_pss = perf$max_pss, + io_in = perf$io_in, + io_out = perf$io_out, + mean_load= perf$mean_load, + cpu_time = perf$cpu_time, + stringsAsFactors = FALSE + ) + } + } + } + do.call(rbind, rows) +} + +records <- jsonlite::fromJSON(json_output, simplifyVector = FALSE) + +fd <- do.call(rbind, lapply(records, flatten_record)) + +head(fd) + +table(fd$param_name) +write.csv(fd, file = 'aggregated_results.csv') + +``` diff --git a/parse_results.py b/parse_results.py index 27f2132..1b09be2 100755 --- a/parse_results.py +++ b/parse_results.py @@ -62,7 +62,7 @@ def parse_result_path(path: Path) -> Dict[str, str]: parts = path.parts result: Dict[str, str] = {} - # Parse out_{backend}_seed_{seed}_run_{run} + # parse out_{backend}_seed_{seed}_run_{run} out_match = re.match( r"out_(?P[a-zA-Z0-9]+)_seed_(?P\d+)_run_(?P\d+)", parts[0]) @@ -155,7 +155,7 @@ def parse_metric_scores(scores_file: Path) -> Optional[Dict[str, float]]: k_strings = [k.strip() for k in lines[0].strip().split(',')] k_values = [] for k_str in k_strings: - match = re.match(r'k=(\d+)', k_str) + match = re.match(r'.*k=(\d+)*', k_str) if match: k_values.append(int(match.group(1))) else: From cd8559d533913e89bd72f519ca0d4b66c422f679 Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Thu, 11 Dec 2025 17:43:56 +0100 Subject: [PATCH 03/18] Add descriptive plots --- .play_minio.json | 1 - analyze_results_izaskun.Rmd | 107 +++++++++++++++++++++++++++++++++++- 2 files changed, 106 insertions(+), 2 deletions(-) delete mode 100644 .play_minio.json diff --git a/.play_minio.json b/.play_minio.json deleted file mode 100644 index 81a2b2b..0000000 --- a/.play_minio.json +++ /dev/null @@ -1 +0,0 @@ -{"access_key": "Q3AM3UQ867SPQQA43P2F", "secret_key": "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"} \ No newline at end of file diff --git a/analyze_results_izaskun.Rmd b/analyze_results_izaskun.Rmd index 4464484..f210ea7 100644 --- a/analyze_results_izaskun.Rmd +++ b/analyze_results_izaskun.Rmd @@ -9,7 +9,7 @@ date: "`r Sys.Date()`" library(knitr) library(tidyverse) library(jsonlite) - +library(ggplot2) knitr::opts_chunk$set( echo = TRUE, @@ -117,3 +117,108 @@ table(fd$param_name) write.csv(fd, file = 'aggregated_results.csv') ``` + +```{r} +print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], + aes(x = backend, y = cpu_time, fill = backend)) + + geom_boxplot(outlier.alpha = 0.3) + + theme_minimal(base_size = 14) + + labs(title = "CPU time by backend", + x = "Backend", + y = "CPU Time (s)") + + scale_fill_brewer(palette = "Set2")) +``` + + +```{r} + +ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], + aes(x = backend, y = metric_value, color = backend)) + + geom_boxplot(outlier.alpha = 0.3) + + facet_wrap(~metric_name, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "Metrics consistency across backends", + x = "Backend", + y = "Metric Value") + + scale_color_brewer(palette = "Set1") +``` + +```{r} +df_run <- fd %>% + select(dataset_name, method, k, metric_name, run, seed, backend, metric_value) + + +# a bit of collapsing here! +wide_run <- fd %>% + filter(backend %in% c("conda","oras","envmodules")) %>% + group_by(dataset_name, method, k, metric_name, run, seed, backend) %>% + summarise(metric_value = mean(as.numeric(metric_value), na.rm = TRUE), .groups = "drop") %>% + pivot_wider(names_from = backend, values_from = metric_value) + +# Check structure +str(wide_run) + +cors_run <- wide_run %>% + group_by(metric_name) %>% + summarise( + cor_conda_oras = cor(conda, oras, use="complete.obs"), + cor_conda_envmodules = cor(conda, envmodules, use="complete.obs"), + cor_oras_envmodules = cor(oras, envmodules, use="complete.obs") + ) + +print(cors_run) + +``` + + +```{r, fig.width = 10, fig.height=10} + +perf_metrics <- c("cpu_time","max_rss","max_vms","max_uss", + "max_pss","io_in","io_out","mean_load") + +fd_long <- fd %>% + filter(backend %in% c("conda","oras","envmodules")) %>% + pivot_longer(cols = all_of(perf_metrics), + names_to = "metric", + values_to = "value") + +# boxplots + jittered scatter, faceted by metric +ggplot(fd_long, aes(x = backend, y = value, fill = backend)) + + geom_boxplot(outlier.alpha = 0.3) + + geom_jitter(width = 0.2, alpha = 0.1, size = 1, color = "black") + + facet_wrap(~metric, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "Performance metrics by backend", + x = "Backend", + y = "Value") + + scale_fill_brewer(palette = "Set2") + +``` + +Choice of `k` - we lack the annotation of the true k in the long CSV + + +```{r, fig.height = 10, fig.width = 10} + +ggplot(fd, aes(x = factor(k), y = metric_value, fill = method)) + + geom_jitter(width = 0.2, alpha = 0.4, size = 1, fill = "black") + + facet_grid(method ~ metric_name + dataset_name, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "Clustering metric sensitivity to k", + x = "number of clusters k (true or not)", + y = "Metric value") +``` + + + +```{r, fig.height = 10, fig.width = 10} +ggplot(fd, aes(x = factor(k), y = metric_value, fill = method)) + + geom_jitter(width = 0.2, alpha = 0.4, size = 1, color = "black") + + facet_grid(dataset_name + method ~ metric_name, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "Clustering metric sensitivity to k", + x = "Number of clusters (k), true or not", + y = "Metric value") + + +``` From 5a6b8f524d238ac7d307a6d955dfebe373c41164 Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Fri, 12 Dec 2025 14:23:59 +0100 Subject: [PATCH 04/18] Update plots --- analyze_results_izaskun.Rmd | 140 +++++++++++++++++++++++++++++++----- 1 file changed, 122 insertions(+), 18 deletions(-) diff --git a/analyze_results_izaskun.Rmd b/analyze_results_izaskun.Rmd index f210ea7..1aa7d15 100644 --- a/analyze_results_izaskun.Rmd +++ b/analyze_results_izaskun.Rmd @@ -1,11 +1,11 @@ --- -title: "Clustbench Performance Analysis" +title: "clustbench figure 2" output: html_document: default date: "`r Sys.Date()`" --- -```{r setup, include=TRUE} +```{r setup, message = FALSE} library(knitr) library(tidyverse) library(jsonlite) @@ -118,6 +118,9 @@ write.csv(fd, file = 'aggregated_results.csv') ``` + +Oops, is the speed of computing the metric, or of running the method? + ```{r} print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], aes(x = backend, y = cpu_time, fill = backend)) + @@ -129,8 +132,48 @@ print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], scale_fill_brewer(palette = "Set2")) ``` +Clearly the method, so all good: + +```{r} +print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], + aes(x = backend, y = cpu_time, fill = backend)) + + geom_boxplot(outlier.alpha = 0.3) + + # add points for each method/params combination + geom_point(alpha = 0.6, position = position_jitter(width = 0.15)) + + # connect points across backends for same module+params + geom_line(aes(group = interaction(method, method_params, seed, run, generator, dataset_name, k)), + alpha = 0.1, color = "grey40") + + facet_wrap(~metric_name, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "CPU time by backend", + x = "Backend", + y = "CPU time (s)") + + scale_color_brewer(palette = "Set1") +) +``` + + ```{r} +print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], + aes(x = backend, y = cpu_time, fill = backend)) + + geom_boxplot(outlier.alpha = 0.3) + + # add points for each method/params combination + geom_point(aes(color = method), + alpha = 0.6, position = position_jitter(width = 0.15)) + + # connect points across backends for same module+params + geom_line(aes(group = interaction(method, method_params, seed, run, generator, dataset_name, k)), + alpha = 0.1, color = "grey40") + + theme_minimal(base_size = 14) + + labs(title = "CPU time by backend", + x = "Backend", + y = "CPU time (s)") + + scale_color_brewer(palette = "Set1") +) +``` + + +```{r, eval = FALSE} ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], aes(x = backend, y = metric_value, color = backend)) + @@ -143,6 +186,33 @@ ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], scale_color_brewer(palette = "Set1") ``` + + +```{r} + +print( + ggplot(fd, aes(x = backend, y = metric_value)) + + # boxplots still colored by backend + geom_boxplot(aes(fill = backend), outlier.alpha = 0.3) + + # points colored by run + geom_point(aes(color = run), + alpha = 0.6, + position = position_jitter(width = 0.15)) + + # connect points across backends for same method/params/run/etc. + geom_line(aes(group = interaction(method, method_params, seed, run, generator, dataset_name, k), + color = run), + alpha = 0.2) + + facet_wrap(dataset_name ~ metric_name, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "Metrics consistency across backends (stage = methods)", + x = "Backend", + y = "Metric Value") + + scale_fill_brewer(palette = "Set1") + + scale_color_brewer(palette = "Dark2")) + +``` + + ```{r} df_run <- fd %>% select(dataset_name, method, k, metric_name, run, seed, backend, metric_value) @@ -195,30 +265,64 @@ ggplot(fd_long, aes(x = backend, y = value, fill = backend)) + ``` -Choice of `k` - we lack the annotation of the true k in the long CSV +Something wrong with method "1" etc here? -```{r, fig.height = 10, fig.width = 10} +```{r, fig.width = 10, fig.height = 10} +fd_long <- fd %>% + pivot_longer(cols = all_of(perf_metrics), + names_to = "metric", + values_to = "value") -ggplot(fd, aes(x = factor(k), y = metric_value, fill = method)) + - geom_jitter(width = 0.2, alpha = 0.4, size = 1, fill = "black") + - facet_grid(method ~ metric_name + dataset_name, scales = "free_y") + +ggplot(fd_long, aes(x = backend, y = value)) + + # boxplots filled by backend + ## geom_boxplot(aes(fill = backend), outlier.alpha = 0.3) + + # points colored by method + geom_point(aes(color = method), + alpha = 0.6, + position = position_jitter(width = 0.15)) + + # connect points across backends for same method/run/etc. + geom_line(aes(group = interaction(method, method_params, seed, run, generator, dataset_name, k), + color = run), + alpha = 0.1) + + facet_wrap(~metric, scales = "free_y") + theme_minimal(base_size = 14) + - labs(title = "Clustering metric sensitivity to k", - x = "number of clusters k (true or not)", - y = "Metric value") + labs(title = "Performance metrics by backend", + x = "Backend", + y = "Value") + + scale_fill_brewer(palette = "Set2") + + scale_color_brewer(palette = "Dark2") + +``` + +```{r} + +table(fd$method) ``` +Choice of `k` - we lack the annotation of the true k in the long CSV + + +Just use deviations vs mean depending on the k. Ideally this should vs the true number of clusters! -```{r, fig.height = 10, fig.width = 10} -ggplot(fd, aes(x = factor(k), y = metric_value, fill = method)) + - geom_jitter(width = 0.2, alpha = 0.4, size = 1, color = "black") + - facet_grid(dataset_name + method ~ metric_name, scales = "free_y") + - theme_minimal(base_size = 14) + - labs(title = "Clustering metric sensitivity to k", - x = "Number of clusters (k), true or not", - y = "Metric value") +```{r, fig.width = 7} +fd_dev_k <- fd %>% + group_by(method, dataset_name, metric_name) %>% + mutate(mean_value = mean(metric_value, na.rm = TRUE), + deviation_k = metric_value - mean_value) %>% + ungroup() +summary(fd_dev_k$deviation_k) + +ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method)) + + geom_point(alpha = 0.6, position = position_jitter(width = 0.15)) + + facet_wrap(dataset_name~metric_name, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "deviation of metric across ks", + subtitle = "caution vs mean(method,dataset,metric) value, not true k!", + x = "k", + y = "perf metric deviation from group mean") + + scale_color_brewer(palette = "Set1") ``` From e2ffa5a0869001376d50eccbd803b20e846b900e Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Fri, 12 Dec 2025 15:01:27 +0100 Subject: [PATCH 05/18] Add plots re: seed and run sensitivity, extend benchmark --- Makefile | 10 +++++---- analyze_results_izaskun.Rmd | 43 +++++++++++++++++++++++++++++++++++++ parse_results.py | 42 +----------------------------------- 3 files changed, 50 insertions(+), 45 deletions(-) diff --git a/Makefile b/Makefile index 21b4c90..57c7bc4 100644 --- a/Makefile +++ b/Makefile @@ -36,8 +36,10 @@ export EASYBUILD_PREFIX OB_CMD = ob run benchmark --local-storage --cores ${MAX_CORES} # actual benchmark plan repository - to be pinned (the commit/tag) -CLUSTERING_REPO = https://github.com/omnibenchmark/clustering_example -CLUSTERING_DIR = clustering_example +CLUSTERING_REPO = https://github.com/omnibenchmark/clustering_example +CLUSTERING_BRANCH = longer_yamls + +CLUSTERING_DIR = clustering_example # legacy reports in the wrong repository; to be moved to this one REPORTS_REPO = https://github.com/imallona/clustering_report @@ -49,10 +51,10 @@ all: clone_yamls clone_reports run_conda run_oras run_envs knit_report clone_yamls: @if [ ! -d "$(CLUSTERING_DIR)" ]; then \ echo "Cloning clustering_example repo..."; \ - git clone --branch easyconfigs_py3126 $(CLUSTERING_REPO); \ + git clone --branch ${CLUSTERING_BRANCH} $(CLUSTERING_REPO); \ else \ echo "clustering_example repo already present, pulling latest..."; \ - cd $(CLUSTERING_DIR) && git fetch && git checkout easyconfigs_py3126 && git pull; \ + cd $(CLUSTERING_DIR) && git fetch && git checkout ${CLUSTERING_BRANCH} && git pull; \ fi # clone the clustering_report repo (mark branch) if not already present diff --git a/analyze_results_izaskun.Rmd b/analyze_results_izaskun.Rmd index 1aa7d15..d489749 100644 --- a/analyze_results_izaskun.Rmd +++ b/analyze_results_izaskun.Rmd @@ -326,3 +326,46 @@ ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method)) + y = "perf metric deviation from group mean") + scale_color_brewer(palette = "Set1") ``` + +What about the consistency across runs and seeds? e.g., do different seeds produce different results? + +First, seeds + +```{r} +fd_seed_summary <- fd %>% + group_by(method, dataset_name, metric_name, k, seed) %>% + summarise(mean_value = mean(metric_value, na.rm = TRUE), # average across runs + sd_runs = sd(metric_value, na.rm = TRUE), # replicate consistency + .groups = "drop") + +fd_seed_var <- fd_seed_summary %>% + group_by(method, dataset_name, metric_name, k) %>% + summarise(sd_seeds = sd(mean_value, na.rm = TRUE), # variability across seeds + mean_sd_runs = mean(sd_runs, na.rm = TRUE), # replicate consistency + .groups = "drop") + + +summary(fd_seed_var) + + +ggplot(fd_seed_var, aes(x = metric_name, y = sd_seeds, fill = method)) + + geom_boxplot(outlier.alpha = 0.3) + + theme_minimal(base_size = 14) + + labs(title = "seed sensitivity across methods", + x = "Metric", + y = "seed-driven sd") + + scale_fill_brewer(palette = "Set2") + +``` + +Second, runs + +```{r} +ggplot(fd_seed_var, aes(x = metric_name, y = mean_sd_runs, fill = method)) + + geom_boxplot(outlier.alpha = 0.3) + + theme_minimal(base_size = 14) + + labs(title = "repeated run sensitivity across methods", + x = "Metric", + y = "run-driven sd") + + scale_fill_brewer(palette = "Set2") +``` diff --git a/parse_results.py b/parse_results.py index 1b09be2..6e797ec 100755 --- a/parse_results.py +++ b/parse_results.py @@ -12,46 +12,6 @@ from pathlib import Path from typing import Dict, List, Optional - -# def parse_result_path(path: Path) -> Dict[str, str]: -# """ -# Parse a result path and extract components. - -# """ -# parts = path.parts - -# # print(pars) - -# result = {} - -# # Parse out-{backend}-{rep} -# out_match = re.match( r"out_(?P[a-zA-Z0-9]+)_seed_(?P\d+)_run_(?P\d+)", parts) -# if out_match: -# result['backend'] = out_match.group(1) -# result['seed'] = out_match.group(2) -# result['rep'] = out_match.group(3) - -# # Find dataset_generator part -# for part in parts: -# if part.startswith('dataset_generator-'): -# # Parse dataset_generator-{generator}_dataset_name-{name} -# dataset_match = re.match(r'dataset_generator-([^_]+)_dataset_name-(.+)', part) -# if dataset_match: -# result['generator'] = dataset_match.group(1) -# result['dataset_name'] = dataset_match.group(2) -# break - -# # The method is the last part (after clustering/) -# if 'clustering' in parts: -# clustering_idx = parts.index('clustering') -# if clustering_idx + 1 < len(parts): -# result['method'] = parts[clustering_idx + 1] - -# result['path'] = str(path) - -# return result - - def parse_result_path(path: Path) -> Dict[str, str]: """ Parse a result path and extract components: @@ -155,7 +115,7 @@ def parse_metric_scores(scores_file: Path) -> Optional[Dict[str, float]]: k_strings = [k.strip() for k in lines[0].strip().split(',')] k_values = [] for k_str in k_strings: - match = re.match(r'.*k=(\d+)*', k_str) + match = re.match(r'k=(\d+)', k_str) if match: k_values.append(int(match.group(1))) else: From 04e4fd9704731877469622240db776f78e72e5a4 Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Fri, 12 Dec 2025 17:29:29 +0100 Subject: [PATCH 06/18] Fix parse_results.py to accept quotes in ks; add extra plots, including Bland Altmans ks vs true k --- Makefile | 8 +- analyze_results_izaskun.Rmd | 144 +++++++++++++++++++++++++++++++----- parse_results.py | 8 +- 3 files changed, 134 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index 57c7bc4..7180fce 100644 --- a/Makefile +++ b/Makefile @@ -16,10 +16,10 @@ # run_conda run conda backend with seeds + repeats # run_oras run oras backend with seeds + repeats # run_envs run envmodules backend with seeds + repeats -# knit_report generate RMarkdown reports - not fully tested +# knit_report generate RMarkdown reports and an aggregated CSV - not fully tested # # Environment: -# - MAX_CORES controls parallelism (default: 50). +# - MAX_CORES controls num concurrent rules # - EASYBUILD_PREFIX needs to be tuned to access the envmodules built extending EESSI <--------------!!!! # see: https://github.com/omnibenchmark/clustering_example/pull/43 # @@ -28,7 +28,7 @@ MAX_CORES ?= 50 -# EasyBuild installation prefix (imallona; edit accordingly) +# EasyBuild installation prefix (imallona; edit accordingly) ## <------------------------------------!!!! EASYBUILD_PREFIX ?= /data/imallona/.local/easybuild export EASYBUILD_PREFIX @@ -38,7 +38,6 @@ OB_CMD = ob run benchmark --local-storage --cores ${MAX_CORES} # actual benchmark plan repository - to be pinned (the commit/tag) CLUSTERING_REPO = https://github.com/omnibenchmark/clustering_example CLUSTERING_BRANCH = longer_yamls - CLUSTERING_DIR = clustering_example # legacy reports in the wrong repository; to be moved to this one @@ -116,7 +115,6 @@ run_envs: clone_yamls done \ ' - knit_report: clone_reports # R -e 'rmarkdown::render("$(REPORTS_DIR)/07_metrics_across_backends.Rmd", params = list(performance_bn = "performance-results.rds", metrics_bn = "metrics-results.rds", clustering_dir = "."))' # R -e 'rmarkdown::render("$(REPORTS_DIR)/08_performances_across_backends.Rmd", params = list(performance_bn = "performance-results.rds", metrics_bn = "metrics-results.rds", clustering_dir = "."))' diff --git a/analyze_results_izaskun.Rmd b/analyze_results_izaskun.Rmd index d489749..276b248 100644 --- a/analyze_results_izaskun.Rmd +++ b/analyze_results_izaskun.Rmd @@ -23,15 +23,6 @@ knitr::opts_chunk$set( ``` -## Load Data - -```{r load-data} -# Run the Python script and capture JSON output -json_output <- system("python3 parse_results.py 2>/dev/null", intern = TRUE) -json_text <- paste(json_output, collapse = "\n") -``` - - ```{r} # flatten configurations into long rows of param_name/param_value flatten_configurations <- function(cfgs) { @@ -59,6 +50,20 @@ flatten_configurations <- function(cfgs) { out } +## given the way we have them, so we aim to provide 5 ks, middle being the true, but pad with 2s on the left +get_true_k <- function(x) { + n <- length(x) + if (n == 5) { + return(x[3]) + } else if (n == 4) { + return(x[3]) + } else if (n == 3) { + return(x[2]) + } else { + return(x[1]) + } +} + flatten_record <- function(rec) { pm <- rec$metrics$partition_metrics perf <- rec$performance @@ -69,6 +74,8 @@ flatten_record <- function(rec) { for (metric_name in names(pm)) { metric_vals <- pm[[metric_name]] + true_k <- get_true_k(names(metric_vals)) + for (k in names(metric_vals)) { for (i in seq_len(nrow(cfg_df))) { idx <- idx + 1 @@ -86,6 +93,7 @@ flatten_record <- function(rec) { param_name = cfg_df$param_name[i], param_value = cfg_df$param_value[i], k = as.integer(k), + true_k = as.integer(true_k), metric_name = metric_name, metric_value = metric_vals[[k]], # performance metrics @@ -107,12 +115,13 @@ flatten_record <- function(rec) { do.call(rbind, rows) } -records <- jsonlite::fromJSON(json_output, simplifyVector = FALSE) +records <- jsonlite::fromJSON('aggregated_results.json', simplifyVector = FALSE) fd <- do.call(rbind, lapply(records, flatten_record)) head(fd) + table(fd$param_name) write.csv(fd, file = 'aggregated_results.csv') @@ -303,27 +312,49 @@ table(fd$method) Choice of `k` - we lack the annotation of the true k in the long CSV -Just use deviations vs mean depending on the k. Ideally this should vs the true number of clusters! + + + + + + + + + + + -```{r, fig.width = 7} + + + + + + + + + + + +```{r} + fd_dev_k <- fd %>% group_by(method, dataset_name, metric_name) %>% - mutate(mean_value = mean(metric_value, na.rm = TRUE), - deviation_k = metric_value - mean_value) %>% + # get the metric_value at the true_k for this group + mutate(true_value = metric_value[k == true_k][1], + deviation_k = metric_value - true_value) %>% ungroup() summary(fd_dev_k$deviation_k) - ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method)) + geom_point(alpha = 0.6, position = position_jitter(width = 0.15)) + - facet_wrap(dataset_name~metric_name, scales = "free_y") + + facet_wrap(dataset_name ~ metric_name, scales = "free_y") + theme_minimal(base_size = 14) + - labs(title = "deviation of metric across ks", - subtitle = "caution vs mean(method,dataset,metric) value, not true k!", + labs(title = "Deviation of metric across ks", + subtitle = "Deviation vs value at true k", x = "k", - y = "perf metric deviation from group mean") + + y = "Perf metric deviation from true k") + scale_color_brewer(palette = "Set1") ``` @@ -369,3 +400,78 @@ ggplot(fd_seed_var, aes(x = metric_name, y = mean_sd_runs, fill = method)) + y = "run-driven sd") + scale_fill_brewer(palette = "Set2") ``` + +Recap + + +```{r, fig.width = 12} +print(ggplot(fd_long, aes(x = backend, y = value, fill = backend)) + + geom_boxplot(outlier.alpha = 0.3) + + geom_jitter(width = 0.2, alpha = 0.1, size = 1) + + facet_wrap(~metric, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "comp perf by backend") + + scale_fill_brewer(palette = "Set2")) + +print(ggplot(fd_seed_summary, + aes(x = seed, y = mean_value, + group = interaction(method, dataset_name, k), + color = method)) + + geom_line(alpha = 0.5) + + geom_point(size = 2) + + geom_errorbar(aes(ymin = mean_value - sd_runs, + ymax = mean_value + sd_runs), + width = 0.2, alpha = 0.4) + + facet_wrap(~metric_name, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "Seed sensitivity", + subtitle = "Error bars depict runs variability")) + +print(ggplot(fd, aes(x = cpu_time, y = metric_value, + color = method, shape = backend)) + + geom_point(alpha = 0.6) + + facet_wrap(~metric_name, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "clustering metrics vs runtime trade‑offs", + x = "CPU time (s)", + y = "clusering metric value")) + +print(ggplot(fd, aes(x = max_rss, y = metric_value, + color = method, shape = backend)) + + geom_point(alpha = 0.6) + + facet_wrap(~metric_name, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "clustering metrics vs RSS trade‑offs", + x = "max RSS (MB)", + y = "clustering metric value")) + +``` + +Bland Altmans, any k vs the true k + +```{r, fig.width =10} +fd_dev_true <- fd %>% + group_by(method, dataset_name, metric_name) %>% + # get the metric value at true_k + mutate(true_value = metric_value[k == true_k][1], + diff_val = metric_value - true_value, + mean_val = (metric_value + true_value)/2) %>% + ungroup() + +# Bland–Altman style plot: any k vs true_k +ggplot(fd_dev_true, aes(x = mean_val, y = diff_val, color = method)) + + geom_point(alpha = 0.6) + + geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE), + linetype = "dashed", color = "blue") + + geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) + 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), + linetype = "dotted", color = "red") + + geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) - 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), + linetype = "dotted", color = "red") + + facet_wrap(~metric_name, scales = "free") + + theme_minimal(base_size = 14) + + labs(title = "Bland–Altman: any k vs true_k", + x = "Mean of k and true_k", + y = "Difference (k - true_k)") + + +``` diff --git a/parse_results.py b/parse_results.py index 6e797ec..af3c35b 100755 --- a/parse_results.py +++ b/parse_results.py @@ -115,7 +115,8 @@ def parse_metric_scores(scores_file: Path) -> Optional[Dict[str, float]]: k_strings = [k.strip() for k in lines[0].strip().split(',')] k_values = [] for k_str in k_strings: - match = re.match(r'k=(\d+)', k_str) + k_str = k_str.strip('"') + match = re.match(r'k=(\d+)', k_str) if match: k_values.append(int(match.group(1))) else: @@ -136,7 +137,10 @@ def parse_metric_scores(scores_file: Path) -> Optional[Dict[str, float]]: raise ValueError(f'Duplicate k value {k} with different scores: {result[k]} vs {score}') else: result[k] = score - + + # ## Find the middle column (index = len//2), that contains the true k, and report it + # mid_idx = len(k_values) // 2 + # result['true_k'] = k_values[mid_idx] return result except Exception as e: From 7ddfb79ca0bad564bab395b47b5b6dc707500be4 Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Mon, 15 Dec 2025 10:48:03 +0100 Subject: [PATCH 07/18] Fixes in methods granularity in jsonizer, associated changes in R --- analyze_results_izaskun.Rmd | 329 +++++++++++++++++++++--------------- parse_results.py | 299 +++++++++++++++----------------- 2 files changed, 323 insertions(+), 305 deletions(-) mode change 100755 => 100644 parse_results.py diff --git a/analyze_results_izaskun.Rmd b/analyze_results_izaskun.Rmd index 276b248..f410e83 100644 --- a/analyze_results_izaskun.Rmd +++ b/analyze_results_izaskun.Rmd @@ -10,6 +10,7 @@ library(knitr) library(tidyverse) library(jsonlite) library(ggplot2) +library(data.table) knitr::opts_chunk$set( echo = TRUE, @@ -17,38 +18,13 @@ knitr::opts_chunk$set( message = TRUE, fig.path = "plots/", dev = c("png", "svg"), - fig.width = 6, - fig.height = 6) + fig.width = 10, + fig.height = 10) ``` ```{r} -# flatten configurations into long rows of param_name/param_value -flatten_configurations <- function(cfgs) { - if (is.null(cfgs) || length(cfgs) == 0) { - return(data.frame(parameter_dir = NA_character_, - param_name = NA_character_, - param_value = NA_character_, - stringsAsFactors = FALSE)) - } - out <- data.frame() - for (cfg in cfgs) { - parameter_dir <- cfg$parameter_dir - params <- cfg$parameters - if (is.data.frame(params)) params <- as.list(params) - flat <- unlist(params, use.names = TRUE) - for (p in names(flat)) { - out <- rbind(out, data.frame( - parameter_dir = parameter_dir, - param_name = p, - param_value = as.character(flat[[p]]), - stringsAsFactors = FALSE - )) - } - } - out -} ## given the way we have them, so we aim to provide 5 ks, middle being the true, but pad with 2s on the left get_true_k <- function(x) { @@ -64,65 +40,121 @@ get_true_k <- function(x) { } } + +`%||%` <- function(a, b) if (!is.null(a)) a else b + +flatten_parameters <- function(params, parameter_dir) { + if (is.null(params) || length(params) == 0) { + return(data.frame(parameter_dir = parameter_dir, + param_name = NA_character_, + param_value = NA_character_, + stringsAsFactors = FALSE)) + } + flat <- unlist(params, use.names = TRUE) + data.frame( + parameter_dir = parameter_dir, + param_name = names(flat), + param_value = as.character(flat), + stringsAsFactors = FALSE + ) +} + flatten_record <- function(rec) { - pm <- rec$metrics$partition_metrics + mets <- rec$metrics perf <- rec$performance - cfg_df <- flatten_configurations(rec$configurations) + params_df <- flatten_parameters(rec$parameters, rec$parameter_dir) rows <- list() - idx <- 0 - - for (metric_name in names(pm)) { - metric_vals <- pm[[metric_name]] - true_k <- get_true_k(names(metric_vals)) - - for (k in names(metric_vals)) { - for (i in seq_len(nrow(cfg_df))) { - idx <- idx + 1 - rows[[idx]] <- data.frame( - backend = rec$backend, - seed = rec$seed, - run = rec$run, - generator = rec$generator, - dataset_name = rec$dataset_name, - method = rec$method, - path = rec$path, - method_params= rec$method_params, - method_full = rec$method_full, - parameter_dir= cfg_df$parameter_dir[i], - param_name = cfg_df$param_name[i], - param_value = cfg_df$param_value[i], - k = as.integer(k), - true_k = as.integer(true_k), - metric_name = metric_name, - metric_value = metric_vals[[k]], - # performance metrics - s = perf$s, - h_m_s = perf[["h:m:s"]], - max_rss = perf$max_rss, - max_vms = perf$max_vms, - max_uss = perf$max_uss, - max_pss = perf$max_pss, - io_in = perf$io_in, - io_out = perf$io_out, - mean_load= perf$mean_load, - cpu_time = perf$cpu_time, - stringsAsFactors = FALSE - ) + idx <- 0 + + for (family in names(mets)) { + fam_list <- mets[[family]] + if (is.null(fam_list)) next + + for (metric_name in names(fam_list)) { + metric_vals <- fam_list[[metric_name]] + if (is.null(metric_vals)) next + + ks <- names(metric_vals) + true_k <- get_true_k(ks) + + for (k in ks) { + for (i in seq_len(nrow(params_df))) { + idx <- idx + 1 + rows[[idx]] <- data.frame( + backend = rec$backend, + seed = rec$seed, + run = rec$run, + generator = rec$generator, + dataset_name = rec$dataset_name, + method = rec$method, # keep "sklearn" + method_full = rec$method_full, # "sklearn_method-birch" or "sklearn_method-kmeans" + parameter_dir = rec$parameter_dir, # "method-birch" or "method-kmeans" + param_name = params_df$param_name[i], + param_value = params_df$param_value[i], + metric_family = family, + metric_name = metric_name, + k = as.integer(k), + true_k = as.integer(true_k), + metric_value = metric_vals[[k]], + s = perf$s, + h_m_s = perf[["h:m:s"]], + max_rss = perf$max_rss, + max_vms = perf$max_vms, + max_uss = perf$max_uss, + max_pss = perf$max_pss, + io_in = perf$io_in, + io_out = perf$io_out, + mean_load = perf$mean_load, + cpu_time = perf$cpu_time, + stringsAsFactors = FALSE + ) + } } } } + do.call(rbind, rows) } -records <- jsonlite::fromJSON('aggregated_results.json', simplifyVector = FALSE) -fd <- do.call(rbind, lapply(records, flatten_record)) +``` + + +```{r} + +records <- fromJSON("aggregated_results.json", simplifyVector = FALSE) + +# +fd_list <- vector("list", length(records)) +for (i in seq_along(records)) { + if (i == 1 || i == 2 || i %% 250 == 0) + cat("Processing record", i, "of", length(records), "\n") + fd_list[[i]] <- flatten_record(records[[i]]) +} +fd <- data.table::rbindlist(fd_list, use.names = TRUE, fill = TRUE) + +dim(fd) + +str(fd) +table(is.na(fd$max_rss)) +table(fd$method_full) +table(vapply(records, function(x) x$method_full, character(1))) -head(fd) +fd <- as.data.frame(fd) + +fd$k <- as.integer(fd$k) +fd$true_k <- as.integer(fd$true_k) + +cols_to_num <- c("max_rss","max_vms","max_uss","max_pss", + "io_in","io_out","mean_load","cpu_time") + +fd[cols_to_num] <- lapply(fd[cols_to_num], function(x) { + x[x == "NA"] <- NA_character_ + as.numeric(x) +}) -table(fd$param_name) write.csv(fd, file = 'aggregated_results.csv') ``` @@ -139,7 +171,7 @@ print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], x = "Backend", y = "CPU Time (s)") + scale_fill_brewer(palette = "Set2")) -``` + ``` Clearly the method, so all good: @@ -150,7 +182,7 @@ print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], # add points for each method/params combination geom_point(alpha = 0.6, position = position_jitter(width = 0.15)) + # connect points across backends for same module+params - geom_line(aes(group = interaction(method, method_params, seed, run, generator, dataset_name, k)), + geom_line(aes(group = interaction(method, method_full, seed, run, generator, dataset_name, k)), alpha = 0.1, color = "grey40") + facet_wrap(~metric_name, scales = "free_y") + theme_minimal(base_size = 14) + @@ -165,13 +197,13 @@ print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], ```{r} print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], - aes(x = backend, y = cpu_time, fill = backend)) + + aes(x = backend, y = cpu_time)) + geom_boxplot(outlier.alpha = 0.3) + # add points for each method/params combination geom_point(aes(color = method), alpha = 0.6, position = position_jitter(width = 0.15)) + # connect points across backends for same module+params - geom_line(aes(group = interaction(method, method_params, seed, run, generator, dataset_name, k)), + geom_line(aes(group = interaction(method, method_full, seed, run, generator, dataset_name, k)), alpha = 0.1, color = "grey40") + theme_minimal(base_size = 14) + labs(title = "CPU time by backend", @@ -181,6 +213,43 @@ print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], ) ``` +```{r} +print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], + aes(x = backend, y = cpu_time)) + + geom_boxplot(outlier.alpha = 0.3) + + # add points for each method/params combination + geom_point(aes(color = method_full), + alpha = 0.6, position = position_jitter(width = 0.15)) + + # connect points across backends for same module+params + geom_line(aes(group = interaction(method, method_full, seed, run, generator, dataset_name, k)), + alpha = 0.1, color = "grey40") + + theme_minimal(base_size = 14) + + labs(title = "CPU time by backend", + x = "Backend", + y = "CPU time (s)") + + scale_color_brewer(palette = "Set1") +) +``` + + +```{r} +print( + ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], + aes(x = backend, y = cpu_time)) + + geom_boxplot(outlier.alpha = 0.3) + + geom_point(aes(color = method_full), + alpha = 0.6, position = position_jitter(width = 0.15)) + + geom_line(aes(group = interaction(method, method_full, seed, run, generator, dataset_name, k)), + alpha = 0.1, color = "grey40") + + theme_minimal(base_size = 14) + + facet_wrap(~method, scales = "free_y") + + labs(title = "CPU time by backend", + x = "Backend", + y = "CPU time (s)") + + scale_color_brewer(palette = "Set1") +) +``` + ```{r, eval = FALSE} @@ -197,7 +266,7 @@ ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], -```{r} +```{r, fig.width = 20, fig.height = 20} print( ggplot(fd, aes(x = backend, y = metric_value)) + @@ -208,7 +277,7 @@ print( alpha = 0.6, position = position_jitter(width = 0.15)) + # connect points across backends for same method/params/run/etc. - geom_line(aes(group = interaction(method, method_params, seed, run, generator, dataset_name, k), + geom_line(aes(group = interaction(method, method_full, seed, run, generator, dataset_name, k), color = run), alpha = 0.2) + facet_wrap(dataset_name ~ metric_name, scales = "free_y") + @@ -234,7 +303,6 @@ wide_run <- fd %>% summarise(metric_value = mean(as.numeric(metric_value), na.rm = TRUE), .groups = "drop") %>% pivot_wider(names_from = backend, values_from = metric_value) -# Check structure str(wide_run) cors_run <- wide_run %>% @@ -261,6 +329,23 @@ fd_long <- fd %>% names_to = "metric", values_to = "value") +# Bland–Altman style plot: any k vs true_k +ggplot(fd_dev_true, aes(x = mean_val, y = diff_val, color = method)) + + geom_point(alpha = 0.6) + + geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE), + linetype = "dashed", color = "blue") + + geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) + 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), + linetype = "dotted", color = "red") + + geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) - 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), + linetype = "dotted", color = "red") + + facet_wrap(~metric_name, scales = "free") + + theme_minimal(base_size = 14) + + labs(title = "Bland–Altman: any k vs true_k", + x = "Mean of k and true_k", + y = "Difference (k - true_k)") + + +``` # boxplots + jittered scatter, faceted by metric ggplot(fd_long, aes(x = backend, y = value, fill = backend)) + geom_boxplot(outlier.alpha = 0.3) + @@ -274,8 +359,6 @@ ggplot(fd_long, aes(x = backend, y = value, fill = backend)) + ``` -Something wrong with method "1" etc here? - ```{r, fig.width = 10, fig.height = 10} fd_long <- fd %>% @@ -291,7 +374,7 @@ ggplot(fd_long, aes(x = backend, y = value)) + alpha = 0.6, position = position_jitter(width = 0.15)) + # connect points across backends for same method/run/etc. - geom_line(aes(group = interaction(method, method_params, seed, run, generator, dataset_name, k), + geom_line(aes(group = interaction(method, method_full, seed, run, generator, dataset_name, k), color = run), alpha = 0.1) + facet_wrap(~metric, scales = "free_y") + @@ -306,46 +389,23 @@ ggplot(fd_long, aes(x = backend, y = value)) + ```{r} -table(fd$method) +table(fd$method, fd$method_full) ``` -Choice of `k` - we lack the annotation of the true k in the long CSV - - - - - - - - - - - - - +Choice of `k` impact in clusering perf metrics. Conda only, to speedup. - - - - - - - - - - - - -```{r} +```{r, fig.width = 20, fig.height = 20} fd_dev_k <- fd %>% - group_by(method, dataset_name, metric_name) %>% + filter(backend == 'conda') %>% + group_by(method_full, dataset_name, metric_name) %>% # get the metric_value at the true_k for this group - mutate(true_value = metric_value[k == true_k][1], - deviation_k = metric_value - true_value) %>% + mutate(perf_at_true_value = metric_value[k == true_k][1], + deviation_k = metric_value - perf_at_true_value) %>% ungroup() summary(fd_dev_k$deviation_k) +str(fd_dev_k) ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method)) + geom_point(alpha = 0.6, position = position_jitter(width = 0.15)) + @@ -360,17 +420,18 @@ ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method)) + What about the consistency across runs and seeds? e.g., do different seeds produce different results? -First, seeds +First, seeds. Again conda only. ```{r} fd_seed_summary <- fd %>% - group_by(method, dataset_name, metric_name, k, seed) %>% + filter(backend == 'conda') %>% + group_by(method_full, dataset_name, metric_name, k, seed) %>% summarise(mean_value = mean(metric_value, na.rm = TRUE), # average across runs sd_runs = sd(metric_value, na.rm = TRUE), # replicate consistency .groups = "drop") fd_seed_var <- fd_seed_summary %>% - group_by(method, dataset_name, metric_name, k) %>% + group_by(method_full, dataset_name, metric_name, k) %>% summarise(sd_seeds = sd(mean_value, na.rm = TRUE), # variability across seeds mean_sd_runs = mean(sd_runs, na.rm = TRUE), # replicate consistency .groups = "drop") @@ -379,7 +440,7 @@ fd_seed_var <- fd_seed_summary %>% summary(fd_seed_var) -ggplot(fd_seed_var, aes(x = metric_name, y = sd_seeds, fill = method)) + +ggplot(fd_seed_var, aes(x = metric_name, y = sd_seeds, fill = method_full)) + geom_boxplot(outlier.alpha = 0.3) + theme_minimal(base_size = 14) + labs(title = "seed sensitivity across methods", @@ -392,7 +453,7 @@ ggplot(fd_seed_var, aes(x = metric_name, y = sd_seeds, fill = method)) + Second, runs ```{r} -ggplot(fd_seed_var, aes(x = metric_name, y = mean_sd_runs, fill = method)) + +ggplot(fd_seed_var, aes(x = metric_name, y = mean_sd_runs, fill = method_full)) + geom_boxplot(outlier.alpha = 0.3) + theme_minimal(base_size = 14) + labs(title = "repeated run sensitivity across methods", @@ -412,11 +473,14 @@ print(ggplot(fd_long, aes(x = backend, y = value, fill = backend)) + theme_minimal(base_size = 14) + labs(title = "comp perf by backend") + scale_fill_brewer(palette = "Set2")) +``` + +```{r, fig.width = 20, fig.height = 20} print(ggplot(fd_seed_summary, aes(x = seed, y = mean_value, - group = interaction(method, dataset_name, k), - color = method)) + + group = interaction(method_full, dataset_name, k), + color = method_full)) + geom_line(alpha = 0.5) + geom_point(size = 2) + geom_errorbar(aes(ymin = mean_value - sd_runs, @@ -426,9 +490,11 @@ print(ggplot(fd_seed_summary, theme_minimal(base_size = 14) + labs(title = "Seed sensitivity", subtitle = "Error bars depict runs variability")) +``` +```{r, fig.width = 12} print(ggplot(fd, aes(x = cpu_time, y = metric_value, - color = method, shape = backend)) + + color = method_full, shape = backend)) + geom_point(alpha = 0.6) + facet_wrap(~metric_name, scales = "free_y") + theme_minimal(base_size = 14) + @@ -437,7 +503,7 @@ print(ggplot(fd, aes(x = cpu_time, y = metric_value, y = "clusering metric value")) print(ggplot(fd, aes(x = max_rss, y = metric_value, - color = method, shape = backend)) + + color = method_full, shape = backend)) + geom_point(alpha = 0.6) + facet_wrap(~metric_name, scales = "free_y") + theme_minimal(base_size = 14) + @@ -451,27 +517,10 @@ Bland Altmans, any k vs the true k ```{r, fig.width =10} fd_dev_true <- fd %>% - group_by(method, dataset_name, metric_name) %>% + group_by(method_full, dataset_name, metric_name) %>% # get the metric value at true_k mutate(true_value = metric_value[k == true_k][1], diff_val = metric_value - true_value, mean_val = (metric_value + true_value)/2) %>% ungroup() -# Bland–Altman style plot: any k vs true_k -ggplot(fd_dev_true, aes(x = mean_val, y = diff_val, color = method)) + - geom_point(alpha = 0.6) + - geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE), - linetype = "dashed", color = "blue") + - geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) + 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), - linetype = "dotted", color = "red") + - geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) - 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), - linetype = "dotted", color = "red") + - facet_wrap(~metric_name, scales = "free") + - theme_minimal(base_size = 14) + - labs(title = "Bland–Altman: any k vs true_k", - x = "Mean of k and true_k", - y = "Difference (k - true_k)") - - -``` diff --git a/parse_results.py b/parse_results.py old mode 100755 new mode 100644 index af3c35b..7dda69f --- a/parse_results.py +++ b/parse_results.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 """ -Simple script to parse clustbench results with glob pattern matching. +Parse clustbench results with glob pattern matching. -Pattern: out-{backend}-{rep}/data/clustbench/dataset_generator-{generator}_dataset_name-{name}/clustering/{method} +Pattern: +out-{backend}_seed-{seed}_run-{run}/data/clustbench/dataset_generator-{generator}_dataset_name-{name}/clustering/{method} """ import csv @@ -12,68 +13,90 @@ from pathlib import Path from typing import Dict, List, Optional -def parse_result_path(path: Path) -> Dict[str, str]: + +def parse_result_path(path: Path) -> List[Dict[str, str]]: """ Parse a result path and extract components: - backend, seed, run (from out_* directories) - dataset generator and dataset name - - method (after clustering/) + - method (immediate folder after clustering/) + - method_full (method + variant symlink/subdir) + + Returns a list of dicts, one per available variant directory under {method}. """ parts = path.parts - result: Dict[str, str] = {} + base_result: Dict[str, str] = {} # parse out_{backend}_seed_{seed}_run_{run} out_match = re.match( r"out_(?P[a-zA-Z0-9]+)_seed_(?P\d+)_run_(?P\d+)", - parts[0]) - + parts[0] + ) if out_match: - result["backend"] = out_match.group("backend") - result["seed"] = out_match.group("seed") - result["run"] = out_match.group("run") + base_result["backend"] = out_match.group("backend") + base_result["seed"] = out_match.group("seed") + base_result["run"] = out_match.group("run") - # Find dataset_generator part + # find dataset_generator part for part in parts: if part.startswith("dataset_generator-"): dataset_match = re.match( r"dataset_generator-([^_]+)_dataset_name-(.+)", part ) if dataset_match: - result["generator"] = dataset_match.group(1) - result["dataset_name"] = dataset_match.group(2) + base_result["generator"] = dataset_match.group(1) + base_result["dataset_name"] = dataset_match.group(2) break - # The method is the last part (after clustering/) + results: List[Dict[str, str]] = [] + + # The method is the folder after clustering/ if "clustering" in parts: clustering_idx = parts.index("clustering") if clustering_idx + 1 < len(parts): - result["method"] = parts[clustering_idx + 1] + method_dir = parts[clustering_idx + 1] + base_result["method"] = method_dir + + method_path = path + if method_path.is_dir(): + for child in method_path.iterdir(): + # skip hidden dirs, hashes, and metrics folder + if child.name.startswith("."): + continue + if re.fullmatch(r"[0-9a-f]{32,}", child.name): + continue + if re.fullmatch(r"[0-9a-f]{8,}", child.name): + continue + if child.name == "metrics": + continue + + if child.is_symlink() or child.is_dir(): + r = base_result.copy() + r["method_full"] = f"{method_dir}_{child.name}" + r["path"] = str(child) + results.append(r) + else: + r = base_result.copy() + r["method_full"] = "/".join(parts[clustering_idx + 1:]) + r["path"] = str(path) + results.append(r) - result["path"] = str(path) - return result + return results def parse_performance_file(perf_file: Path) -> Optional[Dict]: - """ - Parse a clustbench_performance.txt file (TSV format). - - Returns: - Dictionary with performance metrics, or None if file doesn't exist - """ + """Parse a clustbench_performance.txt file (TSV format).""" if not perf_file.exists(): return None try: with open(perf_file, 'r') as f: reader = csv.DictReader(f, delimiter='\t') - # Get the first (and only) data row for row in reader: - # Convert values to appropriate types result = {} for key, value in row.items(): if value: value = value.strip() - # Keep h:m:s as string, convert others to float if key == 'h:m:s': result[key] = value else: @@ -91,212 +114,158 @@ def parse_performance_file(perf_file: Path) -> Optional[Dict]: def parse_metric_scores(scores_file: Path) -> Optional[Dict[str, float]]: - """ - Parse a clustbench.scores.gz file. - - Format: - k=2,k=2,k=2,k=3,k=4 - 1.0,1.0,1.0,0.7671742903354675,0.7289468426413069 - - Returns: - Dictionary mapping k values to scores, or None if file doesn't exist - """ + """Parse a clustbench.scores.gz file into {k: score} dict.""" if not scores_file.exists(): return None - + try: with gzip.open(scores_file, 'rt') as f: lines = f.readlines() - + if len(lines) != 2: return {'error': f'Expected 2 lines, got {len(lines)}'} - - # Parse header (k values) - extract integers from "k=2" format - k_strings = [k.strip() for k in lines[0].strip().split(',')] + + k_strings = [k.strip().strip('"') for k in lines[0].strip().split(',')] k_values = [] for k_str in k_strings: - k_str = k_str.strip('"') - match = re.match(r'k=(\d+)', k_str) - if match: - k_values.append(int(match.group(1))) + m = re.match(r'k=(\d+)', k_str) + if m: + k_values.append(int(m.group(1))) else: return {'error': f'Invalid k format: {k_str}'} - - # Parse scores - scores = [float(s.strip()) for s in lines[1].strip().split(',')] - + + score_strings = [s.strip().strip('"') for s in lines[1].strip().split(',')] + scores = [] + for s in score_strings: + try: + scores.append(float(s)) + except ValueError: + return {'error': f'Invalid score: {s}'} + if len(k_values) != len(scores): return {'error': f'Mismatch: {len(k_values)} k values, {len(scores)} scores'} - - # Build result dict, checking for duplicate k values with different scores + result = {} for k, score in zip(k_values, scores): - if k in result: - # Check if the score is different - if abs(result[k] - score) > 1e-10: - raise ValueError(f'Duplicate k value {k} with different scores: {result[k]} vs {score}') - else: - result[k] = score + if k in result and abs(result[k] - score) > 1e-10: + return {'error': f'Duplicate k {k} with differing scores'} + result[k] = score - # ## Find the middle column (index = len//2), that contains the true k, and report it - # mid_idx = len(k_values) // 2 - # result['true_k'] = k_values[mid_idx] return result - + except Exception as e: return {'error': str(e)} -def parse_metrics(param_dir: Path) -> Dict[str, Dict[str, Dict[str, float]]]: - """ - Parse metrics from a parameter directory. - - Structure: {param_dir}/metrics/{metric_family}/metric-{metric_name}/clustbench.scores.gz - - Returns: - Nested dict: {metric_family: {metric_name: {k: score}}} - """ +def parse_metrics(config_dir: Path) -> Dict[str, Dict[str, Dict[str, float]]]: + """Parse metrics from a configuration directory.""" metrics = {} - metrics_dir = param_dir / 'metrics' - + metrics_dir = config_dir / 'metrics' if not metrics_dir.exists(): return metrics - - # Iterate over metric families + for family_dir in metrics_dir.iterdir(): if not family_dir.is_dir(): continue - family_name = family_dir.name metrics[family_name] = {} - - # Iterate over metrics in this family for metric_dir in family_dir.iterdir(): if not metric_dir.is_dir(): continue - - # Extract metric name from metric-{name} pattern metric_match = re.match(r'metric-(.+)', metric_dir.name) if not metric_match: continue - metric_name = metric_match.group(1) - - # Parse the scores file scores_file = metric_dir / 'clustbench.scores.gz' scores = parse_metric_scores(scores_file) - if scores: metrics[family_name][metric_name] = scores - return metrics -def find_results(base_dir: str = '.', pattern: str = 'out_*/data/clustbench/dataset_generator-*/clustering/*') -> List[Dict[str, str]]: +def find_results( + base_dir: str = ".", + pattern: str = "out_*/data/clustbench/dataset_generator-*/clustering/*" +) -> List[Dict[str, str]]: """ - Find all result directories matching the pattern. - - Args: - base_dir: Base directory to search from - pattern: Glob pattern to match - - Returns: - List of parsed result dictionaries + Return one record per configuration folder with parameters, performance, and metrics. """ base_path = Path(base_dir) - results = [] + results: List[Dict[str, str]] = [] for path in base_path.glob(pattern): - if path.is_dir(): - # Skip hidden directories (starting with .) - if not any(part.startswith('.') for part in path.parts): - parsed = parse_result_path(path) - - # Find all parameter directories (subdirectories with parameter patterns) - param_dirs = [d for d in path.iterdir() if d.is_dir() and not d.name.startswith('.')] - - if param_dirs: - # Parse configurations and their performance - parsed['configurations'] = [] - - # Assume first param_dir for method-level data - first_param_dir = param_dirs[0] - - # Parse performance file at method level - perf_file = first_param_dir / 'clustbench_performance.txt' - performance = parse_performance_file(perf_file) - if performance: - parsed['performance'] = performance - - # Parse metrics at method level - metrics = parse_metrics(first_param_dir) - if metrics: - parsed['metrics'] = metrics - - # Add method_params and method_full at method level - method_params = first_param_dir.name - - # Extract method from method-{method} pattern if present - method_match = re.match(r'method-([^_]+)', method_params) - if method_match: - extracted_method = method_match.group(1) - parsed['method'] = extracted_method - - method_full = f"{parsed.get('method', '')}_{method_params}" - parsed['method_params'] = method_params - parsed['method_full'] = method_full - - for param_dir in param_dirs: - # Load parameters.json if it exists - params_file = param_dir / 'parameters.json' - parameters = None - if params_file.exists(): - try: - with open(params_file, 'r') as f: - parameters = json.load(f) - except Exception as e: - parameters = {'error': str(e)} - - config = { - 'parameter_dir': param_dir.name, - 'parameters': parameters - } + if not path.is_dir(): + continue + if any(part.startswith(".") for part in path.parts): + continue - parsed['configurations'].append(config) + variants = parse_result_path(path) + for variant in variants: + config_dir = Path(variant["path"]) + if not config_dir.is_dir() or config_dir.name == "metrics": + continue - results.append(parsed) + record = variant.copy() + + # Parameters + params_file = config_dir / "parameters.json" + parameters = None + if params_file.exists(): + try: + with open(params_file, "r") as f: + parameters = json.load(f) + except Exception as e: + parameters = {"error": str(e)} + record["parameters"] = parameters + record["parameter_dir"] = config_dir.name + + # Performance + perf_file = config_dir / "clustbench_performance.txt" + performance = parse_performance_file(perf_file) + if performance: + record["performance"] = performance + + # Metrics + metrics = parse_metrics(config_dir) + if metrics: + record["metrics"] = metrics + + # Normalize method name + m = re.match(r"method-([^_]+)", config_dir.name) + if m: + record["method"] = m.group(1) + + # Ensure method_full includes config dir name once + variant_name = record["method_full"] + if config_dir.name not in variant_name: + record["method_full"] = f"{variant_name}_{config_dir.name}" + + record["path"] = str(config_dir) + results.append(record) return results def main(): - """Main function to run the parser.""" - # Find all matching results results = find_results() - - # Print as JSON print(json.dumps(results, indent=2)) - # Print summary - print(f"\n# Found {len(results)} result directories", file=__import__('sys').stderr) - - # Group by backend, generator, method - by_backend = {} - by_generator = {} - by_method = {} + # Summary to stderr + import sys + print(f"\n# Found {len(results)} result directories", file=sys.stderr) + by_backend, by_generator, by_method = {}, {}, {} for r in results: backend = r.get('backend', 'unknown') generator = r.get('generator', 'unknown') method = r.get('method', 'unknown') - by_backend[backend] = by_backend.get(backend, 0) + 1 by_generator[generator] = by_generator.get(generator, 0) + 1 by_method[method] = by_method.get(method, 0) + 1 - print(f"# By backend: {by_backend}", file=__import__('sys').stderr) - print(f"# By generator: {by_generator}", file=__import__('sys').stderr) - print(f"# By method: {by_method}", file=__import__('sys').stderr) + print(f"# By backend: {by_backend}", file=sys.stderr) + print(f"# By generator: {by_generator}", file=sys.stderr) + print(f"# By method: {by_method}", file=sys.stderr) if __name__ == '__main__': From 229b823667eaa27e9d3138cae158ce153a12ec52 Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Mon, 15 Dec 2025 12:12:27 +0100 Subject: [PATCH 08/18] Fix early termination --- analyze_results_izaskun.Rmd | 111 +++++++++++++++++++++++++----------- 1 file changed, 77 insertions(+), 34 deletions(-) diff --git a/analyze_results_izaskun.Rmd b/analyze_results_izaskun.Rmd index f410e83..9bf00e5 100644 --- a/analyze_results_izaskun.Rmd +++ b/analyze_results_izaskun.Rmd @@ -171,7 +171,7 @@ print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], x = "Backend", y = "CPU Time (s)") + scale_fill_brewer(palette = "Set2")) - ``` +``` Clearly the method, so all good: @@ -329,23 +329,7 @@ fd_long <- fd %>% names_to = "metric", values_to = "value") -# Bland–Altman style plot: any k vs true_k -ggplot(fd_dev_true, aes(x = mean_val, y = diff_val, color = method)) + - geom_point(alpha = 0.6) + - geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE), - linetype = "dashed", color = "blue") + - geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) + 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), - linetype = "dotted", color = "red") + - geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) - 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), - linetype = "dotted", color = "red") + - facet_wrap(~metric_name, scales = "free") + - theme_minimal(base_size = 14) + - labs(title = "Bland–Altman: any k vs true_k", - x = "Mean of k and true_k", - y = "Difference (k - true_k)") - -``` # boxplots + jittered scatter, faceted by metric ggplot(fd_long, aes(x = backend, y = value, fill = backend)) + geom_boxplot(outlier.alpha = 0.3) + @@ -356,10 +340,14 @@ ggplot(fd_long, aes(x = backend, y = value, fill = backend)) + x = "Backend", y = "Value") + scale_fill_brewer(palette = "Set2") - ``` + + +Something wrong with method "1" etc here? + + ```{r, fig.width = 10, fig.height = 10} fd_long <- fd %>% pivot_longer(cols = all_of(perf_metrics), @@ -389,25 +377,48 @@ ggplot(fd_long, aes(x = backend, y = value)) + ```{r} -table(fd$method, fd$method_full) +table(fd$method) ``` -Choice of `k` impact in clusering perf metrics. Conda only, to speedup. +Choice of `k` - we lack the annotation of the true k in the long CSV + + + + + + + + + + + + + -```{r, fig.width = 20, fig.height = 20} + + + + + + + + + + + + +```{r} fd_dev_k <- fd %>% - filter(backend == 'conda') %>% group_by(method_full, dataset_name, metric_name) %>% # get the metric_value at the true_k for this group - mutate(perf_at_true_value = metric_value[k == true_k][1], - deviation_k = metric_value - perf_at_true_value) %>% + mutate(true_value = metric_value[k == true_k][1], + deviation_k = metric_value - true_value) %>% ungroup() summary(fd_dev_k$deviation_k) str(fd_dev_k) - -ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method)) + +ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method_full)) + geom_point(alpha = 0.6, position = position_jitter(width = 0.15)) + facet_wrap(dataset_name ~ metric_name, scales = "free_y") + theme_minimal(base_size = 14) + @@ -420,11 +431,10 @@ ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method)) + What about the consistency across runs and seeds? e.g., do different seeds produce different results? -First, seeds. Again conda only. +First, seeds ```{r} fd_seed_summary <- fd %>% - filter(backend == 'conda') %>% group_by(method_full, dataset_name, metric_name, k, seed) %>% summarise(mean_value = mean(metric_value, na.rm = TRUE), # average across runs sd_runs = sd(metric_value, na.rm = TRUE), # replicate consistency @@ -465,6 +475,11 @@ ggplot(fd_seed_var, aes(x = metric_name, y = mean_sd_runs, fill = method_full)) Recap +```{r} +colnames(fd) + +``` + ```{r, fig.width = 12} print(ggplot(fd_long, aes(x = backend, y = value, fill = backend)) + geom_boxplot(outlier.alpha = 0.3) + @@ -473,10 +488,7 @@ print(ggplot(fd_long, aes(x = backend, y = value, fill = backend)) + theme_minimal(base_size = 14) + labs(title = "comp perf by backend") + scale_fill_brewer(palette = "Set2")) -``` - -```{r, fig.width = 20, fig.height = 20} print(ggplot(fd_seed_summary, aes(x = seed, y = mean_value, group = interaction(method_full, dataset_name, k), @@ -490,9 +502,7 @@ print(ggplot(fd_seed_summary, theme_minimal(base_size = 14) + labs(title = "Seed sensitivity", subtitle = "Error bars depict runs variability")) -``` -```{r, fig.width = 12} print(ggplot(fd, aes(x = cpu_time, y = metric_value, color = method_full, shape = backend)) + geom_point(alpha = 0.6) + @@ -515,7 +525,7 @@ print(ggplot(fd, aes(x = max_rss, y = metric_value, Bland Altmans, any k vs the true k -```{r, fig.width =10} +```{r, fig.width = 20, fig.height = 20} fd_dev_true <- fd %>% group_by(method_full, dataset_name, metric_name) %>% # get the metric value at true_k @@ -524,3 +534,36 @@ fd_dev_true <- fd %>% mean_val = (metric_value + true_value)/2) %>% ungroup() +# Bland–Altman style plot: any k vs true_k +ggplot(fd_dev_true, aes(x = mean_val, y = diff_val, color = method_full)) + + geom_point(alpha = 0.6) + + geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE), + linetype = "dashed", color = "blue") + + geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) + 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), + linetype = "dotted", color = "red") + + geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) - 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), + linetype = "dotted", color = "red") + + facet_wrap(~metric_name, scales = "free") + + theme_minimal(base_size = 14) + + labs(title = "Bland–Altman: any k vs true_k", + x = "Mean of k and true_k", + y = "Difference (k - true_k)") + +ggplot(fd_dev_true, aes(x = mean_val, y = diff_val, color = dataset_name)) + + geom_point(alpha = 0.6) + + geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE), + linetype = "dashed", color = "blue") + + geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) + 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), + linetype = "dotted", color = "red") + + geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) - 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), + linetype = "dotted", color = "red") + + facet_wrap(~metric_name, scales = "free") + + theme_minimal(base_size = 14) + + labs(title = "Bland–Altman: any k vs true_k", + x = "Mean of k and true_k", + y = "Difference (k - true_k)") + + + +``` + From 71197a094e49a0c337418b0e2ce0f237f5af8cc0 Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Mon, 15 Dec 2025 14:55:08 +0100 Subject: [PATCH 09/18] Add plots, caching, disable self-contained HTML --- analyze_results_izaskun.Rmd | 225 ++++++++++++++++++++++++++++-------- 1 file changed, 180 insertions(+), 45 deletions(-) diff --git a/analyze_results_izaskun.Rmd b/analyze_results_izaskun.Rmd index 9bf00e5..ae96da1 100644 --- a/analyze_results_izaskun.Rmd +++ b/analyze_results_izaskun.Rmd @@ -1,10 +1,22 @@ --- -title: "clustbench figure 2" +title: "clustbench exploratory / fig 2 making" +author: "Izaskun Mallona" output: - html_document: default -date: "`r Sys.Date()`" + html_document: + theme: readable + toc: true + toc_float: true + code_folding: hide + code_download: true + number_sections: true + df_print: default + highlight: tango + keep_md: true + self_contained: false +date: "`r format(Sys.Date(), '%B %d, %Y')`" --- + ```{r setup, message = FALSE} library(knitr) library(tidyverse) @@ -16,11 +28,12 @@ knitr::opts_chunk$set( echo = TRUE, warning = TRUE, message = TRUE, + fig.width = 12, + fig.height = 12, fig.path = "plots/", dev = c("png", "svg"), - fig.width = 10, - fig.height = 10) - + cache.lazy = FALSE, + cache = TRUE) ``` @@ -173,7 +186,7 @@ print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], scale_fill_brewer(palette = "Set2")) ``` -Clearly the method, so all good: +Clearly the method, so all good. With segments following dataset/method/params. ```{r} print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], @@ -231,6 +244,38 @@ print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], ) ``` +**Let's make sure we don't plot methods/datasets/params when one of the backends is not evern recording performances because the run is so quick**: + +@todo check this and perhaps update others + +```{r, fig.width = 30, fig.height = 30} + + + +fd_complete <- fd %>% + filter(backend %in% c("conda","oras","envmodules")) %>% + group_by(method_full, dataset_name, seed, run, generator, k) %>% + # keep only groups that have all 3 backends and no NA cpu_time, + ## even if that means disregarding the superquick tasks that are not profiled by snmk + filter(n_distinct(backend) == 3, !any(is.na(cpu_time))) %>% + ungroup() + +print(ggplot(fd_complete, aes(x = backend, y = cpu_time)) + + geom_boxplot(outlier.alpha = 0.3) + + geom_point(alpha = 0.6, + position = position_jitter(width = 0.15)) + + geom_line( + aes(group = interaction(method, method_full, seed, run, generator, dataset_name, k)), + alpha = 0.1, + color = "grey40" + ) + + facet_wrap(~method_full, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "CPU time by backend (only complete groups)", + x = "Backend", + y = "CPU time (s)")) + +``` ```{r} print( @@ -251,19 +296,6 @@ print( ``` -```{r, eval = FALSE} - -ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], - aes(x = backend, y = metric_value, color = backend)) + - geom_boxplot(outlier.alpha = 0.3) + - facet_wrap(~metric_name, scales = "free_y") + - theme_minimal(base_size = 14) + - labs(title = "Metrics consistency across backends", - x = "Backend", - y = "Metric Value") + - scale_color_brewer(palette = "Set1") -``` - ```{r, fig.width = 20, fig.height = 20} @@ -291,6 +323,10 @@ print( ``` + + + + ```{r} df_run <- fd %>% select(dataset_name, method, k, metric_name, run, seed, backend, metric_value) @@ -342,7 +378,31 @@ ggplot(fd_long, aes(x = backend, y = value, fill = backend)) + scale_fill_brewer(palette = "Set2") ``` +Now the same but making sure only items that are measured across all backends (so no NAs because they're too quick and not profiled) are shown +```{r, fig.width = 10, fig.height=10} + +fd_long_complete <- fd %>% + filter(backend %in% c("conda","oras","envmodules")) %>% + pivot_longer(cols = all_of(perf_metrics), + names_to = "metric", + values_to = "value") %>% + group_by(method_full, dataset_name, seed, run, generator, k, metric) %>% + filter(n_distinct(backend) == 3, !any(is.na(value))) %>% + ungroup() + +ggplot(fd_long_complete, aes(x = backend, y = value, fill = backend)) + + geom_boxplot(outlier.alpha = 0.3) + + geom_jitter(width = 0.2, alpha = 0.1, size = 1, color = "black") + + facet_wrap(~metric, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "Perf metrics by backend (only complete groups)", + x = "backend", + y = "perf value") + + scale_fill_brewer(palette = "Set2") + + +``` Something wrong with method "1" etc here? @@ -354,6 +414,8 @@ fd_long <- fd %>% names_to = "metric", values_to = "value") +table(fd_long$method, fd_long$method_full) + ggplot(fd_long, aes(x = backend, y = value)) + # boxplots filled by backend ## geom_boxplot(aes(fill = backend), outlier.alpha = 0.3) + @@ -382,31 +444,6 @@ table(fd$method) Choice of `k` - we lack the annotation of the true k in the long CSV - - - - - - - - - - - - - - - - - - - - - - - - - ```{r} fd_dev_k <- fd %>% @@ -567,3 +604,101 @@ ggplot(fd_dev_true, aes(x = mean_val, y = diff_val, color = dataset_name)) + ``` +More Bland Altmans + +```{r, fig.width = 50, fig.height = 30} +ggplot(fd_dev_true, + aes(x = mean_val, y = diff_val, color = method_full)) + # color by method_full + geom_point(alpha = 0.6) + + geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE), + linetype = "dashed", color = "blue") + + geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) + + 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), + linetype = "dotted", color = "red") + + geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) - + 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), + linetype = "dotted", color = "red") + + facet_grid(metric_name ~ dataset_name, scales = "free") + + theme_minimal(base_size = 14) + + labs(title = "Bland–Altman: any k vs true_k", + x = "Mean of k and true_k", + y = "Difference (k - true_k)") + +``` + +ks again + +```{r, fig.width = 30, fig.height = 30} +true_vals <- fd %>% + filter(k == true_k) %>% + select(method_full, dataset_name, metric_name, run, seed, true_value = metric_value) + +fd_dev_k <- fd %>% + group_by(method_full, dataset_name, metric_name, run, seed) %>% + mutate(true_value = first(metric_value[k == true_k]), + deviation_k = metric_value - true_value) %>% + ungroup() + +ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method_full)) + + geom_point(alpha = 0.6, position = position_jitter(width = 0.15)) + + facet_grid(dataset_name ~ metric_name, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "Deviation of metric across ks", + subtitle = "Deviation vs value at true k", + x = "k", + y = "Perf metric deviation from true k") + + scale_color_brewer(palette = "Set1") + +``` + +seeds and runs again, only true k + + +```{r} + +fd_true <- fd %>% filter(k == true_k) + +fd_dev_seed <- fd_true %>% + group_by(method_full, dataset_name, metric_name, run) %>% + mutate( + mean_true_val = mean(metric_value, na.rm = TRUE), + deviation_seed = metric_value - mean_true_val + ) %>% + ungroup() + +## floats strike again... +tol <- sqrt(.Machine$double.eps) +fd_dev_seed <- fd_dev_seed %>% + mutate(deviation_seed = ifelse(abs(deviation_seed) < tol, 0, deviation_seed)) + + +summary(fd_dev_seed) + + +ggplot(fd_dev_seed, aes(x = factor(seed), y = deviation_seed, color = method_full)) + + geom_point(alpha = 0.6) + + theme_minimal(base_size = 14) + + labs(title = "Deviation at true k across seeds", + x = "Seed", + y = "Deviation from mean across seeds") + + scale_color_brewer(palette = "Set1") + + +fd_dev_run <- fd_true %>% + group_by(method_full, dataset_name, metric_name, seed) %>% + mutate( + mean_true_val = mean(metric_value, na.rm = TRUE), + deviation_run = metric_value - mean_true_val + ) %>% + ungroup() + +tol <- sqrt(.Machine$double.eps) +fd_dev_run <- fd_dev_run %>% + mutate(deviation_run = ifelse(abs(deviation_run) < tol, 0, deviation_run)) + +summary(fd_dev_run) + +``` + + +Hmm, either I'm getting only deterministic methods or something in calcs is wrong, let's rerun with more methods From ef63a58202645a497f60781868e4419df56f0b86 Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Mon, 15 Dec 2025 15:42:55 +0100 Subject: [PATCH 10/18] Organize a bit, fix typos --- analyze_results_izaskun.Rmd | 295 ++++++++++++++++-------------------- 1 file changed, 131 insertions(+), 164 deletions(-) diff --git a/analyze_results_izaskun.Rmd b/analyze_results_izaskun.Rmd index ae96da1..2696d0c 100644 --- a/analyze_results_izaskun.Rmd +++ b/analyze_results_izaskun.Rmd @@ -28,8 +28,8 @@ knitr::opts_chunk$set( echo = TRUE, warning = TRUE, message = TRUE, - fig.width = 12, - fig.height = 12, + fig.width = 10, + fig.height = 10, fig.path = "plots/", dev = c("png", "svg"), cache.lazy = FALSE, @@ -138,21 +138,20 @@ flatten_record <- function(rec) { records <- fromJSON("aggregated_results.json", simplifyVector = FALSE) -# fd_list <- vector("list", length(records)) for (i in seq_along(records)) { - if (i == 1 || i == 2 || i %% 250 == 0) + if (i == 1 || i == 2 || i == 100 || i %% 1000 == 0) cat("Processing record", i, "of", length(records), "\n") fd_list[[i]] <- flatten_record(records[[i]]) } fd <- data.table::rbindlist(fd_list, use.names = TRUE, fill = TRUE) -dim(fd) +## dim(fd) -str(fd) -table(is.na(fd$max_rss)) -table(fd$method_full) -table(vapply(records, function(x) x$method_full, character(1))) +## str(fd) +## table(is.na(fd$max_rss)) +## table(fd$method_full) +## table(vapply(records, function(x) x$method_full, character(1))) fd <- as.data.frame(fd) @@ -172,8 +171,9 @@ write.csv(fd, file = 'aggregated_results.csv') ``` +# QC -Oops, is the speed of computing the metric, or of running the method? +Is the speed of computing the metric, or of running the method? ```{r} print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], @@ -206,25 +206,28 @@ print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], ) ``` - - -```{r} -print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], - aes(x = backend, y = cpu_time)) + - geom_boxplot(outlier.alpha = 0.3) + - # add points for each method/params combination - geom_point(aes(color = method), - alpha = 0.6, position = position_jitter(width = 0.15)) + - # connect points across backends for same module+params - geom_line(aes(group = interaction(method, method_full, seed, run, generator, dataset_name, k)), - alpha = 0.1, color = "grey40") + - theme_minimal(base_size = 14) + - labs(title = "CPU time by backend", - x = "Backend", - y = "CPU time (s)") + - scale_color_brewer(palette = "Set1") -) -``` +# CPU by backend + +## Noncomplete observations + + + + + + + + + + + + + + + + + + + ```{r} print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], @@ -244,14 +247,11 @@ print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], ) ``` -**Let's make sure we don't plot methods/datasets/params when one of the backends is not evern recording performances because the run is so quick**: - -@todo check this and perhaps update others - -```{r, fig.width = 30, fig.height = 30} - +## Complete observations +Let's make sure we don't plot methods/datasets/params when one of the backends is not even recording performances because the run is so quick it ends before it's profiled: +```{r, fig.width = 15, fig.height = 15} fd_complete <- fd %>% filter(backend %in% c("conda","oras","envmodules")) %>% group_by(method_full, dataset_name, seed, run, generator, k) %>% @@ -262,7 +262,8 @@ fd_complete <- fd %>% print(ggplot(fd_complete, aes(x = backend, y = cpu_time)) + geom_boxplot(outlier.alpha = 0.3) + - geom_point(alpha = 0.6, + geom_point(aes(color = method_full), + alpha = 0.6, position = position_jitter(width = 0.15)) + geom_line( aes(group = interaction(method, method_full, seed, run, generator, dataset_name, k)), @@ -279,7 +280,7 @@ print(ggplot(fd_complete, aes(x = backend, y = cpu_time)) + ```{r} print( - ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], + ggplot(fd_complete[fd_complete$backend %in% c("conda","oras","envmodules"), ], aes(x = backend, y = cpu_time)) + geom_boxplot(outlier.alpha = 0.3) + geom_point(aes(color = method_full), @@ -290,15 +291,17 @@ print( facet_wrap(~method, scales = "free_y") + labs(title = "CPU time by backend", x = "Backend", - y = "CPU time (s)") + - scale_color_brewer(palette = "Set1") + y = "CPU time (s)") ) ``` +# Results consistency across backends -```{r, fig.width = 20, fig.height = 20} +## Unreadable perf metrics plot + +```{r, fig.width = 30, fig.height = 30} print( ggplot(fd, aes(x = backend, y = metric_value)) + @@ -322,25 +325,20 @@ print( ``` - - - - +# Pairwise correlations ```{r} df_run <- fd %>% select(dataset_name, method, k, metric_name, run, seed, backend, metric_value) -# a bit of collapsing here! +# a bit of collapsing here wide_run <- fd %>% filter(backend %in% c("conda","oras","envmodules")) %>% group_by(dataset_name, method, k, metric_name, run, seed, backend) %>% summarise(metric_value = mean(as.numeric(metric_value), na.rm = TRUE), .groups = "drop") %>% pivot_wider(names_from = backend, values_from = metric_value) -str(wide_run) - cors_run <- wide_run %>% group_by(metric_name) %>% summarise( @@ -353,6 +351,9 @@ print(cors_run) ``` +# Computational performance metrics + +## Non-complete obs ```{r, fig.width = 10, fig.height=10} @@ -378,6 +379,8 @@ ggplot(fd_long, aes(x = backend, y = value, fill = backend)) + scale_fill_brewer(palette = "Set2") ``` +## Complete obs + Now the same but making sure only items that are measured across all backends (so no NAs because they're too quick and not profiled) are shown ```{r, fig.width = 10, fig.height=10} @@ -404,6 +407,7 @@ ggplot(fd_long_complete, aes(x = backend, y = value, fill = backend)) + ``` +## Non-complete observations colored by method provider, with a bug Something wrong with method "1" etc here? @@ -414,7 +418,7 @@ fd_long <- fd %>% names_to = "metric", values_to = "value") -table(fd_long$method, fd_long$method_full) +## table(fd_long$method, fd_long$method_full) ggplot(fd_long, aes(x = backend, y = value)) + # boxplots filled by backend @@ -437,12 +441,11 @@ ggplot(fd_long, aes(x = backend, y = value)) + ``` -```{r} +# Choice of `k` - and its misspecification -table(fd$method) -``` +## Deviation of performance at false k vs true k -Choice of `k` - we lack the annotation of the true k in the long CSV +Mind there is no such a thing as true k, we use the first labelset. ```{r} @@ -454,10 +457,10 @@ fd_dev_k <- fd %>% ungroup() summary(fd_dev_k$deviation_k) -str(fd_dev_k) +## str(fd_dev_k) ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method_full)) + geom_point(alpha = 0.6, position = position_jitter(width = 0.15)) + - facet_wrap(dataset_name ~ metric_name, scales = "free_y") + + facet_wrap(~ metric_name, scales = "free_y") + theme_minimal(base_size = 14) + labs(title = "Deviation of metric across ks", subtitle = "Deviation vs value at true k", @@ -466,79 +469,85 @@ ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method_full)) + scale_color_brewer(palette = "Set1") ``` +# Impact of seeds and repeated runs + + What about the consistency across runs and seeds? e.g., do different seeds produce different results? -First, seeds +Only at true k. -```{r} -fd_seed_summary <- fd %>% - group_by(method_full, dataset_name, metric_name, k, seed) %>% - summarise(mean_value = mean(metric_value, na.rm = TRUE), # average across runs - sd_runs = sd(metric_value, na.rm = TRUE), # replicate consistency - .groups = "drop") -fd_seed_var <- fd_seed_summary %>% - group_by(method_full, dataset_name, metric_name, k) %>% - summarise(sd_seeds = sd(mean_value, na.rm = TRUE), # variability across seeds - mean_sd_runs = mean(sd_runs, na.rm = TRUE), # replicate consistency - .groups = "drop") +```{r} -summary(fd_seed_var) +fd_true <- fd %>% filter(k == true_k) +fd_dev_seed <- fd_true %>% + group_by(method_full, dataset_name, metric_name, run) %>% + mutate( + mean_true_val = mean(metric_value, na.rm = TRUE), + deviation_seed = metric_value - mean_true_val + ) %>% + ungroup() -ggplot(fd_seed_var, aes(x = metric_name, y = sd_seeds, fill = method_full)) + - geom_boxplot(outlier.alpha = 0.3) + - theme_minimal(base_size = 14) + - labs(title = "seed sensitivity across methods", - x = "Metric", - y = "seed-driven sd") + - scale_fill_brewer(palette = "Set2") +## floats strike again... +tol <- sqrt(.Machine$double.eps) +fd_dev_seed <- fd_dev_seed %>% + mutate(deviation_seed = ifelse(abs(deviation_seed) < tol, 0, deviation_seed)) -``` -Second, runs +summary(fd_dev_seed) -```{r} -ggplot(fd_seed_var, aes(x = metric_name, y = mean_sd_runs, fill = method_full)) + - geom_boxplot(outlier.alpha = 0.3) + + +ggplot(fd_dev_seed, aes(x = factor(seed), y = deviation_seed, color = method_full)) + + geom_point(alpha = 0.6) + theme_minimal(base_size = 14) + - labs(title = "repeated run sensitivity across methods", - x = "Metric", - y = "run-driven sd") + - scale_fill_brewer(palette = "Set2") -``` + labs(title = "Deviation at true k across seeds", + x = "Seed", + y = "Deviation from mean across seeds") -Recap +fd_dev_run <- fd_true %>% + group_by(method_full, dataset_name, metric_name, seed) %>% + mutate( + mean_true_val = mean(metric_value, na.rm = TRUE), + deviation_run = metric_value - mean_true_val + ) %>% + ungroup() + +tol <- sqrt(.Machine$double.eps) +fd_dev_run <- fd_dev_run %>% + mutate(deviation_run = ifelse(abs(deviation_run) < tol, 0, deviation_run)) -```{r} -colnames(fd) +summary(fd_dev_run) ``` -```{r, fig.width = 12} -print(ggplot(fd_long, aes(x = backend, y = value, fill = backend)) + - geom_boxplot(outlier.alpha = 0.3) + - geom_jitter(width = 0.2, alpha = 0.1, size = 1) + - facet_wrap(~metric, scales = "free_y") + - theme_minimal(base_size = 14) + - labs(title = "comp perf by backend") + - scale_fill_brewer(palette = "Set2")) +# Comp vs clustering performance trade-offs -print(ggplot(fd_seed_summary, - aes(x = seed, y = mean_value, - group = interaction(method_full, dataset_name, k), - color = method_full)) + - geom_line(alpha = 0.5) + - geom_point(size = 2) + - geom_errorbar(aes(ymin = mean_value - sd_runs, - ymax = mean_value + sd_runs), - width = 0.2, alpha = 0.4) + - facet_wrap(~metric_name, scales = "free_y") + - theme_minimal(base_size = 14) + - labs(title = "Seed sensitivity", - subtitle = "Error bars depict runs variability")) + +```{r, fig.width = 12} +## print(ggplot(fd_long, aes(x = backend, y = value, fill = backend)) + +## geom_boxplot(outlier.alpha = 0.3) + +## geom_jitter(width = 0.2, alpha = 0.1, size = 1) + +## facet_wrap(~metric, scales = "free_y") + +## theme_minimal(base_size = 14) + +## labs(title = "comp perf by backend") + +## scale_fill_brewer(palette = "Set2")) + +## print(ggplot(fd_seed_summary, +## aes(x = seed, y = mean_value, +## group = interaction(method_full, dataset_name, k), +## color = method_full)) + +## geom_line(alpha = 0.5) + +## geom_point(size = 2) + +## geom_errorbar(aes(ymin = mean_value - sd_runs, +## ymax = mean_value + sd_runs), +## width = 0.2, alpha = 0.4) + +## facet_wrap(~metric_name, scales = "free_y") + +## theme_minimal(base_size = 14) + +## labs(title = "Seed sensitivity", +## subtitle = "Error bars depict runs variability")) print(ggplot(fd, aes(x = cpu_time, y = metric_value, color = method_full, shape = backend)) + @@ -547,7 +556,7 @@ print(ggplot(fd, aes(x = cpu_time, y = metric_value, theme_minimal(base_size = 14) + labs(title = "clustering metrics vs runtime trade‑offs", x = "CPU time (s)", - y = "clusering metric value")) + y = "clustering metric value")) print(ggplot(fd, aes(x = max_rss, y = metric_value, color = method_full, shape = backend)) + @@ -560,9 +569,13 @@ print(ggplot(fd, aes(x = max_rss, y = metric_value, ``` -Bland Altmans, any k vs the true k +# Bland Altmans for ks vs true k + +Again there is no such a thing as a true k -```{r, fig.width = 20, fig.height = 20} +## Colored by method + +```{r, fig.width = 15, fig.height = 15} fd_dev_true <- fd %>% group_by(method_full, dataset_name, metric_name) %>% # get the metric value at true_k @@ -585,7 +598,11 @@ ggplot(fd_dev_true, aes(x = mean_val, y = diff_val, color = method_full)) + labs(title = "Bland–Altman: any k vs true_k", x = "Mean of k and true_k", y = "Difference (k - true_k)") +``` + +## Colored by dataset +```{r, fig.width = 15, fig.height = 15} ggplot(fd_dev_true, aes(x = mean_val, y = diff_val, color = dataset_name)) + geom_point(alpha = 0.6) + geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE), @@ -604,6 +621,8 @@ ggplot(fd_dev_true, aes(x = mean_val, y = diff_val, color = dataset_name)) + ``` +## Grid of Bland Altmans - misspecified k + More Bland Altmans ```{r, fig.width = 50, fig.height = 30} @@ -626,7 +645,7 @@ ggplot(fd_dev_true, ``` -ks again +## Effects of misspecifying `k`, unreadable QC plot ```{r, fig.width = 30, fig.height = 30} true_vals <- fd %>% @@ -650,55 +669,3 @@ ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method_full)) + scale_color_brewer(palette = "Set1") ``` - -seeds and runs again, only true k - - -```{r} - -fd_true <- fd %>% filter(k == true_k) - -fd_dev_seed <- fd_true %>% - group_by(method_full, dataset_name, metric_name, run) %>% - mutate( - mean_true_val = mean(metric_value, na.rm = TRUE), - deviation_seed = metric_value - mean_true_val - ) %>% - ungroup() - -## floats strike again... -tol <- sqrt(.Machine$double.eps) -fd_dev_seed <- fd_dev_seed %>% - mutate(deviation_seed = ifelse(abs(deviation_seed) < tol, 0, deviation_seed)) - - -summary(fd_dev_seed) - - -ggplot(fd_dev_seed, aes(x = factor(seed), y = deviation_seed, color = method_full)) + - geom_point(alpha = 0.6) + - theme_minimal(base_size = 14) + - labs(title = "Deviation at true k across seeds", - x = "Seed", - y = "Deviation from mean across seeds") + - scale_color_brewer(palette = "Set1") - - -fd_dev_run <- fd_true %>% - group_by(method_full, dataset_name, metric_name, seed) %>% - mutate( - mean_true_val = mean(metric_value, na.rm = TRUE), - deviation_run = metric_value - mean_true_val - ) %>% - ungroup() - -tol <- sqrt(.Machine$double.eps) -fd_dev_run <- fd_dev_run %>% - mutate(deviation_run = ifelse(abs(deviation_run) < tol, 0, deviation_run)) - -summary(fd_dev_run) - -``` - - -Hmm, either I'm getting only deterministic methods or something in calcs is wrong, let's rerun with more methods From 1995ab67483f4238a4e607387aceae01c1f2eb31 Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Wed, 17 Dec 2025 10:36:43 +0100 Subject: [PATCH 11/18] Add censored cpu_time plot, add consistency checks, document possible bug --- analyze_results_izaskun.Rmd | 171 ++++++++++++++++++++---------------- 1 file changed, 95 insertions(+), 76 deletions(-) diff --git a/analyze_results_izaskun.Rmd b/analyze_results_izaskun.Rmd index 2696d0c..63bd201 100644 --- a/analyze_results_izaskun.Rmd +++ b/analyze_results_izaskun.Rmd @@ -166,7 +166,7 @@ fd[cols_to_num] <- lapply(fd[cols_to_num], function(x) { as.numeric(x) }) - +fd$k_offset <- fd$k - fd$true_k write.csv(fd, file = 'aggregated_results.csv') ``` @@ -206,97 +206,116 @@ print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], ) ``` -# CPU by backend +# Consistency checks -## Noncomplete observations - - - - - - - - - - - - - - - - - - - +Are there cputimes that are redudant/incosistent for repeated runs? ```{r} -print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], - aes(x = backend, y = cpu_time)) + - geom_boxplot(outlier.alpha = 0.3) + - # add points for each method/params combination - geom_point(aes(color = method_full), - alpha = 0.6, position = position_jitter(width = 0.15)) + - # connect points across backends for same module+params - geom_line(aes(group = interaction(method, method_full, seed, run, generator, dataset_name, k)), - alpha = 0.1, color = "grey40") + - theme_minimal(base_size = 14) + - labs(title = "CPU time by backend", - x = "Backend", - y = "CPU time (s)") + - scale_color_brewer(palette = "Set1") -) +str(fd) + +keys <- c("dataset_name", "method", "method_full", "generator", + "backend", "k", "seed", "run") + +fd_inconsistencies <- fd %>% + group_by(across(all_of(keys))) %>% + summarise( + n_cpu_time = n_distinct(cpu_time, na.rm = TRUE), + cpu_times = list(unique(cpu_time)), + .groups = "drop" + ) %>% + filter(n_cpu_time > 1) + +stopifnot(nrow(fd_inconsistencies) == 0) ``` -## Complete observations +Seed consistencies -Let's make sure we don't plot methods/datasets/params when one of the backends is not even recording performances because the run is so quick it ends before it's profiled: +```{r} +keys <- c("dataset_name", "method", "method_full", "generator", + "backend", "run", "k", "metric_name") -```{r, fig.width = 15, fig.height = 15} -fd_complete <- fd %>% - filter(backend %in% c("conda","oras","envmodules")) %>% - group_by(method_full, dataset_name, seed, run, generator, k) %>% - # keep only groups that have all 3 backends and no NA cpu_time, - ## even if that means disregarding the superquick tasks that are not profiled by snmk - filter(n_distinct(backend) == 3, !any(is.na(cpu_time))) %>% - ungroup() +fd_seed_diff <- fd %>% + group_by(across(all_of(keys))) %>% + summarise( + n_seeds = n_distinct(seed), + n_metric_values = n_distinct(metric_value, na.rm = TRUE), + metric_values = list(unique(metric_value)), + .groups = "drop" + ) %>% + filter(n_seeds > 1 & n_metric_values > 1) -print(ggplot(fd_complete, aes(x = backend, y = cpu_time)) + - geom_boxplot(outlier.alpha = 0.3) + - geom_point(aes(color = method_full), - alpha = 0.6, - position = position_jitter(width = 0.15)) + - geom_line( - aes(group = interaction(method, method_full, seed, run, generator, dataset_name, k)), - alpha = 0.1, - color = "grey40" - ) + - facet_wrap(~method_full, scales = "free_y") + - theme_minimal(base_size = 14) + - labs(title = "CPU time by backend (only complete groups)", - x = "Backend", - y = "CPU time (s)")) +stopifnot(nrow(fd_seed_diff) == 0) +``` +`k` inconsistencies + + +```{r} +stopifnot(range(fd$k_offset) == c(-2, 2)) ``` +Are seeds ok? why is `fcps_method-FCPS_MinEnergy_seed-2` a method name, which seed is that? **looks wrong** + ```{r} -print( - ggplot(fd_complete[fd_complete$backend %in% c("conda","oras","envmodules"), ], - aes(x = backend, y = cpu_time)) + - geom_boxplot(outlier.alpha = 0.3) + - geom_point(aes(color = method_full), - alpha = 0.6, position = position_jitter(width = 0.15)) + - geom_line(aes(group = interaction(method, method_full, seed, run, generator, dataset_name, k)), - alpha = 0.1, color = "grey40") + - theme_minimal(base_size = 14) + - facet_wrap(~method, scales = "free_y") + - labs(title = "CPU time by backend", - x = "Backend", - y = "CPU time (s)") -) +table(fd$method_full, fd$seed) ``` + +# CPU by backend + +## Censoring aware : cpu_time < 0.05 plotted as 0.05 + +We impute NA cpu_time as 0.05 s + +```{r} + +min(fd$cpu_time, na.rm = TRUE) +fd$imputed_cpu_time <- ifelse(is.na(fd$cpu_time), no = fd$cpu_time, yes = 0.05) +fd$censored_cpu_time <- is.na(fd$cpu_time) + +## also, we average for repeated runs, seeds and ks + +fd_avg <- fd %>% + filter(backend %in% c("conda","oras","envmodules")) %>% + group_by(backend, method, method_full, generator, dataset_name) %>% + summarise(imputed_cpu_time = mean(imputed_cpu_time, na.rm = TRUE), + censored_cpu_time = any(censored_cpu_time), + .groups = "drop") + + +## str(fd_avg) +## head(fd_avg) + +ggplot(fd_avg, aes(x = backend, y = imputed_cpu_time)) + + geom_boxplot(outlier.alpha = 0.3) + + geom_point(aes(color = method_full, shape = censored_cpu_time), + alpha = 0.6, position = position_jitter(width = 0.15)) + + geom_line(aes(group = interaction(method, method_full, generator, dataset_name)), + alpha = 0.2, color = "grey40") + + theme_minimal(base_size = 14) + + facet_wrap(~method, scales = "free_y") + + labs(title = "CPU time by backend - censored", + x = "Backend", + y = "CPU time (s)") + + scale_y_sqrt() + + +ggplot(fd_avg, aes(x = backend, y = imputed_cpu_time)) + + geom_boxplot(outlier.alpha = 0.3) + + geom_point(aes(color = method_full, shape = censored_cpu_time), + alpha = 0.6, position = position_jitter(width = 0.15)) + + geom_line(aes(group = interaction(method, method_full, generator, dataset_name)), + alpha = 0.2, color = "grey40") + + theme_minimal(base_size = 14) + + facet_wrap(~method, scales = "free_y") + + labs(title = "CPU time by backend - censored", + x = "Backend", + y = "CPU time (s)") + +``` + # Results consistency across backends ## Unreadable perf metrics plot From de9fc914525d07f150d540eb7dd77cf14bdf0610 Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Wed, 17 Dec 2025 13:05:30 +0100 Subject: [PATCH 12/18] Fix seeds in Makefile, before they were ran all with seed 2 --- Makefile | 76 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index 7180fce..0a98e47 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ # ============================================================ -MAX_CORES ?= 50 +MAX_CORES ?= 250 # EasyBuild installation prefix (imallona; edit accordingly) ## <------------------------------------!!!! EASYBUILD_PREFIX ?= /data/imallona/.local/easybuild @@ -44,6 +44,12 @@ CLUSTERING_DIR = clustering_example REPORTS_REPO = https://github.com/imallona/clustering_report REPORTS_DIR = clustering_report +## seeds to explore +SEEDS := 2 54 546 744 1443 + +## repeated runs per seed +RUNS := 1 2 3 + all: clone_yamls clone_reports run_conda run_oras run_envs knit_report # clone the clustering_example repo if not already present @@ -67,56 +73,52 @@ clone_reports: fi run_conda: clone_yamls - @for seed in 2 54 546 744 1443; do \ + mkdir -p results + @for seed in $(SEEDS); do \ echo "Running conda benchmark with seed $$seed..."; \ cp $(CLUSTERING_DIR)/Clustering_conda.yml $(CLUSTERING_DIR)/Clustering_conda_tmp.yml; \ - sed -i "s/--seed, [0-9]\+/--seed, $$seed/" $(CLUSTERING_DIR)/Clustering_conda_tmp.yml; \ - for i in 1 2 3; do \ - echo " Run $$i for seed $$seed..."; \ - ${OB_CMD} -b $(CLUSTERING_DIR)/Clustering_conda_tmp.yml; \ - cp $(CLUSTERING_DIR)/Clustering_conda_tmp.yml out; \ - mv out out_conda_seed_$$seed\_run_$$i; \ + sed -i "s/--seed\",[[:space:]]*[0-9]\+/--seed\", $$seed/" $(CLUSTERING_DIR)/Clustering_conda_tmp.yml; \ + for i in $(RUNS); do \ + echo " Run $$i for seed $$seed and run $$i."; \ + echo "DEST: results/out_conda_seed_$$seed\_run_$$i" ;\ + ${OB_CMD} -b $(CLUSTERING_DIR)/Clustering_conda_tmp.yml --out-dir results/out_conda_seed_$$seed\_run_$$i; \ + mv $(CLUSTERING_DIR)/Clustering_conda_tmp.yml results/out_conda_seed_$$seed\_run_$$i/; \ done; \ - rm $(CLUSTERING_DIR)/Clustering_conda_tmp.yml; \ done run_oras: clone_yamls - @for seed in 2 54 546 744 1443; do \ + @for seed in $(SEEDS); do \ echo "Running oras benchmark with seed $$seed..."; \ cp $(CLUSTERING_DIR)/Clustering_oras.yml $(CLUSTERING_DIR)/Clustering_oras_tmp.yml; \ - sed -i "s/--seed, [0-9]\+/--seed, $$seed/" $(CLUSTERING_DIR)/Clustering_oras_tmp.yml; \ - for i in 1 2 3; do \ - echo " Run $$i for seed $$seed..."; \ - ${OB_CMD} -b $(CLUSTERING_DIR)/Clustering_oras_tmp.yml; \ - cp $(CLUSTERING_DIR)/Clustering_oras_tmp.yml out; \ - mv out out_oras_seed_$$seed\_run_$$i; \ + sed -i "s/--seed\",[[:space:]]*[0-9]\+/--seed\", $$seed/" $(CLUSTERING_DIR)/Clustering_oras_tmp.yml; \ + for i in $(RUNS); do \ + echo " Run $$i for seed $$seed and run $$i."; \ + ${OB_CMD} -b $(CLUSTERING_DIR)/Clustering_oras_tmp.yml --out-dir results/out_oras_seed_$$seed\_run_$$i/; \ + mv $(CLUSTERING_DIR)/Clustering_oras_tmp.yml results/out_oras_seed_$$seed\_run_$$i/; \ done; \ - rm $(CLUSTERING_DIR)/Clustering_oras_tmp.yml; \ done run_envs: clone_yamls @bash -c '\ - source /cvmfs/software.eessi.io/versions/2025.06/init/lmod/bash && \ - module load EESSI-extend/2025.06-easybuild && \ - export MODULEPATH="$(EASYBUILD_PREFIX)/software/modules/all:$$MODULEPATH" && \ - module use $$MODULEPATH && \ - echo $$MODULEPATH && \ - for seed in 2 54 546 744 1443; do \ - echo "Running envmodules benchmark with seed $$seed..."; \ - cp $(CLUSTERING_DIR)/Clustering_envmodules.yml $(CLUSTERING_DIR)/Clustering_envmodules_tmp.yml; \ - sed -i "s/--seed, [0-9]\+/--seed, $$seed/" $(CLUSTERING_DIR)/Clustering_envmodules_tmp.yml; \ - for i in 1 2 3; do \ - echo " Run $$i for seed $$seed..."; \ - ${OB_CMD} -b $(CLUSTERING_DIR)/Clustering_envmodules_tmp.yml; \ - cp $(CLUSTERING_DIR)/Clustering_envmodules_tmp.yml out; \ - mv out out_envmodules_seed_$$seed\_run_$$i; \ - done; \ - rm $(CLUSTERING_DIR)/Clustering_envmodules_tmp.yml; \ - done \ - ' + source /cvmfs/software.eessi.io/versions/2025.06/init/lmod/bash && \ + module load EESSI-extend/2025.06-easybuild && \ + export MODULEPATH="$(EASYBUILD_PREFIX)/software/modules/all:$$MODULEPATH" && \ + module use $$MODULEPATH && \ + echo $$MODULEPATH && \ + for seed in $(SEEDS); do \ + echo "Running envmodules benchmark with seed $$seed..."; \ + cp $(CLUSTERING_DIR)/Clustering_envmodules.yml $(CLUSTERING_DIR)/Clustering_envmodules_tmp.yml; \ + sed -i "s/--seed\",[[:space:]]*[0-9]\+/--seed\", $$seed/" $(CLUSTERING_DIR)/Clustering_envmodules_tmp.yml; \ + for i in $(RUNS); do \ + echo " Run $$i for seed $$seed and run $$i..."; \ + ${OB_CMD} -b $(CLUSTERING_DIR)/Clustering_envmodules_tmp.yml --out-dir results/out_envmodules_seed_$$seed\_run_$$i/; \ + mv $(CLUSTERING_DIR)/Clustering_envmodules_tmp.yml results/out_envmodules_seed_$$seed\_run_$$i/; \ + done; \ + done \ + ' knit_report: clone_reports - # R -e 'rmarkdown::render("$(REPORTS_DIR)/07_metrics_across_backends.Rmd", params = list(performance_bn = "performance-results.rds", metrics_bn = "metrics-results.rds", clustering_dir = "."))' - # R -e 'rmarkdown::render("$(REPORTS_DIR)/08_performances_across_backends.Rmd", params = list(performance_bn = "performance-results.rds", metrics_bn = "metrics-results.rds", clustering_dir = "."))' + ## R -e 'rmarkdown::render("$(REPORTS_DIR)/07_metrics_across_backends.Rmd", params = list(performance_bn = "performance-results.rds", metrics_bn = "metrics-results.rds", clustering_dir = "."))' + ## R -e 'rmarkdown::render("$(REPORTS_DIR)/08_performances_across_backends.Rmd", params = list(performance_bn = "performance-results.rds", metrics_bn = "metrics-results.rds", clustering_dir = "."))' python parse_results.py > aggregated_results.json R -e 'rmarkdown::render("analyze_results_izaskun.Rmd")' From 79a365ef96c7e726bc4f1a5548a87208727175ee Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Wed, 17 Dec 2025 15:15:09 +0100 Subject: [PATCH 13/18] Several bugfixes (we're running extra rounds with 'seeds' for methods not using any seed, now they're flagged) --- Makefile | 6 +++--- analyze_results_izaskun.Rmd | 29 ++++++++++++++++++++++++++++- parse_results.py | 7 ++++--- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 0a98e47..3074d7a 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,7 @@ run_conda: clone_yamls echo " Run $$i for seed $$seed and run $$i."; \ echo "DEST: results/out_conda_seed_$$seed\_run_$$i" ;\ ${OB_CMD} -b $(CLUSTERING_DIR)/Clustering_conda_tmp.yml --out-dir results/out_conda_seed_$$seed\_run_$$i; \ - mv $(CLUSTERING_DIR)/Clustering_conda_tmp.yml results/out_conda_seed_$$seed\_run_$$i/; \ + cp $(CLUSTERING_DIR)/Clustering_conda_tmp.yml results/out_conda_seed_$$seed\_run_$$i/; \ done; \ done @@ -94,7 +94,7 @@ run_oras: clone_yamls for i in $(RUNS); do \ echo " Run $$i for seed $$seed and run $$i."; \ ${OB_CMD} -b $(CLUSTERING_DIR)/Clustering_oras_tmp.yml --out-dir results/out_oras_seed_$$seed\_run_$$i/; \ - mv $(CLUSTERING_DIR)/Clustering_oras_tmp.yml results/out_oras_seed_$$seed\_run_$$i/; \ + cp $(CLUSTERING_DIR)/Clustering_oras_tmp.yml results/out_oras_seed_$$seed\_run_$$i/; \ done; \ done @@ -112,7 +112,7 @@ run_envs: clone_yamls for i in $(RUNS); do \ echo " Run $$i for seed $$seed and run $$i..."; \ ${OB_CMD} -b $(CLUSTERING_DIR)/Clustering_envmodules_tmp.yml --out-dir results/out_envmodules_seed_$$seed\_run_$$i/; \ - mv $(CLUSTERING_DIR)/Clustering_envmodules_tmp.yml results/out_envmodules_seed_$$seed\_run_$$i/; \ + cp $(CLUSTERING_DIR)/Clustering_envmodules_tmp.yml results/out_envmodules_seed_$$seed\_run_$$i/; \ done; \ done \ ' diff --git a/analyze_results_izaskun.Rmd b/analyze_results_izaskun.Rmd index 63bd201..40a06b4 100644 --- a/analyze_results_izaskun.Rmd +++ b/analyze_results_izaskun.Rmd @@ -167,7 +167,7 @@ fd[cols_to_num] <- lapply(fd[cols_to_num], function(x) { }) fd$k_offset <- fd$k - fd$true_k -write.csv(fd, file = 'aggregated_results.csv') +## write.csv(fd, file = 'aggregated_results.csv') ## later, this needs extra cleaning ``` @@ -257,9 +257,36 @@ stopifnot(range(fd$k_offset) == c(-2, 2)) Are seeds ok? why is `fcps_method-FCPS_MinEnergy_seed-2` a method name, which seed is that? **looks wrong** ```{r} + table(fd$method_full, fd$seed) +table(fd$method, fd$seed) ``` +Are we stupidly running some methods more times with different seeds, even if they don't use seeds at all? then remove them + +```{r} +head(fd) +table(fd$method_full) + + +## flag methods that explicitly encode a seed. +## filter(has_seed | seed == 1), keep all rows for seed‑encoded methods, but only seed==1 otherwise +fd_clean <- fd %>% + mutate(has_seed = grepl("seed-", method_full), + seed = ifelse(has_seed, as.character(seed), "none")) %>% + filter(has_seed | seed == "none") + +dim(fd_clean) +dim(fd) +table(fd$seed, useNA = 'always') +table(fd_clean$seed, useNA = 'always') + +fd <- fd_clean + +rm(fd_clean) + +write.csv(fd, file = 'aggregated_results.csv') +``` diff --git a/parse_results.py b/parse_results.py index 7dda69f..c07cebd 100644 --- a/parse_results.py +++ b/parse_results.py @@ -12,7 +12,7 @@ import re from pathlib import Path from typing import Dict, List, Optional - +import sys def parse_result_path(path: Path) -> List[Dict[str, str]]: """ @@ -27,10 +27,11 @@ def parse_result_path(path: Path) -> List[Dict[str, str]]: parts = path.parts base_result: Dict[str, str] = {} + # print("DEBUG parts:", parts, file = sys.stderr) # parse out_{backend}_seed_{seed}_run_{run} out_match = re.match( r"out_(?P[a-zA-Z0-9]+)_seed_(?P\d+)_run_(?P\d+)", - parts[0] + parts[1] ) if out_match: base_result["backend"] = out_match.group("backend") @@ -185,7 +186,7 @@ def parse_metrics(config_dir: Path) -> Dict[str, Dict[str, Dict[str, float]]]: def find_results( base_dir: str = ".", - pattern: str = "out_*/data/clustbench/dataset_generator-*/clustering/*" + pattern: str = "results/out_*/data/clustbench/dataset_generator-*/clustering/*" ) -> List[Dict[str, str]]: """ Return one record per configuration folder with parameters, performance, and metrics. From 564b20849daf0597e00ab70a9a63cba9a4579e1b Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Wed, 17 Dec 2025 15:54:06 +0100 Subject: [PATCH 14/18] Deduplicate repeated runs and plot averages instead --- analyze_results_izaskun.Rmd | 396 +++++++++++++++++++++++++----------- 1 file changed, 274 insertions(+), 122 deletions(-) diff --git a/analyze_results_izaskun.Rmd b/analyze_results_izaskun.Rmd index 40a06b4..754fcb7 100644 --- a/analyze_results_izaskun.Rmd +++ b/analyze_results_izaskun.Rmd @@ -345,31 +345,31 @@ ggplot(fd_avg, aes(x = backend, y = imputed_cpu_time)) + # Results consistency across backends -## Unreadable perf metrics plot - -```{r, fig.width = 30, fig.height = 30} - -print( - ggplot(fd, aes(x = backend, y = metric_value)) + - # boxplots still colored by backend - geom_boxplot(aes(fill = backend), outlier.alpha = 0.3) + - # points colored by run - geom_point(aes(color = run), - alpha = 0.6, - position = position_jitter(width = 0.15)) + - # connect points across backends for same method/params/run/etc. - geom_line(aes(group = interaction(method, method_full, seed, run, generator, dataset_name, k), - color = run), - alpha = 0.2) + - facet_wrap(dataset_name ~ metric_name, scales = "free_y") + - theme_minimal(base_size = 14) + - labs(title = "Metrics consistency across backends (stage = methods)", - x = "Backend", - y = "Metric Value") + - scale_fill_brewer(palette = "Set1") + - scale_color_brewer(palette = "Dark2")) - -``` + + + + + + + + + + + + + + + + + + + + + + + + + # Pairwise correlations @@ -404,7 +404,7 @@ print(cors_run) ```{r, fig.width = 10, fig.height=10} perf_metrics <- c("cpu_time","max_rss","max_vms","max_uss", - "max_pss","io_in","io_out","mean_load") + "max_pss","io_in","io_out","mean_load", "imputed_cpu_time") fd_long <- fd %>% filter(backend %in% c("conda","oras","envmodules")) %>% @@ -412,9 +412,15 @@ fd_long <- fd %>% names_to = "metric", values_to = "value") +## again, averaging repeated runs with the same seed +fd_avg <- fd_long %>% + group_by(backend, seed, run, metric, dataset_name, method_full) %>% + summarise(mean_value = mean(value, na.rm = TRUE), .groups = "drop") +head(fd_avg) + # boxplots + jittered scatter, faceted by metric -ggplot(fd_long, aes(x = backend, y = value, fill = backend)) + +ggplot(fd_avg, aes(x = backend, y = mean_value, fill = backend)) + geom_boxplot(outlier.alpha = 0.3) + geom_jitter(width = 0.2, alpha = 0.1, size = 1, color = "black") + facet_wrap(~metric, scales = "free_y") + @@ -440,7 +446,11 @@ fd_long_complete <- fd %>% filter(n_distinct(backend) == 3, !any(is.na(value))) %>% ungroup() -ggplot(fd_long_complete, aes(x = backend, y = value, fill = backend)) + +fd_avg <- fd_long_complete %>% + group_by(backend, seed, run, metric, dataset_name, method_full) %>% + summarise(mean_value = mean(value, na.rm = TRUE), .groups = "drop") + +ggplot(fd_avg, aes(x = backend, y = mean_value, fill = backend)) + geom_boxplot(outlier.alpha = 0.3) + geom_jitter(width = 0.2, alpha = 0.1, size = 1, color = "black") + facet_wrap(~metric, scales = "free_y") + @@ -449,8 +459,6 @@ ggplot(fd_long_complete, aes(x = backend, y = value, fill = backend)) + x = "backend", y = "perf value") + scale_fill_brewer(palette = "Set2") - - ``` ## Non-complete observations colored by method provider, with a bug @@ -464,17 +472,22 @@ fd_long <- fd %>% names_to = "metric", values_to = "value") -## table(fd_long$method, fd_long$method_full) +fd_avg <- fd_long %>% + group_by(backend, seed, run, metric, dataset_name, method_full) %>% + summarise(mean_value = mean(value, na.rm = TRUE), .groups = "drop") -ggplot(fd_long, aes(x = backend, y = value)) + +head(fd_avg) +table(fd_avg$seed, fd_avg$run) + +ggplot(fd_avg, aes(x = backend, y = mean_value)) + # boxplots filled by backend ## geom_boxplot(aes(fill = backend), outlier.alpha = 0.3) + # points colored by method - geom_point(aes(color = method), + geom_point(aes(color = method_full), alpha = 0.6, position = position_jitter(width = 0.15)) + # connect points across backends for same method/run/etc. - geom_line(aes(group = interaction(method, method_full, seed, run, generator, dataset_name, k), + geom_line(aes(group = interaction(method_full, seed, run, dataset_name), color = run), alpha = 0.1) + facet_wrap(~metric, scales = "free_y") + @@ -503,6 +516,7 @@ fd_dev_k <- fd %>% ungroup() summary(fd_dev_k$deviation_k) + ## str(fd_dev_k) ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method_full)) + geom_point(alpha = 0.6, position = position_jitter(width = 0.15)) + @@ -515,6 +529,77 @@ ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method_full)) + scale_color_brewer(palette = "Set1") ``` +## By k offset + + +```{r} +## str(fd) +fd_dev_k <- fd %>% + group_by(method_full, dataset_name, metric_name) %>% + # get the metric_value at the true_k for this group + mutate(true_value = metric_value[k == true_k][1], + deviation_k = metric_value - true_value) %>% + ungroup() + +## again aggregate across repeated runs - not seeds, given the method_full has them in its values +fd_dev_k_avg <- fd_dev_k %>% + group_by(method_full, dataset_name, metric_name, k, k_offset) %>% + summarise( + mean_deviation_k = mean(deviation_k, na.rm = TRUE), + .groups = "drop" + ) + +head(fd_dev_k_avg) + +ggplot(fd_dev_k_avg, aes(x = k_offset, y = mean_deviation_k, color = method_full)) + + geom_point(alpha = 0.6, position = position_jitter(width = 0.15, height = 0)) + + facet_wrap(~ metric_name, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "Performance impact of k offsets", + subtitle = "Deviation vs value at true k", + x = "Offset from true k", + y = "Perf metric deviation from true k") + + scale_color_brewer(palette = "Set1") + +ggplot(fd_dev_k_avg, aes(x = k_offset, y = mean_deviation_k, color = dataset_name)) + + geom_point(alpha = 0.6, position = position_jitter(width = 0.15, height = 0)) + + facet_wrap(~ metric_name, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "Performance impact of k offsets", + x = "Offset from true k", + y = "Perf metric deviation from true k") + + scale_color_brewer(palette = "Set1") + +``` + +adj Rand index only + +```{r} +# ilter to adjusted_rand_score only +fd_dev_k_ars <- fd_dev_k_avg %>% + filter(metric_name == "adjusted_rand_score") + +# by method_full +ggplot(fd_dev_k_ars, aes(x = k_offset, y = mean_deviation_k, color = method_full)) + + geom_point(alpha = 0.6, position = position_jitter(width = 0.15, height = 0)) + + theme_minimal(base_size = 14) + + labs(title = "Performance impact of k offsets", + subtitle = "Deviation vs value at true k", + x = "Offset from true k", + y = "Deviation in adjusted_rand_score") + + scale_color_brewer(palette = "Set1") + +# by dataset_name +ggplot(fd_dev_k_ars, aes(x = k_offset, y = mean_deviation_k, color = dataset_name)) + + geom_point(alpha = 0.6, position = position_jitter(width = 0.15, height = 0)) + + theme_minimal(base_size = 14) + + labs(title = "Performance impact of k offsets", + x = "Offset from true k", + y = "Deviation in adjusted_rand_score") + + scale_color_brewer(palette = "Set1") + +``` + # Impact of seeds and repeated runs @@ -522,6 +607,8 @@ What about the consistency across runs and seeds? e.g., do different seeds produ Only at true k. +Double check this is correct... + ```{r} @@ -545,12 +632,12 @@ fd_dev_seed <- fd_dev_seed %>% summary(fd_dev_seed) -ggplot(fd_dev_seed, aes(x = factor(seed), y = deviation_seed, color = method_full)) + - geom_point(alpha = 0.6) + - theme_minimal(base_size = 14) + - labs(title = "Deviation at true k across seeds", - x = "Seed", - y = "Deviation from mean across seeds") +## ggplot(fd_dev_seed, aes(x = factor(seed), y = deviation_seed, color = method_full)) + +## geom_point(alpha = 0.6) + +## theme_minimal(base_size = 14) + +## labs(title = "Deviation at true k across seeds", +## x = "Seed", +## y = "Deviation from mean across seeds") fd_dev_run <- fd_true %>% @@ -573,45 +660,59 @@ summary(fd_dev_run) ```{r, fig.width = 12} -## print(ggplot(fd_long, aes(x = backend, y = value, fill = backend)) + -## geom_boxplot(outlier.alpha = 0.3) + -## geom_jitter(width = 0.2, alpha = 0.1, size = 1) + -## facet_wrap(~metric, scales = "free_y") + + +## print(ggplot(fd, aes(x = cpu_time, y = metric_value, +## color = method_full, shape = backend)) + +## geom_point(alpha = 0.6) + +## facet_wrap(~metric_name, scales = "free_y") + ## theme_minimal(base_size = 14) + -## labs(title = "comp perf by backend") + -## scale_fill_brewer(palette = "Set2")) - -## print(ggplot(fd_seed_summary, -## aes(x = seed, y = mean_value, -## group = interaction(method_full, dataset_name, k), -## color = method_full)) + -## geom_line(alpha = 0.5) + -## geom_point(size = 2) + -## geom_errorbar(aes(ymin = mean_value - sd_runs, -## ymax = mean_value + sd_runs), -## width = 0.2, alpha = 0.4) + +## labs(title = "clustering metrics vs runtime trade‑offs", +## x = "CPU time (s)", +## y = "clustering metric value")) + +## print(ggplot(fd, aes(x = max_rss, y = metric_value, +## color = method_full, shape = backend)) + +## geom_point(alpha = 0.6) + ## facet_wrap(~metric_name, scales = "free_y") + ## theme_minimal(base_size = 14) + -## labs(title = "Seed sensitivity", -## subtitle = "Error bars depict runs variability")) +## labs(title = "clustering metrics vs RSS trade‑offs", +## x = "max RSS (MB)", +## y = "clustering metric value")) -print(ggplot(fd, aes(x = cpu_time, y = metric_value, + +# aggregate across runs and ks again... +fd_avg <- fd %>% + group_by(method_full, backend, dataset_name, metric_name) %>% + summarise( + mean_imputed_cpu_time = mean(imputed_cpu_time, na.rm = TRUE), + mean_max_rss = mean(max_rss, na.rm = TRUE), + mean_metric = mean(metric_value, na.rm = TRUE), + .groups = "drop" + ) + +# imputed cpu time vs metric +print( + ggplot(fd_avg, aes(x = mean_imputed_cpu_time, y = mean_metric, color = method_full, shape = backend)) + - geom_point(alpha = 0.6) + - facet_wrap(~metric_name, scales = "free_y") + - theme_minimal(base_size = 14) + - labs(title = "clustering metrics vs runtime trade‑offs", - x = "CPU time (s)", - y = "clustering metric value")) + geom_point(alpha = 0.6) + + facet_wrap(~metric_name, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "Clustering metrics vs runtime trade‑offs", + x = "Mean CPU time (s)", + y = "Mean clustering metric value") +) -print(ggplot(fd, aes(x = max_rss, y = metric_value, +# RSS vs metric, caution here no imputation / no idea how to handle memory for censored cpu_time data +print( + ggplot(fd_avg, aes(x = mean_max_rss, y = mean_metric, color = method_full, shape = backend)) + - geom_point(alpha = 0.6) + - facet_wrap(~metric_name, scales = "free_y") + - theme_minimal(base_size = 14) + - labs(title = "clustering metrics vs RSS trade‑offs", - x = "max RSS (MB)", - y = "clustering metric value")) + geom_point(alpha = 0.6) + + facet_wrap(~metric_name, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "Clustering metrics vs RSS trade‑offs", + x = "Mean max RSS (MB)", + y = "Mean clustering metric value") +) ``` @@ -630,88 +731,139 @@ fd_dev_true <- fd %>% mean_val = (metric_value + true_value)/2) %>% ungroup() -# Bland–Altman style plot: any k vs true_k -ggplot(fd_dev_true, aes(x = mean_val, y = diff_val, color = method_full)) + +## # Bland–Altman style plot: any k vs true_k +## ggplot(fd_dev_true, aes(x = mean_val, y = diff_val, color = method_full)) + +## geom_point(alpha = 0.6) + +## geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE), +## linetype = "dashed", color = "blue") + +## geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) + 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), +## linetype = "dotted", color = "red") + +## geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) - 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), +## linetype = "dotted", color = "red") + +## facet_wrap(~metric_name, scales = "free") + +## theme_minimal(base_size = 14) + +## labs(title = "Bland–Altman: any k vs true_k", +## x = "Mean of k and true_k", +## y = "Difference (k - true_k)") + +# collapse across seeds and runs +fd_dev_true_avg <- fd_dev_true %>% + group_by(method_full, dataset_name, metric_name, k, k_offset) %>% + summarise( + mean_diff_val = mean(diff_val, na.rm = TRUE), + mean_mean_val = mean(mean_val, na.rm = TRUE), + .groups = "drop" + ) + +ggplot(fd_dev_true_avg, aes(x = mean_mean_val, y = mean_diff_val, color = method_full)) + geom_point(alpha = 0.6) + - geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE), + geom_hline(yintercept = mean(fd_dev_true_avg$mean_diff_val, na.rm = TRUE), linetype = "dashed", color = "blue") + - geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) + 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), + geom_hline(yintercept = mean(fd_dev_true_avg$mean_diff_val, na.rm = TRUE) + 1.96*sd(fd_dev_true_avg$mean_diff_val, na.rm = TRUE), linetype = "dotted", color = "red") + - geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) - 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), + geom_hline(yintercept = mean(fd_dev_true_avg$mean_diff_val, na.rm = TRUE) - 1.96*sd(fd_dev_true_avg$mean_diff_val, na.rm = TRUE), linetype = "dotted", color = "red") + facet_wrap(~metric_name, scales = "free") + theme_minimal(base_size = 14) + - labs(title = "Bland–Altman: any k vs true_k", - x = "Mean of k and true_k", - y = "Difference (k - true_k)") + labs(title = "Bland–Altman: any k vs true_k (averaged)", + x = "Mean of k and true_k (averaged)", + y = "Difference (k - true_k, averaged)") + + ``` ## Colored by dataset ```{r, fig.width = 15, fig.height = 15} -ggplot(fd_dev_true, aes(x = mean_val, y = diff_val, color = dataset_name)) + + +ggplot(fd_dev_true_avg, aes(x = mean_mean_val, y = mean_diff_val, color = dataset_name)) + geom_point(alpha = 0.6) + - geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE), + geom_hline(yintercept = mean(fd_dev_true_avg$mean_diff_val, na.rm = TRUE), linetype = "dashed", color = "blue") + - geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) + 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), + geom_hline(yintercept = mean(fd_dev_true_avg$mean_diff_val, na.rm = TRUE) + 1.96*sd(fd_dev_true_avg$mean_diff_val, na.rm = TRUE), linetype = "dotted", color = "red") + - geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) - 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), + geom_hline(yintercept = mean(fd_dev_true_avg$mean_diff_val, na.rm = TRUE) - 1.96*sd(fd_dev_true_avg$mean_diff_val, na.rm = TRUE), linetype = "dotted", color = "red") + facet_wrap(~metric_name, scales = "free") + theme_minimal(base_size = 14) + - labs(title = "Bland–Altman: any k vs true_k", - x = "Mean of k and true_k", - y = "Difference (k - true_k)") - + labs(title = "Bland–Altman: any k vs true_k (averaged)", + x = "Mean of k and true_k (averaged)", + y = "Difference (k - true_k, averaged)") ``` -## Grid of Bland Altmans - misspecified k + + + + + + + + + + + + + + + + + + + + + + + -More Bland Altmans +## Effects of misspecifying `k`, unreadable QC plot -```{r, fig.width = 50, fig.height = 30} -ggplot(fd_dev_true, - aes(x = mean_val, y = diff_val, color = method_full)) + # color by method_full - geom_point(alpha = 0.6) + - geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE), - linetype = "dashed", color = "blue") + - geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) + - 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), - linetype = "dotted", color = "red") + - geom_hline(yintercept = mean(fd_dev_true$diff_val, na.rm = TRUE) - - 1.96*sd(fd_dev_true$diff_val, na.rm = TRUE), - linetype = "dotted", color = "red") + - facet_grid(metric_name ~ dataset_name, scales = "free") + - theme_minimal(base_size = 14) + - labs(title = "Bland–Altman: any k vs true_k", - x = "Mean of k and true_k", - y = "Difference (k - true_k)") +```{r, fig.width = 30, fig.height = 30} +## true_vals <- fd %>% +## filter(k == true_k) %>% +## select(method_full, dataset_name, metric_name, run, seed, true_value = metric_value) -``` +## fd_dev_k <- fd %>% +## group_by(method_full, dataset_name, metric_name, run, seed) %>% +## mutate(true_value = first(metric_value[k == true_k]), +## deviation_k = metric_value - true_value) %>% +## ungroup() -## Effects of misspecifying `k`, unreadable QC plot +## head(fd_dev_k) -```{r, fig.width = 30, fig.height = 30} -true_vals <- fd %>% - filter(k == true_k) %>% - select(method_full, dataset_name, metric_name, run, seed, true_value = metric_value) +## ggplot(fd_dev_k, aes(x = factor(k_offset), y = deviation_k, color = method_full)) + +## geom_point(alpha = 0.6, position = position_jitter(width = 0.15)) + +## facet_grid(dataset_name ~ metric_name, scales = "free_y") + +## theme_minimal(base_size = 14) + +## labs(title = "Deviation of metric across ks", +## subtitle = "Deviation vs value at true k", +## x = "k", +## y = "Perf metric deviation from true k") + +## scale_color_brewer(palette = "Set1") fd_dev_k <- fd %>% group_by(method_full, dataset_name, metric_name, run, seed) %>% - mutate(true_value = first(metric_value[k == true_k]), + mutate(true_value = first(metric_value[k == true_k]), deviation_k = metric_value - true_value) %>% ungroup() -ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method_full)) + - geom_point(alpha = 0.6, position = position_jitter(width = 0.15)) + - facet_grid(dataset_name ~ metric_name, scales = "free_y") + +# deduplicate: average across runs , only ARI +fd_dev_k_avg <- fd_dev_k %>% + filter(metric_name == "adjusted_rand_score") %>% + group_by(method_full, dataset_name, metric_name, k, k_offset) %>% + summarise( + mean_deviation_k = mean(deviation_k, na.rm = TRUE), + .groups = "drop" + ) + +# plot with avg values +ggplot(fd_dev_k_avg, aes(x = k_offset, y = mean_deviation_k, color = method_full)) + + geom_point(alpha = 0.6, position = position_jitter(width = 0.15, height = 0)) + + facet_wrap(~dataset_name, scales = "free_y", ncol = 3) + theme_minimal(base_size = 14) + - labs(title = "Deviation of metric across ks", - subtitle = "Deviation vs value at true k", - x = "k", - y = "Perf metric deviation from true k") + + labs(title = "Deviation of metric across k offsets", + x = "Offset from true k", + y = "Mean ARI deviation from that of true `k`") + scale_color_brewer(palette = "Set1") - ``` From 58ad0aa19a0d82bf87815b15d4ea6545e4124055 Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Thu, 18 Dec 2025 09:55:35 +0100 Subject: [PATCH 15/18] Document, parallelize load --- analyze_results_izaskun.Rmd | 229 ++++++++++++++++-------------------- 1 file changed, 102 insertions(+), 127 deletions(-) diff --git a/analyze_results_izaskun.Rmd b/analyze_results_izaskun.Rmd index 754fcb7..1f43736 100644 --- a/analyze_results_izaskun.Rmd +++ b/analyze_results_izaskun.Rmd @@ -23,6 +23,7 @@ library(tidyverse) library(jsonlite) library(ggplot2) library(data.table) +library(parallel) knitr::opts_chunk$set( echo = TRUE, @@ -137,14 +138,26 @@ flatten_record <- function(rec) { ```{r} records <- fromJSON("aggregated_results.json", simplifyVector = FALSE) +##records <- fromJSON("aggregated_results_full.json", simplifyVector = FALSE) -fd_list <- vector("list", length(records)) -for (i in seq_along(records)) { +## fd_list <- vector("list", length(records)) +## for (i in seq_along(records)) { +## if (i == 1 || i == 2 || i == 100 || i %% 1000 == 0) +## cat("Processing record", i, "of", length(records), "\n") +## fd_list[[i]] <- flatten_record(records[[i]]) +## } +## fd <- data.table::rbindlist(fd_list, use.names = TRUE, fill = TRUE) + + +# parallel apply instead +fd_list <- mclapply(seq_along(records), function(i) { if (i == 1 || i == 2 || i == 100 || i %% 1000 == 0) - cat("Processing record", i, "of", length(records), "\n") - fd_list[[i]] <- flatten_record(records[[i]]) -} -fd <- data.table::rbindlist(fd_list, use.names = TRUE, fill = TRUE) + cat("Processing record", i, "of", length(records), "\n") + flatten_record(records[[i]]) +}, mc.cores = detectCores()) + +fd <- rbindlist(fd_list, use.names = TRUE, fill = TRUE) + ## dim(fd) @@ -168,7 +181,6 @@ fd[cols_to_num] <- lapply(fd[cols_to_num], function(x) { fd$k_offset <- fd$k - fd$true_k ## write.csv(fd, file = 'aggregated_results.csv') ## later, this needs extra cleaning - ``` # QC @@ -254,38 +266,29 @@ stopifnot(nrow(fd_seed_diff) == 0) stopifnot(range(fd$k_offset) == c(-2, 2)) ``` -Are seeds ok? why is `fcps_method-FCPS_MinEnergy_seed-2` a method name, which seed is that? **looks wrong** +We are running some methods more times with different seeds, even if they don't use seeds at all. Removing these different "seeds" that are not such but runs. We have repeated/controlled runs separately. ```{r} -table(fd$method_full, fd$seed) -table(fd$method, fd$seed) -``` - -Are we stupidly running some methods more times with different seeds, even if they don't use seeds at all? then remove them - -```{r} -head(fd) -table(fd$method_full) - - -## flag methods that explicitly encode a seed. -## filter(has_seed | seed == 1), keep all rows for seed‑encoded methods, but only seed==1 otherwise +## seed 2 for seed-unaware methods, all seed for seed-aware methods fd_clean <- fd %>% - mutate(has_seed = grepl("seed-", method_full), - seed = ifelse(has_seed, as.character(seed), "none")) %>% - filter(has_seed | seed == "none") + mutate(has_seed = grepl("seed-", method_full)) %>% + filter( + (has_seed) | (!has_seed & seed == 2) # keep all seeds if encoded, else only seed==2 + ) %>% + mutate(seed = ifelse(has_seed, as.character(seed), "none")) -dim(fd_clean) -dim(fd) -table(fd$seed, useNA = 'always') -table(fd_clean$seed, useNA = 'always') + +table(fd$seed, grepl('seed', fd$method_full), useNA = 'always') +table(fd_clean$seed, grepl('seed', fd_clean$method_full), useNA = 'always') fd <- fd_clean rm(fd_clean) write.csv(fd, file = 'aggregated_results.csv') +# write.csv(fd, file = 'aggregated_results_full.csv') + ``` @@ -345,55 +348,84 @@ ggplot(fd_avg, aes(x = backend, y = imputed_cpu_time)) + # Results consistency across backends - - - - - - - - - - - - - - - - - - - - - - - +# Pairwise correlations - +```{r, fig.width = 9} +wide_run <- fd %>% + filter(backend %in% c("conda","oras","envmodules")) %>% + group_by(dataset_name, method, k, metric_name, run, seed, backend) %>% + summarise(metric_value = mean(as.numeric(metric_value), na.rm = TRUE), .groups = "drop") %>% + tidyr::pivot_wider(names_from = backend, values_from = metric_value, values_fill = NA) -# Pairwise correlations +head(wide_run) -```{r} -df_run <- fd %>% - select(dataset_name, method, k, metric_name, run, seed, backend, metric_value) +# Compute correlations per metric_name AND seed and for pairwise complete obs +cors_seed <- wide_run %>% + group_by(metric_name, seed) %>% + summarise( + cor_conda_oras = if(sum(complete.cases(conda, oras)) > 1) + cor(conda, oras, use = "complete.obs") else NA_real_, + cor_conda_envmodules = if(sum(complete.cases(conda, envmodules)) > 1) + cor(conda, envmodules, use = "complete.obs") else NA_real_, + cor_oras_envmodules = if(sum(complete.cases(oras, envmodules)) > 1) + cor(oras, envmodules, use = "complete.obs") else NA_real_, + .groups = "drop") + + +print(cors_run) + +cors_long <- cors_seed %>% + pivot_longer( + cols = starts_with("cor_"), + names_to = "pair", + values_to = "correlation") -# a bit of collapsing here +ggplot(cors_long, aes(x = pair, y = interaction(metric_name, seed), fill = correlation)) + + geom_tile(color = "white") + + geom_text(aes(label = round(correlation, 2)), color = "black", size = 3) + + scale_fill_gradient2(low = "blue", mid = "white", high = "red", midpoint = 0, + limits = c(-1, 1), na.value = "grey90") + + theme_minimal(base_size = 12) + + labs(title = "Backend correlations per metric and seed", + x = "Backend pair", + y = "Metric and seed", + fill = "cor coef") + +``` + +And the repeated runs? + +```{r} + wide_run <- fd %>% filter(backend %in% c("conda","oras","envmodules")) %>% group_by(dataset_name, method, k, metric_name, run, seed, backend) %>% summarise(metric_value = mean(as.numeric(metric_value), na.rm = TRUE), .groups = "drop") %>% - pivot_wider(names_from = backend, values_from = metric_value) + unite("backend_run", backend, run, sep = "_") %>% # combine backend and run + pivot_wider(names_from = backend_run, values_from = metric_value, values_fill = NA) -cors_run <- wide_run %>% - group_by(metric_name) %>% - summarise( - cor_conda_oras = cor(conda, oras, use="complete.obs"), - cor_conda_envmodules = cor(conda, envmodules, use="complete.obs"), - cor_oras_envmodules = cor(oras, envmodules, use="complete.obs") - ) -print(cors_run) +# only correlating aggregated metrics +num_cols <- wide_run %>% + select(where(is.numeric), -k, -seed) + + +# pairwise cors, so including repeated runs with the same backend +cors_all <- cor(num_cols, use = "pairwise.complete.obs") + +cors_df <- melt(cors_all) + +# heatmap with correlation coefficients +ggplot(cors_df, aes(x = Var1, y = Var2, fill = value)) + + geom_tile(color = "white") + + geom_text(aes(label = round(value, 2)), color = "black", size = 3) + + scale_fill_gradient2(low = "blue", mid = "white", high = "red", midpoint = 0, + limits = c(-1, 1), na.value = "grey90") + + theme_minimal(base_size = 12) + + labs(title = "Metric correlation per backend and repeated run", + x = "backend and run", y = "backend and run", fill = "Correlation") + ``` @@ -506,7 +538,7 @@ ggplot(fd_avg, aes(x = backend, y = mean_value)) + Mind there is no such a thing as true k, we use the first labelset. -```{r} +```{r, fig.height = 7, fig.width = 10} fd_dev_k <- fd %>% group_by(method_full, dataset_name, metric_name) %>% @@ -532,7 +564,7 @@ ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method_full)) + ## By k offset -```{r} +```{r, fig.width = 10, fig.height = 7} ## str(fd) fd_dev_k <- fd %>% group_by(method_full, dataset_name, metric_name) %>% @@ -549,8 +581,6 @@ fd_dev_k_avg <- fd_dev_k %>% .groups = "drop" ) -head(fd_dev_k_avg) - ggplot(fd_dev_k_avg, aes(x = k_offset, y = mean_deviation_k, color = method_full)) + geom_point(alpha = 0.6, position = position_jitter(width = 0.15, height = 0)) + facet_wrap(~ metric_name, scales = "free_y") + @@ -574,7 +604,7 @@ ggplot(fd_dev_k_avg, aes(x = k_offset, y = mean_deviation_k, color = dataset_nam adj Rand index only -```{r} +```{r, fig.height = 6, fig.width = 6} # ilter to adjusted_rand_score only fd_dev_k_ars <- fd_dev_k_avg %>% filter(metric_name == "adjusted_rand_score") @@ -600,61 +630,6 @@ ggplot(fd_dev_k_ars, aes(x = k_offset, y = mean_deviation_k, color = dataset_nam ``` -# Impact of seeds and repeated runs - - -What about the consistency across runs and seeds? e.g., do different seeds produce different results? - -Only at true k. - -Double check this is correct... - - - -```{r} - -fd_true <- fd %>% filter(k == true_k) - -fd_dev_seed <- fd_true %>% - group_by(method_full, dataset_name, metric_name, run) %>% - mutate( - mean_true_val = mean(metric_value, na.rm = TRUE), - deviation_seed = metric_value - mean_true_val - ) %>% - ungroup() - -## floats strike again... -tol <- sqrt(.Machine$double.eps) -fd_dev_seed <- fd_dev_seed %>% - mutate(deviation_seed = ifelse(abs(deviation_seed) < tol, 0, deviation_seed)) - - -summary(fd_dev_seed) - - -## ggplot(fd_dev_seed, aes(x = factor(seed), y = deviation_seed, color = method_full)) + -## geom_point(alpha = 0.6) + -## theme_minimal(base_size = 14) + -## labs(title = "Deviation at true k across seeds", -## x = "Seed", -## y = "Deviation from mean across seeds") - - -fd_dev_run <- fd_true %>% - group_by(method_full, dataset_name, metric_name, seed) %>% - mutate( - mean_true_val = mean(metric_value, na.rm = TRUE), - deviation_run = metric_value - mean_true_val - ) %>% - ungroup() - -tol <- sqrt(.Machine$double.eps) -fd_dev_run <- fd_dev_run %>% - mutate(deviation_run = ifelse(abs(deviation_run) < tol, 0, deviation_run)) - -summary(fd_dev_run) - -``` # Comp vs clustering performance trade-offs @@ -819,7 +794,7 @@ ggplot(fd_dev_true_avg, aes(x = mean_mean_val, y = mean_diff_val, color = datase ## Effects of misspecifying `k`, unreadable QC plot -```{r, fig.width = 30, fig.height = 30} +```{r, fig.width = 15, fig.height = 15} ## true_vals <- fd %>% ## filter(k == true_k) %>% ## select(method_full, dataset_name, metric_name, run, seed, true_value = metric_value) From d7e9f6a08c42a301da1dbeff03d82424f38e2c43 Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Thu, 18 Dec 2025 13:16:44 +0100 Subject: [PATCH 16/18] Simplify, bugfix --- analyze_results_izaskun.Rmd | 209 ++++++++++++------------------------ 1 file changed, 70 insertions(+), 139 deletions(-) diff --git a/analyze_results_izaskun.Rmd b/analyze_results_izaskun.Rmd index 1f43736..89965cd 100644 --- a/analyze_results_izaskun.Rmd +++ b/analyze_results_izaskun.Rmd @@ -34,7 +34,7 @@ knitr::opts_chunk$set( fig.path = "plots/", dev = c("png", "svg"), cache.lazy = FALSE, - cache = TRUE) + cache = FALSE) ``` @@ -138,7 +138,7 @@ flatten_record <- function(rec) { ```{r} records <- fromJSON("aggregated_results.json", simplifyVector = FALSE) -##records <- fromJSON("aggregated_results_full.json", simplifyVector = FALSE) +## records <- fromJSON("aggregated_results_full.json", simplifyVector = FALSE) ## fd_list <- vector("list", length(records)) ## for (i in seq_along(records)) { @@ -223,8 +223,6 @@ print(ggplot(fd[fd$backend %in% c("conda","oras","envmodules"), ], Are there cputimes that are redudant/incosistent for repeated runs? ```{r} -str(fd) - keys <- c("dataset_name", "method", "method_full", "generator", "backend", "k", "seed", "run") @@ -277,9 +275,17 @@ fd_clean <- fd %>% (has_seed) | (!has_seed & seed == 2) # keep all seeds if encoded, else only seed==2 ) %>% mutate(seed = ifelse(has_seed, as.character(seed), "none")) +``` +Before +```{r} table(fd$seed, grepl('seed', fd$method_full), useNA = 'always') +``` + +After + +```{r} table(fd_clean$seed, grepl('seed', fd_clean$method_full), useNA = 'always') fd <- fd_clean @@ -299,7 +305,7 @@ write.csv(fd, file = 'aggregated_results.csv') We impute NA cpu_time as 0.05 s -```{r} +```{r, fig.width = 12, fig.height = 8} min(fd$cpu_time, na.rm = TRUE) fd$imputed_cpu_time <- ifelse(is.na(fd$cpu_time), no = fd$cpu_time, yes = 0.05) @@ -350,16 +356,15 @@ ggplot(fd_avg, aes(x = backend, y = imputed_cpu_time)) + # Pairwise correlations -```{r, fig.width = 9} +```{r, fig.width = 7, fig.height = 14} wide_run <- fd %>% filter(backend %in% c("conda","oras","envmodules")) %>% group_by(dataset_name, method, k, metric_name, run, seed, backend) %>% summarise(metric_value = mean(as.numeric(metric_value), na.rm = TRUE), .groups = "drop") %>% tidyr::pivot_wider(names_from = backend, values_from = metric_value, values_fill = NA) -head(wide_run) -# Compute correlations per metric_name AND seed and for pairwise complete obs +# compute correlations per metric_name AND seed and for pairwise complete obs cors_seed <- wide_run %>% group_by(metric_name, seed) %>% summarise( @@ -372,7 +377,7 @@ cors_seed <- wide_run %>% .groups = "drop") -print(cors_run) +print(cors_seed) cors_long <- cors_seed %>% pivot_longer( @@ -449,7 +454,7 @@ fd_avg <- fd_long %>% group_by(backend, seed, run, metric, dataset_name, method_full) %>% summarise(mean_value = mean(value, na.rm = TRUE), .groups = "drop") -head(fd_avg) +## head(fd_avg) # boxplots + jittered scatter, faceted by metric ggplot(fd_avg, aes(x = backend, y = mean_value, fill = backend)) + @@ -493,12 +498,9 @@ ggplot(fd_avg, aes(x = backend, y = mean_value, fill = backend)) + scale_fill_brewer(palette = "Set2") ``` -## Non-complete observations colored by method provider, with a bug +## Non-complete observations colored by method -Something wrong with method "1" etc here? - - -```{r, fig.width = 10, fig.height = 10} +```{r, fig.width = 15, fig.height = 10} fd_long <- fd %>% pivot_longer(cols = all_of(perf_metrics), names_to = "metric", @@ -508,27 +510,55 @@ fd_avg <- fd_long %>% group_by(backend, seed, run, metric, dataset_name, method_full) %>% summarise(mean_value = mean(value, na.rm = TRUE), .groups = "drop") -head(fd_avg) -table(fd_avg$seed, fd_avg$run) +## head(fd_avg) +## table(fd_avg$seed, fd_avg$run) +## unique(fd_avg$method_full) ggplot(fd_avg, aes(x = backend, y = mean_value)) + - # boxplots filled by backend - ## geom_boxplot(aes(fill = backend), outlier.alpha = 0.3) + - # points colored by method + # points colored by method, seed and other params geom_point(aes(color = method_full), alpha = 0.6, position = position_jitter(width = 0.15)) + # connect points across backends for same method/run/etc. - geom_line(aes(group = interaction(method_full, seed, run, dataset_name), - color = run), + geom_line(aes(group = interaction(method_full, seed, run, dataset_name)), alpha = 0.1) + facet_wrap(~metric, scales = "free_y") + theme_minimal(base_size = 14) + labs(title = "Performance metrics by backend", x = "Backend", - y = "Value") + - scale_fill_brewer(palette = "Set2") + - scale_color_brewer(palette = "Dark2") + y = "Value") + +``` + +## Non-complete observations colored by method provider + +```{r, fig.width = 15, fig.height = 10} +fd_long <- fd %>% + pivot_longer(cols = all_of(perf_metrics), + names_to = "metric", + values_to = "value") + +fd_avg <- fd_long %>% + group_by(backend, seed, run, metric, dataset_name, method_full, method) %>% + summarise(mean_value = mean(value, na.rm = TRUE), .groups = "drop") + +## head(fd_avg) +## table(fd_avg$seed, fd_avg$run) +## unique(fd_avg$method_full) + +ggplot(fd_avg, aes(x = backend, y = mean_value)) + + # points colored by method, seed and other params + geom_point(aes(color = method), + alpha = 0.6, + position = position_jitter(width = 0.15)) + + # connect points across backends for same method/run/etc. + geom_line(aes(group = interaction(method, seed, run, dataset_name)), + alpha = 0.1) + + facet_wrap(~metric, scales = "free_y") + + theme_minimal(base_size = 14) + + labs(title = "Performance metrics by backend", + x = "Backend", + y = "Value") ``` @@ -538,7 +568,7 @@ ggplot(fd_avg, aes(x = backend, y = mean_value)) + Mind there is no such a thing as true k, we use the first labelset. -```{r, fig.height = 7, fig.width = 10} +```{r, fig.height = 10, fig.width = 13} fd_dev_k <- fd %>% group_by(method_full, dataset_name, metric_name) %>% @@ -547,7 +577,7 @@ fd_dev_k <- fd %>% deviation_k = metric_value - true_value) %>% ungroup() -summary(fd_dev_k$deviation_k) +## summary(fd_dev_k$deviation_k) ## str(fd_dev_k) ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method_full)) + @@ -557,14 +587,13 @@ ggplot(fd_dev_k, aes(x = factor(k), y = deviation_k, color = method_full)) + labs(title = "Deviation of metric across ks", subtitle = "Deviation vs value at true k", x = "k", - y = "Perf metric deviation from true k") + - scale_color_brewer(palette = "Set1") + y = "Perf metric deviation from true k") ``` ## By k offset -```{r, fig.width = 10, fig.height = 7} +```{r, fig.width = 12, fig.height = 9} ## str(fd) fd_dev_k <- fd %>% group_by(method_full, dataset_name, metric_name) %>% @@ -588,8 +617,7 @@ ggplot(fd_dev_k_avg, aes(x = k_offset, y = mean_deviation_k, color = method_full labs(title = "Performance impact of k offsets", subtitle = "Deviation vs value at true k", x = "Offset from true k", - y = "Perf metric deviation from true k") + - scale_color_brewer(palette = "Set1") + y = "Perf metric deviation from true k") ggplot(fd_dev_k_avg, aes(x = k_offset, y = mean_deviation_k, color = dataset_name)) + geom_point(alpha = 0.6, position = position_jitter(width = 0.15, height = 0)) + @@ -597,14 +625,13 @@ ggplot(fd_dev_k_avg, aes(x = k_offset, y = mean_deviation_k, color = dataset_nam theme_minimal(base_size = 14) + labs(title = "Performance impact of k offsets", x = "Offset from true k", - y = "Perf metric deviation from true k") + - scale_color_brewer(palette = "Set1") + y = "Perf metric deviation from true k") ``` adj Rand index only -```{r, fig.height = 6, fig.width = 6} +```{r, fig.height = 6, fig.width = 8} # ilter to adjusted_rand_score only fd_dev_k_ars <- fd_dev_k_avg %>% filter(metric_name == "adjusted_rand_score") @@ -616,8 +643,7 @@ ggplot(fd_dev_k_ars, aes(x = k_offset, y = mean_deviation_k, color = method_full labs(title = "Performance impact of k offsets", subtitle = "Deviation vs value at true k", x = "Offset from true k", - y = "Deviation in adjusted_rand_score") + - scale_color_brewer(palette = "Set1") + y = "Deviation in adjusted_rand_score") # by dataset_name ggplot(fd_dev_k_ars, aes(x = k_offset, y = mean_deviation_k, color = dataset_name)) + @@ -625,36 +651,16 @@ ggplot(fd_dev_k_ars, aes(x = k_offset, y = mean_deviation_k, color = dataset_nam theme_minimal(base_size = 14) + labs(title = "Performance impact of k offsets", x = "Offset from true k", - y = "Deviation in adjusted_rand_score") + - scale_color_brewer(palette = "Set1") + y = "Deviation in adjusted_rand_score") ``` # Comp vs clustering performance trade-offs +Caution only CPU's NAs are handled, not other perf metrics -```{r, fig.width = 12} - -## print(ggplot(fd, aes(x = cpu_time, y = metric_value, -## color = method_full, shape = backend)) + -## geom_point(alpha = 0.6) + -## facet_wrap(~metric_name, scales = "free_y") + -## theme_minimal(base_size = 14) + -## labs(title = "clustering metrics vs runtime trade‑offs", -## x = "CPU time (s)", -## y = "clustering metric value")) - -## print(ggplot(fd, aes(x = max_rss, y = metric_value, -## color = method_full, shape = backend)) + -## geom_point(alpha = 0.6) + -## facet_wrap(~metric_name, scales = "free_y") + -## theme_minimal(base_size = 14) + -## labs(title = "clustering metrics vs RSS trade‑offs", -## x = "max RSS (MB)", -## y = "clustering metric value")) - - +```{r, fig.width = 12, fig.height = 12} # aggregate across runs and ks again... fd_avg <- fd %>% group_by(method_full, backend, dataset_name, metric_name) %>% @@ -697,7 +703,7 @@ Again there is no such a thing as a true k ## Colored by method -```{r, fig.width = 15, fig.height = 15} +```{r, fig.width = 15, fig.height = 15, warning = FALSE} fd_dev_true <- fd %>% group_by(method_full, dataset_name, metric_name) %>% # get the metric value at true_k @@ -740,7 +746,7 @@ ggplot(fd_dev_true_avg, aes(x = mean_mean_val, y = mean_diff_val, color = method linetype = "dotted", color = "red") + facet_wrap(~metric_name, scales = "free") + theme_minimal(base_size = 14) + - labs(title = "Bland–Altman: any k vs true_k (averaged)", + labs(title = "Bland Altman: any k vs true_k (averaged)", x = "Mean of k and true_k (averaged)", y = "Difference (k - true_k, averaged)") @@ -749,7 +755,7 @@ ggplot(fd_dev_true_avg, aes(x = mean_mean_val, y = mean_diff_val, color = method ## Colored by dataset -```{r, fig.width = 15, fig.height = 15} +```{r, fig.width = 15, fig.height = 15, warning = FALSE} ggplot(fd_dev_true_avg, aes(x = mean_mean_val, y = mean_diff_val, color = dataset_name)) + geom_point(alpha = 0.6) + @@ -761,84 +767,9 @@ ggplot(fd_dev_true_avg, aes(x = mean_mean_val, y = mean_diff_val, color = datase linetype = "dotted", color = "red") + facet_wrap(~metric_name, scales = "free") + theme_minimal(base_size = 14) + - labs(title = "Bland–Altman: any k vs true_k (averaged)", + labs(title = "Bland Altman: any k vs true_k (averaged)", x = "Mean of k and true_k (averaged)", y = "Difference (k - true_k, averaged)") ``` - - - - - - - - - - - - - - - - - - - - - - - - - -## Effects of misspecifying `k`, unreadable QC plot - -```{r, fig.width = 15, fig.height = 15} -## true_vals <- fd %>% -## filter(k == true_k) %>% -## select(method_full, dataset_name, metric_name, run, seed, true_value = metric_value) - -## fd_dev_k <- fd %>% -## group_by(method_full, dataset_name, metric_name, run, seed) %>% -## mutate(true_value = first(metric_value[k == true_k]), -## deviation_k = metric_value - true_value) %>% -## ungroup() - -## head(fd_dev_k) - -## ggplot(fd_dev_k, aes(x = factor(k_offset), y = deviation_k, color = method_full)) + -## geom_point(alpha = 0.6, position = position_jitter(width = 0.15)) + -## facet_grid(dataset_name ~ metric_name, scales = "free_y") + -## theme_minimal(base_size = 14) + -## labs(title = "Deviation of metric across ks", -## subtitle = "Deviation vs value at true k", -## x = "k", -## y = "Perf metric deviation from true k") + -## scale_color_brewer(palette = "Set1") - -fd_dev_k <- fd %>% - group_by(method_full, dataset_name, metric_name, run, seed) %>% - mutate(true_value = first(metric_value[k == true_k]), - deviation_k = metric_value - true_value) %>% - ungroup() - -# deduplicate: average across runs , only ARI -fd_dev_k_avg <- fd_dev_k %>% - filter(metric_name == "adjusted_rand_score") %>% - group_by(method_full, dataset_name, metric_name, k, k_offset) %>% - summarise( - mean_deviation_k = mean(deviation_k, na.rm = TRUE), - .groups = "drop" - ) - -# plot with avg values -ggplot(fd_dev_k_avg, aes(x = k_offset, y = mean_deviation_k, color = method_full)) + - geom_point(alpha = 0.6, position = position_jitter(width = 0.15, height = 0)) + - facet_wrap(~dataset_name, scales = "free_y", ncol = 3) + - theme_minimal(base_size = 14) + - labs(title = "Deviation of metric across k offsets", - x = "Offset from true k", - y = "Mean ARI deviation from that of true `k`") + - scale_color_brewer(palette = "Set1") -``` From 206e6f207c0dd439c13097bab618efb00f82c9db Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Thu, 18 Dec 2025 18:23:49 +0100 Subject: [PATCH 17/18] Casting and NA filtering --- analyze_results_izaskun.Rmd | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/analyze_results_izaskun.Rmd b/analyze_results_izaskun.Rmd index 89965cd..1ea122b 100644 --- a/analyze_results_izaskun.Rmd +++ b/analyze_results_izaskun.Rmd @@ -138,7 +138,7 @@ flatten_record <- function(rec) { ```{r} records <- fromJSON("aggregated_results.json", simplifyVector = FALSE) -## records <- fromJSON("aggregated_results_full.json", simplifyVector = FALSE) +##records <- fromJSON("aggregated_results_full.json", simplifyVector = FALSE) ## fd_list <- vector("list", length(records)) ## for (i in seq_along(records)) { @@ -172,13 +172,15 @@ fd$k <- as.integer(fd$k) fd$true_k <- as.integer(fd$true_k) cols_to_num <- c("max_rss","max_vms","max_uss","max_pss", - "io_in","io_out","mean_load","cpu_time") + "io_in","io_out","mean_load","cpu_time", + 'metric_value') fd[cols_to_num] <- lapply(fd[cols_to_num], function(x) { x[x == "NA"] <- NA_character_ as.numeric(x) }) +fd <- fd[!is.na(fd$metric_value),] fd$k_offset <- fd$k - fd$true_k ## write.csv(fd, file = 'aggregated_results.csv') ## later, this needs extra cleaning ``` @@ -261,6 +263,7 @@ stopifnot(nrow(fd_seed_diff) == 0) ```{r} +fd <- fd[!is.na(fd$k),] stopifnot(range(fd$k_offset) == c(-2, 2)) ``` @@ -569,6 +572,7 @@ ggplot(fd_avg, aes(x = backend, y = mean_value)) + Mind there is no such a thing as true k, we use the first labelset. ```{r, fig.height = 10, fig.width = 13} +fd$metric_value <- as.numeric(fd$metric_value) fd_dev_k <- fd %>% group_by(method_full, dataset_name, metric_name) %>% From 9af1b779b7035a40a4a4a395b507082daab6eee6 Mon Sep 17 00:00:00 2001 From: Izaskun Mallona Date: Fri, 19 Dec 2025 10:46:11 +0100 Subject: [PATCH 18/18] Add cpu_time scatters by backend; Mark --- analyze_results_izaskun.Rmd | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/analyze_results_izaskun.Rmd b/analyze_results_izaskun.Rmd index 1ea122b..700022f 100644 --- a/analyze_results_izaskun.Rmd +++ b/analyze_results_izaskun.Rmd @@ -304,6 +304,38 @@ write.csv(fd, file = 'aggregated_results.csv') # CPU by backend +## QC by backend scatterplots + + +```{r, fig.width = 7, fig.height = 5} + +wide_cpu <- fd %>% + filter(backend %in% c("conda","envmodules","oras")) %>% + group_by(dataset_name, method, method_full, seed, run, backend) %>% + summarise(cpu_time = mean(as.numeric(cpu_time), na.rm = TRUE), .groups = "drop") %>% + pivot_wider(names_from = backend, values_from = cpu_time) + +ggplot(wide_cpu, aes(x = conda, y = oras, color = method)) + + geom_point(alpha = 0.6) + + labs(title = "CPU time: conda vs oras", + x = "conda", y = "oras") + + theme_minimal() + +ggplot(wide_cpu, aes(x = conda, y = envmodules, color = method)) + + geom_point(alpha = 0.6) + + labs(title = "CPU time: conda vs envmodules", + x = "conda", y = "envmodules") + + theme_minimal() + +ggplot(wide_cpu, aes(x = oras, y = envmodules, color = method)) + + geom_point(alpha = 0.6) + + labs(title = "CPU time: oras vs envmodules", + x = "oras", y = "envmodules") + + theme_minimal() + + +``` + ## Censoring aware : cpu_time < 0.05 plotted as 0.05 We impute NA cpu_time as 0.05 s @@ -635,7 +667,7 @@ ggplot(fd_dev_k_avg, aes(x = k_offset, y = mean_deviation_k, color = dataset_nam adj Rand index only -```{r, fig.height = 6, fig.width = 8} +```{r, fig.height = 6, fig.width = 16} # ilter to adjusted_rand_score only fd_dev_k_ars <- fd_dev_k_avg %>% filter(metric_name == "adjusted_rand_score")