Reproducible Docker build for test cases in src/test

.gitattributes

@@ -0,0 +1,15 @@
+md5sums linguist-generated
+compiled.md5sum linguist-generated
+docker-hash linguist-generated
+*.gts linguist-generated
+*.adt linguist-generated
+*.bir linguist-generated
+*.relf linguist-generated
+
+src/test/compiled.url.txt text eol=lf
+src/test/make/docker-flake.txt text eol=lf
+
+Makefile text eol=lf
+*.sh text eol=lf
+*.mk text eol=lf
+*.md5sum text eol=lf

.github/workflows/run-examples.yml

@@ -79,6 +79,7 @@ jobs:
       - run: dotnet tool install --global boogie --version '3.4.3'
 
       - run: ./mill compile
+      - run: make -C src/test extract
       - run: ./mill test.test -oGD -T 1000000 -n test_util.tags.StandardSystemTest
 
       - uses: actions/upload-artifact@v4
@@ -130,6 +131,7 @@ jobs:
       - run: dotnet tool install --global boogie --version '3.4.3'
 
       - run: ./mill test.compile
+      - run: make -C src/test extract
       - run: ./mill test.test -oGD -T 1000000 -n test_util.tags.UnitTest
 
   UnitTests:
@@ -152,6 +154,7 @@ jobs:
       - run: dotnet tool install --global boogie --version '3.4.3'
 
       - run: ./mill test.compile
+      - run: make -C src/test extract
       - run: ./mill test.test -oGD -T 1000000 -n test_util.tags.UnitTest
 
         # every test with package prefix:
@@ -177,6 +180,7 @@ jobs:
       - run: dotnet tool install --global boogie --version '3.4.3'
 
       - run: ./mill test.compile
+      - run: make -C src/test extract
       - run: ./mill test.test -oGD -T 1000000 -n test_util.tags.AnalysisSystemTest -F 2
 
   AnalysisSystemTests2:
@@ -199,6 +203,7 @@ jobs:
       - run: dotnet tool install --global boogie --version '3.4.3'
 
       - run: ./mill test.compile
+      - run: make -C src/test extract
       - run: ./mill test.test -oGD -T 1000000 -n test_util.tags.AnalysisSystemTest2 -F 2
 
   AnalysisSystemTests3:
@@ -221,6 +226,7 @@ jobs:
       - run: dotnet tool install --global boogie --version '3.4.3'
 
       - run: ./mill test.compile
+      - run: make -C src/test extract
       - run: ./mill test.test -oGD -T 1000000 -n test_util.tags.AnalysisSystemTest3 -F 2
 
   AnalysisSystemTests4:
@@ -243,4 +249,5 @@ jobs:
       - run: dotnet tool install --global boogie --version '3.4.3'
 
       - run: ./mill test.compile
-      - run: ./mill test.test -oGD -T 1000000 -n test_util.tags.AnalysisSystemTest4 -F 2
+      - run: make -C src/test extract
+      - run: ./mill test.test -oGD -T 1000000 -n test_util.tags.AnalysisSystemTest4 -F 2

docs/development/readme.md

@@ -1,10 +1,10 @@
 # BASIL Development
 
 - [project-layout](project-layout.md) Organisation of the source code
-- [editor-setup](editor-setup.md) Guide to basil development in IDEs 
+- [editor-setup](editor-setup.md) Guide to basil development in IDEs
 - [tool-installation](tool-installation.md) Guide to lifter, etc. tool installation
 - [scala](scala.md) Advice on Scala programming.
-- [cfg](cfg.md) Explanation of the old CFG datastructure 
+- [cfg](cfg.md) Explanation of the old CFG datastructure
 - [interpreter](interpreter.md) Explanation of IR interpreter
 - [simplification-solvers](simplification-solvers.md) Explanation of simplification solvers
 
@@ -15,30 +15,30 @@ Basil is implemented in Scala 3.
 
 See also: [Scala Gotchas](scala.md).
 
-Scala is a mixed functional and object-oriented programming language implemented on the JVM. It is a very complicated 
-language with a lot of depth, so it is important to carefully chose the implementation complexity introduced. 
+Scala is a mixed functional and object-oriented programming language implemented on the JVM. It is a very complicated
+language with a lot of depth, so it is important to carefully chose the implementation complexity introduced.
 
-Generally, this means favouring simple standard solutions and choosing functional programming over stateful object oriented style 
+Generally, this means favouring simple standard solutions and choosing functional programming over stateful object oriented style
 (use filter and map rather than loops), prefer immutable case classes and enums to regular mutable classes, etc.
 
 ADTs and functions between ADTs are all you need to solve most problems. Most
-things more complicated than this make the code unnecessarily difficult to maintain. 
+things more complicated than this make the code unnecessarily difficult to maintain.
 
 It is recommended to explore the [Scala documentation](https://docs.scala-lang.org/scala3/book/introduction.html).
-There is also the incomplete [Scala 3 language specification](https://github.com/scala/scala3/tree/main/docs/_spec), 
+There is also the incomplete [Scala 3 language specification](https://github.com/scala/scala3/tree/main/docs/_spec),
 which contains details not present in the documentation, but is not completely updated from Scala 2.
 
 Some general advice:
 
 - Prefer [Enums](https://docs.scala-lang.org/scala3/book/types-adts-gadts.html) over inheritance trees
 - Use functional programming over imperative wherever possible
 - Prefer immutable case classes to regular classes wherever possible
-- Don't unneccessarily use generics or type aliases 
+- Don't unneccessarily use generics or type aliases
 - Correct code should not require explicit casts (`.asInstanceOf`)
 
 #### Code style
 
-We do not have a strict code style however 
+We do not have a strict code style however
 
 - Use two spaces for indentation
 - Use `{}` braces rather than purely indentation-based scoping
@@ -49,17 +49,17 @@ We do not have a strict code style however
 1. To set up an editor for Scala development see [editor-setup](editor-setup.md).
 2. Become familiar with the [project structure](project-layout.md) to start understanding the code.
 3. Install the neccessary tools [here](tool-installation.md), it may be useful to try
-   lifting examples, or just looking at existing examples in the 
+   lifting examples, or just looking at existing examples in the
    [src/test/correct](../../src/test/correct) directory.
-4. Use the below as a guide to common development tasks as they may arise. 
+4. Use the below as a guide to common development tasks as they may arise.
 
 ## Development tasks
 
 Mill should always be run from the git root directory (`git rev-parse --show-toplevel`).
 
 ## Building
 
-To build the project you can use `./mill build`. Often the incremental compilation database won't be properly invalidated on 
+To build the project you can use `./mill build`. Often the incremental compilation database won't be properly invalidated on
 `git switch`/`git checkout`/`git pull`: in these cases `./mill clean` is needed to trigger a full rebuild.
 
 To build a standalone `.jar` file, use `./mill assembly`. This can be found at `out/assembly.dest/out.jar`.
@@ -71,7 +71,7 @@ Printing an analysis result
 Run basil with the `--analysis-result <filepath>` flag to print the results of the analyses to files.
 The `--analysis-result-dot <filepath>` does the same, but outputs a graphviz digraph, which can be viewed by pasting it into [edotor.net](edotor.net), or compiling with `dot`.
 
-This prints the program with the abstract-domain lattice value at each program point following the `::`. 
+This prints the program with the abstract-domain lattice value at each program point following the `::`.
 
 ```
 Procedure main
@@ -84,14 +84,27 @@ Procedure main
 To achieve this for a new result in development use the following method defined in `RunUtils.scala`
 
 ```scala
-def printAnalysisResults(prog: Program, result: Map[CFGPosition, _]): String 
+def printAnalysisResults(prog: Program, result: Map[CFGPosition, _]): String
 ```
 
 
 ## Code Formatting
 
 All Pull Requests must conform to the style produced by `scalafmt`. This is checked automatically in CI, it can be run locally with:
 
+<<<<<<< HEAD
+The [dsl](../basil-ir.md#constructing-programs-in-code) can be used to construct simple example BASIL IR programs, which can then be fed through into the whole pipeline via `IRLoading.load()` in
+`RunUtils.scala`. Prefer to write tests that depend only on the single piece of code under test rather than the whole BASIL translation.
+
+### Integration tests
+
+These are the `SystemTests.scala` test case with the files present in `src/test/correct` for examples that should verify and `src/test/incorrect`
+for examples that should not verify.
+
+These are run via the Makefiles in src/test.
+A Docker image is used to compile and lift examples in a reproducible way.
+See [src/test/readme.md](../src/test/readme.md) for details.
+=======
 ```bash
 ./mill scalafmt.checkFormat
 ```
@@ -105,24 +118,29 @@ Sources can be reformatted with:
 
 ## Tests
 
+The tests are defined as C source files and must be compiled before use.
+Before running any tests, download the pre-compiled test case files with:
+```bash
+$ make -C src/test extract
+```
 
-### Running Tests
-
-See [testing](testing.md).
+For information about running tests, see [testing](testing.md).
 
+For more details on the process of compiling C test cases, including adding and editing test cases,
+see [src/test/readme.md](../src/test/readme.md).
 
 ## Performance profiling
 
-While the first priority is correctness, the performance target for the static 
-analyses is that we can run through the entire 
-[cntlm](https://github.com/versat/cntlm) binary in a reasonable amount of time 
-(seconds), depending on the expected performance of the analysis involved. 
+While the first priority is correctness, the performance target for the static
+analyses is that we can run through the entire
+[cntlm](https://github.com/versat/cntlm) binary in a reasonable amount of time
+(seconds), depending on the expected performance of the analysis involved.
 Loading cntlm requires increasing the heap size by providing the `-Xmx8G` flag.
 
 IntelliJ professional (which can be obtained for free by students) includes a performance profiler.
 
-Alternatively, [async-profiler](https://github.com/async-profiler/async-profiler) can be used to produce a 
-[flame graph](https://brendangregg.com/flamegraphs.html) showing the hot-spots in the program. Download the library from 
+Alternatively, [async-profiler](https://github.com/async-profiler/async-profiler) can be used to produce a
+[flame graph](https://brendangregg.com/flamegraphs.html) showing the hot-spots in the program. Download the library from
 the [releases tab](https://github.com/async-profiler/async-profiler/releases), compile a basil .jar with `mill assembly` and run the jar with the following arguments.
 
 Instructions for Linux and Mac:

docs/development/testing.md

@@ -154,7 +154,12 @@ Both of these will be output in yellow text if your console is using colour.
 
 ## Running Individual suites
 
-From git root directory run `mill test.testOnly`, tests suite names are glob matched against its argument, 
+Before running any tests, download the pre-compiled test case files with:
+```bash
+$ make -C src/test extract
+```
+
+From git root directory run `mill test.testOnly`, tests suite names are glob matched against its argument,
 for example to run all the SystemTest variants use:
 
 ```
@@ -164,19 +169,22 @@ $ ./mill test.testOnly 'SystemTests*'
 
 ### Compiling the Integration test binaries
 
-These are checked in to the respository, but can be recompiled (and new tests compiled) with the following instructions: 
-
 These are the `SystemTests.scala` test case with the files present in `src/test/correct` for examples that should verify and `src/test/incorrect`
-for examples that should not verify. 
+for examples that should not verify.
+
+The tests are defined as C source files and must be compiled and lifted before use.
+These are lifted via the Makefiles, to add another test simply add a directory, c source file, and optionally specification file and run
 
-These are lifted via the Makefiles, to add another test simply add a directory, c source file, and optionally specification file and run 
+Compilation is done in a Docker container environment for consistency.
+A tarball of pre-compiled files is provided to get started quickly.
 
 ```sh
 cd src/test/
+make extract
 make
 ```
 
-The `config.mk` file in the test directory can be used to exclude unnecessary compilers, and change compilation flags. 
+The `config.mk` file in the test directory can be used to exclude unnecessary compilers, and change compilation flags.
 Full details can be found [here](../src/test/readme.md).
 
 

src/main/scala/analysis/data_structure_analysis/Graph.scala

@@ -843,7 +843,7 @@ class Graph(using Counter)(
   }
 
   def SSAVar(posLabel: String, varName: String): Slice = {
-    assert(posLabel.matches("%[0-9a-f]{8}?\\$\\d"), s"posLabel not matching BAP format '$posLabel'")
+    assert(posLabel.matches("""%[0-9a-f]{8}?(:\$\d)?"""), s"posLabel not matching BAP format '$posLabel'")
 
     val res = varToCell.keys.filter(pos => pos.toShortString.startsWith(posLabel))
     assert(res.size == 1, s"failed to get SSAVar for '$posLabel' and '$varName'. matched label: ${res}")

src/main/scala/ir/IRCursor.scala

@@ -7,7 +7,7 @@ import ir.cilvisitor.*
 import collection.mutable
 import scala.annotation.tailrec
 
-/** 
+/**
  *  This file defines functions to get the successor and predecessor of a IR node for control flow.
  */
 

src/main/scala/translating/BAPToIR.scala

@@ -240,10 +240,12 @@ class BAPToIR(var program: BAPProgram, mainAddress: BigInt) {
       val lhs = translateVar(b.lhs)
       val (rhs, load) = translateExpr(b.rhs)
       if (load.isDefined) {
-        val loadWithLabel =
-          MemoryLoad(load.get.lhs, load.get.mem, load.get.index, load.get.endian, load.get.size, Some(b.line + "$0"))
-        val assign = LocalAssign(lhs, rhs, Some(b.line + "$1"))
-        Seq(loadWithLabel, assign)
+        if rhs != load.get.lhs then {
+          Seq(MemoryLoad(load.get.lhs, load.get.mem, load.get.index, load.get.endian, load.get.size, Some(b.line + "$0")),
+          LocalAssign(lhs, rhs, Some(b.line + "$1")))
+        } else {
+          Seq(MemoryLoad(lhs, load.get.mem, load.get.index, load.get.endian, load.get.size, Some(b.line + "$0")))
+        }
       } else {
         val assign = LocalAssign(lhs, rhs, Some(b.line))
         Seq(assign)

src/test/.gitignore

@@ -2,4 +2,17 @@
 *.csv
 *.md
 *.md.part
-*.svg
+*.svg
+
+# files reproducibly generated
+*.relf
+*.adt
+*.bir
+*.gtirb
+*.gts
+docker-hash-new
+!compiled.url.txt
+*.gz
+*.bz2
+*.xz
+*.zst

src/test/Makefile

@@ -1,12 +1,60 @@
-TARGETSS := all verify clean cleanall cleanlift recompile json cleanjson cleangts gts
+TARGETSS := all verify repro-stash repro-check md5sum-check md5sum-update clean cleanall cleanlift recompile json cleanjson cleangts gts
 
-SUBTARGETS = $(wildcard correct/*/ incorrect/*/)
-.PHONY : $(TARGETSS) $(SUBTARGETS) correct incorrect
+# subdirectories of src/test. to be entered into by this makefile.
+DIRS := correct incorrect extraspec_correct extraspec_incorrect indirect_calls memory_regions procedure_summaries
+# dirs with different directory structure: dsa, irreducible_loops
 
-$(TARGETSS): $(SUBTARGETS)
+# non-test dirs: make, scala, unimplemented
 
-correct: $(realpath $(wildcard correct/*))
-incorrect: $(realpath $(wildcard incorrect/*))
+# in case the user specfies DIRS, make sure all dirs exist.
+$(foreach d, $(DIRS), \
+  $(if $(wildcard $(d)/.), \
+    , \
+    $(error user error: directory "$(d)" in DIRS variable does not exist)))
+
+SUBDIRS = $(wildcard $(addsuffix /*/,$(DIRS)))
+.PHONY : $(TARGETSS) $(SUBDIRS) $(DIRS)
+
+# through some unpleasantness, this lets the user specify either DIRS or SUBDIRS
+# on the command line, and the make operation will be narrowed to that directory
+$(TARGETSS): $(SUBDIRS)
+
+$(SUBDIRS):
+	$(MAKE) -C $@ -f $(realpath ./make/lift-directories.mk) $(MAKECMDGOALS)
+
+# concats md5sums files in subdirectories into a compiled.md5sum.
+# check with `md5sum -c compiled.md5sum` in src/test.
+.PHONY: compiled.md5sum
+compiled.md5sum:
+	find $(DIRS) -name '*.md5sum' -exec cat '{}' + | sort -k2 > compiled.md5sum
+
+TARBALL := compiled.tar.zst
+
+$(TARBALL) docker-contents.txt &: compiled.md5sum
+	set -u; $$DOCKER_CMD hash > docker-contents.txt  # before compessing, make sure docker-contents.txt is up to date.
+	md5sum --quiet -c compiled.md5sum  # before compressing, make sure our files match expected hashes.
+	list=`mktemp`; cut -d' ' -f3 compiled.md5sum > $$list && tar caf $(TARBALL) -T $$list && rm $$list
+	sha1sum $(TARBALL)
+
+.PHONY: extract
+extract:
+	# log URL and expected hash
+	{ head -n1 compiled.url.txt; tail -n1 compiled.url.txt; } | cat -v
+	# check existing file, otherwise download fresh copy.
+	{ tail -n1 compiled.url.txt | sha1sum -c - ; } \
+		|| curl "$$(head -n1 compiled.url.txt)" -o $(TARBALL)
+	# check file type.
+	-file $(TARBALL)
+	# validate the hash, otherwise remove the incorrect file and abort.
+	{ tail -n1 compiled.url.txt | sha1sum -c - ; } || { rm -v $(TARBALL); exit 1; }
+	tar xf $(TARBALL) --keep-old-files --touch
+	md5sum --quiet -c compiled.md5sum  # check that extracted files match expected checksums
+
+.PHONY: push
+push:
+	tmp=`mktemp -d` && \
+			git clone git@github.com:UQ-PAC/basil-tests.git $$tmp --single-branch --branch basil-src-test && \
+			cd $$tmp && \
+
+			rm -rf $$tmp
 
-$(SUBTARGETS):
-	-$(MAKE) -C $@ -f $(realpath ./make/lift-directories.mk) $(MAKECMDGOALS)