add docker-helper.sh to lift examples using pac-nix docker image

.gitattributes

@@ -1,3 +1,11 @@
 *.mill linguist-language=Scala
 mill linguist-generated
 mill.bat linguist-generated
+
+src/test/compiled.url.txt text eol=lf
+src/test/make/docker-flake.txt text eol=lf
+
+Makefile text eol=lf
+*.sh text eol=lf
+*.mk text eol=lf
+*.md5sum text eol=lf linguist-generated

src/test/Makefile

@@ -1,12 +1,60 @@
-TARGETSS := all verify clean cleanall cleanlift recompile json cleanjson cleangts gts
+TARGETSS := all verify repro-stash repro-check md5sum-check md5sum-update clean cleanall cleanlift recompile json cleanjson cleangts gts
 
-SUBTARGETS = $(wildcard correct/*/ incorrect/*/)
-.PHONY : $(TARGETSS) $(SUBTARGETS) correct incorrect
+# subdirectories of src/test. to be entered into by this makefile.
+DIRS := correct incorrect extraspec_correct extraspec_incorrect indirect_calls memory_regions procedure_summaries
+# dirs with different directory structure: dsa, irreducible_loops
 
-$(TARGETSS): $(SUBTARGETS)
+# non-test dirs: make, scala, unimplemented
 
-correct: $(realpath $(wildcard correct/*))
-incorrect: $(realpath $(wildcard incorrect/*))
+# in case the user specfies DIRS, make sure all dirs exist.
+$(foreach d, $(DIRS), \
+  $(if $(wildcard $(d)/.), \
+    , \
+    $(error user error: directory "$(d)" in DIRS variable does not exist)))
+
+SUBDIRS = $(wildcard $(addsuffix /*/,$(DIRS)))
+.PHONY : $(TARGETSS) $(SUBDIRS) $(DIRS)
+
+# through some unpleasantness, this lets the user specify either DIRS or SUBDIRS
+# on the command line, and the make operation will be narrowed to that directory
+$(TARGETSS): $(SUBDIRS)
+
+$(SUBDIRS):
+	$(MAKE) -C $@ -f $(realpath ./make/lift-directories.mk) $(MAKECMDGOALS)
+
+# concats md5sums files in subdirectories into a compiled.md5sum.
+# check with `md5sum -c compiled.md5sum` in src/test.
+.PHONY: compiled.md5sum
+compiled.md5sum:
+	find $(DIRS) -name '*.md5sum' -exec cat '{}' + | sort -k2 > compiled.md5sum
+
+TARBALL := compiled.tar.zst
+
+$(TARBALL) docker-contents.txt &: compiled.md5sum
+	set -u; $$DOCKER_CMD hash > docker-contents.txt  # before compessing, make sure docker-contents.txt is up to date.
+	md5sum --quiet -c compiled.md5sum  # before compressing, make sure our files match expected hashes.
+	list=`mktemp`; cut -d' ' -f3 compiled.md5sum > $$list && tar caf $(TARBALL) -T $$list && rm $$list
+	sha1sum $(TARBALL)
+
+.PHONY: extract
+extract:
+	# log URL and expected hash
+	{ head -n1 compiled.url.txt; tail -n1 compiled.url.txt; } | cat -v
+	# check existing file, otherwise download fresh copy.
+	{ tail -n1 compiled.url.txt | sha1sum -c - ; } \
+		|| curl "$$(head -n1 compiled.url.txt)" -o $(TARBALL)
+	# check file type.
+	-file $(TARBALL)
+	# validate the hash, otherwise remove the incorrect file and abort.
+	{ tail -n1 compiled.url.txt | sha1sum -c - ; } || { rm -v $(TARBALL); exit 1; }
+	tar xf $(TARBALL) --keep-old-files --touch
+	md5sum --quiet -c compiled.md5sum  # check that extracted files match expected checksums
+
+.PHONY: push
+push:
+	tmp=`mktemp -d` && \
+			git clone git@github.com:UQ-PAC/basil-tests.git $$tmp --single-branch --branch basil-src-test && \
+			cd $$tmp && \
+
+			rm -rf $$tmp
 
-$(SUBTARGETS):
-	-$(MAKE) -C $@ -f $(realpath ./make/lift-directories.mk) $(MAKECMDGOALS)

src/test/make/bap-normalise.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+# vim: ts=2 sts=2 et sw=2
+
+"""
+BAP .adt / .bir file normaliser:
+
+usage:
+bap-normalise.py ADT-FILE BIR-FILE
+
+both arguments are required in that order!
+files will be modified in-place.
+
++00000540: main_argv :: in out u64 = R1
++00000541: main_result :: out u32 = low:32[R0]
+
+ 00000199:
+ 0000019c: R1 := 0x41F000
+@@ -187,7 +187,7 @@
+ 000001c8: call R30 with noreturn
+
+ 00000515: sub register_tm_clones(register_tm_clones_result)
+-00000529: register_tm_clones_result :: out u32 = low:32[R0]
++00000542: register_tm_clones_result :: out u32 = low:32[R0]
+
+ 0000028e:
+ 00000291: R0 := 0x420000
+@@ -199,13 +199,13 @@
+ 000002b6: R1 := R2 + (R1 ~>> 3)
+ 000002bc: R1 := extend:64[63:1[R1]]
+ 000002c2: when R1 = 0 goto %000002c0
+-00000516: goto %00000337
++0000052f: goto %00000337
+
+ 00000337:
+ 0000033a: R2 := 0x41F000
+ 00000341: R2 := mem[R2 + 0xFF8, el]:u64
+ 00000346: when R2 = 0 goto %000002c0
+-00000517: goto %0000034a
++00000530: goto %0000034a
+
+"""
+
+import sys
+import re
+
+adt_file = sys.argv[1]
+bir_file = sys.argv[2]
+assert len(sys.argv) == 3
+
+string_re = re.compile(rb'''"((?:[^"\\]|\\.)*)"''')
+hexstring_re = re.compile(rb'''"%([\da-fA-F]{8})"''')
+tid_re = re.compile(rb'''Tid\(([_\d]+),''')
+bir_re = re.compile(rb'''(?:^([\da-fA-F]{8}):)|(?: %([\da-fA-F]{8}))''', re.MULTILINE)
+
+with open(adt_file, 'rb') as f:
+  adt = f.read()
+
+tids: dict[int, int] = {}  # map of old tid to their first position in adt
+for match in re.finditer(tid_re, adt):
+  tid = int(match[1].replace(b'_', b''))
+  if tid not in tids:
+    tids[tid] = match.start()
+
+assert tids, f'adt file {adt_file} has no Tid() values??'
+
+keys = list(tids.keys())
+keys.sort(key=tids.__getitem__)
+
+new_tids = {tid: 4*i for i, tid in enumerate(keys)}
+
+# .adt file
+
+def sub_adt(m: re.Match[bytes]) -> bytes:
+  tid = int(m[1].replace(b'_', b''))
+  new = new_tids[tid]
+  return f'Tid({new:_},'.encode('ascii')
+def sub_adt_strings(m: re.Match[bytes]) -> bytes:
+  tid = int(m[1], 16)
+  new = new_tids[tid]
+  return f'"%{new:08x}"'.encode('ascii')
+
+new_adt = re.sub(tid_re, sub_adt, adt)
+new_adt = re.sub(hexstring_re, sub_adt_strings, new_adt)
+
+# .bir file
+
+# print(new_tids)
+bir_seen = set()
+def sub_bir(m: re.Match[bytes]) -> bytes:
+  old = m[1] or m[2]
+  tid = int(old, 16)
+  bir_seen.add(tid)
+  assert tid in new_tids, f"{m}"
+  new = new_tids[tid]
+  return m[0].replace(old, f'{new:08x}'.encode('ascii'))
+
+with open(bir_file, 'rb') as f:
+  bir = f.read()
+
+new_bir = re.sub(bir_re, sub_bir, bir)
+adt_seen = set(new_tids)
+assert bir_seen == adt_seen, f'not equal!\nbir - adt =\n{bir_seen - adt_seen}\nadt - bir =\n{adt_seen - bir_seen}'
+
+assert new_adt != adt
+assert new_bir != bir
+
+# writeback only if both are successful
+
+with open(bir_file, 'wb') as f:
+  f.write(new_bir)
+
+with open(adt_file, 'wb') as f:
+  f.write(new_adt)

src/test/make/docker-flake.txt

@@ -0,0 +1 @@
+github:katrinafyi/pac-nix/569afdf78558de82c24d25e12680157c3b0aa3df#basil-tools-docker

src/test/make/docker-helper.sh

@@ -0,0 +1,172 @@
+#!/usr/bin/env bash
+set -ue
+
+if [[ -z "${GIT_ROOT:-}" ]] && command -v git &>/dev/null; then
+  GIT_ROOT=$(git rev-parse --show-toplevel)
+fi
+: ${GIT_ROOT}
+DIR=$(realpath --relative-to "$GIT_ROOT" .)
+
+: ${DOCKER:=podman}
+: ${DOCKER_PLATFORM:=--platform linux\/amd64}
+: ${DOCKER_USER:=root}
+: ${DOCKER_IMAGE:=ghcr.io/uq-pac/basil-tools-docker}
+
+if [[ $# -lt 1 ]] || [[ "$1" == --help ]]; then
+  echo "usage: $(basename $0) (pull | push | build | start | stop | shell | hash | env [--unset] | COMMAND...)"
+  ! [[ $# -lt 1 ]]
+  exit
+fi
+
+DOCKER_CMD="$(realpath $0)"
+
+if [[ -z "${DOCKER_FLAKE:-}" ]] && [[ -r "$(dirname $DOCKER_CMD)/docker-flake.txt" ]]; then
+  DOCKER_FLAKE="$(cat $(dirname $DOCKER_CMD)/docker-flake.txt)"
+fi
+
+: $DOCKER_FLAKE
+
+# create unique names depending on the flake reference, to ensure the correct container
+# is used.
+# unique names depend only on DOCKER_FLAKE, allowing them to be computed without nix.
+commit=$(printf '%s' "$DOCKER_FLAKE" | grep --only-matching -E '[0-9a-fA-F]{40}' | head -c8)
+flake_hash=flake-$(printf '%s' "$DOCKER_FLAKE" | md5sum | cut -d' ' -f1 | head -c4)
+
+if [[ -z "${DOCKER_TAG:-}" ]]; then
+  DOCKER_TAG="$flake_hash-$commit"
+fi
+
+unique_image="$DOCKER_IMAGE:$DOCKER_TAG"
+unique_container="container-$DOCKER_TAG"
+
+# this allows the env subcommand to output syntax compatible with multiple shells
+shell=$(basename $SHELL)
+if [[ $shell == fish ]]; then
+  unset='set --erase'
+  unalias='functions --erase'
+  eval='('
+else
+  unset=unset
+  unalias=unalias
+  eval='$('
+fi
+
+
+if [[ "$1" == pull ]]; then
+  # pulls the unique image from the registry
+  set -x
+  exec $DOCKER pull $DOCKER_PLATFORM "$unique_image"
+
+elif [[ "$1" == push ]]; then
+  # pushes the unique image to the registry. image must already exist locally.
+  set -x
+  exec $DOCKER push "$unique_image"
+
+elif [[ "$1" == build ]]; then
+  # builds the docker image for running tools.
+  # safe to re-run. if docker image is already up-to-date, should be reasonably fast.
+  nix build "$DOCKER_FLAKE" --no-link
+  nix build "$DOCKER_FLAKE.conf" --no-link
+  conf=$(nix build "$DOCKER_FLAKE.conf" --no-link --print-out-paths)
+  tag=$(nix eval --expr "with builtins; (fromJSON (unsafeDiscardStringContext (readFile $conf))).repo_tag" --impure --raw)
+  if ! [[ "$tag" == "$DOCKER_IMAGE":* ]]; then
+    printf '%s %s %s.\n' \
+      "ERROR: docker image names do not match!" \
+      "nix flake will build '$tag', but" \
+      "DOCKER_IMAGE is '$DOCKER_IMAGE'" >&2
+    exit 1
+  fi
+  set -x
+  $(nix build "$DOCKER_FLAKE" --no-link --print-out-paths) | "$DOCKER" image load
+  $DOCKER image tag "$tag" $unique_image
+  exit
+
+elif [[ "$1" == start ]]; then
+  # starts an instance of the docker image.
+  set -x
+  exec $DOCKER run $DOCKER_PLATFORM -v"$GIT_ROOT:$GIT_ROOT" --rm -td --user $DOCKER_USER --name $unique_container $unique_image
+
+elif [[ "$1" == stop ]]; then
+  # stops the instance of the docker image.
+  set -x
+  exec $DOCKER stop -t 1 $unique_container
+  # since --rm is given to `docker run`, this will also remove the container.
+
+elif [[ "$1" == shell ]]; then
+  # enters an interactive shell within the container.
+  set -x
+  exec $DOCKER exec -it --user $DOCKER_USER -w "$GIT_ROOT/$DIR" -eshell=1 $unique_container /usr/bin/_exec bash
+
+elif [[ "$1" == hash ]]; then
+  # outputs information about the docker image's version to stdout.
+  echo "$DOCKER_FLAKE"
+  echo
+  exec "$DOCKER_CMD" bash -c 'ls -1 /nix/store | sort -k1.33' # sort /nix/store contents by name, not hash
+
+elif [[ "$1" == env ]]; then
+  # outputs commands to set the environment to stdout.
+  # when passed to `eval`, these commands should prepare the shell for running
+  # basil tests through docker.
+
+  # if --unset is used, removes all definitions
+  isunset=$([[ $# -ge 2 ]] && [[ "$2" == --unset ]] && echo true || echo false)
+  # if --reset is used, removes all definitions, then re-adds them based on defaults
+  isreset=$([[ $# -ge 2 ]] && [[ "$2" == --reset ]] && echo true || echo false)
+
+  if $isreset; then
+    isunset=true
+  fi
+
+  function echoexport() {
+    if $isunset; then
+      echo echo "$unset" "$1" ';'
+      echo "$unset" "$1" ';'
+      return
+    fi
+    printf 'echo "%s = %s";\n' "$1" "$2"
+    printf 'export %s="%s";\n' "$1" "$2"
+  }
+
+  echoexport USE_DOCKER "1"
+  echoexport DOCKER_FLAKE "$DOCKER_FLAKE"
+  echoexport DOCKER_IMAGE "$DOCKER_IMAGE"
+  echoexport DOCKER_TAG "$DOCKER_TAG"
+  echoexport DOCKER_PLATFORM "$DOCKER_PLATFORM"
+  echoexport DOCKER "$DOCKER"
+  echoexport DOCKER_USER "$DOCKER_USER"
+  echoexport DOCKER_CMD "$DOCKER_CMD"
+  echoexport GIT_ROOT "$GIT_ROOT"
+  echo 'echo;'
+  echoexport GCC "$DOCKER_CMD aarch64-unknown-linux-gnu-gcc"
+  echoexport CLANG "$DOCKER_CMD aarch64-unknown-linux-gnu-clang"
+  echoexport READELF "$DOCKER_CMD aarch64-unknown-linux-gnu-readelf"
+  echoexport BAP "$DOCKER_CMD bap"
+  echoexport DDISASM "$DOCKER_CMD ddisasm"
+  echoexport PROTO_JSON "$DOCKER_CMD proto-json.py"
+  # echoexport PROTO_JSON "/home/rina/progs/gtirb-semantics/scripts/proto-json.py"
+  echoexport DEBUG_GTS "$DOCKER_CMD debug-gts.py"
+  echoexport GTIRB_SEMANTICS "$DOCKER_CMD gtirb-semantics"
+  echo 'echo;'
+  if $isunset; then
+    echo "echo $unalias docker-helper.sh;"
+    echo "$unalias docker-helper.sh;"
+  else
+    echo "echo alias docker-helper.sh = '$DOCKER_CMD';"
+    echo "alias 'docker-helper.sh=$DOCKER_CMD';"
+  fi
+
+  if $isreset; then
+    echo "eval $eval$DOCKER_CMD env);"
+  fi
+  exit
+fi
+
+if [[ -n "${NIX_BUILD_TOP:-}" ]]; then
+  set -x
+  # if already inside a Nix shell, simply execute
+  exec /usr/bin/_exec "$@"
+else
+  set -x
+  # for other commands, execute within the container.
+  exec $DOCKER exec --user $DOCKER_USER -w "$GIT_ROOT/$DIR" $unique_container /usr/bin/_exec "$@"
+fi

src/test/make/gcc.mk

@@ -1,2 +1,3 @@
 CC=$(GCC)
+CFLAGS += -pie
 include $(GIT_ROOT)/src/test/make/lift.mk

src/test/make/gcc_O2.mk

@@ -1,3 +1,3 @@
 CC=$(GCC)
-CFLAGS += -O2
+CFLAGS += -pie -O2
 include $(GIT_ROOT)/src/test/make/lift.mk

src/test/make/gcc_pic.mk

@@ -1,3 +1,3 @@
 CC=$(GCC)
-CFLAGS += -fpic
+CFLAGS += -pie -fpic
 include $(GIT_ROOT)/src/test/make/lift.mk