update revs

.envrc

@@ -1 +1 @@
-use_nix shell.nix
+use nix

.gitignore

@@ -5,3 +5,6 @@
 .vscode
 qemu
 .Rhistory
+.RData
+blink
+.vagrant

.gitmodules

@@ -1,6 +1,10 @@
 [submodule "LibAFL"]
 	path = LibAFL
-	url = https://github.com/AFLplusplus/LibAFL
+	url = ssh://git@git.cs.tu-dortmund.de:2222/alwin.berger/FRET-LibAFL.git
 [submodule "qemu-libafl-bridge"]
 	path = qemu-libafl-bridge
-	url = https://github.com/AFLplusplus/qemu-libafl-bridge
+	url = ssh://git@git.cs.tu-dortmund.de:2222/alwin.berger/FRET-qemu.git
+[submodule "FreeRTOS"]
+	path = FreeRTOS
+	url = ssh://git@git.cs.tu-dortmund.de:2222/alwin.berger/FRET-FreeRTOS.git
+	branch = wcet-demo

AE.md

@@ -0,0 +1,235 @@
+# FRET Artifact Evaluation
+
+This document provides instructions for reproducing the experimental results from the paper ["Dynamic Fuzzing-Based Whole-System Timing Analysis"](https://sys-sideshow.cs.tu-dortmund.de/downloads/rtss25/paper.pdf).
+
+## Description
+
+**Expected Time**: 5 days (configurable)  
+**Hardware Requirements**: 64 cores, 256-512GB RAM (recommended), minimum 8 cores, 32-64GB RAM  
+**Software Requirements**: Linux x86_64, Nix package manager
+
+### Claims Supported by Artifact
+If you run the benchmarks as described below, it will produce the following files, which correspond to figures in the paper ([link to the paper](https://sys-sideshow.cs.tu-dortmund.de/downloads/rtss25/paper.pdf)).
+
+1. **Figure 3**: (Files: `sql_waters_seq_bytes`, `sql_polycopter_seq_dataflow_full`) This scenario is about mutating just input values. While multiple techniques find the worst case for both scenarios, FRET is the fastest to reach the maximum, particular in the second case. FRET also achieves the highest median result.  
+2. **Figure 4**: (Files: `sql_waters_seq_int`,`sql_release_seq_int`) This scenario is about mutating just interrupt times. While multiple techniques find the worst case for the second scenario, FRET achieves the highest response time on the first one.  
+3. **Figure 5**: (Files: `sql_release_seq_full`, `sql_waters_seq_full`) This scenario is about mutating both kinds of inputs simultaniously. Only FRET achieves the worst possible time when looking at the median results of the first scenario. For the second one, FRET alone reaches the highest response times. Thid demonstrates, that FRET's advantage over other techniques is particularly pronounced when both kinds of inputs are compared.  
+4. **Figure 6**: (File: `all_tasks`) This is a comparison of FRET's advantage over the best other technqiues on each task of the `waters` scenario. FRET ends up above every other technique on each task, validating the comparison in fig. 5 b).  
+
+## Getting Started
+
+### Option 1: VirtualBox Images (Recommended)
+Download our ready-made VM image from [https://sys-sideshow.cs.tu-dortmund.de/downloads/rtss25/fret.ova](https://sys-sideshow.cs.tu-dortmund.de/downloads/rtss25/fret.ova)
+
+- **VM Configuration**: Allocate as much RAM as possible (minimum 32GB, recommended 512GB)  
+- **CPU Allocation**: One core per 4-8GB of RAM (recommended 64 cores, 256-512GB RAM)  
+- **Disk Space**: At least 100GB free space for results  
+- **Login**: Username: `osboxes.org`, Password: `osboxes.org`  
+- **Open a terminal in ~/FRET**  
+   - Load the environemnt using ``direnv relaod`` or ``nix develop``
+- **Ensure you have the right version**: ``git checkout RTSS25-AE && git submodule update --init``  
+
+### Option 2: Setup From Scratch
+**Prerequisites**: Linux x86_64 system with Nix package manager installed
+
+1. Clone the repository:
+```bash
+git clone https://git.cs.tu-dortmund.de/alwin.berger/FRET
+cd FRET
+git checkout RTSS25-AE
+git submodule update --init
+```
+
+2. Install Nix (if not already installed):
+```bash
+curl -L https://nixos.org/nix/install | sh
+source ~/.nix-profile/etc/profile.d/nix.sh
+```
+
+3. Enter the development environment:
+```bash
+nix develop  # or nix-shell for older Nix versions
+```
+
+### Option 3: Docker Setup
+
+1. Clone the repository:
+```bash
+git clone --recursive https://git.cs.tu-dortmund.de/alwin.berger/FRET
+cd FRET
+git checkout RTSS25-AE
+git submodule update --init
+```
+
+2. See [Docker/README.md](Docker/README.md)
+
+### System Requirements
+
+#### Minimum Configuration
+- **CPU**: 8 cores (Intel/AMD x86_64)
+- **Memory**: 32GB RAM
+- **Storage**: 100GB free disk space
+- **OS**: Linux distribution with Nix support
+
+#### Recommended Configuration
+- **CPU**: 64 cores with hyperthreading
+- **Memory**: 256-512GB RAM (allows parallel execution of all configurations)
+- **Time for a full evaluation**: 5 days
+
+## Building and Installation
+
+### Initial Setup
+```bash
+./one_time_setup.sh
+```
+This script builds the default configuration of FRET, along with target systems and additional tools for benchmarking. The script will:
+- Build QEMU with FRET patches
+- Compile target FreeRTOS applications 
+- Build the FRET fuzzer and all dependencies
+- Set up the evaluation environment
+
+### Verification of Installation
+After setup completes, verify the installation:
+```bash
+# Check FRET binary
+cd LibAFL/fuzzers/FRET/benchmark
+target/fret --help
+
+# Check target systems
+ls build
+```
+
+## Step-by-Step Instructions (Optional)
+
+See [README.md](README.md) for reference.
+
+### Manual Usage (Optional)
+You can start using FRET manually. It requires the following inputs:
+
+- **-k**: a FreeRTOS Kernel image  
+- **-c**: a csv file with configuration parameters per target kernel (LibAFL/fuzzers/FRET/benhcmark/target_symbols.csv)  
+
+```bash
+cd LibAFL/fuzzers/FRET
+# Help for arguments
+cargo run -- --help
+# Example
+mkdir -p $TMPDIR
+export DUMP=$(mktemp -d)
+dd if=/dev/random of=$DUMP/input bs=8K count=1
+# fuzz for 10 seconds
+cargo run -- -k benchmark/build/waters_seq_full.elf -c benchmark/target_symbols.csv -n $DUMP/output -ta fuzz -t 10 --seed 123456
+# Produce a trace for the worst case found
+cargo run -- -k benchmark/build/waters_seq_full.elf -c benchmark/target_symbols.csv -n $DUMP/show -trg showmap -i $DUMP/output.case
+# plot the result
+gantt_driver $DUMP/show.trace.ron
+# view the gantt chart
+open $DUMP/show_job.html
+```
+
+## Run the Evaluation
+```sh
+# Feel free to read the script
+./run_eval.sh
+```
+This script will reproduce all figures 3-6 in the eval section of the paper.
+You can edit the environemnt variables at the top to change the following parameters:
+
+- `CORES`: The number of (physical) cores of the VM / fuzzers running in parallel. You will need about 8GB of RAM per fuzzer.  
+- `RUNTIME`: Time spent on each fuzzing run in seconds (the default is 24h). 8h should be sufficient to see results similar to the paper.  
+- `TARGET_REPLICA_NUMBER`: The number of replicas for each regular configuration.  
+- `RANDOM_REPLICA_NUMBER`: The number of replicas for configurations with random fuzzing. These usually deviate very little from each other and thus can be reduced without affecting the results.  
+- `MULTIJOB_REPLICA_NUMBER`: The number of replicas for the figure that compares all techniques for each task of a system. This evluation consists of many configurations, so you can reduce this numer to save a lot of time.  
+
+For complete reproduction of paper results you can use the following configuration, which takes about 5 days on a 64 core machine:
+```bash
+# in run_eval.sh
+export CORES=64 # Number of physical cores
+export RUNTIME=86400 # 24 hours in seconds
+export TARGET_REPLICA_NUMBER=12
+export RANDOM_REPLICA_NUMBER=3
+export MULTIJOB_REPLICA_NUMBER=3
+```
+
+## Results
+All results can be found in `LibAFL/fuzzers/FRET/benchmark` inside a directory `eval_xx-xx-xx`.
+The content should be the following:
+
+- Plots inside the top-level of the directory. You can compare them to the paper according to the hints under "Claims Supported by Artifact" above.  
+- A directory `timedump`, which contains subdirectories for each fuzzer.  
+   - Insude you find files for configuration with different seeds.  
+   - `.time` contain response times of each execution.  
+   - `.case` contains the worst case found by the fuzzer.  
+   - `.trace.ron` contains tracing data of the worst case. Such data can be plotted into agantt chart using the tool ``gantt_driver``  
+
+Please note that fuzzing is a stochastic process and therefore, some of the results may exhibit random variation from the figures in the paper.
+
+An archive of our results is also provided under [https://sys-sideshow.cs.tu-dortmund.de/downloads/rtss25/results.zip](https://sys-sideshow.cs.tu-dortmund.de/downloads/rtss25/results.zip).
+
+
+## Troubleshooting
+
+### Common Issues
+
+#### Out of Memory Errors
+- Reduce `CORES` parameter to match available RAM (up to 8GB per core)
+- Consider using fewer relicas per configuration
+
+#### Build Failures
+```bash
+# Clean and rebuild
+cd LibAFL/fuzzers/FRET
+cargo clean
+```
+
+#### Missing Dependencies
+```bash
+# Ensure Nix environment is active
+nix develop
+# Or for older Nix versions
+nix-shell
+```
+
+#### Incomplete Results
+- Check that snakemake completed without errors
+- Verify all target binaries were built successfully
+- Ensure sufficient disk space (100GB+)
+- If plots appear incomplete remove ``bench.sqlie`` from the benchmark directory.
+- If all else fails: ``snakemake -c1 full_clean``
+
+## Artifact Structure
+
+### Directory Layout
+```
+FRET/
++-- LibAFL/fuzzers/FRET/           # Main FRET fuzzer implementation
+|   +-- src/                       # Source code
+|   +-- benchmark/                 # Evaluation framework
++-- FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC # Our FreeRTOS Demos
++-- one_time_setup.sh             # Initial setup script
++-- run_eval.sh                   # Main evaluation script
++-- AE.md                        # This document
+```
+
+### Key Components
+- **FRET Fuzzer**: Core fuzzing engine with RTOS-aware features
+- **Target Applications**: Real-world FreeRTOS applications for testing
+- **Evaluation Framework**: Snakemake-based automation for reproducible experiments
+
+## Customization
+
+### Adding New Target Applications
+1. Add target configuration to `benchmark/target_symbols.csv`
+2. Update `benchmark/Snakefile` with new target rules
+
+## Further information
+Please consult [README.md](README.md) and [LibAFL/fuzzers/FRET/ARCH.md](LibAFL/fuzzers/FRET/ARCH.md)
+
+## Contact Information
+
+For questions about the artifact or issues during evaluation, please contact:
+- alwin.berger@tu-dortmund.de
+
+## License
+
+This artifact is provided under Apache License Version 2.0.

Docker/Containerfile

@@ -0,0 +1,21 @@
+FROM ubuntu:rolling
+# Nix + systemd
+RUN apt update -y
+RUN apt install curl systemd zsh -y
+RUN curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install linux \
+  --extra-conf "sandbox = false" \
+  --no-start-daemon \
+  --no-confirm
+ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin"
+RUN nix run nixpkgs#hello
+RUN systemctl enable nix-daemon.service
+CMD [ "/usr/lib/systemd/systemd" ]
+# SSH
+RUN apt update && apt install -y openssh-server
+RUN echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config
+RUN echo 'PasswordAuthentication no' >> /etc/ssh/sshd_config
+RUN systemctl enable ssh
+# FRET packages
+COPY ./fret.list /tmp/apt.list
+RUN apt install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" -y  $(grep -vE "^\s*#" /tmp/apt.list  | tr "\n" " ") 
+RUN rm /tmp/apt.list

Docker/README.md

@@ -0,0 +1,44 @@
+# Develop using a Container
+Use podamn oder docker to provide the environment.
+## New to Podman / Docker?
+Install either [Docker](https://www.docker.com/) or [Podman](https://podman.io/get-started) (replace `docker` in the following commands with `podman`)
+## Building the Container
+```sh
+docker build -t fretenv .
+```
+## Creating Container
+```sh
+docker create --name fretbuilder -v fretnixstore:/nix/store -v ../:/root/FRET localhost/fretenv:latest
+```
+The volume ``fretnixstore`` is optional. It is used to cache the packges installed using nix in case you need to re-create the container.
+Additionally, you can pass the following options to access the environment over ssh:
+```sh
+-p 2222:22 # 2222 is the ssh port of the container
+-v $SOMEWHERE:/root # somewhere with a .ssh directory
+```
+## Starting the Container
+```sh
+docker start fretbuilder
+```
+## Entering the Container
+```sh
+docker exec -it fretbuilder bash
+```
+## Using Nix
+```sh
+cd ~/FRET
+nix develop # or nix-shell
+```
+If you want to load the nix-shell automatically:
+```sh
+eval "$(direnv hook bash)"
+direnv allow
+```
+## Removing the Environment
+```sh
+docker stop fretbuilder
+docker container rm fretbuilder
+docker image rm fretenv
+```
+## Potential Issues
+If you run into a limit on threads when using podman, use ``podman create --pids-limit=8192 ...``

Docker/fret.list

@@ -0,0 +1,20 @@
+# Essentials
+git
+vim
+zsh
+rsync
+wget
+tmux
+direnv
+htop
+gdb
+ffmpeg
+# FRET
+sqlite3
+build-essential
+python3-pandas
+python3-jinja2
+python3-matplotlib
+python3-numpy
+python3-reportbug
+python3-scipy

Docker/prep_sys.sh

@@ -0,0 +1 @@
+podman create --memory 350g --memory-swap 351g --name basepod -p 127.0.0.1:8224:22 -v basestore:/nix/store -v /fs/scratch/alwin/containers/home:/root:z -v  /fs/scratch/alwin/FRET:/root/FRET:z localhost/ubuntu-nix-basepod:latest

FreeRTOS

@@ -0,0 +1 @@
+Subproject commit e1179c292dfb5612c8cb7c52f98aac4c8f0371e2

LibAFL

@@ -1 +1 @@
-Subproject commit eeaf7eb43f76fe744617e47807081c8cc447380b
+Subproject commit 25d598f30b342706e63fc2f0b17388c2824ef17f

README.md

@@ -0,0 +1,70 @@
+# FRET
+## Structure
+* `git submodule update --init`
+* LibAFL-based fuzzer under `LibAFL/fuzzers/FRET`
+* FreeRTOS demos under `FreeRTOS/FreeRTOS/Demo/CORTEX_M3_MPS2_QEMU_GCC`
+* QEMU instrumentation under `qemu-libafl-bridge`
+## HowTo
+### Development environment using nix
+Use `nix develop` or `nix-shell` to enter a shell with all required tools.
+### Development environment using podman/docker
+If you don't have nix installed, you can use it though a container.
+See Docker/README.md.
+### Potential Issues
+If you encounter errors where a temporary directory is not found, use `mkdir -p $TMPDIR`
+### Build FRET
+```sh
+cd LibAFL/fuzzers/FRET
+# First time and after changes to QEMU
+sh -c "unset CUSTOM_QEMU_NO_BUILD CUSTOM_QEMU_NO_CONFIGURE && cargo build"
+# Afterwards, simply use
+cargo build
+```
+### Build additional tools
+```sh
+LibAFL/fuzzers/FRET/tools/build.sh
+```
+### Build FreeRTOS Demos
+```sh
+cd LibAFL/fuzzers/FRET/benchmark
+sh build_all_demos.sh
+# see LibAFL/fuzzers/FRET/benchmark/build
+```
+### Example usage
+* Build the demos and additional tools first
+```sh
+cd LibAFL/fuzzers/FRET
+# Help for arguments
+cargo run -- --help
+# Example
+export DUMP=$(mktemp -d)
+dd if=/dev/random of=$DUMP/input bs=8K count=1
+# fuzz for 10 seconds
+cargo run -- -k benchmark/build/waters_seq_full.elf -c benchmark/target_symbols.csv -n $DUMP/output -tag fuzz -t 10 --seed 123456
+# Produce a trace for the worst case found
+cargo run -- -k benchmark/build/waters_seq_full.elf -c benchmark/target_symbols.csv -n $DUMP/show -tr showmap -i $DUMP/output.case
+# plot the result
+../../../state2gantt/driver.sh $DUMP/show.trace.ron
+# view the gantt chart
+open $DUMP/show_job.html
+```
+### Perform canned benchmarks
+* Build the demos and additional tools first
+* Select a benchmark set in `LibAFL/fuzzers/FRET/benchmark/Snakefile`
+* Hardware Requirements:
+    - Recommendation: 512GiB of RAM with 64 physical cores
+    - About 8GB of RAM per Job on average are required to prevent OOMs
+    - The set used for the paper consists of ~270 Jobs, so you will need about five day to reproduce the results
+```sh
+# $BENCHDIR
+cd LibAFL/fuzzers/FRET/benchmark
+# optional
+export BENCHDIR="eval_$(date -I)"
+# Reproduce the evals in the paper e.g.
+snakemake --cores 64 eval_bytes eval_int eval_full waters_multi
+# plot the resutls
+snakemake -c20 plot_benchmarks
+# See images in $BENCHDIR
+snakemake -c20 plot_traces
+# See HTML files in $BENCHDIR/timedump/*/ for traces of the worst cases
+```

edge_compare/.gitignore

@@ -1,6 +0,0 @@
-*.axf
-*.qcow2
-demo
-*.ron
-*.bsp
-target

edge_compare/Cargo.lock

@@ -1,220 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 3
-
-[[package]]
-name = "atty"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
-dependencies = [
- "hermit-abi",
- "libc",
- "winapi",
-]
-
-[[package]]
-name = "autocfg"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
-
-[[package]]
-name = "base64"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
-
-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
-[[package]]
-name = "clap"
-version = "3.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d76c22c9b9b215eeb8d016ad3a90417bd13cb24cf8142756e6472445876cab7"
-dependencies = [
- "atty",
- "bitflags",
- "indexmap",
- "os_str_bytes",
- "strsim",
- "termcolor",
- "textwrap",
-]
-
-[[package]]
-name = "edge_compare"
-version = "0.1.0"
-dependencies = [
- "clap",
- "ron",
- "serde",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
-
-[[package]]
-name = "hermit-abi"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "indexmap"
-version = "1.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
-dependencies = [
- "autocfg",
- "hashbrown",
-]
-
-[[package]]
-name = "libc"
-version = "0.2.119"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4"
-
-[[package]]
-name = "memchr"
-version = "2.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
-
-[[package]]
-name = "os_str_bytes"
-version = "6.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.36"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029"
-dependencies = [
- "unicode-xid",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "ron"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b861ecaade43ac97886a512b360d01d66be9f41f3c61088b42cedf92e03d678"
-dependencies = [
- "base64",
- "bitflags",
- "serde",
-]
-
-[[package]]
-name = "serde"
-version = "1.0.136"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789"
-dependencies = [
- "serde_derive",
-]
-
-[[package]]
-name = "serde_derive"
-version = "1.0.136"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "strsim"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
-
-[[package]]
-name = "syn"
-version = "1.0.86"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-xid",
-]
-
-[[package]]
-name = "termcolor"
-version = "1.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4"
-dependencies = [
- "winapi-util",
-]
-
-[[package]]
-name = "textwrap"
-version = "0.14.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80"
-
-[[package]]
-name = "unicode-xid"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
-
-[[package]]
-name = "winapi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
-dependencies = [
- "winapi-i686-pc-windows-gnu",
- "winapi-x86_64-pc-windows-gnu",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-
-[[package]]
-name = "winapi-util"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
-dependencies = [
- "winapi",
-]
-
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

edge_compare/Cargo.toml

@@ -1,17 +0,0 @@
-[package]
-name = "edge_compare"
-version = "0.1.0"
-authors = [ "Alwin Berger <alwin.berger@tu-dortmund.de>" ]
-edition = "2021"
-
-[features]
-default = ["std"]
-std = []
-
-[profile.release]
-debug = true
-
-[dependencies]
-clap = { version = "3.1.1", features = ["default"] }
-serde = { version = "1.0", default-features = false, features = ["alloc"] } # serialization lib
-ron = "0.7" # write serialized data - including hashmaps

edge_compare/src/main.rs

@@ -1,71 +0,0 @@
-use std::collections::HashMap;
-use std::path::PathBuf;
-use clap::Arg;
-use clap::App;
-use std::{env,fs};
-
-fn main() {
-    let res = match App::new("edge_compare")
-        .version("0.1.0")
-        .author("Alwin Berger")
-        .about("Compare Serialized Edge-Maps.")
-        .arg(
-            Arg::new("a")
-                .short('a')
-                .long("map-a")
-                .required(true)
-                .takes_value(true),
-        )
-        .arg(
-            Arg::new("b")
-                .short('b')
-                .long("map-b")
-                .required(true)
-                .takes_value(true),
-        )
-        .try_get_matches_from(env::args())
-    {
-        Ok(res) => res,
-        Err(err) => {
-            println!(
-                "Syntax: {}, --map-a <input> --map-b <input>\n{:?}",
-                env::current_exe()
-                    .unwrap_or_else(|_| "fuzzer".into())
-                    .to_string_lossy(),
-                err.info,
-            );
-            return;
-        }
-    };
-
-    let path_a = PathBuf::from(res.value_of("a").unwrap().to_string());
-    let path_b = PathBuf::from(res.value_of("b").unwrap().to_string());
-
-    let raw_a = fs::read(path_a).expect("Can not read dumped edges a");
-    let hmap_a : HashMap<(u64,u64),u64> = ron::from_str(&String::from_utf8_lossy(&raw_a)).expect("Can not parse HashMap");
-
-    let raw_b = fs::read(path_b).expect("Can not read dumped edges b");
-    let hmap_b : HashMap<(u64,u64),u64> = ron::from_str(&String::from_utf8_lossy(&raw_b)).expect("Can not parse HashMap");
-
-    let mut a_and_b = Vec::<((u64,u64),u64)>::new();
-    let mut a_and_b_differ = Vec::<((u64,u64),(u64,u64))>::new();
-    let mut a_sans_b = Vec::<((u64,u64),u64)>::new();
-
-    for i_a in hmap_a.clone() {
-        match hmap_b.get(&i_a.0) {
-            None => a_sans_b.push(i_a),
-            Some(x) => if i_a.1 == *x {
-                a_and_b.push(i_a);
-            } else {
-                a_and_b_differ.push((i_a.0,(i_a.1,*x)));
-            }
-        }
-    }
-    let b_sans_a : Vec<((u64,u64),u64)> = hmap_b.into_iter().filter(|x| !hmap_a.contains_key(&x.0) ).collect();
-
-    println!("a_sans_b: {:#?}\na_and_b_differ: {:#?}\nb_sans_a: {:#?}",&a_sans_b,&a_and_b_differ,&b_sans_a);
-    println!("Stats: a\\b: {} a&=b: {} a&!=b: {} b\\a: {} avb: {} jaccarde: {}",
-    a_sans_b.len(),a_and_b.len(),a_and_b_differ.len(),b_sans_a.len(),
-    a_and_b.len()+a_and_b_differ.len()+a_sans_b.len()+b_sans_a.len(),
-    (a_and_b.len()+a_and_b_differ.len())as f64/(a_and_b.len()+a_and_b_differ.len()+a_sans_b.len()+b_sans_a.len()) as f64);
-}

flake.lock

@@ -1,27 +1,15 @@
 {
   "nodes": {
     "flake-utils": {
-      "locked": {
+      "inputs": {
-        "lastModified": 1667395993,
+        "systems": "systems"
-        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
-        "type": "github"
       },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "flake-utils_2": {
       "locked": {
-        "lastModified": 1659877975,
+        "lastModified": 1731533236,
-        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
         "type": "github"
       },
       "original": {
@@ -32,11 +20,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1670064435,
+        "lastModified": 1752950548,
-        "narHash": "sha256-+ELoY30UN+Pl3Yn7RWRPabykwebsVK/kYE9JsIsUMxQ=",
+        "narHash": "sha256-NS6BLD0lxOrnCiEOcvQCDVPXafX1/ek1dfJHX1nUIzc=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "61a8a98e6d557e6dd7ed0cdb54c3a3e3bbc5e25c",
+        "rev": "c87b95e25065c028d31a94f06a62927d18763fdf",
         "type": "github"
       },
       "original": {
@@ -48,11 +36,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1665296151,
+        "lastModified": 1744536153,
-        "narHash": "sha256-uOB0oxqxN9K7XGF1hcnY+PQnlQJ+3bP2vCn/+Ru/bbc=",
+        "narHash": "sha256-awS2zRgF4uTwrOKwwiJcByDzDOdo3Q1rPZbiHQg/N38=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "14ccaaedd95a488dd7ae142757884d8e125b3363",
+        "rev": "18dd725c29603f582cf1900e0d25f9f1063dbf11",
         "type": "github"
       },
       "original": {
@@ -71,15 +59,14 @@
     },
     "rust-overlay": {
       "inputs": {
-        "flake-utils": "flake-utils_2",
         "nixpkgs": "nixpkgs_2"
       },
       "locked": {
-        "lastModified": 1670207212,
+        "lastModified": 1753238793,
-        "narHash": "sha256-uuKbbv0L+QoXiqO7METP9BihY0F7hJqGdKn7xDVfyFw=",
+        "narHash": "sha256-jmQeEpgX+++MEgrcikcwoSiI7vDZWLP0gci7XiWb9uQ=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "18823e511bc85ed27bfabe33cccecb389f9aa92d",
+        "rev": "0ad7ab4ca8e83febf147197e65c006dff60623ab",
         "type": "github"
       },
       "original": {
@@ -87,6 +74,21 @@
         "repo": "rust-overlay",
         "type": "github"
       }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -14,22 +14,36 @@
         pkgs = import nixpkgs {
           inherit system overlays;
         };
-        myrust = pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.default.override {
+        my-python-packages = ps: with ps; [
-          extensions = [ "rust-analyzer" "rust-src" "rust-docs" ];
+          sphinx
-        });
+          sphinx_rtd_theme
+          # other python packages
+        ];
+        R-with-my-packages = pkgs.rWrapper.override{ packages = with pkgs.rPackages; [ ggplot2 readr dplyr plotly mosaic DBI tikzDevice colorspace heatmaply RColorBrewer RSQLite languageserver ]; };
+        clang-all = pkgs.symlinkJoin {
+          name ="clang-all";
+          paths = (with pkgs; [ llvmPackages_19.libclang.out llvmPackages_19.libllvm.out llvmPackages_19.libunwind.out
+          llvmPackages_19.libclang.lib llvmPackages_19.libllvm.lib llvmPackages_19.libunwind
+          llvmPackages_19.libclang.dev llvmPackages_19.libllvm.dev llvmPackages_19.libunwind.dev ]);
+        };
       in
       with pkgs;
       rec {
-        devShell = mkShell.override {stdenv = llvmPackages_14.stdenv;} {  # LibAFL needs LLVM
+        devShell = mkShell.override {stdenv = stdenv;} {  # LibAFL needs LLVM
           buildInputs = [
-            # rust-bin.nightly.latest.default
+            (rust-bin.stable."1.87.0".default.override {
-            myrust
+              extensions = [ "rust-src" "rust-analyzer" ];
-            llvmPackages_14.libclang
+            })
+            cargo-make
+            llvmPackages_19.clangUseLLVM
+            clang-all
+            zlib
+            cargo-flamegraph
             # für qemu
-            python3
+            (pkgs.python3.withPackages my-python-packages)
             meson # unsure
             ninja
-            pkgconfig
+            pkg-config
             glib
             pixman
             # libslirp
@@ -37,41 +51,46 @@
             xxd
             # FreeRTOS
             gcc-arm-embedded
+            openssl_legacy  # gdb needs libcrypt.so.1
             # generate bindings from RTOS to Rust
             rust-bindgen
-            # compare libafl edges
-            packages.edge_compare
             # Debugging
             ddd
             # visualization
             graphviz
-            rstudioWrapper
+            #rstudioWrapper # prefer host packages for R
-            R
+            #R 
+            R-with-my-packages
+            pandoc
             # dependencies for mosaic
             freetype
             fontconfig
+            # benchmarking
+            snakemake
+            vim
+            psmisc
+            sqlite
+            heaptrack
           ];
 
           shellHook = ''
-          export CUSTOM_QEMU_DIR=/home/alwin/code/FRET/qemu-libafl-bridge
+          export INSIDE_DEVSHELL=1
+          export PATH=$PATH:$(pwd)/LibAFL/fuzzers/FRET/tools/bin
+          export CUSTOM_QEMU_DIR=$(pwd)/qemu-libafl-bridge
+          export LIBAFL_QEMU_DIR=$(pwd)/qemu-libafl-bridge
           export CUSTOM_QEMU_NO_BUILD=1
+          export CUSTOM_QEMU_NO_CONFIGURE=1
+          export LIBAFL_EDGES_MAP_SIZE_IN_USE=1048576
           # export EMULATION_MODE=systemmode
           # export CPU_TARGET=arm
           # export CROSS_CC=arm-none-eabi-gcc
-          export LIBCLANG_PATH=${llvmPackages_14.libclang.lib}/lib
+          export LIBCLANG_PATH=${clang-all}/lib
+          export BENCHDIR=bench_default
+
+          export FREERTOS_KERNEL_PATH=$(pwd)/FreeRTOS-Kernel
+          mkdir -p $TMPDIR
           '';
         };
-
-        packages = {
-          edge_compare=rustPlatform.buildRustPackage rec {
-            pname = "edge_compare";
-            version = "0.1.0";
-
-            src = ./edge_compare;
-
-            cargoSha256 = "sha256-47THUU9aKhAwb2Tz8bJWDmVhDokpr+DOiFNZhmsN8Gk=";
-          };
-        };
       }
     );
 }

graph2viz/.gitignore

@@ -1,4 +0,0 @@
-*.csv
-*.png
-*.pdf
-target

graph2viz/Cargo.toml

@@ -1,14 +0,0 @@
-[package]
-name = "graph2viz"
-version = "0.1.0"
-edition = "2021"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-fret = { path = "../LibAFL/fuzzers/FRET" }
-serde = { version = "1.0", default-features = false, features = ["alloc"] } # serialization lib
-hashbrown =  { version = "0.12", features = ["serde", "ahash-compile-time-rng"] } # A faster hashmap, nostd compatible
-petgraph = { version="0.6.0", features = ["serde-1"] }
-ron = "0.7" # write serialized data - including hashmaps
-rand = "0.5"

graph2viz/src/main.rs

@@ -1,23 +0,0 @@
-use std::path::PathBuf;
-use std::{env,fs};
-use fret::systemstate::graph::SysGraphFeedbackState;
-use petgraph::dot::{Dot, Config};
-
-fn main() {
-    let args : Vec<String> = env::args().collect();
-
-    let path_a = PathBuf::from(args[1].clone());
-    let raw_a = fs::read(path_a).expect("Can not read dumped traces b");
-    // let path_b = PathBuf::from(args[2].clone());
-
-    let feedbackstate : SysGraphFeedbackState = ron::from_str(&String::from_utf8_lossy(&raw_a)).expect("Can not parse HashMap");
-
-    let newgraph = feedbackstate.graph.map(
-        |_, n| n.get_taskname(),
-        // |_, n| format!("{} {:?}",n.get_taskname(),n.get_input_counts().iter().min().unwrap_or(&0)),
-        |_, e| e,
-    );
-    // let tempg = format!("{:?}",Dot::with_config(&newgraph, &[Config::EdgeNoLabel]));
-    println!("{:?}",Dot::with_config(&newgraph, &[Config::EdgeNoLabel]));
-
-}

one_time_setup.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+if  [[ -z "$INSIDE_DEVSHELL" ]]; then
+  echo "This script should be run inside a nix-shell. Run 'nix develop' or 'nix-shell' first."
+  exit 1
+fi
+
+# Always use the script's directory as the working directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+# Ensure that all sources are up-to-date
+#git submodule update --init --recursive
+
+# The central directory for the benchmarks
+cd LibAFL/fuzzers/FRET/benchmark
+
+# one-time setup
+# build QEMU for the first time
+snakemake -c 1 rebuild_qemu
+# Build kelper tools to aid the analysis of the benchmarks
+snakemake -c 1 build_tools
+# Build the kernels for the benchmarks
+snakemake -c 1 build_kernels
+
+cd -

qemu-libafl-bridge

@@ -1 +1 @@
-Subproject commit 7f19e258a17d3aa136df4ea990fcf11d263717ac
+Subproject commit fead70c87bb718728847ce2d4ef081f34b57656f

run_eval.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Configuration
+export CORES=64 # Number of physical cores
+export RUNTIME=86400 # 24 hours in seconds
+export TARGET_REPLICA_NUMBER=12
+export RANDOM_REPLICA_NUMBER=3
+export MULTIJOB_REPLICA_NUMBER=3
+
+if  [[ -z "$INSIDE_DEVSHELL" ]]; then
+  echo "This script should be run inside a nix-shell. Run 'nix develop' or 'nix-shell' first."
+  exit 1
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+cd LibAFL/fuzzers/FRET/benchmark
+
+export BENCHDIR="eval_$(date -I)"
+# prepare all fuzzer configurations
+snakemake --keep-incomplete --cores $CORES all_bins
+# Run the eval examples from the paper (eval_bytes = Fig. 3, eval_int = Fig. 4, eval_full = Fig. 5, waters_multi = Fig. 6)
+snakemake --keep-incomplete --cores $CORES eval_bytes eval_int eval_full waters_multi
+# plot the resutls
+snakemake --keep-incomplete --cores $CORES plot_benchmarks
+# See images in $BENCHDIR
+snakemake --keep-incomplete --cores $CORES plot_traces
+# See HTML files in $BENCHDIR/timedump/*/ for traces of the worst cases
+
+cd -

state2gantt/.gitignore

@@ -1,4 +0,0 @@
-*.csv
-*.png
-*.pdf
-target

state2gantt/Cargo.toml

@@ -1,14 +0,0 @@
-[package]
-name = "state2gantt"
-version = "0.1.0"
-edition = "2021"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-fret = { path = "../LibAFL/fuzzers/FRET" }
-serde = { version = "1.0", default-features = false, features = ["alloc"] } # serialization lib
-# hashbrown =  { version = "0.12", features = ["serde", "ahash-compile-time-rng"] } # A faster hashmap, nostd compatible
-# petgraph = { version="0.6.0", features = ["serde-1"] }
-ron = "0.7" # write serialized data - including hashmaps
-rand = "0.5"

state2gantt/Makefile

@@ -1,4 +0,0 @@
-all(%):
-	target/debug/state2gantt $%_afl.ron > $%_afl.csv
-	target/debug/state2gantt $%_state.ron > $%_state.csv
-	target/debug/state2gantt $%_random.ron > $%_random.csv

state2gantt/gantt.R

@@ -1,44 +0,0 @@
-args = commandArgs(trailingOnly=TRUE)
-if (length(args)==0) {
-  filename="~/code/FRET/state2gantt/trace.csv"
-} else {
-  filename=args[1]
-}
-trace <- read.csv(filename)
-
-task_ids = unique(trace[[3]]) # assume this has descending prio order
-prio_from_name <- function(t) {
-  1 + length(task_ids) - Position(function(y) y==t, task_ids)
-}
-
-trace[[3]]=sapply(trace[[3]], prio_from_name )
-
-width = 710
-height = (9/16) * width
-if (length(args)>0) { png(file=sprintf("%s.png",filename), width=width, height=height) }
-# prepare an empty plot
-plot(c(trace[[2]][1],trace[[2]][length(trace[[2]])]),
-     c(0,length(task_ids)),
-     col = "white", xlab = "", ylab = "")
-# draw all segments
-segments(x0 = trace$start,
-         y0 = trace$name,
-         x1 = trace$end,
-         y1 = trace$name,
-         lwd = 3)
-
-highlight_prio <- function(p,col) {
-  interest = trace[which(trace[[3]] == p),]
-  segments(x0 = interest$start,
-           y0 = interest$name,
-           x1 = interest$end,
-           y1 = interest$name,
-           lwd = 3,
-           col = col)
-}
-
-highlight_prio(1,"red")
-#highlight_prio(2,"green")
-
-if (length(args)>0) { dev.off() }
-

state2gantt/src/main.rs

@@ -1,20 +0,0 @@
-use std::collections::HashMap;
-use std::path::PathBuf;
-use std::{env,fs};
-use fret::systemstate::RefinedFreeRTOSSystemState;
-
-fn main() {
-    let args : Vec<String> = env::args().collect();
-
-    let path_a = PathBuf::from(args[1].clone());
-    let raw_a = fs::read(path_a).expect("Can not read dumped traces b");
-    // let path_b = PathBuf::from(args[2].clone());
-
-    let trace : Vec<RefinedFreeRTOSSystemState> = ron::from_str(&String::from_utf8_lossy(&raw_a)).expect("Can not parse HashMap");
-
-    println!("start,end,name");
-    for s in trace {
-        if s.current_task.task_name == "Tmr Svc" {continue;}
-        println!("{},{},{}",s.start_tick,s.end_tick,s.current_task.task_name);
-    }
-}