Provide script to run all internal Jest fuzz tests

These are also executed in the CI system parallel to the normal unit tests.
CodeIntelligenceTesting · Jan 10, 2023 · 99439a1 · 99439a1
1 parent 9da122c
commit 99439a1
Show file tree

Hide file tree

Showing 3 changed files with 137 additions and 2 deletions.
diff --git a/.github/workflows/run-all-tests.yaml b/.github/workflows/run-all-tests.yaml
@@ -19,8 +19,6 @@ jobs:
         run: npm install
       - name: install dependencies with apt
         run: sudo apt-get install --yes clang-tidy
-      - name: build fuzzer
-        run: npm run build --workspace=@jazzer.js/fuzzer
       - name: check formatting and linting
         run: npm run check
   unit_tests:
@@ -60,10 +58,30 @@ jobs:
         run: npm run build --workspace=@jazzer.js/fuzzer
       - name: run all tests
         run: npm run test
+  fuzz_tests:
+    name: fuzz tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+          cache: "npm"
+      - name: install dependencies
+        run: npm install
+      - name: build project
+        run: npm run build
+      - name: build fuzzer
+        run: npm run build --workspace=@jazzer.js/fuzzer
+      - name: run all fuzz tests
+        run: node fuzztests/runFuzzTests.js
   auto-merge:
     needs:
       - linting
       - unit_tests
+      - fuzz_tests
     permissions:
       pull-requests: write
       contents: write

diff --git a/fuzztests/readme.md b/fuzztests/readme.md
@@ -0,0 +1,34 @@
+# Internal fuzz tests
+
+**Note**: These are only internal tests and not intended as examples.
+
+This directory contains fuzz tests created via the Jest integration that test
+internal Jazzer.js functionality.
+
+## Config
+
+General fuzz test configuration is done via `.jazzerjsrc`
+
+## Execution
+
+Use the provided script `runFuzzTests` to execute all tests in the `fuzztests`
+directory. Pass in `async` as first argument to start all tests in parallel. To
+change the working dir, pass in the target directory as second parameter.
+
+**Sync:**
+
+```shell
+./runFuzzTests.js
+```
+
+**Async:**
+
+```shell
+./runFuzzTests.js async
+```
+
+**Other dir:**
+
+```shell
+./runFuzzTests.js async otherDirectory
+```
diff --git a/fuzztests/runFuzzTests.js b/fuzztests/runFuzzTests.js
@@ -0,0 +1,83 @@
+#!/usr/bin/node
+
+// Helper script that searches for Jest fuzz tests in the current directory and
+// executes them in new processes using the found fuzz test names.
+
+/* eslint-disable @typescript-eslint/no-var-requires */
+
+const fs = require("fs/promises");
+const { spawn } = require("child_process");
+
+const fuzzTestFileExtension = "fuzz.js";
+const fuzzTestNameRegex = /it.fuzz\("(.*)"/g;
+
+async function findFuzzTestNamesInFile(file) {
+	const fuzzTestNames = [];
+	if (file.endsWith(fuzzTestFileExtension)) {
+		const content = await fs.readFile(file);
+		for (let match of content.toString().matchAll(fuzzTestNameRegex)) {
+			fuzzTestNames.push(match[1]);
+		}
+	}
+	return fuzzTestNames;
+}
+
+async function findFuzzTestNamesInDir(dir) {
+	const files = await fs.readdir(dir);
+	let fuzzTests = {};
+	for (const file of files) {
+		const fuzzTestNames = await findFuzzTestNamesInFile(file);
+		if (fuzzTestNames.length) {
+			fuzzTests[file] = fuzzTestNames;
+		}
+	}
+	return fuzzTests;
+}
+
+async function executeFuzzTest(file, name) {
+	console.log(`--- Executing fuzz test ${file} > ${name}`);
+	return new Promise((resolve, reject) => {
+		process.env["JAZZER_FUZZ"] = "1";
+		let test = spawn("npm", ["test", "--", `--testNamePattern='${name}'`], {
+			stdio: "inherit",
+		});
+		test.on("error", (error) => {
+			console.log(`ERROR: ${error.message}`);
+			reject(error);
+		});
+		test.on("close", (code) => {
+			console.log(`--- Finished fuzz test ${file} > ${name} with code ${code}`);
+			resolve();
+		});
+	});
+}
+
+async function runFuzzTestsInDir(dir, mode) {
+	const fuzzTests = await findFuzzTestNamesInDir(dir);
+	if (mode === "async") {
+		return Promise.all(
+			Object.entries(fuzzTests).map(([file, names]) =>
+				Promise.all(names.map((name) => executeFuzzTest(file, name)))
+			)
+		);
+	} else {
+		for (const [file, names] of Object.entries(fuzzTests)) {
+			for (let name of names) {
+				await executeFuzzTest(file, name);
+			}
+		}
+	}
+}
+
+// Change into script / requested dir to simplify file handling
+process.chdir(process.argv[3] || __dirname);
+
+runFuzzTestsInDir(".", process.argv[2]).then(
+	() => {
+		console.log("DONE");
+	},
+	(e) => {
+		console.log("ERROR:", e);
+		process.exit(79);
+	}
+);