Add release canary test (#525)

This adds a test to the PR CI workflow to package up jazzer.js and run it against a project that imports in similar form to what it would get if it were importing the released version rather than using local file references. This will catch the issue we had previously where an import worked locally but not in the released version.
CodeIntelligenceTesting · Aug 7, 2023 · 656fb4d · 656fb4d
1 parent f00e77d
commit 656fb4d
Show file tree

Hide file tree

Showing 11 changed files with 270 additions and 0 deletions.
diff --git a/.github/workflows/run-all-tests.yaml b/.github/workflows/run-all-tests.yaml
@@ -75,11 +75,29 @@ jobs:
         run: npm run build --workspace=@jazzer.js/fuzzer
       - name: run all fuzz tests
         run: node fuzztests/runFuzzTests.js
+  end-to-end:
+    name: end-to-end
+    runs-on: ubuntu-20.04
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+          cache: "npm"
+      - name: pack jazzer.js
+        run: cd end-to-end && ./package-jazzer-js.sh
+      - name: build example
+        run: cd end-to-end && npm install --save-dev *.tgz && npm run build
+      - name: run tests
+        run: cd end-to-end && npx jest
   auto-merge:
     needs:
       - linting
       - unit_tests
       - fuzz_tests
+      - end-to-end
     permissions:
       pull-requests: write
       contents: write

diff --git a/end-to-end/.gitignore b/end-to-end/.gitignore
@@ -0,0 +1,2 @@
+**/package-lock.json
+**/.cifuzz-corpus
diff --git a/end-to-end/README.md b/end-to-end/README.md
@@ -0,0 +1,22 @@
+# Jazzer End to End Canary Test
+
+This is the code from `examples/jest_typescript_integration` with a single
+change to `package.json`: the Jazzer.js dependencies now come from
+`jazzer-js-<package name>.tgz` files in this directory. These can be created by
+running `./package-jazzer-js.sh` which will call `npm pack` on the Jazzer
+packages so that we can test for any packaging errors.
+
+The Typescript integration example was chosen as that should exercise more of
+jazzer.js than the other examples.
+
+## Running Locally
+
+```bash
+./package-jazzer-js.sh
+npm install --save-dev *.tgz
+npx jest
+```
+
+_Note_: running just `npm install` may result in caching issues where the
+contents of the tarballs in this directory are ignored and older versions from
+somewhere are used instead.
diff --git a/end-to-end/integration.fuzz.ts b/end-to-end/integration.fuzz.ts
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2023 Code Intelligence GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import "@jazzer.js/jest-runner";
+import * as target from "./target";
+
+describe("Target", () => {
+	it.fuzz("executes sync methods", (data: Buffer) => {
+		target.fuzzMe(data);
+	});
+
+	it.fuzz("executes async methods", async (data: Buffer) => {
+		await target.asyncFuzzMe(data);
+	});
+
+	it.fuzz(
+		"executes methods with a done callback",
+		(data: Buffer, done: (e?: Error) => void) => {
+			target.callbackFuzzMe(data, done);
+		},
+	);
+});
diff --git a/end-to-end/integration.test.ts b/end-to-end/integration.test.ts
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2023 Code Intelligence GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as target from "./target";
+
+describe("My describe", () => {
+	it("My normal Jest test", () => {
+		expect(1).toEqual(1);
+	});
+
+	it("My done callback Jest test", (done) => {
+		expect(1).toEqual(1);
+		done();
+	});
+
+	it("My async Jest test", async () => {
+		expect(1).toEqual(1);
+	});
+
+	it("Test target function", () => {
+		const data = Buffer.from("a");
+		target.fuzzMe(data);
+	});
+});
diff --git a/end-to-end/jest.config.ts b/end-to-end/jest.config.ts
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2023 Code Intelligence GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import type { Config } from "jest";
+
+const config: Config = {
+	verbose: true,
+	projects: [
+		{
+			displayName: "Jest",
+			preset: "ts-jest",
+		},
+		{
+			displayName: {
+				name: "Jazzer.js",
+				color: "cyan",
+			},
+			preset: "ts-jest",
+			runner: "@jazzer.js/jest-runner",
+			testEnvironment: "node",
+			testMatch: ["<rootDir>/*.fuzz.[jt]s"],
+		},
+	],
+	coveragePathIgnorePatterns: ["/node_modules/", "/dist/"],
+	modulePathIgnorePatterns: ["/node_modules", "/dist/"],
+};
+
+export default config;
diff --git a/end-to-end/package-jazzer-js.sh b/end-to-end/package-jazzer-js.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+cd ..
+npm install
+npm run build
+npm run build --workspace='@jazzer.js/fuzzer'
+
+sed_version_and_mv() {
+    while read data; do
+        local no_version=$(echo $data | sed -r -f end-to-end/remove-version.sed)
+        echo "mv $data end-to-end/$no_version"
+        mv $data end-to-end/$no_version
+    done
+}
+
+npm pack --workspaces | sed_version_and_mv 
diff --git a/end-to-end/package.json b/end-to-end/package.json
@@ -0,0 +1,24 @@
+{
+	"name": "jest_typescript_integration",
+	"version": "1.0.0",
+	"description": "An example showing how Jazzer.js integrates with Jest and TypeScript",
+	"scripts": {
+		"build": "tsc",
+		"dryRun": "jest",
+		"fuzz": "JAZZER_FUZZ=1 jest --coverage",
+		"coverage": "jest --coverage"
+	},
+	"devDependencies": {
+		"@jazzer.js/bug-detectors": "file:jazzer.js-bug-detectors.tgz",
+		"@jazzer.js/core": "file:jazzer.js-core.tgz",
+		"@jazzer.js/fuzzer": "file:jazzer.js-fuzzer.tgz",
+		"@jazzer.js/hooking": "file:jazzer.js-hooking.tgz",
+		"@jazzer.js/instrumentor": "file:jazzer.js-instrumentor.tgz",
+		"@jazzer.js/jest-runner": "file:jazzer.js-jest-runner.tgz",
+		"@types/jest": "^29.4.0",
+		"jest": "^29.4.1",
+		"ts-jest": "^29.0.5",
+		"ts-node": "^10.9.1",
+		"typescript": "^4.9.5"
+	}
+}
diff --git a/end-to-end/remove-version.sed b/end-to-end/remove-version.sed
@@ -0,0 +1 @@
+s/-[0-9]+\.[0-9]+\.[0-9]+\.tgz/\.tgz/g
diff --git a/end-to-end/target.ts b/end-to-end/target.ts
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2023 Code Intelligence GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export function fuzzMe(data: Buffer) {
+	const s = data.toString();
+	if (s.length !== 16) {
+		return;
+	}
+	if (
+		s.slice(0, 8) === "Awesome " &&
+		s.slice(8, 15) === "Fuzzing" &&
+		s[15] === "!"
+	) {
+		throw Error("Welcome to Awesome Fuzzing!");
+	}
+}
+
+export function callbackFuzzMe(data: Buffer, done: (e?: Error) => void) {
+	// Use setImmediate here to unblock the event loop but still have better
+	// performance compared to setTimeout.
+	setImmediate(() => {
+		try {
+			fuzzMe(data);
+			done();
+		} catch (e: unknown) {
+			if (e instanceof Error) {
+				done(e);
+			} else {
+				done(new Error(`Error: ${e}`));
+			}
+		}
+	});
+}
+
+export async function asyncFuzzMe(data: Buffer) {
+	return new Promise((resolve, reject) => {
+		callbackFuzzMe(data, (e?: Error) => {
+			if (e) {
+				reject(e);
+			} else {
+				resolve(null);
+			}
+		});
+	});
+}
diff --git a/end-to-end/tsconfig.json b/end-to-end/tsconfig.json
@@ -0,0 +1,17 @@
+{
+	"compilerOptions": {
+		"target": "ES2022",
+		"module": "NodeNext",
+		"moduleResolution": "node",
+		"allowJs": true,
+		"rootDir": ".",
+		"outDir": "./dist",
+		"esModuleInterop": true,
+		"forceConsistentCasingInFileNames": true,
+		"strict": true,
+		"skipLibCheck": true,
+		"declaration": true,
+		"composite": true,
+		"sourceMap": true
+	}
+}