test(wiptest): add a way to test wiptest (#699)

This adds a really basic test runner that will execute test files using
`bun wiptest` and compare the output to make sure it's correct. It could
definitely be improved, especially in the speed department, but it's at
least functional now, which is better than we had before!

Makefile

@@ -518,7 +518,7 @@ docker-push-base:
 
 	docker tag bun-base-with-args ghcr.io/jarred-sumner/bun-base-with-args:latest
 	docker push ghcr.io/jarred-sumner/bun-base-with-args:latest
-	
+
 ifeq ($(POSIX_PKG_MANAGER), brew)
 PKGNAME_NINJA := ninja
 else
@@ -862,6 +862,12 @@ release-bin-without-push: test-all release-bin-check generate-release-bin-as-zip
 release-bin: release-bin-without-push release-bin-push
 
 
+test/wiptest/run.o: test/wiptest/run.cpp
+	$(CXX) -Wall -g -c -std=c++17 -o test/wiptest/run.o test/wiptest/run.cpp
+
+test/wiptest/run: test/wiptest/run.o
+	$(CXX) -Wall -g -o test/wiptest/run test/wiptest/run.o
+
 
 release-bin-dir:
 	echo $(PACKAGE_DIR)
@@ -898,8 +904,10 @@ test-dev-bun-snapshot:
 	touch test/bun.js/snapshots.debug.js
 	$(foreach i,$(wildcard test/bun.js/*.snapshot.*),echo "" >> test/bun.js/snapshots.debug.js; echo "// $i" >> test/bun.js/snapshots.debug.js; $(DEBUG_BUN) build $i --platform=bun >> test/bun.js/snapshots.debug.js;)
 
+test-bun-wiptest: test/wiptest/run
+	cd test/wiptest && BUN_BIN=$(DEBUG_BUN) ./run ./fixtures
 
-test-all: test-install test-bun-snapshot test-with-hmr test-no-hmr test-create-next test-create-react test-bun-run test-bun-install test-bun-dev
+test-all: test-install test-bun-snapshot test-with-hmr test-no-hmr test-create-next test-create-react test-bun-run test-bun-install test-bun-dev test-bun-wiptest
 
 copy-test-node-modules:
 	rm -rf test/snippets/package-json-exports/node_modules || echo "";

test/wiptest/.gitignore

@@ -0,0 +1,2 @@
+run.o
+run

test/wiptest/README.md

@@ -0,0 +1,77 @@
+# wiptest tests
+
+## To run these tests
+
+Go back up to the main directory of this repo and run
+
+```bash
+make test-bun-wiptest
+```
+
+## Developing these tests
+
+These tests are special. They test the test runner itself. Since we are testing the test runner, we unfortunately can't rely _on_ the test runner for these tests, since a bug in the runner could result in false passes. Instead this relies on a very small and very specialized test runner which processes the files in the `fixtures/` directory. The tests this runner actually performs are meta tests: it runs the `wiptest` runner on the file, and compares the output to what it expects to see. These expectations are given through special comment macros. All macros can be placed anywhere within the file, but MUST be preceded by `// ` (e.g. `// STATUS: PASS` is valid, but `//STATUS: PASS`, `/* STATUS: PASS */` and `# STATUS: PASS` are not).
+
+| name          | description                                                                                                                                                                              |
+| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `STATUS`      | The expected exit status of the `wiptest` command. Must be `PASS` or `FAIL`. May only be declared once per file.                                                                         |
+| `EXPECT`      | Some text that must be in the output of the `wiptest` command.                                                                                                                           |
+| `EXPECTNOT`   | Same as `EXPECT`, but the text must NOT appear in the output.                                                                                                                            |
+| `TESTPATTERN` | Override the test pattern that will be passed to `wiptest`. It can be used for testing different types of file resolution, such as partial matching. May only be declared once per file. |
+
+Each test file must contain a `STATUS` macro and at least one `EXPECT` and/or `EXPECTNOT`.
+
+### Examples
+
+#### Example 1
+
+```js
+// STATUS: FAIL
+import { expect, it } from "bun:test";
+
+it("should fail", () => {
+  // EXPECT: Expected: 2
+  // EXPECT: Received: 1
+  // EXPECT: toBe.mismatch.test.js:8:2
+  expect(1).toBe(2);
+});
+
+// EXPECT: 0 pass
+// EXPECT: 1 fail
+```
+
+This will expect the test to fail (non-zero exit code), and the output must contain "Expected: 2", "Received: 1", "toBe.test.js:8:2", "0 pass", and "1 fail".
+
+#### Example 2
+
+```js
+// STATUS: PASS
+import { expect, it } from "bun:test";
+
+it("should pass", () => {
+  // EXPECTNOT: toBe.match.test.js:6:2
+  expect(1).toBe(1);
+});
+
+// EXPECT: 1 pass
+// EXPECT: 0 fail
+```
+
+This will expect the test to pass (exit code `0`), and the output must contain "1 pass", and "0 fail", and must NOT contain "toBe.test.js:6:2"
+
+#### Example 3
+
+```js
+// EXPECT: 0 fail
+import { expect, it } from "bun:test";
+
+// STATUS: PASS
+it("should pass", () => {
+  expect(1).toBe(1);
+});
+
+// EXPECT: 1 pass
+// EXPECTNOT: toBe.match.test.js:6:2
+```
+
+This results in the exact same test as Example 2. As you can see, the macros can be defined in any order. It's advised to place them next to the section of the code that generates the error, but they can be placed wherever makes the most sense for the test at hand.

test/wiptest/fixtures/toBe.match.test.js

@@ -0,0 +1,10 @@
+// STATUS: PASS
+import { expect, it } from "bun:test";
+
+it("should pass", () => {
+  // EXPECTNOT: toBe.match.test.js:6:2
+  expect(1).toBe(1);
+});
+
+// EXPECT: 1 pass
+// EXPECT: 0 fail

test/wiptest/fixtures/toBe.mismatch.test.js

@@ -0,0 +1,12 @@
+// STATUS: FAIL
+import { expect, it } from "bun:test";
+
+it("should fail", () => {
+  // EXPECT: Expected: 2
+  // EXPECT: Received: 1
+  // EXPECT: toBe.mismatch.test.js:8:2
+  expect(1).toBe(2);
+});
+
+// EXPECT: 0 pass
+// EXPECT: 1 fail

test/wiptest/fixtures/toHaveLength.match.test.js

@@ -0,0 +1,20 @@
+// STATUS: PASS
+import { expect, it } from "bun:test";
+
+it("should work with arrays", () => {
+  // EXPECTNOT: toHaveLength.match.test.js:6:2
+  expect(["a", "b", "c"]).toHaveLength(3);
+});
+
+it("should work with strings", () => {
+  // EXPECTNOT: toHaveLength.match.test.js:11:2
+  expect("abcd").toHaveLength(4);
+});
+
+it("should work with arbitrary objects", () => {
+  // EXPECTNOT: toHaveLength.match.test.js:16:2
+  expect({ length: 42 }).toHaveLength(42);
+});
+
+// EXPECT: 3 pass
+// EXPECT: 0 fail

test/wiptest/fixtures/toHaveLength.mismatch.test.js

@@ -0,0 +1,20 @@
+// STATUS: FAIL
+import { expect, it } from "bun:test";
+
+it("should work with arrays", () => {
+  // EXPECT: toHaveLength.mismatch.test.js:6:2
+  expect(["a", "b", "c"]).toHaveLength(2);
+});
+
+it("should work with strings", () => {
+  // EXPECT: toHaveLength.mismatch.test.js:11:2
+  expect("abcd").toHaveLength(5);
+});
+
+it("should work with arbitrary objects", () => {
+  // EXPECT: toHaveLength.mismatch.test.js:16:2
+  expect({ length: 42 }).toHaveLength(24);
+});
+
+// EXPECT: 0 pass
+// EXPECT: 3 fail

test/wiptest/run.cpp

@@ -0,0 +1,300 @@
+#include <dirent.h>
+#include <filesystem>
+#include <iostream>
+#include <fstream>
+#include <unistd.h>
+#include <vector>
+
+int passed = 0;
+int failed = 0;
+
+/**
+ * @brief Run an executable and capture the output.
+ *
+ * @param path The path to the executable.
+ * @param argv The null-terminated arguments to pass to the executable.
+ * @param output [out] The string into which to place the output. This string will NOT be cleared first.
+ * @param exitCode [out] The return code of the executable.
+ * @return -1 if something went wrong, otherwise 0.
+ */
+int exec(char const* const path, char const* const argv[], std::string* const output, int* const exitCode) {
+  int stdpipe[2];
+  if (pipe(stdpipe) == -1) {
+    std::cerr << "ERROR: Unable to capture pipe" << std::endl;
+    return -1;
+  }
+
+  pid_t pid = fork();
+  if (pid == -1) {
+    std::cerr << "ERROR: Unable to capture pipe for stderr"  << std::endl;
+    return -1;
+  } else if (pid == 0) {
+    close(stdpipe[0]); // Child doesn't need to read the pipe.
+    // Capture stdout and stderr
+    dup2(stdpipe[1], 1);
+    dup2(stdpipe[1], 2);
+    execv(const_cast<char const*>(path), const_cast<char* const*>(argv));
+    return -1; // This is just to make the compiler happy.
+  } else {
+    close(stdpipe[1]); // Parent doesn't need to write.
+
+    if (output != nullptr) {
+      char buf[8];
+      int count;
+      while((count = read(stdpipe[0], buf, 8)) > 0) {
+        output->append(buf, count);
+      }
+    }
+    close(stdpipe[0]); // Done
+
+    // parent
+    int status;
+    if (waitpid(pid, &status, 0) == -1) {
+      std::cerr << "ERROR: waitpid failed somehow" << std::endl;
+      return -1;
+    }
+
+    if (exitCode != nullptr)
+      *exitCode = WEXITSTATUS(status);
+
+    return 0;
+  }
+}
+
+int execTest(char const* const bunBin, char const* const testMatch, std::string* const output, int* const exitCode) {
+  char const* args[] = {bunBin, "wiptest", testMatch, NULL};
+  return exec(bunBin, args, output, exitCode);
+}
+
+void parseMacros(std::string const filePath, int* const expectPass, std::vector<std::string const>& expects, std::vector<std::string const>& expectNots, std::string& testPattern, std::vector<std::string const>& errors) {
+  std::ifstream fstream(filePath);
+
+  if (!fstream.is_open()) {
+    errors.push_back("Unable to open file");
+    return;
+  }
+
+  std::string line;
+  size_t idx;
+  char const* rest;
+  while (fstream) {
+    std::getline(fstream, line);
+    idx = line.find("// ");
+    if (idx == std::string::npos)
+      continue;
+
+    idx += 3;
+
+    if (idx >= line.length())
+      continue;
+
+    switch (line.at(idx)) {
+    // STATUS
+    case 'S':
+      rest = line.c_str() + (idx);
+      if (strncmp(rest, "STATUS: ", 8) != 0)
+        continue;
+      if (idx + 8 >= line.length())
+        continue;
+      rest += 8;
+      if (strcmp(rest, "PASS") == 0)
+        *expectPass = 1;
+      else if (strcmp(rest, "FAIL") == 0)
+        *expectPass = 0;
+      else {
+        std::string err = "Invalid STATUS: '";
+        err += rest;
+        err += "', must be PASS or FAIL";
+        errors.push_back(err);
+      }
+      break;
+
+    // EXPECT
+    // EXPECTNOT
+    case 'E': {
+      rest = line.c_str() + (idx);
+      if (strncmp(rest, "EXPECT", 6) != 0)
+        continue;
+      idx += 6;
+      if (idx >= line.length())
+        continue;
+      rest += 6;
+      bool isNot = strncmp(rest, "NOT", 3) == 0;
+      if (isNot) {
+        idx += 3;
+        if (idx >= line.length())
+          continue;
+        rest += 3;
+      }
+
+      if (strncmp(rest, ": ", 2) != 0)
+        continue;
+
+      idx += 2;
+      if (idx >= line.length()) {
+        std::string err = isNot ? "EXPECTNOT" : "EXPECT";
+        err += " must not be empty";
+        errors.push_back(err);
+        continue;
+      }
+      rest += 2;
+
+      if (isNot)
+        expectNots.push_back(rest);
+      else
+        expects.push_back(rest);
+      break;
+    }
+
+    // TESTPATTERN
+    case 'T':
+      rest = line.c_str() + (idx);
+      if (strncmp(rest, "TESTPATTERN: ", 13) != 0)
+        continue;
+      if (idx + 13 >= line.length()) {
+        errors.push_back("TESTPATTERN must not be empty");
+        continue;
+      }
+      rest += 13;
+      testPattern.assign(rest);
+      break;
+
+    default:
+      break;
+    }
+  }
+}
+
+/**
+ * @brief Execute a single test.
+ *
+ * @param bunBin The path to the Bun binary to execute.
+ * @param baseDir The directory in which the test file is stored.
+ * @param testFile The name of the test file.
+ * @return The output of the test command (stdout + stderr).
+ */
+void runTest(char const* const bunBin, char const* const baseDir, char const* const testFile) {
+  printf("Running test '%s'...", testFile);
+  std::string filePath;
+  filePath += baseDir;
+  filePath += "/";
+  filePath += testFile;
+
+  std::vector<std::string const> errors;
+
+  int expectPass = 2;
+  std::vector<std::string const> expects;
+  std::vector<std::string const> expectNots;
+  parseMacros(filePath, &expectPass, expects, expectNots, filePath, errors);
+
+  if (expectPass == 2)
+    errors.push_back("Missing STATUS macro");
+
+  if (expects.size() == 0 && expectNots.size() == 0)
+    errors.push_back("File must contain at least one EXPECT or EXPECTNOT macro");
+
+  // Only run the test suite if we haven't yet failed.
+  if (errors.size() == 0) {
+    int exitCode;
+    std::string output;
+    if (execTest(bunBin, filePath.c_str(), &output, &exitCode) < 0) {
+      errors.push_back("Unable to parse test file");
+    }
+
+    auto didPass = exitCode == 0;
+    if (expectPass != didPass) {
+      if (expectPass) {
+        std::string err = "Expected exit code to be 0, got ";
+        err += exitCode;
+        errors.push_back(err);
+      } else
+        errors.push_back("Expected non-zero exit code");
+    }
+
+
+    for (auto ex : expects) {
+      if (output.find(ex) == std::string::npos) {
+        std::string msg = "Output does not contain '";
+        msg += ex;
+        msg += "'";
+        errors.push_back(msg);
+      }
+    }
+    for (auto ex : expectNots){
+      if (output.find(ex) != std::string::npos) {
+        std::string msg = "Output contains '";
+        msg += ex;
+        msg += "'";
+        errors.push_back(msg);
+      }
+    }
+  }
+
+  if (errors.size() > 0) {
+    ++failed;
+    printf(" Fail\n");
+    for (auto err : errors)
+      printf("  ERROR: %s\n", err.c_str());
+    printf("\n");
+  } else {
+    ++passed;
+    printf(" Pass\n");
+  }
+}
+
+/**
+ * @brief Run all tests in a directory.
+ *
+ * @param dir The directory to run tests in.
+ * @param bunBin The path to the Bun binary to execute.
+ */
+int runAll(char const* const dir, char const* const bunBin) {
+  struct dirent *entry = nullptr;
+
+  auto* dp = opendir(dir);
+  if (dp == nullptr) {
+    std::cerr << "ERROR: Unable to open directory '" << dir << "'" << std::endl;
+    return -1;
+  }
+
+  while ((entry = readdir(dp))) {
+    if (!strcmp(entry->d_name, ".") || !strcmp(entry->d_name, ".."))
+      continue;
+    runTest(bunBin, dir, entry->d_name);
+  }
+
+  if (closedir(dp) < 0) {
+    std::cerr << "ERROR: Unable to close directory '" << dir << "'" << std::endl;
+    return -1;
+  }
+
+  return 0;
+}
+
+int main(int argc, char const* argv[]) {
+  if (argc != 2) {
+    std::cerr << "Must provide path to test files" << std::endl;
+    return 1;
+  }
+
+  auto bunBin = std::getenv("BUN_BIN");
+  if (bunBin == nullptr) {
+    std::cerr << "ERROR: `$BUN_BIN` is not defined. Either set it manually or run this file via `make`'" << std::endl;
+    return 1;
+  }
+
+  if (!std::filesystem::exists(bunBin)) {
+    std::cerr << "ERROR: " << bunBin << " does not exist. Did you forget to run `make dev`?" << std::endl;
+    return 1;
+  }
+
+  char testDir[PATH_MAX];
+  realpath(argv[1], testDir);
+
+  if (runAll(testDir, bunBin) < 0)
+    return -1;
+
+  printf("\n\n\nFinished running tests.\nTotal: %d\nPassed: %d\nFailed: %d\n", passed + failed, passed, failed);
+
+  return failed;
+}