Continue running tests on failure. Print failures.

.gitignore

@@ -109,3 +109,5 @@ scripts/verdaccio.sh
 
 # Don't ignore the specific Points.dist() docs folder
 !markdown/dev/reference/api/point/dist
+
+.test-failures.log

package.json

@@ -21,7 +21,7 @@
     "reconfigure": "all-contributors generate && node scripts/reconfigure.js",
     "prerelease": "lerna version --no-git-tag-version --no-push && yarn reconfigure && yarn buildall",
     "buildall": "lerna run cibuild_step1 && lerna run cibuild_step2",
-    "testall": "lerna run testci",
+    "testall": "node scripts/testall.js",
     "release": "lerna exec -- npm publish",
     "postrelease": "git add . && git commit -m ':bookmark: v$npm_package_version' && git tag -a v$npm_package_version -m ':bookmark: FreeSewing v$npm_package_version'",
     "ship": "lerna exec --no-bail -- npm publish",

scripts/test-failure-collector.js

@@ -0,0 +1,69 @@
+/*
+ * Used to collect test failures in a file. Use by specifying --file to Mocha.
+ *
+ * See https://mochajs.org/#command-line-usage
+ */
+
+const path = require('path');
+const projectRoot = path.normalize(path.join(__dirname, '..'));
+const outputLog = path.join(projectRoot, '.test-failures.log');
+
+const red = function(string) {
+  return `\x1b[31m${string}\x1b[0m`;
+};
+
+const green = function(string) {
+  return `\x1b[32m${string}\x1b[0m`;
+};
+
+const dim = function(string) {
+  return `\x1b[2m${string}\x1b[0m`;
+};
+
+// Mapping of test file name to array of failing tests.
+const failuresPerFile = {};
+
+afterEach(function () {
+  if (this.currentTest.state === "failed") {
+    failuresPerFile[this.currentTest.file] = failuresPerFile[this.currentTest.file] || [];
+    failuresPerFile[this.currentTest.file].push(this.currentTest);
+  }
+});
+
+after(function () {
+  if (Object.keys(failuresPerFile).length === 0) return;
+
+  const fs = require('fs')
+  const logger = fs.createWriteStream(outputLog, { flags: 'a' });
+  const writeLine = (line) => logger.write(`${line}\n`);
+
+  for (let file in failuresPerFile) {
+    const failures = failuresPerFile[file];
+
+    // Remove project root from file path to keep log lines shorter.
+    if (file.startsWith(projectRoot)) {
+      file = file.substr(projectRoot.length + 1, file.length - projectRoot.length - 1)
+    }
+
+    // Print each failure.
+    failures.forEach(function (failure, i) {
+      const stack = failure.err.stack.split('\n');
+      writeLine(`${file}:  ${i + 1}\) ${failure.title}:`);
+      writeLine(`${file}:`);
+      writeLine(`${file}:      ${red(stack[0].trim())}`);
+      writeLine(`${file}:      ${green('+ expected')} ${red('- actual')}`);
+      writeLine(`${file}:`);
+      writeLine(`${file}:      ${red("-" + failure.err.actual)}`);
+      writeLine(`${file}:      ${green("+" + failure.err.expected)}`);
+      writeLine(`${file}:`);
+      stack.slice(1).forEach(function (stackLine) {
+        writeLine(`${file}:      ${dim(stackLine.trim())}`);
+      });
+      if (i < failures.length - 1) {
+        writeLine(`${file}:`);
+      }
+    });
+  }
+
+  logger.end();
+});

scripts/testall.js

@@ -0,0 +1,25 @@
+const fs = require('fs')
+const path = require('path');
+const spawn = require('child_process').spawn
+
+const projectRoot = path.normalize(path.join(__dirname, '..'));
+const outputLog = path.join(projectRoot, '.test-failures.log');
+const collectorScript = path.join(projectRoot, 'scripts', 'test-failure-collector.js');
+
+// Start with a fresh output log on each run.
+if (fs.existsSync(outputLog)) {
+  fs.unlinkSync(outputLog);
+}
+
+// Run all tests, specifying the collector script.
+spawn('lerna', ['run', '--no-bail', 'testci', '--', '--file', `${collectorScript}`], { stdio: 'inherit' })
+  .on('exit', function(code) {
+    // If a failure occurred, the log file will have been created. Print it.
+    if (fs.existsSync(outputLog)) {
+      console.error(fs.readFileSync(outputLog, 'utf8').trim());
+    }
+
+    // Propagate the exit code.
+    process.exit(code);
+  });
+