[internal] Cache build results (#4693)

.circleci/config.yml

@@ -22,6 +22,18 @@ aliases:
         - node_modules
       key: v1-yarn-deps-{{ checksum "yarn.lock" }}
 
+  - &restore_build_cache
+    restore_cache:
+      keys:
+        - v1-build-cache-{{ .Branch }}
+        - v1-build-cache-master
+
+  - &save_build_cache
+    save_cache:
+      paths:
+        - .cache
+      key: v1-build-cache-{{ .Branch }}
+
   # Default
   - &defaults
     working_directory: ~/prettier
@@ -54,7 +66,9 @@ jobs:
     steps:
       - attach_workspace:
           at: ~/prettier
+      - *restore_build_cache
       - run: yarn build
+      - *save_build_cache
       - persist_to_workspace:
           root: .
           paths:

.gitignore

@@ -1,3 +1,4 @@
+/.cache
 /node_modules
 /scripts/release/node_modules
 *.log

package.json

@@ -73,6 +73,7 @@
     "eslint-plugin-import": "2.9.0",
     "eslint-plugin-prettier": "2.6.0",
     "eslint-plugin-react": "7.7.0",
+    "execa": "0.10.0",
     "jest": "21.1.0",
     "mkdirp": "0.5.1",
     "prettier": "1.13.5",

scripts/build/build.js

@@ -1,10 +1,15 @@
 "use strict";
 
 const chalk = require("chalk");
+const execa = require("execa");
+const minimist = require("minimist");
+const path = require("path");
 const stringWidth = require("string-width");
+
 const bundler = require("./bundler");
 const bundleConfigs = require("./config");
 const util = require("./util");
+const Cache = require("./cache");
 
 // Errors in promises should be fatal.
 const loggedErrors = new Set();
@@ -16,8 +21,9 @@ process.on("unhandledRejection", err => {
   process.exit(1);
 });
 
-const OK = chalk.reset.inverse.bold.green(" DONE ");
-const FAIL = chalk.reset.inverse.bold.red(" FAIL ");
+const CACHED = chalk.bgYellow.black(" CACHED ");
+const OK = chalk.bgGreen.black("  DONE  ");
+const FAIL = chalk.bgRed.black("  FAIL  ");
 
 function fitTerminal(input) {
   const columns = Math.min(process.stdout.columns || 40, 80);
@@ -28,18 +34,22 @@ function fitTerminal(input) {
   return input;
 }
 
-async function createBundle(bundleConfig) {
+async function createBundle(bundleConfig, cache) {
   const { output } = bundleConfig;
   process.stdout.write(fitTerminal(output));
 
-  try {
-    await bundler(bundleConfig, output);
-  } catch (error) {
-    process.stdout.write(`${FAIL}\n\n`);
-    handleError(error);
-  }
-
-  process.stdout.write(`${OK}\n`);
+  return bundler(bundleConfig, cache)
+    .catch(error => {
+      console.log(FAIL + "\n");
+      handleError(error);
+    })
+    .then(result => {
+      if (result.cached) {
+        console.log(CACHED);
+      } else {
+        console.log(OK);
+      }
+    });
 }
 
 function handleError(error) {
@@ -48,6 +58,22 @@ function handleError(error) {
   throw error;
 }
 
+async function cacheFiles() {
+  // Copy built files to .cache
+  try {
+    await execa("rm", ["-rf", path.join(".cache", "files")]);
+    await execa("mkdir", ["-p", path.join(".cache", "files")]);
+    for (const bundleConfig of bundleConfigs) {
+      await execa("cp", [
+        path.join("dist", bundleConfig.output),
+        path.join(".cache", "files")
+      ]);
+    }
+  } catch (err) {
+    // Don't fail the build
+  }
+}
+
 async function preparePackage() {
   const pkg = await util.readJson("package.json");
   pkg.bin = "./bin-prettier.js";
@@ -64,15 +90,30 @@ async function preparePackage() {
   await util.copyFile("./README.md", "./dist/README.md");
 }
 
-async function run() {
-  await util.asyncRimRaf("dist");
+async function run(params) {
+  await execa("rm", ["-rf", "dist"]);
+  await execa("mkdir", ["-p", "dist"]);
+
+  if (params["purge-cache"]) {
+    await execa("rm", ["-rf", ".cache"]);
+  }
+
+  const bundleCache = new Cache(".cache/", "v1");
+  await bundleCache.load();
 
   console.log(chalk.inverse(" Building packages "));
   for (const bundleConfig of bundleConfigs) {
-    await createBundle(bundleConfig);
+    await createBundle(bundleConfig, bundleCache);
   }
 
+  await bundleCache.save();
+  await cacheFiles();
+
   await preparePackage();
 }
 
-run();
+run(
+  minimist(process.argv.slice(2), {
+    boolean: ["purge-cache"]
+  })
+);

scripts/build/bundler.js

@@ -1,5 +1,6 @@
 "use strict";
 
+const execa = require("execa");
 const path = require("path");
 const { rollup } = require("rollup");
 const webpack = require("webpack");
@@ -186,11 +187,29 @@ function runWebpack(config) {
   });
 }
 
-module.exports = async function createBundle(bundle) {
+module.exports = async function createBundle(bundle, cache) {
+  const useCache = await cache.checkBundle(
+    bundle.output,
+    getRollupConfig(bundle)
+  );
+  if (useCache) {
+    try {
+      await execa("cp", [
+        path.join(cache.cacheDir, "files", bundle.output),
+        "dist"
+      ]);
+      return { cached: true };
+    } catch (err) {
+      // Proceed to build
+    }
+  }
+
   if (bundle.bundler === "webpack") {
     await runWebpack(getWebpackConfig(bundle));
   } else {
     const result = await rollup(getRollupConfig(bundle));
     await result.write(getRollupOutputOptions(bundle));
   }
+
+  return { bundled: true };
 };

scripts/build/cache.js

@@ -0,0 +1,129 @@
+"use strict";
+
+const util = require("util");
+const assert = require("assert");
+const crypto = require("crypto");
+const fs = require("fs");
+const path = require("path");
+const { rollup } = require("rollup");
+
+const readFile = util.promisify(fs.readFile);
+const writeFile = util.promisify(fs.writeFile);
+
+const ROOT = path.join(__dirname, "..", "..");
+
+function Cache(cacheDir, version) {
+  this.cacheDir = path.resolve(cacheDir || required("cacheDir"));
+  this.manifest = path.join(this.cacheDir, "manifest.json");
+  this.version = version || required("version");
+  this.checksums = {};
+  this.files = {};
+  this.updated = {
+    version: this.version,
+    checksums: {},
+    files: {}
+  };
+}
+
+// Loads the manifest.json file with the information from the last build
+Cache.prototype.load = async function() {
+  // This should never throw, if it does, let it fail the build
+  const lockfile = await readFile("yarn.lock", "utf-8");
+  const lockfileHash = hashString(lockfile);
+  this.updated.checksums["yarn.lock"] = lockfileHash;
+
+  try {
+    const manifest = await readFile(this.manifest, "utf-8");
+    const { version, checksums, files } = JSON.parse(manifest);
+
+    // Ignore the cache if the version changed
+    assert.equal(this.version, version);
+
+    assert.ok(typeof checksums === "object");
+    // If yarn.lock changed, rebuild everything
+    assert.equal(lockfileHash, checksums["yarn.lock"]);
+    this.checksums = checksums;
+
+    assert.ok(typeof files === "object");
+    this.files = files;
+
+    for (const files of Object.values(this.files)) {
+      assert.ok(Array.isArray(files));
+    }
+  } catch (err) {
+    this.checksums = {};
+    this.files = {};
+  }
+};
+
+// Run rollup to get the list of files included in the bundle and check if
+// any (or the list itself) have changed.
+// This takes the same rollup config used for bundling to include files that are
+// resolved by specific plugins.
+Cache.prototype.checkBundle = async function(output, rollupConfig) {
+  const result = await rollup(getRollupConfig(rollupConfig));
+  const modules = result.modules
+    .filter(mod => !/\0/.test(mod.id))
+    .map(mod => [path.relative(ROOT, mod.id), mod.originalCode]);
+
+  const files = new Set(this.files[output]);
+  const newFiles = (this.updated.files[output] = []);
+
+  let dirty = false;
+
+  for (const [id, code] of modules) {
+    newFiles.push(id);
+    // If we already checked this file for another bundle, reuse the hash
+    if (!this.updated.checksums[id]) {
+      this.updated.checksums[id] = hashString(code);
+    }
+    const hash = this.updated.checksums[id];
+
+    // Check if this file changed
+    if (!this.checksums[id] || this.checksums[id] !== hash) {
+      dirty = true;
+    }
+
+    // Check if this file is new
+    if (!files.delete(id)) {
+      dirty = true;
+    }
+  }
+
+  // Final check: if any file was removed, `files` is not empty
+  return !dirty && files.size === 0;
+};
+
+Cache.prototype.save = async function() {
+  try {
+    await writeFile(this.manifest, JSON.stringify(this.updated, null, 2));
+  } catch (err) {
+    // Don't fail the build
+  }
+};
+
+function required(name) {
+  throw new Error(name + " is required");
+}
+
+function hashString(string) {
+  return crypto
+    .createHash("md5")
+    .update(string)
+    .digest("hex");
+}
+
+function getRollupConfig(rollupConfig) {
+  return Object.assign({}, rollupConfig, {
+    onwarn() {},
+    plugins: rollupConfig.plugins.filter(
+      plugin =>
+        // We're not interested in dependencies, we already check `yarn.lock`
+        plugin.name !== "node-resolve" &&
+        // This is really slow, we need this "preflight" to be fast
+        plugin.name !== "babel"
+    )
+  });
+}
+
+module.exports = Cache;

scripts/build/util.js

@@ -1,7 +1,6 @@
 "use strict";
 
 const fs = require("fs");
-const rimraf = require("rimraf");
 const promisify = require("util").promisify;
 
 const readFile = promisify(fs.readFile);
@@ -23,7 +22,6 @@ async function copyFile(from, to) {
 }
 
 module.exports = {
-  asyncRimRaf: promisify(rimraf),
   readJson,
   writeJson,
   copyFile,

scripts/release/steps/generate-bundles.js

@@ -4,7 +4,7 @@ const chalk = require("chalk");
 const { runYarn, logPromise, readJson } = require("../utils");
 
 module.exports = async function({ version }) {
-  await logPromise("Generating bundles", runYarn("build"));
+  await logPromise("Generating bundles", runYarn(["build", "--purge-cache"]));
 
   const builtPkg = await readJson("dist/package.json");
   if (builtPkg.version !== version) {

scripts/release/utils.js

@@ -34,7 +34,10 @@ function logPromise(name, promise) {
 }
 
 function runYarn(script) {
-  return execa("yarn", ["--silent", script]).catch(error => {
+  if (typeof script === "string") {
+    script = [script];
+  }
+  return execa("yarn", ["--silent"].concat(script)).catch(error => {
     throw Error(`\`yarn ${script}\` failed\n${error.stdout}`);
   });
 }

yarn.lock

@@ -1651,6 +1651,16 @@ cross-spawn@^5.0.1, cross-spawn@^5.1.0:
     shebang-command "^1.2.0"
     which "^1.2.9"
 
+cross-spawn@^6.0.0:
+  version "6.0.5"
+  resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"
+  dependencies:
+    nice-try "^1.0.4"
+    path-key "^2.0.1"
+    semver "^5.5.0"
+    shebang-command "^1.2.0"
+    which "^1.2.9"
+
 cryptiles@2.x.x:
   version "2.0.5"
   resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-2.0.5.tgz#3bdfecdc608147c1c67202fa291e7dca59eaa3b8"
@@ -2239,6 +2249,18 @@ exec-sh@^0.2.0:
   dependencies:
     merge "^1.1.3"
 
+execa@0.10.0:
+  version "0.10.0"
+  resolved "https://registry.yarnpkg.com/execa/-/execa-0.10.0.tgz#ff456a8f53f90f8eccc71a96d11bdfc7f082cb50"
+  dependencies:
+    cross-spawn "^6.0.0"
+    get-stream "^3.0.0"
+    is-stream "^1.1.0"
+    npm-run-path "^2.0.0"
+    p-finally "^1.0.0"
+    signal-exit "^3.0.0"
+    strip-eof "^1.0.0"
+
 execa@^0.7.0:
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/execa/-/execa-0.7.0.tgz#944becd34cc41ee32a63a9faf27ad5a65fc59777"
@@ -4065,6 +4087,10 @@ next-tick@1:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c"
 
+nice-try@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.4.tgz#d93962f6c52f2c1558c0fbda6d512819f1efe1c4"
+
 node-fetch@^1.0.1:
   version "1.7.3"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz#980f6f72d85211a5347c6b2bc18c5b84c3eb47ef"
@@ -4422,7 +4448,7 @@ path-is-inside@^1.0.1, path-is-inside@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/path-is-inside/-/path-is-inside-1.0.2.tgz#365417dede44430d1c11af61027facf074bdfc53"
 
-path-key@^2.0.0:
+path-key@^2.0.0, path-key@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40"
 
@@ -5146,7 +5172,7 @@ sax@^1.2.4:
   version "5.4.1"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e"
 
-semver@5.5.0, semver@^5.4.1:
+semver@5.5.0, semver@^5.4.1, semver@^5.5.0:
   version "5.5.0"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab"