Use execa to orchestrate running npm scripts, and add a lage-debug command

change/change-3938e9f7-033d-42a7-b3cb-d40316a9382f.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "type": "patch",
+      "comment": "Remove unused glob-hasher dep",
+      "packageName": "@lage-run/cache",
+      "email": "elcraig@microsoft.com",
+      "dependentChangeType": "patch"
+    }
+  ]
+}

change/change-8ab7af26-370a-4496-87d7-b41eb0117e71.json

@@ -0,0 +1,25 @@
+{
+  "changes": [
+    {
+      "type": "patch",
+      "comment": "Add shebang to CLI entry point",
+      "packageName": "@lage-run/cli",
+      "email": "elcraig@microsoft.com",
+      "dependentChangeType": "patch"
+    },
+    {
+      "type": "minor",
+      "comment": "Use execa to run scripts, and add a lage-debug command which runs with a non-minified build",
+      "packageName": "lage",
+      "email": "elcraig@microsoft.com",
+      "dependentChangeType": "patch"
+    },
+    {
+      "type": "minor",
+      "comment": "Use execa to run scripts",
+      "packageName": "@lage-run/scheduler",
+      "email": "elcraig@microsoft.com",
+      "dependentChangeType": "patch"
+    }
+  ]
+}

packages/cache/package.json

@@ -21,8 +21,7 @@
     "@lage-run/logger": "^1.3.0",
     "backfill-cache": "5.7.1",
     "backfill-config": "6.4.1",
-    "backfill-logger": "5.2.1",
-    "glob-hasher": "^1.3.0"
+    "backfill-logger": "5.2.1"
   },
   "devDependencies": {
     "@lage-run/monorepo-fixture": "*",

packages/cli/src/cli.ts

@@ -1,3 +1,4 @@
+#!/usr/bin/env node
 import { Command } from "commander";
 
 import { runCommand } from "./commands/run/index.js";

packages/lage/package.json

@@ -9,11 +9,11 @@
   "main": "dist/main.js",
   "types": "dist/index.d.ts",
   "bin": {
-    "lage": "dist/lage.js"
+    "lage": "dist/lage.js",
+    "lage-debug": "dist/debug/lage.js"
   },
   "scripts": {
-    "prebundle": "node scripts/prebuild.js",
-    "bundle": "yarn dts-bundle && node scripts/bundle.mjs",
+    "bundle": "rimraf dist && yarn dts-bundle && node scripts/bundle.mjs",
     "dts-bundle": "dts-bundle-generator --config ./dts-bundle.config.js && node ./scripts/update-dts-bundle.js"
   },
   "dependencies": {
@@ -28,14 +28,11 @@
     "backfill-config": "6.4.1",
     "dts-bundle-generator": "^7.2.0",
     "workspace-tools": "0.36.4",
-    "esbuild": "^0.17.18"
+    "esbuild": "^0.17.18",
+    "rimraf": "^5.0.5"
   },
   "files": [
-    "dist/*.d.ts",
-    "dist/lage.js",
-    "dist/lage.js.map",
-    "dist/runners/**",
-    "dist/workers/**"
+    "dist"
   ],
   "publishConfig": {
     "access": "public"

packages/lage/scripts/bundle.mjs

@@ -1,28 +1,82 @@
+// @ts-check
 import * as esbuild from "esbuild";
+import fs from "fs";
+import { createRequire } from "module";
+import path from "path";
+import { findPackageRoot } from "workspace-tools";
 
-async function bundle(entry, outfile, addBanner = false) {
-  await esbuild.build({
-    entryPoints: [entry],
+const localRequire = createRequire(import.meta.url);
+
+async function bundle() {
+  console.log("\nBundling lage with esbuild...");
+
+  const packageRoot = findPackageRoot(process.cwd()) || process.cwd();
+  const packageJson = JSON.parse(fs.readFileSync(path.join(packageRoot, "package.json"), "utf-8"));
+
+  // Mapping from output path (relative to dist, no extension) to input path (relative to package root)
+  const entryPoints = {
+    lage: "@lage-run/cli/lib/cli.js",
+    main: "./index.js",
+    "workers/targetWorker": "@lage-run/scheduler/lib/workers/targetWorker.js",
+  };
+  // List of external modules that should not be bundled
+  const external = [
+    // Externalize deps and optional deps, which are native packages
+    ...Object.keys(packageJson.dependencies),
+    ...Object.keys(packageJson.optionalDependencies),
+    // Also don't re-bundle targetWorker from the other entry points
+    "./workers/targetWorker",
+  ];
+
+  // Due to the fact that workers require the runner to be in the same directory,
+  // add the runners to the entry points, as well as the externals (so the file is preserved).
+  const pkgToRunnerDir = {
+    "@lage-run/scheduler": "lib/runners",
+    "@lage-run/cli": "lib/commands/cache/runners",
+  };
+  for (const [pkg, runnerDir] of Object.entries(pkgToRunnerDir)) {
+    const pkgPath = path.dirname(localRequire.resolve(`${pkg}/package.json`));
+    for (const runner of fs.readdirSync(path.join(pkgPath, runnerDir))) {
+      // By convention, only include things that end with "Runner.js"
+      if (runner.endsWith("Runner.js")) {
+        const runnerInput = `${pkgPath}/${runnerDir}/${runner}`;
+        const runnerOutput = `runners/${runner}`;
+        entryPoints[runnerOutput.replace(/\.js$/, "")] = runnerInput;
+        external.push(`./${runnerOutput}`);
+      }
+    }
+  }
+
+  /** @type {import('esbuild').BuildOptions} */
+  const esbuildOptions = {
+    absWorkingDir: packageRoot,
+    entryPoints,
     bundle: true,
     platform: "node",
     target: ["node16"],
-    outfile,
+    logLevel: "info",
+    external,
+  };
+
+  // Minified build
+  console.log("\nCreating minified bundles under dist");
+  await esbuild.build({
+    ...esbuildOptions,
+    outdir: "dist",
     sourcemap: true,
-    external: [
-      "fsevents",
-      "glob-hasher",
-      "./runners/NpmScriptRunner.js",
-      "./workers/targetWorker",
-      "./runners/NoOpRunner.js",
-      "./runners/WorkerRunner.js",
-    ],
-    ...(addBanner && { banner: { js: "#!/usr/bin/env node" } }),
     minify: true,
   });
+
+  // Debug build
+  console.log("\nCreating debug bundles under dist/debug");
+  await esbuild.build({
+    ...esbuildOptions,
+    outdir: "dist/debug",
+    minify: false,
+  });
 }
 
-await Promise.all([
-  bundle("@lage-run/cli/lib/cli.js", "dist/lage.js", true),
-  bundle("./index.js", "dist/main.js"),
-  bundle("@lage-run/scheduler/lib/workers/targetWorker.js", "dist/workers/targetWorker.js"),
-]);
+await bundle().catch((err) => {
+  console.error(err.stack || err);
+  process.exit(1);
+});

packages/scheduler/package.json

@@ -22,7 +22,8 @@
     "@lage-run/cache": "^1.1.5",
     "@lage-run/config": "^0.3.5",
     "@lage-run/hasher": "^1.1.0",
-    "@lage-run/worker-threads-pool": "^0.8.0"
+    "@lage-run/worker-threads-pool": "^0.8.0",
+    "execa": "5.1.1"
   },
   "devDependencies": {
     "@lage-run/scheduler-types": "^0.3.13",

packages/scheduler/src/runners/NpmScriptRunner.ts

@@ -1,7 +1,6 @@
-import { existsSync } from "fs";
 import { join } from "path";
 import { readFile } from "fs/promises";
-import { spawn, type ChildProcess } from "child_process";
+import execa from "execa";
 import type { TargetRunner, TargetRunnerOptions } from "@lage-run/scheduler-types";
 import type { Target } from "@lage-run/target-graph";
 
@@ -56,12 +55,9 @@ export class NpmScriptRunner implements TargetRunner {
     const { nodeOptions, npmCmd, taskArgs } = this.options;
     const task = target.options?.script ?? target.task;
 
-    let childProcess: ChildProcess | undefined;
+    let childProcess: execa.ExecaChildProcess | undefined;
 
-    /**
-     * Handling abort signal from the abort controller. Gracefully kills the process,
-     * will be handled by exit handler separately to resolve the promise.
-     */
+    // Handling abort signal from the abort controller. This gracefully kills the process.
     if (abortSignal) {
       if (abortSignal.aborted) {
         return;
@@ -70,9 +66,7 @@ export class NpmScriptRunner implements TargetRunner {
       const abortSignalHandler = () => {
         abortSignal.removeEventListener("abort", abortSignalHandler);
         if (childProcess && !childProcess.killed) {
-          const pid = childProcess.pid;
-
-          process.stdout.write(`Abort signal detected, attempting to killing process id ${pid}\n`);
+          process.stdout.write(`Abort signal detected, attempting to killing process id ${childProcess.pid}\n`);
 
           childProcess.kill("SIGTERM");
 
@@ -84,23 +78,19 @@ export class NpmScriptRunner implements TargetRunner {
           }, NpmScriptRunner.gracefulKillTimeout);
 
           // Remember that even this timeout needs to be unref'ed, otherwise the process will hang due to this timeout
-          if (t.unref) {
-            t.unref();
-          }
+          t.unref?.();
         }
       };
 
       abortSignal.addEventListener("abort", abortSignalHandler);
     }
 
-    /**
-     * Actually spawn the npm client to run the task
-     */
+    // Actually spawn the npm client to run the task
     const npmRunArgs = this.getNpmArgs(task, taskArgs);
     const npmRunNodeOptions = [nodeOptions, target.options?.nodeOptions].filter((str) => str).join(" ");
 
-    await new Promise<void>((resolve, reject) => {
-      childProcess = spawn(npmCmd, npmRunArgs, {
+    try {
+      childProcess = execa(npmCmd, npmRunArgs, {
         cwd: target.cwd,
         stdio: ["inherit", "pipe", "pipe"],
         // This is required for Windows due to https://nodejs.org/en/blog/vulnerability/april-2024-security-releases-2
@@ -115,41 +105,25 @@ export class NpmScriptRunner implements TargetRunner {
         },
       });
 
-      let exitHandled = false;
-
-      const handleChildProcessExit = (code: number) => {
-        childProcess?.off("exit", handleChildProcessExit);
-        childProcess?.off("error", handleChildProcessExit);
-
-        if (exitHandled) {
-          return;
-        }
+      process.stdout.write(`Running ${[npmCmd, ...npmRunArgs].join(" ")}, pid: ${childProcess.pid}\n`);
 
-        exitHandled = true;
+      childProcess.stdout?.pipe(process.stdout);
+      childProcess.stderr?.pipe(process.stderr);
 
+      await childProcess;
+    } catch (err) {
+      const execaError = err as execa.ExecaError;
+      throw new Error(
+        `NPM Script Runner: ${npmCmd} ${npmRunArgs.join(" ")} exited with ${execaError.exitCode ? `code ${execaError.exitCode}` : "error"}`
+      );
+    } finally {
+      try {
         childProcess?.stdout?.destroy();
         childProcess?.stderr?.destroy();
-        childProcess?.stdin?.destroy();
-
-        if (code === 0) {
-          return resolve();
-        }
-
-        reject(new Error(`NPM Script Runner: ${npmCmd} ${npmRunArgs.join(" ")} exited with code ${code}`));
-      };
-
-      const { pid } = childProcess;
-
-      process.stdout.write(`Running ${[npmCmd, ...npmRunArgs].join(" ")}, pid: ${pid}\n`);
-
-      const stdout = childProcess.stdout!;
-      const stderr = childProcess.stderr!;
-
-      stdout.pipe(process.stdout);
-      stderr.pipe(process.stderr);
-
-      childProcess.on("exit", handleChildProcessExit);
-      childProcess.on("error", () => handleChildProcessExit(1));
-    });
+        childProcess = undefined;
+      } catch {
+        // ignore
+      }
+    }
   }
 }