Exclude tasks that should not run from the dep graph

change/change-64a60bc2-b0f9-434b-8ce5-47a7d48f3c1b.json

@@ -0,0 +1,18 @@
+{
+  "changes": [
+    {
+      "type": "minor",
+      "comment": "Exclude tasks that should not run from the dep graph",
+      "packageName": "@lage-run/cli",
+      "email": "dobes@formative.com",
+      "dependentChangeType": "patch"
+    },
+    {
+      "type": "minor",
+      "comment": "Exclude tasks that should not run from the dep graph",
+      "packageName": "@lage-run/target-graph",
+      "email": "dobes@formative.com",
+      "dependentChangeType": "patch"
+    }
+  ]
+}

packages/cli/src/commands/info/action.ts

@@ -11,9 +11,9 @@ import fs from "fs";
 import { parse } from "shell-quote";
 
 import type { ReporterInitOptions } from "../../types/ReporterInitOptions.js";
-import { type Target, getStartTargetId } from "@lage-run/target-graph";
+import { type Target, getStartTargetId, type TargetConfig } from "@lage-run/target-graph";
 import { initializeReporters } from "../initializeReporters.js";
-import { TargetRunnerPicker } from "@lage-run/runners";
+import { TargetRunnerPicker, type TargetRunnerPickerOptions } from "@lage-run/runners";
 import { getBinPaths } from "../../getBinPaths.js";
 import { runnerPickerOptions } from "../../runnerPickerOptions.js";
 import { parseServerOption } from "../parseServerOption.js";
@@ -22,6 +22,7 @@ import { glob } from "@lage-run/globby";
 import { FileHasher } from "@lage-run/hasher/lib/FileHasher.js";
 import { hashStrings } from "@lage-run/hasher";
 import { getGlobalInputHashFilePath } from "../targetHashFilePath.js";
+import { shouldRun } from "../shouldRun";
 
 interface InfoActionOptions extends ReporterInitOptions {
   dependencies: boolean;
@@ -109,6 +110,8 @@ export async function infoAction(options: InfoActionOptions, command: Command) {
 
   const { tasks, taskArgs } = filterArgsForTasks(command.args);
 
+  const pickerOptions = runnerPickerOptions(options.nodeArg, config.npmClient, taskArgs);
+
   const targetGraph = await createTargetGraph({
     logger,
     root,
@@ -123,6 +126,7 @@ export async function infoAction(options: InfoActionOptions, command: Command) {
     tasks,
     packageInfos,
     priorities: config.priorities,
+    shouldRun: shouldRun(pickerOptions),
   });
 
   const scope = getFilteredPackages({
@@ -137,8 +141,6 @@ export async function infoAction(options: InfoActionOptions, command: Command) {
     sinceIgnoreGlobs: options.ignore.concat(config.ignore),
   });
 
-  const pickerOptions = runnerPickerOptions(options.nodeArg, config.npmClient, taskArgs);
-
   const runnerPicker = new TargetRunnerPicker(pickerOptions);
 
   // This is a temporary flag to allow backwards compatibility with the old lage graph format used by BuildXL (formerly known as Domino).

packages/cli/src/commands/run/createTargetGraph.ts

@@ -1,5 +1,5 @@
 import type { Logger } from "@lage-run/logger";
-import { WorkspaceTargetGraphBuilder } from "@lage-run/target-graph";
+import { type Target, type TargetConfig, WorkspaceTargetGraphBuilder } from "@lage-run/target-graph";
 import type { PackageInfos } from "workspace-tools";
 import { getBranchChanges, getDefaultRemoteBranch, getStagedChanges, getUnstagedChanges, getUntrackedChanges } from "workspace-tools";
 import { getFilteredPackages } from "../../filter/getFilteredPackages.js";
@@ -20,6 +20,7 @@ interface CreateTargetGraphOptions {
   tasks: string[];
   packageInfos: PackageInfos;
   priorities: Priority[];
+  shouldRun: (config: TargetConfig, target: Target) => boolean | Promise<boolean>;
 }
 
 function getChangedFiles(since: string, cwd: string) {
@@ -52,9 +53,10 @@ export async function createTargetGraph(options: CreateTargetGraphOptions) {
     tasks,
     packageInfos,
     priorities,
+    shouldRun,
   } = options;
 
-  const builder = new WorkspaceTargetGraphBuilder(root, packageInfos);
+  const builder = new WorkspaceTargetGraphBuilder(root, packageInfos, shouldRun);
 
   const packages = getFilteredPackages({
     root,
@@ -80,7 +82,7 @@ export async function createTargetGraph(options: CreateTargetGraphOptions) {
 
   for (const [id, definition] of Object.entries(pipeline)) {
     if (Array.isArray(definition)) {
-      builder.addTargetConfig(
+      await builder.addTargetConfig(
         id,
         {
           cache: true,
@@ -91,7 +93,7 @@ export async function createTargetGraph(options: CreateTargetGraphOptions) {
         changedFiles
       );
     } else {
-      builder.addTargetConfig(id, definition, changedFiles);
+      await builder.addTargetConfig(id, definition, changedFiles);
     }
   }
 

packages/cli/src/commands/run/runAction.ts

@@ -13,11 +13,13 @@ import createLogger from "@lage-run/logger";
 import type { ReporterInitOptions } from "../../types/ReporterInitOptions.js";
 import type { FilterOptions } from "../../types/FilterOptions.js";
 import type { SchedulerRunSummary } from "@lage-run/scheduler-types";
-import type { TargetGraph } from "@lage-run/target-graph";
+import type { Target, TargetConfig, TargetGraph } from "@lage-run/target-graph";
 import { NoTargetFoundError } from "../../types/errors.js";
 import { createCache } from "../../cache/createCacheProvider.js";
 import { runnerPickerOptions } from "../../runnerPickerOptions.js";
 import { optimizeTargetGraph } from "../../optimizeTargetGraph.js";
+import type { TargetRunnerPickerOptions } from "@lage-run/runners";
+import { shouldRun } from "../shouldRun";
 
 interface RunOptions extends ReporterInitOptions, FilterOptions {
   concurrency: number;
@@ -50,6 +52,11 @@ export async function runAction(options: RunOptions, command: Command) {
 
   const { tasks, taskArgs } = filterArgsForTasks(command.args);
 
+  const pickerOptions: TargetRunnerPickerOptions = {
+    ...runnerPickerOptions(options.nodeArg, config.npmClient, taskArgs),
+    ...config.runners,
+  };
+
   const targetGraph = await createTargetGraph({
     logger,
     root,
@@ -64,6 +71,7 @@ export async function runAction(options: RunOptions, command: Command) {
     tasks,
     packageInfos,
     priorities: config.priorities,
+    shouldRun: shouldRun(pickerOptions),
   });
 
   validateTargetGraph(targetGraph, allowNoTargetRuns);
@@ -93,10 +101,7 @@ export async function runAction(options: RunOptions, command: Command) {
       taskArgs,
       skipLocalCache: options.skipLocalCache,
       cacheOptions: config.cacheOptions,
-      runners: {
-        ...runnerPickerOptions(options.nodeArg, config.npmClient, taskArgs),
-        ...config.runners,
-      },
+      runners: pickerOptions,
     },
     maxWorkersPerTask: new Map([...getMaxWorkersPerTask(filteredPipeline, concurrency), ...maxWorkersPerTaskMap]),
     hasher,

packages/cli/src/commands/run/watchAction.ts

@@ -13,10 +13,12 @@ import createLogger, { LogLevel } from "@lage-run/logger";
 
 import type { ReporterInitOptions } from "../../types/ReporterInitOptions.js";
 import type { SchedulerRunSummary } from "@lage-run/scheduler-types";
-import type { Target } from "@lage-run/target-graph";
+import type { Target, TargetConfig } from "@lage-run/target-graph";
 import type { FilterOptions } from "../../types/FilterOptions.js";
 import { createCache } from "../../cache/createCacheProvider.js";
 import { runnerPickerOptions } from "../../runnerPickerOptions.js";
+import type { TargetRunnerPickerOptions } from "@lage-run/runners";
+import { shouldRun } from "../shouldRun";
 
 interface RunOptions extends ReporterInitOptions, FilterOptions {
   concurrency: number;
@@ -48,6 +50,11 @@ export async function watchAction(options: RunOptions, command: Command) {
 
   const { tasks, taskArgs } = filterArgsForTasks(command.args);
 
+  const pickerOptions: TargetRunnerPickerOptions = {
+    ...runnerPickerOptions(options.nodeArg, config.npmClient, taskArgs),
+    ...config.runners,
+  };
+
   const targetGraph = await createTargetGraph({
     logger,
     root,
@@ -62,6 +69,7 @@ export async function watchAction(options: RunOptions, command: Command) {
     tasks,
     packageInfos,
     priorities: config.priorities,
+    shouldRun: shouldRun(pickerOptions),
   });
 
   // Make sure we do not attempt writeRemoteCache in watch mode
@@ -88,10 +96,7 @@ export async function watchAction(options: RunOptions, command: Command) {
       taskArgs,
       skipLocalCache: options.skipLocalCache,
       cacheOptions: config.cacheOptions,
-      runners: {
-        ...runnerPickerOptions(options.nodeArg, config.npmClient, taskArgs),
-        ...config.runners,
-      },
+      runners: pickerOptions,
     },
     shouldCache: options.cache,
     shouldResetCache: options.resetCache,

packages/cli/src/commands/server/lageService.ts

@@ -16,6 +16,8 @@ import { formatDuration, hrToSeconds, hrtimeDiff } from "@lage-run/format-hrtime
 import path from "path";
 import fs from "fs";
 import { getGlobalInputHashFilePath, getHashFilePath } from "../targetHashFilePath.js";
+import { shouldRun } from "../shouldRun";
+import type { TargetRunnerPickerOptions } from "@lage-run/runners";
 
 interface LageServiceContext {
   config: ConfigOptions;
@@ -62,6 +64,11 @@ async function createInitializedPromise({ cwd, logger, serverControls, nodeArg,
 
   const packageInfos = getPackageInfos(root);
 
+  const pickerOptions: TargetRunnerPickerOptions = {
+    ...runnerPickerOptions(nodeArg, config.npmClient, taskArgs),
+    ...config.runners,
+  };
+
   logger.info("Initializing target graph");
   const targetGraph = await createTargetGraph({
     logger,
@@ -77,6 +84,7 @@ async function createInitializedPromise({ cwd, logger, serverControls, nodeArg,
     tasks,
     packageInfos,
     priorities: config.priorities,
+    shouldRun: shouldRun(pickerOptions)
   });
 
   const targetHasher = new TargetHasher({
@@ -98,6 +106,7 @@ async function createInitializedPromise({ cwd, logger, serverControls, nodeArg,
   const filteredPipeline = filterPipelineDefinitions(targetGraph.targets.values(), config.pipeline);
 
   logger.info("Initializing Pool");
+
   const pool = new AggregatedPool({
     logger,
     maxWorkersByGroup: new Map([...getMaxWorkersPerTask(filteredPipeline, maxWorkers)]),
@@ -109,8 +118,7 @@ async function createInitializedPromise({ cwd, logger, serverControls, nodeArg,
       stderr: true,
       workerData: {
         runners: {
-          ...runnerPickerOptions(nodeArg, config.npmClient, taskArgs),
-          ...config.runners,
+          ...pickerOptions,
           shouldCache: false,
           shouldResetCache: false,
         },

packages/cli/src/commands/shouldRun.ts

@@ -0,0 +1,22 @@
+import type { Target, TargetConfig } from "@lage-run/target-graph";
+import { TargetRunner, TargetRunnerPicker, TargetRunnerPickerOptions } from "@lage-run/runners";
+
+// Generate a shouldRun function we can provide to the graph builder to prune tasks
+// This allows the runners configured in the project to return whether the task should
+// run.  If a task should not run, it also should not cause anything it depends on to
+// run.
+export function shouldRun(pickerOptions: TargetRunnerPickerOptions) {
+  const picker = new TargetRunnerPicker(pickerOptions);
+  return async (config: TargetConfig, target: Target) => {
+    if (typeof config.shouldRun === "function" && !await config.shouldRun(target)) {
+      return false;
+    }
+
+    const runner = await picker.pick(target);
+    if(!runner?.shouldRun(target)) {
+      return false;
+    }
+
+    return true;
+  }
+}

packages/target-graph/src/WorkspaceTargetGraphBuilder.ts

@@ -13,6 +13,15 @@ import { TargetFactory } from "./TargetFactory.js";
 import pLimit from "p-limit";
 
 const DEFAULT_STAGED_TARGET_THRESHOLD = 50;
+
+const defaultShouldRun = (config: TargetConfig, target: Target) => {
+  if (typeof config.shouldRun === "function") {
+    return config.shouldRun(target);
+  }
+
+  return true;
+}
+
 /**
  * TargetGraphBuilder class provides a builder API for registering target configs. It exposes a method called `generateTargetGraph` to
  * generate a topological graph of targets (package + task) and their dependencies.
@@ -45,7 +54,7 @@ export class WorkspaceTargetGraphBuilder {
    * @param root the root directory of the workspace
    * @param packageInfos the package infos for the workspace
    */
-  constructor(root: string, private packageInfos: PackageInfos) {
+  constructor(root: string, private packageInfos: PackageInfos, private shouldRun: (config: TargetConfig, target: Target) => boolean | Promise<boolean> = defaultShouldRun) {
     this.dependencyMap = createDependencyMap(packageInfos, { withDevDependencies: true, withPeerDependencies: false });
     this.graphBuilder = new TargetGraphBuilder();
     this.targetFactory = new TargetFactory({
@@ -75,23 +84,25 @@ export class WorkspaceTargetGraphBuilder {
       this.targetConfigMap.set(id, config);
       this.hasRootTarget = true;
 
-      this.processStagedConfig(target, config, changedFiles);
+      await this.processStagedConfig(target, config, changedFiles);
     } else if (id.includes("#")) {
       const { packageName, task } = getPackageAndTask(id);
       const target = this.targetFactory.createPackageTarget(packageName!, task, config);
       this.graphBuilder.addTarget(target);
       this.targetConfigMap.set(id, config);
 
-      this.processStagedConfig(target, config, changedFiles);
+      await this.processStagedConfig(target, config, changedFiles);
     } else {
       const packages = Object.keys(this.packageInfos);
       for (const packageName of packages) {
         const task = id;
         const target = this.targetFactory.createPackageTarget(packageName!, task, config);
-        this.graphBuilder.addTarget(target);
-        this.targetConfigMap.set(id, config);
+        if(await this.shouldRun(config, target)) {
+          this.graphBuilder.addTarget(target);
+          this.targetConfigMap.set(id, config);
 
-        this.processStagedConfig(target, config, changedFiles);
+          await this.processStagedConfig(target, config, changedFiles);
+        }
       }
     }
   }
@@ -145,14 +156,6 @@ export class WorkspaceTargetGraphBuilder {
     }
   }
 
-  shouldRun(config: TargetConfig, target: Target) {
-    if (typeof config.shouldRun === "function") {
-      return config.shouldRun(target);
-    }
-
-    return true;
-  }
-
   /**
    * Builds a scoped target graph for given tasks and packages
    *
@@ -225,7 +228,7 @@ export class WorkspaceTargetGraphBuilder {
       if (config) {
         setShouldRunPromises.push(
           limit(async () => {
-            target.shouldRun = await this.shouldRun(config, target);
+            target.shouldRun = await defaultShouldRun(config, target);
           })
         );
       }