diff --git a/change/change-64a60bc2-b0f9-434b-8ce5-47a7d48f3c1b.json b/change/change-64a60bc2-b0f9-434b-8ce5-47a7d48f3c1b.json new file mode 100644 index 000000000..a0a1b27c2 --- /dev/null +++ b/change/change-64a60bc2-b0f9-434b-8ce5-47a7d48f3c1b.json @@ -0,0 +1,18 @@ +{ + "changes": [ + { + "type": "minor", + "comment": "Exclude tasks that should not run from the dep graph", + "packageName": "@lage-run/cli", + "email": "dobes@formative.com", + "dependentChangeType": "patch" + }, + { + "type": "minor", + "comment": "Exclude tasks that should not run from the dep graph", + "packageName": "@lage-run/target-graph", + "email": "dobes@formative.com", + "dependentChangeType": "patch" + } + ] +} \ No newline at end of file diff --git a/packages/cli/src/commands/info/action.ts b/packages/cli/src/commands/info/action.ts index 4f1c71d89..67795bd68 100644 --- a/packages/cli/src/commands/info/action.ts +++ b/packages/cli/src/commands/info/action.ts @@ -11,9 +11,9 @@ import fs from "fs"; import { parse } from "shell-quote"; import type { ReporterInitOptions } from "../../types/ReporterInitOptions.js"; -import { type Target, getStartTargetId } from "@lage-run/target-graph"; +import { type Target, getStartTargetId, type TargetConfig } from "@lage-run/target-graph"; import { initializeReporters } from "../initializeReporters.js"; -import { TargetRunnerPicker } from "@lage-run/runners"; +import { TargetRunnerPicker, type TargetRunnerPickerOptions } from "@lage-run/runners"; import { getBinPaths } from "../../getBinPaths.js"; import { runnerPickerOptions } from "../../runnerPickerOptions.js"; import { parseServerOption } from "../parseServerOption.js"; @@ -22,6 +22,7 @@ import { glob } from "@lage-run/globby"; import { FileHasher } from "@lage-run/hasher/lib/FileHasher.js"; import { hashStrings } from "@lage-run/hasher"; import { getGlobalInputHashFilePath } from "../targetHashFilePath.js"; +import { shouldRun } from "../shouldRun"; interface InfoActionOptions extends ReporterInitOptions { dependencies: boolean; @@ -109,6 +110,8 @@ export async function infoAction(options: InfoActionOptions, command: Command) { const { tasks, taskArgs } = filterArgsForTasks(command.args); + const pickerOptions = runnerPickerOptions(options.nodeArg, config.npmClient, taskArgs); + const targetGraph = await createTargetGraph({ logger, root, @@ -123,6 +126,7 @@ export async function infoAction(options: InfoActionOptions, command: Command) { tasks, packageInfos, priorities: config.priorities, + shouldRun: shouldRun(pickerOptions), }); const scope = getFilteredPackages({ @@ -137,8 +141,6 @@ export async function infoAction(options: InfoActionOptions, command: Command) { sinceIgnoreGlobs: options.ignore.concat(config.ignore), }); - const pickerOptions = runnerPickerOptions(options.nodeArg, config.npmClient, taskArgs); - const runnerPicker = new TargetRunnerPicker(pickerOptions); // This is a temporary flag to allow backwards compatibility with the old lage graph format used by BuildXL (formerly known as Domino). diff --git a/packages/cli/src/commands/run/createTargetGraph.ts b/packages/cli/src/commands/run/createTargetGraph.ts index 770ebe347..51c222924 100644 --- a/packages/cli/src/commands/run/createTargetGraph.ts +++ b/packages/cli/src/commands/run/createTargetGraph.ts @@ -1,5 +1,5 @@ import type { Logger } from "@lage-run/logger"; -import { WorkspaceTargetGraphBuilder } from "@lage-run/target-graph"; +import { type Target, type TargetConfig, WorkspaceTargetGraphBuilder } from "@lage-run/target-graph"; import type { PackageInfos } from "workspace-tools"; import { getBranchChanges, getDefaultRemoteBranch, getStagedChanges, getUnstagedChanges, getUntrackedChanges } from "workspace-tools"; import { getFilteredPackages } from "../../filter/getFilteredPackages.js"; @@ -20,6 +20,7 @@ interface CreateTargetGraphOptions { tasks: string[]; packageInfos: PackageInfos; priorities: Priority[]; + shouldRun: (config: TargetConfig, target: Target) => boolean | Promise; } function getChangedFiles(since: string, cwd: string) { @@ -52,9 +53,10 @@ export async function createTargetGraph(options: CreateTargetGraphOptions) { tasks, packageInfos, priorities, + shouldRun, } = options; - const builder = new WorkspaceTargetGraphBuilder(root, packageInfos); + const builder = new WorkspaceTargetGraphBuilder(root, packageInfos, shouldRun); const packages = getFilteredPackages({ root, @@ -80,7 +82,7 @@ export async function createTargetGraph(options: CreateTargetGraphOptions) { for (const [id, definition] of Object.entries(pipeline)) { if (Array.isArray(definition)) { - builder.addTargetConfig( + await builder.addTargetConfig( id, { cache: true, @@ -91,7 +93,7 @@ export async function createTargetGraph(options: CreateTargetGraphOptions) { changedFiles ); } else { - builder.addTargetConfig(id, definition, changedFiles); + await builder.addTargetConfig(id, definition, changedFiles); } } diff --git a/packages/cli/src/commands/run/runAction.ts b/packages/cli/src/commands/run/runAction.ts index 67b25a7d0..87369cb2f 100644 --- a/packages/cli/src/commands/run/runAction.ts +++ b/packages/cli/src/commands/run/runAction.ts @@ -13,11 +13,13 @@ import createLogger from "@lage-run/logger"; import type { ReporterInitOptions } from "../../types/ReporterInitOptions.js"; import type { FilterOptions } from "../../types/FilterOptions.js"; import type { SchedulerRunSummary } from "@lage-run/scheduler-types"; -import type { TargetGraph } from "@lage-run/target-graph"; +import type { Target, TargetConfig, TargetGraph } from "@lage-run/target-graph"; import { NoTargetFoundError } from "../../types/errors.js"; import { createCache } from "../../cache/createCacheProvider.js"; import { runnerPickerOptions } from "../../runnerPickerOptions.js"; import { optimizeTargetGraph } from "../../optimizeTargetGraph.js"; +import type { TargetRunnerPickerOptions } from "@lage-run/runners"; +import { shouldRun } from "../shouldRun"; interface RunOptions extends ReporterInitOptions, FilterOptions { concurrency: number; @@ -50,6 +52,11 @@ export async function runAction(options: RunOptions, command: Command) { const { tasks, taskArgs } = filterArgsForTasks(command.args); + const pickerOptions: TargetRunnerPickerOptions = { + ...runnerPickerOptions(options.nodeArg, config.npmClient, taskArgs), + ...config.runners, + }; + const targetGraph = await createTargetGraph({ logger, root, @@ -64,6 +71,7 @@ export async function runAction(options: RunOptions, command: Command) { tasks, packageInfos, priorities: config.priorities, + shouldRun: shouldRun(pickerOptions), }); validateTargetGraph(targetGraph, allowNoTargetRuns); @@ -93,10 +101,7 @@ export async function runAction(options: RunOptions, command: Command) { taskArgs, skipLocalCache: options.skipLocalCache, cacheOptions: config.cacheOptions, - runners: { - ...runnerPickerOptions(options.nodeArg, config.npmClient, taskArgs), - ...config.runners, - }, + runners: pickerOptions, }, maxWorkersPerTask: new Map([...getMaxWorkersPerTask(filteredPipeline, concurrency), ...maxWorkersPerTaskMap]), hasher, diff --git a/packages/cli/src/commands/run/watchAction.ts b/packages/cli/src/commands/run/watchAction.ts index 134f3892f..737b41f0d 100644 --- a/packages/cli/src/commands/run/watchAction.ts +++ b/packages/cli/src/commands/run/watchAction.ts @@ -13,10 +13,12 @@ import createLogger, { LogLevel } from "@lage-run/logger"; import type { ReporterInitOptions } from "../../types/ReporterInitOptions.js"; import type { SchedulerRunSummary } from "@lage-run/scheduler-types"; -import type { Target } from "@lage-run/target-graph"; +import type { Target, TargetConfig } from "@lage-run/target-graph"; import type { FilterOptions } from "../../types/FilterOptions.js"; import { createCache } from "../../cache/createCacheProvider.js"; import { runnerPickerOptions } from "../../runnerPickerOptions.js"; +import type { TargetRunnerPickerOptions } from "@lage-run/runners"; +import { shouldRun } from "../shouldRun"; interface RunOptions extends ReporterInitOptions, FilterOptions { concurrency: number; @@ -48,6 +50,11 @@ export async function watchAction(options: RunOptions, command: Command) { const { tasks, taskArgs } = filterArgsForTasks(command.args); + const pickerOptions: TargetRunnerPickerOptions = { + ...runnerPickerOptions(options.nodeArg, config.npmClient, taskArgs), + ...config.runners, + }; + const targetGraph = await createTargetGraph({ logger, root, @@ -62,6 +69,7 @@ export async function watchAction(options: RunOptions, command: Command) { tasks, packageInfos, priorities: config.priorities, + shouldRun: shouldRun(pickerOptions), }); // Make sure we do not attempt writeRemoteCache in watch mode @@ -88,10 +96,7 @@ export async function watchAction(options: RunOptions, command: Command) { taskArgs, skipLocalCache: options.skipLocalCache, cacheOptions: config.cacheOptions, - runners: { - ...runnerPickerOptions(options.nodeArg, config.npmClient, taskArgs), - ...config.runners, - }, + runners: pickerOptions, }, shouldCache: options.cache, shouldResetCache: options.resetCache, diff --git a/packages/cli/src/commands/server/lageService.ts b/packages/cli/src/commands/server/lageService.ts index d8eccd372..ce7c51de5 100644 --- a/packages/cli/src/commands/server/lageService.ts +++ b/packages/cli/src/commands/server/lageService.ts @@ -16,6 +16,8 @@ import { formatDuration, hrToSeconds, hrtimeDiff } from "@lage-run/format-hrtime import path from "path"; import fs from "fs"; import { getGlobalInputHashFilePath, getHashFilePath } from "../targetHashFilePath.js"; +import { shouldRun } from "../shouldRun"; +import type { TargetRunnerPickerOptions } from "@lage-run/runners"; interface LageServiceContext { config: ConfigOptions; @@ -62,6 +64,11 @@ async function createInitializedPromise({ cwd, logger, serverControls, nodeArg, const packageInfos = getPackageInfos(root); + const pickerOptions: TargetRunnerPickerOptions = { + ...runnerPickerOptions(nodeArg, config.npmClient, taskArgs), + ...config.runners, + }; + logger.info("Initializing target graph"); const targetGraph = await createTargetGraph({ logger, @@ -77,6 +84,7 @@ async function createInitializedPromise({ cwd, logger, serverControls, nodeArg, tasks, packageInfos, priorities: config.priorities, + shouldRun: shouldRun(pickerOptions) }); const targetHasher = new TargetHasher({ @@ -98,6 +106,7 @@ async function createInitializedPromise({ cwd, logger, serverControls, nodeArg, const filteredPipeline = filterPipelineDefinitions(targetGraph.targets.values(), config.pipeline); logger.info("Initializing Pool"); + const pool = new AggregatedPool({ logger, maxWorkersByGroup: new Map([...getMaxWorkersPerTask(filteredPipeline, maxWorkers)]), @@ -109,8 +118,7 @@ async function createInitializedPromise({ cwd, logger, serverControls, nodeArg, stderr: true, workerData: { runners: { - ...runnerPickerOptions(nodeArg, config.npmClient, taskArgs), - ...config.runners, + ...pickerOptions, shouldCache: false, shouldResetCache: false, }, diff --git a/packages/cli/src/commands/shouldRun.ts b/packages/cli/src/commands/shouldRun.ts new file mode 100644 index 000000000..876c8eb89 --- /dev/null +++ b/packages/cli/src/commands/shouldRun.ts @@ -0,0 +1,22 @@ +import type { Target, TargetConfig } from "@lage-run/target-graph"; +import { TargetRunner, TargetRunnerPicker, TargetRunnerPickerOptions } from "@lage-run/runners"; + +// Generate a shouldRun function we can provide to the graph builder to prune tasks +// This allows the runners configured in the project to return whether the task should +// run. If a task should not run, it also should not cause anything it depends on to +// run. +export function shouldRun(pickerOptions: TargetRunnerPickerOptions) { + const picker = new TargetRunnerPicker(pickerOptions); + return async (config: TargetConfig, target: Target) => { + if (typeof config.shouldRun === "function" && !await config.shouldRun(target)) { + return false; + } + + const runner = await picker.pick(target); + if(!runner?.shouldRun(target)) { + return false; + } + + return true; + } +} diff --git a/packages/target-graph/src/WorkspaceTargetGraphBuilder.ts b/packages/target-graph/src/WorkspaceTargetGraphBuilder.ts index 9a5ec3b48..11dfe4d4b 100644 --- a/packages/target-graph/src/WorkspaceTargetGraphBuilder.ts +++ b/packages/target-graph/src/WorkspaceTargetGraphBuilder.ts @@ -13,6 +13,15 @@ import { TargetFactory } from "./TargetFactory.js"; import pLimit from "p-limit"; const DEFAULT_STAGED_TARGET_THRESHOLD = 50; + +const defaultShouldRun = (config: TargetConfig, target: Target) => { + if (typeof config.shouldRun === "function") { + return config.shouldRun(target); + } + + return true; +} + /** * TargetGraphBuilder class provides a builder API for registering target configs. It exposes a method called `generateTargetGraph` to * generate a topological graph of targets (package + task) and their dependencies. @@ -45,7 +54,7 @@ export class WorkspaceTargetGraphBuilder { * @param root the root directory of the workspace * @param packageInfos the package infos for the workspace */ - constructor(root: string, private packageInfos: PackageInfos) { + constructor(root: string, private packageInfos: PackageInfos, private shouldRun: (config: TargetConfig, target: Target) => boolean | Promise = defaultShouldRun) { this.dependencyMap = createDependencyMap(packageInfos, { withDevDependencies: true, withPeerDependencies: false }); this.graphBuilder = new TargetGraphBuilder(); this.targetFactory = new TargetFactory({ @@ -75,23 +84,25 @@ export class WorkspaceTargetGraphBuilder { this.targetConfigMap.set(id, config); this.hasRootTarget = true; - this.processStagedConfig(target, config, changedFiles); + await this.processStagedConfig(target, config, changedFiles); } else if (id.includes("#")) { const { packageName, task } = getPackageAndTask(id); const target = this.targetFactory.createPackageTarget(packageName!, task, config); this.graphBuilder.addTarget(target); this.targetConfigMap.set(id, config); - this.processStagedConfig(target, config, changedFiles); + await this.processStagedConfig(target, config, changedFiles); } else { const packages = Object.keys(this.packageInfos); for (const packageName of packages) { const task = id; const target = this.targetFactory.createPackageTarget(packageName!, task, config); - this.graphBuilder.addTarget(target); - this.targetConfigMap.set(id, config); + if(await this.shouldRun(config, target)) { + this.graphBuilder.addTarget(target); + this.targetConfigMap.set(id, config); - this.processStagedConfig(target, config, changedFiles); + await this.processStagedConfig(target, config, changedFiles); + } } } } @@ -145,14 +156,6 @@ export class WorkspaceTargetGraphBuilder { } } - shouldRun(config: TargetConfig, target: Target) { - if (typeof config.shouldRun === "function") { - return config.shouldRun(target); - } - - return true; - } - /** * Builds a scoped target graph for given tasks and packages * @@ -225,7 +228,7 @@ export class WorkspaceTargetGraphBuilder { if (config) { setShouldRunPromises.push( limit(async () => { - target.shouldRun = await this.shouldRun(config, target); + target.shouldRun = await defaultShouldRun(config, target); }) ); } diff --git a/packages/target-graph/tests/WorkspaceTargetGraphBuilder.test.ts b/packages/target-graph/tests/WorkspaceTargetGraphBuilder.test.ts index a9806dc7d..53b27977a 100644 --- a/packages/target-graph/tests/WorkspaceTargetGraphBuilder.test.ts +++ b/packages/target-graph/tests/WorkspaceTargetGraphBuilder.test.ts @@ -39,7 +39,7 @@ describe("workspace target graph builder", () => { }); const builder = new WorkspaceTargetGraphBuilder(root, packageInfos); - builder.addTargetConfig("build", { + await builder.addTargetConfig("build", { dependsOn: ["^build"], }); @@ -81,8 +81,8 @@ describe("workspace target graph builder", () => { }); const builder = new WorkspaceTargetGraphBuilder(root, packageInfos); - builder.addTargetConfig("test"); - builder.addTargetConfig("lint"); + await builder.addTargetConfig("test"); + await builder.addTargetConfig("lint"); const targetGraph = await builder.build(["test", "lint"]); @@ -121,11 +121,11 @@ describe("workspace target graph builder", () => { const builder = new WorkspaceTargetGraphBuilder(root, packageInfos); - builder.addTargetConfig("build", { + await builder.addTargetConfig("build", { dependsOn: ["^build"], }); - builder.addTargetConfig("a#build", { + await builder.addTargetConfig("a#build", { dependsOn: [], }); @@ -163,11 +163,11 @@ describe("workspace target graph builder", () => { const builder = new WorkspaceTargetGraphBuilder(root, packageInfos); - builder.addTargetConfig("build", { + await builder.addTargetConfig("build", { dependsOn: ["^build"], }); - builder.addTargetConfig("a#build", { + await builder.addTargetConfig("a#build", { dependsOn: [], }); @@ -197,11 +197,11 @@ describe("workspace target graph builder", () => { const builder = new WorkspaceTargetGraphBuilder(root, packageInfos); - builder.addTargetConfig("bundle", { + await builder.addTargetConfig("bundle", { dependsOn: ["^^transpile"], }); - builder.addTargetConfig("transpile"); + await builder.addTargetConfig("transpile"); const targetGraph = await builder.build(["bundle"], ["a"]); expect(getGraphFromTargets(targetGraph)).toMatchInlineSnapshot(` @@ -242,12 +242,12 @@ describe("workspace target graph builder", () => { const builder = new WorkspaceTargetGraphBuilder(root, packageInfos); - builder.addTargetConfig("build", { + await builder.addTargetConfig("build", { dependsOn: ["common#copy", "^build"], }); - builder.addTargetConfig("common#copy"); - builder.addTargetConfig("common#build"); + await builder.addTargetConfig("common#copy"); + await builder.addTargetConfig("common#build"); const targetGraph = await builder.build(["build"]); expect(getGraphFromTargets(targetGraph)).toMatchInlineSnapshot(` @@ -297,11 +297,11 @@ describe("workspace target graph builder", () => { }); const builder = new WorkspaceTargetGraphBuilder(root, packageInfos); - builder.addTargetConfig("build", { + await builder.addTargetConfig("build", { dependsOn: ["^build", "#global:task"], }); - builder.addTargetConfig("#global:task", { + await builder.addTargetConfig("#global:task", { dependsOn: [], }); @@ -346,11 +346,11 @@ describe("workspace target graph builder", () => { }); const builder = new WorkspaceTargetGraphBuilder(root, packageInfos); - builder.addTargetConfig("build", { + await builder.addTargetConfig("build", { dependsOn: ["^build", "#global:task"], }); - builder.addTargetConfig("#global:task", { + await builder.addTargetConfig("#global:task", { dependsOn: [], }); @@ -375,11 +375,11 @@ describe("workspace target graph builder", () => { }); const builder = new WorkspaceTargetGraphBuilder(root, packageInfos); - builder.addTargetConfig("build", { + await builder.addTargetConfig("build", { dependsOn: ["^build"], }); - builder.addTargetConfig("#global:task", { + await builder.addTargetConfig("#global:task", { dependsOn: [], });