From 4d3fe6c95a00b1de443a4b3072cbe67ecc598793 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 18 Mar 2025 09:13:50 +0100 Subject: [PATCH 1/2] Use canStartBuild rather than nextBuild for starting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is some evidence that a delay in nextBuild can leak to unrelated workers. Switch to canStartBuild, whose docs show an example to put the worker in “quarantine” and avoid the quarantine's exponential backoff mechanism. --- master/master.cfg | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/master/master.cfg b/master/master.cfg index 2e72096a..a31b673a 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -223,21 +223,22 @@ def get_delay(now, end): def no_builds_between(start, end): start = datetime.strptime(start, "%H:%M").time() end = datetime.strptime(end, "%H:%M").time() - def f(builder, requests): + @defer.inlineCallbacks + def canStartBuild(builder, wfb, request): now = datetime.now().time() if is_within_time_range(now, start, end): delay = get_delay(now, end) - # Schedule the build later - deferred = defer.Deferred() - builder.master.reactor.callLater( - int(delay), - deferred.callback, - requests[0], - ) - return deferred + # Adapted from: https://docs.buildbot.net/current/manual/customization.html#canstartbuild-functions + wfb.worker.quarantine_timeout = delay + wfb.worker.putInQuarantine() + # This does not take the worker out of quarantine, it only resets + # the timeout value to default (restarting the default + # exponential backoff) + wfb.worker.resetQuarantine() + return False # Schedule the build now - return requests[0] - return f + return True + return canStartBuild github_status_builders = [] @@ -323,7 +324,7 @@ for branch_num, (git_url, branchname, git_branch) in enumerate(git_branches): # This worker runs pyperformance at 12am. If a build is scheduled between # 10pm and 2am, it will be delayed at 2am. if worker_name == "diegorusso-aarch64-bigmem": - builder.nextBuild = no_builds_between("22:00", "2:00") + builder.canStartBuild = no_builds_between("22:00", "2:00") c["builders"].append(builder) @@ -392,7 +393,7 @@ for name, worker_name, buildfactory, stability, tier in BUILDERS: # This worker runs pyperformance at 12am. If a build is scheduled between # 10pm and 2am, it will be delayed at 2am. if worker_name == "diegorusso-aarch64-bigmem": - builder.nextBuild = no_builds_between("22:00", "2:00") + builder.canStartBuild = no_builds_between("22:00", "2:00") c["builders"].append(builder) From d084a327af9ca38d2021733357f5da5e49bd6c9c Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 18 Mar 2025 22:47:21 +0100 Subject: [PATCH 2/2] Update master/master.cfg --- master/master.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/master/master.cfg b/master/master.cfg index a31b673a..b5f44242 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -223,7 +223,6 @@ def get_delay(now, end): def no_builds_between(start, end): start = datetime.strptime(start, "%H:%M").time() end = datetime.strptime(end, "%H:%M").time() - @defer.inlineCallbacks def canStartBuild(builder, wfb, request): now = datetime.now().time() if is_within_time_range(now, start, end):