mirror your GitHub repos to tangled.org automatically
synchub.to
mirror
sync
tangled
github
git
fix: retry transient knot failures indefinitely instead of retiring them
Daniel Roe
(Jul 12, 2026, 3:20 PM +0200)
8f2f3215
d4c4b2e4
+39
-5
+8
-1
server/api/jobs/run.get.ts
+8
-1
server/api/jobs/run.get.ts
···
1
1
import crypto from 'node:crypto'
2
+
import { isTransientTransport } from '#server/utils/git-wire/errors'
2
3
import { dispatch } from '#server/utils/job-handlers'
3
4
import { claim, cleanupOldJobs, complete, fail } from '#server/utils/queue'
4
5
···
91
92
}
92
93
catch (err) {
93
94
failed++
95
+
// A flaky knot resets connections for minutes at a time; a push to a
96
+
// real branch must survive that and land once the knot recovers, not
97
+
// be retired to `failed` after 8 unlucky attempts. Keep transient
98
+
// transport failures (and the per-job wall-clock cap, which usually
99
+
// means "slow right now") retrying indefinitely on the capped backoff.
100
+
const retryForever = isTransientTransport(err) || err instanceof JobTimeoutError
94
101
// eslint-disable-next-line no-await-in-loop
95
-
await fail(job.id, job.attempts, err)
102
+
await fail(job.id, job.attempts, err, { retryForever })
96
103
}
97
104
98
105
processed++
+18
-3
server/utils/queue.ts
+18
-3
server/utils/queue.ts
···
129
129
return affected.rowCount ?? affected.affectedRows ?? 0
130
130
}
131
131
132
+
export interface FailOptions {
133
+
maxAttempts?: number
134
+
/**
135
+
* When true, the job keeps re-queuing past `maxAttempts` instead of being
136
+
* retired to `failed`. For failures that are temporary by nature (a flaky
137
+
* knot SSH endpoint resetting connections), giving up permanently would
138
+
* abandon legitimate work: a push to a real branch must land eventually, not
139
+
* be dropped after 8 unlucky attempts. Backoff still caps at 1h, so an
140
+
* indefinitely-retrying job polls at most hourly rather than hammering.
141
+
*/
142
+
retryForever?: boolean
143
+
}
144
+
132
145
/**
133
146
* Record a failure. Re-queues with exponential backoff until `maxAttempts`,
134
-
* after which the job is marked `failed` and stays put for inspection.
147
+
* after which the job is marked `failed` and stays put for inspection, unless
148
+
* `retryForever` is set (see `FailOptions`).
135
149
*/
136
-
export async function fail(id: number, attempts: number, err: unknown, maxAttempts = MAX_ATTEMPTS) {
150
+
export async function fail(id: number, attempts: number, err: unknown, opts: FailOptions = {}) {
151
+
const { maxAttempts = MAX_ATTEMPTS, retryForever = false } = opts
137
152
const db = useDb()
138
153
const message = err instanceof Error ? err.message : String(err)
139
154
140
-
if (attempts >= maxAttempts) {
155
+
if (!retryForever && attempts >= maxAttempts) {
141
156
await db.update(job)
142
157
.set({ status: 'failed', lastError: message, lockedBy: null, lockedUntil: null, updatedAt: new Date() })
143
158
.where(sql`${job.id} = ${id}`)
+13
-1
test/unit/queue.spec.ts
+13
-1
test/unit/queue.spec.ts
···
115
115
it('marks failed once attempts >= maxAttempts', async () => {
116
116
await enqueue('github.push', {})
117
117
const claimed = await claim('worker-1', 60_000)
118
-
await fail(claimed!.id, 5, new Error('terminal'), 5)
118
+
await fail(claimed!.id, 5, new Error('terminal'), { maxAttempts: 5 })
119
119
120
120
const db = useDb()
121
121
const rows = await db.select().from(job).where(sql`${job.id} = ${claimed!.id}`)
122
122
expect(rows[0]?.status).toBe('failed')
123
123
expect(rows[0]?.lastError).toBe('terminal')
124
+
})
125
+
126
+
it('keeps re-queuing past maxAttempts when retryForever is set', async () => {
127
+
await enqueue('github.push', {})
128
+
const claimed = await claim('worker-1', 60_000)
129
+
// Well past the ceiling, but a transient transport failure must not retire.
130
+
await fail(claimed!.id, 99, new Error('ssh error: read ECONNRESET'), { retryForever: true })
131
+
132
+
const db = useDb()
133
+
const rows = await db.select().from(job).where(sql`${job.id} = ${claimed!.id}`)
134
+
expect(rows[0]?.status).toBe('queued')
135
+
expect(new Date(rows[0]?.runAfter ?? 0).getTime()).toBeGreaterThan(Date.now())
124
136
})
125
137
})
126
138