fix(level-blocks): validate collector refs off the compiled spec; exhaustive dispatch · russ.fugl.dev/levelset@4ff45d4

+1 -1

README.md

··· 266 266 - **`analyzeRun(resolved, constraints, { scorers, search })`** — static validation _before_ searching: unknown resource/task/unit references, contradictory or unfittable deadline/release pairs, precedence and unit-precedence cycles, kinds the chosen search won't enforce. Issues carry stable `LEVELSET_E###`/`W###` codes and attribute to the **named constraint instance** (`name` on any constraint; instanceIds fill in for config-built runs). `runFromConfig` throws `RunAnalysisError` on errors; warnings ride along. 267 267 - **`enabled: false`** on a constraint or block — the present-but-inactive lever for what-if comparisons; searches skip it and record the skip as a `disabledConstraints` annotation, so "disabled" stays distinguishable from "absent". 268 268 - **`weightedScorer` / `bestByLex`** — multi-objective combination as data: a weight map (with `null` = reported-but-unweighted) or ordered lexicographic tiers. 269 - - **Collector blocks (`KBestBlock`, `ParetoBlock`)** — the selection layer, one step above scoring: where a scorer ranks a single schedule, a collector picks a _set_ out of the candidate stream. `KBest` (`levelset.KBest.v1`, params `{ k, by: [scorerInstanceId…] }`) keeps the best `k` schedules ranked lexicographically over the referenced scorers; `Pareto` (`levelset.Pareto.v1`, params `{ over: [scorerInstanceId…] }`) keeps the non-dominated set. They reference scorers by instanceId (resolved at run-wiring time, like `objective.lexicographic`), so the spec stays plain JSON. `run.collectors()` runs every enabled collector against a fresh stream and returns each one's schedule set keyed by instanceId. Since `serialSGS` emits a single schedule today, a collector currently selects that one schedule — collectors become a genuine lever once a multi-candidate search lands. 269 + - **Collector blocks (`KBestBlock`, `ParetoBlock`)** — the selection layer, one step above scoring: where a scorer ranks a single schedule, a collector picks a _set_ out of the candidate stream. `KBest` (`levelset.KBest.v1`, params `{ k, by: [scorerInstanceId…] }`) keeps the best `k` schedules ranked lexicographically over the referenced scorers; `Pareto` (`levelset.Pareto.v1`, params `{ over: [scorerInstanceId…] }`) keeps the non-dominated set. They reference scorers by instanceId (resolved at run-wiring time, like `objective.lexicographic`), so the spec stays plain JSON. `run.collectors()` runs every enabled collector against a fresh stream and returns each one's schedule set, each result labeled by its instanceId. Since `serialSGS` emits a single schedule today, a collector currently selects that one schedule — collectors become a genuine lever once a multi-candidate search lands. 270 270 - **`scheduleProfiles(schedule)`** — shared derived data (per-resource day histograms, per-unit open spans) computed once per schedule; every scoring block reads these slots instead of recomputing. 271 271 - **Provenance** — `run.materialize(schedule)` embeds a `levelset` block in the output `ProjectFile`: engine version, the _filled_ run config (defaults expanded — rerunning it reproduces the schedule), seed, achieved scores, warnings, and search annotations. The JSON writer/reader round-trips it, so "why does this schedule look like this?" is answerable from the file itself. 272 272

+3 -1

src/level-blocks/pareto.ts

··· 3 3 // Keeps the non-dominated set across the referenced scorer instances (the 4 4 // Pareto frontier): a schedule survives unless another is at least as good 5 5 // on every scorer and strictly better on at least one. Two or more scorers 6 - // are the sensible case; one degenerates to "the best by that scorer". 6 + // are the sensible case; with a single scorer nothing dominates a co-best 7 + // schedule, so the frontier keeps every schedule tied at that scorer's best 8 + // value. 7 9 // 8 10 // The `over` entries are scorer instanceIds from the same pipeline, not 9 11 // Scorer closures: they resolve to compiled scorers only at run-wiring time

+59 -16

src/level-blocks/runConfig.ts

··· 30 30 Scorer, 31 31 Search, 32 32 } from "../level-core/types.ts"; 33 + import { assertNeverCollector } from "../level-core/types.ts"; 33 34 import { weightedScorer } from "../level-core/weightedScorer.ts"; 34 35 import type { RunProvenance } from "../schema/provenance.ts"; 35 36 import { LEVELSET_VERSION } from "../version.ts"; ··· 124 125 /** Drain the stream and keep the best schedule by the objective. */ 125 126 best(): Promise<Schedule | null>; 126 127 /** Run every enabled collector block against a fresh search stream, 127 - * returning each one's selected schedule set keyed by graph instanceId. 128 - * Empty when the pipeline declares no collectors. */ 128 + * returning each one's selected schedule set, each result labeled by its 129 + * graph instanceId. Empty when the pipeline declares no collectors. */ 129 130 collectors(): Promise<ReadonlyArray<CollectorResult>>; 130 131 /** Every scorer's raw value, keyed by instanceId — the reported view. */ 131 132 scores(schedule: Schedule): Record<string, number>; ··· 151 152 ); 152 153 // Collector blocks reference scorers by instanceId (KBest.by / Pareto.over); 153 154 // like weights and lexicographic tiers those refs must land on scorer-family 154 - // instances. Read the parsed params so defaults/coercions are applied. 155 + // instances. Read the refs off the *compiled* Collector spec (via apply) 156 + // rather than duck-typing the parsed params: the closed Collector ADT 157 + // guarantees by/over are string[], so a custom collector that names its refs 158 + // under a different param still gets layer-1 validation and a non-array 159 + // by/over can't leak through as a raw TypeError. apply() is pure and 160 + // compileGraph re-applies it later, so calling it here costs nothing. 155 161 const collectorRefs = validated.graph.blocks 156 162 .filter((b) => reg.get(b.code)?.family === "collector") 157 163 .flatMap((b) => { 158 - const params = validated.parsedParams.get(b.instanceId) as { 159 - by?: ReadonlyArray<string>; 160 - over?: ReadonlyArray<string>; 161 - }; 162 - const refs = params.by ?? params.over ?? []; 163 - return refs.map((k) => ({ k, where: `collector "${b.instanceId}" scorer reference` })); 164 + const def = reg.get(b.code)!; 165 + const params = validated.parsedParams.get(b.instanceId); 166 + let collector: Collector; 167 + try { 168 + collector = (def.block as { apply(input: unknown): Collector }).apply(params); 169 + } catch (err) { 170 + throw new GraphValidationError([ 171 + { 172 + instanceId: b.instanceId, 173 + code: b.code, 174 + message: `collector "${b.instanceId}" failed to compile its spec: ${ 175 + err instanceof Error ? err.message : String(err) 176 + }`, 177 + }, 178 + ]); 179 + } 180 + return collectorScorerRefs(collector).map((k) => ({ 181 + k, 182 + where: `collector "${b.instanceId}" scorer reference`, 183 + })); 164 184 }); 165 185 const badRefs = [ 166 186 ...Object.keys(parsed.weights ?? {}).map((k) => ({ k, where: "weights key" })), ··· 312 332 313 333 // Same active-scorer requirement for collector references: fillRunConfig 314 334 // proved they point at scorer instances; an *enabled* collector can only 315 - // select against *enabled* scorers. 335 + // select against *enabled* scorers. Distinguish disabled from truly absent: 336 + // the ref exists in the full scorer set (compiled away by `enabled: false`) 337 + // versus never a scorer instance at all. 338 + const allScorerIds = new Set( 339 + filled.pipeline.blocks 340 + .filter((b) => registry.get(b.code)?.family === "scorer") 341 + .map((b) => b.instanceId), 342 + ); 316 343 const inactiveCollectorRefs = collectors.flatMap((c) => 317 344 collectorScorerRefs(c.collector) 318 345 .filter((id) => !activeScorerIds.has(id)) ··· 322 349 throw new GraphValidationError( 323 350 inactiveCollectorRefs.map(({ instanceId, id }) => ({ 324 351 instanceId, 325 - message: `collector "${instanceId}" references a disabled scorer instance "${id}"`, 352 + message: allScorerIds.has(id) 353 + ? `collector "${instanceId}" references a disabled scorer instance "${id}"` 354 + : `collector "${instanceId}" references scorer instance "${id}", which does not exist`, 326 355 })), 327 356 ); 328 357 } ··· 363 392 return Promise.all( 364 393 collectors.map(async (c): Promise<CollectorResult> => { 365 394 const refs = collectorScorerRefs(c.collector).map((id) => byId.get(id)!); 366 - const schedules = 367 - c.collector.kind === "KBest" 368 - ? await run.stream().kBestByLex(refs, c.collector.k) 369 - : await run.stream().paretoFrontier(refs); 395 + let schedules: ReadonlyArray<Schedule>; 396 + switch (c.collector.kind) { 397 + case "KBest": 398 + schedules = await run.stream().kBestByLex(refs, c.collector.k); 399 + break; 400 + case "Pareto": 401 + schedules = await run.stream().paretoFrontier(refs); 402 + break; 403 + default: 404 + assertNeverCollector(c.collector); 405 + } 370 406 return { instanceId: c.instanceId, kind: c.collector.kind, schedules }; 371 407 }), 372 408 ); ··· 399 435 400 436 /** The scorer instanceIds a collector references, whatever its kind. */ 401 437 function collectorScorerRefs(collector: Collector): ReadonlyArray<string> { 402 - return collector.kind === "KBest" ? collector.by : collector.over; 438 + switch (collector.kind) { 439 + case "KBest": 440 + return collector.by; 441 + case "Pareto": 442 + return collector.over; 443 + default: 444 + return assertNeverCollector(collector); 445 + } 403 446 } 404 447 405 448 // Search.run returns a Failure as the generator's return value;

+1 -1

src/level-core/scheduleStream.ts

··· 92 92 for await (const s of this.source) { 93 93 scored.push({ schedule: s, scores: scorers.map((sc) => sc.score(s)) }); 94 94 } 95 - // Stable sort (V8/Bun guarantee it): equal-on-all-tiers keeps stream 95 + // Stable sort (ES2019 guarantees it): equal-on-all-tiers keeps stream 96 96 // order, matching bestByLex where first seen wins. 97 97 scored.sort((a, b) => 98 98 lexBetter(a.scores, b.scores, scorers) ? -1 : lexBetter(b.scores, a.scores, scorers) ? 1 : 0,

+2 -2

src/level-core/types.ts

··· 313 313 } 314 314 315 315 /** Keep the non-dominated set across the `over` scorers (Pareto frontier). 316 - * Two or more scorers are the sensible case; one degenerates to "the best 317 - * by that scorer". */ 316 + * Two or more scorers are the sensible case; with a single scorer the 317 + * frontier keeps every schedule tied at that scorer's best value. */ 318 318 export interface ParetoCollector { 319 319 readonly kind: "Pareto"; 320 320 /** Scorer instanceIds spanning the objective space. */

+31

test/level-blocks/collectors.test.ts

··· 28 28 direction: "min", 29 29 score: (s) => s.annotations.get("cost") as number, 30 30 }; 31 + // A max-direction scorer: higher is better (unlike makespan/cost). 32 + const byGain: Scorer = { 33 + name: "gain", 34 + direction: "max", 35 + score: (s) => s.annotations.get("gain") as number, 36 + }; 31 37 32 38 function streamOf(schedules: ReadonlyArray<Schedule>): ScheduleStreamImpl { 33 39 return new ScheduleStreamImpl({ ··· 93 99 const all = await streamOf([sched(2, 1), sched(1, 2)]).kBestByLex([byMakespan], 9); 94 100 expect(all).toHaveLength(2); 95 101 expect(await streamOf([sched(1, 1)]).kBestByLex([byMakespan], 0)).toEqual([]); 102 + }); 103 + 104 + test("honors a direction:max tier when breaking ties (higher wins)", async () => { 105 + // makespan ties at 5 between the two gain schedules; the max-direction 106 + // gain scorer breaks it, so the higher gain (30) ranks above the lower. 107 + const lowGain: Schedule = { ...sched(5, 0), annotations: new Map([["gain", 10]]) }; 108 + const highGain: Schedule = { ...sched(5, 0), annotations: new Map([["gain", 30]]) }; 109 + const clearWin: Schedule = { ...sched(4, 0), annotations: new Map([["gain", 1]]) }; 110 + const top = await streamOf([lowGain, highGain, clearWin]).kBestByLex([byMakespan, byGain], 2); 111 + expect(top.map((s) => [s.makespan, s.annotations.get("gain")])).toEqual([ 112 + [4, 1], 113 + [5, 30], 114 + ]); 115 + }); 116 + 117 + test("a full tie on every tier keeps first-seen stream order", async () => { 118 + // All three tie on makespan and cost; a stable sort must keep them in 119 + // stream order, so kBest returns the first two seen (by identity). 120 + const first = sched(5, 10); 121 + const second = sched(5, 10); 122 + const third = sched(5, 10); 123 + const top = await streamOf([first, second, third]).kBestByLex([byMakespan, byCost], 2); 124 + expect(top).toHaveLength(2); 125 + expect(top[0]).toBe(first); 126 + expect(top[1]).toBe(second); 96 127 }); 97 128 });

+49 -1

test/level-blocks/runConfig.test.ts

··· 2 2 // runFromConfig end-to-end, weights, enabled levers, and coded failures. 3 3 4 4 import { describe, expect, test } from "bun:test"; 5 + import { z } from "zod"; 5 6 6 7 import { GraphValidationError } from "../../src/level-blocks/compile.ts"; 8 + import type { BlockDef } from "../../src/level-blocks/registry.ts"; 7 9 import { fillRunConfig, runFromConfig } from "../../src/level-blocks/runConfig.ts"; 8 - import { STANDARD_CODES } from "../../src/level-blocks/standard.ts"; 10 + import { STANDARD_CODES, standardRegistry } from "../../src/level-blocks/standard.ts"; 9 11 import { RunAnalysisError } from "../../src/level-core/errors.ts"; 10 12 import { LEVELSET_VERSION } from "../../src/version.ts"; 11 13 ··· 253 255 params: { over: ["nope"] }, 254 256 }); 255 257 expect(() => runFromConfig(crewProject(), config)).toThrow(GraphValidationError); 258 + }); 259 + 260 + // A custom collector whose input carries scorer refs under a param name 261 + // other than by/over: its apply() maps `scorers` into the KBest `by` field. 262 + // fillRunConfig must extract the ref off this compiled spec (not the param 263 + // shape), so a dangling ref is a GraphValidationError — not a silent pass 264 + // (the old duck-typing missed non-by/over params) and not a raw TypeError. 265 + const pickByDef: BlockDef = { 266 + code: "myapp.PickBy.v1", 267 + family: "collector", 268 + tier: "engineer", 269 + block: { 270 + id: "PickBy", 271 + schema: { 272 + input: z.object({ scorers: z.array(z.string()).min(1) }), 273 + output: z.object({ 274 + kind: z.literal("KBest"), 275 + k: z.number(), 276 + by: z.array(z.string()), 277 + }), 278 + }, 279 + apply: ({ scorers }: { scorers: string[] }) => ({ kind: "KBest", k: 1, by: scorers }), 280 + toMiniZinc: () => ({ text: "% pick" }), 281 + doc: { nl: "pick", pseudocode: "pick" }, 282 + } as never, 283 + defaultShadow: { scorers: ["makespan"] }, 284 + }; 285 + 286 + test("a custom collector's refs are validated off the compiled spec, not the param name", () => { 287 + const registry = standardRegistry([pickByDef]); 288 + 289 + const dangling = baseConfig(); 290 + dangling.pipeline.blocks.push({ 291 + instanceId: "pick", 292 + code: "myapp.PickBy.v1", 293 + params: { scorers: ["ghost"] }, 294 + }); 295 + expect(() => fillRunConfig(dangling, registry)).toThrow(GraphValidationError); 296 + 297 + const valid = baseConfig(); 298 + valid.pipeline.blocks.push({ 299 + instanceId: "pick", 300 + code: "myapp.PickBy.v1", 301 + params: { scorers: ["makespan"] }, 302 + }); 303 + expect(() => fillRunConfig(valid, registry)).not.toThrow(); 256 304 }); 257 305 258 306 test("a collector referencing a disabled scorer fails validation", () => {

Configure Feed

Configure Feed