[READ-ONLY] Mirror of https://github.com/Schniz/sprits. type-safe concurrent pipeline for ambitious applications
wip
Gal Schlezinger
(Oct 26, 2024, 2:02 PM -0700)
c81663de
+482
+175
.gitignore
+175
.gitignore
···
1
+
# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
2
+
3
+
# Logs
4
+
5
+
logs
6
+
_.log
7
+
npm-debug.log_
8
+
yarn-debug.log*
9
+
yarn-error.log*
10
+
lerna-debug.log*
11
+
.pnpm-debug.log*
12
+
13
+
# Caches
14
+
15
+
.cache
16
+
17
+
# Diagnostic reports (https://nodejs.org/api/report.html)
18
+
19
+
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
20
+
21
+
# Runtime data
22
+
23
+
pids
24
+
_.pid
25
+
_.seed
26
+
*.pid.lock
27
+
28
+
# Directory for instrumented libs generated by jscoverage/JSCover
29
+
30
+
lib-cov
31
+
32
+
# Coverage directory used by tools like istanbul
33
+
34
+
coverage
35
+
*.lcov
36
+
37
+
# nyc test coverage
38
+
39
+
.nyc_output
40
+
41
+
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
42
+
43
+
.grunt
44
+
45
+
# Bower dependency directory (https://bower.io/)
46
+
47
+
bower_components
48
+
49
+
# node-waf configuration
50
+
51
+
.lock-wscript
52
+
53
+
# Compiled binary addons (https://nodejs.org/api/addons.html)
54
+
55
+
build/Release
56
+
57
+
# Dependency directories
58
+
59
+
node_modules/
60
+
jspm_packages/
61
+
62
+
# Snowpack dependency directory (https://snowpack.dev/)
63
+
64
+
web_modules/
65
+
66
+
# TypeScript cache
67
+
68
+
*.tsbuildinfo
69
+
70
+
# Optional npm cache directory
71
+
72
+
.npm
73
+
74
+
# Optional eslint cache
75
+
76
+
.eslintcache
77
+
78
+
# Optional stylelint cache
79
+
80
+
.stylelintcache
81
+
82
+
# Microbundle cache
83
+
84
+
.rpt2_cache/
85
+
.rts2_cache_cjs/
86
+
.rts2_cache_es/
87
+
.rts2_cache_umd/
88
+
89
+
# Optional REPL history
90
+
91
+
.node_repl_history
92
+
93
+
# Output of 'npm pack'
94
+
95
+
*.tgz
96
+
97
+
# Yarn Integrity file
98
+
99
+
.yarn-integrity
100
+
101
+
# dotenv environment variable files
102
+
103
+
.env
104
+
.env.development.local
105
+
.env.test.local
106
+
.env.production.local
107
+
.env.local
108
+
109
+
# parcel-bundler cache (https://parceljs.org/)
110
+
111
+
.parcel-cache
112
+
113
+
# Next.js build output
114
+
115
+
.next
116
+
out
117
+
118
+
# Nuxt.js build / generate output
119
+
120
+
.nuxt
121
+
dist
122
+
123
+
# Gatsby files
124
+
125
+
# Comment in the public line in if your project uses Gatsby and not Next.js
126
+
127
+
# https://nextjs.org/blog/next-9-1#public-directory-support
128
+
129
+
# public
130
+
131
+
# vuepress build output
132
+
133
+
.vuepress/dist
134
+
135
+
# vuepress v2.x temp and cache directory
136
+
137
+
.temp
138
+
139
+
# Docusaurus cache and generated files
140
+
141
+
.docusaurus
142
+
143
+
# Serverless directories
144
+
145
+
.serverless/
146
+
147
+
# FuseBox cache
148
+
149
+
.fusebox/
150
+
151
+
# DynamoDB Local files
152
+
153
+
.dynamodb/
154
+
155
+
# TernJS port file
156
+
157
+
.tern-port
158
+
159
+
# Stores VSCode versions used for testing VSCode extensions
160
+
161
+
.vscode-test
162
+
163
+
# yarn v2
164
+
165
+
.yarn/cache
166
+
.yarn/unplugged
167
+
.yarn/build-state.yml
168
+
.yarn/install-state.gz
169
+
.pnp.*
170
+
171
+
# IntelliJ based IDEs
172
+
.idea
173
+
174
+
# Finder (MacOS) folder config
175
+
.DS_Store
+15
README.md
+15
README.md
+30
biome.json
+30
biome.json
···
1
+
{
2
+
"$schema": "https://biomejs.dev/schemas/1.9.4/schema.json",
3
+
"vcs": {
4
+
"enabled": false,
5
+
"clientKind": "git",
6
+
"useIgnoreFile": false
7
+
},
8
+
"files": {
9
+
"ignoreUnknown": false,
10
+
"ignore": []
11
+
},
12
+
"formatter": {
13
+
"enabled": true,
14
+
"indentStyle": "tab"
15
+
},
16
+
"organizeImports": {
17
+
"enabled": true
18
+
},
19
+
"linter": {
20
+
"enabled": true,
21
+
"rules": {
22
+
"recommended": true
23
+
}
24
+
},
25
+
"javascript": {
26
+
"formatter": {
27
+
"quoteStyle": "double"
28
+
}
29
+
}
30
+
}
bun.lockb
bun.lockb
This is a binary file and will not be displayed.
+17
package.json
+17
package.json
···
1
+
{
2
+
"name": "sprits",
3
+
"module": "src/index.ts",
4
+
"type": "module",
5
+
"devDependencies": {
6
+
"@biomejs/biome": "^1.9.4",
7
+
"@types/bun": "latest"
8
+
},
9
+
"peerDependencies": {
10
+
"typescript": "^5.0.0"
11
+
},
12
+
"dependencies": {
13
+
"effect": "^3.10.4",
14
+
"ts-graphviz": "^2.1.4"
15
+
}
16
+
}
17
+
+27
tsconfig.json
+27
tsconfig.json
···
1
+
{
2
+
"compilerOptions": {
3
+
// Enable latest features
4
+
"lib": ["ESNext", "DOM"],
5
+
"target": "ESNext",
6
+
"module": "ESNext",
7
+
"moduleDetection": "force",
8
+
"jsx": "react-jsx",
9
+
"allowJs": true,
10
+
11
+
// Bundler mode
12
+
"moduleResolution": "bundler",
13
+
"allowImportingTsExtensions": true,
14
+
"verbatimModuleSyntax": true,
15
+
"noEmit": true,
16
+
17
+
// Best practices
18
+
"strict": true,
19
+
"skipLibCheck": true,
20
+
"noFallthroughCasesInSwitch": true,
21
+
22
+
// Some stricter flags (disabled by default)
23
+
"noUnusedLocals": false,
24
+
"noUnusedParameters": false,
25
+
"noPropertyAccessFromIndexSignature": false
26
+
}
27
+
}
+85
src/index.ts
+85
src/index.ts
···
1
+
import { Effect, Console, type Duration } from "effect";
2
+
import * as Step from "./step";
3
+
4
+
class SomeService extends Effect.Tag("SomeService")<
5
+
SomeService,
6
+
{ hello: string }
7
+
>() {}
8
+
9
+
class Random extends Effect.Tag("Random")<SomeService, { next: number }>() {}
10
+
11
+
const debugging =
12
+
(dur: Duration.DurationInput) =>
13
+
<A, E, R>(effect: Effect.Effect<A, E, R>) =>
14
+
Effect.gen(function* () {
15
+
yield* Effect.logInfo("start");
16
+
const value = yield* effect;
17
+
yield* Effect.sleep(dur);
18
+
yield* Effect.logInfo("end");
19
+
return value;
20
+
});
21
+
22
+
const grandparent = Step.make({
23
+
name: "grandparent",
24
+
inputs: [],
25
+
run: Effect.gen(function* () {
26
+
yield* Console.log("hi from grandparent");
27
+
return 666;
28
+
}).pipe(debugging("400 millis")),
29
+
});
30
+
31
+
const step1 = Step.make({
32
+
name: "step1",
33
+
inputs: [grandparent],
34
+
run: Effect.gen(function* () {
35
+
yield* SomeService;
36
+
if ((yield* Random).next > 0.5) {
37
+
yield* Effect.fail("oh no (from step 1)" as const);
38
+
}
39
+
yield* Console.log("hi from step1");
40
+
return "step1" as const;
41
+
}).pipe(debugging("400 millis")),
42
+
});
43
+
44
+
const step_parent = Step.make({
45
+
name: "step_parent",
46
+
inputs: [],
47
+
run: Effect.gen(function* () {
48
+
yield* SomeService;
49
+
if ((yield* Random).next > 0.5) {
50
+
yield* Effect.fail("oh no (parent)" as const);
51
+
}
52
+
yield* Console.log("hi from step_parent");
53
+
return "parent" as const;
54
+
}).pipe(debugging("400 millis")),
55
+
});
56
+
57
+
const step2 = Step.make({
58
+
name: "step2",
59
+
inputs: [step1, step_parent],
60
+
run: Effect.gen(function* () {
61
+
if ((yield* Random).next > 0.5) {
62
+
yield* Effect.fail("oh no" as const);
63
+
}
64
+
yield* step1.read;
65
+
yield* step_parent.read;
66
+
yield* Console.log("hi from step2");
67
+
return "step2" as const;
68
+
}).pipe(debugging("400 millis")),
69
+
});
70
+
71
+
const beforeUnwrapping = step2.run;
72
+
// ^?
73
+
74
+
const effect = Step.run(step2);
75
+
// ^?
76
+
77
+
await effect.pipe(
78
+
Effect.provideService(SomeService, { hello: "world" }),
79
+
Effect.provideService(Random, { next: 0 }),
80
+
Effect.tapBoth({
81
+
onSuccess: Console.log,
82
+
onFailure: Console.error,
83
+
}),
84
+
Effect.runPromise,
85
+
);
+133
src/step.ts
+133
src/step.ts
···
1
+
import { Context, Deferred, Effect } from "effect";
2
+
import { type NodeModel, digraph, toDot as toGraphvizDot } from "ts-graphviz";
3
+
4
+
export const make = <
5
+
const Name extends string,
6
+
A,
7
+
E,
8
+
R,
9
+
const Inputs extends AnyStep[],
10
+
>(
11
+
opts: Omit<Step<Name, A, E, R, Inputs>, "read">,
12
+
): Step<Name, A, E, R, Inputs> => {
13
+
return {
14
+
...opts,
15
+
read: Context.GenericTag<StepContext<Name>, A>(`step/${opts.name}`),
16
+
};
17
+
};
18
+
19
+
interface Step<Name extends string, A, E, R, Inputs extends AnyStep[]> {
20
+
name: Name;
21
+
readonly inputs: Inputs;
22
+
run: Effect.Effect<A, E, R>;
23
+
read: Effect.Effect<A, E, StepContext<Name> | R>;
24
+
}
25
+
26
+
type StepContext<Name extends string> = `step/${Name}`;
27
+
28
+
// biome-ignore lint/suspicious/noExplicitAny: no one cares
29
+
type AnyStep = Step<any, any, any, any, any[]>;
30
+
// biome-ignore lint/suspicious/noExplicitAny: no one cares
31
+
type Tail<Ts> = Ts extends [any, ...infer T] ? T : [];
32
+
33
+
type _ParentStepContexts<Ss extends AnyStep[], Current = never> = {
34
+
empty: Current;
35
+
nonempty: _ParentStepContexts<Tail<Ss>, StepContext<Ss[0]["name"]> | Current>;
36
+
}[Ss extends [] ? "empty" : "nonempty"];
37
+
type ParentStepContexts<S extends AnyStep> = _ParentStepContexts<S["inputs"]>;
38
+
39
+
const getDependencies = (s: AnyStep) =>
40
+
Effect.gen(function* () {
41
+
const dependencies = new Map<
42
+
AnyStep,
43
+
{ deps: Set<AnyStep>; whenResolved: Deferred.Deferred<unknown, unknown> }
44
+
>();
45
+
const stack = [s];
46
+
const visited = new Set<AnyStep>();
47
+
48
+
while (true) {
49
+
const item = stack.pop();
50
+
if (!item) break;
51
+
visited.add(item);
52
+
53
+
const current = dependencies.get(item) || {
54
+
deps: new Set<AnyStep>(),
55
+
whenResolved: yield* Deferred.make<unknown, unknown>(),
56
+
};
57
+
dependencies.set(item, current);
58
+
59
+
for (const input of item.inputs) {
60
+
current.deps.add(input);
61
+
62
+
if (!visited.has(input)) {
63
+
stack.push(input);
64
+
}
65
+
}
66
+
}
67
+
return dependencies;
68
+
});
69
+
70
+
export const run = <S extends AnyStep>(
71
+
step: S,
72
+
): Effect.Effect<
73
+
Effect.Effect.Success<S["run"]>,
74
+
Effect.Effect.Error<S["run"]>,
75
+
Exclude<Effect.Effect.Context<S["run"]>, ParentStepContexts<S>>
76
+
> =>
77
+
// @ts-expect-error
78
+
Effect.gen(function* () {
79
+
yield* Effect.yieldNow();
80
+
const dependencies = yield* getDependencies(step);
81
+
82
+
yield* Effect.forEach(
83
+
dependencies,
84
+
([step, { deps, whenResolved }]) =>
85
+
Effect.gen(function* () {
86
+
let context = Context.empty();
87
+
88
+
if (deps.size) {
89
+
yield* Effect.forEach(
90
+
deps,
91
+
(dep) =>
92
+
Effect.gen(function* () {
93
+
const dependency = yield* Effect.fromNullable(
94
+
dependencies.get(dep),
95
+
).pipe(Effect.orDieWith(() => "Dependency not found?!"));
96
+
const value = yield* dependency.whenResolved;
97
+
const ctx = dep.read as Context.Tag<unknown, unknown>;
98
+
context = Context.add(context, ctx, value);
99
+
}),
100
+
{
101
+
discard: true,
102
+
},
103
+
);
104
+
}
105
+
106
+
yield* step.run.pipe(
107
+
Effect.provide(context),
108
+
Effect.onExit((exit) => Deferred.done(whenResolved, exit)),
109
+
Effect.annotateLogs({ step: step.name }),
110
+
);
111
+
}),
112
+
{ concurrency: "unbounded" },
113
+
);
114
+
});
115
+
116
+
export const toDot = (step: AnyStep) =>
117
+
Effect.gen(function* () {
118
+
const dependencies = yield* getDependencies(step);
119
+
120
+
const graph = digraph((g) => {
121
+
const nodes = {} as Record<string, NodeModel>;
122
+
for (const [step] of dependencies) {
123
+
nodes[step.name] = g.node(step.name);
124
+
}
125
+
for (const [step, { deps }] of dependencies) {
126
+
for (const dep of deps) {
127
+
g.edge([nodes[dep.name], nodes[step.name]]);
128
+
}
129
+
}
130
+
});
131
+
132
+
return toGraphvizDot(graph);
133
+
});