leaderless gossip-based service discovery w/ consul compatibility
idfk, simplify agent? maybe?
+174
-249
+50
-186
agent/agent.go
+50
-186
agent/agent.go
···
25
25
// dns resolver for services
26
26
// read-write exception for maintenance mode?
27
27
// opentelemetry metrics
28
+
// how should cascade handle name conflicts? defaulting to just blowing up
29
+
// for now, but we _could_ take the tailscale route & append -1 or whatever
30
+
// to the node. that would be more user friendly.
28
31
29
32
// key differences from consul
30
33
// - services are gossip'd
···
52
55
import (
53
56
"fmt"
54
57
"os"
55
-
"os/signal"
56
-
"strings"
57
58
"sync"
58
-
"syscall"
59
59
"time"
60
60
61
61
"github.com/hashicorp/memberlist"
···
63
63
"golang.org/x/exp/slog"
64
64
)
65
65
66
-
const (
67
-
// gracefulTimeout controls how long we wait before forcefully terminating
68
-
// note that this value interacts with serfConfig.LeavePropagateDelay
69
-
gracefulTimeout = 10 * time.Second
70
-
)
71
-
72
-
func Run() {
73
-
config := readConfig()
74
-
logger := config.SetupLogger()
75
-
agent := setupAgent(config, logger)
76
-
if err := agent.Start(); err != nil {
77
-
fmt.Println(err)
78
-
os.Exit(1)
79
-
}
80
-
defer agent.Shutdown()
81
-
// join any specified startup nodes
82
-
if err := startupJoin(config, agent); err != nil {
83
-
fmt.Println(err)
84
-
os.Exit(1)
85
-
}
86
-
if err := handleSignals(config, agent); err != nil {
87
-
fmt.Println(err)
88
-
os.Exit(1)
89
-
}
90
-
}
91
-
92
-
// handleSignals blocks until we get an exit-causing signal
93
-
func handleSignals(config *Config, agent *Agent) error {
94
-
signalCh := make(chan os.Signal, 4)
95
-
signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)
96
-
97
-
// Wait for a signal
98
-
var sig os.Signal
99
-
select {
100
-
case s := <-signalCh:
101
-
sig = s
102
-
case <-agent.ShutdownCh():
103
-
// Agent is already shutdown!
104
-
return nil
105
-
}
106
-
agent.logger.Info("caught signal", "signal", sig)
107
-
108
-
// Check if we should do a graceful leave
109
-
graceful := false
110
-
if sig == os.Interrupt || sig == syscall.SIGTERM {
111
-
graceful = true
112
-
}
113
-
114
-
// Bail fast if not doing a graceful leave
115
-
if !graceful {
116
-
agent.logger.Warn("leave cluster with zero grace")
117
-
return nil
118
-
}
66
+
// Agent starts and manages a Serf & adds service discovery (TODO)
67
+
type Agent struct {
68
+
// this is the agent's configuration
69
+
Config *Config
70
+
serfConfig *serf.Config
119
71
120
-
// Attempt a graceful leave
121
-
gracefulCh := make(chan struct{})
122
-
agent.logger.Info("shut down agent gracefully")
123
-
go func() {
124
-
if err := agent.Leave(); err != nil {
125
-
agent.logger.Error("Error: %s", err)
126
-
return
127
-
}
128
-
close(gracefulCh)
129
-
}()
72
+
// This is the underlying Serf we are wrapping
73
+
serf *serf.Serf
130
74
131
-
// Wait for leave or another signal
132
-
select {
133
-
case <-signalCh:
134
-
return fmt.Errorf("idfk")
135
-
case <-time.After(gracefulTimeout):
136
-
return fmt.Errorf("leave timed out")
137
-
case <-gracefulCh:
138
-
return nil
139
-
}
140
-
}
75
+
// We receive serf events on this channel
76
+
eventCh chan serf.Event
77
+
logger *slog.Logger
141
78
142
-
func readConfig() *Config {
143
-
config := DefaultConfig()
144
-
// CASCADE_LOGLEVEL=info
145
-
l := os.Getenv("CASCADE_LOGLEVEL")
146
-
if l != "" {
147
-
config.LogLevel = l
148
-
if l == "DEBUG" {
149
-
// config.
150
-
}
151
-
}
152
-
// CASCADE_BIND=192.168.0.15:12345
153
-
if os.Getenv("CASCADE_BIND") != "" {
154
-
config.BindAddr = os.Getenv("CASCADE_BIND")
155
-
}
156
-
// CASCADE_JOIN=127.0.0.1,127.0.0.5
157
-
if os.Getenv("CASCADE_JOIN") != "" {
158
-
config.StartJoin = strings.Split(os.Getenv("CASCADE_JOIN"), ",")
159
-
}
160
-
// CASCADE_NAME=nostromo.j3s.sh
161
-
if os.Getenv("CASCADE_NAME") != "" {
162
-
config.NodeName = os.Getenv("CASCADE_NAME")
163
-
}
164
-
return config
79
+
// if this channel recieves a signal, cascade shuts down
80
+
shutdown bool
81
+
shutdownCh chan struct{}
82
+
shutdownLock sync.Mutex
165
83
}
166
84
167
-
func startupJoin(config *Config, agent *Agent) error {
168
-
if len(config.StartJoin) == 0 {
169
-
return nil
170
-
}
85
+
// New returns an agent lmao
86
+
func New(config *Config) *Agent {
87
+
agent := Agent{}
88
+
serfConfig := serf.DefaultConfig()
89
+
serfConfig.MemberlistConfig = memberlist.DefaultWANConfig()
171
90
172
-
n, err := agent.Join(config.StartJoin)
173
-
if err != nil {
174
-
return err
175
-
}
176
-
if n > 0 {
177
-
agent.logger.Info("issue join request", "nodes", n)
178
-
}
91
+
// XXX: why do serf and cascade use the same event channel?
92
+
eventCh := make(chan serf.Event, 1024)
93
+
agent.eventCh = eventCh
94
+
serfConfig.EventCh = eventCh
179
95
180
-
return nil
181
-
}
96
+
// TODO: make log level configurable
97
+
// XXX: reconsider this agent-scoped logger? what does consul do?
98
+
// logLogger is a bridge from the log.Logger to slog.Logger
99
+
h := slog.HandlerOptions{Level: slog.LevelDebug}.NewTextHandler(os.Stderr)
100
+
logger := slog.New(h)
101
+
slog.SetDefault(logger)
102
+
agent.logger = logger
103
+
serfConfig.Logger = slog.NewLogLogger(h, slog.LevelInfo)
182
104
183
-
func setupAgent(config *Config, logger *slog.Logger) *Agent {
184
-
bindIP, bindPort, err := config.AddrParts(config.BindAddr)
185
-
if err != nil {
186
-
panic(err)
187
-
}
188
-
189
-
serfConfig := serf.DefaultConfig()
190
-
// logLogger is just a bridge from the old logger to the
191
-
// new slog handler.
192
-
logLogger := slog.NewLogLogger(logger.Handler(), config.GetLogLevel())
193
-
serfConfig.Logger = logLogger
194
-
serfConfig.NodeName = config.NodeName
195
-
serfConfig.ProtocolVersion = uint8(serf.ProtocolVersionMax)
196
-
// TODO: how should cascade handle name conflicts?
197
-
// defaulting to just blowing up for now, but
198
-
// we _could_ take the tailscale route & append
199
-
// -1 or whatever to the node. that would be more user friendly.
200
-
// TODO: some of these serf settings were pulled
105
+
// TODO: some of these serf settings were cargo-culted
201
106
// from consul[1]. re-examine them eventually.
202
107
serfConfig.EnableNameConflictResolution = false
203
108
serfConfig.LeavePropagateDelay = 3 * time.Second
109
+
serfConfig.MemberlistConfig.BindAddr = config.BindAddr.IP.String()
110
+
serfConfig.MemberlistConfig.BindPort = config.BindAddr.Port
111
+
serfConfig.MemberlistConfig.DeadNodeReclaimTime = 30 * time.Second
204
112
serfConfig.MinQueueDepth = 4096
113
+
serfConfig.NodeName = config.NodeName
114
+
serfConfig.ProtocolVersion = uint8(serf.ProtocolVersionMax)
205
115
serfConfig.QueueDepthWarning = 1000000
206
116
serfConfig.ReconnectTimeout = 3 * 24 * time.Hour
207
117
208
-
serfConfig.MemberlistConfig = memberlist.DefaultWANConfig()
209
-
serfConfig.MemberlistConfig.Logger = logLogger
210
-
serfConfig.MemberlistConfig.DeadNodeReclaimTime = 30 * time.Second
211
-
serfConfig.MemberlistConfig.BindAddr = bindIP
212
-
serfConfig.MemberlistConfig.BindPort = bindPort
213
-
214
-
agent := Create(config, serfConfig, logger)
215
-
agent.logger.Info("setup cascade agent", "nodename", config.NodeName, "bindport", bindPort, "bindip", bindIP, "loglevel", config.GetLogLevel())
216
-
return agent
217
-
}
218
-
219
-
// [1]: sources for consul serf tweaks
220
-
// https://github.com/hashicorp/consul/blob/v1.14.4/agent/consul/config.go
221
-
// https://github.com/hashicorp/consul/blob/v1.14.4/lib/serf/serf.go
222
-
223
-
// Agent starts and manages a Serf & adds service discovery (TODO)
224
-
type Agent struct {
225
-
serfConf *serf.Config
226
-
eventCh chan serf.Event
227
-
// This is the underlying Serf we are wrapping
228
-
serf *serf.Serf
229
-
// a lil logger used by the agent, you know how it be
230
-
logger *slog.Logger
231
-
// this channel receiving a signal = shutdown
232
-
shutdown bool
233
-
shutdownCh chan struct{}
234
-
shutdownLock sync.Mutex
235
-
}
236
-
237
-
func Create(config *Config, serfConf *serf.Config, logger *slog.Logger) *Agent {
238
-
// Create a channel to listen for events from Serf
239
-
eventCh := make(chan serf.Event, 64)
240
-
serfConf.EventCh = eventCh
241
-
242
-
// Setup the agent
243
-
agent := &Agent{
244
-
serfConf: serfConf,
245
-
eventCh: eventCh,
246
-
logger: logger,
247
-
shutdownCh: make(chan struct{}),
248
-
}
249
-
250
-
return agent
118
+
agent.Config = config
119
+
agent.serfConfig = serfConfig
120
+
agent.shutdownCh = make(chan struct{})
121
+
return &agent
251
122
}
252
123
253
124
func (a *Agent) eventLoop() {
···
268
139
}
269
140
}
270
141
142
+
// Start creates and starts an agent's serf client, eventloop, api, and more!
143
+
// The agent config should not be modified after an agend has been started.
271
144
func (a *Agent) Start() error {
272
145
// Create serf first
273
-
serf, err := serf.Create(a.serfConf)
146
+
serf, err := serf.Create(a.serfConfig)
274
147
if err != nil {
275
148
return err
276
149
}
···
337
210
return a.shutdownCh
338
211
}
339
212
340
-
func (c *Config) SetupLogger() *slog.Logger {
341
-
opts := slog.HandlerOptions{
342
-
Level: c.GetLogLevel(),
343
-
}
344
-
if c.LogLevel == "DEBUG" {
345
-
opts.AddSource = true
346
-
}
347
-
handler := opts.NewTextHandler(os.Stderr)
348
-
logger := slog.New(handler)
349
-
slog.SetDefault(logger)
350
-
return logger
351
-
}
213
+
// [1]: sources for consul serf tweaks
214
+
// https://github.com/hashicorp/consul/blob/v1.14.4/agent/consul/config.go
215
+
// https://github.com/hashicorp/consul/blob/v1.14.4/lib/serf/serf.go
+10
-61
agent/config.go
+10
-61
agent/config.go
···
1
1
package agent
2
2
3
3
import (
4
-
"fmt"
5
4
"net"
6
5
"os"
7
-
8
-
"golang.org/x/exp/slog"
9
6
)
10
7
11
-
const DefaultBindPort int = 4440
8
+
const DefaultBindPort int = 4443
9
+
const DefaultClientPort int = 4443
12
10
13
11
func DefaultConfig() *Config {
14
12
hostname, err := os.Hostname()
···
16
14
panic(err)
17
15
}
18
16
19
-
return &Config{
20
-
BindAddr: "0.0.0.0",
21
-
ClientAddr: "0.0.0.0",
22
-
LogLevel: "INFO",
23
-
NodeName: hostname,
24
-
}
25
-
}
17
+
// TODO: figure out how to default the listeners
18
+
cfg := Config{}
19
+
cfg.BindAddr = &net.TCPAddr{IP: []byte{0, 0, 0, 0}, Port: DefaultBindPort}
20
+
cfg.ClientAddr = &net.TCPAddr{IP: []byte{127, 0, 0, 1}, Port: DefaultClientPort}
21
+
cfg.NodeName = hostname
26
22
27
-
// Services are built on top of serf
28
-
// tags. The tag format under the hood
29
-
// is service=name=<name>,port=<port>
30
-
type Service struct {
31
-
name string
32
-
port int
23
+
return &cfg
33
24
}
34
25
35
26
type Config struct {
36
-
BindAddr string
37
-
ClientAddr string
38
-
LogLevel string
27
+
BindAddr *net.TCPAddr
28
+
ClientAddr *net.TCPAddr
39
29
NodeName string
40
30
StartJoin []string
41
-
Services []Service
42
-
}
43
-
44
-
// lifted from serf, could be simplified
45
-
func (c *Config) AddrParts(address string) (string, int, error) {
46
-
checkAddr := address
47
-
48
-
START:
49
-
_, _, err := net.SplitHostPort(checkAddr)
50
-
if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
51
-
checkAddr = fmt.Sprintf("%s:%d", checkAddr, DefaultBindPort)
52
-
goto START
53
-
}
54
-
if err != nil {
55
-
return "", 0, err
56
-
}
57
-
58
-
// Get the address
59
-
addr, err := net.ResolveTCPAddr("tcp", checkAddr)
60
-
if err != nil {
61
-
return "", 0, err
62
-
}
63
-
64
-
return addr.IP.String(), addr.Port, nil
65
-
}
66
-
67
-
func (c *Config) GetLogLevel() slog.Level {
68
-
var lvl slog.Level
69
-
if c.LogLevel == "DEBUG" {
70
-
lvl = slog.LevelDebug
71
-
}
72
-
if c.LogLevel == "INFO" {
73
-
lvl = slog.LevelInfo
74
-
}
75
-
if c.LogLevel == "WARN" || c.LogLevel == "WARNING" {
76
-
lvl = slog.LevelWarn
77
-
}
78
-
if c.LogLevel == "ERR" || c.LogLevel == "ERROR" {
79
-
lvl = slog.LevelError
80
-
}
81
-
return lvl
82
31
}
+114
-2
main.go
+114
-2
main.go
···
2
2
3
3
import (
4
4
"fmt"
5
+
"log"
5
6
"os"
7
+
"os/signal"
8
+
"strings"
9
+
"syscall"
10
+
"time"
6
11
7
12
"git.j3s.sh/cascade/agent"
8
13
"git.j3s.sh/cascade/list"
9
14
)
10
15
11
-
// TODO: rename agent to something cooler
12
16
const usage = `cascade agent start a cascade agent
13
17
cascade list|ls list nodes or services
14
18
cascade members show serf cluster members
15
19
cascade rtt estimate latency between nodes
16
20
`
17
21
22
+
// gracefulTimeout controls how long we wait before forcefully terminating
23
+
// note that this value interacts with serf's LeavePropagateDelay config
24
+
const gracefulTimeout = 10 * time.Second
25
+
26
+
// TODO: rename agent to something cooler
18
27
func main() {
19
28
if len(os.Args) == 1 {
20
29
fmt.Fprintf(os.Stderr, "%s", usage)
···
27
36
func run(command string, args []string) {
28
37
switch command {
29
38
case "agent":
30
-
agent.Run()
39
+
handleAgent()
31
40
case "list", "ls":
32
41
list.Run(args)
33
42
default:
···
35
44
os.Exit(1)
36
45
}
37
46
}
47
+
48
+
func handleAgent() {
49
+
config := getAgentConfig()
50
+
agent := agent.New(config)
51
+
if err := agent.Start(); err != nil {
52
+
fmt.Println(err)
53
+
os.Exit(1)
54
+
}
55
+
defer agent.Shutdown()
56
+
// join any specified startup nodes
57
+
if err := startupJoin(agent); err != nil {
58
+
fmt.Println(err)
59
+
os.Exit(1)
60
+
}
61
+
if err := handleSignals(agent); err != nil {
62
+
fmt.Println(err)
63
+
os.Exit(1)
64
+
}
65
+
}
66
+
67
+
// handleSignals blocks until we get an exit-causing signal
68
+
func handleSignals(agent *agent.Agent) error {
69
+
signalCh := make(chan os.Signal, 4)
70
+
signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)
71
+
72
+
// Wait for a signal
73
+
var sig os.Signal
74
+
select {
75
+
case s := <-signalCh:
76
+
sig = s
77
+
case <-agent.ShutdownCh():
78
+
// Agent is already shutdown!
79
+
return nil
80
+
}
81
+
fmt.Fprintf(os.Stderr, "caught signal %s", sig)
82
+
83
+
// Check if we should do a graceful leave
84
+
graceful := false
85
+
if sig == os.Interrupt || sig == syscall.SIGTERM {
86
+
graceful = true
87
+
}
88
+
89
+
// Bail fast if not doing a graceful leave
90
+
if !graceful {
91
+
fmt.Fprintf(os.Stderr, "leave cluster with zero grace")
92
+
return nil
93
+
}
94
+
95
+
// Attempt a graceful leave
96
+
gracefulCh := make(chan struct{})
97
+
fmt.Printf("shutting down gracefully")
98
+
go func() {
99
+
if err := agent.Leave(); err != nil {
100
+
fmt.Println("Error: ", err)
101
+
return
102
+
}
103
+
close(gracefulCh)
104
+
}()
105
+
106
+
// Wait for leave or another signal
107
+
select {
108
+
case <-signalCh:
109
+
return fmt.Errorf("idfk")
110
+
case <-time.After(gracefulTimeout):
111
+
return fmt.Errorf("leave timed out")
112
+
case <-gracefulCh:
113
+
return nil
114
+
}
115
+
}
116
+
117
+
func startupJoin(a *agent.Agent) error {
118
+
if len(a.Config.StartJoin) == 0 {
119
+
return nil
120
+
}
121
+
122
+
n, err := a.Join(a.Config.StartJoin)
123
+
if err != nil {
124
+
return err
125
+
}
126
+
if n > 0 {
127
+
log.Printf("issue join request nodes=%d\n", n)
128
+
}
129
+
130
+
return nil
131
+
}
132
+
133
+
func getAgentConfig() *agent.Config {
134
+
config := agent.DefaultConfig()
135
+
// CASCADE_BIND=192.168.0.15:12345
136
+
if os.Getenv("CASCADE_BIND") != "" {
137
+
//TODO
138
+
// config.BindAddr = os.Getenv("CASCADE_BIND")
139
+
}
140
+
// CASCADE_JOIN=127.0.0.1,127.0.0.5
141
+
if os.Getenv("CASCADE_JOIN") != "" {
142
+
config.StartJoin = strings.Split(os.Getenv("CASCADE_JOIN"), ",")
143
+
}
144
+
// CASCADE_NAME=nostromo.j3s.sh
145
+
if os.Getenv("CASCADE_NAME") != "" {
146
+
config.NodeName = os.Getenv("CASCADE_NAME")
147
+
}
148
+
return config
149
+
}