···11-package main
22-33-import (
44- "fmt"
55- "sync"
66-77- "github.com/hashicorp/serf/serf"
88- "golang.org/x/exp/slog"
99-)
1010-1111-// Agent starts and manages a Serf & adds service discovery (TODO)
1212-type Agent struct {
1313- serfConf *serf.Config
1414- eventCh chan serf.Event
1515- // This is the underlying Serf we are wrapping
1616- serf *serf.Serf
1717- // a lil logger used by the agent, you know how it be
1818- logger *slog.Logger
1919- // this channel receiving a signal = shutdown
2020- shutdown bool
2121- shutdownCh chan struct{}
2222- shutdownLock sync.Mutex
2323-}
2424-2525-func Create(config *Config, serfConf *serf.Config, logger *slog.Logger) *Agent {
2626- // Create a channel to listen for events from Serf
2727- eventCh := make(chan serf.Event, 64)
2828- serfConf.EventCh = eventCh
2929-3030- // Setup the agent
3131- agent := &Agent{
3232- serfConf: serfConf,
3333- eventCh: eventCh,
3434- logger: logger,
3535- shutdownCh: make(chan struct{}),
3636- }
3737-3838- return agent
3939-}
4040-4141-func (a *Agent) eventLoop() {
4242- serfShutdownCh := a.serf.ShutdownCh()
4343- for {
4444- select {
4545- case e := <-a.eventCh:
4646- a.logger.Info("receive event", "event", e.String())
4747-4848- case <-serfShutdownCh:
4949- a.logger.Info("detect serf shutdown, turning off event loop")
5050- a.Shutdown()
5151- return
5252-5353- case <-a.shutdownCh:
5454- return
5555- }
5656- }
5757-}
5858-5959-func (a *Agent) Start() error {
6060- a.logger.Info("start serf agent")
6161-6262- // Create serf first
6363- serf, err := serf.Create(a.serfConf)
6464- if err != nil {
6565- return err
6666- }
6767- a.serf = serf
6868-6969- // Start event loop
7070- go a.eventLoop()
7171- return nil
7272-}
7373-7474-// Shutdown closes this agent and all of its processes. Should be preceded
7575-// by a Leave for a graceful shutdown.
7676-func (a *Agent) Shutdown() error {
7777- a.shutdownLock.Lock()
7878- defer a.shutdownLock.Unlock()
7979-8080- if a.shutdown {
8181- return nil
8282- }
8383-8484- if a.serf == nil {
8585- goto EXIT
8686- }
8787-8888- a.logger.Info("request serf shutdown")
8989- if err := a.serf.Shutdown(); err != nil {
9090- return err
9191- }
9292-9393-EXIT:
9494- a.logger.Info("complete serf shutdown")
9595- a.shutdown = true
9696- close(a.shutdownCh)
9797- return nil
9898-}
9999-100100-// Join asks the Serf instance to join. See the Serf.Join function.
101101-func (a *Agent) Join(addrs []string) (n int, err error) {
102102- a.logger.Info("issue join request", "addrs", addrs)
103103- // we always ignore old events because cascade don't
104104- // care about the past
105105- n, err = a.serf.Join(addrs, true)
106106- if err != nil {
107107- return n, fmt.Errorf("Error joining: %v\n", err)
108108- }
109109- // TODO: when joining fails, we don't get an error here - serf & memberlist
110110- // just print to stdout and serf.Join returns without issue.
111111- return n, err
112112-}
113113-114114-// Leave prepares for a graceful shutdown of the agent and its processes
115115-func (a *Agent) Leave() error {
116116- if a.serf == nil {
117117- return nil
118118- }
119119-120120- a.logger.Info("request graceful serf leave")
121121- return a.serf.Leave()
122122-}
123123-124124-// ShutdownCh returns a channel that can be selected to wait
125125-// for the agent to perform a shutdown.
126126-func (a *Agent) ShutdownCh() <-chan struct{} {
127127- return a.shutdownCh
128128-}
+351
agent/agent.go
···11+// design touchstones
22+// simple to configure/implement
33+// - minimal configurable options
44+// - sparse use of commands/flags/env vars
55+// - fast to start/join a cluster
66+// - http-only api
77+// api auth/encryption is best implemented by a reverse proxy (like caddy or nginx)
88+// - no RPC (it's too annoying to deal with)
99+// predictable state
1010+// - no runtime configuration allowed (no reload support)
1111+// for a gossip cluster to work, nodes in the cluster must
1212+// be the source of truth. we do not allow the user to change
1313+// the cluster's view of reality in any way - the user may
1414+// only inspect it.
1515+// - HTTP API only supports read-only commands
1616+// this ensures that cascade's starting state never
1717+// diverges from its running state.
1818+// (restarts are very fast + interruptionless)
1919+// this also helps with security
2020+// one massive global cluster
2121+// - for simplicity, operability, and because gossip can handle it
2222+// compatability
2323+// - attempt to have a consul-compatible API where it matters?
2424+// todo
2525+// dns resolver for services
2626+// read-write exception for maintenance mode?
2727+// opentelemetry metrics
2828+2929+// key differences from consul
3030+// - services are gossip'd
3131+3232+// in cascade, node == member, and catalog == agent
3333+// agent api (http-only)
3434+// GET /v1/agent/metrics -> opentelemetry metrics for this agent
3535+// GET /v1/agent/members -> show serf cluster membership state
3636+// GET /v1/agent/services -> show services this agent owns
3737+// aliases: GET /v1/catalog/nodes
3838+// catalog api (http & dns)
3939+// GET /v1/catalog/nodes
4040+// GET /v1/catalog/services
4141+// dns: dig @127.0.0.1 -p 8600 nostromo.node.cascade. ANY
4242+// dns: dig @127.0.0.1 -p 8600 redis.service.cascade. ANY
4343+4444+// consul uses /catalog because the masters maintain a service
4545+// catalog separate than the agent state
4646+4747+// in cascade, the agents _are_ expected to maintain service catalogues,
4848+// so we do away with the catalog concept entirely.
4949+5050+package agent
5151+5252+import (
5353+ "fmt"
5454+ "os"
5555+ "os/signal"
5656+ "strings"
5757+ "sync"
5858+ "syscall"
5959+ "time"
6060+6161+ "github.com/hashicorp/memberlist"
6262+ "github.com/hashicorp/serf/serf"
6363+ "golang.org/x/exp/slog"
6464+)
6565+6666+const (
6767+ // gracefulTimeout controls how long we wait before forcefully terminating
6868+ // note that this value interacts with serfConfig.LeavePropagateDelay
6969+ gracefulTimeout = 10 * time.Second
7070+)
7171+7272+func Run() {
7373+ config := readConfig()
7474+ logger := config.SetupLogger()
7575+ agent := setupAgent(config, logger)
7676+ if err := agent.Start(); err != nil {
7777+ fmt.Println(err)
7878+ os.Exit(1)
7979+ }
8080+ defer agent.Shutdown()
8181+ // join any specified startup nodes
8282+ if err := startupJoin(config, agent); err != nil {
8383+ fmt.Println(err)
8484+ os.Exit(1)
8585+ }
8686+ if err := handleSignals(config, agent); err != nil {
8787+ fmt.Println(err)
8888+ os.Exit(1)
8989+ }
9090+}
9191+9292+// handleSignals blocks until we get an exit-causing signal
9393+func handleSignals(config *Config, agent *Agent) error {
9494+ signalCh := make(chan os.Signal, 4)
9595+ signal.Notify(signalCh, os.Interrupt, syscall.SIGTERM)
9696+9797+ // Wait for a signal
9898+ var sig os.Signal
9999+ select {
100100+ case s := <-signalCh:
101101+ sig = s
102102+ case <-agent.ShutdownCh():
103103+ // Agent is already shutdown!
104104+ return nil
105105+ }
106106+ agent.logger.Info("caught signal", "signal", sig)
107107+108108+ // Check if we should do a graceful leave
109109+ graceful := false
110110+ if sig == os.Interrupt || sig == syscall.SIGTERM {
111111+ graceful = true
112112+ }
113113+114114+ // Bail fast if not doing a graceful leave
115115+ if !graceful {
116116+ agent.logger.Warn("leave cluster with zero grace")
117117+ return nil
118118+ }
119119+120120+ // Attempt a graceful leave
121121+ gracefulCh := make(chan struct{})
122122+ agent.logger.Info("shut down agent gracefully")
123123+ go func() {
124124+ if err := agent.Leave(); err != nil {
125125+ agent.logger.Error("Error: %s", err)
126126+ return
127127+ }
128128+ close(gracefulCh)
129129+ }()
130130+131131+ // Wait for leave or another signal
132132+ select {
133133+ case <-signalCh:
134134+ return fmt.Errorf("idfk")
135135+ case <-time.After(gracefulTimeout):
136136+ return fmt.Errorf("leave timed out")
137137+ case <-gracefulCh:
138138+ return nil
139139+ }
140140+}
141141+142142+func readConfig() *Config {
143143+ config := DefaultConfig()
144144+ // CASCADE_LOGLEVEL=info
145145+ l := os.Getenv("CASCADE_LOGLEVEL")
146146+ if l != "" {
147147+ config.LogLevel = l
148148+ if l == "DEBUG" {
149149+ // config.
150150+ }
151151+ }
152152+ // CASCADE_BIND=192.168.0.15:12345
153153+ if os.Getenv("CASCADE_BIND") != "" {
154154+ config.BindAddr = os.Getenv("CASCADE_BIND")
155155+ }
156156+ // CASCADE_JOIN=127.0.0.1,127.0.0.5
157157+ if os.Getenv("CASCADE_JOIN") != "" {
158158+ config.StartJoin = strings.Split(os.Getenv("CASCADE_JOIN"), ",")
159159+ }
160160+ // CASCADE_NAME=nostromo.j3s.sh
161161+ if os.Getenv("CASCADE_NAME") != "" {
162162+ config.NodeName = os.Getenv("CASCADE_NAME")
163163+ }
164164+ return config
165165+}
166166+167167+func startupJoin(config *Config, agent *Agent) error {
168168+ if len(config.StartJoin) == 0 {
169169+ return nil
170170+ }
171171+172172+ n, err := agent.Join(config.StartJoin)
173173+ if err != nil {
174174+ return err
175175+ }
176176+ if n > 0 {
177177+ agent.logger.Info("issue join request", "nodes", n)
178178+ }
179179+180180+ return nil
181181+}
182182+183183+func setupAgent(config *Config, logger *slog.Logger) *Agent {
184184+ bindIP, bindPort, err := config.AddrParts(config.BindAddr)
185185+ if err != nil {
186186+ panic(err)
187187+ }
188188+189189+ serfConfig := serf.DefaultConfig()
190190+ // logLogger is just a bridge from the old logger to the
191191+ // new slog handler.
192192+ logLogger := slog.NewLogLogger(logger.Handler(), config.GetLogLevel())
193193+ serfConfig.Logger = logLogger
194194+ serfConfig.NodeName = config.NodeName
195195+ serfConfig.ProtocolVersion = uint8(serf.ProtocolVersionMax)
196196+ // TODO: how should cascade handle name conflicts?
197197+ // defaulting to just blowing up for now, but
198198+ // we _could_ take the tailscale route & append
199199+ // -1 or whatever to the node. that would be more user friendly.
200200+ // TODO: some of these serf settings were pulled
201201+ // from consul[1]. re-examine them eventually.
202202+ serfConfig.EnableNameConflictResolution = false
203203+ serfConfig.LeavePropagateDelay = 3 * time.Second
204204+ serfConfig.MinQueueDepth = 4096
205205+ serfConfig.QueueDepthWarning = 1000000
206206+ serfConfig.ReconnectTimeout = 3 * 24 * time.Hour
207207+208208+ serfConfig.MemberlistConfig = memberlist.DefaultWANConfig()
209209+ serfConfig.MemberlistConfig.Logger = logLogger
210210+ serfConfig.MemberlistConfig.DeadNodeReclaimTime = 30 * time.Second
211211+ serfConfig.MemberlistConfig.BindAddr = bindIP
212212+ serfConfig.MemberlistConfig.BindPort = bindPort
213213+214214+ agent := Create(config, serfConfig, logger)
215215+ agent.logger.Info("setup cascade agent", "nodename", config.NodeName, "bindport", bindPort, "bindip", bindIP, "loglevel", config.GetLogLevel())
216216+ return agent
217217+}
218218+219219+// [1]: sources for consul serf tweaks
220220+// https://github.com/hashicorp/consul/blob/v1.14.4/agent/consul/config.go
221221+// https://github.com/hashicorp/consul/blob/v1.14.4/lib/serf/serf.go
222222+223223+// Agent starts and manages a Serf & adds service discovery (TODO)
224224+type Agent struct {
225225+ serfConf *serf.Config
226226+ eventCh chan serf.Event
227227+ // This is the underlying Serf we are wrapping
228228+ serf *serf.Serf
229229+ // a lil logger used by the agent, you know how it be
230230+ logger *slog.Logger
231231+ // this channel receiving a signal = shutdown
232232+ shutdown bool
233233+ shutdownCh chan struct{}
234234+ shutdownLock sync.Mutex
235235+}
236236+237237+func Create(config *Config, serfConf *serf.Config, logger *slog.Logger) *Agent {
238238+ // Create a channel to listen for events from Serf
239239+ eventCh := make(chan serf.Event, 64)
240240+ serfConf.EventCh = eventCh
241241+242242+ // Setup the agent
243243+ agent := &Agent{
244244+ serfConf: serfConf,
245245+ eventCh: eventCh,
246246+ logger: logger,
247247+ shutdownCh: make(chan struct{}),
248248+ }
249249+250250+ return agent
251251+}
252252+253253+func (a *Agent) eventLoop() {
254254+ serfShutdownCh := a.serf.ShutdownCh()
255255+ for {
256256+ select {
257257+ case e := <-a.eventCh:
258258+ a.logger.Info("receive event", "event", e.String())
259259+260260+ case <-serfShutdownCh:
261261+ a.logger.Info("detect serf shutdown, turning off event loop")
262262+ a.Shutdown()
263263+ return
264264+265265+ case <-a.shutdownCh:
266266+ return
267267+ }
268268+ }
269269+}
270270+271271+func (a *Agent) Start() error {
272272+ // Create serf first
273273+ serf, err := serf.Create(a.serfConf)
274274+ if err != nil {
275275+ return err
276276+ }
277277+ a.serf = serf
278278+279279+ // Start event loop
280280+ go a.eventLoop()
281281+ return nil
282282+}
283283+284284+// Shutdown closes this agent and all of its processes. Should be preceded
285285+// by a Leave for a graceful shutdown.
286286+func (a *Agent) Shutdown() error {
287287+ a.shutdownLock.Lock()
288288+ defer a.shutdownLock.Unlock()
289289+290290+ if a.shutdown {
291291+ return nil
292292+ }
293293+294294+ if a.serf == nil {
295295+ goto EXIT
296296+ }
297297+298298+ a.logger.Info("request serf shutdown")
299299+ if err := a.serf.Shutdown(); err != nil {
300300+ return err
301301+ }
302302+303303+EXIT:
304304+ a.logger.Info("complete serf shutdown")
305305+ a.shutdown = true
306306+ close(a.shutdownCh)
307307+ return nil
308308+}
309309+310310+// Join asks the Serf instance to join. See the Serf.Join function.
311311+func (a *Agent) Join(addrs []string) (n int, err error) {
312312+ a.logger.Info("issue join request", "addrs", addrs)
313313+ // we always ignore old events because cascade don't
314314+ // care about the past
315315+ n, err = a.serf.Join(addrs, true)
316316+ if err != nil {
317317+ return n, fmt.Errorf("Error joining: %v\n", err)
318318+ }
319319+ // TODO: when joining fails, we don't get an error here - serf & memberlist
320320+ // just print to stdout and serf.Join returns without issue.
321321+ return n, err
322322+}
323323+324324+// Leave prepares for a graceful shutdown of the agent and its processes
325325+func (a *Agent) Leave() error {
326326+ if a.serf == nil {
327327+ return nil
328328+ }
329329+330330+ a.logger.Info("request graceful serf leave")
331331+ return a.serf.Leave()
332332+}
333333+334334+// ShutdownCh returns a channel that can be selected to wait
335335+// for the agent to perform a shutdown.
336336+func (a *Agent) ShutdownCh() <-chan struct{} {
337337+ return a.shutdownCh
338338+}
339339+340340+func (c *Config) SetupLogger() *slog.Logger {
341341+ opts := slog.HandlerOptions{
342342+ Level: c.GetLogLevel(),
343343+ }
344344+ if c.LogLevel == "DEBUG" {
345345+ opts.AddSource = true
346346+ }
347347+ handler := opts.NewTextHandler(os.Stderr)
348348+ logger := slog.New(handler)
349349+ slog.SetDefault(logger)
350350+ return logger
351351+}