leaderless gossip-based service discovery w/ consul compatibility
5

Configure Feed

Select the types of activity you want to include in your feed.

import preliminary API structs & methods

Jes Olson (Feb 20, 2023, 7:00 PM -0800) 8b9dc8ea 2d61578f

+649
+106
api/agent.go
··· 1 + package api 2 + 3 + // AgentMember represents a cluster member known to the agent 4 + type AgentMember struct { 5 + Name string 6 + Addr string 7 + Port uint16 8 + Tags map[string]string 9 + // Status of the Member which corresponds to github.com/hashicorp/serf/serf.MemberStatus 10 + // Value is one of: 11 + // 12 + // AgentMemberNone = 0 13 + // AgentMemberAlive = 1 14 + // AgentMemberLeaving = 2 15 + // AgentMemberLeft = 3 16 + // AgentMemberFailed = 4 17 + Status int 18 + ProtocolMin uint8 19 + ProtocolMax uint8 20 + ProtocolCur uint8 21 + DelegateMin uint8 22 + DelegateMax uint8 23 + DelegateCur uint8 24 + } 25 + 26 + // XXX: this may be needed for consul compatability, but I'm leaving it 27 + // out for now in case it isn't. 28 + // const AllSegments = "_all" 29 + 30 + // Agent can be used to query the Agent endpoints 31 + type Agent struct { 32 + c *Client 33 + 34 + // cache the node name 35 + nodeName string 36 + } 37 + 38 + // Agent returns a handle to the agent endpoints 39 + func (c *Client) Agent() *Agent { 40 + return &Agent{c: c} 41 + } 42 + 43 + // Self is used to query the agent we are speaking to for 44 + // information about itself 45 + func (a *Agent) Self() (map[string]map[string]interface{}, error) { 46 + r := a.c.newRequest("GET", "/v1/agent/self") 47 + _, resp, err := requireOK(a.c.doRequest(r)) 48 + if err != nil { 49 + return nil, err 50 + } 51 + defer closeResponseBody(resp) 52 + 53 + var out map[string]map[string]interface{} 54 + if err := decodeBody(resp, &out); err != nil { 55 + return nil, err 56 + } 57 + return out, nil 58 + } 59 + 60 + // Host is used to retrieve information about the host the 61 + // agent is running on such as CPU, memory, and disk. Requires 62 + // a operator:read ACL token. 63 + func (a *Agent) Host() (map[string]interface{}, error) { 64 + r := a.c.newRequest("GET", "/v1/agent/host") 65 + _, resp, err := requireOK(a.c.doRequest(r)) 66 + if err != nil { 67 + return nil, err 68 + } 69 + defer closeResponseBody(resp) 70 + 71 + var out map[string]interface{} 72 + if err := decodeBody(resp, &out); err != nil { 73 + return nil, err 74 + } 75 + return out, nil 76 + } 77 + 78 + // NodeName is used to get the node name of the agent 79 + func (a *Agent) NodeName() (string, error) { 80 + if a.nodeName != "" { 81 + return a.nodeName, nil 82 + } 83 + info, err := a.Self() 84 + if err != nil { 85 + return "", err 86 + } 87 + name := info["Config"]["NodeName"].(string) 88 + a.nodeName = name 89 + return name, nil 90 + } 91 + 92 + // Members returns the known serf gossip members 93 + func (a *Agent) Members() ([]*AgentMember, error) { 94 + r := a.c.newRequest("GET", "/v1/agent/members") 95 + _, resp, err := requireOK(a.c.doRequest(r)) 96 + if err != nil { 97 + return nil, err 98 + } 99 + defer closeResponseBody(resp) 100 + 101 + var out []*AgentMember 102 + if err := decodeBody(resp, &out); err != nil { 103 + return nil, err 104 + } 105 + return out, nil 106 + }
+483
api/api.go
··· 1 + package api 2 + 3 + import ( 4 + "bytes" 5 + "context" 6 + "encoding/json" 7 + "fmt" 8 + "io" 9 + "io/ioutil" 10 + "net" 11 + "net/http" 12 + "net/url" 13 + "strconv" 14 + "strings" 15 + "sync" 16 + "time" 17 + 18 + "git.j3s.sh/cascade/lib/cleanhttp" 19 + ) 20 + 21 + // Config is used to configure the creation of a client 22 + type Config struct { 23 + // Address is the address of the Consul server 24 + Address string 25 + 26 + // Scheme is the URI scheme for the Consul server 27 + Scheme string 28 + 29 + // Transport is the Transport to use for the http client. 30 + Transport *http.Transport 31 + 32 + // HttpClient is the client to use. Default will be 33 + // used if not provided. 34 + HttpClient *http.Client 35 + 36 + // WaitTime limits how long a Watch will block. If not provided, 37 + // the agent default values will be used. 38 + WaitTime time.Duration 39 + } 40 + 41 + // DefaultConfig returns a default configuration for the client. By default this 42 + // will pool and reuse idle connections to cascade. If you have a long-lived 43 + // client object, this is the desired behavior and should make the most efficient 44 + // use of the connections to cascade. If you don't reuse a client object, which 45 + // is not recommended, then you may notice idle connections building up over 46 + // time. To avoid this, use the DefaultNonPooledConfig() instead. 47 + func DefaultConfig() *Config { 48 + return defaultConfig(cleanhttp.DefaultPooledTransport) 49 + } 50 + 51 + // DefaultNonPooledConfig returns a default configuration for the client which 52 + // does not pool connections. This isn't a recommended configuration because it 53 + // will reconnect to cascade on every request, but this is useful to avoid the 54 + // accumulation of idle connections if you make many client objects during the 55 + // lifetime of your application. 56 + func DefaultNonPooledConfig() *Config { 57 + return defaultConfig(cleanhttp.DefaultTransport) 58 + } 59 + 60 + // defaultConfig returns the default configuration for the client, using the 61 + // given function to make the transport. 62 + func defaultConfig(transportFn func() *http.Transport) *Config { 63 + config := &Config{ 64 + Address: "127.0.0.1:8500", 65 + Scheme: "http", 66 + Transport: transportFn(), 67 + } 68 + 69 + return config 70 + } 71 + 72 + // Client provides a client to the Consul API 73 + type Client struct { 74 + modifyLock sync.RWMutex 75 + headers http.Header 76 + 77 + config Config 78 + } 79 + 80 + // Headers gets the current set of headers used for requests. This returns a 81 + // copy; to modify it call AddHeader or SetHeaders. 82 + func (c *Client) Headers() http.Header { 83 + c.modifyLock.RLock() 84 + defer c.modifyLock.RUnlock() 85 + 86 + if c.headers == nil { 87 + return nil 88 + } 89 + 90 + ret := make(http.Header) 91 + for k, v := range c.headers { 92 + for _, val := range v { 93 + ret[k] = append(ret[k], val) 94 + } 95 + } 96 + 97 + return ret 98 + } 99 + 100 + // AddHeader allows a single header key/value pair to be added 101 + // in a race-safe fashion. 102 + func (c *Client) AddHeader(key, value string) { 103 + c.modifyLock.Lock() 104 + defer c.modifyLock.Unlock() 105 + c.headers.Add(key, value) 106 + } 107 + 108 + // SetHeaders clears all previous headers and uses only the given 109 + // ones going forward. 110 + func (c *Client) SetHeaders(headers http.Header) { 111 + c.modifyLock.Lock() 112 + defer c.modifyLock.Unlock() 113 + c.headers = headers 114 + } 115 + 116 + // NewClient returns a new client 117 + func NewClient(config *Config) (*Client, error) { 118 + // bootstrap the config 119 + defConfig := DefaultConfig() 120 + 121 + if config.Address == "" { 122 + config.Address = defConfig.Address 123 + } 124 + 125 + if config.Scheme == "" { 126 + config.Scheme = defConfig.Scheme 127 + } 128 + 129 + if config.Transport == nil { 130 + config.Transport = defConfig.Transport 131 + } 132 + 133 + if config.HttpClient == nil { 134 + config.HttpClient = NewHttpClient(config.Transport) 135 + } 136 + 137 + parts := strings.SplitN(config.Address, "://", 2) 138 + if len(parts) == 2 { 139 + switch parts[0] { 140 + case "http": 141 + config.Scheme = "http" 142 + // TODO: unix socket support? do i care? no??? 143 + default: 144 + return nil, fmt.Errorf("Unknown protocol scheme: %s", parts[0]) 145 + } 146 + config.Address = parts[1] 147 + } 148 + 149 + return &Client{config: *config, headers: make(http.Header)}, nil 150 + } 151 + 152 + // NewHttpClient returns an http client configured with the given Transport 153 + func NewHttpClient(transport *http.Transport) *http.Client { 154 + client := &http.Client{ 155 + Transport: transport, 156 + } 157 + return client 158 + } 159 + 160 + // request is used to help build up a request 161 + type request struct { 162 + config *Config 163 + method string 164 + url *url.URL 165 + params url.Values 166 + body io.Reader 167 + header http.Header 168 + obj interface{} 169 + ctx context.Context 170 + } 171 + 172 + // QueryMeta is used to return meta data about a query 173 + type QueryMeta struct { 174 + // LastIndex. This can be used as a WaitIndex to perform 175 + // a blocking query 176 + LastIndex uint64 177 + 178 + // LastContentHash. This can be used as a WaitHash to perform a blocking query 179 + // for endpoints that support hash-based blocking. Endpoints that do not 180 + // support it will return an empty hash. 181 + LastContentHash string 182 + 183 + // How long did the request take 184 + RequestTime time.Duration 185 + 186 + // Is address translation enabled for HTTP responses on this agent 187 + AddressTranslationEnabled bool 188 + } 189 + 190 + // QueryOptions are used to parameterize a query 191 + type QueryOptions struct { 192 + // WaitIndex is used to enable a blocking query. Waits 193 + // until the timeout or the next index is reached 194 + WaitIndex uint64 195 + 196 + // WaitHash is used by some endpoints instead of WaitIndex to perform blocking 197 + // on state based on a hash of the response rather than a monotonic index. 198 + // This is required when the state being blocked on is not stored in Raft, for 199 + // example agent-local proxy configuration. 200 + WaitHash string 201 + 202 + // WaitTime is used to bound the duration of a wait. 203 + // Defaults to that of the Config, but can be overridden. 204 + WaitTime time.Duration 205 + 206 + // Near is used to provide a node name that will sort the results 207 + // in ascending order based on the estimated round trip time from 208 + // that node. Setting this to "_agent" will use the agent's node 209 + // for the sort. 210 + Near string 211 + 212 + // NodeMeta is used to filter results by nodes with the given 213 + // metadata key/value pairs. Currently, only one key/value pair can 214 + // be provided for filtering. 215 + NodeMeta map[string]string 216 + 217 + // ctx is an optional context pass through to the underlying HTTP 218 + // request layer. Use Context() and WithContext() to manage this. 219 + ctx context.Context 220 + 221 + // Filter requests filtering data prior to it being returned. The string 222 + // is a go-bexpr compatible expression. 223 + Filter string 224 + } 225 + 226 + // setQueryOptions is used to annotate the request with 227 + // additional query options 228 + func (r *request) setQueryOptions(q *QueryOptions) { 229 + if q == nil { 230 + return 231 + } 232 + if q.WaitIndex != 0 { 233 + r.params.Set("index", strconv.FormatUint(q.WaitIndex, 10)) 234 + } 235 + if q.WaitTime != 0 { 236 + r.params.Set("wait", durToMsec(q.WaitTime)) 237 + } 238 + if q.WaitHash != "" { 239 + r.params.Set("hash", q.WaitHash) 240 + } 241 + if q.Near != "" { 242 + r.params.Set("near", q.Near) 243 + } 244 + if q.Filter != "" { 245 + r.params.Set("filter", q.Filter) 246 + } 247 + if len(q.NodeMeta) > 0 { 248 + for key, value := range q.NodeMeta { 249 + r.params.Add("node-meta", key+":"+value) 250 + } 251 + } 252 + 253 + r.ctx = q.ctx 254 + } 255 + 256 + // durToMsec converts a duration to a millisecond specified string. If the 257 + // user selected a positive value that rounds to 0 ms, then we will use 1 ms 258 + // so they get a short delay, otherwise Consul will translate the 0 ms into 259 + // a huge default delay. 260 + func durToMsec(dur time.Duration) string { 261 + ms := dur / time.Millisecond 262 + if dur > 0 && ms == 0 { 263 + ms = 1 264 + } 265 + return fmt.Sprintf("%dms", ms) 266 + } 267 + 268 + // serverError is a string we look for to detect 500 errors. 269 + const serverError = "Unexpected response code: 500" 270 + 271 + // IsRetryableError returns true for 500 errors from the Consul servers, and 272 + // network connection errors. These are usually retryable at a later time. 273 + // This applies to reads but NOT to writes. This may return true for errors 274 + // on writes that may have still gone through, so do not use this to retry 275 + // any write operations. 276 + func IsRetryableError(err error) bool { 277 + if err == nil { 278 + return false 279 + } 280 + 281 + if _, ok := err.(net.Error); ok { 282 + return true 283 + } 284 + 285 + // TODO (slackpad) - Make a real error type here instead of using 286 + // a string check. 287 + return strings.Contains(err.Error(), serverError) 288 + } 289 + 290 + // toHTTP converts the request to an HTTP request 291 + func (r *request) toHTTP() (*http.Request, error) { 292 + // Encode the query parameters 293 + r.url.RawQuery = r.params.Encode() 294 + 295 + // Check if we should encode the body 296 + if r.body == nil && r.obj != nil { 297 + b, err := encodeBody(r.obj) 298 + if err != nil { 299 + return nil, err 300 + } 301 + r.body = b 302 + } 303 + 304 + // Create the HTTP request 305 + req, err := http.NewRequest(r.method, r.url.RequestURI(), r.body) 306 + if err != nil { 307 + return nil, err 308 + } 309 + 310 + req.URL.Host = r.url.Host 311 + req.URL.Scheme = r.url.Scheme 312 + req.Host = r.url.Host 313 + req.Header = r.header 314 + 315 + // Content-Type must always be set when a body is present 316 + // See https://github.com/hashicorp/consul/issues/10011 317 + if req.Body != nil && req.Header.Get("Content-Type") == "" { 318 + req.Header.Set("Content-Type", "application/json") 319 + } 320 + 321 + if r.ctx != nil { 322 + return req.WithContext(r.ctx), nil 323 + } 324 + 325 + return req, nil 326 + } 327 + 328 + // newRequest is used to create a new request 329 + func (c *Client) newRequest(method, path string) *request { 330 + r := &request{ 331 + config: &c.config, 332 + method: method, 333 + url: &url.URL{ 334 + Scheme: c.config.Scheme, 335 + Host: c.config.Address, 336 + Path: path, 337 + }, 338 + params: make(map[string][]string), 339 + header: c.Headers(), 340 + } 341 + 342 + if c.config.WaitTime != 0 { 343 + r.params.Set("wait", durToMsec(r.config.WaitTime)) 344 + } 345 + 346 + return r 347 + } 348 + 349 + // doRequest runs a request with our client 350 + func (c *Client) doRequest(r *request) (time.Duration, *http.Response, error) { 351 + req, err := r.toHTTP() 352 + if err != nil { 353 + return 0, nil, err 354 + } 355 + start := time.Now() 356 + resp, err := c.config.HttpClient.Do(req) 357 + diff := time.Since(start) 358 + return diff, resp, err 359 + } 360 + 361 + // Query is used to do a GET request against an endpoint 362 + // and deserialize the response into an interface using 363 + // standard cascade conventions. 364 + func (c *Client) query(endpoint string, out interface{}, q *QueryOptions) (*QueryMeta, error) { 365 + r := c.newRequest("GET", endpoint) 366 + r.setQueryOptions(q) 367 + rtt, resp, err := c.doRequest(r) 368 + if err != nil { 369 + return nil, err 370 + } 371 + defer closeResponseBody(resp) 372 + 373 + qm := &QueryMeta{} 374 + err = parseQueryMeta(resp, qm) 375 + if err != nil { 376 + return nil, err 377 + } 378 + qm.RequestTime = rtt 379 + 380 + if err := decodeBody(resp, out); err != nil { 381 + return nil, err 382 + } 383 + return qm, nil 384 + } 385 + 386 + // parseQueryMeta is used to help parse query meta-data 387 + func parseQueryMeta(resp *http.Response, q *QueryMeta) error { 388 + header := resp.Header 389 + 390 + // Parse the X-Consul-Index (if it's set - hash based blocking queries don't 391 + // set this) 392 + // if indexStr := header.Get("X-Consul-Index"); indexStr != "" { 393 + // index, err := strconv.ParseUint(indexStr, 10, 64) 394 + // if err != nil { 395 + // return fmt.Errorf("Failed to parse X-Consul-Index: %v", err) 396 + // } 397 + // q.LastIndex = index 398 + // } 399 + // q.LastContentHash = header.Get("X-Consul-ContentHash") 400 + 401 + // Parse X-Consul-Translate-Addresses 402 + switch header.Get("X-Consul-Translate-Addresses") { 403 + case "true": 404 + q.AddressTranslationEnabled = true 405 + default: 406 + q.AddressTranslationEnabled = false 407 + } 408 + 409 + return nil 410 + } 411 + 412 + // decodeBody is used to JSON decode a body 413 + func decodeBody(resp *http.Response, out interface{}) error { 414 + dec := json.NewDecoder(resp.Body) 415 + return dec.Decode(out) 416 + } 417 + 418 + // encodeBody is used to encode a request body 419 + func encodeBody(obj interface{}) (io.Reader, error) { 420 + buf := bytes.NewBuffer(nil) 421 + enc := json.NewEncoder(buf) 422 + if err := enc.Encode(obj); err != nil { 423 + return nil, err 424 + } 425 + return buf, nil 426 + } 427 + 428 + // requireOK is used to wrap doRequest and check for a 200 429 + func requireOK(d time.Duration, resp *http.Response, e error) (time.Duration, *http.Response, error) { 430 + if e != nil { 431 + if resp != nil { 432 + closeResponseBody(resp) 433 + } 434 + return d, nil, e 435 + } 436 + if resp.StatusCode != 200 { 437 + return d, nil, generateUnexpectedResponseCodeError(resp) 438 + } 439 + return d, resp, nil 440 + } 441 + 442 + // closeResponseBody reads resp.Body until EOF, and then closes it. The read 443 + // is necessary to ensure that the http.Client's underlying RoundTripper is able 444 + // to re-use the TCP connection. See godoc on net/http.Client.Do. 445 + func closeResponseBody(resp *http.Response) error { 446 + _, _ = io.Copy(ioutil.Discard, resp.Body) 447 + return resp.Body.Close() 448 + } 449 + 450 + func (req *request) filterQuery(filter string) { 451 + if filter == "" { 452 + return 453 + } 454 + 455 + req.params.Set("filter", filter) 456 + } 457 + 458 + // generateUnexpectedResponseCodeError consumes the rest of the body, closes 459 + // the body stream and generates an error indicating the status code was 460 + // unexpected. 461 + func generateUnexpectedResponseCodeError(resp *http.Response) error { 462 + var buf bytes.Buffer 463 + io.Copy(&buf, resp.Body) 464 + closeResponseBody(resp) 465 + return fmt.Errorf("Unexpected response code: %d (%s)", resp.StatusCode, buf.Bytes()) 466 + } 467 + 468 + func requireNotFoundOrOK(d time.Duration, resp *http.Response, e error) (bool, time.Duration, *http.Response, error) { 469 + if e != nil { 470 + if resp != nil { 471 + closeResponseBody(resp) 472 + } 473 + return false, d, nil, e 474 + } 475 + switch resp.StatusCode { 476 + case 200: 477 + return true, d, resp, nil 478 + case 404: 479 + return false, d, resp, nil 480 + default: 481 + return false, d, nil, generateUnexpectedResponseCodeError(resp) 482 + } 483 + }
+60
lib/cleanhttp/cleanhttp.go
··· 1 + // Copyright (c) HashiCorp, Inc. 2 + // SPDX-License-Identifier: MPL-2.0 3 + 4 + package cleanhttp 5 + 6 + import ( 7 + "net" 8 + "net/http" 9 + "runtime" 10 + "time" 11 + ) 12 + 13 + // DefaultTransport returns a new http.Transport with similar default values to 14 + // http.DefaultTransport, but with idle connections and keepalives disabled. 15 + func DefaultTransport() *http.Transport { 16 + transport := DefaultPooledTransport() 17 + transport.DisableKeepAlives = true 18 + transport.MaxIdleConnsPerHost = -1 19 + return transport 20 + } 21 + 22 + // DefaultPooledTransport returns a new http.Transport with similar default 23 + // values to http.DefaultTransport. Do not use this for transient transports as 24 + // it can leak file descriptors over time. Only use this for transports that 25 + // will be re-used for the same host(s). 26 + func DefaultPooledTransport() *http.Transport { 27 + transport := &http.Transport{ 28 + Proxy: http.ProxyFromEnvironment, 29 + DialContext: (&net.Dialer{ 30 + Timeout: 30 * time.Second, 31 + KeepAlive: 30 * time.Second, 32 + DualStack: true, 33 + }).DialContext, 34 + MaxIdleConns: 100, 35 + IdleConnTimeout: 90 * time.Second, 36 + ExpectContinueTimeout: 1 * time.Second, 37 + ForceAttemptHTTP2: true, 38 + MaxIdleConnsPerHost: runtime.GOMAXPROCS(0) + 1, 39 + } 40 + return transport 41 + } 42 + 43 + // DefaultClient returns a new http.Client with similar default values to 44 + // http.Client, but with a non-shared Transport, idle connections disabled, and 45 + // keepalives disabled. 46 + func DefaultClient() *http.Client { 47 + return &http.Client{ 48 + Transport: DefaultTransport(), 49 + } 50 + } 51 + 52 + // DefaultPooledClient returns a new http.Client with similar default values to 53 + // http.Client, but with a shared Transport. Do not use this function for 54 + // transient clients as it can leak file descriptors over time. Only use this 55 + // for clients that will be re-used for the same host(s). 56 + func DefaultPooledClient() *http.Client { 57 + return &http.Client{ 58 + Transport: DefaultPooledTransport(), 59 + } 60 + }