Authenticate Hex package requests with HEXPM_READ_API_KEY · gleam.run/gleam@e1a67f5

+7

CHANGELOG.md

··· 92 92 informative and helpful. 93 93 ([Moritz Böhme](https://github.com/MoritzBoehme)) 94 94 95 + - The build tool can now authenticate requests to Hex with the API key from the 96 + `HEXPM_READ_API_KEY` environment variable, when resolving and downloading 97 + dependencies. This raises the request rate limit from the stricter per-IP 98 + limit to the higher per-user limit, avoiding "rate limit exceeded" errors 99 + when building large projects. 100 + ([John Downey](https://github.com/jtdowney)) 101 + 95 102 ### Language server 96 103 97 104 - The language server now supports go-to-definition, find-references and rename

+15 -3

compiler-cli/src/dependencies.rs

··· 510 510 // If we need to download at-least one package 511 511 if missing_hex_packages.peek().is_some() || !missing_git_packages.is_empty() { 512 512 let http = HttpClient::boxed(); 513 - let downloader = hex::Downloader::new(fs.clone(), fs, http, Untar::boxed(), paths.clone()); 513 + let downloader = hex::Downloader::new( 514 + fs.clone(), 515 + fs, 516 + http, 517 + Untar::boxed(), 518 + crate::hex::read_env_readonly_api_key(), 519 + paths.clone(), 520 + ); 514 521 let start = Instant::now(); 515 522 telemetry.downloading_package("packages"); 516 523 downloader ··· 1646 1653 name: String, 1647 1654 version: Version, 1648 1655 provided: &HashMap<EcoString, ProvidedPackage>, 1656 + credentials: Option<&hexpm::Credentials>, 1649 1657 ) -> Result<ManifestPackage> { 1650 1658 match provided.get(name.as_str()) { 1651 1659 Some(provided_package) => Ok(provided_package.to_manifest_package(name.as_str())), 1652 1660 None => { 1653 1661 let config = hexpm::Config::new(); 1654 1662 let release = 1655 - hex::get_package_release(&name, &version, &config, &HttpClient::new()).await?; 1663 + hex::get_package_release(&name, &version, credentials, &config, &HttpClient::new()) 1664 + .await?; 1656 1665 let build_tools = release 1657 1666 .meta 1658 1667 .build_tools ··· 1682 1691 runtime_cache: RefCell<HashMap<String, Rc<hexpm::Package>>>, 1683 1692 runtime: tokio::runtime::Handle, 1684 1693 http: HttpClient, 1694 + credentials: Option<hexpm::Credentials>, 1685 1695 } 1686 1696 1687 1697 impl PackageFetcher { ··· 1690 1700 runtime_cache: RefCell::new(HashMap::new()), 1691 1701 runtime, 1692 1702 http: HttpClient::new(), 1703 + credentials: crate::hex::read_env_readonly_api_key(), 1693 1704 } 1694 1705 } 1695 1706 ··· 1741 1752 1742 1753 tracing::debug!(package = package, "looking_up_hex_package"); 1743 1754 let config = hexpm::Config::new(); 1744 - let request = hexpm::repository_v2_get_package_request(package, None, &config); 1755 + let request = 1756 + hexpm::repository_v2_get_package_request(package, self.credentials.as_ref(), &config); 1745 1757 let response = self 1746 1758 .runtime 1747 1759 .block_on(self.http.send(request))

+8 -5

compiler-cli/src/dependencies/dependency_manager.rs

··· 308 308 )?; 309 309 310 310 // Convert the hex packages and local packages into manifest packages 311 - let manifest_packages = self.runtime.block_on(future::try_join_all( 312 - resolved 313 - .into_iter() 314 - .map(|(name, version)| lookup_package(name, version, &provided_packages)), 315 - ))?; 311 + let credentials = crate::hex::read_env_readonly_api_key(); 312 + let manifest_packages = 313 + self.runtime 314 + .block_on(future::try_join_all(resolved.into_iter().map( 315 + |(name, version)| { 316 + lookup_package(name, version, &provided_packages, credentials.as_ref()) 317 + }, 318 + )))?; 316 319 317 320 let manifest = Manifest { 318 321 packages: manifest_packages,

+1 -1

compiler-cli/src/hex.rs

··· 11 11 paths::ProjectPaths, 12 12 }; 13 13 14 - pub use auth::HexAuthentication; 14 + pub use auth::{HexAuthentication, read_env_readonly_api_key}; 15 15 16 16 /// Prepare credentials for user for write actions. 17 17 /// This will prompt for a one-time-password if needed.

+46 -10

compiler-cli/src/hex/auth.rs

··· 16 16 17 17 pub const LOCAL_PASS_PROMPT: &str = "Local password"; 18 18 pub const API_ENV_NAME: &str = "HEXPM_API_KEY"; 19 + pub const READONLY_API_ENV_NAME: &str = "HEXPM_READ_API_KEY"; 19 20 20 21 #[derive(Debug)] 21 22 pub struct EncryptedLegacyApiKey { ··· 185 186 /// an access token. 186 187 /// 3. The OAuth flow. 187 188 pub fn get_or_create_api_credentials(&mut self) -> Result<hexpm::Credentials> { 188 - if let Some(key) = Self::read_env_api_key()? { 189 + if let Some(key) = read_env_api_key() { 189 190 return Ok(key); 190 191 } 191 192 ··· 206 207 self.create_and_store_new_credentials_via_oauth() 207 208 } 208 209 209 - fn read_env_api_key() -> Result<Option<hexpm::Credentials>> { 210 - let api_key = std::env::var(API_ENV_NAME).unwrap_or_default(); 211 - if api_key.trim().is_empty() { 212 - Ok(None) 213 - } else { 214 - Ok(Some(hexpm::Credentials::ApiKey(EcoString::from(api_key)))) 215 - } 216 - } 217 - 218 210 /// Read, decrypt, and refresh OAuth keys stored on the filesystem. 219 211 /// 220 212 /// The new refresh is encrypted and stored on the file system for next use. ··· 367 359 struct StoredOAuthCredentials { 368 360 hexpm: StoredOAuthRepoCredentials, 369 361 } 362 + 363 + /// Read a Hex API key from the `HEXPM_API_KEY` environment variable, if one is set. 364 + /// 365 + /// This authenticates write commands such as `gleam publish`. 366 + fn read_env_api_key() -> Option<hexpm::Credentials> { 367 + api_key_credentials(&std::env::var(API_ENV_NAME).unwrap_or_default()) 368 + } 369 + 370 + /// Read a Hex API key from the `HEXPM_READ_API_KEY` environment variable, if one 371 + /// is set. 372 + /// 373 + /// This is used to authenticate otherwise anonymous read requests (dependency 374 + /// resolution and package downloads) so that they are subject to Hex's higher 375 + /// per-user rate limits rather than the stricter per-IP limits. 376 + pub fn read_env_readonly_api_key() -> Option<hexpm::Credentials> { 377 + api_key_credentials(&std::env::var(READONLY_API_ENV_NAME).unwrap_or_default()) 378 + } 379 + 380 + fn api_key_credentials(api_key: &str) -> Option<hexpm::Credentials> { 381 + let api_key = api_key.trim(); 382 + if api_key.is_empty() { 383 + None 384 + } else { 385 + Some(hexpm::Credentials::ApiKey(EcoString::from(api_key))) 386 + } 387 + } 388 + 389 + #[cfg(test)] 390 + mod tests { 391 + use super::*; 392 + 393 + #[test] 394 + fn api_key_credentials_trims_surrounding_whitespace() { 395 + assert_eq!( 396 + api_key_credentials(" secret\n"), 397 + Some(hexpm::Credentials::ApiKey("secret".into())), 398 + ); 399 + } 400 + 401 + #[test] 402 + fn api_key_credentials_blank_is_none() { 403 + assert_eq!(api_key_credentials(" \n"), None); 404 + } 405 + }

+16

compiler-cli/src/lib.rs

··· 149 149 )] 150 150 pub enum Command { 151 151 /// Build the project 152 + /// 153 + /// This command optionally accepts the environment variable 154 + /// `HEXPM_READ_API_KEY`, which can hold a Hex API key to authenticate 155 + /// with Hex with a higher rate limit. 156 + /// 157 + #[command(verbatim_doc_comment)] 152 158 Build { 153 159 /// Consider the build failed if the package contains any warnings 154 160 #[arg(long)] ··· 274 280 /// and for applications made of multiple packages in a single version 275 281 /// control repository. 276 282 /// 283 + /// This command optionally accepts the environment variable 284 + /// `HEXPM_READ_API_KEY`, which can hold a Hex API key to authenticate 285 + /// with Hex with a higher rate limit. 286 + /// 277 287 #[command(subcommand, verbatim_doc_comment)] 278 288 Deps(Dependencies), 279 289 280 290 /// Update dependency packages to their latest versions 291 + /// 292 + /// This command optionally accepts the environment variable 293 + /// `HEXPM_READ_API_KEY`, which can hold a Hex API key to authenticate 294 + /// with Hex with a higher rate limit. 295 + /// 296 + #[command(verbatim_doc_comment)] 281 297 Update(UpdateOptions), 282 298 283 299 /// Work with the Hex package manager

+80 -2

compiler-core/src/hex.rs

··· 194 194 http: DebugIgnore<Box<dyn HttpClient>>, 195 195 untar: DebugIgnore<Box<dyn TarUnpacker>>, 196 196 hex_config: hexpm::Config, 197 + credentials: DebugIgnore<Option<hexpm::Credentials>>, 197 198 paths: ProjectPaths, 198 199 } 199 200 ··· 203 204 fs_writer: Box<dyn FileSystemWriter>, 204 205 http: Box<dyn HttpClient>, 205 206 untar: Box<dyn TarUnpacker>, 207 + credentials: Option<hexpm::Credentials>, 206 208 paths: ProjectPaths, 207 209 ) -> Self { 208 210 Self { ··· 211 213 http: DebugIgnore(http), 212 214 untar: DebugIgnore(untar), 213 215 hex_config: hexpm::Config::new(), 216 + credentials: DebugIgnore(credentials), 214 217 paths, 215 218 } 216 219 } ··· 244 247 let request = hexpm::repository_get_package_tarball_request( 245 248 &package.name, 246 249 &package.version.to_string(), 247 - None, 250 + self.credentials.as_ref(), 248 251 &self.hex_config, 249 252 ); 250 253 let response = self.http.send(request).await?; ··· 360 363 pub async fn get_package_release<Http: HttpClient>( 361 364 name: &str, 362 365 version: &Version, 366 + credentials: Option<&hexpm::Credentials>, 363 367 config: &hexpm::Config, 364 368 http: &Http, 365 369 ) -> Result<hexpm::Release<hexpm::ReleaseMeta>> { ··· 369 373 version = version.as_str(), 370 374 "looking_up_package_release" 371 375 ); 372 - let request = hexpm::api_get_package_release_request(name, &version, None, config); 376 + let request = hexpm::api_get_package_release_request(name, &version, credentials, config); 373 377 let response = http.send(request).await?; 374 378 hexpm::api_get_package_release_response(response).map_err(Error::hex) 375 379 } 380 + 381 + #[cfg(test)] 382 + mod tests { 383 + use super::*; 384 + use std::sync::Mutex; 385 + 386 + /// A fake `HttpClient` that records the `authorization` header of the last 387 + /// request it was sent and replies with a canned package release. 388 + #[derive(Default)] 389 + struct AuthorizationCapturingHttpClient { 390 + last_authorization: Mutex<Option<String>>, 391 + } 392 + 393 + #[async_trait::async_trait] 394 + impl HttpClient for AuthorizationCapturingHttpClient { 395 + async fn send( 396 + &self, 397 + request: http::Request<Vec<u8>>, 398 + ) -> Result<http::Response<Vec<u8>>, Error> { 399 + let authorization = request 400 + .headers() 401 + .get("authorization") 402 + .map(|value| value.to_str().unwrap().to_string()); 403 + *self.last_authorization.lock().unwrap() = authorization; 404 + 405 + let body = serde_json::json!({ 406 + "version": "1.0.0", 407 + "checksum": "960090c2fb391784bb34267b099dc9315cc1b1f6013e7415bc763cef1905d7d3", 408 + "requirements": {}, 409 + "meta": { "app": "gleam_stdlib", "build_tools": ["gleam"] } 410 + }) 411 + .to_string() 412 + .into_bytes(); 413 + 414 + Ok(http::Response::builder().status(200).body(body).unwrap()) 415 + } 416 + } 417 + 418 + #[test] 419 + fn get_package_release_authenticates_with_api_key() { 420 + let http = AuthorizationCapturingHttpClient::default(); 421 + let credentials = hexpm::Credentials::ApiKey("secret-key".into()); 422 + 423 + let _ = futures::executor::block_on(get_package_release( 424 + "gleam_stdlib", 425 + &Version::new(1, 0, 0), 426 + Some(&credentials), 427 + &hexpm::Config::new(), 428 + &http, 429 + )) 430 + .unwrap(); 431 + 432 + assert_eq!( 433 + http.last_authorization.lock().unwrap().as_deref(), 434 + Some("secret-key") 435 + ); 436 + } 437 + 438 + #[test] 439 + fn get_package_release_is_anonymous_without_api_key() { 440 + let http = AuthorizationCapturingHttpClient::default(); 441 + 442 + let _ = futures::executor::block_on(get_package_release( 443 + "gleam_stdlib", 444 + &Version::new(1, 0, 0), 445 + None, 446 + &hexpm::Config::new(), 447 + &http, 448 + )) 449 + .unwrap(); 450 + 451 + assert_eq!(http.last_authorization.lock().unwrap().as_deref(), None); 452 + } 453 + }

+6 -2

hexpm/src/lib.rs

··· 110 110 } 111 111 } 112 112 113 - #[derive(Debug, Clone)] 113 + #[derive(Debug, Clone, PartialEq, Eq)] 114 114 pub enum Credentials { 115 115 // Short lived credential from OAuth 116 116 OAuthAccessToken(EcoString), ··· 370 370 371 371 match parts.status { 372 372 StatusCode::OK => (), 373 + StatusCode::TOO_MANY_REQUESTS => return Err(ApiError::RateLimited), 374 + StatusCode::UNAUTHORIZED => return Err(unauthorised_response(&parts.headers)), 373 375 StatusCode::FORBIDDEN => return Err(ApiError::NotFound), 374 376 StatusCode::NOT_FOUND => return Err(ApiError::NotFound), 375 377 status => { ··· 438 440 let (parts, body) = response.into_parts(); 439 441 match parts.status { 440 442 StatusCode::OK => (), 443 + StatusCode::TOO_MANY_REQUESTS => return Err(ApiError::RateLimited), 444 + StatusCode::UNAUTHORIZED => return Err(unauthorised_response(&parts.headers)), 441 445 StatusCode::FORBIDDEN => return Err(ApiError::NotFound), 442 446 StatusCode::NOT_FOUND => return Err(ApiError::NotFound), 443 447 status => { ··· 732 736 #[error(transparent)] 733 737 Io(#[from] std::io::Error), 734 738 735 - #[error("The rate limit for the Hex API has been exceeded for this IP")] 739 + #[error("The rate limit for the Hex API has been exceeded")] 736 740 RateLimited, 737 741 738 742 #[error("Invalid authentication credentials")]

+25

hexpm/src/tests.rs

··· 592 592 } 593 593 594 594 #[test] 595 + fn get_package_response_unauthorized() { 596 + let response = make_response(401, vec![]); 597 + let error = crate::repository_v2_get_package_response( 598 + response, 599 + std::include_bytes!("../test/public_key"), 600 + ) 601 + .unwrap_err(); 602 + 603 + assert_eq!(error.to_string(), "Invalid authentication credentials"); 604 + } 605 + 606 + #[test] 595 607 fn get_package_from_bytes_ok() { 596 608 let response_body = std::include_bytes!("../test/package_exfmt"); 597 609 let mut uncompressed = Vec::new(); ··· 746 758 let err = crate::repository_get_package_tarball_response(response, &checksum).unwrap_err(); 747 759 748 760 assert_eq!(err.to_string(), "Resource was not found"); 761 + } 762 + 763 + #[test] 764 + fn get_repository_tarball_response_rate_limited() { 765 + let checksum = vec![1, 2, 3, 4, 5]; 766 + 767 + let response = make_response(429, vec![]); 768 + let err = crate::repository_get_package_tarball_response(response, &checksum).unwrap_err(); 769 + 770 + assert_eq!( 771 + err.to_string(), 772 + "The rate limit for the Hex API has been exceeded" 773 + ); 749 774 } 750 775 751 776 #[test]

Configure Feed

Configure Feed