{ "@context": "https://schema.org", "@type": "Report", "url": "https://optimuslabs.io/research/briefings/ai-provider-key-custodians-breached-oct-2026", "identifier": "ai-provider-key-custodians-breached-oct-2026", "headline": "AI Provider Key Custodians Breached One After Another in October", "name": "AI Provider Key Custodians Breached One After Another in October", "abstract": "Fireworks, OpenCode, LiteLLM and Railway hold customers' AI provider keys or environment variables, and each has been breached or backdoored. Warp lost agent trace and telemetry data. Wiz counts at least nine such platforms hit in October, including Andromeda, Zeabur, Lightning AI, CommandCode and Northflank, and has watched stolen keys resold within days. Attribution is unconfirmed.", "description": "Live incidents, as of Nov 1, 2026. On Oct 6, Fireworks found that someone had used internal credentials to read environment variables, including API keys, in a limited number of customer accounts. On Oct 17, OpenCode said a breach of its hosted Console may have exposed provider keys for 1.4% of Console users. On the morning of Oct 21, a poisoned LiteLLM release went out through PyPI and Docker with a credential stealer built to take the provider keys a proxy stores. The same day, Railway said someone may have read decrypted customer environment variables and paused all deployments. In its preliminary report, Railway says the attacker was after AI provider keys for distillation. LiteLLM's attack chain starts with a master key committed to a public repository on May 27 and never revoked. No one has published a root cause that links all four. Wiz counts at least nine AI developer platforms that disclosed incidents in October.", "datePublished": "2026-10-21", "dateModified": "2026-10-21", "inLanguage": "en", "isAccessibleForFree": true, "author": { "@type": "Organization", "name": "Optimus Labs · Civilizations", "url": "https://optimuslabs.io/research/briefings" }, "publisher": { "@type": "Organization", "name": "Optimus Labs", "url": "https://optimuslabs.io" }, "keywords": [ "Fireworks AI", "OpenCode", "LiteLLM", "Railway", "Warp", "Andromeda", "Zeabur", "Lightning AI", "CommandCode", "Northflank", "NVIDIA Brev", "Anthropic", "OpenAI", "Google", "Credential", "Inference", "AI Asset Supply Chain Security", "Cyber Hygiene", "agentic AI security", "AI agent supply chain" ], "about": [ { "@type": "Thing", "name": "Fireworks AI" }, { "@type": "Thing", "name": "OpenCode" }, { "@type": "Thing", "name": "LiteLLM" }, { "@type": "Thing", "name": "Railway" }, { "@type": "Thing", "name": "Warp" }, { "@type": "Thing", "name": "Andromeda" }, { "@type": "Thing", "name": "Zeabur" }, { "@type": "Thing", "name": "Lightning AI" }, { "@type": "Thing", "name": "CommandCode" }, { "@type": "Thing", "name": "Northflank" }, { "@type": "Thing", "name": "NVIDIA Brev" }, { "@type": "Thing", "name": "Anthropic" }, { "@type": "Thing", "name": "OpenAI" }, { "@type": "Thing", "name": "Google" } ], "citation": [ { "@type": "CreativeWork", "name": "Andromeda: October 2026 security incident", "url": "https://andromeda.ai/security-updates/october-2026-security-incident" }, { "@type": "CreativeWork", "name": "Zeabur: October 2026 security incident", "url": "https://zeabur.com/blogs/october-2026-security-incident" }, { "@type": "CreativeWork", "name": "Lightning AI: October 2026 incident", "url": "https://lightning.ai/blog/oct-2026-incident" }, { "@type": "CreativeWork", "name": "CommandCode on X", "url": "https://x.com/CommandCodeAI/status/2115583532055826770" }, { "@type": "CreativeWork", "name": "Northflank status page", "url": "https://status.northflank.com/" }, { "@type": "CreativeWork", "name": "Zach Lloyd (Warp CEO) on X, Oct 26", "url": "https://x.com/zachlloydtweets/status/2114800787716972873" }, { "@type": "CreativeWork", "name": "International Cyber Digest on X: NVIDIA Brev (unconfirmed)", "url": "https://x.com/IntCyberDigest/status/2116580207197094359" }, { "@type": "CreativeWork", "name": "Wiz: Black October for AI developer platforms", "url": "https://www.wiz.io/blog/black-october-for-ai-developer-platforms" }, { "@type": "CreativeWork", "name": "Railway security disclosure, Oct 21", "url": "https://blog.railway.com/p/security-disclosure-wednesday-october-21st" }, { "@type": "CreativeWork", "name": "Railway preliminary report, Oct 22", "url": "https://blog.railway.com/p/preliminary-report-october-security-incident" }, { "@type": "CreativeWork", "name": "Railway incident status", "url": "https://status.railway.com/incident/QH8L8B3E" }, { "@type": "CreativeWork", "name": "LiteLLM incident report, Oct 21", "url": "https://docs.litellm.ai/blog/security-incident-oct-21-2026" }, { "@type": "CreativeWork", "name": "LiteLLM attack chain, Oct 23", "url": "https://docs.litellm.ai/blog/security-incident-oct-21-2026-attack-chain" }, { "@type": "CreativeWork", "name": "LiteLLM security issue #47826", "url": "https://github.com/BerriAI/litellm/issues/47826" }, { "@type": "CreativeWork", "name": "The Hacker News: LiteLLM compromised", "url": "https://thehackernews.com/2026/10/litellm-compromised-in-supply-chain.html" }, { "@type": "CreativeWork", "name": "The Register: LiteLLM supply chain attack", "url": "https://www.theregister.com/security/2026/10/21/litellm-supply-chain-attack-hoovers-up-passwords-and-secrets/5304548" }, { "@type": "CreativeWork", "name": "JFrog: LiteLLM hijacked again", "url": "https://research.jfrog.com/post/litellm-hijacked-again/" }, { "@type": "CreativeWork", "name": "Socket: LiteLLM compromise", "url": "https://socket.dev/blog/litellm-compromise" }, { "@type": "CreativeWork", "name": "Wiz: LiteLLM compromised, steals credentials", "url": "https://www.wiz.io/blog/litellm-compromised-steals-credentials" }, { "@type": "CreativeWork", "name": "Endor Labs: LiteLLM 1.107.0", "url": "https://www.endorlabs.com/learn/litellm-1-107-0-compromised-python-startup-hooks-steal-cloud-ai-and-proxy-credentials" }, { "@type": "CreativeWork", "name": "Fireworks October 2026 security incident", "url": "https://fireworks.ai/security/bulletins/october-2026-security-incident" }, { "@type": "CreativeWork", "name": "LAVA: Fireworks AI incident", "url": "https://lava.security/blog/fireworks-ai-security-incident" }, { "@type": "CreativeWork", "name": "OpenCode statement on X, Oct 17", "url": "https://x.com/opencode/status/2111519637049319818" }, { "@type": "CreativeWork", "name": "Optimus Labs briefing #13: OpenCode Console", "url": "https://optimuslabs.io/research/briefings/opencode-console-breach-provider-key-exposure" }, { "@type": "CreativeWork", "name": "Optimus Labs briefing: Salesloft Drift OAuth supply chain", "url": "https://optimuslabs.io/research/briefings/salesloft-drift-oauth-supply-chain" }, { "@type": "CreativeWork", "name": "Optimus Labs briefing: arrayref poisoned Rust crate", "url": "https://optimuslabs.io/research/briefings/arrayref-rust-crate-build-time-rce" }, { "@type": "CreativeWork", "name": "Optimus Labs briefing: Coder registry hijack", "url": "https://optimuslabs.io/research/briefings/coder-registry-infrastructure-hijack" }, { "@type": "CreativeWork", "name": "Cloud Security Alliance: TeamPCP and AI infrastructure", "url": "https://labs.cloudsecurityalliance.org/research/csa-research-note-teampcp-ai-infra-supply-chain-20260406-csa/" }, { "@type": "CreativeWork", "name": "Ars Technica: terabytes of credentials leaked", "url": "https://arstechnica.com/security/2026/08/terabytes-of-credentials-leaked-in-massive-supply-chain-attack/" }, { "@type": "CreativeWork", "name": "Anthropic threat report, Sep 2026", "url": "https://www.anthropic.com/threat-intelligence-report-september-2026" }, { "@type": "CreativeWork", "name": "OWASP Top 10 for Agentic Applications 2026", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/" }, { "@type": "CreativeWork", "name": "OWASP Top 10 for LLM Applications 2026", "url": "https://genai.owasp.org/resource/owasp-genai-llm-top-10-2026/" } ], "encoding": [ { "@type": "MediaObject", "encodingFormat": "text/markdown", "contentUrl": "https://optimuslabs.io/research/briefings/ai-provider-key-custodians-breached-oct-2026.md" }, { "@type": "MediaObject", "encodingFormat": "application/json", "contentUrl": "https://optimuslabs.io/research/briefings/ai-provider-key-custodians-breached-oct-2026.json" } ], "additionalProperty": [ { "@type": "PropertyValue", "name": "severity", "value": "CRITICAL · LIVE INCIDENTS · KEY CUSTODIANS" }, { "@type": "PropertyValue", "name": "blastRadius", "value": "Any team that stored provider keys or environment variables with one of the four: Fireworks evaluation jobs, OpenCode Console workspaces, a LiteLLM proxy or host that ran version 1.107.0, or Railway service variables. LiteLLM receives about 23 million downloads a week, and its proxy keeps every upstream provider key in one database table." } ], "briefing": { "slug": "ai-provider-key-custodians-breached-oct-2026", "number": 14, "title": "AI Provider Key Custodians Breached One After Another in October", "dek": "Fireworks, OpenCode, LiteLLM and Railway hold customers' AI provider keys or environment variables, and each has been breached or backdoored. Warp lost agent trace and telemetry data. Wiz counts at least nine such platforms hit in October, including Andromeda, Zeabur, Lightning AI, CommandCode and Northflank, and has watched stolen keys resold within days. Attribution is unconfirmed.", "tldr": "Live incidents as of Nov 1, 2026. Four companies that sit between teams and AI model providers had customer secrets exposed or targeted between Oct 6 and Oct 21: Fireworks, OpenCode, LiteLLM and Railway. OpenCode and Railway tie their breaches to distillation. LiteLLM traces its release to a master key leaked in May and never revoked. Fireworks has named no motive. Wiz counts at least nine AI developer platforms that disclosed incidents in October, and saw stolen enterprise keys tested within 30 minutes and resold within a week. Revoke at the provider, not at the custodian.", "date": "2026-10-21", "severity": "critical", "severityLabel": "CRITICAL · LIVE INCIDENTS · KEY CUSTODIANS", "types": [ "credential", "inference" ], "relatedUseCase": "secure-ai-agent-adoption", "categories": [ "AI", "CH" ], "vendors": [ "Fireworks AI", "OpenCode", "LiteLLM", "Railway", "Warp", "Andromeda", "Zeabur", "Lightning AI", "CommandCode", "Northflank", "NVIDIA Brev", "Anthropic", "OpenAI", "Google" ], "faq": [ { "q": "Which AI developer platforms were breached in October 2026?", "a": "Wiz counts at least nine that disclosed incidents: Fireworks AI (Oct 6), Andromeda (Oct 9), Zeabur (Oct 14), Lightning AI (Oct 15), OpenCode (Oct 16 to 17), Railway (Oct 20), Warp (Oct 23), CommandCode (Oct 28) and Northflank (Oct 30). LiteLLM shipped a malicious release on Oct 21. NVIDIA Brev is reported as a tenth but not confirmed." }, { "q": "I used Andromeda, Zeabur, Lightning AI, CommandCode or Northflank. What should I do?", "a": "Treat any provider key or environment variable stored there as exposed. Revoke it at the provider that issued it, not only in the platform dashboard, check billing since Oct 6, and set a hard spend cap on the replacement key." }, { "q": "Is the NVIDIA Brev breach confirmed?", "a": "No. As of Nov 1, 2026 it is reported by International Cyber Digest, and NVIDIA has not published a notice we could check." } ], "featured": false, "liveIncident": { "asOf": "2026-11-01", "note": "Live incidents, as of Nov 1, 2026. Wiz has published downstream abuse of keys stolen in these breaches; Railway is still investigating and this page will be updated." }, "blastRadius": "Any team that stored provider keys or environment variables with one of the four: Fireworks evaluation jobs, OpenCode Console workspaces, a LiteLLM proxy or host that ran version 1.107.0, or Railway service variables. LiteLLM [receives about 23 million downloads a week](https://www.theregister.com/security/2026/10/21/litellm-supply-chain-attack-hoovers-up-passwords-and-secrets/5304548), and its proxy keeps every upstream provider key in one database table.", "summary": "Live incidents, as of Nov 1, 2026. On Oct 6, [Fireworks found](https://fireworks.ai/security/bulletins/october-2026-security-incident) that someone had used internal credentials to read environment variables, including API keys, in a limited number of customer accounts. On Oct 17, [OpenCode said](https://x.com/opencode/status/2111519637049319818) a breach of its hosted Console may have exposed provider keys for 1.4% of Console users. On the morning of Oct 21, a poisoned [LiteLLM release](https://docs.litellm.ai/blog/security-incident-oct-21-2026) went out through PyPI and Docker with a credential stealer built to take the provider keys a proxy stores. The same day, [Railway said](https://blog.railway.com/p/security-disclosure-wednesday-october-21st) someone may have read decrypted customer environment variables and paused all deployments. In its [preliminary report](https://blog.railway.com/p/preliminary-report-october-security-incident), Railway says the attacker was after AI provider keys for distillation. LiteLLM's [attack chain](https://docs.litellm.ai/blog/security-incident-oct-21-2026-attack-chain) starts with a master key committed to a public repository on May 27 and never revoked. No one has published a root cause that links all four. [Wiz counts](https://www.wiz.io/blog/black-october-for-ai-developer-platforms) at least nine AI developer platforms that disclosed incidents in October.", "affected": { "checks": [ "Did a host, CI runner or container of yours run litellm 1.107.0, or pull the latest, main-latest or main-stable LiteLLM image, between 11:39 and 14:20 UTC on Oct 21?", "Do you keep API keys or registry credentials in Railway variables, run Fireworks evaluation jobs with secrets, or save provider keys in OpenCode Console ([briefing #13](https://optimuslabs.io/research/briefings/opencode-console-breach-provider-key-exposure))?", "Did you store provider keys or environment variables on Andromeda, Zeabur, Lightning AI, CommandCode or Northflank, or use NVIDIA Brev, in October?", "Does provider billing since Oct 6 show models, volumes or regions you do not recognise, or calls through global cross-region inference profiles you never use?" ], "notAffected": [ "LiteLLM [says its private enterprise release](https://docs.litellm.ai/blog/security-incident-oct-21-2026) and every version before 1.107.0 are safe. Fireworks says no notice means it has not identified your account. Both are the vendors' own claims." ], "note": "Rotate stored keys at the provider that issued them. [Fireworks](https://fireworks.ai/security/bulletins/october-2026-security-incident) says removing a stored secret does not invalidate the underlying credential, and the same holds for all four custodians." }, "timeline": [ { "label": "LiteLLM master key committed to a public repository", "date": "May 27", "note": "A config file held the master key to the internal LiteLLM Agent Platform. The commit was force-pushed over, but the key was never revoked ([LiteLLM](https://docs.litellm.ai/blog/security-incident-oct-21-2026-attack-chain))." }, { "label": "Fireworks finds unauthorized use of internal credentials", "date": "Oct 6", "note": "Environment variables in evaluation jobs were captured in a limited number of accounts. It published its bulletin on Oct 9 ([Fireworks](https://fireworks.ai/security/bulletins/october-2026-security-incident))." }, { "label": "Attacker takes LiteLLM owner GitHub token", "date": "Oct 10", "note": "The Agent Platform returned an organization owner's GitHub CLI token. Throwaway workflows then posted two maintainers' tokens to the attacker ([LiteLLM](https://docs.litellm.ai/blog/security-incident-oct-21-2026-attack-chain))." }, { "label": "LiteLLM look-alike domain registered", "date": "Oct 13", "note": "litellm[.]dev was created Oct 13 and updated Oct 18, per [JFrog](https://research.jfrog.com/post/litellm-hijacked-again/). The real project uses litellm.ai." }, { "label": "Andromeda discloses an incident", "date": "Oct 9", "note": "Calls it part of an \"industry-wide attack\" ([Andromeda](https://andromeda.ai/security-updates/october-2026-security-incident))." }, { "label": "Zeabur discloses a breach", "date": "Oct 14", "note": "Credential access; attacker activity Oct 12 to 14. Zeabur is shutting down ([Zeabur](https://zeabur.com/blogs/october-2026-security-incident))." }, { "label": "Lightning AI discloses an incident", "date": "Oct 15", "note": "Credential access, per [Lightning AI](https://lightning.ai/blog/oct-2026-incident)." }, { "label": "OpenCode Console keys exposed and breach disclosed", "date": "Oct 17", "note": "Earliest possible exposure 00:30 UTC. 1.4% of Console users, \"[related to a large scale distillation attack](https://x.com/opencode/status/2111519637049319818)\"." }, { "label": "Railway attack window opens", "date": "Oct 20, 13:58 UTC", "note": "Elevated latency paged on-call; Railway found privilege escalation in its build infrastructure and halted builds ([Railway](https://blog.railway.com/p/preliminary-report-october-security-incident))." }, { "label": "LiteLLM release protections overridden", "date": "Oct 21, 10:08 UTC", "note": "Branch protection rejected a change to the private release workflow. Five seconds later a privileged developer credential overrode it ([LiteLLM](https://docs.litellm.ai/blog/security-incident-oct-21-2026))." }, { "label": "Malicious litellm 1.107.0 live", "date": "Oct 21, 11:39 UTC", "note": "Seven PyPI wheels until 13:23 UTC. Docker tags latest, main-latest, main-stable and v1.107.0 were poisoned from 11:46 to 14:20 UTC." }, { "label": "Railway disclosure and paused deployments", "date": "Oct 21", "note": "Railway [notified upstream providers to revoke keys](https://blog.railway.com/p/security-disclosure-wednesday-october-21st) and paused all deployments." }, { "label": "Railway confirms attacker out", "date": "Oct 22, 03:32 UTC", "note": "Railway says this is when it confirmed the attacker no longer had access to internal systems ([Railway](https://blog.railway.com/p/preliminary-report-october-security-incident))." }, { "label": "CommandCode discloses an incident", "date": "Oct 28", "note": "Attributed to a distillation attempt ([CommandCode](https://x.com/CommandCodeAI/status/2115583532055826770))." }, { "label": "Northflank discloses an incident", "date": "Oct 30", "note": "Posted on the [Northflank status page](https://status.northflank.com/); dates per Wiz." }, { "label": "Wiz links abused keys to the breaches", "date": "Oct 30", "note": "Wiz reports two enterprise AWS Bedrock keys stolen in one of these vendor breaches, abused for about 2.5 million calls before rotation ([Wiz](https://www.wiz.io/blog/black-october-for-ai-developer-platforms))." } ], "narrativeSections": [ { "heading": "Three hosted platforms and one package", "paragraphs": [ "**Fireworks**, **OpenCode** and **Railway** are hosted platforms that store customer secrets, and an intruder reached each one. [Fireworks](https://fireworks.ai/security/bulletins/october-2026-security-incident) says internal credentials were used to capture evaluation-job variables and that it found no access to customer models, datasets or inference traffic. [OpenCode's Console](https://optimuslabs.io/research/briefings/opencode-console-breach-provider-key-exposure) was reached through a legacy system being retired. [Railway says](https://blog.railway.com/p/preliminary-report-october-security-incident) the attacker escalated privileges in its build infrastructure and pulled secrets in encrypted form, and that a subset of customers had secrets exposed that need rotating.", "**LiteLLM** is different. It is the open source gateway many teams run themselves, and the poisoned release ran inside their own environments. It [reads the proxy's provider-key table directly](https://www.endorlabs.com/learn/litellm-1-107-0-compromised-python-startup-hooks-steal-cloud-ai-and-proxy-credentials), takes cloud credentials, SSH keys and environment variables, and installs a backdoor that [checks in hourly](https://www.theregister.com/security/2026/10/21/litellm-supply-chain-attack-hoovers-up-passwords-and-secrets/5304548)." ] }, { "heading": "How the LiteLLM release was taken", "paragraphs": [ "[LiteLLM's attack chain](https://docs.litellm.ai/blog/security-incident-oct-21-2026-attack-chain) starts on May 27, when a config file committed to a public repository held the master key to its internal LiteLLM Agent Platform. The commit was force-pushed over, but the key was never revoked. On Oct 10 the attacker used it to read the Agent Platform, which returned a GitHub CLI token belonging to an organization owner. LiteLLM says that token was not scoped down and skips GitHub's re-authentication for sensitive actions.", "With owner access, the attacker pushed throwaway workflows that posted two maintainers' tokens, stored as CI secrets, to the attacker's endpoint, and took tokens for other internal systems, with a focus on upstream inference API keys. Release approval had been relaxed that week from a device-locked account to any two engineers, so one token triggered the release and another approved it; a third deleted the run records. The build ran from a look-alike repository, Berri-AI instead of BerriAI, and the official source tag stayed clean. [JFrog notes](https://research.jfrog.com/post/litellm-hijacked-again/) that reading the source would have missed it.", "Hijacked publishing access is a repeat pattern in our own briefings: in [arrayref](https://optimuslabs.io/research/briefings/arrayref-rust-crate-build-time-rce), a compromised maintainer account pushed malicious crates to crates.io." ] }, { "heading": "Why the middleman holds the prize", "paragraphs": [ "OpenCode and Railway tie their breaches to distillation, the pulling of a strong model's answers at scale to train a cheaper copy. [Anthropic says](https://www.anthropic.com/threat-intelligence-report-september-2026) distillers often use stolen API credentials from legitimate customers. [Railway adds](https://blog.railway.com/p/preliminary-report-october-security-incident) that the attacker appeared to have abliterated models and substantial compute. LiteLLM says the attacker's activity focused on upstream inference API tokens. Fireworks has named no motive, and no source links the four incidents to one actor.", "**Our read, not the vendors':** a gateway or platform holds keys for many customers in one place, so one intrusion yields a pile of paid, aged accounts with normal usage history. That is plausible, not proven. Stored tokens as the way in is not new: in [Salesloft Drift](https://optimuslabs.io/research/briefings/salesloft-drift-oauth-supply-chain), attackers stole the OAuth tokens a vendor held for customers and used them against customer CRMs." ] }, { "heading": "Part of a wider October", "paragraphs": [ "[Wiz counts](https://www.wiz.io/blog/black-october-for-ai-developer-platforms) at least nine AI developer platforms that disclosed incidents in October: Fireworks AI, Andromeda, Zeabur, Lightning AI, OpenCode, Railway, Warp, CommandCode and Northflank. Fireworks, OpenCode and Warp describe an \"agent swarm\" attack, and OpenCode and CommandCode attribute theirs to distillation. Root causes differ: credential access at Fireworks, Zeabur and Lightning, build infrastructure at Railway, a third-party dependency at Warp, a legacy system at OpenCode. Every public report describes attackers hunting stored environment variables, especially AI access. Zeabur is shutting down.", "[Warp's CEO said](https://x.com/zachlloydtweets/status/2114800787716972873) on Oct 26 that an agent swarm, likely starting with a hole in a vendor dependency, reached a limited amount of agent trace and telemetry data, with an upper bound of roughly 1% of users. Warp has not reported provider keys taken, and is working with CrowdStrike and law enforcement.", "NVIDIA Brev may be a tenth. [International Cyber Digest reports](https://x.com/IntCyberDigest/status/2116580207197094359) that NVIDIA told users of unauthorised activity from Oct 4 to 20 that may have exposed the SSH keys Brev set up for their VMs. We have not seen NVIDIA's notice, so treat this as reported, not confirmed. Andromeda, Zeabur, Lightning AI, CommandCode and Northflank have published less, and no single vulnerability links them." ] }, { "heading": "The other October incidents", "paragraphs": [ "Five more platforms disclosed incidents in October, per [Wiz](https://www.wiz.io/blog/black-october-for-ai-developer-platforms). Details are thin, and the dates are Wiz's.", "**Andromeda** (Oct 9): its [incident report](https://andromeda.ai/security-updates/october-2026-security-incident) calls the breach part of an \"industry-wide attack\".", "**Zeabur** (Oct 14): its [write-up](https://zeabur.com/blogs/october-2026-security-incident) puts the root cause at credential access and names attacker address 23[.]106[.]46[.]239, active Oct 12 to 14. Zeabur is shutting down because of the incident.", "**Lightning AI** (Oct 15): its [notice](https://lightning.ai/blog/oct-2026-incident) also points to credential access. Wiz notes Lightning was hit earlier in the Mini Shai-Hulud campaign.", "**CommandCode** (Oct 28): [its post on X](https://x.com/CommandCodeAI/status/2115583532055826770) ties the incident to an attempt at distillation, as OpenCode did.", "**Northflank** (Oct 30): disclosed on its [status page](https://status.northflank.com/). No root cause is public yet.", "**NVIDIA Brev** (reported, not confirmed): see above. NVIDIA has not published a notice we can check.", "If you stored keys or environment variables on any of these, treat them as exposed and rotate them at the provider that issued them." ] }, { "heading": "What happens to a stolen key", "paragraphs": [ "Wiz followed two enterprise AWS Bedrock keys it ties to one of these vendor breaches. Each was tested within 30 minutes of the theft by a single model call from a residential address, in a one-minute sweep across several organizations. A small checker then pinged both keys every day to confirm they still worked.", "Four days later, the first key was flooded: about 800,000 Bedrock calls in 30 hours, mostly Opus models. Its policy allowed global cross-region inference profiles, so only about 6% of calls were throttled. The second key was restricted to US profiles. A reseller pool failed for hours until a checker found the working model IDs, then one address made about 1.7 million calls in 13 hours, with most of the rest throttled by regional quotas. Wiz estimates over $100,000 in usage at list price across the two.", "Wiz traced checks on both keys to two open resale sites selling discounted frontier-model access, one at an address Zeabur named in its own breach. Wiz cannot tie either site to the original theft. **Our read:** key scope is a blast-radius control. The restricted key still leaked, but it cost the attacker a day and ran into quotas." ] }, { "heading": "LiteLLM was already hit in March", "paragraphs": [ "On Mar 24, versions 1.82.7 and 1.82.8 reached PyPI for about 40 minutes after attackers took LiteLLM's publishing token through a poisoned Trivy scanner ([Cloud Security Alliance](https://labs.cloudsecurityalliance.org/research/csa-research-note-teampcp-ai-infra-supply-chain-20260406-csa/)). CloudSEK later [counted more than 2,500 organizations](https://arstechnica.com/security/2026/08/terabytes-of-credentials-leaked-in-massive-supply-chain-attack/) in the captured data, which means potential exposure, not confirmed compromise. JFrog says the October release uses different payloads and infrastructure and that [no evidence links it to the March actor](https://research.jfrog.com/post/litellm-hijacked-again/). Both went after the same asset: the keys a gateway holds." ] }, { "heading": "How it maps to OWASP", "paragraphs": [ "In the [Agentic Applications Top 10](https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/), ASI04 Agentic Supply Chain Vulnerabilities covers LiteLLM and the hosted custodians, and ASI03 Identity & Privilege Abuse covers the stolen keys. In the [LLM Applications Top 10](https://genai.owasp.org/resource/owasp-genai-llm-top-10-2026/), LLM02 Sensitive Information Disclosure names API keys as protected information, and LLM06 Unbounded Consumption covers what stolen keys are used for, including distillation theft.", "*The OWASP GenAI Security Project does not endorse any product, service or tool, including Optimus Labs. We cite its lists to classify these incidents.*" ] }, { "heading": "If you hold other people's keys", "lede": "Provider keys are crown jewels.", "paragraphs": [ "Any vendor that stores a customer's API key for them (a model gateway, an agent platform, an observability or CI tool) is one intrusion away from these same incidents. None of the four vendors has said how stored keys were protected, so none of this is a finding about them. It is the path we would expect a vendor to follow, drawn from [OWASP](https://cheatsheetseries.owasp.org/cheatsheets/Secrets_Management_Cheat_Sheet.html) and [NIST key-management guidance](https://csrc.nist.gov/pubs/sp/800/57/pt1/r5/final).", "LiteLLM adds a question for any vendor: who can change what you ship? **Our read:** keep an organization owner's token out of developer tooling, require approval from an account not reachable the same way, and copy workflow logs where an account cannot delete them. LiteLLM's chain also shows that a leaked key which was \"cleaned up\" but never revoked is still live: practices 6 and 8 apply to your own keys too. Environment variables are a favorite place to look: in the [Coder registry hijack](https://optimuslabs.io/research/briefings/coder-registry-infrastructure-hijack), poisoned modules scraped credentials from environment variables, config files and shell history, and no CVE was assigned. The eight practices below cover the stored keys themselves." ], "practices": [ { "phase": "Hold less", "icon": "KeyRound", "title": "Store less", "body": "Use short-lived, scoped credentials or delegated access wherever a provider offers them. Many model providers still issue long-lived static keys, so a gateway often has to store them. Where the design allows, pass a key through for one request without saving it." }, { "phase": "At rest", "icon": "Lock", "title": "Use envelope encryption", "body": "A unique data key per credential, AES-256-GCM, with the data key wrapped by a master key in a KMS or HSM. OWASP says not to keep encryption keys next to the secrets they protect unless those keys are themselves encrypted." }, { "phase": "At rest", "icon": "Layers", "title": "Separate customers", "body": "One master key per customer, and bind each ciphertext to its customer and record so a copied blob will not decrypt in another row. Deleting a customer's key then makes their stored credentials unrecoverable." }, { "phase": "At rest", "icon": "EyeOff", "title": "Make stored keys write-only", "body": "No reveal button and no export. Show the last four characters and the date it was set. Never log a key: strip authorization headers from logs and error reports." }, { "phase": "In use", "icon": "Activity", "title": "Decrypt per use, and watch it", "body": "Decrypt only for the outbound call, wipe the plaintext from memory, and keep any cache short. Log every decrypt and alert on spikes, so a bulk pull stands out." }, { "phase": "Beyond the vault", "icon": "Archive", "title": "Retire systems completely", "body": "Before a system is decommissioned, move or delete the credentials it holds and confirm they are gone. Any system still holding live keys is still in scope." }, { "phase": "Beyond the vault", "icon": "Gauge", "title": "Give customers the brakes", "body": "Per-key spend caps that stop traffic, model and IP allowlists, last-used time and source for every key, and plain revoke steps for each provider." }, { "phase": "Beyond the vault", "icon": "Siren", "title": "Plan the breach before it happens", "body": "Keep a per-customer list of stored keys by last four characters, agree a revocation contact with each provider ahead of time, and tell customers in hours, not days. Anthropic and Google acted on Oct 17, within about a day of the first alert, and a pre-agreed route makes that routine." } ] }, { "heading": "What we still don't know", "paragraphs": [ "As of Nov 1, 2026, unpublished: how many Railway accounts were affected and the specific exploits used against Railway, which it says it will disclose later, how the Fireworks intruder got the internal credentials, whether the four (or the nine Wiz counts) share an actor, who ran the floods Wiz observed, and how many of the stolen keys are in use today." ] } ], "whatHappened": [ "Fireworks, OpenCode, LiteLLM and Railway each held customer provider keys or environment variables, and each had them exposed or targeted between Oct 6 and Oct 21.", "LiteLLM's release pipeline was taken through a master key leaked in May and never revoked, which led to three engineers' GitHub tokens, and shipped a credential stealer for about two hours." ], "whyItMatters": [ "Your model accounts now depend on a chain of custodians. A weak link anywhere exposes the keys.", "Deleting a key at the custodian does not kill a copy. Revoking at the issuing provider does." ], "whatToDo": [ "Revoke and replace every provider key that touched one of the four, or Andromeda, Zeabur, Lightning AI, CommandCode, Northflank or NVIDIA Brev, at the provider, and confirm the old key is dead.", "Pin LiteLLM to 1.106.2 by digest, and never pull latest on a gateway that holds keys.", "Set a hard spend cap per key, use one key per tool, and scope cloud model keys to the regions and models you use, so the next leak stays small." ], "remediation": [ { "window": "Contain now · 0-24h", "actions": [ "Rotate keys at the issuing provider (Anthropic, OpenAI, Google and others), not only in the custodian's dashboard, and check billing since Oct 6.", "LiteLLM: isolate any host or container that ran 1.107.0. [Do not start with pip](https://docs.litellm.ai/blog/security-incident-oct-21-2026), because any Python start re-runs the startup hooks. Delete cook.pth, cook.py and sitecustomize.py, rebuild from verified artifacts, and rotate everything the host could reach: proxy provider keys, master and virtual keys, DATABASE_URL, the IAM role and every environment secret.", "Railway and Fireworks: follow their notices. [Railway](https://blog.railway.com/p/preliminary-report-october-security-incident) recommends that every customer who signed up before Oct 21 rotate keys stored on Railway, including environment variables and registry credentials. [Fireworks](https://fireworks.ai/security/bulletins/october-2026-security-incident) says to rotate the third-party credentials named in your notice." ] }, { "window": "Investigate · 24-72h", "actions": [ "Block and search for www[.]litellm[.]dev and the GitHub dead drop in the indicators below. The marker /tmp/.pc_once_72129118c59f5636c0bb proves exfiltration if present, but its absence proves nothing.", "Check cloud audit logs for the instance role being used from outside your infrastructure after install time. LiteLLM suggests starting with CloudTrail GetCallerIdentity.", "AWS Bedrock: search CloudTrail for InvokeModel from unfamiliar addresses, Bedrock or EC2 quota listing, and PutAccountDataRetention calls. [Wiz](https://www.wiz.io/blog/black-october-for-ai-developer-platforms) saw all three on stolen keys." ] }, { "window": "Harden the agent surface", "actions": [ "Pin gateways and base images by digest. Do not pull a mutable tag such as latest, and verify the exact wheel, not only the source tag.", "Keep provider keys in a secrets manager you control, one key per tool, with a hard spend cap that stops traffic. On Bedrock, deny global cross-region inference profiles unless you need them.", "Ask every vendor that holds your keys how they are encrypted, how releases are approved, and how fast you will be told about a breach." ] } ], "indicators": [ "Malicious package: litellm 1.107.0 (PyPI, all seven wheels). Last safe version 1.106.2.", "Docker tags on Oct 21: v1.107.0, latest, main-latest, main-stable (digest sha256:1bbcb1443bc962e45f5aad53fe3d2c30600e0379715fb6c664cdd3ac126c339e)", "Files in site-packages: cook.pth, cook.py, sitecustomize.py", "Exfiltration: hxxp://www[.]litellm[.]dev/healthcheck. Dead drop: raw[.]githubusercontent[.]com/letters9813/pythonclient-test/main/domain.txt", "Key-checking addresses reported by Wiz (not tied directly to the original breaches): 23[.]106[.]46[.]239, 38[.]246[.]114[.]118, 104[.]243[.]32[.]50, 31[.]57[.]201[.]201", "Lock file /tmp/remote_code_loader.lock. Look-alike source in build provenance: Berri-AI/litellm (legitimate: BerriAI/litellm)" ], "sourcing": { "confirmed": [ "The vendors' own statements: [Railway](https://blog.railway.com/p/security-disclosure-wednesday-october-21st) and its [preliminary report](https://blog.railway.com/p/preliminary-report-october-security-incident), [LiteLLM](https://docs.litellm.ai/blog/security-incident-oct-21-2026) and its [attack chain](https://docs.litellm.ai/blog/security-incident-oct-21-2026-attack-chain), [Fireworks](https://fireworks.ai/security/bulletins/october-2026-security-incident) and [OpenCode](https://x.com/opencode/status/2111519637049319818).", "Technical analysis of the LiteLLM release by [JFrog](https://research.jfrog.com/post/litellm-hijacked-again/), [Socket](https://socket.dev/blog/litellm-compromise), [Wiz](https://www.wiz.io/blog/litellm-compromised-steals-credentials) and [Endor Labs](https://www.endorlabs.com/learn/litellm-1-107-0-compromised-python-startup-hooks-steal-cloud-ai-and-proxy-credentials), and reporting by [The Hacker News](https://thehackernews.com/2026/10/litellm-compromised-in-supply-chain.html) and [The Register](https://www.theregister.com/security/2026/10/21/litellm-supply-chain-attack-hoovers-up-passwords-and-secrets/5304548)." ], "assessed": [ "That the four incidents form one pattern aimed at provider keys is our read; Wiz also groups them, with five more, as one month of attacks. OpenCode and Railway name distillation. No source links the four to one actor.", "Counts of exposed accounts, keys and organizations are the vendors' or researchers' figures and are not independently verified. Railway is still investigating." ] }, "image": "/research/diagrams/ai-provider-key-custodians-trust-boundaries-nov01.png", "pdfUrl": "/__l5e/assets-v1/287c9d06-5cf8-456c-8b2a-ae54a29ba32a/ai-provider-key-custodians-breached-oct-2026-brief.pdf", "imageAlt": "Trust-boundary diagram of AI provider key custodians breached in October 2026, as of Nov 1, 2026. A customer tenant hands keys, environment variables and agent data to five custodians: Fireworks AI, OpenCode Console, the LiteLLM proxy, Railway and Warp, with other platforms reported by Wiz and an unconfirmed NVIDIA Brev report noted. Unknown threat actors reach them, and Wiz traced stolen AWS Bedrock keys tested within 30 minutes and resold", "sources": [ { "label": "Andromeda: October 2026 security incident", "url": "https://andromeda.ai/security-updates/october-2026-security-incident" }, { "label": "Zeabur: October 2026 security incident", "url": "https://zeabur.com/blogs/october-2026-security-incident" }, { "label": "Lightning AI: October 2026 incident", "url": "https://lightning.ai/blog/oct-2026-incident" }, { "label": "CommandCode on X", "url": "https://x.com/CommandCodeAI/status/2115583532055826770" }, { "label": "Northflank status page", "url": "https://status.northflank.com/" }, { "label": "Zach Lloyd (Warp CEO) on X, Oct 26", "url": "https://x.com/zachlloydtweets/status/2114800787716972873" }, { "label": "International Cyber Digest on X: NVIDIA Brev (unconfirmed)", "url": "https://x.com/IntCyberDigest/status/2116580207197094359" }, { "label": "Wiz: Black October for AI developer platforms", "url": "https://www.wiz.io/blog/black-october-for-ai-developer-platforms" }, { "label": "Railway security disclosure, Oct 21", "url": "https://blog.railway.com/p/security-disclosure-wednesday-october-21st" }, { "label": "Railway preliminary report, Oct 22", "url": "https://blog.railway.com/p/preliminary-report-october-security-incident" }, { "label": "Railway incident status", "url": "https://status.railway.com/incident/QH8L8B3E" }, { "label": "LiteLLM incident report, Oct 21", "url": "https://docs.litellm.ai/blog/security-incident-oct-21-2026" }, { "label": "LiteLLM attack chain, Oct 23", "url": "https://docs.litellm.ai/blog/security-incident-oct-21-2026-attack-chain" }, { "label": "LiteLLM security issue #47826", "url": "https://github.com/BerriAI/litellm/issues/47826" }, { "label": "The Hacker News: LiteLLM compromised", "url": "https://thehackernews.com/2026/10/litellm-compromised-in-supply-chain.html" }, { "label": "The Register: LiteLLM supply chain attack", "url": "https://www.theregister.com/security/2026/10/21/litellm-supply-chain-attack-hoovers-up-passwords-and-secrets/5304548" }, { "label": "JFrog: LiteLLM hijacked again", "url": "https://research.jfrog.com/post/litellm-hijacked-again/" }, { "label": "Socket: LiteLLM compromise", "url": "https://socket.dev/blog/litellm-compromise" }, { "label": "Wiz: LiteLLM compromised, steals credentials", "url": "https://www.wiz.io/blog/litellm-compromised-steals-credentials" }, { "label": "Endor Labs: LiteLLM 1.107.0", "url": "https://www.endorlabs.com/learn/litellm-1-107-0-compromised-python-startup-hooks-steal-cloud-ai-and-proxy-credentials" }, { "label": "Fireworks October 2026 security incident", "url": "https://fireworks.ai/security/bulletins/october-2026-security-incident" }, { "label": "LAVA: Fireworks AI incident", "url": "https://lava.security/blog/fireworks-ai-security-incident" }, { "label": "OpenCode statement on X, Oct 17", "url": "https://x.com/opencode/status/2111519637049319818" }, { "label": "Optimus Labs briefing #13: OpenCode Console", "url": "https://optimuslabs.io/research/briefings/opencode-console-breach-provider-key-exposure" }, { "label": "Optimus Labs briefing: Salesloft Drift OAuth supply chain", "url": "https://optimuslabs.io/research/briefings/salesloft-drift-oauth-supply-chain" }, { "label": "Optimus Labs briefing: arrayref poisoned Rust crate", "url": "https://optimuslabs.io/research/briefings/arrayref-rust-crate-build-time-rce" }, { "label": "Optimus Labs briefing: Coder registry hijack", "url": "https://optimuslabs.io/research/briefings/coder-registry-infrastructure-hijack" }, { "label": "Cloud Security Alliance: TeamPCP and AI infrastructure", "url": "https://labs.cloudsecurityalliance.org/research/csa-research-note-teampcp-ai-infra-supply-chain-20260406-csa/" }, { "label": "Ars Technica: terabytes of credentials leaked", "url": "https://arstechnica.com/security/2026/08/terabytes-of-credentials-leaked-in-massive-supply-chain-attack/" }, { "label": "Anthropic threat report, Sep 2026", "url": "https://www.anthropic.com/threat-intelligence-report-september-2026" }, { "label": "OWASP Top 10 for Agentic Applications 2026", "url": "https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/" }, { "label": "OWASP Top 10 for LLM Applications 2026", "url": "https://genai.owasp.org/resource/owasp-genai-llm-top-10-2026/" } ] } }