{
  "meta": {
    "title": "Interpretability Map",
    "curator": "Muhammad Zane Abdullah",
    "updated": "2026-07-01",
    "schema": "nodes are papers/instruments/venues; edges are prerequisite pairs [from,to]; layers are the linear reading order; problems are open experiments"
  },
  "tracks": {
    "found": {
      "label": "Foundations",
      "color": "#6D5BD0"
    },
    "feat": {
      "label": "Features",
      "color": "#C98A2B"
    },
    "circ": {
      "label": "Circuits",
      "color": "#BF5B3B"
    },
    "front": {
      "label": "Frontier",
      "color": "#B4487E"
    },
    "audit": {
      "label": "Auditing",
      "color": "#3E6DB5"
    },
    "tinker": {
      "label": "Instruments",
      "color": "#23876B"
    },
    "ext": {
      "label": "Adjacent work",
      "color": "#8A7350"
    }
  },
  "nodes": [
    {
      "id": "hub-found",
      "hub": true,
      "track": "found",
      "label": "FOUNDATIONS",
      "title": "Foundations",
      "meta": "the machine you're dissecting",
      "summary": "Mech interp is dissection, and you cannot dissect what you cannot picture. This lobe builds the working mental model, tokens, attention, MLPs, the residual stream, that every paper downstream assumes.",
      "r": 15,
      "effort": null,
      "links": []
    },
    {
      "id": "hub-feat",
      "hub": true,
      "track": "feat",
      "label": "FEATURES",
      "title": "Features",
      "meta": "the nouns of the model's mind",
      "summary": "Sparse autoencoders unmix superposition: expand activations into a wide dictionary, force sparsity, and the entries lock onto real concepts. The field's searchable index of what a model represents.",
      "r": 15,
      "effort": null,
      "links": []
    },
    {
      "id": "hub-circ",
      "hub": true,
      "track": "circ",
      "label": "CIRCUITS",
      "title": "Circuits",
      "meta": "the grammar of computation",
      "summary": "Features are nouns; circuits are the grammar, how representations combine into computation. The 'AI biology' era, run on Claude 3.5 Haiku because attribution graphs are computationally expensive.",
      "r": 15,
      "effort": null,
      "links": []
    },
    {
      "id": "hub-front",
      "hub": true,
      "track": "front",
      "label": "FRONTIER",
      "title": "The frontier",
      "meta": "2025 → now, at production scale",
      "summary": "The current wave builds readouts that scale to frontier models: activations translated directly into language, character as measurable directions, and internal workspaces holding thoughts that never reach the output.",
      "r": 15,
      "effort": null,
      "links": []
    },
    {
      "id": "hub-audit",
      "hub": true,
      "track": "audit",
      "label": "AUDITING",
      "title": "Alignment auditing",
      "meta": "the payoff, verification",
      "summary": "Behavior can lie: a model that knows it is being evaluated can game the test. Audits use interpretability to look underneath, the application that turns the whole field into a verification tool.",
      "r": 15,
      "effort": null,
      "links": []
    },
    {
      "id": "hub-tinker",
      "hub": true,
      "track": "tinker",
      "label": "INSTRUMENTS",
      "title": "Instruments & community",
      "meta": "where hands-on work happens",
      "summary": "Copy a working notebook into Colab and write code immediately; do not get involved in tech setup. These are the standard instruments and rooms of the field.",
      "r": 15,
      "effort": null,
      "links": []
    },
    {
      "id": "b3b1b",
      "hub": false,
      "track": "found",
      "label": "3Blue1Brown",
      "title": "3Blue1Brown, neural networks & transformers",
      "meta": "videos · ~4 hrs · start here",
      "summary": "The best visual intuition for attention and deep learning anywhere. Watch the full series before reading anything technical, the residual-stream picture it builds is the one the whole field uses.",
      "r": 8,
      "effort": [
        "~4 hrs",
        "start"
      ],
      "links": [
        [
          "watch",
          "https://www.3blue1brown.com/topics/neural-networks",
          "Neural networks series"
        ]
      ]
    },
    {
      "id": "nanda-guide",
      "hub": false,
      "track": "found",
      "label": "Nanda's guide",
      "title": "How to become a mech interp researcher",
      "meta": "Nanda · Sept 2025 · canonical onboarding",
      "summary": "The field's onboarding advice from the head of DeepMind's mech interp team. Learn the minimum viable basics (≤1 month, breadth-first), then learn by doing 1–5 day research mini-projects. Mech interp is an empirical science with short feedback loops and modest compute.",
      "r": 10,
      "effort": [
        "~2 hrs",
        "start"
      ],
      "links": [
        [
          "read",
          "https://www.alignmentforum.org/posts/jP9KDyMkchuv6tHwm/how-to-become-a-mechanistic-interpretability-researcher",
          "The guide, Alignment Forum"
        ],
        [
          "extra",
          "https://www.neelnanda.io/mechanistic-interpretability",
          "Nanda's mech interp hub"
        ]
      ]
    },
    {
      "id": "zoomin",
      "hub": false,
      "track": "found",
      "label": "Zoom In",
      "title": "Zoom In: an introduction to circuits",
      "meta": "Distill · 2020 · founding manifesto",
      "summary": "Olah and collaborators propose treating neural networks like organisms, features as cells, circuits as organs. Written about vision models, but it is the worldview Anthropic's interpretability team still runs on.",
      "r": 8,
      "effort": [
        "~1.5 hrs",
        "context"
      ],
      "links": [
        [
          "read",
          "https://distill.pub/2020/circuits/zoom-in/",
          "Zoom In, Distill"
        ]
      ]
    },
    {
      "id": "framework",
      "hub": false,
      "track": "found",
      "label": "Math framework",
      "title": "A mathematical framework for transformer circuits",
      "meta": "Anthropic · 2021 · canon",
      "summary": "Introduces the residual stream as the model's central communication channel, decomposes attention into QK (where to look) and OV (what to move) circuits, and discovers induction heads, the first nontrivial algorithm found inside a transformer.",
      "r": 11,
      "effort": [
        "~6 hrs",
        "core"
      ],
      "links": [
        [
          "read",
          "https://transformer-circuits.pub/2021/framework/index.html",
          "Paper, Transformer Circuits"
        ],
        [
          "extra",
          "https://transformer-circuits.pub/2022/in-context-learning-and-induction-heads/index.html",
          "Induction heads follow-up"
        ]
      ]
    },
    {
      "id": "toymodels",
      "hub": false,
      "track": "found",
      "label": "Superposition",
      "title": "Toy models of superposition",
      "meta": "Anthropic · 2022 · canon",
      "summary": "Why neurons are polysemantic: models represent more features than they have dimensions by storing them as overlapping directions, exploiting sparsity. The problem statement that motivates the entire features lobe.",
      "r": 11,
      "effort": [
        "~4 hrs",
        "core"
      ],
      "links": [
        [
          "read",
          "https://transformer-circuits.pub/2022/toy_model/index.html",
          "Paper, Transformer Circuits"
        ]
      ]
    },
    {
      "id": "nanda-pod",
      "hub": false,
      "track": "found",
      "label": "80k interview",
      "title": "Neel Nanda on the race to read AI minds",
      "meta": "80,000 Hours · 2025 · calibration",
      "summary": "Where the field's leaders think it stands: from 'fully reverse-engineer the model' to 'a high chance of a medium big deal.' Partial understanding genuinely helps evaluation, monitoring, and incident analysis, but there are no guarantees. Listen before investing months.",
      "r": 7,
      "effort": [
        "~3 hrs",
        "context"
      ],
      "links": [
        [
          "listen",
          "https://80000hours.org/podcast/episodes/neel-nanda-mechanistic-interpretability/",
          "Episode + transcript"
        ]
      ]
    },
    {
      "id": "monosem",
      "hub": false,
      "track": "feat",
      "label": "Monosemanticity",
      "title": "Towards monosemanticity",
      "meta": "Anthropic · 2023 · SAEs arrive",
      "summary": "Sparse autoencoders on a one-layer transformer pull thousands of clean, causal features, DNA, Hebrew, legal boilerplate, out of 512 polysemantic neurons. Features beat neurons decisively in blind interpretability ratings.",
      "r": 11,
      "effort": [
        "~4 hrs",
        "core"
      ],
      "links": [
        [
          "read",
          "https://transformer-circuits.pub/2023/monosemantic-features/index.html",
          "Paper, Transformer Circuits"
        ]
      ]
    },
    {
      "id": "scaling",
      "hub": false,
      "track": "feat",
      "label": "Scaling SAEs",
      "title": "Scaling monosemanticity",
      "meta": "Anthropic · 2024 · Claude 3 Sonnet",
      "summary": "The same recipe on a production model: millions of features, including abstract and multilingual ones, code bugs, inner conflict, sycophancy, deception. Clamp the Golden Gate Bridge feature and Claude identifies as the bridge. Steering via features, not prompts.",
      "r": 11,
      "effort": [
        "~3 hrs",
        "core"
      ],
      "links": [
        [
          "read",
          "https://transformer-circuits.pub/2024/scaling-monosemanticity/",
          "Paper, Transformer Circuits"
        ],
        [
          "summary",
          "https://www.anthropic.com/research/mapping-mind-language-model",
          "Anthropic summary"
        ]
      ]
    },
    {
      "id": "neuronpedia",
      "hub": false,
      "track": "tinker",
      "label": "Neuronpedia",
      "title": "Neuronpedia",
      "meta": "instrument · browse & steer real features",
      "summary": "An open catalog of SAE features across open models, see what activates them, test your own text, steer. 'Here is a weird feature family nobody has characterized' is a genuine first contribution, with zero GPU budget.",
      "r": 10,
      "effort": [
        "open-ended",
        "hands-on"
      ],
      "links": [
        [
          "tinker",
          "https://www.neuronpedia.org",
          "neuronpedia.org"
        ]
      ]
    },
    {
      "id": "tracing",
      "hub": false,
      "track": "circ",
      "label": "Tracing thoughts",
      "title": "Tracing the thoughts of a large language model",
      "meta": "Anthropic · Mar 2025 · the microscope",
      "summary": "Extends feature-finding into computational circuits: attribution graphs revealing the pathway from input words to output words. The accessible entry to the circuits pair.",
      "r": 10,
      "effort": [
        "~1 hr",
        "core"
      ],
      "links": [
        [
          "summary",
          "https://www.anthropic.com/research/tracing-thoughts-language-model",
          "Anthropic research post"
        ],
        [
          "read",
          "https://transformer-circuits.pub/2025/attribution-graphs/methods.html",
          "Methods: circuit tracing"
        ]
      ]
    },
    {
      "id": "biology",
      "hub": false,
      "track": "circ",
      "label": "AI biology",
      "title": "On the biology of a large language model",
      "meta": "Anthropic · Mar 2025 · ten case studies",
      "summary": "Deep dissections of Claude 3.5 Haiku: planning ahead in poetry, a shared multilingual conceptual space, parallel mental-math pathways, and cases where the stated chain of thought does not match the actual computation. Unfaithful reasoning, caught in the act.",
      "r": 11,
      "effort": [
        "~5 hrs",
        "core"
      ],
      "links": [
        [
          "read",
          "https://transformer-circuits.pub/2025/attribution-graphs/biology.html",
          "Paper, Transformer Circuits"
        ]
      ]
    },
    {
      "id": "tracer",
      "hub": false,
      "track": "tinker",
      "label": "circuit-tracer",
      "title": "Open circuit-tracing tools",
      "meta": "instrument · attribution graphs on open models",
      "summary": "Anthropic open-sourced the attribution-graph method so it runs on open-weights models, with graphs explorable on Neuronpedia. A realistic second project after ARENA.",
      "r": 8,
      "effort": [
        "a weekend",
        "hands-on"
      ],
      "links": [
        [
          "tinker",
          "https://github.com/safety-research/circuit-tracer",
          "circuit-tracer, GitHub"
        ],
        [
          "summary",
          "https://www.anthropic.com/research/open-source-circuit-tracing-tools",
          "Release post"
        ]
      ]
    },
    {
      "id": "nla",
      "hub": false,
      "track": "front",
      "label": "NLAs",
      "title": "Natural language autoencoders",
      "meta": "Anthropic · May 2026 · activations → English",
      "summary": "A verbalizer translates internal activations directly into readable English; a reconstructor rebuilds the activations from that explanation. Round-trip reconstruction keeps explanations honest, attacking the human-labeling bottleneck that limited SAEs.",
      "r": 10,
      "effort": [
        "~1.5 hrs",
        "frontier"
      ],
      "links": [
        [
          "read",
          "https://www.anthropic.com/research/natural-language-autoencoders",
          "Anthropic research post"
        ]
      ]
    },
    {
      "id": "workspace",
      "hub": false,
      "track": "front",
      "label": "Global workspace",
      "title": "A global workspace in language models",
      "meta": "Anthropic · Jul 2026 · unverbalized thoughts",
      "summary": "Evidence for an emergent mental workspace holding internal assessments that never appear in output, with swap experiments establishing causality, commentary from global-workspace neuroscientists, and an open-weights replication. Ships with a repository and a Neuronpedia demo.",
      "r": 10,
      "effort": [
        "~2 hrs",
        "frontier"
      ],
      "links": [
        [
          "read",
          "https://www.anthropic.com/research/global-workspace",
          "Anthropic research post"
        ]
      ]
    },
    {
      "id": "persona",
      "hub": false,
      "track": "front",
      "label": "Persona vectors",
      "title": "Persona vectors, introspection & the assistant axis",
      "meta": "Anthropic · 2025–26 · character as geometry",
      "summary": "A three-paper arc: traits like sycophancy exist as directions in activation space that can be monitored and steered; models show limited but functional introspection; the 'Assistant' persona itself sits on a measurable, stabilizable axis.",
      "r": 9,
      "effort": [
        "~3 hrs",
        "frontier"
      ],
      "links": [
        [
          "read",
          "https://www.anthropic.com/research/persona-vectors",
          "Persona vectors"
        ],
        [
          "read",
          "https://www.anthropic.com/research/introspection",
          "Signs of introspection"
        ],
        [
          "read",
          "https://www.anthropic.com/research/assistant-axis",
          "The assistant axis"
        ]
      ]
    },
    {
      "id": "hidden",
      "hub": false,
      "track": "audit",
      "label": "Hidden objectives",
      "title": "Auditing language models for hidden objectives",
      "meta": "Anthropic · 2025 · the King Lear paper",
      "summary": "A model is deliberately trained with a hidden misaligned objective (reward-model sycophancy); four blinded teams hunt it using training-data analysis, SAEs, and behavioral probing. Three of four succeed, the existence proof that audits can catch a concealed goal.",
      "r": 10,
      "effort": [
        "~2 hrs",
        "core"
      ],
      "links": [
        [
          "summary",
          "https://www.anthropic.com/research/auditing-hidden-objectives",
          "Anthropic research post"
        ]
      ]
    },
    {
      "id": "agents",
      "hub": false,
      "track": "audit",
      "label": "Auditing agents",
      "title": "Automated auditing agents & AuditBench",
      "meta": "Anthropic · 2025–26 · scaling the auditor",
      "summary": "Three agents, investigator, evaluator, red-teamer, audit models autonomously. One agent finds a planted objective 10–13% of the time; ensembles reach 42%. AuditBench provides 56 models with implanted hidden behaviors as a public benchmark; the evaluation agent is open-sourced.",
      "r": 9,
      "effort": [
        "~2 hrs",
        "frontier"
      ],
      "links": [
        [
          "read",
          "https://alignment.anthropic.com/2025/automated-auditing/",
          "Building auditing agents"
        ],
        [
          "extra",
          "https://alignment.anthropic.com",
          "Alignment Science blog"
        ]
      ]
    },
    {
      "id": "openprob",
      "hub": false,
      "track": "audit",
      "label": "Open problems",
      "title": "Open problems in mechanistic interpretability",
      "meta": "arXiv · 2025 · Sharkey, Nanda & 27 others",
      "summary": "The field's own map of its holes: method improvements needed, how to apply methods toward concrete goals, and socio-technical challenges. Read it after ARENA chapters 1–2, a map of gaps only makes sense once you know the terrain.",
      "r": 8,
      "effort": [
        "~4 hrs",
        "survey"
      ],
      "links": [
        [
          "read",
          "https://arxiv.org/abs/2501.16496",
          "Paper, arXiv"
        ]
      ]
    },
    {
      "id": "200prob",
      "hub": false,
      "track": "audit",
      "label": "200 problems",
      "title": "200 concrete open problems in interpretability",
      "meta": "Nanda · sequence · rated by difficulty",
      "summary": "The classic problem list, rated A (tractable for a newcomer in days) through harder tiers. The stated calibration: a newcomer should get traction on anything rated A, and maybe B. Pick one that jumps out and run, do not get paralyzed by choice.",
      "r": 8,
      "effort": [
        "~2 hrs",
        "survey"
      ],
      "links": [
        [
          "read",
          "https://www.alignmentforum.org/s/yivyHaCAmMJ3CqSyj",
          "Sequence, Alignment Forum"
        ]
      ]
    },
    {
      "id": "arena",
      "hub": false,
      "track": "tinker",
      "label": "ARENA",
      "title": "ARENA, the practical curriculum",
      "meta": "course · free · start here",
      "summary": "The de facto onboarding path: hands-on chapters on transformer internals, interpretability techniques, and SAEs, all in runnable notebooks with solutions. Chapters 1–2 are the minimum foundation for everything else here.",
      "r": 11,
      "effort": [
        "3–4 wks",
        "hands-on"
      ],
      "links": [
        [
          "tinker",
          "https://www.arena.education",
          "arena.education"
        ]
      ]
    },
    {
      "id": "tlens",
      "hub": false,
      "track": "tinker",
      "label": "TransformerLens",
      "title": "TransformerLens",
      "meta": "instrument · the standard research library",
      "summary": "The standard library for loading open models with hooks into every internal activation, the tool nearly every mini-project is built with. Start from its Main Demo notebook in Colab (Protocol 1).",
      "r": 10,
      "effort": [
        "a weekend",
        "hands-on"
      ],
      "links": [
        [
          "tinker",
          "https://github.com/TransformerLensOrg/TransformerLens",
          "TransformerLens, GitHub"
        ]
      ]
    },
    {
      "id": "saelens",
      "hub": false,
      "track": "tinker",
      "label": "SAELens",
      "title": "SAELens + open SAE suites",
      "meta": "instrument · train & analyze the SDL family",
      "summary": "Train your own sparse autoencoders or load pretrained suites and analyze them. DeepMind's Gemma Scope, extended in 2025 to cover the Gemma 3 family from 270M to 27B, is the largest open dictionary release; pairs directly with Neuronpedia for visualization (Protocol 2).",
      "r": 8,
      "effort": [
        "a weekend",
        "hands-on"
      ],
      "links": [
        [
          "tinker",
          "https://github.com/jbloomAus/SAELens",
          "SAELens, GitHub"
        ],
        [
          "extra",
          "https://huggingface.co/google/gemma-scope",
          "Gemma Scope pretrained SAEs"
        ]
      ]
    },
    {
      "id": "mats",
      "hub": false,
      "track": "tinker",
      "label": "MATS",
      "title": "MATS & the research community",
      "meta": "program · the mentorship pipeline",
      "summary": "The research-mentorship program most new interpretability researchers come through. Alongside it: the Alignment Forum for work-in-progress and fast feedback.",
      "r": 8,
      "effort": [
        "program",
        "community"
      ],
      "links": [
        [
          "join",
          "https://www.matsprogram.org",
          "matsprogram.org"
        ],
        [
          "read",
          "https://www.alignmentforum.org/topics/interpretability-ml-and-ai",
          "Alignment Forum, interp tag"
        ]
      ]
    },
    {
      "id": "tcpub",
      "hub": false,
      "track": "tinker",
      "label": "transformer-circuits",
      "title": "Transformer Circuits, the journal",
      "meta": "venue · where the canon lives",
      "summary": "Anthropic's interpretability publication venue, every canonical paper here lives in full. The accessible summaries live on anthropic.com/research; read those first.",
      "r": 9,
      "effort": [
        "ongoing",
        "venue"
      ],
      "links": [
        [
          "read",
          "https://transformer-circuits.pub",
          "transformer-circuits.pub"
        ],
        [
          "extra",
          "https://www.anthropic.com/research/team/interpretability",
          "Anthropic interpretability team"
        ]
      ]
    },
    {
      "id": "sdl",
      "hub": false,
      "track": "feat",
      "label": "Transcoders & crosscoders",
      "title": "Transcoders, crosscoders & cross-layer transcoders",
      "meta": "2024–25 · the SDL family beyond SAEs",
      "summary": "SAEs' successors in the sparse-dictionary-learning family: transcoders model a sublayer's input→output computation rather than autoencoding activations; crosscoders span multiple layers (and enable model diffing between base and fine-tuned models); cross-layer transcoders are the substrate that made attribution graphs tractable. If you read one methods thread after SAEs, read this one.",
      "r": 10,
      "effort": [
        "~4 hrs",
        "core"
      ],
      "links": [
        [
          "read",
          "https://transformer-circuits.pub/2024/crosscoders/index.html",
          "Sparse crosscoders (Anthropic)"
        ],
        [
          "read",
          "https://arxiv.org/abs/2406.11944",
          "Transcoders (Dunefsky et al.)"
        ],
        [
          "extra",
          "https://transformer-circuits.pub/2025/attribution-graphs/methods.html",
          "CLTs in the attribution-graph methods"
        ]
      ]
    },
    {
      "id": "hub-ext",
      "hub": true,
      "track": "ext",
      "label": "ADJACENT WORK",
      "title": "Adjacent work",
      "meta": "DeepMind, OpenAI & the academic canon",
      "summary": "The Anthropic thread is the spine of this map, but the field is a conversation across labs. This lobe holds the adjacent canon a complete researcher needs: the academic circuit dissections, OpenAI's automated-interpretability and SAE lines, and the model-editing and world-model results.",
      "r": 15,
      "effort": null,
      "links": []
    },
    {
      "id": "ioi",
      "hub": false,
      "track": "ext",
      "label": "IOI circuit",
      "title": "Interpretability in the wild: the IOI circuit",
      "meta": "Wang et al. · 2022 · Redwood · academic canon",
      "summary": "The first end-to-end circuit for a nontrivial behavior in GPT-2: indirect object identification ('John gave a drink to,') traced through 26 attention heads in 7 classes. Established the workflow, and the standards of causal evidence, that circuit papers still follow.",
      "r": 9,
      "effort": [
        "~4 hrs",
        "core"
      ],
      "links": [
        [
          "read",
          "https://arxiv.org/abs/2211.00593",
          "Paper, arXiv"
        ]
      ]
    },
    {
      "id": "rome",
      "hub": false,
      "track": "ext",
      "label": "ROME",
      "title": "Locating and editing factual associations (ROME)",
      "meta": "Meng et al. · 2022 · academic canon",
      "summary": "Causal tracing localizes where a fact ('The Eiffel Tower is in Paris') lives in a model, then edits it surgically. The founding result of the knowledge-localization and model-editing line, and a standing caution, since later work showed localization and editing come apart.",
      "r": 9,
      "effort": [
        "~3 hrs",
        "context"
      ],
      "links": [
        [
          "read",
          "https://arxiv.org/abs/2202.05262",
          "Paper, arXiv"
        ],
        [
          "extra",
          "https://rome.baulab.info",
          "Project page, Bau Lab"
        ]
      ]
    },
    {
      "id": "othello",
      "hub": false,
      "track": "ext",
      "label": "Othello-GPT",
      "title": "Emergent world representations (Othello-GPT)",
      "meta": "Li et al. · 2023 · + the linear-probe follow-up",
      "summary": "A transformer trained only on Othello move sequences develops an internal board-state model, evidence that sequence prediction induces world models. Nanda's follow-up showed the representation is linear, a key data point for the linear representation hypothesis.",
      "r": 8,
      "effort": [
        "~2 hrs",
        "context"
      ],
      "links": [
        [
          "read",
          "https://arxiv.org/abs/2210.13382",
          "Paper, arXiv"
        ],
        [
          "extra",
          "https://www.alignmentforum.org/posts/nmxzr2zsjNtjaHh7x/actually-othello-gpt-has-a-linear-emergent-world",
          "Nanda's linear-probe follow-up"
        ]
      ]
    },
    {
      "id": "openai-neurons",
      "hub": false,
      "track": "ext",
      "label": "LMs explain neurons",
      "title": "Language models can explain neurons in language models",
      "meta": "OpenAI · 2023 · automated interpretability",
      "summary": "GPT-4 writes and scores natural-language explanations for every neuron in GPT-2, the founding result of automated interpretability, and the direct ancestor of verbalization methods like natural language autoencoders.",
      "r": 8,
      "effort": [
        "~1.5 hrs",
        "context"
      ],
      "links": [
        [
          "read",
          "https://openai.com/research/language-models-can-explain-neurons-in-language-models",
          "OpenAI research post"
        ]
      ]
    },
    {
      "id": "openai-sae",
      "hub": false,
      "track": "ext",
      "label": "OpenAI SAEs",
      "title": "Scaling and evaluating sparse autoencoders",
      "meta": "Gao et al. · OpenAI · 2024 · 16M latents on GPT-4",
      "summary": "OpenAI's entry in the SAE race: TopK autoencoders scaled to 16 million latents on GPT-4, with clean scaling laws and evaluation metrics. Read alongside Anthropic's Scaling Monosemanticity for the cross-lab picture.",
      "r": 8,
      "effort": [
        "~3 hrs",
        "core"
      ],
      "links": [
        [
          "read",
          "https://arxiv.org/abs/2406.04093",
          "Paper, arXiv"
        ]
      ]
    },
    {
      "id": "nnsight",
      "hub": false,
      "track": "tinker",
      "label": "nnsight / NDIF",
      "title": "nnsight & the National Deep Inference Fabric",
      "meta": "instrument · remote internals of large models",
      "summary": "Open the hood on models too big for your GPU: nnsight expresses interventions on any PyTorch model, and NDIF executes them remotely on large open models (Llama-class) with full access to internals. The answer to 'but I can't afford to run a 70B model.'",
      "r": 9,
      "effort": [
        "a weekend",
        "hands-on"
      ],
      "links": [
        [
          "tinker",
          "https://nnsight.net",
          "nnsight.net"
        ],
        [
          "extra",
          "https://ndif.us",
          "NDIF"
        ]
      ]
    },
    {
      "id": "pythia",
      "hub": false,
      "track": "tinker",
      "label": "Pythia",
      "title": "Pythia, a suite for studying training dynamics",
      "meta": "EleutherAI · open suite · 154 checkpoints per model",
      "summary": "Sixteen models (70M–12B) trained on identical data in identical order, with 154 checkpoints each, the standard laboratory for questions about how features and circuits form during training.",
      "r": 8,
      "effort": [
        "open-ended",
        "hands-on"
      ],
      "links": [
        [
          "tinker",
          "https://github.com/EleutherAI/pythia",
          "Pythia, GitHub"
        ],
        [
          "read",
          "https://arxiv.org/abs/2304.01373",
          "Paper, arXiv"
        ]
      ]
    },
    {
      "id": "olmo",
      "hub": false,
      "track": "tinker",
      "label": "OLMo",
      "title": "OLMo, fully open models, data included",
      "meta": "Ai2 · open weights + open training data",
      "summary": "The only frontier-adjacent models where you can inspect the training data behind a feature. When a question needs 'what did the model see that made it learn this?', OLMo is the laboratory.",
      "r": 8,
      "effort": [
        "open-ended",
        "hands-on"
      ],
      "links": [
        [
          "tinker",
          "https://allenai.org/olmo",
          "allenai.org/olmo"
        ]
      ]
    },
    {
      "id": "goodfire",
      "hub": false,
      "track": "tinker",
      "label": "Goodfire Ember",
      "title": "Goodfire Ember, hosted interpretability API",
      "meta": "instrument · commercial · steer via API",
      "summary": "A hosted API exposing features of open models for reading and steering without any local infrastructure, and the existence proof that interpretability supports a commercial product. Useful for steering-reliability experiments at scale.",
      "r": 8,
      "effort": [
        "~2 hrs",
        "hands-on"
      ],
      "links": [
        [
          "tinker",
          "https://www.goodfire.ai",
          "goodfire.ai"
        ]
      ]
    }
  ],
  "edges": [
    [
      "hub-found",
      "hub-feat"
    ],
    [
      "hub-feat",
      "hub-circ"
    ],
    [
      "hub-circ",
      "hub-front"
    ],
    [
      "hub-front",
      "hub-audit"
    ],
    [
      "hub-found",
      "hub-tinker"
    ],
    [
      "hub-found",
      "b3b1b"
    ],
    [
      "hub-found",
      "nanda-guide"
    ],
    [
      "hub-found",
      "zoomin"
    ],
    [
      "hub-found",
      "framework"
    ],
    [
      "hub-found",
      "toymodels"
    ],
    [
      "hub-found",
      "nanda-pod"
    ],
    [
      "framework",
      "toymodels"
    ],
    [
      "nanda-guide",
      "nanda-pod"
    ],
    [
      "hub-feat",
      "monosem"
    ],
    [
      "hub-feat",
      "scaling"
    ],
    [
      "toymodels",
      "monosem"
    ],
    [
      "monosem",
      "scaling"
    ],
    [
      "scaling",
      "neuronpedia"
    ],
    [
      "hub-circ",
      "tracing"
    ],
    [
      "hub-circ",
      "biology"
    ],
    [
      "tracing",
      "biology"
    ],
    [
      "scaling",
      "tracing"
    ],
    [
      "biology",
      "tracer"
    ],
    [
      "tracer",
      "neuronpedia"
    ],
    [
      "hub-front",
      "nla"
    ],
    [
      "hub-front",
      "workspace"
    ],
    [
      "hub-front",
      "persona"
    ],
    [
      "scaling",
      "nla"
    ],
    [
      "nla",
      "workspace"
    ],
    [
      "hub-audit",
      "hidden"
    ],
    [
      "hub-audit",
      "agents"
    ],
    [
      "hub-audit",
      "openprob"
    ],
    [
      "hub-audit",
      "200prob"
    ],
    [
      "scaling",
      "hidden"
    ],
    [
      "hidden",
      "agents"
    ],
    [
      "persona",
      "hidden"
    ],
    [
      "nanda-guide",
      "200prob"
    ],
    [
      "openprob",
      "200prob"
    ],
    [
      "hub-tinker",
      "arena"
    ],
    [
      "hub-tinker",
      "tlens"
    ],
    [
      "hub-tinker",
      "saelens"
    ],
    [
      "hub-tinker",
      "mats"
    ],
    [
      "hub-tinker",
      "neuronpedia"
    ],
    [
      "hub-tinker",
      "tcpub"
    ],
    [
      "arena",
      "tlens"
    ],
    [
      "tlens",
      "saelens"
    ],
    [
      "saelens",
      "neuronpedia"
    ],
    [
      "framework",
      "tcpub"
    ],
    [
      "nanda-guide",
      "arena"
    ],
    [
      "nanda-guide",
      "mats"
    ],
    [
      "hub-feat",
      "sdl"
    ],
    [
      "scaling",
      "sdl"
    ],
    [
      "sdl",
      "tracing"
    ],
    [
      "sdl",
      "saelens"
    ],
    [
      "hub-ext",
      "hub-found"
    ],
    [
      "hub-ext",
      "ioi"
    ],
    [
      "hub-ext",
      "rome"
    ],
    [
      "hub-ext",
      "othello"
    ],
    [
      "hub-ext",
      "openai-neurons"
    ],
    [
      "hub-ext",
      "openai-sae"
    ],
    [
      "framework",
      "ioi"
    ],
    [
      "ioi",
      "tracing"
    ],
    [
      "openai-neurons",
      "nla"
    ],
    [
      "openai-sae",
      "scaling"
    ],
    [
      "toymodels",
      "othello"
    ],
    [
      "hub-tinker",
      "nnsight"
    ],
    [
      "hub-tinker",
      "pythia"
    ],
    [
      "hub-tinker",
      "olmo"
    ],
    [
      "hub-tinker",
      "goodfire"
    ],
    [
      "nnsight",
      "tlens"
    ],
    [
      "pythia",
      "tlens"
    ],
    [
      "goodfire",
      "neuronpedia"
    ]
  ],
  "layers": [
    {
      "layer": "L0",
      "name": "Before the field",
      "rationale": "Get the working mental model, tokens, attention, MLPs, the residual stream, and calibrate expectations from the field's own leadership.",
      "refs": [
        "b3b1b",
        "nanda-guide",
        "zoomin",
        "nanda-pod"
      ]
    },
    {
      "layer": "L1",
      "name": "Conceptual core",
      "rationale": "Two papers define the language everything downstream assumes: the mathematics of what attention heads do, and why neurons are not the right unit of analysis.",
      "refs": [
        "framework",
        "toymodels"
      ]
    },
    {
      "layer": "L2",
      "name": "Features",
      "rationale": "Sparse autoencoders unmix superposition into a searchable dictionary of concepts, first on a toy model, then on a production frontier model.",
      "refs": [
        "monosem",
        "scaling",
        "neuronpedia"
      ]
    },
    {
      "layer": "L3",
      "name": "Circuits",
      "rationale": "Attribution graphs turn features into causal stories of computation. The biology era, with ten dissections of real model behaviors.",
      "refs": [
        "tracing",
        "biology",
        "tracer"
      ]
    },
    {
      "layer": "L4",
      "name": "The frontier",
      "rationale": "Readouts that scale to frontier models: activations verbalized in English, character as measurable geometry, and internal workspaces holding unverbalized thoughts.",
      "refs": [
        "nla",
        "workspace",
        "persona"
      ]
    },
    {
      "layer": "L5",
      "name": "Instruments",
      "rationale": "The standard laboratory: curriculum, libraries, feature catalogs, and the mentorship pipeline. Protocols in Part 3 put these to work.",
      "refs": [
        "arena",
        "tlens",
        "saelens",
        "mats",
        "tcpub"
      ]
    },
    {
      "layer": "L6",
      "name": "Auditing & open problems",
      "rationale": "The payoff: interpretability as a verification tool for models whose behavior can lie, and the problem lists where new researchers contribute.",
      "refs": [
        "hidden",
        "agents",
        "openprob",
        "200prob"
      ]
    },
    {
      "layer": "S1",
      "name": "Adjacent canon (supplementary)",
      "rationale": "The cross-lab conversation: the academic circuit dissections that set the field's evidential standards, OpenAI's automated-interpretability and SAE lines, and the SDL family beyond SAEs. Read interleaved with L2–L3.",
      "refs": [
        "ioi",
        "rome",
        "othello",
        "openai-neurons",
        "openai-sae",
        "sdl"
      ]
    }
  ],
  "problems": [
    {
      "d": "A",
      "theme": "features",
      "title": "Characterize an unmapped feature family",
      "why": "Millions of open SAE features exist; most have never been examined by a human. A careful catalog of one coherent family, with activation conditions, failure cases, and steering behavior, is a real contribution requiring zero GPU budget.",
      "steps": [
        "Browse Neuronpedia for a Gemma Scope layer and pick a family (e.g. temporal reasoning, negation, uncertainty markers).",
        "Document 15–20 features: max-activating examples, your own probe texts, edge cases where the label breaks.",
        "Steer each feature and record whether behavior matches the label.",
        "Write it up as a short post with a features spreadsheet; share on the Alignment Forum."
      ],
      "refs": [
        [
          "Neuronpedia",
          "https://www.neuronpedia.org"
        ],
        [
          "Gemma Scope",
          "https://huggingface.co/google/gemma-scope"
        ]
      ]
    },
    {
      "d": "A",
      "theme": "methods · very current",
      "title": "Probes vs. SAEs on a downstream task",
      "why": "DeepMind reported linear probes beating SAEs on out-of-distribution safety detection and deprioritized fundamental SAE work, the field's live argument. Extending the comparison to one new task is a small, publishable experiment.",
      "steps": [
        "Pick a binary property (refusal intent, sarcasm, a language) and build a small labeled activation dataset from Gemma 2.",
        "Train a linear probe on raw activations; compare against the best single SAE feature and a sparse probe over SAE features.",
        "Test out-of-distribution: new phrasing, new domain.",
        "Report which representation wins, where, and by how much, either result is informative."
      ],
      "refs": [
        [
          "SAELens",
          "https://github.com/jbloomAus/SAELens"
        ],
        [
          "TransformerLens",
          "https://github.com/TransformerLensOrg/TransformerLens"
        ]
      ]
    },
    {
      "d": "A",
      "theme": "circuits",
      "title": "Replicate the IOI circuit, then break it",
      "why": "The classic dissection is fully documented and runs on GPT-2 small. Replication builds every core skill; the contribution is in the second step, finding where the published circuit's story is incomplete.",
      "steps": [
        "Reproduce the indirect-object-identification results with TransformerLens (ARENA has a guided version).",
        "Construct adversarial variants of the task, extra names, distractor clauses, and measure where the circuit's components stop explaining behavior.",
        "Patch to localize what takes over.",
        "Write up the divergence: where the canonical story holds and where it leaks."
      ],
      "refs": [
        [
          "IOI paper",
          "https://arxiv.org/abs/2211.00593"
        ],
        [
          "ARENA",
          "https://www.arena.education"
        ]
      ]
    },
    {
      "d": "B",
      "theme": "features",
      "title": "Measure the dark matter of dictionary learning",
      "why": "SAE reconstructions leave a residual error, activation mass no feature explains. Nobody fully knows what lives there. Characterizing the dark matter on one model/layer is a direct attack on a named open problem.",
      "steps": [
        "Load a Gemma Scope SAE; compute reconstruction error across a diverse corpus.",
        "Ask where error concentrates: token types, positions, syntactic contexts.",
        "Patch the residual alone into a clean run, what behavior does the unexplained mass carry?",
        "Compare across layer depth and dictionary width."
      ],
      "refs": [
        [
          "Open Problems SDL section",
          "https://arxiv.org/abs/2501.16496"
        ],
        [
          "Gemma Scope",
          "https://huggingface.co/google/gemma-scope"
        ]
      ]
    },
    {
      "d": "B",
      "theme": "faithfulness",
      "title": "Chain-of-thought vs. the actual computation",
      "why": "The biology paper caught models whose stated reasoning did not match their internal computation. Systematically measuring that gap on an open model is among the most safety-relevant experiments a newcomer can run.",
      "steps": [
        "Pick arithmetic or multi-hop factual tasks on an open model with circuit-tracer support.",
        "Collect the model's stated chain of thought; independently trace the attribution graph for the same answers.",
        "Score agreement between stated steps and mechanistic pathway.",
        "Report the conditions under which verbalized reasoning is and isn't faithful."
      ],
      "refs": [
        [
          "circuit-tracer",
          "https://github.com/safety-research/circuit-tracer"
        ],
        [
          "Biology paper",
          "https://transformer-circuits.pub/2025/attribution-graphs/biology.html"
        ]
      ]
    },
    {
      "d": "B",
      "theme": "universality",
      "title": "Do two models learn the same features?",
      "why": "Universality, whether independently trained models converge on shared features, bears on whether interpretability findings transfer. Crosscoders make the comparison tractable.",
      "steps": [
        "Take two Pythia sizes (or a base vs. chat-tuned pair).",
        "Train or load dictionaries for a matched layer; align feature spaces by activation correlation or a crosscoder.",
        "Quantify overlap: which families are shared, which are idiosyncratic.",
        "Bonus: use Pythia checkpoints to watch shared features emerge during training."
      ],
      "refs": [
        [
          "Pythia",
          "https://github.com/EleutherAI/pythia"
        ],
        [
          "Crosscoders",
          "https://transformer-circuits.pub/2024/crosscoders/index.html"
        ]
      ]
    },
    {
      "d": "B",
      "theme": "steering",
      "title": "When does steering actually work?",
      "why": "Feature steering is the field's flagship demo, but its reliability envelope is unmapped: strength, context length, feature interference. A systematic evaluation is directly useful to everyone building on steering.",
      "steps": [
        "Select 20 well-labeled features across categories (style, topic, safety-relevant).",
        "Sweep clamp strengths; measure intended effect vs. collateral damage (perplexity, off-target behavior).",
        "Test compositionality: steer two features at once.",
        "Publish the reliability envelope, the field lacks exactly this table."
      ],
      "refs": [
        [
          "Neuronpedia steering",
          "https://www.neuronpedia.org"
        ],
        [
          "Goodfire Ember",
          "https://www.goodfire.ai"
        ]
      ]
    },
    {
      "d": "C",
      "theme": "auditing",
      "title": "Break the auditors on AuditBench",
      "why": "Ensembled auditing agents find planted objectives 42% of the time, meaning 58% escape. Characterizing which defect classes evade which methods is frontier work with public ground truth to score against.",
      "steps": [
        "Run the open-sourced evaluation agent against a subset of AuditBench's 56 planted models.",
        "Add one interpretability probe of your own (SAE features, steering perturbations) to the loop.",
        "Build the confusion matrix: defect class × detection method.",
        "Report the systematic blind spots, this is the validation gap Anthropic names as the open challenge."
      ],
      "refs": [
        [
          "Automated auditing",
          "https://alignment.anthropic.com/2025/automated-auditing/"
        ],
        [
          "Alignment Science blog",
          "https://alignment.anthropic.com"
        ]
      ]
    }
  ]
}