@inproceedings{vaswani2017attention, title={Attention is all you need}, author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia}, booktitle={Advances in neural information processing systems}, pages={5998--6008}, year={2017}, url={https://proceedings.neurips.cc/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf} } @article{cammarata2020thread, author = {Cammarata, Nick and Carter, Shan and Goh, Gabriel and Olah, Chris and Petrov, Michael and Schubert, Ludwig and Voss, Chelseagur and Egan, Ben and Lim, Swee Kiat}, title = {Thread: Circuits}, journal = {Distill}, year = {2020}, url = {https://distill.pub/2020/circuits} } @inproceedings{mikolov2013linguistic, title={Linguistic regularities in continuous space word representations}, author={Mikolov, Tom{\'a}{\v{s}} and Yih, Wen-tau and Zweig, Geoffrey}, booktitle={Proceedings of the 2013 conference of the north american chapter of the association for computational linguistics: Human language technologies}, pages={746--751}, year={2013}, url={https://aclanthology.org/N13-1090.pdf} } @article{brown2020language, title={Language models are few-shot learners}, author={Brown, Tom B and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and others}, journal={arXiv preprint arXiv:2005.14165}, year={2020}, url={https://arxiv.org/pdf/2005.14165} } @article{raffel2019exploring, title={Exploring the limits of transfer learning with a unified text-to-text transformer}, author={Raffel, Colin and Shazeer, Noam and Roberts, Adam and Lee, Katherine and Narang, Sharan and Matena, Michael and Zhou, Yanqi and Li, Wei and Liu, Peter J}, journal={arXiv preprint arXiv:1910.10683}, year={2019}, url={https://www.jmlr.org/papers/volume21/20-074/20-074.pdf} } @article{rae2021scaling, title={Scaling Language Models: Methods, Analysis & Insights from Training Gopher}, author={Rae, Jack W. and Borgeaud, Sebastian and Cai, Trevor and Millican, Katie and Hoffmann, Jordan and Song, Francis and Aslanides, John and Henderson, Sarah and Ring, Roman and Young, Susannah and Rutherford, Eliza and Hennigan, Tom and Menick, Jacob and Cassirer, Albin and Powell, Richard and Driessche, George van den and Hendricks, Lisa Anne and Rauh, Maribeth and Huang, Po-Sen and Glaese, Amelia and Welbl, Johannes and Dathathri, Sumanth and Huang, Saffron and Uesato, Jonathan and Mellor, John and Higgins, Irina and Creswell, Antonia and McAleese, Nat and Wu, Amy and Elsen, Erich and Jayakumar, Siddhant and Buchatskaya, Elena and Budden, David and Sutherland, Esme and Simonyan, Karen and Paganini, Michela and Sifre, Laurent and Martens, Lena and Li, Xiang Lorraine and Kuncoro, Adhiguna and Nematzadeh, Aida and Gribovskaya, Elena and Donato, Domenic and Lazaridou, Angeliki and Mensch, Arthur and Lespiau, Jean-Baptiste and Tsimpoukelli, Maria and Grigorev, Nikolai and Fritz, Doug and Sottiaux, Thibault and Pajarskas, Mantas and Pohlen, Toby and Gong, Zhitao and Toyama, Daniel and d’Autume, Cyprien de Masson and Li, Yujia and Terzi, Tayfun and Mikulik, Vladimir and Babuschkin, Igor and Clark, Aidan and Casas, Diego de Las and Guy, Aurelia and Jones, Chris and Bradbury, James and Johnson, Matthew and Hechtman, Blake and Weidinger, Laura and Gabriel, Iason and Isaac, William and Lockhart, Ed and Osindero, Simon and Rimell, Laura and Dyer, Chris and Vinyals, Oriol and Ayoub, Kareem and Stanway, Jeff and Bennett, Lorrayne and Hassabis, Demis and Kavukcuoglu, Koray and Irving, Geoffrey}, journal={Preprint}, year={2021}, url={https://storage.googleapis.com/deepmind-media/research/language-research/Training%20Gopher.pdf} } @article{nakatsukasa2019low, title={The low-rank eigenvalue problem}, author={Nakatsukasa, Yuji}, journal={arXiv preprint arXiv:1905.11490}, year={2019}, url={https://arxiv.org/pdf/1905.11490} } @article{tarnowski2021real, title={Real spectra of large real asymmetric random matrices}, author={Tarnowski, Wojciech}, journal={arXiv preprint arXiv:2104.02584}, year={2021}, url={https://arxiv.org/pdf/2104.02584} } @article{vig2019multiscale, title={A multiscale visualization of attention in the transformer model}, author={Vig, Jesse}, journal={arXiv preprint arXiv:1906.05714}, year={2019}, url={https://arxiv.org/pdf/1906.05714} } @article{voita2019analyzing, title={Analyzing multi-head self-attention: Specialized heads do the heavy lifting, the rest can be pruned}, author={Voita, Elena and Talbot, David and Moiseev, Fedor and Sennrich, Rico and Titov, Ivan}, journal={arXiv preprint arXiv:1905.09418}, year={2019}, url={https://arxiv.org/pdf/1905.09418} } @article{clark2019does, title={What does bert look at? an analysis of bert's attention}, author={Clark, Kevin and Khandelwal, Urvashi and Levy, Omer and Manning, Christopher D}, journal={arXiv preprint arXiv:1906.04341}, year={2019}, url={https://arxiv.org/pdf/1906.04341} } @misc{jones2017tensor2tensor, title={Tensor2tensor transformer visualization}, author={Jones, Llion}, year={2017}, url={https://github.com/tensorflow/tensor2tensor/tree/master/tensor2tensor/visualization} } @article{goh2021multimodal, author = {Goh, Gabriel and Cammarata, Nick and Voss, Chelsea and Carter, Shan and Petrov, Michael and Schubert, Ludwig and Radford, Alec and Olah, Chris}, title = {Multimodal Neurons in Artificial Neural Networks}, journal = {Distill}, year = {2021}, url = {https://distill.pub/2021/multimodal-neurons}, doi = {10.23915/distill.00030} } @article{dong2021attention, title={Attention is not all you need: Pure attention loses rank doubly exponentially with depth}, author={Dong, Yihe and Cordonnier, Jean-Baptiste and Loukas, Andreas}, journal={arXiv preprint arXiv:2103.03404}, year={2021}, url={https://arxiv.org/pdf/2103.03404} } @article{shazeer2020talking, title={Talking-heads attention}, author={Shazeer, Noam and Lan, Zhenzhong and Cheng, Youlong and Ding, Nan and Hou, Le}, journal={arXiv preprint arXiv:2003.02436}, year={2020}, url={https://arxiv.org/pdf/2003.02436} } @inproceedings{he2016deep, title={Deep residual learning for image recognition}, author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian}, booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, pages={770--778}, year={2016}, url={https://arxiv.org/pdf/1512.03385} } @article{srivastava2015highway, title={Highway networks}, author={Srivastava, Rupesh Kumar and Greff, Klaus and Schmidhuber, J{\"u}rgen}, journal={arXiv preprint arXiv:1505.00387}, year={2015}, url={https://arxiv.org/pdf/1505.00387} } @article{hochreiter1997long, title={Long short-term memory}, author={Hochreiter, Sepp and Schmidhuber, J{\"u}rgen}, journal={Neural computation}, volume={9}, number={8}, pages={1735--1780}, year={1997}, publisher={MIT Press}, url={http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.676.4320&rep=rep1&type=pdf} } @article{hendrycks2016gaussian, title={Gaussian error linear units (gelus)}, author={Hendrycks, Dan and Gimpel, Kevin}, journal={arXiv preprint arXiv:1606.08415}, year={2016}, url={https://arxiv.org/pdf/1606.08415} } @article{klambauer2017self, title={Self-normalizing neural networks}, author={Klambauer, G{\"u}nter and Unterthiner, Thomas and Mayr, Andreas and Hochreiter, Sepp}, journal={Advances in neural information processing systems}, volume={30}, year={2017}, url={https://proceedings.neurips.cc/paper/2017/file/5d44ee6f2c3f71b73125876103c8f6c4-Paper.pdf} } @misc{bauerle2020what, title={What does BERT dream of?}, author={Bauerle, Alex and Wexler, James}, year={2020}, url={https://pair-code.github.io/interpretability/text-dream/blogpost/} } @inproceedings{koh2017understanding, title={Understanding black-box predictions via influence functions}, author={Koh, Pang Wei and Liang, Percy}, booktitle={International Conference on Machine Learning}, pages={1885--1894}, year={2017}, organization={PMLR}, url={https://arxiv.org/pdf/1703.04730} } @incollection{kindermans2019reliability, title={The (un) reliability of saliency methods}, author={Kindermans, Pieter-Jan and Hooker, Sara and Adebayo, Julius and Alber, Maximilian and Schutt, Kristof T and Dahne, Sven and Erhan, Dumitru and Kim, Been}, booktitle={Explainable AI: Interpreting, Explaining and Visualizing Deep Learning}, pages={267--280}, year={2019}, publisher={Springer}, url={https://arxiv.org/pdf/1711.00867} } @article{basu2020influence, title={Influence functions in deep learning are fragile}, author={Basu, Samyadeep and Pope, Philip and Feizi, Soheil}, journal={arXiv preprint arXiv:2006.14651}, year={2020}, url={https://arxiv.org/pdf/2006.14651} } @article{kokhlikyan2021investigating, title={Investigating sanity checks for saliency maps with image and text classification}, author={Kokhlikyan, Narine and Miglani, Vivek and Alsallakh, Bilal and Martin, Miguel and Reblitz-Richardson, Orion}, journal={arXiv preprint arXiv:2106.07475}, year={2021}, url={https://arxiv.org/pdf/2106.07475} } @inproceedings{ancona2019explaining, title={Explaining deep neural networks with a polynomial time algorithm for shapley value approximation}, author={Ancona, Marco and Oztireli, Cengiz and Gross, Markus}, booktitle={International Conference on Machine Learning}, pages={272--281}, year={2019}, organization={PMLR}, url={https://proceedings.mlr.press/v97/ancona19a/ancona19a.pdf} } @article{adebayo2018sanity, title={Sanity checks for saliency maps}, author={Adebayo, Julius and Gilmer, Justin and Muelly, Michael and Goodfellow, Ian and Hardt, Moritz and Kim, Been}, journal={arXiv preprint arXiv:1810.03292}, year={2018}, url={https://arxiv.org/pdf/1810.03292} } @inproceedings{zhou2016learning, title={Learning deep features for discriminative localization}, author={Zhou, Bolei and Khosla, Aditya and Lapedriza, Agata and Oliva, Aude and Torralba, Antonio}, booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition}, pages={2921--2929}, year={2016}, url={http://cnnlocalization.csail.mit.edu/Zhou_Learning_Deep_Features_CVPR_2016_paper.pdf}, doi={10.1109/cvpr.2016.319} } @article{selvaraju2016grad, title={Grad-cam: Why did you say that? visual explanations from deep networks via gradient-based localization}, author={Selvaraju, Ramprasaath R and Das, Abhishek and Vedantam, Ramakrishna and Cogswell, Michael and Parikh, Devi and Batra, Dhruv}, journal={arXiv preprint arXiv:1610.02391}, year={2016}, url={https://arxiv.org/pdf/1610.02391} } @article{kim2017tcav, title={TCAV: Relative concept importance testing with Linear Concept Activation Vectors}, author={Kim, Been and Gilmer, Justin and Viegas, Fernanda and Erlingsson, Ulfar and Wattenberg, Martin}, journal={arXiv preprint arXiv:1711.11279}, year={2017}, url={https://arxiv.org/pdf/1711.11279} } @incollection{raghu2017svcca, title = {SVCCA: Singular Vector Canonical Correlation Analysis for Deep Learning Dynamics and Interpretability}, author = {Raghu, Maithra and Gilmer, Justin and Yosinski, Jason and Sohl-Dickstein, Jascha}, booktitle = {Advances in Neural Information Processing Systems 30}, editor = {I. Guyon and U. V. Luxburg and S. Bengio and H. Wallach and R. Fergus and S. Vishwanathan and R. Garnett}, pages = {6078--6087}, year = {2017}, publisher = {Curran Associates, Inc.}, url = {http://papers.nips.cc/paper/7188-svcca-singular-vector-canonical-correlation-analysis-for-deep-learning-dynamics-and-interpretability.pdf} } @article{nielsen2016thought, title={Thought as a Technology}, author={Michael Nielsen}, year={2016}, url={http://cognitivemedium.com/tat/index.html} } @article{carter2017using, title={Using Artificial Intelligence to Augment Human Intelligence}, author={Carter, Shan and Nielsen, Michael}, journal={Distill}, year={2017}, url={https://distill.pub/2017/aia/}, doi={10.23915/distill.00009} } @article{olah2015visualizing, title={Visualizing Representations: Deep Learning and Human Beings}, author={Olah, Chris}, year={2015}, url={http://colah.github.io/posts/2015-01-Visualizing-Representations/} } @article{olah2017feature, author = {Olah, Chris and Mordvintsev, Alexander and Schubert, Ludwig}, title = {Feature Visualization}, journal = {Distill}, year = {2017}, url = {https://distill.pub/2017/feature-visualization}, doi = {10.23915/distill.00007} } @inproceedings{nguyen2015deep, title={Deep neural networks are easily fooled: High confidence predictions for unrecognizable images}, author={Nguyen, Anh and Yosinski, Jason and Clune, Jeff}, booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition}, pages={427--436}, year={2015}, doi={10.1109/cvpr.2015.7298640}, url={https://arxiv.org/pdf/1412.1897} } @article{mordvintsev2015inceptionism, title={Inceptionism: Going deeper into neural networks}, author={Mordvintsev, Alexander and Olah, Christopher and Tyka, Mike}, journal={Google Research Blog}, year={2015}, url={https://research.googleblog.com/2015/06/inceptionism-going-deeper-into-neural.html} } @article{simonyan2013deep, title={Deep inside convolutional networks: Visualising image classification models and saliency maps}, author={Simonyan, Karen and Vedaldi, Andrea and Zisserman, Andrew}, journal={arXiv preprint arXiv:1312.6034}, year={2013}, url={https://arxiv.org/pdf/1312.6034} } @article{nguyen2016plug, title={Plug & play generative networks: Conditional iterative generation of images in latent space}, author={Nguyen, Anh and Clune, Jeff and Bengio, Yoshua and Dosovitskiy, Alexey and Yosinski, Jason}, journal={arXiv preprint arXiv:1612.00005}, year={2016}, url={https://arxiv.org/pdf/1612.00005} } @article{fong2017interpretable, title={Interpretable Explanations of Black Boxes by Meaningful Perturbation}, author={Fong, Ruth and Vedaldi, Andrea}, journal={arXiv preprint arXiv:1704.03296}, year={2017}, url={https://arxiv.org/pdf/1704.03296} } @article{kindermans2017patternnet, title={PatternNet and PatternLRP--Improving the interpretability of neural networks}, author={Kindermans, Pieter-Jan and Sch{\"u}tt, Kristof T and Alber, Maximilian and M{\"u}ller, Klaus-Robert and D{\"a}hne, Sven}, journal={arXiv preprint arXiv:1705.05598}, year={2017}, url={https://arxiv.org/pdf/1705.05598}, doi={10.1007/978-3-319-10590-1_53} } @inproceedings{zeiler2014visualizing, title={Visualizing and understanding convolutional networks}, author={Zeiler, Matthew D and Fergus, Rob}, booktitle={European conference on computer vision}, pages={818--833}, year={2014}, organization={Springer}, url={https://arxiv.org/pdf/1311.2901} } @article{springenberg2014striving, title={Striving for simplicity: The all convolutional net}, author={Springenberg, Jost Tobias and Dosovitskiy, Alexey and Brox, Thomas and Riedmiller, Martin}, journal={arXiv preprint arXiv:1412.6806}, year={2014}, url={https://arxiv.org/pdf/1412.6806} } @article{kindermans2017reliability, title={The (Un)reliability of saliency methods}, author={Kindermans, Pieter-Jan and Hooker, Sara and Adebayo, Julius and Alber, Maximilian and Sch{\"u}tt, Kristof T and D{\"a}hne, Sven and Erhan, Dumitru and Kim, Been}, journal={arXiv preprint arXiv:1711.00867}, year={2017}, url={https://arxiv.org/pdf/1711.00867} } @article{maaten2008visualizing, title={Visualizing data using t-SNE}, author={Maaten, Laurens van der and Hinton, Geoffrey}, journal={Journal of Machine Learning Research}, volume={9}, number={Nov}, pages={2579--2605}, year={2008}, url={http://www.jmlr.org/papers/volume9/vandermaaten08a/vandermaaten08a.pdf} } @article{pirolli1999information, title={Information foraging}, author={Pirolli, Peter and Card, Stuart}, journal={Psychological review}, volume={106}, number={4}, pages={643}, year={1999}, publisher={American Psychological Association}, url={http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.31.5407&rep=rep1&type=pdf}, doi={10.1037//0033-295x.106.4.643} } @inproceedings{bau2017network, title={Network dissection: Quantifying interpretability of deep visual representations}, author={Bau, David and Zhou, Bolei and Khosla, Aditya and Oliva, Aude and Torralba, Antonio}, booktitle={Computer Vision and Pattern Recognition (CVPR), 2017 IEEE Conference on}, pages={3319--3327}, year={2017}, organization={IEEE}, url={https://arxiv.org/pdf/1704.05796}, doi={10.1109/cvpr.2017.354} } @article{mackinlay1986automating, title={Automating the design of graphical presentations of relational information}, author={Mackinlay, Jock}, journal={Acm Transactions On Graphics (Tog)}, volume={5}, number={2}, pages={110--141}, year={1986}, publisher={ACM}, url={http://www2.parc.com/istl/groups/uir/publications/items/UIR-1986-02-Mackinlay-TOG-Automating.pdf}, doi={10.1145/22949.22950} } @article{nguyen2016multifaceted, title={Multifaceted feature visualization: Uncovering the different types of features learned by each neuron in deep neural networks}, author={Nguyen, Anh and Yosinski, Jason and Clune, Jeff}, journal={arXiv preprint arXiv:1602.03616}, year={2016}, url={https://arxiv.org/pdf/1602.03616} } @article{strobelt2018lstmvis, title={LSTMVis: A tool for visual analysis of hidden state dynamics in recurrent neural networks}, author={Strobelt, Hendrik and Gehrmann, Sebastian and Pfister, Hanspeter and Rush, Alexander M}, journal={IEEE Transactions on Visualization and Computer Graphics}, volume={24}, number={1}, pages={667--676}, year={2018}, publisher={IEEE}, url={https://arxiv.org/pdf/1606.07461}, doi={10.1109/tvcg.2017.2744158} } @article{kahng2018cti, title={ActiVis: Visual Exploration of Industry-Scale Deep Neural Network Models}, author={Kahng, Minsuk and Andrews, Pierre Y and Kalro, Aditya and Chau, Duen Horng Polo}, journal={IEEE Transactions on Visualization and Computer Graphics}, volume={24}, number={1}, pages={88--97}, year={2018}, publisher={IEEE}, url={https://arxiv.org/pdf/1704.01942}, doi={10.1109/tvcg.2017.2744718} } @article{bilal2018convolutional, title={Do convolutional neural networks learn class hierarchy?}, author={Bilal, Alsallakh and Jourabloo, Amin and Ye, Mao and Liu, Xiaoming and Ren, Liu}, journal={IEEE Transactions on Visualization and Computer Graphics}, volume={24}, number={1}, pages={152--162}, year={2018}, publisher={IEEE}, url={https://arxiv.org/pdf/1710.06501}, doi={10.1109/tvcg.2017.2744683} } @inproceedings{amershi2015modeltracker, title={Modeltracker: Redesigning performance analysis tools for machine learning}, author={Amershi, Saleema and Chickering, Max and Drucker, Steven M and Lee, Bongshin and Simard, Patrice and Suh, Jina}, booktitle={Proceedings of the 33rd Annual ACM Conference on Human Factors in Computing Systems}, pages={337--346}, year={2015}, organization={ACM}, url={https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/amershi.CHI2015.ModelTracker.pdf}, doi={10.1145/2702123.2702509} } @inproceedings{kapoor2010interactive, title={Interactive optimization for steering machine classification}, author={Kapoor, Ashish and Lee, Bongshin and Tan, Desney and Horvitz, Eric}, booktitle={Proceedings of the SIGCHI Conference on Human Factors in Computing Systems}, pages={1343--1352}, year={2010}, organization={ACM}, url={http://erichorvitz.com/steering_classification_2010.pdf}, doi={10.1145/1753326.1753529} } @inproceedings{krause2016interacting, title={Interacting with predictions: Visual inspection of black-box machine learning models}, author={Krause, Josua and Perer, Adam and Ng, Kenney}, booktitle={Proceedings of the 2016 CHI Conference on Human Factors in Computing Systems}, pages={5686--5697}, year={2016}, organization={ACM}, url={http://perer.org/papers/adamPerer-Prospector-CHI2016.pdf}, doi={10.1145/2858036.2858529} } @inproceedings{szegedy2015going, title={Going deeper with convolutions}, author={Szegedy, Christian and Liu, Wei and Jia, Yangqing and Sermanet, Pierre and Reed, Scott and Anguelov, Dragomir and Erhan, Dumitru and Vanhoucke, Vincent and Rabinovich, Andrew and others}, year={2015}, organization={CVPR}, url={https://arxiv.org/pdf/1409.4842}, doi={10.1109/cvpr.2015.7298594} } @article{szegedy2013intriguing, title={Intriguing properties of neural networks}, author={Szegedy, Christian and Zaremba, Wojciech and Sutskever, Ilya and Bruna, Joan and Erhan, Dumitru and Goodfellow, Ian and Fergus, Rob}, journal={arXiv preprint arXiv:1312.6199}, year={2013}, url={https://arxiv.org/pdf/1312.6199} } @article{mikolov2013efficient, title={Efficient estimation of word representations in vector space}, author={Mikolov, Tomas and Chen, Kai and Corrado, Greg and Dean, Jeffrey}, journal={arXiv preprint arXiv:1301.3781}, year={2013}, url={https://arxiv.org/pdf/1301.3781} } @article{jo2017measuring, title={Measuring the tendency of CNNs to Learn Surface Statistical Regularities}, author={Jo, Jason and Bengio, Yoshua}, journal={arXiv preprint arXiv:1711.11561}, year={2017}, url={https://arxiv.org/pdf/1711.11561} } @article{sabour2015adversarial, title={Adversarial manipulation of deep representations}, author={Sabour, Sara and Cao, Yanshuai and Faghri, Fartash and Fleet, David J}, journal={arXiv preprint arXiv:1511.05122}, year={2015}, url={https://arxiv.org/pdf/1511.05122} } @article{sundararajan2017axiomatic, title={Axiomatic attribution for deep networks}, author={Sundararajan, Mukund and Taly, Ankur and Yan, Qiqi}, journal={arXiv preprint arXiv:1703.01365}, year={2017}, url={https://arxiv.org/pdf/1703.01365} } @article{odena2016deconvolution, author = {Odena, Augustus and Dumoulin, Vincent and Olah, Chris}, title = {Deconvolution and Checkerboard Artifacts}, journal = {Distill}, year = {2016}, url = {http://distill.pub/2016/deconv-checkerboard}, doi = {10.23915/distill.00003} } @inproceedings{christiano2017deep, title={Deep reinforcement learning from human preferences}, author={Christiano, Paul F and Leike, Jan and Brown, Tom and Martic, Miljan and Legg, Shane and Amodei, Dario}, booktitle={Advances in Neural Information Processing Systems}, pages={4302--4310}, year={2017}, url={https://proceedings.neurips.cc/paper_files/paper/2017/file/d5e2c0adad503c91f91df240d0cd4e49-Paper.pdf} } @article{poplin2018prediction, author = {Poplin, Ryan and Varadarajan, Avinash V. and Blumer, Katy and Liu, Yun and McConnell, Michael V. and Corrado, Greg S. and Peng, Lily and Webster, Dale R.}, year = {2018}, title = {Prediction of cardiovascular risk factors from retinal fundus photographs via deep learning}, journal = {Nature Biomedical Engineering}, publisher = {Nature Publishing Group}, doi = {10.1038/s41551-018-0195-0}, month = {2}, pages = {1--7}, url = {https://doi.org/10.1038/s41551-018-0195-0}, } @inproceedings{hardt2016equality, title={Equality of opportunity in supervised learning}, author={Hardt, Moritz and Price, Eric and Srebro, Nati and others}, booktitle={Advances in neural information processing systems}, pages={3315--3323}, year={2016}, url={https://proceedings.neurips.cc/paper/2016/file/9d2682367c3935defcb1f9e247a97c0d-Paper.pdf} } @inproceedings{corbett2017algorithmic, title={Algorithmic decision making and the cost of fairness}, author={Corbett-Davies, Sam and Pierson, Emma and Feller, Avi and Goel, Sharad and Huq, Aziz}, booktitle={Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining}, pages={797--806}, year={2017}, organization={ACM}, url={https://dl.acm.org/doi/pdf/10.1145/3097983.3098095}, doi={10.1145/3097983.3098095} } @article{card1991morphological, title={A morphological analysis of the design space of input devices}, author={Card, Stuart K and Mackinlay, Jock D and Robertson, George G}, journal={ACM Transactions on Information Systems (TOIS)}, volume={9}, number={2}, pages={99--122}, year={1991}, publisher={ACM}, url={https://dl.acm.org/doi/pdf/10.1145/123078.128726}, doi={10.1145/123078.128726} } @book{wilkinson2006grammar, title={The grammar of graphics}, author={Wilkinson, Leland}, year={2006}, publisher={Springer Science \& Business Media}, url={https://link.springer.com/book/10.1007/0-387-28695-0}, doi={10.1007/0-387-28695-0} } @article{fong2018net2vec, title={Net2Vec: Quantifying and Explaining how Concepts are Encoded by Filters in Deep Neural Networks}, author={Fong, Ruth and Vedaldi, Andrea}, journal={arXiv preprint arXiv:1801.03454}, year={2018}, url={https://arxiv.org/pdf/1801.03454} } @article{adiwardana2020towards, title={Towards a human-like open-domain chatbot}, author={Adiwardana, Daniel and Luong, Minh-Thang and So, David R and Hall, Jamie and Fiedel, Noah and Thoppilan, Romal and Yang, Zi and Kulshreshtha, Apoorv and Nemade, Gaurav and Lu, Yifeng and others}, journal={arXiv preprint arXiv:2001.09977}, year={2020}, url={https://arxiv.org/pdf/2001.09977} } @misc{LaMDA, title={LaMDA: our breakthrough conversation technology}, author={Collins, Eli and Ghahramani, Zoubin}, year={2021}, url={https://blog.google/technology/ai/lamda/} } @article{rogers2020primer, title={A primer in bertology: What we know about how bert works}, author={Rogers, Anna and Kovaleva, Olga and Rumshisky, Anna}, journal={Transactions of the Association for Computational Linguistics}, volume={8}, pages={842--866}, year={2020}, publisher={MIT Press}, url={https://direct.mit.edu/tacl/article/doi/10.1162/tacl_a_00349/96482/A-Primer-in-BERTology-What-We-Know-About-How-BERT}, doi={10.1162/tacl_a_00349} } @article{jain2019attention, title={Attention is not explanation}, author={Jain, Sarthak and Wallace, Byron C}, journal={arXiv preprint arXiv:1902.10186}, year={2019}, url={https://arxiv.org/pdf/1902.10186} } @article{serrano2019attention, title={Is attention interpretable?}, author={Serrano, Sofia and Smith, Noah A}, journal={arXiv preprint arXiv:1906.03731}, year={2019}, url={https://arxiv.org/pdf/1906.03731} } @article{wiegreffe2019attention, title={Attention is not not explanation}, author={Wiegreffe, Sarah and Pinter, Yuval}, journal={arXiv preprint arXiv:1908.04626}, year={2019}, url={https://arxiv.org/pdf/1908.04626} } @article{devlin2018bert, title={Bert: Pre-training of deep bidirectional transformers for language understanding}, author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina}, journal={arXiv preprint arXiv:1810.04805}, year={2018}, url={https://arxiv.org/pdf/1810.04805} } @article{so2021primer, title={Primer: Searching for efficient transformers for language modeling}, author={So, David R and Ma{\'n}ke, Wojciech and Liu, Hanxiao and Dai, Zihang and Shazeer, Noam and Le, Quoc V}, journal={arXiv preprint arXiv:2109.08668}, year={2021}, url={https://arxiv.org/pdf/2109.08668} } @article{lu2021influence, title={Influence Patterns for Explaining Information Flow in BERT}, author={Lu, Kaiji and Wang, Zifan and Mardziel, Piotr and Datta, Anupam}, journal={Advances in Neural Information Processing Systems}, volume={34}, year={2021}, url={https://arxiv.org/pdf/2011.00740} } @article{brunner2019identifiability, title={On identifiability in transformers}, author={Brunner, Gino and Liu, Yang and Pascual, Damian and Richter, Oliver and Ciaramita, Massimiliano and Wattenhofer, Roger}, journal={arXiv preprint arXiv:1908.04211}, year={2019}, url={https://arxiv.org/pdf/1908.04211} } @article{abnar2020quantifying, title={Quantifying attention flow in transformers}, author={Abnar, Samira and Zuidema, Willem}, journal={arXiv preprint arXiv:2005.00928}, year={2020}, url={https://arxiv.org/pdf/2005.00928} } @article{htut2019attention, title={Do attention heads in bert track syntactic dependencies?}, author={Htut, Phu Mon and Phang, Jason and Bordia, Shikha and Bowman, Samuel R}, journal={arXiv preprint arXiv:1911.12246}, year={2019}, url={https://arxiv.org/pdf/1911.12246} } @article{kobayashi2020attention, title={Attention is not only a weight: Analyzing transformers with vector norms}, author={Kobayashi, Goro and Kuribayashi, Tatsuki and Yokoi, Sho and Inui, Kentaro}, journal={arXiv preprint arXiv:2004.10102}, year={2020}, url={https://arxiv.org/pdf/2004.10102} } @article{voss2021visualizing, author = {Voss, Chelsea and Cammarata, Nick and Goh, Gabriel and Petrov, Michael and Schubert, Ludwig and Egan, Ben and Lim, Swee Kiat and Olah, Chris}, title = {Visualizing Weights}, journal = {Distill}, year = {2021}, url = {https://distill.pub/2020/circuits/visualizing-weights}, doi = {10.23915/distill.00024.007} } @article{cammarata2020curve, author = {Cammarata, Nick and Goh, Gabriel and Carter, Shan and Schubert, Ludwig and Petrov, Michael and Olah, Chris}, title = {Curve Detectors}, journal = {Distill}, year = {2020}, url = {https://distill.pub/2020/circuits/curve-detectors} } @article{cammarata2021curve, author = {Cammarata, Nick and Goh, Gabriel and Carter, Shan and Voss, Chelsea and Schubert, Ludwig and Olah, Chris}, title = {Curve Circuits}, journal = {Distill}, year = {2021}, url = {https://distill.pub/2020/circuits/curve-circuits} } @article{karpathy2015visualizing, title={Visualizing and understanding recurrent networks}, author={Karpathy, Andrej and Johnson, Justin and Fei-Fei, Li}, journal={arXiv preprint arXiv:1506.02078}, url={https://arxiv.org/pdf/1506.02078}, year={2015} } @article{radford2017learning, title={Learning to generate reviews and discovering sentiment}, author={Radford, Alec and Jozefowicz, Rafal and Sutskever, Ilya}, journal={arXiv preprint arXiv:1704.01444}, url={https://arxiv.org/pdf/1704.01444}, year={2017} } @article{zhou2014object, title={Object detectors emerge in deep scene cnns}, author={Zhou, Bolei and Khosla, Aditya and Lapedriza, Agata and Oliva, Aude and Torralba, Antonio}, journal={arXiv preprint arXiv:1412.6856}, url={https://arxiv.org/pdf/1412.6856}, year={2014} } @inproceedings{netdissect2017, title={Network Dissection: Quantifying Interpretability of Deep Visual Representations}, author={Bau, David and Zhou, Bolei and Khosla, Aditya and Oliva, Aude and Torralba, Antonio}, booktitle={Computer Vision and Pattern Recognition}, url={https://arxiv.org/pdf/1704.05796}, year={2017} } @article{morcos2018importance, title={On the importance of single directions for generalization}, author={Morcos, Ari S and Barrett, David GT and Rabinowitz, Neil C and Botvinick, Matthew}, journal={arXiv preprint arXiv:1803.06959}, url={https://arxiv.org/pdf/1803.06959}, year={2018} } @article{erhan2009visualizing, title={Visualizing higher-layer features of a deep network}, author={Erhan, Dumitru and Bengio, Yoshua and Courville, Aaron and Vincent, Pascal}, journal={University of Montreal}, volume={1341}, pages={3}, year={2009}, url={https://www.researchgate.net/profile/Aaron_Courville/publication/265022827_Visualizing_Higher-Layer_Features_of_a_Deep_Network/links/53ff82b00cf24c81027da530.pdf} } @article{zimmermann2021well, title={How Well do Feature Visualizations Support Causal Understanding of CNN Activations?}, author={Zimmermann, Roland and Borowski, Judy and Geirhos, Robert and Bethge, Matthias and Wallis, Thomas and Brendel, Wieland}, journal={Advances in Neural Information Processing Systems}, volume={34}, year={2021}, url={https://proceedings.neurips.cc/paper/2021/file/618faa1728eb2ef6e3733645273ab145-Paper.pdf} } @article{carter2016experiments, author = {Carter, Shan and Ha, David and Johnson, Ian and Olah, Chris}, title = {Experiments in Handwriting with a Neural Network}, journal = {Distill}, year = {2016}, url = {http://distill.pub/2016/handwriting}, doi = {10.23915/distill.00004} } @article{olah2018the, author = {Olah, Chris and Satyanarayan, Arvind and Johnson, Ian and Carter, Shan and Schubert, Ludwig and Ye, Katherine and Mordvintsev, Alexander}, title = {The Building Blocks of Interpretability}, journal = {Distill}, year = {2018}, url = {https://distill.pub/2018/building-blocks}, doi = {10.23915/distill.00010} } @article{carter2019activation, author = {Carter, Shan and Armstrong, Zan and Schubert, Ludwig and Johnson, Ian and Olah, Chris}, title = {Activation Atlas}, journal = {Distill}, year = {2019}, url = {https://distill.pub/2019/activation-atlas}, doi = {10.23915/distill.00015} } @misc{smilkov2017playground, title={TensorFlow Playground}, author={Smilkov, Daniel and Carter, Shan and Sculley, D and Vi{\'e}gas, Fernanda B and Wattenberg, Martin}, year={2017}, url={https://playground.tensorflow.org/} } @article{yosinski2015understanding, title={Understanding neural networks through deep visualization}, author={Yosinski, Jason and Clune, Jeff and Nguyen, Anh and Fuchs, Thomas and Lipson, Hod}, journal={arXiv preprint arXiv:1506.06579}, year={2015}, url={https://arxiv.org/pdf/1506.06579} } @article{hoover2019exbert, title={exbert: A visual analysis tool to explore learned representations in transformers models}, author={Hoover, Benjamin and Strobelt, Hendrik and Gehrmann, Sebastian}, journal={arXiv preprint arXiv:1910.05276}, year={2019}, url={https://arxiv.org/pdf/1910.05276} } @misc{alammar2020explaining, title={Interfaces for Explaining Transformer Language Models}, author={Alammar, J}, year={2020}, url={https://jalammar.github.io/explaining-transformers/} } @article{dai2021knowledge, title={Knowledge neurons in pretrained transformers}, author={Dai, Damai and Dong, Li and Hao, Yaru and Sui, Zhifang and Wei, Furu}, journal={arXiv preprint arXiv:2104.08696}, year={2021}, url={https://arxiv.org/pdf/2104.08696} } @article{geva2020transformer, title={Transformer feed-forward layers are key-value memories}, author={Geva, Mor and Schuster, Roei and Berant, Jonathan and Levy, Omer}, journal={arXiv preprint arXiv:2012.14913}, year={2020}, url={https://arxiv.org/pdf/2012.14913} } @article{poerner2018interpretable, title={Interpretable textual neuron representations for NLP}, author={Poerner, Nina and Roth, Benjamin and Sch{\"u}tze, Hinrich}, journal={arXiv preprint arXiv:1809.07291}, year={2018}, url={https://arxiv.org/pdf/1809.07291} } @article{tenney2020language, title={The language interpretability tool: Extensible, interactive visualizations and analysis for NLP models}, author={Tenney, Ian and Wexler, James and Bastings, Jasmijn and Bolukbasi, Tolga and Coenen, Andy and Gehrmann, Sebastian and Jiang, Ellen and Pushkarna, Mahima and Radebaugh, Carey and Reif, Emily and others}, journal={arXiv preprint arXiv:2008.05122}, year={2020}, url={https://arxiv.org/pdf/2008.05122} } @inproceedings{aken2020visbert, title={Visbert: Hidden-state visualizations for transformers}, author={Aken, Betty van and Winter, Benjamin and L{\"o}ser, Alexander and Gers, Felix A}, booktitle={Companion Proceedings of the Web Conference 2020}, pages={207--211}, year={2020}, url={https://arxiv.org/pdf/2011.04507} } @article{wang2021dodrio, title={Dodrio: Exploring Transformer Models with Interactive Visualization}, author={Wang, Zijie J and Turko, Robert and Chau, Duen Horng}, journal={arXiv preprint arXiv:2103.14625}, year={2021}, url={https://arxiv.org/pdf/2103.14625} } @misc{kehlbeck2021demystifying, title={Demystifying the Embedding Space of Language Models}, author={Kehlbeck, Rebecca and Sevastjanova, Rita and Spinner, Thilo and Stahle, Tobias and El-Assady Mennatallah}, journal={VISxAI 2021}, url={https://bert-vs-gpt2.dbvis.de/} } @misc{pearce2021what, title={What Have Language Models Learned?}, author={Adam Pearce}, year={2021}, url={https://pair.withgoogle.com/explorables/fill-in-the-blank/} } @article{joshi2020transformers, author = {Joshi, Chaitanya}, title = {Transformers are Graph Neural Networks}, journal = {The Gradient}, year = {2020}, url = {https://thegradient.pub/transformers-are-gaph-neural-networks/}, } @article{cordonnier2019relationship, title={On the relationship between self-attention and convolutional layers}, author={Cordonnier, Jean-Baptiste and Loukas, Andreas and Jaggi, Martin}, journal={arXiv preprint arXiv:1911.03584}, year={2019}, url={https://arxiv.org/pdf/1911.03584} } @misc{olah2014groups, title={Groups & Group Convolutions}, author={Christopher Olah}, url={https://colah.github.io/posts/2014-12-Groups-Convolution/} } @article{press2020shortformer, title={Shortformer: Better language modeling using shorter inputs}, author={Press, Ofir and Smith, Noah A and Lewis, Mike}, journal={arXiv preprint arXiv:2012.15832}, year={2020}, url={https://arxiv.org/pdf/2012.15832} } @article{su2021roformer, title={Roformer: Enhanced transformer with rotary position embedding}, author={Su, Jianlin and Lu, Yu and Pan, Shengfeng and Wen, Bo and Liu, Yunfeng}, journal={arXiv preprint arXiv:2104.09864}, year={2021}, url={https://arxiv.org/pdf/2104.09864} } @article{askell2021general, title={A General Language Assistant as a Laboratory for Alignment}, author={Askell, Amanda and Bai, Yuntao and Chen, Anna and Drain, Dawn and Ganguli, Deep and Henighan, Tom and Jones, Andy and Joseph, Nicholas and Mann, Ben and DasSarma, Nova and others}, journal={arXiv preprint arXiv:2112.00861}, year={2021}, url={https://arxiv.org/pdf/2112.00861} } @article{o2021context, title={What Context Features Can Transformer Language Models Use?}, author={O'Connor, Joe and Andreas, Jacob}, journal={arXiv preprint arXiv:2106.08367}, year={2021}, url={https://arxiv.org/pdf/2106.08367} } @article{khandelwal2018sharp, title={Sharp nearby, fuzzy far away: How neural language models use context}, author={Khandelwal, Urvashi and He, He and Qi, Peng and Jurafsky, Dan}, journal={arXiv preprint arXiv:1805.04623}, year={2018}, url={https://arxiv.org/pdf/1805.04623} } @article{radford2019language, title={Language models are unsupervised multitask learners}, author={Radford, Alec and Wu, Jeffrey and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya and others}, journal={OpenAI blog}, volume={1}, number={8}, pages={9}, year={2019}, url={https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf} } @article{olah2020zoom, author = {Olah, Chris and Cammarata, Nick and Schubert, Ludwig and Goh, Gabriel and Petrov, Michael and Carter, Shan}, title = {Zoom In: An Introduction to Circuits}, journal = {Distill}, year = {2020}, note = {https://distill.pub/2020/circuits/zoom-in}, doi = {10.23915/distill.00024.001}, url = {https://distill.pub/2020/circuits/zoom-in} } @inproceedings{li2015convergent, title={Convergent learning: Do different neural networks learn the same representations?}, author={Li, Yixuan and Yosinski, Jason and Clune, Jeff and Lipson, Hod and Hopcroft, John E and others}, booktitle={FE@ NIPS}, pages={196--212}, year={2015}, url={https://arxiv.org/pdf/1511.07543} } @article{schubert2021highlow, author = {Schubert, Ludwig and Voss, Chelsea and Cammarata, Nick and Goh, Gabriel and Olah, Chris}, title = {High-Low Frequency Detectors}, journal = {Distill}, year = {2021}, note = {https://distill.pub/2020/circuits/frequency-edges}, doi = {10.23915/distill.00024.005}, url = {https://distill.pub/2020/circuits/frequency-edges/} } @article{kaplan2020scaling, title={Scaling laws for neural language models}, author={Kaplan, Jared and McCandlish, Sam and Henighan, Tom and Brown, Tom B and Chess, Benjamin and Child, Rewon and Gray, Scott and Radford, Alec and Wu, Jeffrey and Amodei, Dario}, journal={arXiv preprint arXiv:2001.08361}, year={2020}, url={https://arxiv.org/pdf/2001.08361} } @article{xie2021explanation, title={An Explanation of In-context Learning as Implicit Bayesian Inference}, author={Xie, Sang Michael and Raghunathan, Aditi and Liang, Percy and Ma, Tengyu}, journal={arXiv preprint arXiv:2111.02080}, year={2021}, url={https://arxiv.org/pdf/2111.02080} } @article{power2022grokking, title={Grokking: Generalization beyond overfitting on small algorithmic datasets}, author={Power, Alethea and Burda, Yuri and Edwards, Harri and Babuschkin, Igor and Misra, Vedant}, journal={arXiv preprint arXiv:2201.02177}, year={2022}, url={https://arxiv.org/pdf/2201.02177} } @article{belkin2019reconciling, title={Reconciling modern machine-learning practice and the classical bias--variance trade-off}, author={Belkin, Mikhail and Hsu, Daniel and Ma, Siyuan and Mandal, Soumik}, journal={Proceedings of the National Academy of Sciences}, volume={116}, number={32}, pages={15849--15854}, year={2019}, publisher={National Acad Sciences}, url={https://www.pnas.org/doi/pdf/10.1073/pnas.1903070116} } @article{nakkiran2021deep, title={Deep double descent: Where bigger models and more data hurt}, author={Nakkiran, Preetum and Kaplun, Gal and Bansal, Yamini and Yang, Tristan and Barak, Boaz and Sutskever, Ilya}, journal={Journal of Statistical Mechanics: Theory and Experiment}, volume={2021}, number={12}, pages={124003}, year={2021}, publisher={IOP Publishing}, url={https://arxiv.org/pdf/1912.02292} } @article{yamins2014performance, title={Performance-optimized hierarchical models predict neural responses in higher visual cortex}, author={Yamins, Daniel LK and Hong, Ha and Cadieu, Charles F and Solomon, Ethan A and Seibert, Darren and DiCarlo, James J}, journal={Proceedings of the National Academy of Sciences}, volume={111}, number={23}, pages={8619--8624}, year={2014}, publisher={National Acad Sciences}, url={https://www.pnas.org/doi/pdf/10.1073/pnas.1403112111} } @article{eickenberg2017seeing, title={Seeing it all: Convolutional network layers map the function of the human visual system}, author={Eickenberg, Michael and Gramfort, Alexandre and Varoquaux, Ga{\"e}l and Thirion, Bertrand}, journal={NeuroImage}, volume={152}, pages={184--194}, year={2017}, publisher={Elsevier}, url={https://inria.hal.science/hal-01389809/file/neuroimage.pdf} } @article{gucclu2015deep, title={Deep neural networks reveal a gradient in the complexity of neural representations across the ventral stream}, author={Güçlü, Umut and van Gerven, Marcel AJ}, journal={Journal of Neuroscience}, volume={35}, number={27}, pages={10005--10014}, year={2015}, publisher={Soc Neuroscience}, url={https://citeseerx.ist.psu.edu/document?repid=rep1&type=pdf&doi=9ad604babb27153c20240dbf4cb8bb357aaff325} } @article{liu2021makes, title={What Makes Good In-Context Examples for GPT-$3 $?}, author={Liu, Jiachang and Shen, Dinghan and Zhang, Yizhe and Dolan, Bill and Carin, Lawrence and Chen, Weizhu}, journal={arXiv preprint arXiv:2101.06804}, year={2021}, url={https://arxiv.org/pdf/2101.06804} } @article{yang2021empirical, title={An empirical study of gpt-3 for few-shot knowledge-based vqa}, author={Yang, Zhengyuan and Gan, Zhe and Wang, Jianfeng and Hu, Xiaowei and Lu, Yumao and Liu, Zicheng and Wang, Lijuan}, journal={arXiv preprint arXiv:2109.05014}, year={2021}, url={https://arxiv.org/pdf/2109.05014} } @misc{steinhardt2021, title={Future ML Systems Will Be Qualitatively Different}, author={Steinhardt, Jacob}, journal={Bounded Regret}, year={2021}, url={https://bounded-regret.ghost.io/future-ml-systems-will-be-qualitatively-different/} } @article{dosovitskiy2020image, author = {Alexey Dosovitskiy and Lucas Beyer and Alexander Kolesnikov and Dirk Weissenborn and Xiaohua Zhai and Thomas Unterthiner and Mostafa Dehghani and Matthias Minderer and Georg Heigold and Sylvain Gelly and Jakob Uszkoreit and Neil Houlsby}, title = {An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale}, journal = {CoRR}, volume = {abs/2010.11929}, year = {2020}, url = {https://arxiv.org/pdf/2010.11929}, } @inproceedings{chen2020generative, title={Generative pretraining from pixels}, author={Chen, Mark and Radford, Alec and Child, Rewon and Wu, Jeffrey and Jun, Heewoo and Luan, David and Sutskever, Ilya}, booktitle={International Conference on Machine Learning}, pages={1691--1703}, year={2020}, organization={PMLR}, url={http://proceedings.mlr.press/v119/chen20s/chen20s.pdf} } @article{jumper2021highly, title={Highly accurate protein structure prediction with AlphaFold}, author={Jumper, John and Evans, Richard and Pritzel, Alexander and Green, Tim and Figurnov, Michael and Ronneberger, Olaf and Tunyasuvunakool, Kathryn and Bates, Russ and {\v{Z}}{\'\i}dek, Augustin and Potapenko, Anna and others}, journal={Nature}, volume={596}, number={7873}, pages={583--589}, year={2021}, publisher={Nature Publishing Group}, url={https://www.nature.com/articles/s41586-021-03819-2%3C/p%3E%3Cp%3E-AlphaFold} } @article{chen2021evaluating, title={Evaluating large language models trained on code}, author={Chen, Mark and Tworek, Jerry and Jun, Heewoo and Yuan, Qiming and Pinto, Henrique Ponde de Oliveira and Kaplan, Jared and Edwards, Harri and Burda, Yuri and Joseph, Nicholas and Brockman, Greg and others}, journal={arXiv preprint arXiv:2107.03374}, year={2021}, url={https://arxiv.org/pdf/2107.03374} } @article{thoppilan2022lamda, title={LaMDA: Language Models for Dialog Applications}, author={Thoppilan, Romal and De Freitas, Daniel and Hall, Jamie and Shazeer, Noam and Kulshreshtha, Apoorv and Cheng, Heng-Tze and Jin, Alicia and Bos, Taylor and Baker, Leslie and Du, Yu and others}, journal={arXiv preprint arXiv:2201.08239}, year={2022}, url={https://arxiv.org/pdf/2201.08239} } @misc{omohundro2008basic, title={The basic AI drives}, author={Omohundro, Stephen M}, year={2008}, url={https://steveomohundro.com/wp-content/uploads/2009/12/ai_drives_final.pdf} } @book{Bostrom14, address = {Oxford, UK}, author = {Bostrom, Nick}, groups = {public}, isbn = {978-0-19-967811-2}, keywords = {01801 103 book ai cognitive science theory}, publisher = {Oxford University Press}, title = {Superintelligence: Paths, Dangers, Strategies}, year = {2014} } @book{russell2019human, title={Human Compatible: ​​AI and the Problem of Control}, author={Russell, Stuart}, year={2019}, url={https://static.fnac-static.com/multimedia/PT/pdf/9780141987507.pdf} } @article{bommasani2021opportunities, title={On the opportunities and risks of foundation models}, author={Bommasani, Rishi and Hudson, Drew A and Adeli, Ehsan and Altman, Russ and Arora, Simran and von Arx, Sydney and Bernstein, Michael S and Bohg, Jeannette and Bosselut, Antoine and Brunskill, Emma and others}, journal={arXiv preprint arXiv:2108.07258}, year={2021}, url={https://arxiv.org/pdf/2108.07258} } @misc{commoncrawl2024common, author = {The Common Crawl Foundation}, title = {Common Crawl}, url = {https://commoncrawl.org}, } @misc{gao2020pile, title={The Pile: An 800GB Dataset of Diverse Text for Language Modeling}, author={Leo Gao and Stella Biderman and Sid Black and Laurence Golding and Travis Hoppe and Charles Foster and Jason Phang and Horace He and Anish Thite and Noa Nabeshima and Shawn Presser and Connor Leahy}, year={2020}, eprint={2101.00027}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/pdf/2101.00027} } @article{nelhage2021mathematical, title={A Mathematical Framework for Transformer Circuits}, author={Elhage, Nelson and Nanda, Neel and Olsson, Catherine and Henighan, Tom and Joseph, Nicholas and Mann, Ben and Askell, Amanda and Bai, Yuntao and Chen, Anna and Conerly, Tom and DasSarma, Nova and Drain, Dawn and Ganguli, Deep and Hatfield-Dodds, Zac and Hernandez, Danny and Jones, Andy and Kernion, Jackson and Lovitt, Liane and Ndousse, Kamal and Amodei, Dario and Brown, Tom and Clark, Jack and Kaplan, Jared and McCandlish, Sam and Olah, Chris}, year={2021}, journal={Transformer Circuits Thread}, url={https://transformer-circuits.pub/2021/framework/index.html} } @article{olsson2022context, title={In-context Learning and Induction Heads}, author={Olsson, Catherine and Elhage, Nelson and Nanda, Neel and Joseph, Nicholas and DasSarma, Nova and Henighan, Tom and Mann, Ben and Askell, Amanda and Bai, Yuntao and Chen, Anna and Conerly, Tom and Drain, Dawn and Ganguli, Deep and Hatfield-Dodds, Zac and Hernandez, Danny and Johnston, Scott and Jones, Andy and Kernion, Jackson and Lovitt, Liane and Ndousse, Kamal and Amodei, Dario and Brown, Tom and Clark, Jack and Kaplan, Jared and McCandlish, Sam and Olah, Chris}, year={2022}, journal={Transformer Circuits Thread}, url={https://transformer-circuits.pub/2022/in-context-learning-and-induction-heads/index.html} } @inproceedings{pan2022effects, title={The Effects of Reward Misspecification: Mapping and Mitigating Misaligned Models}, author={Alexander Pan and Kush Bhatia and Jacob Steinhardt}, booktitle={International Conference on Learning Representations}, year={2022}, url={https://openreview.net/forum?id=JYtwGwIL7ye} } @misc{zhao2021calibrate, title={Calibrate Before Use: Improving Few-Shot Performance of Language Models}, author={Tony Z. Zhao and Eric Wallace and Shi Feng and Dan Klein and Sameer Singh}, year={2021}, eprint={2102.09690}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/pdf/2102.09690} } @misc{gao2021making, title={Making Pre-trained Language Models Better Few-shot Learners}, author={Tianyu Gao and Adam Fisch and Danqi Chen}, year={2021}, eprint={2012.15723}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/pdf/2012.15723} } @misc{saxe2014exact, title={Exact solutions to the nonlinear dynamics of learning in deep linear neural networks}, author={Andrew M. Saxe and James L. McClelland and Surya Ganguli}, year={2014}, eprint={1312.6120}, archivePrefix={arXiv}, primaryClass={cs.NE}, url={https://arxiv.org/pdf/1312.6120} } @misc{reynolds2021prompt, title={Prompt Programming for Large Language Models: Beyond the Few-Shot Paradigm}, author={Laria Reynolds and Kyle McDonell}, year={2021}, eprint={2102.07350}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/pdf/2102.07350} } @misc{hubinger2021risks, title={Risks from Learned Optimization in Advanced Machine Learning Systems}, author={Evan Hubinger and Chris van Merwijk and Vladimir Mikulik and Joar Skalse and Scott Garrabrant}, year={2021}, eprint={1906.01820}, archivePrefix={arXiv}, primaryClass={cs.AI}, url={https://arxiv.org/pdf/1906.01820} } @misc{elsayed2018adversarial, title={Adversarial Reprogramming of Neural Networks}, author={Gamaleldin F. Elsayed and Ian Goodfellow and Jascha Sohl-Dickstein}, year={2018}, eprint={1806.11146}, archivePrefix={arXiv}, primaryClass={cs.LG}, url={https://arxiv.org/pdf/1806.11146} } @inproceedings{erhan2010does, title={Why does unsupervised pre-training help deep learning?}, author={Erhan, Dumitru and Courville, Aaron and Bengio, Yoshua and Vincent, Pascal}, booktitle={Proceedings of the thirteenth international conference on artificial intelligence and statistics}, pages={201--208}, year={2010}, organization={JMLR Workshop and Conference Proceedings}, url={https://www.jmlr.org/papers/volume11/erhan10a/erhan10a} } @misc{gwern2020scaling, title={The Scaling Hypothesis}, author={Gwern Branwen}, year={2020}, url={https://www.gwern.net/Scaling-hypothesis} } @article{goodfellow2014qualitatively, title={Qualitatively characterizing neural network optimization problems}, author={Goodfellow, Ian J and Vinyals, Oriol and Saxe, Andrew M}, journal={arXiv preprint arXiv:1412.6544}, year={2014}, url={https://arxiv.org/pdf/1412.6544} } @article{lucas2021analyzing, title={Analyzing monotonic linear interpolation in neural network loss landscapes}, author={Lucas, James and Bae, Juhan and Zhang, Michael R and Fort, Stanislav and Zemel, Richard and Grosse, Roger}, journal={arXiv preprint arXiv:2104.11044}, year={2021}, url={https://arxiv.org/pdf/2104.11044} } @inproceedings{pennington2017geometry, title={Geometry of neural network loss surfaces via random matrix theory}, author={Pennington, Jeffrey and Bahri, Yasaman}, booktitle={International Conference on Machine Learning}, pages={2798--2806}, year={2017}, organization={PMLR}, url={https://proceedings.mlr.press/v70/pennington17a/pennington17a.pdf} } @article{li2018visualizing, title={Visualizing the loss landscape of neural nets}, author={Li, Hao and Xu, Zheng and Taylor, Gavin and Studer, Christoph and Goldstein, Tom}, journal={Advances in neural information processing systems}, volume={31}, year={2018}, url={https://proceedings.neurips.cc/paper/2018/file/a41b3bb3e6b050b6c9067c67f663b915-Paper.pdf} } @article{saxe2019mathematical, title={A mathematical theory of semantic development in deep neural networks}, author={Saxe, Andrew M and McClelland, James L and Ganguli, Surya}, journal={Proceedings of the National Academy of Sciences}, volume={116}, number={23}, pages={11537--11546}, year={2019}, publisher={National Acad Sciences}, url={https://www.pnas.org/doi/pdf/10.1073/pnas.1820226116}, doi={10.1073/pnas.1820226116} } @article{bahdanau2014neural, title={Neural machine translation by jointly learning to align and translate}, author={Bahdanau, Dzmitry and Cho, Kyunghyun and Bengio, Yoshua}, journal={arXiv preprint arXiv:1409.0473}, year={2014}, url={https://arxiv.org/pdf/1409.0473} } @article{chan2015listen, title={Listen, attend and spell}, author={Chan, William and Jaitly, Navdeep and Le, Quoc V and Vinyals, Oriol}, journal={arXiv preprint arXiv:1508.01211}, year={2015}, url={https://arxiv.org/pdf/1508.01211} } @article{min2022rethinking, title={Rethinking the Role of Demonstrations: What Makes In-Context Learning Work?}, author={Min, Sewon and Lyu, Xinxi and Holtzman, Ari and Artetxe, Mikel and Lewis, Mike and Hajishirzi, Hannaneh and Zettlemoyer, Luke}, journal={arXiv preprint arXiv:2202.12837}, year={2022}, url={https://arxiv.org/pdf/2202.12837} } @article{you2017deep, title={Deep lattice networks and partial monotonic functions}, author={You, Seungil and Ding, David and Canini, Kevin and Pfeifer, Jan and Gupta, Maya}, journal={Advances in neural information processing systems}, volume={30}, year={2017}, url={https://proceedings.neurips.cc/paper_files/paper/2017/file/464d828b85b0bed98e80ade0a5c43b0f-Paper.pdf} } @article{milani2016fast, title={Fast and flexible monotonic functions with ensembles of lattices}, author={Milani Fard, Mahdi and Canini, Kevin and Cotter, Andrew and Pfeifer, Jan and Gupta, Maya}, journal={Advances in neural information processing systems}, volume={29}, year={2016}, url={https://proceedings.neurips.cc/paper_files/paper/2016/file/c913303f392ffc643f7240b180602652-Paper.pdf} } @inproceedings{wang2015falling, title={Falling rule lists}, author={Wang, Fulton and Rudin, Cynthia}, booktitle={Artificial intelligence and statistics}, pages={1013--1022}, year={2015}, organization={PMLR}, url={https://proceedings.mlr.press/v38/wang15a.pdf} } @inproceedings{caruana2015intelligible, title={Intelligible models for healthcare: Predicting pneumonia risk and hospital 30-day readmission}, author={Caruana, Rich and Lou, Yin and Gehrke, Johannes and Koch, Paul and Sturm, Marc and Elhadad, Noemie}, booktitle={Proceedings of the 21th ACM SIGKDD international conference on knowledge discovery and data mining}, pages={1721--1730}, year={2015}, url={https://dl.acm.org/doi/pdf/10.1145/2783258.2788613}, doi={10.1145/2783258.2788613} } @article{olshausen1997sparse, title={Sparse coding with an overcomplete basis set: A strategy employed by V1?}, author={Olshausen, Bruno A and Field, David J}, journal={Vision research}, volume={37}, number={23}, pages={3311--3325}, year={1997}, publisher={Elsevier}, url={https://www.sciencedirect.com/science/article/pii/S0042698997001697}, doi={10.1016/S0042-6989(97)00169-7} } @article{olah2020early, author = {Olah, Chris and Cammarata, Nick and Schubert, Ludwig and Goh, Gabriel and Petrov, Michael and Carter, Shan}, title = {An Overview of Early Vision in InceptionV1}, journal = {Distill}, year = {2020}, url = {https://distill.pub/2020/circuits/early-vision}, doi = {10.23915/distill.00024.002} } @article{openai2019solving, title={Solving Rubik’s Cube with a Robot Hand: A Preprint}, author={OpenAI, Ilge Akkaya and Andrychowicz, Marcin and Chociej, Maciek and Litwin, Mateusz and McGrew, Bob and Petron, Arthur and Paino, Alex and Plappert, Matthias and Powell, Glenn and Ribas, Raphael and others}, url={https://arxiv.org/pdf/1910.07113}, year={2019} } @article{hilton2020understanding, author = {Hilton, Jacob and Cammarata, Nick and Carter, Shan and Goh, Gabriel and Olah, Chris}, title = {Understanding RL Vision}, journal = {Distill}, year = {2020}, url = {https://distill.pub/2020/understanding-rl-vision}, doi = {10.23915/distill.00029} } @article{lawton1971self, title={Self modeling curve resolution}, author={Lawton, William H and Sylvestre, Edward A}, journal={Technometrics}, volume={13}, number={3}, pages={617--633}, year={1971}, publisher={Taylor \& Francis}, url={https://www.jstor.org/stable/1267173} } @article{lee1999learning, title={Learning the parts of objects by non-negative matrix factorization}, author={Lee, Daniel D and Seung, H Sebastian}, journal={Nature}, volume={401}, number={6755}, pages={788--791}, year={1999}, publisher={Nature Publishing Group}, url={http://belohlavek.inf.upol.cz/vyuka/Lee-Seung-NMF-1999-p.pdf} } @article{lee2000algorithms, title={Algorithms for non-negative matrix factorization}, author={Lee, Daniel and Seung, H Sebastian}, journal={Advances in neural information processing systems}, volume={13}, year={2000}, url={https://proceedings.neurips.cc/paper_files/paper/2000/file/f9d1152547c0bde01830b7e8bd60024c-Paper.pdf} } @article{shleifer2021normformer, title={NormFormer: Improved Transformer Pretraining with Extra Normalization}, author={Shleifer, Sam and Weston, Jason and Ott, Myle}, journal={arXiv preprint arXiv:2110.09456}, year={2021}, url={https://arxiv.org/pdf/2110.09456} } @article{ilyas2019adversarial, title={Adversarial examples are not bugs, they are features}, author={Ilyas, Andrew and Santurkar, Shibani and Tsipras, Dimitris and Engstrom, Logan and Tran, Brandon and Madry, Aleksander}, journal={Advances in neural information processing systems}, volume={32}, year={2019}, url={https://arxiv.org/pdf/1905.02175} } @article{engstrom2019a, author = {Engstrom, Logan and Gilmer, Justin and Goh, Gabriel and Hendrycks, Dan and Ilyas, Andrew and Madry, Aleksander and Nakano, Reiichiro and Nakkiran, Preetum and Santurkar, Shibani and Tran, Brandon and Tsipras, Dimitris and Wallace, Eric}, title = {A Discussion of 'Adversarial Examples Are Not Bugs, They Are Features'}, journal = {Distill}, year = {2019}, url = {https://distill.pub/2019/advex-bugs-discussion} } @article{johnson1984extensions, title={Extensions of Lipschitz mappings into a Hilbert space 26}, author={Johnson, William B and Lindenstrauss, Joram}, journal={Contemporary mathematics}, volume={26}, pages={28}, year={1984}, url={https://www.academia.edu/download/99288805/JL-Johnson.pdf} } @article{bai2022training, title={Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback}, author={Bai, Yuntao and Jones, Andy and Ndousse, Kamal and Askell, Amanda and Chen, Anna and DasSarma, Nova and Drain, Dawn and Fort, Stanislav and Ganguli, Deep and Henighan, Tom and others}, journal={arXiv preprint arXiv:2204.05862}, year={2022}, url={https://arxiv.org/pdf/2204.05862} } @article{reif2019visualizing, title={Visualizing and measuring the geometry of BERT}, author = {Coenen, Andy and Reif, Emily and Yuan, Ann and Kim, Been and Pearce, Adam and Viégas, Fernanda and Wattenberg, Martin}, journal={Advances in Neural Information Processing Systems}, volume={32}, year={2019}, url={https://proceedings.neurips.cc/paper_files/paper/2019/file/159c1ffe5b61b41b3c4d8f4c2150f6c4-Paper.pdf} } @article{olah2020naturally, author = {Olah, Chris and Cammarata, Nick and Voss, Chelsea and Schubert, Ludwig and Goh, Gabriel}, title = {Naturally Occurring Equivariance in Neural Networks}, journal = {Distill}, year = {2020}, url = {https://distill.pub/2020/circuits/equivariance}, doi = {10.23915/distill.00024.004} } @article{geva2022transformer, title={Transformer Feed-Forward Layers Build Predictions by Promoting Concepts in the Vocabulary Space}, author={Geva, Mor and Caciularu, Avi and Wang, Kevin Ro and Goldberg, Yoav}, journal={arXiv preprint arXiv:2203.14680}, year={2022}, url={https://arxiv.org/pdf/2203.14680} } @article{geva2022lm, title={LM-Debugger: An Interactive Tool for Inspection and Intervention in Transformer-Based Language Models}, author={Geva, Mor and Caciularu, Avi and Dar, Guy and Roit, Paul and Sadde, Shoval and Shlain, Micah and Tamir, Bar and Goldberg, Yoav}, journal={arXiv preprint arXiv:2204.12130}, year={2022}, url={https://arxiv.org/pdf/2204.12130} } @article{meng2022locating, title={Locating and editing factual knowledge in gpt}, author={Meng, Kevin and Bau, David and Andonian, Alex and Belinkov, Yonatan}, journal={arXiv preprint arXiv:2202.05262}, year={2022}, url={https://arxiv.org/pdf/2202.05262} } @article{bolukbasi2021interpretability, title={An interpretability illusion for bert}, author={Bolukbasi, Tolga and Pearce, Adam and Yuan, Ann and Coenen, Andy and Reif, Emily and Vi{\'e}gas, Fernanda and Wattenberg, Martin}, journal={arXiv preprint arXiv:2104.07143}, year={2021}, url={https://arxiv.org/pdf/2104.07143} } @inproceedings{panigrahi2019word2sense, title={Word2Sense: sparse interpretable word embeddings}, author={Panigrahi, Abhishek and Simhadri, Harsha Vardhan and Bhattacharyya, Chiranjib}, booktitle={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics}, pages={5692--5705}, year={2019}, url={https://aclanthology.org/P19-1570.pdf}, doi={10.18653/v1/P19-1570} } @inproceedings{subramanian2018spine, title={Spine: Sparse interpretable neural embeddings}, author={Subramanian, Anant and Pruthi, Danish and Jhamtani, Harsh and Berg-Kirkpatrick, Taylor and Hovy, Eduard}, booktitle={Proceedings of the AAAI Conference on Artificial Intelligence}, volume={32}, number={1}, year={2018}, url={https://cdn.aaai.org/ojs/11935/11935-13-15463-1-2-20201228.pdf} } @inproceedings{murphy2012learning, title={Learning effective and interpretable semantic models using non-negative sparse embedding}, author={Murphy, Brian and Talukdar, Partha and Mitchell, Tom}, booktitle={Proceedings of COLING 2012}, pages={1933--1950}, year={2012}, url={https://aclanthology.org/C12-1118.pdf} } @article{olah2014neural, title={Neural Networks, Manifolds, and Topology}, author={Olah, Chris}, url={https://colah.github.io/posts/2014-03-NN-Manifolds-Topology/}, year={2014} } @article{paperno2016lambada, title={The LAMBADA dataset: Word prediction requiring a broad discourse context}, author={Paperno, Denis and Kruszewski, Germ{\'a}n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern{\'a}ndez, Raquel}, journal={arXiv preprint arXiv:1606.06031}, year={2016}, url={https://arxiv.org/pdf/1606.06031} } @article{clark2018think, title={Think you have solved question answering? try arc, the ai2 reasoning challenge}, author={Clark, Peter and Cowhey, Isaac and Etzioni, Oren and Khot, Tushar and Sabharwal, Ashish and Schoenick, Carissa and Tafjord, Oyvind}, journal={arXiv preprint arXiv:1803.05457}, year={2018}, url={https://arxiv.org/pdf/1803.05457} } @article{mihaylov2018can, title={Can a suit of armor conduct electricity? a new dataset for open book question answering}, author={Mihaylov, Todor and Clark, Peter and Khot, Tushar and Sabharwal, Ashish}, journal={arXiv preprint arXiv:1809.02789}, year={2018}, url={https://arxiv.org/pdf/1809.02789} } @article{joshi2017triviaqa, title={Triviaqa: A large scale distantly supervised challenge dataset for reading comprehension}, author={Joshi, Mandar and Choi, Eunsol and Weld, Daniel S and Zettlemoyer, Luke}, journal={arXiv preprint arXiv:1705.03551}, year={2017}, url={https://arxiv.org/pdf/1705.03551} } @article{hendrycks2020measuring, title={Measuring massive multitask language understanding}, author={Hendrycks, Dan and Burns, Collin and Basart, Steven and Zou, Andy and Mazeika, Mantas and Song, Dawn and Steinhardt, Jacob}, journal={arXiv preprint arXiv:2009.03300}, year={2020}, url={https://arxiv.org/pdf/2009.03300} } @article{zellers2019hellaswag, title={HellaSwag: Can a machine really finish your sentence?}, author={Zellers, Rowan and Holtzman, Ari and Bisk, Yonatan and Farhadi, Ali and Choi, Yejin}, journal={arXiv preprint arXiv:1905.07830}, year={2019}, url={https://arxiv.org/pdf/1905.07830} } @article{press2021train, title={Train short, test long: Attention with linear biases enables input length extrapolation}, author={Press, Ofir and Smith, Noah A and Lewis, Mike}, journal={arXiv preprint arXiv:2108.12409}, year={2021}, url={https://arxiv.org/pdf/2108.12409} } @article{radford2015unsupervised, title={Unsupervised representation learning with deep convolutional generative adversarial networks}, author={Radford, Alec and Metz, Luke and Chintala, Soumith}, journal={arXiv preprint arXiv:1511.06434}, year={2015}, url={https://arxiv.org/pdf/1511.06434} } @article{yi2019firingrate, title={Average firing rate rather than temporal pattern determines metabolic cost of activity in thalamocortical relay neurons}, author={Yi, Guosheng and Grill, Warren}, journal={Scientific reports}, volume={9}, number={1}, pages={6940}, year={2019}, doi={10.1038/s41598-019-43460-8}, url={https://www.nature.com/articles/s41598-019-43460-8.pdf} } @article{thorpe1989coding, title={Local vs. Distributed Coding}, author={Thorpe, Simon J.}, journal={Intellectica}, volume={8}, pages={3--40}, year={1989}, url={https://www.persee.fr/doc/intel_0769-4113_1989_num_8_2_873?ref=cognitiverevolution.ai} } @article{plate2003distributed, author = {Plate, Tony}, year = {2003}, month = {01}, pages = {1-15}, title = {Distributed representations}, journal = {Cognitive Science}, url = {https://citeseerx.ist.psu.edu/document?repid=rep1&type=pdf&doi=54aa797e64793d52c95a381600f590daceecaf1f} } @article{ganguli2012compressed, author = {Ganguli, Surya and Sompolinsky, Haim}, title = {Compressed Sensing, Sparsity, and Dimensionality in Neuronal Information Processing and Data Analysis}, journal = {Annual Review of Neuroscience}, volume = {35}, number = {1}, pages = {485-508}, year = {2012}, doi = {10.1146/annurev-neuro-062111-150410}, note ={PMID: 22483042}, URL = {https://doi.org/10.1146/annurev-neuro-062111-150410} }, @article{gilmer2018adversarial, title={Adversarial spheres}, author={Gilmer, Justin and Metz, Luke and Faghri, Fartash and Schoenholz, Samuel S and Raghu, Maithra and Wattenberg, Martin and Goodfellow, Ian}, journal={arXiv preprint arXiv:1801.02774}, year={2018}, url={https://arxiv.org/pdf/1801.02774} } @misc{nanda2022grokking, author = {Nanda, Neel and Lieberum, Tom}, title={A Mechanistic Interpretability Analysis of Grokking}, year={2022}, url={https://www.lesswrong.com/posts/N6WM6hs7RQMKDhYjB/a-mechanistic-interpretability-analysis-of-grokking} } @misc{nanda2023factualrecall, author = {Nanda, Neel and Rajamanoharan, Senthooran, Kramár, János and Shah, Rohin}, title={Fact Finding: Attempting to Reverse-Engineer Factual Recall on the Neuron Level}, year={2023}, url={https://www.alignmentforum.org/posts/iGuwZTHWb6DFY3sKB/fact-finding-attempting-to-reverse-engineer-factual-recall} } @article{engstrom2019adversarial, title={Adversarial robustness as a prior for learned representations}, author={Engstrom, Logan and Ilyas, Andrew and Santurkar, Shibani and Tsipras, Dimitris and Tran, Brandon and Madry, Aleksander}, journal={arXiv preprint arXiv:1906.00945}, year={2019}, url={https://arxiv.org/pdf/1906.00945} } @article{liu2016delving, title={Delving into transferable adversarial examples and black-box attacks}, author={Liu, Yanpei and Chen, Xinyun and Liu, Chang and Song, Dawn}, journal={arXiv preprint arXiv:1611.02770}, year={2016}, url={https://arxiv.org/pdf/1611.02770} } @article{elhage2022solu, title={Softmax Linear Units}, author={Elhage, Nelson and Hume, Tristan and Olsson, Catherine and Nanda, Neel and Henighan, Tom and Johnston, Scott and ElShowk, Sheer and Joseph, Nicholas and DasSarma, Nova and Mann, Ben and Hernandez, Danny and Askell, Amanda and Ndousse, Kamal and Jones, Andy and Drain, Dawn and Chen, Anna and Bai, Yuntao and Ganguli, Deep and Lovitt, Liane and Hatfield-Dodds, Zac and Kernion, Jackson and Conerly, Tom and Kravec, Shauna and Fort, Stanislav and Kadavath, Saurav and Jacobson, Josh and Tran-Johnson, Eli and Kaplan, Jared and Clark, Jack and Brown, Tom and McCandlish, Sam and Amodei, Dario and Olah, Christopher}, year={2022}, journal={Transformer Circuits Thread}, url={https://transformer-circuits.pub/2022/solu/index.html} } @article{bengio2013representation, title={Representation learning: A review and new perspectives}, author={Bengio, Yoshua and Courville, Aaron and Vincent, Pascal}, journal={IEEE transactions on pattern analysis and machine intelligence}, volume={35}, number={8}, pages={1798--1828}, year={2013}, publisher={IEEE}, url={https://arxiv.org/pdf/1206.5538} } @article{cheung2019superposition, title={Superposition of many models into one}, author={Cheung, Brian and Terekhov, Alexander and Chen, Yubei and Agrawal, Pulkit and Olshausen, Bruno}, journal={Advances in neural information processing systems}, volume={32}, year={2019}, url={https://proceedings.neurips.cc/paper/2019/file/4c7a167bb329bd92580a99ce422d6fa6-Paper.pdf} } @book{alon2019introduction, title={An introduction to systems biology: design principles of biological circuits}, author={Alon, Uri}, year={2019}, publisher={CRC press}, doi={10.1201/9781420011432}, url={https://www.taylorfrancis.com/books/mono/10.1201/9781420011432/introduction-systems-biology-uri-alon} } @inproceedings{kim2018interpretability, title={Interpretability beyond feature attribution: Quantitative testing with concept activation vectors (tcav)}, author={Kim, Been and Wattenberg, Martin and Gilmer, Justin and Cai, Carrie and Wexler, James and Viegas, Fernanda and others}, booktitle={International conference on machine learning}, pages={2668--2677}, year={2018}, organization={PMLR}, url={http://proceedings.mlr.press/v80/kim18d/kim18d.pdf} } @article{fedus2022review, title={A Review of Sparse Expert Models in Deep Learning}, author={Fedus, William and Dean, Jeff and Zoph, Barret}, journal={arXiv preprint arXiv:2209.01667}, year={2022}, url={https://arxiv.org/pdf/2209.01667} } @article{higgins2016beta, title={beta-vae: Learning basic visual concepts with a constrained variational framework}, author={Higgins, Irina and Matthey, Loic and Pal, Arka and Burgess, Christopher and Glorot, Xavier and Botvinick, Matthew and Mohamed, Shakir and Lerchner, Alexander}, year={2016}, url={https://openreview.net/pdf?id=Sy2fzU9gl} } @article{donoho2006compressed, title={Compressed sensing}, author={Donoho, David L}, journal={IEEE Transactions on information theory}, volume={52}, number={4}, pages={1289--1306}, year={2006}, publisher={IEEE}, url={https://www.cmor-faculty.rice.edu/~yzhang/caam699/Image%20papers/CompSensing.pdf} } @book{lakatos1963proofs, title={Proofs and refutations}, author={Lakatos, Imre}, year={1963}, publisher={Nelson London} } @article{hu2020surprising, title={The surprising simplicity of the early-time learning dynamics of neural networks}, author={Hu, Wei and Xiao, Lechao and Adlam, Ben and Pennington, Jeffrey}, journal={Advances in Neural Information Processing Systems}, volume={33}, pages={17116--17128}, year={2020}, url={https://proceedings.neurips.cc/paper_files/paper/2020/file/c6dfc6b7c601ac2978357b7a81e2d7ae-Paper.pdf} } @article{candes2005decoding, title={Decoding by linear programming}, author={Candes, Emmanuel J and Tao, Terence}, journal={IEEE transactions on information theory}, volume={51}, number={12}, pages={4203--4215}, year={2005}, publisher={IEEE}, url={https://arxiv.org/pdf/math.mg/0502327} } @inproceedings{levy2014linguistic, title={Linguistic regularities in sparse and explicit word representations}, author={Levy, Omer and Goldberg, Yoav}, booktitle={Proceedings of the eighteenth conference on computational natural language learning}, pages={171--180}, year={2014}, url={https://aclanthology.org/W14-1618.pdf} } @article{bau2020understanding, title={Understanding the role of individual units in a deep neural network}, author={Bau, David and Zhu, Jun-Yan and Strobelt, Hendrik and Lapedriza, Agata and Zhou, Bolei and Torralba, Antonio}, journal={Proceedings of the National Academy of Sciences}, volume={117}, number={48}, pages={30071--30078}, year={2020}, publisher={National Acad Sciences}, url={https://www.pnas.org/doi/pdf/10.1073/pnas.1907375117} } @inproceedings{donnelly2019interpretability, title={On Interpretability and Feature Representations: An Analysis of the Sentiment Neuron}, author={Donnelly, Jonathan and Roegiest, Adam}, booktitle={European Conference on Information Retrieval}, pages={795--802}, year={2019}, organization={Springer}, url={https://kirasystems.com/files/science/Interpretability_and_Feature_Representations.pdf} } @article{papernot2016towards, title={Towards the science of security and privacy in machine learning}, author={Papernot, Nicolas and McDaniel, Patrick and Sinha, Arunesh and Wellman, Michael}, journal={arXiv preprint arXiv:1611.03814}, year={2016}, url={https://arxiv.org/pdf/1611.03814} } @article{chen2016infogan, title={Infogan: Interpretable representation learning by information maximizing generative adversarial nets}, author={Chen, Xi and Duan, Yan and Houthooft, Rein and Schulman, John and Sutskever, Ilya and Abbeel, Pieter}, journal={Advances in neural information processing systems}, volume={29}, year={2016}, url={https://proceedings.neurips.cc/paper_files/paper/2016/file/7c9d0b1f96aebd7b5eca8c3edaa19ebb-Paper.pdf} } @inproceedings{kim2018disentangling, title={Disentangling by factorising}, author={Kim, Hyunjik and Mnih, Andriy}, booktitle={International Conference on Machine Learning}, pages={2649--2658}, year={2018}, organization={PMLR}, url={http://proceedings.mlr.press/v80/kim18b/kim18b.pdf} } @article{donoho2001uncertainty, title={Uncertainty principles and ideal atomic decomposition}, author={Donoho, David L and Huo, Xiaoming and others}, journal={IEEE transactions on information theory}, volume={47}, number={7}, pages={2845--2862}, year={2001}, publisher={Citeseer}, url={https://citeseerx.ist.psu.edu/document?repid=rep1&type=pdf&doi=41025fae456fc1526d53cf3cbe74e6f88d53a5fe} } @article{cohen2009compressed, title={Compressed sensing and best 𝑘-term approximation}, author={Cohen, Albert and Dahmen, Wolfgang and DeVore, Ronald}, journal={Journal of the American mathematical society}, volume={22}, number={1}, pages={211--231}, year={2009}, url={https://www.math.mcgill.ca/yyang/ml/paper/cohen2009.pdf} } @article{kashin2007remark, title={A remark on compressed sensing}, author={Kashin, Boris S and Temlyakov, Vladimir N}, journal={Mathematical notes}, volume={82}, number={5}, pages={748--755}, year={2007}, publisher={Springer}, url={http://www.mi-ras.ru/~kashin/download/70.pdf} } @inproceedings{wainwright2007information, title={Information-theoretic bounds on sparsity recovery in the high-dimensional and noisy setting}, author={Wainwright, Martin}, booktitle={2007 IEEE International Symposium on Information Theory}, pages={961--965}, year={2007}, organization={IEEE}, url={https://arxiv.org/pdf/math.ST/0702301} } @inproceedings{ba2010lower, title={Lower bounds for sparse recovery}, author={Do Ba, Khanh and Indyk, Piotr and Price, Eric and Woodruff, David P}, booktitle={Proceedings of the twenty-first annual ACM-SIAM symposium on Discrete Algorithms}, pages={1190--1197}, year={2010}, organization={SIAM}, url={https://epubs.siam.org/doi/pdf/10.1137/1.9781611973075.95} } @techreport{donoho2005neighborly, title={Neighborly polytopes and sparse solution of underdetermined linear equations}, author={Donoho, David L}, institution={Stanford University}, year={2005}, url={https://www.researchgate.net/profile/David-Donoho/publication/246925385_Neighborly_Polytopes_and_Sparse_Solution_of_Underdetermined_Linear_Equations/links/00b7d537fa3627c6fa000000/Neighborly-Polytopes-and-Sparse-Solution-of-Underdetermined-Linear-Equations.pdf} } @article{blanchard2009compressed, title={Compressed sensing: How sharp is the RIP}, author={Blanchard, Jeffrey D and Cartis, Coralia and Tanner, Jared}, journal={SIAM Rev., accepted}, volume={10}, pages={090748160}, year={2009}, url={https://optimization-online.org/wp-content/uploads/2009/08/2383.pdf} } @inproceedings{bora2017compressed, title = {Compressed Sensing using Generative Models}, author = {Ashish Bora and Ajil Jalal and Eric Price and Alexandros G. Dimakis}, booktitle = {Proceedings of the 34th International Conference on Machine Learning}, pages = {537--546}, year = {2017}, editor = {Precup, Doina and Teh, Yee Whye}, volume = {70}, series = {Proceedings of Machine Learning Research}, month = {06--11 Aug}, publisher = {PMLR}, pdf = {http://proceedings.mlr.press/v70/bora17a/bora17a.pdf}, url = {https://proceedings.mlr.press/v70/bora17a.html} } @article{metzler2017learned, title={Learned D-AMP: Principled neural network based compressive image recovery}, author={Metzler, Chris and Mousavi, Ali and Baraniuk, Richard}, journal={Advances in Neural Information Processing Systems}, volume={30}, year={2017}, url={https://proceedings.neurips.cc/paper/2017/file/8597a6cfa74defcbde3047c891d78f90-Paper.pdf} } @inproceedings{mousavi2015deep, title={A deep learning approach to structured signal recovery}, author={Mousavi, Ali and Patel, Ankit B and Baraniuk, Richard G}, booktitle={2015 53rd annual allerton conference on communication, control, and computing (Allerton)}, pages={1336--1343}, year={2015}, organization={IEEE}, url={https://arxiv.org/pdf/1508.04065} } @article{blumensath2009iterative, title={Iterative hard thresholding for compressed sensing}, author={Blumensath, Thomas and Davies, Mike E}, journal={Applied and computational harmonic analysis}, volume={27}, number={3}, pages={265--274}, year={2009}, publisher={Elsevier}, url={https://www.sciencedirect.com/science/article/pii/S1063520309000384/pdf?md5=eeeede2b5b88fdcae6120aa9a9a97086&pid=1-s2.0-S1063520309000384-main.pdf&_valck=1} } @misc{goh2016decoding, title={Decoding The Thought Vector}, author={Gabriel Goh}, year={2016}, url={https://gabgoh.github.io/ThoughtVectors/} } @article{arora2018linear, title={Linear algebraic structure of word senses, with applications to polysemy}, author={Arora, Sanjeev and Li, Yuanzhi and Liang, Yingyu and Ma, Tengyu and Risteski, Andrej}, journal={Transactions of the Association for Computational Linguistics}, volume={6}, pages={483--495}, year={2018}, publisher={MIT Press}, url={https://aclanthology.org/Q18-1034.pdf} } @article{kovacevic2007life, title={Life beyond bases: The advent of frames (Part I)}, author={Kovacevic, Jelena and Chebira, Amina}, journal={IEEE Signal Processing Magazine}, volume={24}, number={4}, pages={86--104}, year={2007}, publisher={IEEE}, url={https://sites.gatech.edu/wp-content/uploads/sites/436/2011/04/kovacevic07framesI.pdf} } @article{schlegel2022comparison, title={A comparison of vector symbolic architectures}, author={Schlegel, Kenny and Neubert, Peer and Protzel, Peter}, journal={Artificial Intelligence Review}, volume={55}, number={6}, pages={4523--4555}, year={2022}, publisher={Springer}, url={https://link.springer.com/content/pdf/10.1007/s10462-021-10110-3.pdf} } @article{kleyko2021survey, title={A Survey on Hyperdimensional Computing aka Vector Symbolic Architectures, Part I: Models and Data Transformations}, author={Kleyko, Denis and Rachkovskij, Dmitri A and Osipov, Evgeny and Rahimi, Abbas}, journal={ACM Computing Surveys (CSUR)}, year={2021}, publisher={ACM New York, NY}, url={https://dl.acm.org/doi/pdf/10.1145/3538531} } @article{elhage2022superposition, title={Toy Models of Superposition}, author={Elhage, Nelson and Hume, Tristan and Olsson, Catherine and Schiefer, Nicholas and Henighan, Tom and Kravec, Shauna and Hatfield-Dodds, Zac and Lasenby, Robert and Drain, Dawn and Chen, Carol and Grosse, Roger and McCandlish, Sam and Kaplan, Jared and Amodei, Dario and Wattenberg, Martin and Olah, Christopher}, year={2022}, journal={Transformer Circuits Thread}, url={https://transformer-circuits.pub/2022/toy_model/index.html} } @article{henighan2023superposition, title={Superposition, Memorization, and Double Descent}, author={Henighan, Tom and Carter, Shan and Hume, Tristan and Elhage, Nelson and Lasenby, Robert and Fort, Stanislav and Schiefer, Nicholas and Olah, Christopher}, year={2023}, journal={Transformer Circuits Thread}, url={https://transformer-circuits.pub/2023/toy-double-descent/index.html} } @article{hernandez2022scaling, title={Scaling Laws and Interpretability of Learning from Repeated Data}, author={Hernandez, Danny and Brown, Tom and Conerly, Tom and DasSarma, Nova and Drain, Dawn and El-Showk, Sheer and Elhage, Nelson and Hatfield-Dodds, Zac and Henighan, Tom and Hume, Tristan and others}, journal={arXiv preprint arXiv:2205.10487}, year={2022}, url={https://arxiv.org/pdf/2205.10487} } @inproceedings{glorot2010understanding, title={Understanding the difficulty of training deep feedforward neural networks}, author={Glorot, Xavier and Bengio, Yoshua}, booktitle={Proceedings of the thirteenth international conference on artificial intelligence and statistics}, pages={249--256}, year={2010}, organization={JMLR Workshop and Conference Proceedings}, url={http://proceedings.mlr.press/v9/glorot10a/glorot10a.pdf} } @article{loshchilov2017decoupled, title={Decoupled weight decay regularization}, author={Loshchilov, Ilya and Hutter, Frank}, journal={arXiv preprint arXiv:1711.05101}, year={2017}, url={https://arxiv.org/pdf/1711.05101} } @article{advani2020high, title={High-dimensional dynamics of generalization error in neural networks}, author={Advani, Madhu S and Saxe, Andrew M and Sompolinsky, Haim}, journal={Neural Networks}, volume={132}, pages={428--446}, year={2020}, publisher={Elsevier}, url={https://www.sciencedirect.com/science/article/pii/S0893608020303117} } @article{geiger2019jamming, title={Jamming transition as a paradigm to understand the loss landscape of deep neural networks}, author={Geiger, Mario and Spigler, Stefano and d'Ascoli, St{\'e}phane and Sagun, Levent and Baity-Jesi, Marco and Biroli, Giulio and Wyart, Matthieu}, journal={Physical Review E}, volume={100}, number={1}, pages={012115}, year={2019}, publisher={APS}, url={https://arxiv.org/pdf/1809.09349} } @article{hastie2022surprises, title={Surprises in high-dimensional ridgeless least squares interpolation}, author={Hastie, Trevor and Montanari, Andrea and Rosset, Saharon and Tibshirani, Ryan J}, journal={The Annals of Statistics}, volume={50}, number={2}, pages={949--986}, year={2022}, publisher={Institute of Mathematical Statistics}, url={https://pmc.ncbi.nlm.nih.gov/articles/PMC9481183/pdf/nihms-1830540.pdf} } @article{maloney2022solvable, title={A Solvable Model of Neural Scaling Laws}, author={Maloney, Alexander and Roberts, Daniel A and Sully, James}, journal={arXiv preprint arXiv:2210.16859}, year={2022}, url={https://arxiv.org/pdf/2210.16859} } @book{elad2010sparse, title={Sparse and redundant representations: from theory to applications in signal and image processing}, author={Elad, Michael}, volume={2}, number={1}, year={2010}, publisher={Springer}, } @inproceedings{engan1999method, title={Method of optimal directions for frame design}, author={Engan, Kjersti and Aase, Sven Ole and Husoy, J Hakon}, booktitle={1999 IEEE International Conference on Acoustics, Speech, and Signal Processing. Proceedings. ICASSP99 (Cat. No. 99CH36258)}, volume={5}, pages={2443--2446}, year={1999}, organization={IEEE} } @article{aharon2006k, title={K-SVD: An algorithm for designing overcomplete dictionaries for sparse representation}, author={Aharon, Michal and Elad, Michael and Bruckstein, Alfred}, journal={IEEE Transactions on signal processing}, volume={54}, number={11}, pages={4311--4322}, year={2006}, publisher={IEEE}, url={https://freddy.cs.technion.ac.il/wp-content/uploads/2017/12/K-SVD-An-Algorithm-for-Designing-Overcomplete.pdf} } @inproceedings{Gregor2010LearningFA, title={Learning Fast Approximations of Sparse Coding}, author={Karol Gregor and Yann LeCun}, booktitle={International Conference on Machine Learning}, year={2010}, url={https://icml.cc/Conferences/2010/papers/449.pdf} } @article{Barello2018SparseCodingVA, title={Sparse-Coding Variational Auto-Encoders}, author={G. Barello and Adam S. Charles and Jonathan W. Pillow}, journal={bioRxiv}, year={2018}, url={https://www.biorxiv.org/content/biorxiv/early/2018/08/23/399246.full.pdf?%3Fcollection=} } @misc{sharkey2022interim, title={[Interim research report] Taking features out of superposition with sparse autoencoders}, author={Lee Sharkey and Dan Braun and Beren Millidge}, year={2022}, url={https://www.lesswrong.com/posts/z6QQJbtpkEAX3Aojj/interim-research-report-taking-features-out-of-superposition} } @misc{li2023inferencetime, title={Inference-Time Intervention: Eliciting Truthful Answers from a Language Model}, author={Kenneth Li and Oam Patel and Fernanda Viégas and Hanspeter Pfister and Martin Wattenberg}, year={2023}, eprint={2306.03341}, archivePrefix={arXiv}, primaryClass={cs.LG}, url={https://arxiv.org/pdf/2306.03341} } @misc{turner2023activation, title={Activation Addition: Steering Language Models Without Optimization}, author={Alexander Matt Turner and Lisa Thiergart and David Udell and Gavin Leech and Ulisse Mini and Monte MacDiarmid}, year={2023}, eprint={2308.10248}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/pdf/2308.10248} } @article{marks2023geometry, title={The geometry of truth: Emergent linear structure in large language model representations of true/false datasets}, author={Marks, Samuel and Tegmark, Max}, journal={arXiv preprint arXiv:2310.06824}, url={https://arxiv.org/pdf/2310.06824}, year={2023} } @misc{rimsky2024steering, title={Steering Llama 2 via Contrastive Activation Addition}, author={Nina Rimsky and Nick Gabrieli and Julian Schulz and Meg Tong and Evan Hubinger and Alexander Matt Turner}, year={2024}, eprint={2312.06681}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/pdf/2312.06681} } @article{alain2016understanding, title={Understanding intermediate layers using linear classifier probes}, author={Alain, Guillaume and Bengio, Yoshua}, journal={arXiv preprint arXiv:1610.01644}, year={2016}, url={https://arxiv.org/pdf/1610.01644} } @article{viegas2023system, title={The System Model and the User Model: Exploring AI Dashboard Design}, author={Viégas, Fernanda and Wattenberg, Martin}, journal={arXiv preprint arXiv:2305.02469}, year={2023}, url={https://arxiv.org/pdf/2305.02469} } @article{li2022emergent, title={Emergent world representations: Exploring a sequence model trained on a synthetic task}, author={Li, Kenneth and Hopkins, Aspen K and Bau, David and Viégas, Fernanda and Pfister, Hanspeter and Wattenberg, Martin}, journal={arXiv preprint arXiv:2210.13382}, year={2022}, url={https://arxiv.org/pdf/2210.13382} } @article{faruqui2015sparse, title={Sparse overcomplete word vector representations}, author={Faruqui, Manaal and Tsvetkov, Yulia and Yogatama, Dani and Dyer, Chris and Smith, Noah}, journal={arXiv preprint arXiv:1506.02004}, year={2015}, url={https://arxiv.org/pdf/1506.02004} } @misc{belrose2023leace, title={LEACE: Perfect linear concept erasure in closed form}, author={Nora Belrose and David Schneider-Joseph and Shauli Ravfogel and Ryan Cotterell and Edward Raff and Stella Biderman}, year={2023}, eprint={2306.03819}, archivePrefix={arXiv}, primaryClass={cs.LG}, url={https://arxiv.org/pdf/2306.03819} } @misc{bills2023language, title={Language models can explain neurons in language models}, author={ Bills, Steven and Cammarata, Nick and Mossing, Dan and Tillman, Henk and Gao, Leo and Goh, Gabriel and Sutskever, Ilya and Leike, Jan and Wu, Jeff and Saunders, William }, year={2023}, url ={https://openaipublic.blob.core.windows.net/neuron-explainer/paper/index.html} } @misc{olah2023distributed, title={Distributed Representations: Composition \& Superposition}, author={Olah, Christopher}, year={2023}, url={https://transformer-circuits.pub/2023/superposition-composition/index.html} } @article{scherlis2022polysemanticity, title={Polysemanticity and capacity in neural networks}, author={Scherlis, Adam and Sachan, Kshitij and Jermyn, Adam S and Benton, Joe and Shlegeris, Buck}, journal={arXiv preprint arXiv:2210.01892}, year={2022}, url={https://arxiv.org/pdf/2210.01892} } @article{gurnee2023finding, title={Finding Neurons in a Haystack: Case Studies with Sparse Probing}, author={Gurnee, Wes and Nanda, Neel and Pauly, Matthew and Harvey, Katherine and Troitskii, Dmitrii and Bertsimas, Dimitris}, journal={arXiv preprint arXiv:2305.01610}, year={2023}, url={https://arxiv.org/pdf/2305.01610} } @article{yun2021transformer, title={Transformer visualization via dictionary learning: contextualized embedding as a linear superposition of transformer factors}, author={Yun, Zeyu and Chen, Yubei and Olshausen, Bruno A and LeCun, Yann}, journal={arXiv preprint arXiv:2103.15949}, year={2021}, url={https://arxiv.org/pdf/2103.15949} } @article{mahinpei2021promises, title={Promises and pitfalls of black-box concept learning models}, author={Mahinpei, Anita and Clark, Justin and Lage, Isaac and Doshi-Velez, Finale and Pan, Weiwei}, journal={arXiv preprint arXiv:2106.13314}, year={2021}, url={https://arxiv.org/pdf/2106.13314} } @article{krotov2019unsupervised, title={Unsupervised learning by competing hidden units}, author={Krotov, Dmitry and Hopfield, John J}, journal={Proceedings of the National Academy of Sciences}, volume={116}, number={16}, pages={7723--7731}, year={2019}, publisher={National Acad Sciences}, url={https://www.pnas.org/doi/pdf/10.1073/pnas.1820458116} } @inproceedings{rauker2023toward, title={Toward transparent ai: A survey on interpreting the inner structures of deep neural networks}, author={Räuker, Tilman and Ho, Anson and Casper, Stephen and Hadfield-Menell, Dylan}, booktitle={2023 IEEE Conference on Secure and Trustworthy Machine Learning (SaTML)}, pages={464--483}, year={2023}, organization={IEEE}, url={https://arxiv.org/pdf/2207.13243} } @article{jermyn2022engineering, title={Engineering monosemanticity in toy models}, author={Jermyn, Adam S and Schiefer, Nicholas and Hubinger, Evan}, journal={arXiv preprint arXiv:2211.09169}, year={2022}, url={https://arxiv.org/pdf/2211.09169} } @article{zhang2019word, title={Word embedding visualization via dictionary learning}, author={Zhang, Juexiao and Chen, Yubei and Cheung, Brian and Olshausen, Bruno A}, journal={arXiv preprint arXiv:1910.03833}, year={2019}, url={https://arxiv.org/pdf/1910.03833} } @misc{cunningham2023replication, title={[Replication] Conjecture's Sparse Coding in Toy Models}, author={Cunningham, Hoagy and Smith, Logan}, year={2023}, url={https://www.lesswrong.com/posts/E8imxQo96WgDCMxkA/replication-conjecture-s-sparse-coding-in-toy-models} } @misc{huben2023update, title={[Research Update] Sparse Autoencoder features are bimodal}, author={Huben, Robert}, year={2023}, url={https://aizi.substack.com/p/research-update-sparse-autoencoder} } @misc{smith2023found, title={(tentatively) Found 600+ Monosemantic Features in a Small LM Using Sparse Autoencoders}, author={Smith, Logan}, year={2023}, url={https://www.lesswrong.com/posts/wqRqb7h6ZC48iDgfK/tentatively-found-600-monosemantic-features-in-a-small-lm} } @misc{smith2023strong, title={Really Strong Features Found in Residual Stream}, author={Smith, Logan}, year={2023}, url={https://www.lesswrong.com/posts/Q76CpqHeEMykKpFdB/really-strong-features-found-in-residual-stream} } @misc{cunningham2023autointerpretation, title={AutoInterpretation Finds Sparse Coding Beats Alternatives}, author={Cunningham, Hoagy}, year={2023}, url={https://www.lesswrong.com/posts/ursraZGcpfMjCXtnn/autointerpretation-finds-sparse-coding-beats-alternatives} } @misc{lsgos2023dropout, title={Dropout can create a privileged basis in the ReLU output model}, author={lsgos}, year={2023}, url={https://www.lesswrong.com/posts/uSdFFTATPFJz4pQyB/dropout-can-create-a-privileged-basis-in-the-relu-output} } @misc{hobbhahn2023more, title={More findings on Memorization and double descent}, author={Hobbhahn, Marius}, year={2023}, url={https://www.alignmentforum.org/posts/KzwB4ovzrZ8DYWgpw/more-findings-on-memorization-and-double-descent} } @misc{braunE2E2024, title={Identifying Functionally Important Features with End-to-End Sparse Dictionary Learning}, author={Braun, Dan and Taylor, Jordan and Goldowsky-Dill, Nicholas and Sharkey, Lee}, year={2024}, url={https://publications.apolloresearch.ai/end_to_end_sparse_dictionary_learning.pdf} } @article{braun2025identifying, title={Identifying functionally important features with end-to-end sparse dictionary learning}, author={Braun, Dan and Taylor, Jordan and Goldowsky-Dill, Nicholas and Sharkey, Lee}, journal={Advances in Neural Information Processing Systems}, volume={37}, pages={107286--107325}, year={2025}, url={https://publications.apolloresearch.ai/end_to_end_sparse_dictionary_learning.pdf} } @misc{nanda2023actually, title={Actually, Othello-GPT Has A Linear Emergent World Representation}, author={Nanda, Neel}, year={2023}, url={https://www.neelnanda.io/mechanistic-interpretability/othello} } @article{kingma2014adam, title={Adam: A method for stochastic optimization}, author={Kingma, Diederik P and Ba, Jimmy}, journal={arXiv preprint arXiv:1412.6980}, url={https://arxiv.org/pdf/1412.6980}, year={2014} } @article{cunningham2023sparse, title={Sparse Autoencoders Find Highly Interpretable Model Directions}, author={Cunningham, Hoagy and Ewart, Aidan and Smith, Logan and Huben, Robert and Sharkey, Lee}, journal={arXiv preprint arXiv:2309.08600}, url={https://arxiv.org/pdf/2309.08600}, year={2023} } @misc{rumbelow2023solid, title={SolidGoldMagikarp (plus, prompt generation)}, author={Rumbelow, Jessica and Watkins, Matthew}, year={2023}, url={https://www.lesswrong.com/posts/aPeJE8bSo6rAFoLqg/solidgoldmagikarp-plus-prompt-generation} } @article{burns2022discovering, title={Discovering latent knowledge in language models without supervision}, author={Burns, Collin and Ye, Haotian and Klein, Dan and Steinhardt, Jacob}, journal={arXiv preprint arXiv:2212.03827}, year={2022}, url={https://arxiv.org/pdf/2212.03827} } @article{mcgrath2022acquisition, title={Acquisition of chess knowledge in alphazero}, author={McGrath, Thomas and Kapishnikov, Andrei and Toma{\v{s}}ev, Nenad and Pearce, Adam and Wattenberg, Martin and Hassabis, Demis and Kim, Been and Paquet, Ulrich and Kramnik, Vladimir}, journal={Proceedings of the National Academy of Sciences}, volume={119}, number={47}, pages={e2206625119}, year={2022}, publisher={National Acad Sciences}, url={https://www.pnas.org/doi/pdf/10.1073/pnas.2206625119} } @article{frey2014topk, title={k-Sparse Autoencoders}, author={Alireza Makhzani and Brendan J. Frey}, journal={CoRR}, year={2013}, volume={abs/1312.5663}, url={https://api.semanticscholar.org/CorpusID:14850799} } @article{nanda2023emergent, title={Emergent Linear Representations in World Models of Self-Supervised Sequence Models}, author={Nanda, Neel and Lee, Andrew and Wattenberg, Martin}, journal={arXiv preprint arXiv:2309.00941}, year={2023}, url={https://arxiv.org/pdf/2309.00941} } @inproceedings{he2015delving, title={Delving deep into rectifiers: Surpassing human-level performance on imagenet classification}, author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian}, booktitle={Proceedings of the IEEE international conference on computer vision}, pages={1026--1034}, year={2015}, url={https://openaccess.thecvf.com/content_iccv_2015/papers/He_Delving_Deep_into_ICCV_2015_paper.pdf} } @article{frankle2018lottery, title={The lottery ticket hypothesis: Finding sparse, trainable neural networks}, author={Frankle, Jonathan and Carbin, Michael}, journal={arXiv preprint arXiv:1803.03635}, year={2018}, url={https://arxiv.org/pdf/1803.03635} } @article{bricken2023emergence, title={Emergence of Sparse Representations from Noise}, author={Bricken, Trenton and Schaeffer, Rylan and Olshausen, Bruno and Kreiman, Gabriel}, year={2023}, url={https://openreview.net/pdf?id=cxYaBAXVKg} } @article{miao2021incorporating, title={On incorporating inductive biases into VAEs}, author={Miao, Ning and Mathieu, Emile and Siddharth, N and Teh, Yee Whye and Rainforth, Tom}, journal={arXiv preprint arXiv:2106.13746}, year={2021}, url={https://arxiv.org/pdf/2106.13746} } @article{rentzeperis2023beyond, title={Beyond ℓ₁ sparse coding in V1}, author={Rentzeperis, Ilias and Calatroni, Luca and Perrinet, Laurent and Prandi, Dario}, journal={arXiv preprint arXiv:2301.10002}, year={2023}, url={https://arxiv.org/pdf/2301.10002} } @article{hurley2009comparing, title={Comparing measures of sparsity}, author={Hurley, Niall and Rickard, Scott}, journal={IEEE Transactions on Information Theory}, volume={55}, number={10}, pages={4723--4741}, year={2009}, publisher={IEEE}, url={https://arxiv.org/pdf/0811.4706} } @misc{nanda2023patching, title={Attribution Patching: Activation Patching At Industrial Scale}, author={Nanda, Neel}, year={2023}, url={https://www.neelnanda.io/mechanistic-interpretability/attribution-patching} } @article{grosse2023studying, title={Studying Large Language Model Generalization with Influence Functions}, author={Grosse, Roger and Bae, Juhan and Anil, Cem and Elhage, Nelson and Tamkin, Alex and Tajdini, Amirhossein and Steiner, Benoit and Li, Dustin and Durmus, Esin and Perez, Ethan and others}, journal={arXiv preprint arXiv:2308.03296}, year={2023}, url={https://arxiv.org/pdf/2308.03296} } @inproceedings{hernandez2021natural, title={Natural language descriptions of deep visual features}, author={Hernandez, Evan and Schwettmann, Sarah and Bau, David and Bagashvili, Teona and Torralba, Antonio and Andreas, Jacob}, booktitle={International Conference on Learning Representations}, year={2021}, url={https://arxiv.org/pdf/2201.11114} } @misc{christiano2022anomaly, title={Mechanistic anomaly detection and ELK}, author={Christiano, Paul}, year={2022}, url={https://ai-alignment.com/mechanistic-anomaly-detection-and-elk-fb84f4c6d0dc} } @misc{wright2024suppression, title={Addressing Feature Suppression in SAEs}, author={Wright, Benjamin and Sharkey, Lee}, year={2024}, url={https://www.lesswrong.com/posts/3JuSjTZyMzaSeTxKk/addressing-feature-suppression-in-saes} } @article{rajamanoharan2024improving, title={Improving Dictionary Learning with Gated Sparse Autoencoders}, author={Rajamanoharan, Senthooran and Conmy, Arthur and Smith, Lewis and Lieberum, Tom and Varma, Vikrant and Kram{\'a}r, J{\'a}nos and Shah, Rohin and Nanda, Neel}, journal={arXiv preprint arXiv:2404.16014}, year={2024}, url={https://arxiv.org/pdf/2404.16014} } @misc{vaintrob2024computation, title={Toward A Mathematical Framework for Computation in Superposition}, author={Vaintrob, Dmitry and Mendel, Jake and H\"{a}nni, Kaarel}, year={2024}, url={https://www.lesswrong.com/posts/2roZtSr5TGmLjXMnT/toward-a-mathematical-framework-for-computation-in} } @article{he2024dictionary, title={Dictionary Learning Improves Patch-Free Circuit Discovery in Mechanistic Interpretability: A Case Study on Othello-GPT}, author={He, Zhengfu and Ge, Xuyang and Tang, Qiong and Sun, Tianxiang and Cheng, Qinyuan and Qiu, Xipeng}, journal={arXiv preprint arXiv:2402.12201}, year={2024}, url={https://arxiv.org/pdf/2402.12201} } @misc{riggs2024improvingsae, title={Improving SAE's by Sqrt()-ing L1 \& Removing Lowest Activating Features}, author={Riggs, Logan and Brinkmann, Jannik}, year={2024}, url={https://www.lesswrong.com/posts/YiGs8qJ8aNBgwt2YN/improving-sae-s-by-sqrt-ing-l1-and-removing-lowest} } @misc{fry2024vision, title={Towards Multimodal Interpretability: Learning Sparse Interpretable Features in Vision Transformers}, author={Fry, Hugo}, year={2024}, url={https://www.lesswrong.com/posts/bCtbuWraqYTDtuARg/towards-multimodal-interpretability-learning-sparse-2} } @misc{kissane2024attention, title={Sparse Autoencoders Work on Attention Layer Outputs}, author={Kissane, Connor and robertzk and Conmy, Arthur and Nanda, Neel}, year={2024}, url={https://www.lesswrong.com/posts/DtdzGwFh9dCfsekZZ/sparse-autoencoders-work-on-attention-layer-outputs} } @article{bricken2023monosemanticity, title={Towards Monosemanticity: Decomposing Language Models With Dictionary Learning}, author={Bricken, Trenton and Templeton, Adly and Batson, Joshua and Chen, Brian and Jermyn, Adam and Conerly, Tom and Turner, Nick and Anil, Cem and Denison, Carson and Askell, Amanda and Lasenby, Robert and Wu, Yifan and Kravec, Shauna and Schiefer, Nicholas and Maxwell, Tim and Joseph, Nicholas and Hatfield-Dodds, Zac and Tamkin, Alex and Nguyen, Karina and McLean, Brayden and Burke, Josiah E and Hume, Tristan and Carter, Shan and Henighan, Tom and Olah, Christopher}, year={2023}, journal={Transformer Circuits Thread}, url={https://transformer-circuits.pub/2023/monosemantic-features/index.html}, note={https://transformer-circuits.pub/2023/monosemantic-features/index.html} } @article{elhage23basis, title={Privileged Bases in the Transformer Residual Stream}, author={Elhage, Nelson and Lasenby, Robert and Olah, Christopher}, journal={Transformer Circuits Thread}, year={2023}, url={https://transformer-circuits.pub/2023/privileged-basis/index.html} } @misc{olsson2021mlp, title={MLP Neurons - 40L Preliminary Investigation [rough early thoughts]}, author={Olsson, Catherine and Elhage, Nelson and Olah, Chris}, url={https://www.youtube.com/watch?v=8wYNsoycM1U} } @misc{40l2021l, title={MLP Neurons - 40L Preliminary Investigation [rough early thoughts]}, author={Olsson, Catherine and Elhage, Nelson and Olah, Chris}, url={https://www.youtube.com/watch?v=8wYNsoycM1U} } @article{marks2024sparse, title={Sparse Feature Circuits: Discovering and Editing Interpretable Causal Graphs in Language Models}, author={Marks, Samuel and Rager, Can and Michaud, Eric J and Belinkov, Yonatan and Bau, David and Mueller, Aaron}, journal={arXiv preprint arXiv:2403.19647}, year={2024}, url={https://arxiv.org/pdf/2403.19647} } @misc{till2024true, title={Do sparse autoencoders find "true features"?}, author={Till, Demian}, year={2024}, url={https://www.lesswrong.com/posts/QoR8noAB3Mp2KBA4B/do-sparse-autoencoders-find-true-features} } @misc{aizi2024only, title={Research Report: Sparse Autoencoders find only 9/180 board state features in OthelloGPT}, author={AIZI, Robert}, year={2024}, url={https://www.lesswrong.com/posts/BduCMgmjJnCtc7jKc/research-report-sparse-autoencoders-find-only-9-180-board} } @misc{anders2024composed, title={Sparse autoencoders find composed features in small toy models}, author={Anders,Evan and Neo, Clement and Hoelscher-Obermaier, Jason and Howard, Jessica N.}, year={2024}, url={https://www.lesswrong.com/posts/a5wwqza2cY3W7L9cj/sparse-autoencoders-find-composed-features-in-small-toy} } @misc{bloom2024open, title={Open Source Sparse Autoencoders for all Residual Stream Layers of GPT2-Small}, author={Bloom, Joseph}, year={2024}, url={https://www.lesswrong.com/posts/f9EgfLSurAiqRJySD/open-source-sparse-autoencoders-for-all-residual-stream} } @misc{openai2024debugger, title={Transformer Debugger}, author={Mossing, Dan and Bills, Steven and Tillman, Henk and Dupré la Tour, Tom and Cammarata, Nick and Gao, Leo and Achiam, Joshua and Yeh, Catherine and Leike, Jan and Wu, Jeff and Saunders, William}, year={2024}, url={https://github.com/openai/transformer-debugger} } @misc{gurnee2024pathological, title={SAE reconstruction errors are (empirically) pathological}, author={Gurnee, Wes}, year={2024}, url={https://www.lesswrong.com/posts/rZPiuFxESMxCDHe4B/sae-reconstruction-errors-are-empirically-pathological} } @misc{lindsey2024how, title={How Strongly do Dictionary Learning Features Influence Model Behavior?}, author={Lindsey, Jack}, year={2024}, url={https://transformer-circuits.pub/2024/april-update/index.html#ablation-exps} } @misc{jermyn20248l, title={Features in an 8-layer Model}, author={Jermyn, Adam and Conerly, Tom and Bricken, Trenton and Templeton, Adly}, year={2024}, url={https://transformer-circuits.pub/2024/jan-update/index.html#dict-learning} } @article{zou2023representation, title={Representation engineering: A top-down approach to ai transparency}, author={Zou, Andy and Phan, Long and Chen, Sarah and Campbell, James and Guo, Phillip and Ren, Richard and Pan, Alexander and Yin, Xuwang and Mazeika, Mantas and Dombrowski, Ann-Kathrin and others}, journal={arXiv preprint arXiv:2310.01405}, year={2023}, url={https://arxiv.org/pdf/2310.01405} } @article{kadavath2022language, title={Language models (mostly) know what they know}, author={Kadavath, Saurav and Conerly, Tom and Askell, Amanda and Henighan, Tom and Drain, Dawn and Perez, Ethan and Schiefer, Nicholas and Hatfield-Dodds, Zac and DasSarma, Nova and Tran-Johnson, Eli and others}, journal={arXiv preprint arXiv:2207.05221}, url={https://arxiv.org/pdf/2207.05221}, year={2022} } @article{jahanian2019steerability, title={On the "steerability" of generative adversarial networks}, author={Jahanian, Ali and Chai, Lucy and Isola, Phillip}, journal={arXiv preprint arXiv:1907.07171}, year={2019}, url={https://arxiv.org/pdf/1907.07171} } @inproceedings{upchurch2017deep, title={Deep feature interpolation for image content changes}, author={Upchurch, Paul and Gardner, Jacob and Pleiss, Geoff and Pless, Robert and Snavely, Noah and Bala, Kavita and Weinberger, Kilian}, booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, pages={7064--7073}, year={2017}, url={https://openaccess.thecvf.com/content_cvpr_2017/papers/Upchurch_Deep_Feature_Interpolation_CVPR_2017_paper.pdf} } @inproceedings{dev2020measuring, title={On measuring and mitigating biased inferences of word embeddings}, author={Dev, Sunipa and Li, Tao and Phillips, Jeff M and Srikumar, Vivek}, booktitle={Proceedings of the AAAI Conference on Artificial Intelligence}, volume={34}, number={05}, pages={7659--7666}, year={2020}, url={https://ojs.aaai.org/index.php/AAAI/article/view/6267/6123} } @article{bolukbasi2016man, title={Man is to computer programmer as woman is to homemaker? debiasing word embeddings}, author={Bolukbasi, Tolga and Chang, Kai-Wei and Zou, James Y and Saligrama, Venkatesh and Kalai, Adam T}, journal={Advances in neural information processing systems}, volume={29}, year={2016}, url={https://proceedings.neurips.cc/paper_files/paper/2016/file/a486cd07e4ac3d270571622f4f316ec5-Paper.pdf} } @article{christiano2021eliciting, title={Eliciting latent knowledge: How to tell if your eyes deceive you}, author={Christiano, Paul and Cotra, Ajeya and Xu, Mark}, journal={Google Docs, December}, year={2021}, url={https://docs.google.com/document/d/1WwsnJQstPq91_Yh-Ch2XRL8H_EpsnjrC1dwZXR37PC8/edit?tab=t.0#heading=h.kkaua0hwmp1d} } @article{tamkin2023codebook, title={Codebook features: Sparse and discrete interpretability for neural networks}, author={Tamkin, Alex and Taufeeque, Mohammad and Goodman, Noah D}, journal={arXiv preprint arXiv:2310.17230}, url={https://arxiv.org/pdf/2310.17230}, year={2023} } @misc{conmy2024steering, title={Activation Steering with SAEs}, author={Conmy, Arthur and Nanda, Neel}, year={2024}, url={https://www.lesswrong.com/posts/C5KAZQib3bzzpeyrg/full-post-progress-update-1-from-the-gdm-mech-interp-team#Activation_Steering_with_SAEs} } @misc{batson2024easy, title={Using Features For Easy Circuit Identification}, author={Batson, Joshua and Chen, Brian and Jones, Andy}, year={2024}, url={https://transformer-circuits.pub/2024/march-update/index.html#feature-heads} } @misc{lindsey2024evals, title={Interpretability Evals for Dictionary Learning}, author={Lindsey, Jack and Cunningham, Hoagy and Conerly, Tom, and Templeton, Adly}, year={2024}, url={https://transformer-circuits.pub/2024/august-update/index.html#interp-evals} } @misc{turner2024sensitivity, title={Measuring feature sensitivity using dataset filtering}, author={Turner, Nicholas L and Jermyn, Adam and Batson, Joshua}, year={2024}, url={https://transformer-circuits.pub/2024/july-update/index.html#feature-sensitivity} } @article{todd2023function, title={Function vectors in large language models}, author={Todd, Eric and Li, Millicent L and Sharma, Arnab Sen and Mueller, Aaron and Wallace, Byron C and Bau, David}, journal={arXiv preprint arXiv:2310.15213}, url={https://arxiv.org/pdf/2310.15213}, year={2023} } @article{hoffmann2022training, title={Training compute-optimal large language models}, author={Hoffmann, Jordan and Borgeaud, Sebastian and Mensch, Arthur and Buchatskaya, Elena and Cai, Trevor and Rutherford, Eliza and Casas, Diego de Las and Hendricks, Lisa Anne and Welbl, Johannes and Clark, Aidan and others}, journal={arXiv preprint arXiv:2203.15556}, url={https://arxiv.org/pdf/2203.15556}, year={2022} } @online{macdiarmid2024sleeperagentprobes, author = {Monte MacDiarmid and Timothy Maxwell and Nicholas Schiefer and Jesse Mu and Jared Kaplan and David Duvenaud and Sam Bowman and Alex Tamkin and Ethan Perez and Mrinank Sharma and Carson Denison and Evan Hubinger}, title = {Simple probes can catch sleeper agents}, date = {2024-04-23}, year = {2024}, url = {https://www.anthropic.com/news/probes-catch-sleeper-agents}, } @article{mallen2023eliciting, title={Eliciting Latent Knowledge from Quirky Language Models}, author={Mallen, Alex and Belrose, Nora}, journal={arXiv preprint arXiv:2312.01037}, url={https://arxiv.org/pdf/2312.01037}, year={2023} } @misc{gurnee2024language, title={Language Models Represent Space and Time}, author={Wes Gurnee and Max Tegmark}, year={2024}, eprint={2310.02207}, archivePrefix={arXiv}, url={https://arxiv.org/pdf/2310.02207} } @article{le2011ica, title={ICA with reconstruction cost for efficient overcomplete feature learning}, author={Le, Quoc and Karpenko, Alexandre and Ngiam, Jiquan and Ng, Andrew}, journal={Advances in neural information processing systems}, volume={24}, year={2011}, url={https://proceedings.neurips.cc/paper_files/paper/2011/file/233509073ed3432027d48b1a83f5fbd2-Paper.pdf} } @article{nye2021show, title={Show your work: Scratchpads for intermediate computation with language models}, author={Nye, Maxwell and Andreassen, Anders Johan and Gur-Ari, Guy and Michalewski, Henryk and Austin, Jacob and Bieber, David and Dohan, David and Lewkowycz, Aitor and Bosma, Maarten and Luan, David and others}, journal={arXiv preprint arXiv:2112.00114}, url={https://arxiv.org/pdf/2112.00114}, year={2021} } @article{hubinger2024sleeperagents, title={Sleeper Agents: Training Deceptive LLMs that Persist Through Safety Training}, author={Hubinger, Evan and Denison, Carson and Mu, Jesse and Lambert, Mike and Tong, Meg and MacDiarmid, Monte and Lanham, Tamera and Ziegler, Daniel M. and Maxwell, Tim and Cheng, Newton and Jermyn, Adam and Askell, Amanda and Radhakrishnan, Ansh and Anil, Cem and Duvenaud, David and Ganguli, Deep and Barez, Fazl and Clark, Jack and Ndousse, Kamal and Sachan, Kshitij and Sellitto, Michael and Sharma, Mrinank and DasSarma, Nova and Grosse, Roger and Kravec, Shauna and Bai, Yuntao and Witten, Zachary and Favaro, Marina and Brauner, Jan and Karnofsky, Holden and Christiano, Paul and Bowman, Samuel R. and Graham, Logan and Kaplan, Jared and Mindermann, Sören and Greenblatt, Ryan and Shlegeris, Buck and Schiefer, Nicholas and Perez, Ethan}, journal={arXiv preprint arXiv:2401.05566}, year={2024}, url={https://arxiv.org/pdf/2401.05566} } @article{conmy2023towards, title={Towards automated circuit discovery for mechanistic interpretability}, author={Conmy, Arthur and Mavor-Parker, Augustine and Lynch, Aengus and Heimersheim, Stefan and Garriga-Alonso, Adri{\`a}}, journal={Advances in Neural Information Processing Systems}, volume={36}, pages={16318--16352}, year={2023}, url={https://proceedings.neurips.cc/paper_files/paper/2023/file/34e1dbe95d34d7ebaf99b9bcaeb5b2be-Paper-Conference.pdf} } @article{syed2023attributionpatching, title={​​Attribution Patching Outperforms Automated Circuit Discovery}, author={Syed, Aaquib and Rager, Can and Conmy, Arthur}, journal={arXiv preprint arXiv:2310.10348}, year={2023}, url={https://arxiv.org/pdf/2310.10348} } @article{kramar2024atpstar, title={AtP*: An efficient and scalable method for localizing LLM behaviour to components}, author={Kramár, János and Lieberum, Tom and Shah, Rohin and Nanda, Neel}, journal={arXiv preprint arXiv:2403.00745}, year={2024}, url={https://arxiv.org/pdf/2403.00745} } @misc{templeton2024predicting, title={Predicting Future Activations}, author={Templeton, Adly and Batson, Joshua and Jermyn, Adam and Olah, Chris}, year={2024}, url={https://transformer-circuits.pub/2024/jan-update/index.html#predict-future} } @article{dunefsky2024transcoders, title={Transcoders find interpretable LLM feature circuits}, author={Dunefsky, Jacob and Chlenski, Philippe and Nanda, Neel}, journal={Advances in Neural Information Processing Systems}, volume={37}, pages={24375--24410}, year={2025}, url={https://arxiv.org/abs/2406.11944} } @misc{marks2024dictionary, title={dictionary\_learning Github Repository}, journal={Github}, author={Marks, Samuel and Karvonen, Adam and Mueller, Aaron}, year={2024}, url={https://github.com/saprmarks/dictionary_learning} } @article{fedus2021switch, title={Switch Transformers: Scaling to Trillion Parameter Models with Simple and Efficient Sparsity}, author={Fedus, William and Zoph, Barret and Shazeer, Noam}, year={2021}, journal={arXiv preprint arXiv:2101.03961}, url={https://arxiv.org/pdf/2101.03961} } @inproceedings{shah2023modeldiff, title={Modeldiff: A framework for comparing learning algorithms}, author={Shah, Harshay and Park, Sung Min and Ilyas, Andrew and Madry, Aleksander}, booktitle={International Conference on Machine Learning}, pages={30646--30688}, year={2023}, organization={PMLR}, url={https://proceedings.mlr.press/v202/shah23a/shah23a.pdf} } @article{ding2023bipartite, title={Bipartite invariance in mouse primary visual cortex}, author={Ding, Zhiwei and Tran, Dat T and Ponder, Kayla and Cobos, Erick and Ding, Zhuokun and Fahey, Paul G and Wang, Eric and Muhammad, Taliah and Fu, Jiakun and Cadena, Santiago A and others}, journal={bioRxiv}, year={2023}, publisher={Cold Spring Harbor Laboratory Preprints}, url={https://www.biorxiv.org/content/10.1101/2023.03.15.532836v1}, } @misc{kissane2024transfer, title={SAEs (usually) Transfer Between Base and Chat Models}, author={Kissane, Connor and Krzyzanowski, Robert and Conmy, Arthur and Nanda, Neel}, year={2024}, url={https://www.lesswrong.com/posts/fmwk6qxrpW8d4jvbd/saes-usually-transfer-between-base-and-chat-models} } @misc{kissane2024base, author={Connor Kissane and Robert Krzyzanowski and Arthur Conmy and Neel Nanda}, year={2024}, howpublished={Alignment Forum}, title={Base LLMs Refuse Too}, url={https://www.alignmentforum.org/posts/YWo2cKJgL7Lg8xWjj/base-llms-refuse-too} } @misc{heimersheim2024exponentially, title={Residual stream norms grow exponentially over the forward pass}, author={Heimersheim, Stefan and Turner, Alex}, year={2024}, url={https://www.alignmentforum.org/posts/8mizBCm3dyc432nK8/residual-stream-norms-grow-exponentially-over-the-forward} } @article{gao2024scaling, title={Scaling and evaluating sparse autoencoders}, author={Gao, Leo and la Tour, Tom Dupr{\'e} and Tillman, Henk and Goh, Gabriel and Troll, Rajan and Radford, Alec and Sutskever, Ilya and Leike, Jan and Wu, Jeffrey}, journal={arXiv preprint arXiv:2406.04093}, year={2024}, url={https://arxiv.org/pdf/2406.04093} } @inproceedings{lenc2015understanding, title={Understanding image representations by measuring their equivariance and equivalence}, author={Lenc, Karel and Vedaldi, Andrea}, booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition}, pages={991--999}, year={2015}, url={https://www.cv-foundation.org/openaccess/content_cvpr_2015/papers/Lenc_Understanding_Image_Representations_2015_CVPR_paper.pdf} } @article{bansal2021revisiting, title={Revisiting model stitching to compare neural representations}, author={Bansal, Yamini and Nakkiran, Preetum and Barak, Boaz}, journal={Advances in neural information processing systems}, volume={34}, pages={225--236}, year={2021}, url={https://proceedings.neurips.cc/paper/2021/file/01ded4259d101feb739b06c399e9cd9c-Paper.pdf} } @article{morcos2018insights, title={Insights on representational similarity in neural networks with canonical correlation}, author={Morcos, Ari and Raghu, Maithra and Bengio, Samy}, journal={Advances in neural information processing systems}, volume={31}, year={2018}, url={https://proceedings.neurips.cc/paper/2018/file/a7a3d70c6d17a73140918996d03c014f-Paper.pdf} } @article{laakso2000content, title={Content and cluster analysis: assessing representational similarity in neural systems}, author={Laakso, Aarre and Cottrell, Garrison}, journal={Philosophical psychology}, volume={13}, number={1}, pages={47--76}, year={2000}, publisher={Taylor \& Francis}, url={https://www.academia.edu/download/37134/2httu4en2ophecv7af79.pdf} } @article{barannikov2021representation, title={Representation topology divergence: A method for comparing neural network representations}, author={Barannikov, Serguei and Trofimov, Ilya and Balabin, Nikita and Burnaev, Evgeny}, journal={arXiv preprint arXiv:2201.00058}, year={2021}, url={https://arxiv.org/pdf/2201.00058} } @inproceedings{tenney2019bert, title={BERT Rediscovers the Classical NLP Pipeline}, author={Tenney, Ian and Das, Dipanjan and Pavlick, Ellie}, booktitle={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics}, pages={4593--4601}, year={2019}, publisher={Association for Computational Linguistics}, address={Florence, Italy}, doi={10.18653/v1/P19-1452}, url={https://aclanthology.org/P19-1452} } @inproceedings{hewitt2019structural, title={A Structural Probe for Finding Syntax in Word Representations}, author={Hewitt, John and Manning, Christopher D}, booktitle={Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)}, pages={4129--4138}, year={2019}, publisher={Association for Computational Linguistics}, address={Minneapolis, Minnesota}, url={https://aclanthology.org/N19-1419} } @article{prakash2024fine, title={Fine-tuning enhances existing mechanisms: A case study on entity tracking}, author={Prakash, Nikhil and Shaham, Tamar Rott and Haklay, Tal and Belinkov, Yonatan and Bau, David}, journal={arXiv preprint arXiv:2402.14811}, year={2024}, url={https://arxiv.org/pdf/2402.14811} } @article{jain2023mechanistically, title={Mechanistically analyzing the effects of fine-tuning on procedurally defined tasks}, author={Jain, Samyak and Kirk, Robert and Lubana, Ekdeep Singh and Dick, Robert P and Tanaka, Hidenori and Grefenstette, Edward and Rockt{\"a}schel, Tim and Krueger, David Scott}, journal={arXiv preprint arXiv:2311.12786}, year={2023}, url={https://arxiv.org/pdf/2311.12786} } @article{lee2024mechanistic, title={A mechanistic understanding of alignment algorithms: A case study on dpo and toxicity}, author={Lee, Andrew and Bai, Xiaoyan and Pres, Itamar and Wattenberg, Martin and Kummerfeld, Jonathan K and Mihalcea, Rada}, journal={arXiv preprint arXiv:2401.01967}, year={2024}, url={https://arxiv.org/pdf/2401.01967} } @inproceedings{Manning1999BookRF, title={Book Reviews: Foundations of Statistical Natural Language Processing}, author={Christopher D. Manning and Hinrich Sch{\"u}tze}, booktitle={International Conference on Computational Logic}, year={1999}, url={https://api.semanticscholar.org/CorpusID:52800448} } @article{Brown1992ClassBasedNM, title={Class-Based n-gram Models of Natural Language}, author={Peter F. Brown and Vincent J. Della Pietra and Peter V. de Souza and Jennifer C. Lai and Robert L. Mercer}, journal={Comput. Linguistics}, year={1992}, volume={18}, pages={467-479}, url={https://api.semanticscholar.org/CorpusID:10986188} } @misc{lee2024learningdiverseattackslarge, title={Learning diverse attacks on large language models for robust red-teaming and safety tuning}, author={Seanie Lee and Minsu Kim and Lynn Cherif and David Dobre and Juho Lee and Sung Ju Hwang and Kenji Kawaguchi and Gauthier Gidel and Yoshua Bengio and Nikolay Malkin and Moksh Jain}, year={2024}, eprint={2405.18540}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/pdf/2405.18540}, } @article{templeton2024scaling, title={Scaling Monosemanticity: Extracting Interpretable Features from Claude 3 Sonnet}, author={Templeton, Adly and Conerly, Tom and Marcus, Jonathan and Lindsey, Jack and Bricken, Trenton and Chen, Brian and Pearce, Adam and Citro, Craig and Ameisen, Emmanuel and Jones, Andy and Cunningham, Hoagy and Turner, Nicholas L and McDougall, Callum and MacDiarmid, Monte and Freeman, C. Daniel and Sumers, Theodore R. and Rees, Edward and Batson, Joshua and Jermyn, Adam and Carter, Shan and Olah, Chris and Henighan, Tom}, year={2024}, journal={Transformer Circuits Thread}, url={https://transformer-circuits.pub/2024/scaling-monosemanticity/index.html} } @misc{tigges2023linearrepresentationssentimentlarge, title={Linear Representations of Sentiment in Large Language Models}, author={Curt Tigges and Oskar John Hollinsworth and Atticus Geiger and Neel Nanda}, year={2023}, eprint={2310.15154}, archivePrefix={arXiv}, primaryClass={cs.LG}, url={https://arxiv.org/pdf/2310.15154}, } @article{Alain2016UnderstandingIL, title={Understanding intermediate layers using linear classifier probes}, author={Guillaume Alain and Yoshua Bengio}, journal={ArXiv}, year={2016}, volume={abs/1610.01644}, url={https://api.semanticscholar.org/CorpusID:9794990} } @article{lan2024sparse, title={Sparse Autoencoders Reveal Universal Feature Spaces Across Large Language Models}, author={Lan, Michael and Torr, Philip and Meek, Austin and Khakzar, Ashkan and Krueger, David and Barez, Fazl}, journal={arXiv preprint arXiv:2410.06981}, year={2024}, url={https://arxiv.org/pdf/2410.06981} } @article{gorton2024missing, title={The Missing Curve Detectors of InceptionV1: Applying Sparse Autoencoders to InceptionV1 Early Vision}, author={Gorton, Liv}, journal={arXiv preprint arXiv:2406.03662}, year={2024}, url={https://arxiv.org/pdf/2406.03662} } @article{rajamanoharan2024jumping, title={Jumping ahead: Improving reconstruction fidelity with jumprelu sparse autoencoders}, author={Rajamanoharan, Senthooran and Lieberum, Tom and Sonnerat, Nicolas and Conmy, Arthur and Varma, Vikrant and Kram{\'a}r, J{\'a}nos and Nanda, Neel}, journal={arXiv preprint arXiv:2407.14435}, year={2024}, url={https://arxiv.org/pdf/2407.14435} } @article{hubinger2024sleeperagentstrainingdeceptive, title={Sleeper Agents: Training Deceptive LLMs that Persist Through Safety Training}, author={Evan Hubinger and Carson Denison and Jesse Mu and Mike Lambert and Meg Tong and Monte MacDiarmid and Tamera Lanham and Daniel M. Ziegler and Tim Maxwell and Newton Cheng and Adam Jermyn and Amanda Askell and Ansh Radhakrishnan and Cem Anil and David Duvenaud and Deep Ganguli and Fazl Barez and Jack Clark and Kamal Ndousse and Kshitij Sachan and Michael Sellitto and Mrinank Sharma and Nova DasSarma and Roger Grosse and Shauna Kravec and Yuntao Bai and Zachary Witten and Marina Favaro and Jan Brauner and Holden Karnofsky and Paul Christiano and Samuel R. Bowman and Logan Graham and Jared Kaplan and Sören Mindermann and Ryan Greenblatt and Buck Shlegeris and Nicholas Schiefer and Ethan Perez}, year={2024}, eprint={2401.05566}, archivePrefix={arXiv}, primaryClass={cs.CR}, url={https://arxiv.org/pdf/2401.05566}, } @misc{lindsey2024crosscoders, title={Sparse Crosscoders for Cross-Layer Features and Model Diffing}, author={Lindsey, Jack and Templeton, Adly and Marcus, Jonathan and Conerly, Thomas and Batson, Joshua and Olah, Christopher}, year={2024}, url={https://transformer-circuits.pub/2024/crosscoders/index.html} } @article{brinkmann2025large, title={Large Language Models Share Representations of Latent Grammatical Concepts Across Typologically Diverse Languages}, author={Brinkmann, Jannik and Wendler, Chris and Bartelt, Christian and Mueller, Aaron}, journal={arXiv preprint arXiv:2501.06346}, year={2025}, url={https://arxiv.org/pdf/2501.06346} } @misc{stefanhex2023resstream, title={Residual stream norms grow exponentially over the forward pass}, author={Heimersheim, Stefan and Turner, Alex}, year={2023}, url={https://www.lesswrong.com/posts/8mizBCm3dyc432nK8/residual-stream-norms-grow-exponentially-over-the-forward} } @misc{bricken2024stage, title={Stage-Wise Model Diffing}, author={Trenton Bricken, Siddharth Mishra-Sharma, Jonathan Marcus, Adam Jermyn, Christopher Olah, Kelley Rivoire, Thomas Henighan}, year={2024}, url={https://transformer-circuits.pub/2024/model-diffing/index.html} } @misc{jermyn2024tanh, title={Tanh Penalty in Dictionary Learning}, author={Adam Jermyn, Adly Templeton, Joshua Batson, Trenton Bricken}, year={2024}, url={https://transformer-circuits.pub/2024/feb-update/index.html#dict-learning-tanh} } @article{bussman2024batchtopk, title={BatchTopK Sparse Autoencoders}, author={Bart Bussmann, Patrick Leask, Neel Nanda}, journal={arXiv preprint arXiv:2412.06410}, year={2024}, url={https://arxiv.org/pdf/2412.06410} } @misc{kissane2024open, title={Open Source Replication of Anthropic’s Crosscoder paper for model-diffing}, author={Connor Kissane, robertzk, Arthur Conmy, Neel Nanda}, year={2024}, url={https://www.lesswrong.com/posts/srt6JXsRMtmqAJavD/open-source-replication-of-anthropic-s-crosscoder-paper-for} } @misc{bricken2024oversampling, title={Oversampling a Topic in the SAE Training Set Results in More Detailed Features Related to that Topic}, author={Trenton Bricken, Jonathan Marcus, Kelley Rivoire, Thomas Henighan}, year={2024}, url={https://transformer-circuits.pub/2024/september-update/index.html#oversampling} } @misc{schut2025do, title={Do Multilingual LLMs Think In English?}, author={Lisa Schut and Yarin Gal and Sebastian Farquhar}, year={2025}, eprint={2502.15603}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/pdf/2502.15603}, } @inproceedings{wendler2024llamas, title={Do llamas work in english? on the latent language of multilingual transformers}, author={Wendler, Chris and Veselovsky, Veniamin and Monea, Giovanni and West, Robert}, booktitle={Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, pages={15366--15394}, year={2024}, url={https://aclanthology.org/2024.acl-long.820.pdf} } @inproceedings{dumas2024llamas, title={How do llamas process multilingual text? a latent exploration through activation patching}, author={Dumas, Clement and Veselovsky, Veniamin and Monea, Giovanni and West, Robert and Wendler, Chris}, booktitle={ICML 2024 Workshop on Mechanistic Interpretability}, year={2024}, url={https://openreview.net/pdf?id=0ku2hIm4BS} } @article{turpin2023language, title={Language models don't always say what they think: Unfaithful explanations in chain-of-thought prompting}, author={Turpin, Miles and Michael, Julian and Perez, Ethan and Bowman, Samuel}, journal={Advances in Neural Information Processing Systems}, volume={36}, pages={74952--74965}, year={2023}, url={https://proceedings.neurips.cc/paper_files/paper/2023/file/ed3fea9033a80fea1376299fa7863f4a-Paper-Conference.pdf} } @article{frankfurt2009bullshit, title={On bullshit}, author={Frankfurt, Harry G}, year={2009}, publisher={Princeton University Press} , url = {https://press.princeton.edu/books/hardcover/9780691122946/on-bullshit} } @article{sharma2023towards, title={Towards understanding sycophancy in language models}, author={Sharma, Mrinank and Tong, Meg and Korbak, Tomasz and Duvenaud, David and Askell, Amanda and Bowman, Samuel R and Cheng, Newton and Durmus, Esin and Hatfield-Dodds, Zac and Johnston, Scott R and others}, journal={arXiv preprint arXiv:2310.13548}, year={2023}, url={https://arxiv.org/pdf/2310.13548} } @article{arditi2025refusal, title={Refusal in language models is mediated by a single direction}, author={Arditi, Andy and Obeso, Oscar and Syed, Aaquib and Paleka, Daniel and Panickssery, Nina and Gurnee, Wes and Nanda, Neel}, journal={Advances in Neural Information Processing Systems}, volume={37}, pages={136037--136083}, year={2025}, url={https://proceedings.neurips.cc/paper_files/paper/2024/file/f545448535dfde4f9786555403ab7c49-Paper-Conference.pdf} } @article{marshall2024refusal, title={Refusal in LLMs is an Affine Function}, author={Marshall, Thomas and Scherlis, Adam and Belrose, Nora}, journal={arXiv preprint arXiv:2411.09003}, year={2024}, url={https://arxiv.org/pdf/2411.09003} } @article{jain2025makes, title={What makes and breaks safety fine-tuning? a mechanistic study}, author={Jain, Samyak and Lubana, Ekdeep S and Oksuz, Kemal and Joy, Tom and Torr, Philip and Sanyal, Amartya and Dokania, Puneet}, journal={Advances in Neural Information Processing Systems}, volume={37}, pages={93406--93478}, year={2025}, url={https://proceedings.neurips.cc/paper_files/paper/2024/file/a9bef53eb7b0e5950d4f2d9c74a16006-Paper-Conference.pdf} } @misc{lee2025finding, title={Finding Features Causally Upstream of Refusal}, author={Lee, Daniel and Breck, Eric and Arditi, Andy}, url={https://www.lesswrong.com/posts/Zwg4q8XTaLXRQofEt/finding-features-causally-upstream-of-refusal}, year={2025} } @misc{nikankin2024arithmetic, title={Arithmetic Without Algorithms: Language Models Solve Math With a Bag of Heuristics}, author={Yaniv Nikankin and Anja Reusch and Aaron Mueller and Yonatan Belinkov}, year={2024}, eprint={2410.21272}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/pdf/2410.21272}, } @misc{kantamneni2025trig, title={Language Models Use Trigonometry to Do Addition}, author={Subhash Kantamneni and Max Tegmark}, year={2025}, eprint={2502.00873}, archivePrefix={arXiv}, primaryClass={cs.AI}, url={https://arxiv.org/pdf/2502.00873}, } @article{greenblatt2024alignment, title={Alignment faking in large language models}, author={Greenblatt, Ryan and Denison, Carson and Wright, Benjamin and Roger, Fabien and MacDiarmid, Monte and Marks, Sam and Treutlein, Johannes and Belonax, Tim and Chen, Jack and Duvenaud, David and others}, journal={arXiv preprint arXiv:2412.14093}, year={2024}, url={https://arxiv.org/pdf/2412.14093} } @article{betley2025tell, title={Tell me about yourself: LLMs are aware of their learned behaviors}, author={Betley, Jan and Bao, Xuchan and Soto, Mart{\'\i}n and Sztyber-Betley, Anna and Chua, James and Evans, Owain}, journal={arXiv preprint arXiv:2501.11120}, year={2025}, url={https://arxiv.org/pdf/2501.11120} } @article{paulo2025transcoders, title={Transcoders Beat Sparse Autoencoders for Interpretability}, author={Paulo, Gon{\c{c}}alo and Shabalin, Stepan and Belrose, Nora}, journal={arXiv preprint arXiv:2501.18823}, year={2025}, url={https://arxiv.org/pdf/2501.18823} } @article{shi2025hypothesis, title={Hypothesis testing the circuit hypothesis in LLMs}, author={Shi, Claudia and Beltran Velez, Nicolas and Nazaret, Achille and Zheng, Carolina and Garriga-Alonso, Adri{\`a} and Jesson, Andrew and Makar, Maggie and Blei, David}, journal={Advances in Neural Information Processing Systems}, volume={37}, pages={94539--94567}, year={2025}, url={https://proceedings.neurips.cc/paper_files/paper/2024/file/abccb8a90b30d45b948360ba41f5a20f-Paper-Conference.pdf} } @article{tigges2024llm, title={LLM circuit analyses are consistent across training and scale}, author={Tigges, Curt and Hanna, Michael and Yu, Qinan and Biderman, Stella}, journal={arXiv preprint arXiv:2407.10827}, year={2024}, url={https://arxiv.org/pdf/2407.10827} } @article{wang2022interpretability, title={Interpretability in the wild: a circuit for indirect object identification in gpt-2 small}, author={Wang, Kevin and Variengien, Alexandre and Conmy, Arthur and Shlegeris, Buck and Steinhardt, Jacob}, journal={arXiv preprint arXiv:2211.00593}, year={2022}, url={https://arxiv.org/pdf/2211.00593} } @article{merullo2023circuit, title={Circuit component reuse across tasks in transformer language models}, author={Merullo, Jack and Eickhoff, Carsten and Pavlick, Ellie}, journal={arXiv preprint arXiv:2310.08744}, year={2023}, url={https://arxiv.org/pdf/2310.08744} } @article{sharkey2025open, title={Open Problems in Mechanistic Interpretability}, author={Sharkey, Lee and Chughtai, Bilal and Batson, Joshua and Lindsey, Jack and Wu, Jeff and Bushnaq, Lucius and Goldowsky-Dill, Nicholas and Heimersheim, Stefan and Ortega, Alejandro and Bloom, Joseph and others}, journal={arXiv preprint arXiv:2501.16496}, year={2025}, url={https://arxiv.org/pdf/2501.16496} } @article{bereska2024mechanistic, title={Mechanistic Interpretability for AI Safety--A Review}, author={Bereska, Leonard and Gavves, Efstratios}, journal={arXiv preprint arXiv:2404.14082}, year={2024}, url={https://arxiv.org/pdf/2404.14082} } @article{ferrando2024primer, title={A Primer on the Inner Workings of Transformer-based Language Models}, author={Ferrando, Javier and Sarti, Gabriele and Bisazza, Arianna and Costa-juss{\`a}, Marta R}, journal={arXiv preprint arXiv:2405.00208}, year={2024}, url={https://arxiv.org/pdf/2405.00208} } @article{goldowsky2023localizing, title={Localizing model behavior with path patching}, author={Goldowsky-Dill, Nicholas and MacLeod, Chris and Sato, Lucas and Arora, Aryaman}, journal={arXiv preprint arXiv:2304.05969}, year={2023}, url={https://arxiv.org/pdf/2304.05969} } @article{chan2022causal, title={Causal scrubbing, a method for rigorously testing interpretability hypotheses}, author={Chan, Lawrence and Garriga-Alonso, Adrià and Goldwosky-Dill, Nicholas and Greenblatt, Ryan and Nitishinskaya, Jenny and Radhakrishnan, Ansh and Shlegeris, Buck and Thomas, Nate}, year={2022}, journal={AI Alignment Forum}, url={https://www.alignmentforum.org/posts/JvZhhzycHu2Yd57RN/causal-scrubbing-a-method-for-rigorously-testing} } @article{miller2024transformer, title={Transformer circuit faithfulness metrics are not robust}, author={Miller, Joseph and Chughtai, Bilal and Saunders, William}, journal={arXiv preprint arXiv:2407.08734}, year={2024}, url={https://arxiv.org/pdf/2407.08734} } @article{hanna2024have, title={Have faith in faithfulness: Going beyond circuit overlap when finding model mechanisms}, author={Hanna, Michael and Pezzelle, Sandro and Belinkov, Yonatan}, journal={arXiv preprint arXiv:2403.17806}, year={2024}, url={https://arxiv.org/pdf/2403.17806} } @article{ferrando2024information, title={Information flow routes: Automatically interpreting language models at scale}, author={Ferrando, Javier and Voita, Elena}, journal={arXiv preprint arXiv:2403.00824}, year={2024}, url={https://arxiv.org/pdf/2403.00824} } @article{lieberum2023does, title={Does circuit analysis interpretability scale? evidence from multiple choice capabilities in chinchilla}, author={Lieberum, Tom and Rahtz, Matthew and Kram{\'a}r, J{\'a}nos and Nanda, Neel and Irving, Geoffrey and Shah, Rohin and Mikulik, Vladimir}, journal={arXiv preprint arXiv:2307.09458}, year={2023}, url={https://arxiv.org/pdf/2307.09458} } @article{bushnaq2024local, title={The local interaction basis: Identifying computationally-relevant and sparsely interacting features in neural networks}, author={Bushnaq, Lucius and Heimersheim, Stefan and Goldowsky-Dill, Nicholas and Braun, Dan and Mendel, Jake and H{\"a}nni, Kaarel and Griffin, Avery and St{\"o}hler, J{\"o}rn and Wache, Magdalena and Hobbhahn, Marius}, journal={arXiv preprint arXiv:2405.10928}, year={2024}, url={https://arxiv.org/pdf/2405.10928} } @article{bushnaq2024using, title={Using degeneracy in the loss landscape for mechanistic interpretability}, author={Bushnaq, Lucius and Mendel, Jake and Heimersheim, Stefan and Braun, Dan and Goldowsky-Dill, Nicholas and H{\"a}nni, Kaarel and Wu, Cindy and Hobbhahn, Marius}, journal={arXiv preprint arXiv:2405.10927}, year={2024}, url={https://arxiv.org/pdf/2405.10927} } @article{braun2025interpretability, title={Interpretability in Parameter Space: Minimizing Mechanistic Description Length with Attribution-based Parameter Decomposition}, author={Braun, Dan and Bushnaq, Lucius and Heimersheim, Stefan and Mendel, Jake and Sharkey, Lee}, journal={arXiv preprint arXiv:2501.14926}, year={2025}, url={https://arxiv.org/pdf/2501.14926} } @article{feng2023language, title={How do language models bind entities in context?}, author={Feng, Jiahai and Steinhardt, Jacob}, journal={arXiv preprint arXiv:2310.17191}, year={2023}, url={https://arxiv.org/pdf/2310.17191} } @article{jenner2025evidence, title={Evidence of learned look-ahead in a chess-playing neural network}, author={Jenner, Erik and Kapur, Shreyas and Georgiev, Vasil and Allen, Cameron and Emmons, Scott and Russell, Stuart J}, journal={Advances in Neural Information Processing Systems}, volume={37}, pages={31410--31437}, year={2025}, url={https://proceedings.neurips.cc/paper_files/paper/2024/file/37d9f19150fce07bced2a81fc87d47a6-Paper-Conference.pdf} } @article{pal2023future, title={Future lens: Anticipating subsequent tokens from a single hidden state}, author={Pal, Koyena and Sun, Jiuding and Yuan, Andrew and Wallace, Byron C and Bau, David}, journal={arXiv preprint arXiv:2311.04897}, year={2023}, url={https://arxiv.org/pdf/2311.04897} } @article{wu2024language, title={Do language models plan ahead for future tokens?}, author={Wu, Wilson and Morris, John X and Levine, Lionel}, journal={arXiv preprint arXiv:2404.00859}, year={2024}, url={https://arxiv.org/pdf/2404.00859} } @article{chanin2024absorption, title={A is for absorption: Studying feature splitting and absorption in sparse autoencoders}, author={Chanin, David and Wilken-Smith, James and Dulka, Tom{\'a}{\v{s}} and Bhatnagar, Hardik and Bloom, Joseph}, journal={arXiv preprint arXiv:2409.14507}, year={2024}, url={https://arxiv.org/pdf/2409.14507} } @misc{nabeshima2024matryoshka, title={Matryoshka Sparse Autoencoders}, author={Nabeshima, Noa}, year={2024}, url={https://www.lesswrong.com/posts/zbebxYCqsryPALh8C/matryoshka-sparse-autoencoders#fn-gTupTd3B3GQegCM2j-4} } @misc{bussmann2024matryoshka, title={Learning Multi-Level Features with Matryoshka SAEs}, author={Bussmann, Bart and Leask, Patrick and Nanda, Neel}, year={2024}, url={https://www.lesswrong.com/posts/rKM9b6B2LqwSB5ToN/learning-multi-level-features-with-matryoshka-saes} } @misc{bussmann2024meta, title={Showing SAE Latents Are Not Atomic Using Meta-SAEs}, author={Bussmann, Bart and Pearce, Michael and Leask, Patrick and Bloom, Joseph and Sharkey, Lee and Nanda, Neel}, year={2024}, url={https://www.alignmentforum.org/posts/TMAmHh4DdMr4nCSr5/showing-sae-latents-are-not-atomic-using-meta-saes} } @misc{conerly2025optimization, title={Dictionary Learning Optimization Techniques}, author={Conerly, Tom and Cunningham, Hoagy and Templeton, Adly and Lindsey, Jack and Hosmer, Basil and Jermyn, Adam}, year={2024}, url={https://transformer-circuits.pub/2025/january-update/index.html#DL} } @article{ge2024automatically, title={Automatically identifying local and global circuits with linear computation graphs}, author={Ge, Xuyang and Zhu, Fukang and Shu, Wentao and Wang, Junxuan and He, Zhengfu and Qiu, Xipeng}, journal={arXiv preprint arXiv:2405.13868}, year={2024}, url={https://arxiv.org/pdf/2405.13868} } @article{o2024sparse, title={Sparse autoencoders enable scalable and reliable circuit identification in language models}, author={O'Neill, Charles and Bui, Thang}, journal={arXiv preprint arXiv:2405.12522}, year={2024}, url={https://arxiv.org/pdf/2405.12522} } @article{zhang2025eap, title={EAP-GP: Mitigating Saturation Effect in Gradient-based Automated Circuit Identification}, author={Zhang, Lin and Dong, Wenshuo and Zhang, Zhuoran and Yang, Shu and Hu, Lijie and Liu, Ninghao and Zhou, Pan and Wang, Di}, journal={arXiv preprint arXiv:2502.06852}, year={2025}, url={https://arxiv.org/pdf/2502.06852} } @article{dumas2024separating, title={Separating Tongue from Thought: Activation Patching Reveals Language-Agnostic Concept Representations in Transformers}, author={Dumas, Cl{\'e}ment and Wendler, Chris and Veselovsky, Veniamin and Monea, Giovanni and West, Robert}, journal={arXiv preprint arXiv:2411.08745}, year={2024}, url={https://arxiv.org/pdf/2411.08745} } @article{lepori2023uncovering, title={Uncovering intermediate variables in transformers using circuit probing}, author={Lepori, Michael A and Serre, Thomas and Pavlick, Ellie}, journal={arXiv preprint arXiv:2311.04354}, year={2023}, url={https://arxiv.org/pdf/2311.04354} } @article{katz2023visit, title={VISIT: Visualizing and interpreting the semantic information flow of transformers}, author={Katz, Shahar and Belinkov, Yonatan}, journal={arXiv preprint arXiv:2305.13417}, year={2023}, url={https://arxiv.org/pdf/2305.13417} } @article{bhaskar2025finding, title={Finding transformer circuits with edge pruning}, author={Bhaskar, Adithya and Wettig, Alexander and Friedman, Dan and Chen, Danqi}, journal={Advances in Neural Information Processing Systems}, volume={37}, pages={18506--18534}, year={2025}, url={https://arxiv.org/abs/2406.16778} } @misc{ kharlapenko2025scaling, title={Scaling Sparse Feature Circuits For Studying In-Context Learning}, author={Dmitrii Kharlapenko and Stepan Shabalin and Fazl Barez and Neel Nanda and Arthur Conmy}, year={2025}, url={https://openreview.net/forum?id=Pa1vr1Prww} } @article{davies2023discovering, title={Discovering variable binding circuitry with desiderata}, author={Davies, Xander and Nadeau, Max and Prakash, Nikhil and Shaham, Tamar Rott and Bau, David}, journal={arXiv preprint arXiv:2307.03637}, year={2023}, url={https://arxiv.org/pdf/2307.03637} } @inproceedings{heimersheim2023circuit, title={A circuit for Python docstrings in a 4-layer attention-only transformer}, author={Heimersheim, Stefan and Janiak, Jett}, booktitle={Alignment Forum}, year={2023}, url={https://www.lesswrong.com/posts/u6KXXmKFbXfWzoAXn/a-circuit-for-python-docstrings-in-a-4-layer-attention-only} } @article{hanna2023does, title={How does GPT-2 compute greater-than?: Interpreting mathematical abilities in a pre-trained language model}, author={Hanna, Michael and Liu, Ollie and Variengien, Alexandre}, journal={Advances in Neural Information Processing Systems}, volume={36}, pages={76033--76060}, year={2023}, url={https://proceedings.neurips.cc/paper_files/paper/2023/file/efbba7719cc5172d175240f24be11280-Paper-Conference.pdf} } @article{ferrando2024know, title={Do I Know This Entity? Knowledge Awareness and Hallucinations in Language Models}, author={Ferrando, Javier and Obeso, Oscar and Rajamanoharan, Senthooran and Nanda, Neel}, journal={arXiv preprint arXiv:2411.14257}, year={2024}, url={https://arxiv.org/pdf/2411.14257} } @article{kantamneni2025sparse, title={Are Sparse Autoencoders Useful? A Case Study in Sparse Probing}, author={Kantamneni, Subhash and Engels, Joshua and Rajamanoharan, Senthooran and Tegmark, Max and Nanda, Neel}, journal={arXiv preprint arXiv:2502.16681}, year={2025}, url={https://arxiv.org/pdf/2502.16681} } @article{wu2025axbench, title={AXBENCH: Steering LLMs? Even Simple Baselines Outperform Sparse Autoencoders}, author={Wu, Zhengxuan and Arora, Aryaman and Geiger, Atticus and Wang, Zheng and Huang, Jing and Jurafsky, Dan and Manning, Christopher D and Potts, Christopher}, journal={arXiv preprint arXiv:2501.17148}, year={2025}, url={https://arxiv.org/pdf/2501.17148} } @misc{wright2024feature, title={Addressing Feature Suppression in SAEs}, author={Wright, Benjamin and Sharkey, Lee}, year={2024}, month={Feb}, url={https://www.lesswrong.com/posts/3JuSjTZyMzaSeTxKk/addressing-feature-suppression-in-saes} } @article{farnik2025jacobian, title={Jacobian Sparse Autoencoders: Sparsify Computations, Not Just Activations}, author={Farnik, Lucy and Lawson, Tim and Houghton, Conor and Aitchison, Laurence}, journal={arXiv preprint arXiv:2502.18147}, year={2025}, url={https://arxiv.org/pdf/2502.18147} } @article{lawson2024residual, title={Residual Stream Analysis with Multi-Layer SAEs}, author={Lawson, Tim and Farnik, Lucy and Houghton, Conor and Aitchison, Laurence}, journal={arXiv preprint arXiv:2409.04185}, year={2024}, url={https://arxiv.org/pdf/2409.04185} } @article{olmo2024features, title={Features that Make a Difference: Leveraging Gradients for Improved Dictionary Learning}, author={Olmo, Jeffrey and Wilson, Jared and Forsey, Max and Hepner, Bryce and Howe, Thomas Vin and Wingate, David}, journal={arXiv preprint arXiv:2411.10397}, year={2024}, url={https://arxiv.org/pdf/2411.10397} } @article{mudide2024efficient, title={Efficient dictionary learning with switch sparse autoencoders}, author={Mudide, Anish and Engels, Joshua and Michaud, Eric J and Tegmark, Max and de Witt, Christian Schroeder}, journal={arXiv preprint arXiv:2410.08201}, year={2024}, url={https://arxiv.org/pdf/2410.08201} } @article{leask2025sparse, title={Sparse Autoencoders Do Not Find Canonical Units of Analysis}, author={Leask, Patrick and Bussmann, Bart and Pearce, Michael and Bloom, Joseph and Tigges, Curt and Moubayed, Noura Al and Sharkey, Lee and Nanda, Neel}, journal={arXiv preprint arXiv:2502.04878}, year={2025}, url={https://arxiv.org/pdf/2502.04878} } @article{heap2025sparse, title={Sparse Autoencoders Can Interpret Randomly Initialized Transformers}, author={Heap, Thomas and Lawson, Tim and Farnik, Lucy and Aitchison, Laurence}, journal={arXiv preprint arXiv:2501.17727}, year={2025}, url={https://arxiv.org/pdf/2501.17727} } @inproceedings{kissane2024saes, title={Saes are highly dataset dependent: A case study on the refusal direction}, author={Kissane, Connor and Krzyzanowski, Robert and Nanda, Neel and Conmy, Arthur}, booktitle={Alignment Forum}, year={2024}, url={https://www.lesswrong.com/posts/rtp6n7Z23uJpEH7od/saes-are-highly-dataset-dependent-a-case-study-on-the} } @article{paulo2024automatically, title={Automatically interpreting millions of features in large language models}, author={Paulo, Gon{\c{c}}alo and Mallen, Alex and Juang, Caden and Belrose, Nora}, journal={arXiv preprint arXiv:2410.13928}, year={2024}, url={https://arxiv.org/pdf/2410.13928} } @article{paulo2025sparse, title={Sparse Autoencoders Trained on the Same Data Learn Different Features}, author={Paulo, Gon{\c{c}}alo and Belrose, Nora}, journal={arXiv preprint arXiv:2501.16615}, year={2025}, url={https://arxiv.org/pdf/2501.16615} } @article{haklay2025position, title={Position-aware Automatic Circuit Discovery}, author={Haklay, Tal and Orgad, Hadas and Bau, David and Mueller, Aaron and Belinkov, Yonatan}, journal={arXiv preprint arXiv:2502.04577}, year={2025}, url={https://arxiv.org/pdf/2502.04577} } @article{mueller2024quest, title={The quest for the right mediator: A history, survey, and theoretical grounding of causal interpretability}, author={Mueller, Aaron and Brinkmann, Jannik and Li, Millicent and Marks, Samuel and Pal, Koyena and Prakash, Nikhil and Rager, Can and Sankaranarayanan, Aruna and Sharma, Arnab Sen and Sun, Jiuding and others}, journal={arXiv preprint arXiv:2408.01416}, year={2024}, url={https://arxiv.org/pdf/2408.01416} } @article{karvonen2025measuring, title={Measuring progress in dictionary learning for language model interpretability with board game models}, author={Karvonen, Adam and Wright, Benjamin and Rager, Can and Angell, Rico and Brinkmann, Jannik and Smith, Logan and Mayrink Verdun, Claudio and Bau, David and Marks, Samuel}, journal={Advances in Neural Information Processing Systems}, volume={37}, pages={83091--83118}, year={2025}, url={https://proceedings.neurips.cc/paper_files/paper/2024/file/9736acf007760cc2b47948ae3cf06274-Paper-Conference.pdf} } @article{geiger2021causal, title={Causal abstractions of neural networks}, author={Geiger, Atticus and Lu, Hanson and Icard, Thomas and Potts, Christopher}, journal={Advances in Neural Information Processing Systems}, volume={34}, pages={9574--9586}, year={2021}, url={https://proceedings.neurips.cc/paper_files/paper/2021/file/4f5c422f4d49a5a807eda27434231040-Paper.pdf} } @article{wu2023interpretability, title={Interpretability at scale: Identifying causal mechanisms in alpaca}, author={Wu, Zhengxuan and Geiger, Atticus and Icard, Thomas and Potts, Christopher and Goodman, Noah}, journal={Advances in neural information processing systems}, volume={36}, pages={78205--78226}, year={2023}, url={https://proceedings.neurips.cc/paper_files/paper/2023/file/f6a8b109d4d4fd64c75e94aaf85d9697-Paper-Conference.pdf} } @inproceedings{geiger2024finding, title={Finding alignments between interpretable causal variables and distributed neural representations}, author={Geiger, Atticus and Wu, Zhengxuan and Potts, Christopher and Icard, Thomas and Goodman, Noah}, booktitle={Causal Learning and Reasoning}, pages={160--187}, year={2024}, organization={PMLR}, url={https://proceedings.mlr.press/v236/geiger24a/geiger24a.pdf} } @article{geiger2023causal, title={Causal abstraction: A theoretical foundation for mechanistic interpretability}, author={Geiger, Atticus and Ibeling, Duligur and Zur, Amir and Chaudhary, Maheep and Chauhan, Sonakshi and Huang, Jing and Arora, Aryaman and Wu, Zhengxuan and Goodman, Noah and Potts, Christopher and others}, journal={arXiv preprint arXiv:2301.04709}, year={2023}, url={https://arxiv.org/pdf/2301.04709} } @article{chaudhary2024evaluating, title={Evaluating open-source sparse autoencoders on disentangling factual knowledge in gpt-2 small}, author={Chaudhary, Maheep and Geiger, Atticus}, journal={arXiv preprint arXiv:2409.04478}, year={2024}, url={https://arxiv.org/pdf/2409.04478} } @article{karvonensaebench, title={SAEBench: A comprehensive benchmark for sparse autoencoders, December 2024}, author={Karvonen, A and Rager, C and Lin, J and Tigges, C and Bloom, J and Chanin, D and Lau, YT and Farrell, E and Conmy, A and Mc-Dougall, C and others}, journal={URL https://www. neuronpedia. org/sae-bench/info}, url={https://www.neuronpedia.org/sae-bench/info} } @article{karvonen2024evaluating, title={Evaluating Sparse Autoencoders on Targeted Concept Erasure Tasks}, author={Karvonen, Adam and Rager, Can and Marks, Samuel and Nanda, Neel}, journal={arXiv preprint arXiv:2411.18895}, year={2024}, url={https://arxiv.org/pdf/2411.18895} } @article{makelov2024towards, title={Towards principled evaluations of sparse autoencoders for interpretability and control}, author={Makelov, Aleksandar and Lange, George and Nanda, Neel}, journal={arXiv preprint arXiv:2405.08366}, year={2024}, url={https://arxiv.org/pdf/2405.08366} } @article{nanda2023progress, title={Progress measures for grokking via mechanistic interpretability}, author={Nanda, Neel and Chan, Lawrence and Lieberum, Tom and Smith, Jess and Steinhardt, Jacob}, journal={arXiv preprint arXiv:2301.05217}, year={2023}, url={https://arxiv.org/pdf/2301.05217} } @article{heimersheim2024use, title={How to use and interpret activation patching}, author={Heimersheim, Stefan and Nanda, Neel}, journal={arXiv preprint arXiv:2404.15255}, year={2024}, url={https://arxiv.org/pdf/2404.15255} } @article{zhang2023towards, title={Towards best practices of activation patching in language models: Metrics and methods}, author={Zhang, Fred and Nanda, Neel}, journal={arXiv preprint arXiv:2309.16042}, year={2023}, url={https://arxiv.org/pdf/2309.16042} } @article{cao2021low, title={Low-complexity probing via finding subnetworks}, author={Cao, Steven and Sanh, Victor and Rush, Alexander M}, journal={arXiv preprint arXiv:2104.03514}, year={2021}, url={https://arxiv.org/pdf/2104.03514} } @article{shah2024decomposing, title={Decomposing and editing predictions by modeling model computation}, author={Shah, Harshay and Ilyas, Andrew and Madry, Aleksander}, journal={arXiv preprint arXiv:2404.11534}, year={2024}, url={https://arxiv.org/pdf/2404.11534} } @article{mondorf2024circuit, title={Circuit Compositions: Exploring Modular Structures in Transformer-Based Language Models}, author={Mondorf, Philipp and Wold, Sondre and Plank, Barbara}, journal={arXiv preprint arXiv:2410.01434}, year={2024}, url={https://arxiv.org/pdf/2410.01434} } @article{rajaram2024automatic, title={Automatic discovery of visual circuits}, author={Rajaram, Achyuta and Chowdhury, Neil and Torralba, Antonio and Andreas, Jacob and Schwettmann, Sarah}, journal={arXiv preprint arXiv:2404.14349}, year={2024}, url={https://arxiv.org/pdf/2404.14349} } @article{li2024fourier, title={Fourier circuits in neural networks and transformers: A case study of modular arithmetic with multiple inputs}, author={Li, Chenyang and Liang, Yingyu and Shi, Zhenmei and Song, Zhao and Zhou, Tianyi}, journal={arXiv preprint arXiv:2402.09469}, year={2024}, url={https://arxiv.org/pdf/2402.09469} } @article{stander2023grokking, title={Grokking group multiplication with cosets}, author={Stander, Dashiell and Yu, Qinan and Fan, Honglu and Biderman, Stella}, journal={arXiv preprint arXiv:2312.06581}, year={2023}, url={https://arxiv.org/pdf/2312.06581} } @inproceedings{chughtai2023toy, title={A toy model of universality: Reverse engineering how networks learn group operations}, author={Chughtai, Bilal and Chan, Lawrence and Nanda, Neel}, booktitle={International Conference on Machine Learning}, pages={6243--6267}, year={2023}, organization={PMLR}, url={https://proceedings.mlr.press/v202/chughtai23a/chughtai23a.pdf} } @article{zhong2023clock, title={The clock and the pizza: Two stories in mechanistic explanation of neural networks}, author={Zhong, Ziqian and Liu, Ziming and Tegmark, Max and Andreas, Jacob}, journal={Advances in neural information processing systems}, volume={36}, pages={27223--27250}, year={2023}, url = {https://proceedings.neurips.cc/paper_files/paper/2023/file/56cbfbf49937a0873d451343ddc8c57d-Paper-Conference.pdf} } @article{vig2020investigating, title={Investigating gender bias in language models using causal mediation analysis}, author={Vig, Jesse and Gehrmann, Sebastian and Belinkov, Yonatan and Qian, Sharon and Nevo, Daniel and Singer, Yaron and Shieber, Stuart}, journal={Advances in neural information processing systems}, volume={33}, pages={12388--12401}, year={2020}, url={https://proceedings.neurips.cc/paper/2020/file/92650b2e92217715fe312e6fa7b90d82-Paper.pdf} } @article{mathwin2023identifying, title={Identifying a preliminary circuit for predicting gendered pronouns in gpt-2 small}, author={Mathwin, Chris and Corlouer, Guillaume and Kran, Esben and Barez, Fazl and Nanda, Neel}, journal={URL: https://itch.io/jam/mechint/rate/1889871}, year={2023}, url={https://www.apartresearch.com/project/identifying-a-preliminary-circuit-for-predicting-gendered-pronouns-in-gpt-2-small} } @article{chintam2023identifying, title={Identifying and adapting transformer-components responsible for gender bias in an English language model}, author={Chintam, Abhijith and Beloch, Rahel and Zuidema, Willem and Hanna, Michael and Van Der Wal, Oskar}, journal={arXiv preprint arXiv:2310.12611}, year={2023}, url={https://arxiv.org/pdf/2310.12611} } @inproceedings{wu2023causal, title={Causal proxy models for concept-based model explanations}, author={Wu, Zhengxuan and D’Oosterlinck, Karel and Geiger, Atticus and Zur, Amir and Potts, Christopher}, booktitle={International conference on machine learning}, pages={37313--37334}, year={2023}, organization={PMLR}, url={https://proceedings.mlr.press/v202/wu23b/wu23b.pdf} } @article{huang2024ravel, title={Ravel: Evaluating interpretability methods on disentangling language model representations}, author={Huang, Jing and Wu, Zhengxuan and Potts, Christopher and Geva, Mor and Geiger, Atticus}, journal={arXiv preprint arXiv:2402.17700}, year={2024}, url={https://arxiv.org/pdf/2402.17700} } @article{wollschlager2025geometry, title={The Geometry of Refusal in Large Language Models: Concept Cones and Representational Independence}, author={Wollschl{\"a}ger, Tom and Elstner, Jannes and Geisler, Simon and Cohen-Addad, Vincent and G{\"u}nnemann, Stephan and Gasteiger, Johannes}, journal={arXiv preprint arXiv:2502.17420}, year={2025}, url={https://arxiv.org/pdf/2502.17420} } @article{yu2024robust, title={Robust LLM safeguarding via refusal feature adversarial training}, author={Yu, Lei and Do, Virginie and Hambardzumyan, Karen and Cancedda, Nicola}, journal={arXiv preprint arXiv:2409.20089}, year={2024}, url={https://arxiv.org/pdf/2409.20089} } @article{dutta2024think, title={How to think step-by-step: A mechanistic understanding of chain-of-thought reasoning}, author={Dutta, Subhabrata and Singh, Joykirat and Chakrabarti, Soumen and Chakraborty, Tanmoy}, journal={arXiv preprint arXiv:2402.18312}, year={2024}, url={https://arxiv.org/pdf/2402.18312} } @article{cabannes2025iteration, title={Iteration head: A mechanistic study of chain-of-thought}, author={Cabannes, Vivien and Arnal, Charles and Bouaziz, Wassim and Yang, Xingyu and Charton, Francois and Kempe, Julia}, journal={Advances in Neural Information Processing Systems}, volume={37}, pages={109101--109122}, year={2025}, url={https://arxiv.org/abs/2406.02128} } @article{brinkmann2024mechanistic, title={A mechanistic analysis of a transformer trained on a symbolic multi-step reasoning task}, author={Brinkmann, Jannik and Sheshadri, Abhay and Levoso, Victor and Swoboda, Paul and Bartelt, Christian}, journal={arXiv preprint arXiv:2402.11917}, year={2024}, url={https://arxiv.org/pdf/2402.11917} } @article{stolfo2023mechanistic, title={A mechanistic interpretation of arithmetic reasoning in language models using causal mediation analysis}, author={Stolfo, Alessandro and Belinkov, Yonatan and Sachan, Mrinmaya}, journal={arXiv preprint arXiv:2305.15054}, year={2023}, url={https://arxiv.org/pdf/2305.15054} } @article{he2024jailbreaklens, title={Jailbreaklens: Interpreting jailbreak mechanism in the lens of representation and circuit}, author={He, Zeqing and Wang, Zhibo and Chu, Zhixuan and Xu, Huiyu and Zheng, Rui and Ren, Kui and Chen, Chun}, journal={arXiv preprint arXiv:2411.11114}, year={2024}, url={https://arxiv.org/pdf/2411.11114} } @inproceedings{hou2023towards, title={Towards a Mechanistic Interpretation of Multi-Step Reasoning Capabilities of Language Models}, author={Hou, Yifan and Li, Jiaoda and Fei, Yu and Stolfo, Alessandro and Zhou, Wangchunshu and Zeng, Guangtao and Bosselut, Antoine and Sachan, Mrinmaya}, booktitle={Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing}, pages={4902--4919}, year={2023}, url={https://aclanthology.org/2023.emnlp-main.299.pdf} } @article{yang2024large, title={Do large language models latently perform multi-hop reasoning?}, author={Yang, Sohee and Gribovskaya, Elena and Kassner, Nora and Geva, Mor and Riedel, Sebastian}, journal={arXiv preprint arXiv:2402.16837}, year={2024}, url={https://arxiv.org/pdf/2402.16837} } @article{biran2024hopping, title={Hopping too late: Exploring the limitations of large language models on multi-hop queries}, author={Biran, Eden and Gottesman, Daniela and Yang, Sohee and Geva, Mor and Globerson, Amir}, journal={arXiv preprint arXiv:2406.12775}, year={2024}, url={https://arxiv.org/pdf/2406.12775} } @article{yu2025back, title={Back Attention: Understanding and Enhancing Multi-Hop Reasoning in Large Language Models}, author={Yu, Zeping and Belinkov, Yonatan and Ananiadou, Sophia}, journal={arXiv preprint arXiv:2502.10835}, year={2025}, url={https://arxiv.org/pdf/2502.10835} } @article{xie2024sorry, title={Sorry-bench: Systematically evaluating large language model safety refusal behaviors}, author={Xie, Tinghao and Qi, Xiangyu and Zeng, Yi and Huang, Yangsibo and Sehwag, Udari Madhushani and Huang, Kaixuan and He, Luxi and Wei, Boyi and Li, Dacheng and Sheng, Ying and others}, journal={arXiv preprint arXiv:2406.14598}, year={2024}, url={https://arxiv.org/pdf/2406.14598} } @misc{meng2024monitor, title={Monitor: An AI-Driven Observability Interface}, author={Meng, Kevin and Huang, Vincent and Chowdhury, Neil and Choi, Dami and Steinhardt, Jacob and Schwettmann, Sarah}, year={2024}, month={Oct}, url={https://transluce.org/observability-interface} } @misc{gould2023successor, title={Successor Heads: Recurring, Interpretable Attention Heads In The Wild}, author={Rhys Gould and Euan Ong and George Ogden and Arthur Conmy}, year={2023}, eprint={2312.09230}, archivePrefix={arXiv}, primaryClass={cs.LG}, url={https://arxiv.org/pdf/2312.09230} } @misc{kissane2024attncirc, title={Attention Output SAEs Improve Circuit Analysis}, author={Connor Kissane and Robert Krzyzanowski and Arthur Conmy and Neel Nanda}, year={2024}, howpublished={Alignment Forum}, url={https://www.lesswrong.com/posts/EGvtgB7ctifzxZg6v/attention-output-saes-improve-circuit-analysis} } @article{ng2011sparse, title={Sparse autoencoder}, author={Ng, Andrew and others}, journal={CS294A Lecture notes}, volume={72}, number={2011}, pages={1--19}, year={2011}, url={https://graphics.stanford.edu/courses/cs233-21-spring/ReferencedPapers/SAE.pdf} } @misc{zheng2023lmsyschat1m, title={LMSYS-Chat-1M: A Large-Scale Real-World LLM Conversation Dataset}, author={Lianmin Zheng and Wei-Lin Chiang and Ying Sheng and Tianle Li and Siyuan Zhuang and Zhanghao Wu and Yonghao Zhuang and Zhuohan Li and Zi Lin and Eric. P Xing and Joseph E. Gonzalez and Ion Stoica and Hao Zhang}, year={2023}, eprint={2309.11998}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/pdf/2309.11998} } @inproceedings{ribeiro2016should, title={" Why should i trust you?" Explaining the predictions of any classifier}, author={Ribeiro, Marco Tulio and Singh, Sameer and Guestrin, Carlos}, booktitle={Proceedings of the 22nd ACM SIGKDD international conference on knowledge discovery and data mining}, pages={1135--1144}, year={2016}, url={https://dl.acm.org/doi/pdf/10.1145/2939672.2939778?} } @article{liu2022towards, title={Towards understanding grokking: An effective theory of representation learning}, author={Liu, Ziming and Kitouni, Ouail and Nolte, Niklas S and Michaud, Eric and Tegmark, Max and Williams, Mike}, journal={Advances in Neural Information Processing Systems}, volume={35}, pages={34651--34663}, year={2022} , url = {https://papers.neurips.cc/paper_files/paper/2022/file/dfc310e81992d2e4cedc09ac47eff13e-Paper-Conference.pdf} } @article{zhou2024pre, title={Pre-trained large language models use fourier features to compute addition}, author={Zhou, Tianyi and Fu, Deqing and Sharan, Vatsal and Jia, Robin}, journal={arXiv preprint arXiv:2406.03445}, year={2024}, url={https://arxiv.org/pdf/2406.03445} } @article{marks2025auditing, title={Auditing Language Models for Hidden Objectives}, author={Marks, Samuel and Treutlein, Johannes and Bricken, Trenton and Lindsey, Jack and Marcus, Jonathan and Mishra-Sharma, Siddharth and Ziegler, Daniel and Ameisen, Emmanuel and Batson, Joshua and Carter, Shan and Chen, Brian and Cunningham, Hoagy and Dietz, Florien and Golechha, Satvik and Kirchner, Jan and Meek, Austin and Nishimura-Gasparian, Kei and Ong, Euan and Olah, Christopher and Pearce, Adam and Roger, Fabien and Salle, Jeanne and Tong, Meg and Thomas, Drake and Riviore, Kelley and Jermyn, Adam and MacDiarmid, Monte and Henighan, Tom and Hubinger, Evan}, year={2025}, url={https://arxiv.org/pdf/2503.10965}, } @article{geva2023dissecting, title={Dissecting recall of factual associations in auto-regressive language models}, author={Geva, Mor and Bastings, Jasmijn and Filippova, Katja and Globerson, Amir}, journal={arXiv preprint arXiv:2304.14767}, year={2023}, url={https://arxiv.org/pdf/2304.14767} } @article{lanham2023measuring, title={Measuring faithfulness in chain-of-thought reasoning}, author={Lanham, Tamera and Chen, Anna and Radhakrishnan, Ansh and Steiner, Benoit and Denison, Carson and Hernandez, Danny and Li, Dustin and Durmus, Esin and Hubinger, Evan and Kernion, Jackson and others}, journal={arXiv preprint arXiv:2307.13702}, year={2023}, url={https://arxiv.org/pdf/2307.13702} } @article{radhakrishnan2023question, title={Question decomposition improves the faithfulness of model-generated reasoning}, author={Radhakrishnan, Ansh and Nguyen, Karina and Chen, Anna and Chen, Carol and Denison, Carson and Hernandez, Danny and Durmus, Esin and Hubinger, Evan and Kernion, Jackson and others}, journal={arXiv preprint arXiv:2307.11768}, year={2023}, url={https://arxiv.org/pdf/2307.11768} } @article{geng2023survey, title={A survey of confidence estimation and calibration in large language models}, author={Geng, Jiahui and Cai, Fengyu and Wang, Yuxia and Koeppl, Heinz and Nakov, Preslav and Gurevych, Iryna}, journal={arXiv preprint arXiv:2311.08298}, year={2023}, url={https://arxiv.org/pdf/2311.08298} } @article{gawlikowski2023survey, title={A survey of uncertainty in deep neural networks}, author={Gawlikowski, Jakob and Tassi, Cedrique Rovile Njieutcheu and Ali, Mohsin and Lee, Jongseok and Humt, Matthias and Feng, Jianxiang and Kruspe, Anna and Triebel, Rudolph and Jung, Peter and Roscher, Ribana and others}, journal={Artificial Intelligence Review}, volume={56}, number={Suppl 1}, pages={1513--1589}, year={2023}, publisher={Springer}, url={https://link.springer.com/article/10.1007/s10462-023-10562-9} } @article{gurnee2024universal, title={Universal neurons in gpt2 language models}, author={Gurnee, Wes and Horsley, Theo and Guo, Zifan Carl and Kheirkhah, Tara Rezaei and Sun, Qinyi and Hathaway, Will and Nanda, Neel and Bertsimas, Dimitris}, journal={arXiv preprint arXiv:2401.12181}, year={2024}, url={https://arxiv.org/pdf/2401.12181} } @article{ahdritz2024distinguishing, title={Distinguishing the knowable from the unknowable with language models}, author={Ahdritz, Gustaf and Qin, Tian and Vyas, Nikhil and Barak, Boaz and Edelman, Benjamin L}, journal={arXiv preprint arXiv:2402.03563}, year={2024}, url={https://arxiv.org/pdf/2402.03563} } @article{luo2024jailbreak, title={Jailbreak Instruction-Tuned LLMs via end-of-sentence MLP Re-weighting}, author={Luo, Yifan and Zhou, Zhennan and Wang, Meitan and Dong, Bin}, journal={arXiv preprint arXiv:2410.10150}, year={2024}, url={https://arxiv.org/pdf/2410.10150} } @article{wei2023jailbroken, title={Jailbroken: How does llm safety training fail?}, author={Wei, Alexander and Haghtalab, Nika and Steinhardt, Jacob}, journal={Advances in Neural Information Processing Systems}, volume={36}, pages={80079--80110}, year={2023}, url={https://proceedings.neurips.cc/paper_files/paper/2023/file/fd6613131889a4b656206c50a8bd7790-Paper-Conference.pdf} } @article{anil2025many, title={Many-shot jailbreaking}, author={Anil, Cem and Durmus, Esin and Panickssery, Nina and Sharma, Mrinank and Benton, Joe and Kundu, Sandipan and Batson, Joshua and Tong, Meg and Mu, Jesse and Ford, Daniel and others}, journal={Advances in Neural Information Processing Systems}, volume={37}, pages={129696--129742}, year={2025} , url = {https://proceedings.neurips.cc/paper_files/paper/2024/file/ea456e232efb72d261715e33ce25f208-Paper-Conference.pdf} } @article{andriushchenko2024jailbreaking, title={Jailbreaking leading safety-aligned llms with simple adaptive attacks}, author={Andriushchenko, Maksym and Croce, Francesco and Flammarion, Nicolas}, journal={arXiv preprint arXiv:2404.02151}, year={2024}, url={https://arxiv.org/pdf/2404.02151} } @article{stolfo2025confidence, title={Confidence regulation neurons in language models}, author={Stolfo, Alessandro and Wu, Ben and Gurnee, Wes and Belinkov, Yonatan and Song, Xingyi and Sachan, Mrinmaya and Nanda, Neel}, journal={Advances in Neural Information Processing Systems}, volume={37}, pages={125019--125049}, year={2025}, url={https://proceedings.neurips.cc/paper_files/paper/2024/file/e21955c93dede886af1d0d362c756757-Paper-Conference.pdf} } @article{wu2024semantic, title={The Semantic Hub Hypothesis: Language Models Share Semantic Representations Across Languages and Modalities}, author={Wu, Zhaofeng and Yu, Xinyan Velocity and Yogatama, Dani and Lu, Jiasen and Kim, Yoon}, journal={arXiv preprint arXiv:2411.04986}, year={2024}, url={https://arxiv.org/pdf/2411.04986} } @article{pires2019multilingual, title={How multilingual is multilingual BERT?}, author={Pires, Telmo and Schlinger, Eva and Garrette, Dan}, journal={arXiv preprint arXiv:1906.01502}, year={2019}, url={https://arxiv.org/pdf/1906.01502} } @article{zhao2025large, title={How do large language models handle multilingualism?}, author={Zhao, Yiran and Zhang, Wenxuan and Chen, Guizhen and Kawaguchi, Kenji and Bing, Lidong}, journal={Advances in Neural Information Processing Systems}, volume={37}, pages={15296--15319}, year={2025}, url={https://proceedings.neurips.cc/paper_files/paper/2024/file/1bd359b32ab8b2a6bbafa1ed2856cf40-Paper-Conference.pdf} } @article{ferrando2024similarity, title={On the similarity of circuits across languages: a case study on the subject-verb agreement task}, author={Ferrando, Javier and Costa-juss{\`a}, Marta R}, journal={arXiv preprint arXiv:2410.06496}, year={2024}, url={https://arxiv.org/pdf/2410.06496} } @article{taufeeque2024planning, title={Planning in a recurrent neural network that plays Sokoban}, author={Taufeeque, Mohammad and Quirke, Philip and Li, Maximilian and Cundy, Chris and Tucker, Aaron David and Gleave, Adam and Garriga-Alonso, Adri{\`a}}, journal={arXiv preprint arXiv:2407.15421}, year={2024}, url={https://arxiv.org/pdf/2407.15421} } @inproceedings{bush2025interpreting, title={Interpreting Emergent Planning in Model-Free Reinforcement Learning}, author={Thomas Bush and Stephen Chung and Usman Anwar and Adri{\`a} Garriga-Alonso and David Krueger}, booktitle={The Thirteenth International Conference on Learning Representations}, url={https://openreview.net/pdf/e8ceadfe0b16829299aebe5ed2c5bcd1e660ba74.pdf} } @inproceedings{guez2019investigation, title={An investigation of model-free planning}, author={Guez, Arthur and Mirza, Mehdi and Gregor, Karol and Kabra, Rishabh and Racani{\`e}re, S{\'e}bastien and Weber, Th{\'e}ophane and Raposo, David and Santoro, Adam and Orseau, Laurent and Eccles, Tom and others}, booktitle={International conference on machine learning}, pages={2464--2473}, year={2019}, organization={PMLR} , url = {https://proceedings.mlr.press/v97/guez19a.html} } @article{lundberg2017unified, title={A unified approach to interpreting model predictions}, author={Lundberg, Scott M and Lee, Su-In}, journal={Advances in neural information processing systems}, volume={30}, year={2017}, url={https://proceedings.neurips.cc/paper/2017/file/8a20a8621978632d76c43dfd28b67767-Paper.pdf} } @misc{olah2023manifold, title={Feature Manifold Toy Model}, author={Olah, Christopher and Batson, Josh}, year={2023}, url={https://transformer-circuits.pub/2023/may-update/index.html#feature-manifolds} } @misc{olah2024multidimensional, title={What is a Linear Representation? What is a Multidimensional Feature?}, author={Olah, Christopher}, year={2024}, url={https://transformer-circuits.pub/2024/july-update/index.html#linear-representations} } @misc{gorton2024manifold, title={Curve Detector Manifolds in InceptionV1}, author={Gorton, Olivia}, year={2024}, url={https://livgorton.com/curve-detector-manifolds/} } @article{engels2024not, title={Not all language model features are linear}, author={Engels, Joshua and Michaud, Eric J and Liao, Isaac and Gurnee, Wes and Tegmark, Max}, journal={arXiv preprint arXiv:2405.14860}, year={2024}, url={https://arxiv.org/abs/2405.14860} } @misc{pochinkov2025planpara, title={ParaScopes: Do Language Models Plan the Upcoming Paragraph?}, author={Pochinkov, Nicky}, year={2025}, url={https://www.lesswrong.com/posts/9NqgYesCutErskdmu/parascope-do-language-models-plan-the-upcoming-paragraph} } @article{pochinkov2024extracting, title={Extracting Paragraphs from LLM Token Activations}, author={Pochinkov, Nicholas and Benoit, Angelo and Agarwal, Lovkush and Majid, Zainab Ali and Ter-Minassian, Lucile}, journal={arXiv preprint arXiv:2409.06328}, year={2024}, url={https://arxiv.org/pdf/2409.06328} } @article{mu2023learning, title={Learning to compress prompts with gist tokens}, author={Mu, Jesse and Li, Xiang and Goodman, Noah}, journal={Advances in Neural Information Processing Systems}, volume={36}, pages={19327--19352}, year={2023}, url={https://proceedings.neurips.cc/paper_files/paper/2023/file/3d77c6dcc7f143aa2154e7f4d5e22d68-Paper-Conference.pdf} } @article{arcuschin2025chain, title={Chain-of-Thought Reasoning In The Wild Is Not Always Faithful}, author={Arcuschin, Iv{\'a}n and Janiak, Jett and Krzyzanowski, Robert and Rajamanoharan, Senthooran and Nanda, Neel and Conmy, Arthur}, journal={arXiv preprint arXiv:2503.08679}, year={2025}, url={https://arxiv.org/pdf/2503.08679} } @article{goh2024large, title={Large language model influence on diagnostic reasoning: a randomized clinical trial}, author={Goh, Ethan and Gallo, Robert and Hom, Jason and Strong, Eric and Weng, Yingjie and Kerman, Hannah and Cool, Jos{\'e}phine A and Kanjee, Zahir and Parsons, Andrew S and Ahuja, Neera and others}, journal={JAMA Network Open}, volume={7}, number={10}, pages={e2440969--e2440969}, year={2024}, publisher={American Medical Association}, url={https://jamanetwork.com/journals/jamanetworkopen/articlepdf/2825395/goh_2024_oi_241182_1729527081.343.pdf} } @article{mcduff2023towards, title={Towards accurate differential diagnosis with large language models}, author={McDuff, Daniel and Schaekermann, Mike and Tu, Tao and Palepu, Anil and Wang, Amy and Garrison, Jake and Singhal, Karan and Sharma, Yash and Azizi, Shekoofeh and Kulkarni, Kavita and others}, journal={arXiv preprint arXiv:2312.00164}, year={2023}, url={https://jakegarrison.me/doc/Towards%20Accurate%20Differential%20Diagnosis%20with%20Large%20Language%20Models.pdf} } @article{reese2024limitations, title={On the limitations of large language models in clinical diagnosis}, author={Reese, Justin T and Danis, Daniel and Caufield, J Harry and Groza, Tudor and Casiraghi, Elena and Valentini, Giorgio and Mungall, Christopher J and Robinson, Peter N}, journal={medRxiv}, pages={2023--07}, year={2024}, url={https://pmc.ncbi.nlm.nih.gov/articles/PMC10370243/pdf/nihpp-2023.07.13.23292613v1.pdf} } @article{savage2024diagnostic, title={Diagnostic reasoning prompts reveal the potential for large language model interpretability in medicine}, author={Savage, Thomas and Nayak, Ashwin and Gallo, Robert and Rangan, Ekanath and Chen, Jonathan H}, journal={NPJ Digital Medicine}, volume={7}, number={1}, pages={20}, year={2024}, publisher={Nature Publishing Group UK London}, url={https://www.nature.com/articles/s41746-024-01010-1.pdf} } @article{strong2023chatbot, title={Chatbot vs medical student performance on free-response clinical reasoning examinations}, author={Strong, Eric and DiGiammarino, Alicia and Weng, Yingjie and Kumar, Andre and Hosamani, Poonam and Hom, Jason and Chen, Jonathan H}, journal={JAMA internal medicine}, volume={183}, number={9}, pages={1028--1030}, year={2023}, publisher={American Medical Association}, url={https://jamanetwork.com/journals/jamainternalmedicine/articlepdf/2806980/jamainternal_strong_2023_ld_230023_1693517126.00366.pdf} } @article{kanjee2023accuracy, title={Accuracy of a generative artificial intelligence model in a complex diagnostic challenge}, author={Kanjee, Zahir and Crowe, Byron and Rodman, Adam}, journal={Jama}, volume={330}, number={1}, pages={78--80}, year={2023}, publisher={American Medical Association}, url={https://jamanetwork.com/journals/jama/articlepdf/2806457/jama_kanjee_2023_ld_230037_1687532972.65484.pdf} } @article{amann2020explainability, title={Explainability for artificial intelligence in healthcare: a multidisciplinary perspective}, author={Amann, Julia and Blasimme, Alessandro and Vayena, Effy and Frey, Dietmar and Madai, Vince I and Precise4Q Consortium}, journal={BMC medical informatics and decision making}, volume={20}, pages={1--9}, year={2020}, publisher={Springer}, url={https://link.springer.com/content/pdf/10.1186/s12911-020-01332-6.pdf} } @article{band2023application, title={Application of explainable artificial intelligence in medical health: A systematic review of interpretability methods}, author={Band, Shahab S and Yarahmadi, Atefeh and Hsu, Chung-Chian and Biyari, Meghdad and Sookhak, Mehdi and Ameri, Rasoul and Dehzangi, Iman and Chronopoulos, Anthony Theodore and Liang, Huey-Wen}, journal={Informatics in Medicine Unlocked}, volume={40}, pages={101286}, year={2023}, publisher={Elsevier}, url={https://www.sciencedirect.com/science/article/pii/S2352914823001302} } @article{lieberum2024gemmascope, title={Gemma Scope: Open Sparse Autoencoders Everywhere All At Once on Gemma 2}, author={Tom Lieberum and Senthooran Rajamanoharan and Arthur Conmy and Lewis Smith and Nicolas Sonnerat and Vikrant Varma and János Kramár and Anca Dragan and Rohin Shah and Neel Nanda}, year={2024}, eprint={2408.05147}, archivePrefix={arXiv}, primaryClass={cs.LG}, url={https://arxiv.org/abs/2408.05147} } @article{team2024gemma, title={Gemma 2: Improving open language models at a practical size}, author={Team, Gemma and Riviere, Morgane and Pathak, Shreya and Sessa, Pier Giuseppe and Hardin, Cassidy and Bhupatiraju, Surya and Hussenot, L{\'e}onard and Mesnard, Thomas and Shahriari, Bobak and Ram{\'e}, Alexandre and others}, journal={arXiv preprint arXiv:2408.00118}, year={2024}, url={https://arxiv.org/abs/2408.00118} } @article{zhang2024same, title={The Same But Different: Structural Similarities and Differences in Multilingual Language Modeling}, author={Zhang, Ruochen and Yu, Qinan and Zang, Matianyu and Eickhoff, Carsten and Pavlick, Ellie}, journal={arXiv preprint arXiv:2410.09223}, year={2024} } @article{fierro2024multilingual, title={How Do Multilingual Models Remember? Investigating Multilingual Factual Recall Mechanisms}, author={Fierro, Constanza and Foroutan, Negar and Elliott, Desmond and S{\o}gaard, Anders}, journal={arXiv preprint arXiv:2410.14387}, year={2024} } @article{lindsey2025biology, author={Lindsey, Jack and Gurnee, Wes and Ameisen, Emmanuel and Chen, Brian and Pearce, Adam and Turner, Nicholas L. and Citro, Craig and Abrahams, David and Carter, Shan and Hosmer, Basil and Marcus, Jonathan and Sklar, Michael and Templeton, Adly and Bricken, Trenton and McDougall, Callum and Cunningham, Hoagy and Henighan, Thomas and Jermyn, Adam and Jones, Andy and Persic, Andrew and Qi, Zhenyi and Thompson, T. Ben and Zimmerman, Sam and Rivoire, Kelley and Conerly, Thomas and Olah, Chris and Batson, Joshua}, title={On the Biology of a Large Language Model}, journal={Transformer Circuits Thread}, year={2025}, url={https://transformer-circuits.pub/2025/attribution-graphs/biology.html} } @article{ameisen2025circuit, author={Ameisen, Emmanuel and Lindsey, Jack and Pearce, Adam and Gurnee, Wes and Turner, Nicholas L. and Chen, Brian and Citro, Craig and Abrahams, David and Carter, Shan and Hosmer, Basil and Marcus, Jonathan and Sklar, Michael and Templeton, Adly and Bricken, Trenton and McDougall, Callum and Cunningham, Hoagy and Henighan, Thomas and Jermyn, Adam and Jones, Andy and Persic, Andrew and Qi, Zhenyi and Ben Thompson, T. and Zimmerman, Sam and Rivoire, Kelley and Conerly, Thomas and Olah, Chris and Batson, Joshua}, title={Circuit Tracing: Revealing Computational Graphs in Language Models}, journal={Transformer Circuits}, year={2025}, url={https://transformer-circuits.pub/2025/attribution-graphs/methods.html} } @article{hohman2019summit, author={Hohman, Fred and Park, Haekyu and Robinson, Caleb and Chau, Duen Horng Polo}, title={SUMMIT: Scaling Deep Learning Interpretability by Visualizing Activation and Attribution Summarizations}, year={2019} } @article{mcgrath2023hydra, title={The hydra effect: Emergent self-repair in language model computations}, author={McGrath, Thomas and Rahtz, Matthew and Kramar, Janos and Mikulik, Vladimir and Legg, Shane}, journal={arXiv preprint arXiv:2307.15771}, year={2023}, url={https://arxiv.org/pdf/2307.15771} } @misc{wynroe2024bilinear, title={ Decomposing the QK circuit with Bilinear Sparse Dictionary Learning }, author={Wynroe, Keith and Sharkey, Lee}, year={2024}, url={https://www.alignmentforum.org/posts/2ep6FGjTQoGDRnhrq/decomposing-the-qk-circuit-with-bilinear-sparse-dictionary} } @article{prakash2025language, title={Language models use lookbacks to track beliefs}, author={Prakash, Nikhil and Shapira, Natalie and Sharma, Arnab Sen and Riedl, Christoph and Belinkov, Yonatan and Shaham, Tamar Rott and Bau, David and Geiger, Atticus}, journal={arXiv preprint arXiv:2505.14685}, year={2025}, url={https://arxiv.org/pdf/2505.14685} } @article{bertolazzi2025validationgap, title={The Validation Gap: A Mechanistic Analysis of How Language Models Compute Arithmetic but Fail to Validate It}, author={Leonardo Bertolazzi and Philipp Mondorf and Barbara Plank and Raffaella Bernardi}, journal={arXiv preprint arXiv:2502.11771}, year={2025}, url={https://arxiv.org/pdf/2502.11771} } @article{hafting2005microstructureOA, title={Microstructure of a spatial map in the entorhinal cortex}, author={Torkel Hafting and Marianne Fyhn and Sturla Molden and May-Britt Moser and Edvard I. Moser}, journal={Nature}, year={2005}, volume={436}, pages={801-806}, url={https://api.semanticscholar.org/CorpusID:4405184} } @article{fiete2008grid, title={What grid cells convey about rat location}, author={Fiete, Ila R and Burak, Yoram and Brookings, Ted}, journal={Journal of Neuroscience}, volume={28}, number={27}, pages={6858--6871}, year={2008}, publisher={Society for Neuroscience} } @article{li2025language, title={(How) Do Language Models Track State?}, author={Li, Belinda Z and Guo, Zifan Carl and Andreas, Jacob}, journal={arXiv preprint arXiv:2503.02854}, year={2025} } @article{lad2024remarkable, title={The remarkable robustness of llms: Stages of inference?}, author={Lad, Vedang and Lee, Jin Hwa and Gurnee, Wes and Tegmark, Max}, journal={arXiv preprint arXiv:2406.19384}, year={2024} } @article{kamath2025tracing, author={Kamath, Harish and Ameisen, Emmanuel and Kauvar, Isaac and Luger, Rodrigo and Gurnee, Wes and Pearce, Adam and Zimmerman, Sam and Batson, Joshua and Conerly, Thomas and Olah, Chris and Lindsey, Jack}, title={Tracing Attention Computation Through Feature Interactions}, journal={Transformer Circuits Thread}, year={2025}, url={https://transformer-circuits.pub/2025/attention-qk/index.html} } @article{chen2025persona, title={Persona vectors: Monitoring and controlling character traits in language models}, author={Chen, Runjin and Arditi, Andy and Sleight, Henry and Evans, Owain and Lindsey, Jack}, journal={arXiv preprint arXiv:2507.21509}, year={2025} } @misc{bricken2025automating, author = {Bricken, Trenton and Wang, Rowan and Bowman, Sam and Ong, Euan and Treutlein, Johannes and Wu, Jeff and Hubinger, Evan and Marks, Samuel}, title = {Building and evaluating alignment auditing agents}, year = {2025}, month = {July}, day = {24}, howpublished = {\url{https://alignment.anthropic.com/2025/automated-auditing/}} } @inproceedings{shaham2024multimodal, title={A multimodal automated interpretability agent}, author={Shaham, Tamar Rott and Schwettmann, Sarah and Wang, Franklin and Rajaram, Achyuta and Hernandez, Evan and Andreas, Jacob and Torralba, Antonio}, booktitle={Forty-first International Conference on Machine Learning}, year={2024} } @article{park2024geometry, title={The geometry of categorical and hierarchical concepts in large language models}, author={Park, Kiho and Choe, Yo Joong and Jiang, Yibo and Veitch, Victor}, journal={arXiv preprint arXiv:2406.01506}, year={2024} } @article{park2023linear, title={The linear representation hypothesis and the geometry of large language models}, author={Park, Kiho and Choe, Yo Joong and Veitch, Victor}, journal={arXiv preprint arXiv:2311.03658}, year={2023} } @inproceedings{ engels2025not, title={Not All Language Model Features Are One-Dimensionally Linear}, author={Joshua Engels and Eric J Michaud and Isaac Liao and Wes Gurnee and Max Tegmark}, booktitle={The Thirteenth International Conference on Learning Representations}, year={2025}, url={https://openreview.net/forum?id=d63a4AM4hb} } @article{morwani2023feature, title={Feature emergence via margin maximization: case studies in algebraic tasks}, author={Morwani, Depen and Edelman, Benjamin L and Oncescu, Costin-Andrei and Zhao, Rosie and Kakade, Sham}, journal={arXiv preprint arXiv:2311.07568}, year={2023} } @inproceedings{kornblith2019similarity, title={Similarity of neural network representations revisited}, author={Kornblith, Simon and Norouzi, Mohammad and Lee, Honglak and Hinton, Geoffrey}, booktitle={International conference on machine learning}, pages={3519--3529}, year={2019}, organization={PMlR} } @inproceedings{michaud2023the, title={The Quantization Model of Neural Scaling}, author={Eric J Michaud and Ziming Liu and Uzay Girit and Max Tegmark}, booktitle={Thirty-seventh Conference on Neural Information Processing Systems}, year={2023}, url={https://openreview.net/forum?id=3tbTw2ga8K} } @article{alquboj2025number, title={Number Representations in LLMs: A Computational Parallel to Human Perception}, author={AlquBoj, HV and AlQuabeh, Hilal and Bojkovic, Velibor and Hiraoka, Tatsuya and El-Shangiti, Ahmed Oumar and Nwadike, Munachiso and Inui, Kentaro}, journal={arXiv preprint arXiv:2502.16147}, year={2025} } @article{heinzerling2024monotonic, title={Monotonic representation of numeric properties in language models}, author={Heinzerling, Benjamin and Inui, Kentaro}, journal={arXiv preprint arXiv:2403.10381}, year={2024} } @inproceedings{shi2016neural, title={Why neural translations are the right length}, author={Shi, Xing and Knight, Kevin and Yuret, Deniz}, booktitle={Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing}, pages={2278--2282}, year={2016} } @article{moon2025length, title={Length Representations in Large Language Models}, author={Moon, Sangjun and Choi, Dasom and Kwon, Jingun and Kamigaito, Hidetaka and Okumura, Manabu}, journal={arXiv preprint arXiv:2507.20398}, year={2025} } @article{chang2024language, title={Language models need inductive biases to count inductively}, author={Chang, Yingshan and Bisk, Yonatan}, journal={arXiv preprint arXiv:2405.20131}, year={2024} } @article{suzgun2019lstm, title={LSTM networks can perform dynamic counting}, author={Suzgun, Mirac and Gehrmann, Sebastian and Belinkov, Yonatan and Shieber, Stuart M}, journal={arXiv preprint arXiv:1906.03648}, year={2019} } @incollection{rosenthal1998two, title={Two concepts of consciousness}, author={Rosenthal, David M}, booktitle={Consciousness and Emotion in Cognitive Science}, pages={1--31}, year={1998}, publisher={Routledge} } @inproceedings{rosenthal1993thinking, title={Thinking that one thinks.}, author={Rosenthal, David M}, booktitle={Previous drafts were read at Tufts University, Columbia University, University of Wisconsin at Milwaukee, and the Joint Conference of the Sociedad Filos{\'o}fica Ibero-Americana and the Sociedad Argentina de An{\'a}lisis Filos{\'o}fico, in Buenos Aires, Aug 1989.}, year={1993}, organization={Blackwell Publishing} } @article{howe2005muller, title={The M{\"u}ller-Lyer illusion explained by the statistics of image--source relationships}, author={Howe, Catherine Q and Purves, Dale}, journal={Proceedings of the National Academy of Sciences}, volume={102}, number={4}, pages={1234--1239}, year={2005}, publisher={National Academy of Sciences} } @article{schwartz2007space, title={Space and time in visual context}, author={Schwartz, Odelia and Hsu, Anne and Dayan, Peter}, journal={Nature Reviews Neuroscience}, volume={8}, number={7}, pages={522--535}, year={2007}, publisher={Nature Publishing Group UK London} } @article{gibson1937adaptation, title={Adaptation, after-effect and contrast in the perception of tilted lines. I. Quantitative studies.}, author={Gibson, James J and Radner, Minnie}, journal={Journal of experimental psychology}, volume={20}, number={5}, pages={453}, year={1937}, publisher={Psychological Review Company} } @misc{chughtai2024understanding, title = {Understanding positional features in layer 0 SAEs}, author = {Chughtai, Bilal and Lau, Yeu-Tong}, year = {2024}, month = {07}, day = {29}, url = {https://bilalchughtai.co.uk/pos-sae/}, note = {Blog post} } @misc{yedidia2023gpt2, title = {GPT-2's positional embedding matrix is a helix}, author = {Yedidia, Adam}, year = {2023}, month = {07}, day = {20}, url={https://www.lesswrong.com/posts/qvWP3aBDBaqXvPNhS/gpt-2-s-positional-embedding-matrix-is-a-helix}, note = {LessWrong post} } @article{voita2023neurons, title={Neurons in large language models: Dead, n-gram, positional}, author={Voita, Elena and Ferrando, Javier and Nalmpantis, Christoforos}, journal={arXiv preprint arXiv:2309.04827}, year={2023} } @article{modell2025origins, title={The Origins of Representation Manifolds in Large Language Models}, author={Modell, Alexander and Rubin-Delanchy, Patrick and Whiteley, Nick}, journal={arXiv preprint arXiv:2505.18235}, year={2025} } @article{wattenberg2024relational, title={Relational composition in neural networks: A survey and call to action}, author={Wattenberg, Martin and Vi{\'e}gas, Fernanda B}, journal={arXiv preprint arXiv:2407.14662}, year={2024} } @article{gur2025enhancing, title={Enhancing automated interpretability with output-centric feature descriptions}, author={Gur-Arieh, Yoav and Mayan, Roy and Agassy, Chen and Geiger, Atticus and Geva, Mor}, journal={arXiv preprint arXiv:2501.08319}, year={2025} } @article{li2025geometry, title={The geometry of concepts: Sparse autoencoder feature structure}, author={Li, Yuxiao and Michaud, Eric J and Baek, David D and Engels, Joshua and Sun, Xiaoqing and Tegmark, Max}, journal={Entropy}, volume={27}, number={4}, pages={344}, year={2025}, publisher={MDPI} } @article{chang2022geometry, title={The geometry of multilingual language model representations}, author={Chang, Tyler A and Tu, Zhuowen and Bergen, Benjamin K}, journal={arXiv preprint arXiv:2205.10964}, year={2022} } @article{prakash2025language, title={Language models use lookbacks to track beliefs}, author={Prakash, Nikhil and Shapira, Natalie and Sharma, Arnab Sen and Riedl, Christoph and Belinkov, Yonatan and Shaham, Tamar Rott and Bau, David and Geiger, Atticus}, journal={arXiv preprint arXiv:2505.14685}, year={2025} } @article{hindupur2025projecting, title={Projecting assumptions: The duality between sparse autoencoders and concept geometry}, author={Hindupur, Sai Sumedh R and Lubana, Ekdeep Singh and Fel, Thomas and Ba, Demba}, journal={arXiv preprint arXiv:2503.01822}, year={2025} } @article{Moser2008PlaceCG, title={Place cells, grid cells, and the brain's spatial representation system.}, author={Edvard I. Moser and Emilio Kropff and May-Britt Moser}, journal={Annual review of neuroscience}, year={2008}, volume={31}, pages={69-89}, url={https://api.semanticscholar.org/CorpusID:16036900} } @article{smith2001estimating, title={Estimating receptive field size from fMRI data in human striate and extrastriate visual cortex}, author={Smith, Andrew T and Singh, Krish D and Williams, Adrian L and Greenlee, Mark W}, journal={Cerebral cortex}, volume={11}, number={12}, pages={1182--1190}, year={2001}, publisher={Oxford University Press} } @article{dumoulin2008population, title={Population receptive field estimates in human visual cortex}, author={Dumoulin, Serge O and Wandell, Brian A}, journal={Neuroimage}, volume={39}, number={2}, pages={647--660}, year={2008}, publisher={Elsevier} } @article{dehaene2003neural, title={The neural basis of the Weber--Fechner law: a logarithmic mental number line}, author={Dehaene, Stanislas}, journal={Trends in cognitive sciences}, volume={7}, number={4}, pages={145--147}, year={2003}, publisher={Elsevier} } @article{solstad2008representation, title={Representation of geometric borders in the entorhinal cortex}, author={Solstad, Trygve and Boccara, Charlotte N and Kropff, Emilio and Moser, May-Britt and Moser, Edvard I}, journal={Science}, volume={322}, number={5909}, pages={1865--1868}, year={2008}, publisher={American Association for the Advancement of Science} } @article{semedo2019cortical, title={Cortical areas interact through a communication subspace}, author={Semedo, Jo{\~a}o D and Zandvakili, Amin and Machens, Christian K and Yu, Byron M and Kohn, Adam}, journal={Neuron}, volume={102}, number={1}, pages={249--259}, year={2019}, publisher={Elsevier} } @article{macdowell2025multiplexed, title={Multiplexed subspaces route neural activity across brain-wide networks}, author={MacDowell, Camden J and Libby, Alexandra and Jahn, Caroline I and Tafazoli, Sina and Ardalan, Adel and Buschman, Timothy J}, journal={Nature communications}, volume={16}, number={1}, pages={3359}, year={2025}, publisher={Nature Publishing Group UK London} } @inproceedings{leshinskaya2025cognitively, title={Cognitively Inspired Interpretability in Large Neural Networks}, author={Leshinskaya, Anna and Webb, Taylor and Pavlick, Ellie and Feng, Jiahai and Opielka, Gustaw and Stevenson, Claire and Blank, Idan A}, booktitle={Proceedings of the Annual Meeting of the Cognitive Science Society}, volume={47}, year={2025} } @article{he2024multilevel, title={Multilevel interpretability of artificial neural networks: leveraging framework and methods from neuroscience}, author={He, Zhonghao and Achterberg, Jascha and Collins, Katie and Nejad, Kevin and Akarca, Danyal and Yang, Yinzhu and Gurnee, Wes and Sucholutsky, Ilia and Tang, Yuhan and Ianov, Rebeca and others}, journal={arXiv preprint arXiv:2408.12664}, year={2024} } @article{vilas2024position, title={Position: An inner interpretability framework for AI inspired by lessons from cognitive neuroscience}, author={Vilas, Martina G and Adolfi, Federico and Poeppel, David and Roig, Gemma}, journal={arXiv preprint arXiv:2406.01352}, year={2024} } @article{lepori2024beyond, title={Beyond the doors of perception: Vision transformers represent relations between objects}, author={Lepori, Michael and Tartaglini, Alexa and Vong, Wai Keen and Serre, Thomas and Lake, Brenden M and Pavlick, Ellie}, journal={Advances in Neural Information Processing Systems}, volume={37}, pages={131503--131544}, year={2024} } @article{piazza2004tuning, title={Tuning curves for approximate numerosity in the human intraparietal sulcus}, author={Piazza, Manuela and Izard, V{\'e}ronique and Pinel, Philippe and Le Bihan, Denis and Dehaene, Stanislas}, journal={Neuron}, volume={44}, number={3}, pages={547--555}, year={2004}, publisher={Elsevier} } @article{lau2011empirical, title={Empirical support for higher-order theories of conscious awareness}, author={Lau, Hakwan and Rosenthal, David}, journal={Trends in cognitive sciences}, volume={15}, number={8}, pages={365--373}, year={2011}, publisher={Elsevier} } @book{rosenthal2005consciousness, title={Consciousness and mind}, author={Rosenthal, David}, year={2005}, publisher={Clarendon Press} } @article{carruthers2017higher, title={Higher-order theories of consciousness}, author={Carruthers, Peter}, journal={The Blackwell companion to consciousness}, pages={288--297}, year={2017}, publisher={Wiley Online Library} } @book{baars1993cognitive, title={A cognitive theory of consciousness}, author={Baars, Bernard J}, year={1993}, publisher={Cambridge University Press} } @article{baars1997theatre, title={In the theatre of consciousness. Global workspace theory, a rigorous scientific theory of consciousness}, author={Baars, Bernard J}, journal={Journal of consciousness Studies}, volume={4}, number={4}, pages={292--309}, year={1997}, publisher={Imprint Academic} } @article{chalmers1995facing, title={Facing up to the problem of consciousness}, author={Chalmers, David J}, journal={Journal of consciousness studies}, volume={2}, number={3}, pages={200--219}, year={1995}, publisher={Imprint Academic} } @book{searle1992rediscovery, title={The rediscovery of the mind}, author={Searle, John R}, year={1992}, publisher={MIT press} } @inproceedings{crick1990towards, title={Towards a neurobiological theory of consciousness}, author={Crick, Francis and Koch, Christof and others}, booktitle={Seminars in the Neurosciences}, volume={2}, number={263-275}, pages={203}, year={1990}, organization={London} } @article{seth2024conscious, title={Conscious artificial intelligence and biological naturalism}, author={Seth, Anil K}, journal={Behavioral and Brain Sciences}, pages={1--42}, year={2024}, publisher={Cambridge University Press} } @article{tononi2004information, title={An information integration theory of consciousness}, author={Tononi, Giulio}, journal={BMC neuroscience}, volume={5}, number={1}, pages={42}, year={2004}, publisher={Springer} } @article{albantakis2023integrated, title={Integrated information theory (IIT) 4.0: Formulating the properties of phenomenal existence in physical terms}, author={Albantakis, Larissa and Barbosa, Leonardo and Findlay, Graham and Grasso, Matteo and Haun, Andrew M and Marshall, William and Mayner, William GP and Zaeemzadeh, Alireza and Boly, Melanie and Juel, Bj{\o}rn E and others}, journal={PLoS computational biology}, volume={19}, number={10}, pages={e1011465}, year={2023}, publisher={Public Library of Science San Francisco, CA USA} } @article{findlay2024dissociating, title={Dissociating artificial intelligence from artificial consciousness}, author={Findlay, Graham and Marshall, William and Albantakis, Larissa and David, Isaac and Mayner, William GP and Koch, Christof and Tononi, Giulio}, journal={arXiv preprint arXiv:2412.04571}, year={2024} } @article{long2024taking, title={Taking AI welfare seriously}, author={Long, Robert and Sebo, Jeff and Butlin, Patrick and Finlinson, Kathleen and Fish, Kyle and Harding, Jacqueline and Pfau, Jacob and Sims, Toni and Birch, Jonathan and Chalmers, David}, journal={arXiv preprint arXiv:2411.00986}, year={2024} } @article{merriamwebster2025introspection, title={Introspection}, author={{Merriam-Webster}}, howpublished={Merriam-Webster.com Dictionary}, year={2025}, url={ https://www.merriam-webster.com/dictionary/introspection }, note={Accessed August 31, 2025} } @article{long2024taking, title={Taking AI welfare seriously}, author={Long, Robert and Sebo, Jeff and Butlin, Patrick and Finlinson, Kathleen and Fish, Kyle and Harding, Jacqueline and Pfau, Jacob and Sims, Toni and Birch, Jonathan and Chalmers, David}, journal={arXiv preprint arXiv:2411.00986}, year={2024} } @article{song2025privileged, title={Privileged Self-Access Matters for Introspection in AI}, author={Song, Siyuan and Lederman, Harvey and Hu, Jennifer and Mahowald, Kyle}, journal={arXiv preprint arXiv:2508.14802}, year={2025} } @article{binder2024looking, title={Looking inward: Language models can learn about themselves by introspection}, author={Binder, Felix J and Chua, James and Korbak, Tomek and Sleight, Henry and Hughes, John and Long, Robert and Perez, Ethan and Turpin, Miles and Evans, Owain}, journal={arXiv preprint arXiv:2410.13787}, year={2024} } @article{song2025language, title={Language models fail to introspect about their knowledge of language}, author={Song, Siyuan and Hu, Jennifer and Mahowald, Kyle}, journal={arXiv preprint arXiv:2503.07513}, year={2025} } @article{panickssery2024llm, title={LLM evaluators recognize and favor their own generations}, author={Panickssery, Arjun and Bowman, Samuel and Feng, Shi}, journal={Advances in Neural Information Processing Systems}, volume={37}, pages={68772--68802}, year={2024} } @article{wang2025simple, title={Simple Mechanistic Explanations for Out-Of-Context Reasoning}, author={Wang, Atticus and Engels, Joshua and Clive-Griffin, Oliver}, journal={arXiv preprint arXiv:2507.08218}, year={2025} } @article{davidson2024self, title={Self-recognition in language models}, author={Davidson, Tim R and Surkov, Viacheslav and Veselovsky, Veniamin and Russo, Giuseppe and West, Robert and Gulcehre, Caglar}, journal={arXiv preprint arXiv:2407.06946}, year={2024} } @article{laine2024me, title={Me, myself, and ai: The situational awareness dataset (sad) for llms}, author={Laine, Rudolf and Chughtai, Bilal and Betley, Jan and Hariharan, Kaivalya and Balesni, Mikita and Scheurer, J{\'e}r{\'e}my and Hobbhahn, Marius and Meinke, Alexander and Evans, Owain}, journal={Advances in Neural Information Processing Systems}, volume={37}, pages={64010--64118}, year={2024} } @article{comsa2025does, title={Does It Make Sense to Speak of Introspection in Large Language Models?}, author={Comsa, Iulia M and Shanahan, Murray}, journal={arXiv preprint arXiv:2506.05068}, year={2025} } @article{kapoor2024large, title={Large language models must be taught to know what they don’t know}, author={Kapoor, Sanyam and Gruver, Nate and Roberts, Manley and Collins, Katie and Pal, Arka and Bhatt, Umang and Weller, Adrian and Dooley, Samuel and Goldblum, Micah and Wilson, Andrew G}, journal={Advances in Neural Information Processing Systems}, volume={37}, pages={85932--85972}, year={2024} } @article{lin2022teaching, title={Teaching models to express their uncertainty in words}, author={Lin, Stephanie and Hilton, Jacob and Evans, Owain}, journal={arXiv preprint arXiv:2205.14334}, year={2022} } @article{cheng2024can, title={Can AI assistants know what they don't know?}, author={Cheng, Qinyuan and Sun, Tianxiang and Liu, Xiangyang and Zhang, Wenwei and Yin, Zhangyue and Li, Shimin and Li, Linyang and He, Zhengfu and Chen, Kai and Qiu, Xipeng}, journal={arXiv preprint arXiv:2401.13275}, year={2024} } @article{ji2025language, title={Language Models Are Capable of Metacognitive Monitoring and Control of Their Internal Activations}, author={Ji-An, Li and Xiong, Hua-Dong and Wilson, Robert C and Mattar, Marcelo G and Benna, Marcus K}, journal={arXiv preprint arXiv:2505.13763}, year={2025} } @article{chen2024selfie, title={Selfie: Self-interpretation of large language model embeddings}, author={Chen, Haozhe and Vondrick, Carl and Mao, Chengzhi}, journal={arXiv preprint arXiv:2403.10949}, year={2024} } @article{ghandeharioun2024patchscopes, title={Patchscopes: A unifying framework for inspecting hidden representations of language models}, author={Ghandeharioun, Asma and Caciularu, Avi and Pearce, Adam and Dixon, Lucas and Geva, Mor}, journal={arXiv preprint arXiv:2401.06102}, year={2024} } @article{perich2025neural, title={A neural manifold view of the brain}, author={Perich, Matthew G and Narain, Devika and Gallego, Juan A}, journal={Nature Neuroscience}, pages={1--16}, year={2025}, publisher={Nature Publishing Group US New York} } @article{ziv2013long, title={Long-term dynamics of CA1 hippocampal place codes}, author={Ziv, Yaniv and Burns, Laurie D and Cocker, Eric D and Hamel, Elizabeth O and Ghosh, Kunal K and Kitch, Lacey J and Gamal, Abbas El and Schnitzer, Mark J}, journal={Nature neuroscience}, volume={16}, number={3}, pages={264--266}, year={2013}, publisher={Nature Publishing Group US New York} } @article{michaud2025understanding, title={Understanding sparse autoencoder scaling in the presence of feature manifolds}, author={Eric J. Michaud and Liv Gorton and Tom McGrath}, year={2025}, eprint={2509.02565}, archivePrefix={arXiv}, primaryClass={cs.LG}, url={https://arxiv.org/abs/2509.02565}, } @article{bau2018gan, title={Gan dissection: Visualizing and understanding generative adversarial networks}, author={Bau, David and Zhu, Jun-Yan and Strobelt, Hendrik and Zhou, Bolei and Tenenbaum, Joshua B and Freeman, William T and Torralba, Antonio}, journal={arXiv preprint arXiv:1811.10597}, year={2018} } @misc{gorton2024curve, title={Curve Detector Manifolds in InceptionV1}, author={Gorton, Liv}, year={2024}, month={8}, day={1}, url={https://livgorton.com/curve-detector-manifolds/}, } @article{yildiz2022review, title={A review on various explanations of Ponzo-like illusions}, author={Yildiz, Gizem Y and Sperandio, Irene and Kettle, Christine and Chouinard, Philippe A}, journal={Psychonomic Bulletin \& Review}, volume={29}, number={2}, pages={293--320}, year={2022}, publisher={Springer} } @misc{olah2025toy, title={A Toy Model of Interference Weights}, author={Olah, Chris and Turner, Nicholas L. and Conerly, Tom}, year={2025}, month={July}, day={29}, url={https://transformer-circuits.pub/2025/interference-weights/index.html}, } @article{yedidia2023positional, author = {Yedidia, Adam}, title = {The positional embedding matrix and previous-token heads: how do they actually work?}, journal = {Alignment Forum}, year = {2023}, month = {August}, day = {9}, url={https://www.alignmentforum.org/posts/zRA8B2FJLtTYRgie6/the-positional-embedding-matrix-and-previous-token-heads-how}, urldate = {2025-10-06} } @article{hu2025understanding, title={Understanding In-context Learning of Addition via Activation Subspaces}, author={Hu, Xinyan and Yin, Kayo and Jordan, Michael I and Steinhardt, Jacob and Chen, Lijie}, journal={arXiv preprint arXiv:2505.05145}, year={2025} } @article{huang2025decomposing, title={Decomposing Representation Space into Interpretable Subspaces with Unsupervised Learning}, author={Huang, Xinting and Hahn, Michael}, journal={arXiv preprint arXiv:2508.01916}, year={2025} } @article{plunkett2025self, title={Self-Interpretability: LLMs Can Describe Complex Internal Processes that Drive Their Decisions, and Improve with Training}, author={Plunkett, Dillon and Morris, Adam and Reddy, Keerthi and Morales, Jorge}, journal={arXiv preprint arXiv:2505.17120}, year={2025} } @article{long2025claude, author = {Long, Robert}, title = {Why model self-reports are insufficient---and why we studied them anyway: Notes on Claude 4 model welfare interviews}, month = may, year = {2025}, url = {https://eleosai.org/post/claude-4-interview-notes/}, } @article{kammerer2023forms, title={What forms could introspective systems take? A research programme}, author={Kammerer, Fran{\c{c}}ois and Frankish, Keith}, journal={Journal of Consciousness Studies}, volume={30}, number={9-10}, pages={13--48}, year={2023}, publisher={Imprint Academic} } @article{long2023introspective, title={Introspective capabilities in large language models}, author={Long, Robert}, journal={Journal of Consciousness Studies}, volume={30}, number={9-10}, pages={143--153}, year={2023}, publisher={Imprint Academic} } @article{butlin2023consciousness, title={Consciousness in Artificial Intelligence: Insights from the Science of Consciousness}, author={Butlin, Patrick and Long, Robert and Elmoznino, Eric and Bengio, Yoshua and Birch, Jonathan and Constant, Axel and Deane, George and Fleming, Stephen M. and Frith, Chris and Ji, Xu and Kanai, Ryota and Klein, Colin and Lindsay, Grace and Michel, Matthias and Mudrik, Liad and Peters, Megan A. K. and Schwitzgebel, Eric and Simon, Jonathan and VanRullen, Rufin}, journal={arXiv preprint arXiv:2308.08708}, year={2023}, url={https://arxiv.org/abs/2308.08708} } @article{chalmers2023could, title={Could a large language model be conscious?}, author={Chalmers, David J}, journal={arXiv preprint arXiv:2303.07103}, year={2023} } @article{costa2025flat, title={From Flat to Hierarchical: Extracting Sparse Representations with Matching Pursuit}, author={Costa, Val{\'e}rie and Fel, Thomas and Lubana, Ekdeep Singh and Tolooshams, Bahareh and Ba, Demba}, journal={arXiv preprint arXiv:2506.03093}, year={2025} } @misc{olah2023interpretability, author = {Olah, Chris}, title = {Interpretability Dreams}, howpublished = {Transformer Circuits Thread}, year = {2023}, month = {May}, day = {24}, url = {https://transformer-circuits.pub/2023/interpretability-dreams/index.html}, } @article{russell1977evidence, title={Evidence for a three-factor theory of emotions}, author={Russell, James A and Mehrabian, Albert}, journal={Journal of research in Personality}, volume={11}, number={3}, pages={273--294}, year={1977}, publisher={Elsevier} } @article{lu2026assistant, title={The Assistant Axis: Situating and Stabilizing the Default Persona of Language Models}, author={Lu, Christina and Gallagher, Jack and Michala, Jonathan and Fish, Kyle and Lindsey, Jack}, journal={arXiv preprint arXiv:2601.10387}, year={2026} } @article{lynch2025agentic, title={Agentic misalignment: How llms could be insider threats}, author={Lynch, Aengus and Wright, Benjamin and Larson, Caleb and Ritchie, Stuart J and Mindermann, Soren and Hubinger, Evan and Perez, Ethan and Troy, Kevin}, journal={arXiv preprint arXiv:2510.05179}, year={2025} } @article{zhong2025impossiblebench, title={ImpossibleBench: Measuring LLMs' Propensity of Exploiting Test Cases}, author={Zhong, Ziqian and Raghunathan, Aditi and Carlini, Nicholas}, journal={arXiv preprint arXiv:2510.20270}, year={2025} } @article{li2023large, title={Large language models understand and can be enhanced by emotional stimuli}, author={Li, Cheng and Wang, Jindong and Zhang, Yixuan and Zhu, Kaijie and Hou, Wenxin and Lian, Jianxun and Luo, Fang and Yang, Qiang and Xie, Xing}, journal={arXiv preprint arXiv:2307.11760}, year={2023} } @article{ishikawa2025ai, title={AI with Emotions: Exploring Emotional Expressions in Large Language Models}, author={Ishikawa, Shin-nosuke and Yoshino, Atsushi}, journal={arXiv preprint arXiv:2504.14706}, year={2025} } @article{tak2025mechanistic, title={Mechanistic Interpretability of Emotion Inference in Large Language Models}, author={Tak, Ala N and Banayeeanzade, Amin and Bolourani, Anahita and Kian, Mina and Jia, Robin and Gratch, Jonathan}, journal={arXiv preprint arXiv:2502.05489}, year={2025} } @article{zhang2025decoding, title={Decoding Emotion in the Deep: A Systematic Study of How LLMs Represent, Retain, and Express Emotion}, author={Zhang, Jingxiang and Zhong, Lujia}, journal={arXiv preprint arXiv:2510.04064}, year={2025} } @article{reichman2025emotions, title={Emotions Where Art Thou: Understanding and Characterizing the Emotional Latent Space of Large Language Models}, author={Reichman, Benjamin and Avsian, Adar and Heck, Larry}, journal={arXiv preprint arXiv:2510.22042}, year={2025} } @article{adolphs2017should, title={How should neuroscience study emotions? By distinguishing emotion states, concepts, and experiences}, author={Adolphs, Ralph}, journal={Social cognitive and affective neuroscience}, volume={12}, number={1}, pages={24--31}, year={2017}, publisher={Oxford University Press} } @incollection{darwin2025expression, title={The expression of the emotions in man and animals}, author={Darwin, Charles}, booktitle={Death, Loss, Memory and Mourning in the Long Nineteenth Century, 1780--1914}, pages={163--177}, year={2025}, publisher={Routledge} } @article{bradley2000measuring, title={Measuring emotion: Behavior, feeling, and physiology.}, author={Bradley, Margaret M and Lang, Peter J}, year={2000}, publisher={Oxford University Press} } @article{james1884emotion, author = {James, William}, title = {What is an Emotion?}, journal = {Mind}, } @book{lange1885om, title={Om sindsbevaegelser; et psyko-fysiologisk studie}, author={Lange, Carl Georg}, year={1885}, publisher={Lund} } @article{hsueh2023cardiogenic, title={Cardiogenic control of affective behavioural state}, author={Hsueh, Brian and Chen, Ritchie and Jo, YoungJu and Tang, Daniel and Raffiee, Misha and Kim, Yoon Seok and Inoue, Masatoshi and Randles, Sawyer and Ramakrishnan, Charu and Patel, Sneha and others}, journal={Nature}, volume={615}, number={7951}, pages={292--299}, year={2023}, publisher={Nature Publishing Group UK London} } @article{yang2023hypothalamic, title={Hypothalamic neurons that mirror aggression}, author={Yang, Taehong and Bayless, Daniel W and Wei, Yichao and Landayan, Dan and Marcelo, Ivo M and Wang, Yangpeng and DeNardo, Laura A and Luo, Liqun and Druckmann, Shaul and Shah, Nirao M}, journal={Cell}, volume={186}, number={6}, pages={1195--1211}, year={2023}, publisher={Elsevier} } @article{anderson2014framework, title={A framework for studying emotions across species}, author={Anderson, David J and Adolphs, Ralph}, journal={Cell}, volume={157}, number={1}, pages={187--200}, year={2014}, publisher={Elsevier} } @article{kauvar2025conserved, title={Conserved brain-wide emergence of emotional response from sensory experience in humans and mice}, author={Kauvar, Isaac and Richman, Ethan B and Liu, Tony X and Li, Chelsea and Vesuna, Sam and Chibukhchyan, Adelaida and Yamada, Lisa and Fogarty, Adam and Solomon, Ethan and Choi, Eun Young and others}, journal={Science}, volume={20}, number={XX}, pages={eadt3971}, year={2025}, publisher={American Association for the Advancement of Science} } @article{barrett2017theory, title={The theory of constructed emotion: an active inference account of interoception and categorization}, author={Barrett, Lisa Feldman}, journal={Social cognitive and affective neuroscience}, volume={12}, number={1}, pages={1--23}, year={2017}, publisher={Oxford University Press} } @article{hoemann2019emotion, title={Emotion words, emotion concepts, and emotional development in children: A constructionist hypothesis.}, author={Hoemann, Katie and Xu, Fei and Barrett, Lisa Feldman}, journal={Developmental psychology}, volume={55}, number={9}, pages={1830}, year={2019}, publisher={American Psychological Association} } @article{betley2025emergent, title={Emergent Misalignment: Narrow finetuning can produce broadly misaligned LLMs}, author={Betley, Jan and Tan, Daniel and Warncke, Niels and Sztyber-Betley, Anna and Bao, Xuchan and Soto, Mart{\'\i}n and Labenz, Nathan and Evans, Owain}, journal={arXiv preprint arXiv:2502.17424}, year={2025} } @article{macdiarmid2025natural, title={Natural emergent misalignment from reward hacking in production rl}, author={MacDiarmid, Monte and Wright, Benjamin and Uesato, Jonathan and Benton, Joe and Kutasov, Jon and Price, Sara and Bouscal, Naia and Bowman, Sam and Bricken, Trenton and Cloud, Alex and others}, journal={arXiv preprint arXiv:2511.18397}, year={2025} } @article{panickssery2024steering, title={Steering llama 2 via contrastive activation addition, 2024}, author={Panickssery, Nina and Gabrieli, Nick and Schulz, Julian and Tong, Meg and Hubinger, Evan and Turner, Alexander Matt}, journal={URL https://arxiv. org/abs/2312.06681}, volume={3} } @article{shanahan2023role, title={Role play with large language models}, author={Shanahan, Murray and McDonell, Kyle and Reynolds, Laria}, journal={Nature}, volume={623}, number={7987}, pages={493--498}, year={2023}, publisher={Nature Publishing Group UK London} } @article{chen2024persona, title={From persona to personalization: A survey on role-playing language agents}, author={Chen, Jiangjie and Wang, Xintao and Xu, Rui and Yuan, Siyu and Zhang, Yikai and Shi, Wei and Xie, Jian and Li, Shuang and Yang, Ruihan and Zhu, Tinghui and others}, journal={arXiv preprint arXiv:2404.18231}, year={2024} } @article{strachan2024testing, title={Testing theory of mind in large language models and humans}, author={Strachan, James WA and Albergo, Dalila and Borghini, Giulia and Pansardi, Oriana and Scaliti, Eugenio and Gupta, Saurabh and Saxena, Krati and Rufo, Alessandro and Panzeri, Stefano and Manzi, Guido and others}, journal={Nature Human Behaviour}, volume={8}, number={7}, pages={1285--1295}, year={2024}, publisher={Nature Publishing Group UK London} } @article{street2025llms, title={LLMs achieve adult human performance on higher-order theory of mind tasks}, author={Street, Winnie and Siy, John Oliver and Keeling, Geoff and Baranes, Adrien and Barnett, Benjamin and McKibben, Michael and Kanyere, Tatenda and Lentz, Alison and Arcas, Blaise Ag{\"u}era y and Dunbar, Robin IM}, journal={Frontiers in Human Neuroscience}, volume={19}, pages={1633272}, year={2025}, publisher={Frontiers Media SA} } @article{zhu2024language, title={Language models represent beliefs of self and others}, author={Zhu, Wentao and Zhang, Zhining and Wang, Yizhou}, journal={arXiv preprint arXiv:2402.18496}, year={2024} } @article{chen2024designing, title={Designing a dashboard for transparency and control of conversational AI}, author={Chen, Yida and Wu, Aoyu and DePodesta, Trevor and Yeh, Catherine and Li, Kenneth and Marin, Nicholas Castillo and Patel, Oam and Riecke, Jan and Raval, Shivam and Seow, Olivia and others}, journal={arXiv preprint arXiv:2406.07882}, year={2024} } @article{sharma2023towards, title={Towards understanding sycophancy in language models}, author={Sharma, Mrinank and Tong, Meg and Korbak, Tomasz and Duvenaud, David and Askell, Amanda and Bowman, Samuel R and Cheng, Newton and Durmus, Esin and Hatfield-Dodds, Zac and Johnston, Scott R and others}, journal={arXiv preprint arXiv:2310.13548}, year={2023} } @misc{openai2025sycophancy, title={Sycophancy in GPT-4o: What happened and what we’re doing about it}, author={OpenAI}, year={2025}, publisher={OpenAI: Product} } @article{amodei2016concrete, title={Concrete problems in AI safety}, author={Amodei, Dario and Olah, Chris and Steinhardt, Jacob and Christiano, Paul and Schulman, John and Man{\'e}, Dan}, journal={arXiv preprint arXiv:1606.06565}, year={2016} } @misc{vonarx2025rewardhacking, author = {Von Arx, Sydney and Chan, Lawrence and Barnes, Elizabeth}, title = {Recent Frontier Models Are Reward Hacking}, howpublished = {\url{https://metr.org/blog/2025-06-05-recent-reward-hacking/}}, year = {2025} } @misc{baker2025detecting, title={Detecting misbehavior in frontier reasoning models}, author={Baker, B and Huizinga, J and Madry, A and Zaremba, W and Pachocki, J and Farhi, D}, year={2025}, publisher={OpenAI} } @article{mcinnes2018umap, title={Umap: Uniform manifold approximation and projection for dimension reduction}, author={McInnes, Leland and Healy, John and Melville, James}, journal={arXiv preprint arXiv:1802.03426}, year={2018} } @article{kriegeskorte2008representational, title={Representational similarity analysis-connecting the branches of systems neuroscience}, author={Kriegeskorte, Nikolaus and Mur, Marieke and Bandettini, Peter A}, journal={Frontiers in systems neuroscience}, volume={2}, pages={249}, year={2008}, publisher={Frontiers} } @misc{marks2026persona, title={The Persona Selection Model: Why AI Assistants might Behave like Humans}, author={Sam Marks and Jack Lindsey and Christopher Olah}, year={2026}, howpublished={Anthropic Alignment Science Blog}, url={ https://alignment.anthropic.com/2026/psm/ } } @misc{chowdhury2025truthfulness, author = {Chowdhury, Neil and Johnson, Daniel and Huang, Vincent and Steinhardt, Jacob and Schwettmann, Sarah}, title = {Investigating truthfulness in a pre-release o3 model}, year = {2025}, month = {April}, day = {16}, howpublished = {\url{https://transluce.org/investigating-o3-truthfulness}} } @article{soligo2026gemma, title={Gemma Needs Help: Investigating and Mitigating Emotional Instability in LLMs}, author={Soligo, Anna and Mikulik, Vladimir and Saunders, William}, journal={arXiv preprint arXiv:2603.10011}, year={2026} } @article{zhao2025emergence, title={Emergence of Hierarchical Emotion Organization in Large Language Models}, author={Zhao, Bo and Okawa, Maya and Bigelow, Eric J and Yu, Rose and Ullman, Tomer and Lubana, Ekdeep Singh and Tanaka, Hidenori}, journal={arXiv preprint arXiv:2507.10599}, year={2025} } @article{wang2025llms, title={Do LLMs" Feel"? Emotion Circuits Discovery and Control}, author={Wang, Chenxi and Zhang, Yixuan and Yu, Ruiji and Zheng, Yufei and Gao, Lang and Song, Zirui and Xu, Zixiang and Xia, Gus and Zhang, Huishuai and Zhao, Dongyan and others}, journal={arXiv preprint arXiv:2510.11328}, year={2025} } @inproceedings{subramani2022extracting, title={Extracting latent steering vectors from pretrained language models}, author={Subramani, Nishant and Suresh, Nivedita and Peters, Matthew E}, booktitle={Findings of the Association for Computational Linguistics: ACL 2022}, pages={566--581}, year={2022} } @article{wu2025ai, title={AI shares emotion with humans across languages and cultures}, author={Wu, Xiuwen and Wang, Hao and Yan, Zhiang and Tang, Xiaohan and Xu, Pengfei and Siok, Wai-Ting and Li, Ping and Gao, Jia-Hong and Lyu, Bingjiang and Qin, Lang}, journal={arXiv preprint arXiv:2506.13978}, year={2025} } @incollection{posner1975attention, title={Attention and cognitive control}, author={Posner, Michael I and Snyder, Charles R R}, booktitle={Information Processing and Cognition: The Loyola Symposium}, editor={Solso, Robert L}, pages={55--85}, year={1975}, publisher={Lawrence Erlbaum}, address={Hillsdale, NJ} } @article{schneider1977controlled, title={Controlled and automatic human information processing: I. Detection, search, and attention}, author={Schneider, Walter and Shiffrin, Richard M}, journal={Psychological Review}, volume={84}, number={1}, pages={1--66}, year={1977}, publisher={American Psychological Association} } @incollection{norman1986attention, title={Attention to action: Willed and automatic control of behavior}, author={Norman, Donald A and Shallice, Tim}, booktitle={Consciousness and Self-Regulation: Advances in Research and Theory}, editor={Davidson, Richard J and Schwartz, Gary E and Shapiro, David}, volume={4}, pages={1--18}, year={1986}, publisher={Plenum Press}, address={New York}, doi={10.1007/978-1-4757-0629-1_1} } @article{dehaene2001towards, title={Towards a cognitive neuroscience of consciousness: basic evidence and a workspace framework}, author={Dehaene, Stanislas and Naccache, Lionel}, journal={Cognition}, volume={79}, number={1--2}, pages={1--37}, year={2001}, publisher={Elsevier} } @book{baars1988cognitive, title={A Cognitive Theory of Consciousness}, author={Baars, Bernard J}, year={1988}, publisher={Cambridge University Press}, address={Cambridge} } @article{mashour2020conscious, title={Conscious processing and the global neuronal workspace hypothesis}, author={Mashour, George A and Roelfsema, Pieter and Changeux, Jean-Pierre and Dehaene, Stanislas}, journal={Neuron}, volume={105}, number={5}, pages={776--798}, year={2020}, publisher={Elsevier}, doi={10.1016/j.neuron.2020.01.026} } @article{block1995confusion, title={On a confusion about a function of consciousness}, author={Block, Ned}, journal={Behavioral and Brain Sciences}, volume={18}, number={2}, pages={227--247}, year={1995}, publisher={Cambridge University Press}, doi={10.1017/S0140525X00038188} } @article{nagel1974what, title={What Is It Like to Be a Bat?}, author={Nagel, Thomas}, journal={The Philosophical Review}, volume={83}, number={4}, pages={435--450}, year={1974}, doi={10.2307/2183914} } @article{seth2022theories, title={Theories of consciousness}, author={Seth, Anil K. and Bayne, Tim}, journal={Nature Reviews Neuroscience}, volume={23}, number={7}, pages={439--452}, year={2022}, doi={10.1038/s41583-022-00587-4} } @article{dehaene2017consciousness, title={What is consciousness, and could machines have it?}, author={Dehaene, Stanislas and Lau, Hakwan and Kouider, Sid}, journal={Science}, volume={358}, number={6362}, pages={486--492}, year={2017}, publisher={American Association for the Advancement of Science}, doi={10.1126/science.aan8871} } @article{dehaene2006conscious, title={Conscious, preconscious, and subliminal processing: a testable taxonomy}, author={Dehaene, Stanislas and Changeux, Jean-Pierre and Naccache, Lionel and Sackur, J{\'e}r{\^o}me and Sergent, Claire}, journal={Trends in Cognitive Sciences}, volume={10}, number={5}, pages={204--211}, year={2006}, publisher={Elsevier}, doi={10.1016/j.tics.2006.03.007} } @article{dehaene1998neuronal, title={A neuronal model of a global workspace in effortful cognitive tasks}, author={Dehaene, Stanislas and Kerszberg, Michel and Changeux, Jean-Pierre}, journal={Proceedings of the National Academy of Sciences}, volume={95}, number={24}, pages={14529--14534}, year={1998} } @article{gurnee2025when, author={Gurnee, Wes and Ameisen, Emmanuel and Kauvar, Isaac and Tarng ,Julius and Pearce, Adam and Olah, Chris and Batson, Joshua}, title={When Models Manipulate Manifolds: The Geometry of a Counting Task}, journal={Transformer Circuits Thread}, year={2025}, url={https://transformer-circuits.pub/2025/linebreaks/index.html} } @article{sofroniew2026twheemotion, author={Sofroniew, Nicholas and Kauvar, Isaac and Saunders, William and Chen, Runjin and Henighan, Tom and Hydrie, Sasha and Citro, Craig and Pearce, Adam and Tarng, Julius and Gurnee, Wes and Batson, Joshua and Zimmerman, Sam and Rivoire, Kelley and Fish, Kyle and Olah, Chris and Lindsey, Jack}, title={Emotion Concepts and their Function in a Large Language Model}, journal={Transformer Circuits Thread}, year={2026}, url={https://transformer-circuits.pub/2026/emotions/index.html} } @article{lu2026assistant, title={The Assistant Axis: Situating and Stabilizing the Default Persona of Language Models}, author={Lu, Christina and Gallagher, Jack and Michala, Jonathan and Fish, Kyle and Lindsey, Jack}, journal={arXiv preprint arXiv:2601.10387}, year={2026}, url={https://arxiv.org/abs/2601.10387} } @techreport{anthropic2025sonnet45, title={Claude Sonnet 4.5 System Card}, author={Anthropic}, year={2025}, institution={Anthropic}, url={https://www.anthropic.com/claude-sonnet-4-5-system-card} } @article{needham2025evaluation, title={Large Language Models Often Know When They Are Being Evaluated}, author={Needham, Joe and Edkins, Giles and Pimpale, Govind and Bartsch, Henning and Hobbhahn, Marius}, journal={arXiv preprint arXiv:2505.23836}, year={2025}, url={https://arxiv.org/abs/2505.23836} } @misc{nostalgebraist2020logitlens, author={nostalgebraist}, title={interpreting GPT: the logit lens}, year={2020}, howpublished={LessWrong}, url={https://www.lesswrong.com/posts/AcKRB8wDpdaN6v6ru/interpreting-gpt-the-logit-lens} } @article{belrose2023eliciting, title={Eliciting Latent Predictions from Transformers with the Tuned Lens}, author={Belrose, Nora and Furman, Zach and Smith, Logan and Halawi, Danny and Ostrovsky, Igor and McKinney, Lev and Biderman, Stella and Steinhardt, Jacob}, journal={arXiv preprint arXiv:2303.08112}, year={2023}, url={https://arxiv.org/abs/2303.08112} } @inproceedings{yomdin2024jump, title={Jump to Conclusions: Short-Cutting Transformers with Linear Transformations}, author={Yom Din, Alexander and Karidi, Taelin and Choshen, Leshem and Geva, Mor}, booktitle={Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2024)}, pages={9615--9625}, year={2024}, url={https://arxiv.org/abs/2303.09435} } @inproceedings{pal2023future, title={Future Lens: Anticipating Subsequent Tokens from a Single Hidden State}, author={Pal, Koyena and Sun, Jiuding and Yuan, Andrew and Wallace, Byron C and Bau, David}, booktitle={Proceedings of the 27th Conference on Computational Natural Language Learning (CoNLL)}, pages={548--560}, year={2023}, address={Singapore}, publisher={Association for Computational Linguistics}, doi={10.18653/v1/2023.conll-1.37}, url={https://aclanthology.org/2023.conll-1.37} } @inproceedings{geva2022transformer, title={Transformer Feed-Forward Layers Build Predictions by Promoting Concepts in the Vocabulary Space}, author={Geva, Mor and Caciularu, Avi and Wang, Kevin Ro and Goldberg, Yoav}, booktitle={Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing}, pages={30--45}, year={2022}, address={Abu Dhabi, United Arab Emirates}, publisher={Association for Computational Linguistics}, doi={10.18653/v1/2022.emnlp-main.3}, url={https://aclanthology.org/2022.emnlp-main.3} } @inproceedings{dar2023analyzing, title={Analyzing Transformers in Embedding Space}, author={Dar, Guy and Geva, Mor and Gupta, Ankit and Berant, Jonathan}, booktitle={Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, pages={16124--16170}, year={2023}, address={Toronto, Canada}, publisher={Association for Computational Linguistics}, doi={10.18653/v1/2023.acl-long.893}, url={https://aclanthology.org/2023.acl-long.893} } @misc{nanda2024progress, author={Nanda, Neel and Conmy, Arthur and Smith, Lewis and Rajamanoharan, Senthooran and Lieberum, Tom and Kram{\'a}r, J{\'a}nos and Varma, Vikrant}, title={Progress Update #1 from the GDM Mech Interp Team}, year={2024}, url={https://www.alignmentforum.org/posts/HpAr8k74mW4ivCvCu/summary-progress-update-1-from-the-gdm-mech-interp-team} } @article{blumensath2008gradient, title={Gradient Pursuits}, author={Blumensath, Thomas and Davies, Mike E}, journal={IEEE Transactions on Signal Processing}, volume={56}, number={6}, pages={2370--2382}, year={2008}, doi={10.1109/TSP.2007.916124} } @article{lindsey2025emergent, author={Lindsey, Jack}, title={Emergent Introspective Awareness in Large Language Models}, journal={Transformer Circuits Thread}, year={2025}, url={https://transformer-circuits.pub/2025/introspection/index.html} } @article{chun2011taxonomy, title={A taxonomy of external and internal attention}, author={Chun, Marvin M and Golomb, Julie D and Turk-Browne, Nicholas B}, journal={Annual Review of Psychology}, volume={62}, pages={73--101}, year={2011}, doi={10.1146/annurev.psych.093008.100427} } @article{baddeley2003working, title={Working memory: looking back and looking forward}, author={Baddeley, Alan}, journal={Nature Reviews Neuroscience}, volume={4}, number={10}, pages={829--839}, year={2003}, doi={10.1038/nrn1201} } @article{wegner1987paradoxical, title={Paradoxical effects of thought suppression}, author={Wegner, Daniel M and Schneider, David J and Carter, Samuel R and White, Teri L}, journal={Journal of Personality and Social Psychology}, volume={53}, number={1}, pages={5--13}, year={1987}, doi={10.1037/0022-3514.53.1.5} } @article{wegner1994ironic, title={Ironic processes of mental control}, author={Wegner, Daniel M}, journal={Psychological Review}, volume={101}, number={1}, pages={34--52}, year={1994}, doi={10.1037/0033-295X.101.1.34} } @article{miller1956magical, title={The magical number seven, plus or minus two: Some limits on our capacity for processing information}, author={Miller, George A}, journal={Psychological Review}, volume={63}, number={2}, pages={81--97}, year={1956}, doi={10.1037/h0043158} } @article{cowan2001magical, title={The magical number 4 in short-term memory: A reconsideration of mental storage capacity}, author={Cowan, Nelson}, journal={Behavioral and Brain Sciences}, volume={24}, number={1}, pages={87--114}, year={2001}, doi={10.1017/S0140525X01003922} } @article{lynch2025agentic, title={Agentic misalignment: How llms could be insider threats}, author={Lynch, Aengus and Wright, Benjamin and Larson, Caleb and Ritchie, Stuart J and Mindermann, Soren and Hubinger, Evan and Perez, Ethan and Troy, Kevin}, journal={arXiv preprint arXiv:2510.05179}, year={2025} } @techreport{anthropic2025opus45, title={Claude Opus 4.5 System Card}, author={Anthropic}, year={2025}, month={November}, institution={Anthropic}, url={https://www.anthropic.com/claude-opus-4-5-system-card} } @article{macdiarmid2025natural, title={Natural Emergent Misalignment from Reward Hacking in Production RL}, author={MacDiarmid, Monte and Wright, Benjamin and Uesato, Jonathan and Benton, Joe and Kutasov, Jon and Price, Sara and Bouscal, Naia and Bowman, Samuel R. and Bricken, Trenton and Cloud, Alex and Denison, Carson and Gasteiger, Johannes and Greenblatt, Ryan and Leike, Jan and Lindsey, Jack and Mikulik, Vlad and Perez, Ethan and Rodrigues, Alex and Thomas, Drake and Webson, Albert and Ziegler, Daniel and Hubinger, Evan}, journal={arXiv preprint arXiv:2511.18397}, year={2025}, url={https://arxiv.org/abs/2511.18397} } @book{weiskrantz1986blindsight, title={Blindsight: A Case Study and Implications}, author={Weiskrantz, Lawrence}, year={1986}, publisher={Oxford University Press} } @article{graziano2011human, title={Human consciousness and its relationship to social neuroscience: A novel hypothesis}, author={Graziano, Michael S. A. and Kastner, Sabine}, journal={Cognitive Neuroscience}, volume={2}, number={2}, pages={98--113}, year={2011}, doi={10.1080/17588928.2011.565121} } @article{graziano2015attention, title={The attention schema theory: a mechanistic account of subjective awareness}, author={Graziano, Michael S. A. and Webb, Taylor W.}, journal={Frontiers in Psychology}, volume={6}, pages={500}, year={2015}, doi={10.3389/fpsyg.2015.00500} } @article{lamme2006towards, title={Towards a true neural stance on consciousness}, author={Lamme, Victor A. F.}, journal={Trends in Cognitive Sciences}, volume={10}, number={11}, pages={494--501}, year={2006}, doi={10.1016/j.tics.2006.09.001} } @article{lamme2000distinct, title={The distinct modes of vision offered by feedforward and recurrent processing}, author={Lamme, Victor A. F. and Roelfsema, Pieter R.}, journal={Trends in Neurosciences}, volume={23}, number={11}, pages={571--579}, year={2000}, doi={10.1016/S0166-2236(00)01657-X} } @article{bai2022constitutional, title={Constitutional AI: Harmlessness from AI Feedback}, author={Bai, Yuntao and Kadavath, Saurav and Kundu, Sandipan and Askell, Amanda and Kernion, Jackson and Jones, Andy and Chen, Anna and Goldie, Anna and Mirhoseini, Azalia and McKinnon, Cameron and others}, journal={arXiv preprint arXiv:2212.08073}, year={2022}, url={https://arxiv.org/abs/2212.08073} } @inproceedings{sun2023principle, title={Principle-Driven Self-Alignment of Language Models from Scratch with Minimal Human Supervision}, author={Sun, Zhiqing and Shen, Yikang and Zhou, Qinhong and Zhang, Hongxin and Chen, Zhenfang and Cox, David and Yang, Yiming and Gan, Chuang}, booktitle={Advances in Neural Information Processing Systems}, volume={36}, year={2023}, url={https://arxiv.org/abs/2305.03047} } @article{guan2024deliberative, title={Deliberative Alignment: Reasoning Enables Safer Language Models}, author={Guan, Melody Y. and Joglekar, Manas and Wallace, Eric and Jain, Saachi and Barak, Boaz and Helyar, Alec and Dias, Rachel and Vallone, Andrea and Ren, Hongyu and Wei, Jason and others}, journal={arXiv preprint arXiv:2412.16339}, year={2024}, url={https://arxiv.org/abs/2412.16339} } @inproceedings{zelikman2022star, title={STaR: Bootstrapping Reasoning With Reasoning}, author={Zelikman, Eric and Wu, Yuhuai and Mu, Jesse and Goodman, Noah D.}, booktitle={Advances in Neural Information Processing Systems}, volume={35}, year={2022}, url={https://arxiv.org/abs/2203.14465} } @article{zelikman2024quietstar, title={Quiet-STaR: Language Models Can Teach Themselves to Think Before Speaking}, author={Zelikman, Eric and Harik, Georges and Shao, Yijia and Jayasiri, Varuna and Haber, Nick and Goodman, Noah D.}, journal={arXiv preprint arXiv:2403.09629}, year={2024}, url={https://arxiv.org/abs/2403.09629} } @article{deng2024stepwise, title={From Explicit CoT to Implicit CoT: Learning to Internalize CoT Step by Step}, author={Deng, Yuntian and Choi, Yejin and Shieber, Stuart}, journal={arXiv preprint arXiv:2405.14838}, year={2024}, url={https://arxiv.org/abs/2405.14838} } @article{berglund2023outofcontext, title={Taken out of context: On measuring situational awareness in LLMs}, author={Berglund, Lukas and Stickland, Asa Cooper and Balesni, Mikita and Kaufmann, Max and Tong, Meg and Korbak, Tomasz and Kokotajlo, Daniel and Evans, Owain}, journal={arXiv preprint arXiv:2309.00667}, year={2023}, url={https://arxiv.org/abs/2309.00667} } @article{arora2026language, title={Language Model Circuits Are Sparse in the Neuron Basis}, author={Arora, Aryaman and Wu, Zhengxuan and Steinhardt, Jacob and Schwettmann, Sarah}, journal={arXiv preprint arXiv:2601.22594}, year={2026} } @article{bengio2017consciousness, title={The Consciousness Prior}, author={Bengio, Yoshua}, journal={arXiv preprint arXiv:1709.08568}, year={2017}, url={https://arxiv.org/abs/1709.08568} } @inproceedings{goyal2022coordination, title={Coordination Among Neural Modules Through a Shared Global Workspace}, author={Goyal, Anirudh and Didolkar, Aniket and Lamb, Alex and Badola, Kartikeya and Ke, Nan Rosemary and Rahaman, Nasim and Binas, Jonathan and Blundell, Charles and Mozer, Michael and Bengio, Yoshua}, booktitle={International Conference on Learning Representations}, year={2022}, url={https://arxiv.org/abs/2103.01197} } @article{vanrullen2021deep, title={Deep learning and the Global Workspace Theory}, author={VanRullen, Rufin and Kanai, Ryota}, journal={Trends in Neurosciences}, volume={44}, number={9}, pages={692--704}, year={2021}, doi={10.1016/j.tins.2021.04.005}, url={https://arxiv.org/abs/2012.10390} } @article{mongillo2008synaptic, title={Synaptic Theory of Working Memory}, author={Mongillo, Gianluigi and Barak, Omri and Tsodyks, Misha}, journal={Science}, volume={319}, number={5869}, pages={1543--1546}, year={2008}, doi={10.1126/science.1150769} } @article{stokes2015activity, title={`Activity-silent' working memory in prefrontal cortex: a dynamic coding framework}, author={Stokes, Mark G}, journal={Trends in Cognitive Sciences}, volume={19}, number={7}, pages={394--405}, year={2015}, doi={10.1016/j.tics.2015.05.004} } @inproceedings{ba2016fast, title={Using Fast Weights to Attend to the Recent Past}, author={Ba, Jimmy and Hinton, Geoffrey and Mnih, Volodymyr and Leibo, Joel Z and Ionescu, Catalin}, booktitle={Advances in Neural Information Processing Systems}, volume={29}, year={2016}, url={https://arxiv.org/abs/1610.06258} } @inproceedings{schlag2021linear, title={Linear Transformers Are Secretly Fast Weight Programmers}, author={Schlag, Imanol and Irie, Kazuki and Schmidhuber, J{\"u}rgen}, booktitle={Proceedings of the 38th International Conference on Machine Learning}, year={2021}, organization={PMLR}, url={https://arxiv.org/abs/2102.11174} } @inproceedings{krotov2021large, title={Large Associative Memory Problem in Neurobiology and Machine Learning}, author={Krotov, Dmitry and Hopfield, John J}, booktitle={International Conference on Learning Representations}, year={2021}, url={https://arxiv.org/abs/2008.06996} } @article{kozachkov2023building, title={Building transformers from neurons and astrocytes}, author={Kozachkov, Leo and Kastanenka, Ksenia V and Krotov, Dmitry}, journal={Proceedings of the National Academy of Sciences}, volume={120}, number={34}, pages={e2219150120}, year={2023}, doi={10.1073/pnas.2219150120} } @article{irie2025fastweight, title={Fast weight programming and linear transformers: from machine learning to neurobiology}, author={Irie, Kazuki and Gershman, Samuel J}, journal={arXiv preprint arXiv:2508.08435}, year={2025}, url={https://arxiv.org/abs/2508.08435} } @article{milliere2020varieties, title={The varieties of selflessness}, author={Milli{\`e}re, Rapha{\"e}l}, journal={Philosophy and the Mind Sciences}, volume={1}, number={I}, pages={1--41}, year={2020}, doi={10.33735/phimisci.2020.I.48} } @article{metzinger2020minimal, title={Minimal phenomenal experience: Meditation, tonic alertness, and the phenomenology of "pure" consciousness}, author={Metzinger, Thomas}, journal={Philosophy and the Mind Sciences}, volume={1}, number={I}, pages={1--44}, year={2020}, doi={10.33735/phimisci.2020.I.46} } @article{letheby2017self, title={Self unbound: ego dissolution in psychedelic experience}, author={Letheby, Chris and Gerrans, Philip}, journal={Neuroscience of Consciousness}, volume={2017}, number={1}, pages={nix016}, year={2017}, doi={10.1093/nc/nix016} } @inproceedings{hernandez2024linearity, title={Linearity of Relation Decoding in Transformer Language Models}, author={Hernandez, Evan and Sharma, Arnab Sen and Haklay, Tal and Meng, Kevin and Wattenberg, Martin and Andreas, Jacob and Belinkov, Yonatan and Bau, David}, booktitle={International Conference on Learning Representations}, year={2024}, url={https://arxiv.org/abs/2308.09124} } @inproceedings{katz2024backward, title={Backward Lens: Projecting Language Model Gradients into the Vocabulary Space}, author={Katz, Shahar and Belinkov, Yonatan and Geva, Mor and Wolf, Lior}, booktitle={Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing}, year={2024}, url={https://arxiv.org/abs/2402.12865} } @inproceedings{montufar2014number, title={On the Number of Linear Regions of Deep Neural Networks}, author={Mont{\'u}far, Guido and Pascanu, Razvan and Cho, Kyunghyun and Bengio, Yoshua}, booktitle={Advances in Neural Information Processing Systems}, year={2014}, url={https://arxiv.org/abs/1402.1869} } @article{golden2025equivalent, title={Equivalent Linear Mappings of Large Language Models}, author={Golden, James R}, journal={arXiv preprint arXiv:2505.24293}, year={2025}, url={https://arxiv.org/abs/2505.24293} } @article{atad2026tensorlens, title={TensorLens: End-to-End Transformer Analysis via High-Order Attention Tensors}, author={Atad, Ido Andrew and Zimerman, Itamar and Katz, Shahar and Wolf, Lior}, journal={arXiv preprint arXiv:2601.17958}, year={2026}, url={https://arxiv.org/abs/2601.17958} } @article{sandmann2025iterative, title={Iterative Inference in a Chess-Playing Neural Network}, author={Sandmann, Elias and Lapuschkin, Sebastian and Samek, Wojciech}, journal={arXiv preprint arXiv:2508.21380}, year={2025}, url={https://arxiv.org/abs/2508.21380} } @inproceedings{merullo2024language, title={Language Models Implement Simple Word2Vec-style Vector Arithmetic}, author={Merullo, Jack and Eickhoff, Carsten and Pavlick, Ellie}, booktitle={Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies}, pages={5030--5047}, year={2024}, url={https://arxiv.org/abs/2305.16130} } @article{chughtai2024summing, title={Summing Up the Facts: Additive Mechanisms Behind Factual Recall in LLMs}, author={Chughtai, Bilal and Cooney, Alan and Nanda, Neel}, journal={arXiv preprint arXiv:2402.07321}, year={2024}, url={https://arxiv.org/abs/2402.07321} } @article{zhang2024hallucination, title={Knowledge Overshadowing Causes Amalgamated Hallucination in Large Language Models}, author={Zhang, Yuji and Li, Sha and Liu, Jiateng and Yu, Pengfei and Fung, Yi R and Li, Jing and Li, Manling and Ji, Heng}, journal={arXiv preprint arXiv:2407.08039}, year={2024}, url={https://arxiv.org/abs/2407.08039} } @inproceedings{li2024understanding, title={Understanding and Patching Compositional Reasoning in LLMs}, author={Li, Zhaoyi and Jiang, Gangwei and Xie, Hong and Song, Linqi and Lian, Defu and Wei, Ying}, booktitle={Findings of the Association for Computational Linguistics: ACL 2024}, pages={9668--9688}, year={2024}, url={https://arxiv.org/abs/2402.14328} } @article{lu2025latent, title={Latent Chain-of-Thought? Decoding the Depth-Recurrent Transformer}, author={Lu, Wenquan and Wu, Yuechuan and Yang, Xiaodong and Liu, Kyle and Ren, Hongyu}, journal={arXiv preprint arXiv:2507.02199}, year={2025}, url={https://arxiv.org/abs/2507.02199} } @inproceedings{yan2025addition, title={Addition in Four Movements: Mapping Layer-wise Information Trajectories in LLMs}, author={Yan, Yao}, booktitle={Findings of the Association for Computational Linguistics: EMNLP 2025}, year={2025}, url={https://arxiv.org/abs/2506.07824} } @article{zhong2024beyond, title={Beyond English-Centric LLMs: What Language Do Multilingual Language Models Think in?}, author={Zhong, Chengzhi and Cheng, Fei and Liu, Qianying and Jiang, Junfeng and Wan, Zhen and Chu, Chenhui and Murawaki, Yugo and Kurohashi, Sadao}, journal={arXiv preprint arXiv:2408.10811}, year={2024}, url={https://arxiv.org/abs/2408.10811} } @inproceedings{saji2025romanlens, title={RomanLens: The Role Of Latent Romanization In Multilinguality In LLMs}, author={Saji, Alan and Husain, Jaavid Aktar and Jayakumar, Thanmay and Dabre, Raj and Kunchukuttan, Anoop and Puduppully, Ratish}, booktitle={Findings of the Association for Computational Linguistics: ACL 2025}, year={2025}, url={https://arxiv.org/abs/2502.07424} } @article{bode2025aya, title={What Language(s) Does Aya-23 Think In? How Multilinguality Affects Internal Language Representations}, author={Trinley, Katharina and Nakai, Toshiki and Anikina, Tatiana and Baeumel, Tanja}, journal={arXiv preprint arXiv:2507.20279}, year={2025}, url={https://arxiv.org/abs/2507.20279} } @inproceedings{liu2025paths, title={Paths Not Taken: Understanding and Mending the Multilingual Factual Recall Pipeline}, author={Lu, Meng and Zhang, Ruochen and Eickhoff, Carsten and Pavlick, Ellie}, booktitle={Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing}, pages={15066--15096}, year={2025}, url={https://arxiv.org/abs/2505.20546} } @inproceedings{halawi2024overthinking, title={Overthinking the Truth: Understanding how Language Models Process False Demonstrations}, author={Halawi, Danny and Denain, Jean-Stanislas and Steinhardt, Jacob}, booktitle={International Conference on Learning Representations}, year={2024}, url={https://arxiv.org/abs/2307.09476} } @inproceedings{zhou2024alignment, title={How Alignment and Jailbreak Work: Explain LLM Safety through Intermediate Hidden States}, author={Zhou, Zhenhong and Yu, Haiyang and Zhang, Xinghua and Xu, Rongwu and Huang, Fei and Li, Yongbin}, booktitle={Findings of the Association for Computational Linguistics: EMNLP 2024}, pages={2461--2488}, year={2024}, url={https://arxiv.org/abs/2406.05644} } @article{jiang2024interpreting, title={Interpreting and Editing Vision-Language Representations to Mitigate Hallucinations}, author={Jiang, Nick and Kachinthaya, Anish and Petryk, Suzie and Gandelsman, Yossi}, journal={arXiv preprint arXiv:2410.02762}, year={2024}, url={https://arxiv.org/abs/2410.02762} } @article{li2025hidden, title={The Hidden Life of Tokens: Reducing Hallucination of Large Vision-Language Models via Visual Information Steering}, author={Li, Zhuowei and Shi, Haizhou and Gao, Yunhe and Liu, Di and Wang, Zhenting and Chen, Yuxiao and Liu, Ting and Zhao, Long and Wang, Hao and Metaxas, Dimitris N}, journal={arXiv preprint arXiv:2502.03628}, year={2025}, url={https://arxiv.org/abs/2502.03628} } @inproceedings{gong2024devils, title={Devils in Middle Layers of Large Vision-Language Models: Interpreting, Detecting and Mitigating Object Hallucinations via Attention Lens}, author={Jiang, Zhangqi and Chen, Junkai and Zhu, Beier and Luo, Tingjin and Shen, Yankun and Yang, Xu}, booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, year={2025}, url={https://arxiv.org/abs/2411.16724} } @inproceedings{toker2024diffusion, title={Diffusion Lens: Interpreting Text Encoders in Text-to-Image Pipelines}, author={Toker, Michael and Orgad, Hadas and Ventura, Mor and Arad, Dana and Belinkov, Yonatan}, booktitle={Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, pages={9713--9728}, year={2024}, url={https://arxiv.org/abs/2403.05846} } @inproceedings{takatsuki2025diffusion, title={Decoding Vision Transformers: the Diffusion Steering Lens}, author={Takatsuki, Ryota and Joseph, Sonia and Fujisawa, Ippei and Kanai, Ryota}, booktitle={CVPR Workshop on Mechanistic Interpretability for Vision}, year={2025}, url={https://arxiv.org/abs/2504.13763} } @inproceedings{chuang2024dola, title={DoLa: Decoding by Contrasting Layers Improves Factuality in Large Language Models}, author={Chuang, Yung-Sung and Xie, Yujia and Luo, Hongyin and Kim, Yoon and Glass, James and He, Pengcheng}, booktitle={International Conference on Learning Representations}, year={2024}, url={https://arxiv.org/abs/2309.03883} } @inproceedings{gera2023benefits, title={The Benefits of Bad Advice: Autocontrastive Decoding across Model Layers}, author={Gera, Ariel and Friedman, Roni and Arviv, Ofir and Gunasekara, Chulaka and Sznajder, Benjamin and Slonim, Noam and Shnarch, Eyal}, booktitle={Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)}, year={2023}, url={https://arxiv.org/abs/2305.01628} } @inproceedings{hewitt2019designing, title={Designing and Interpreting Probes with Control Tasks}, author={Hewitt, John and Liang, Percy}, booktitle={Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)}, pages={2733--2743}, year={2019}, url={https://arxiv.org/abs/1909.03368} } @article{elazar2021amnesic, title={Amnesic Probing: Behavioral Explanation with Amnesic Counterfactuals}, author={Elazar, Yanai and Ravfogel, Shauli and Jacovi, Alon and Goldberg, Yoav}, journal={Transactions of the Association for Computational Linguistics}, volume={9}, pages={160--175}, year={2021}, url={https://arxiv.org/abs/2006.00995} } @article{pan2024latentqa, title={LatentQA: Teaching LLMs to Decode Activations Into Natural Language}, author={Pan, Alexander and Chen, Lijie and Steinhardt, Jacob}, journal={arXiv preprint arXiv:2412.08686}, year={2024}, url={https://arxiv.org/abs/2412.08686} } @article{karvonen2025oracles, title={Activation Oracles: Training and Evaluating LLMs as General-Purpose Activation Explainers}, author={Karvonen, Adam and Chua, James and Dumas, Cl{\'e}ment and Fraser-Taliente, Kit and Kantamneni, Subhash and Minder, Julian and Ong, Euan and Sharma, Arnab Sen and Wen, Daniel and Evans, Owain and Marks, Samuel}, journal={arXiv preprint arXiv:2512.15674}, year={2025}, url={https://arxiv.org/abs/2512.15674} } @article{frasertaliente2026nla, title={Natural Language Autoencoders Produce Unsupervised Explanations of LLM Activations}, author={Fraser-Taliente, Kit and Kantamneni, Subhash and Ong, Euan and Mossing, Dan and Lu, Christina and Bogdan, Paul C and Ameisen, Emmanuel and Chen, James and Kishylau, Dzmitry and Pearce, Adam and Tarng, Julius and Wu, Alex and Wu, Jeff and Zhang, Yang and Ziegler, Daniel M and Hubinger, Evan and Batson, Joshua and Lindsey, Jack and Zimmerman, Samuel and Marks, Samuel}, journal={Transformer Circuits Thread}, year={2026}, url={https://transformer-circuits.pub/2026/nla/index.html} } @article{hendrycks2021measuring, title={Measuring Massive Multitask Language Understanding}, author={Hendrycks, Dan and Burns, Collin and Basart, Steven and Zou, Andy and Mazeika, Mantas and Song, Dawn and Steinhardt, Jacob}, journal={International Conference on Learning Representations (ICLR)}, year={2021}, url={https://arxiv.org/abs/2009.03300} } @inproceedings{rajpurkar2016squad, title={SQuAD: 100,000+ Questions for Machine Comprehension of Text}, author={Rajpurkar, Pranav and Zhang, Jian and Lopyrev, Konstantin and Liang, Percy}, booktitle={Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing (EMNLP)}, pages={2383--2392}, year={2016}, url={https://arxiv.org/abs/1606.05250} } @article{warstadt2019cola, title={Neural Network Acceptability Judgments}, author={Warstadt, Alex and Singh, Amanpreet and Bowman, Samuel R.}, journal={Transactions of the Association for Computational Linguistics}, volume={7}, pages={625--641}, year={2019}, url={https://arxiv.org/abs/1805.12471} } @inproceedings{joshi2017triviaqa, title={TriviaQA: A Large Scale Distantly Supervised Challenge Dataset for Reading Comprehension}, author={Joshi, Mandar and Choi, Eunsol and Weld, Daniel S. and Zettlemoyer, Luke}, booktitle={Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (ACL)}, pages={1601--1611}, year={2017}, url={https://arxiv.org/abs/1705.03551} } @article{cobbe2021gsm8k, title={Training Verifiers to Solve Math Word Problems}, author={Cobbe, Karl and Kosaraju, Vineet and Bavarian, Mohammad and Chen, Mark and Jun, Heewoo and Kaiser, Lukasz and Plappert, Matthias and Tworek, Jerry and Hilton, Jacob and Nakano, Reiichiro and Hesse, Christopher and Schulman, John}, journal={arXiv preprint arXiv:2110.14168}, year={2021}, url={https://arxiv.org/abs/2110.14168} } @inproceedings{stiennon2020learning, title={Learning to Summarize from Human Feedback}, author={Stiennon, Nisan and Ouyang, Long and Wu, Jeff and Ziegler, Daniel M. and Lowe, Ryan and Voss, Chelsea and Radford, Alec and Amodei, Dario and Christiano, Paul}, booktitle={Advances in Neural Information Processing Systems (NeurIPS)}, year={2020}, url={https://arxiv.org/abs/2009.01325} } @inproceedings{volske2017tldr, title={TL;DR: Mining Reddit to Learn Automatic Summarization}, author={V{\"o}lske, Michael and Potthast, Martin and Syed, Shahbaz and Stein, Benno}, booktitle={Proceedings of the Workshop on New Frontiers in Summarization}, pages={59--63}, year={2017}, publisher={Association for Computational Linguistics}, doi={10.18653/v1/W17-4508}, url={https://aclanthology.org/W17-4508/} } @article{nye2021scratchpad, title={Show Your Work: Scratchpads for Intermediate Computation with Language Models}, author={Nye, Maxwell and Andreassen, Anders Johan and Gur-Ari, Guy and Michalewski, Henryk and Austin, Jacob and Bieber, David and Dohan, David and Lewkowycz, Aitor and Bosma, Maarten and Luan, David and Sutton, Charles and Odena, Augustus}, journal={arXiv preprint arXiv:2112.00114}, year={2021}, url={https://arxiv.org/abs/2112.00114} } @inproceedings{wei2022chain, title={Chain-of-Thought Prompting Elicits Reasoning in Large Language Models}, author={Wei, Jason and Wang, Xuezhi and Schuurmans, Dale and Bosma, Maarten and Ichter, Brian and Xia, Fei and Chi, Ed H. and Le, Quoc V. and Zhou, Denny}, booktitle={Advances in Neural Information Processing Systems (NeurIPS)}, year={2022}, url={https://arxiv.org/abs/2201.11903} } @inproceedings{leask2025sparse, title={Sparse autoencoders do not find canonical units of analysis}, author={Leask, Patrick and Bussmann, Bart and Pearce, Michael and Bloom, Joseph and Tigges, Curt and Al Moubayed, Noura and Sharkey, Lee and Nanda, Neel}, booktitle={International Conference on Learning Representations}, volume={2025}, pages={53617--53642}, year={2025} } @article{butlin2025identifying, title={Identifying indicators of consciousness in AI systems}, author={Butlin, Patrick and Long, Robert and Bayne, Tim and Bengio, Yoshua and Birch, Jonathan and Chalmers, David and Constant, Axel and Deane, George and Elmoznino, Eric and Fleming, Stephen M and others}, journal={Trends in Cognitive Sciences}, year={2025}, publisher={Elsevier} } @article{sackur2009cognitive, title={The cognitive architecture for chaining of two mental operations}, author={Sackur, J{\'e}r{\^o}me and Dehaene, Stanislas}, journal={Cognition}, volume={111}, number={2}, pages={187--211}, year={2009}, publisher={Elsevier} } @article{dehaene2011experimental, title={Experimental and theoretical approaches to conscious processing}, author={Dehaene, Stanislas and Changeux, Jean-Pierre}, journal={Neuron}, volume={70}, number={2}, pages={200--227}, year={2011}, publisher={Elsevier} } @article{marois2005capacity, title={Capacity limits of information processing in the brain}, author={Marois, Ren{\'e} and Ivanoff, Jason}, journal={Trends in cognitive sciences}, volume={9}, number={6}, pages={296--305}, year={2005}, publisher={Elsevier} } @inproceedings{liu2019linguistic, title={Linguistic knowledge and transferability of contextual representations}, author={Liu, Nelson F and Gardner, Matt and Belinkov, Yonatan and Peters, Matthew E and Smith, Noah A}, booktitle={Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)}, pages={1073--1094}, year={2019} } @article{jacobs2025block, title={Block-Recurrent Dynamics in Vision Transformers}, author={Jacobs, Mozes and Fel, Thomas and Hakim, Richard and Brondetta, Alessandra and Ba, Demba and Keller, T Andy}, journal={arXiv preprint arXiv:2512.19941}, year={2025}, url={https://arxiv.org/abs/2512.19941} } @article{fleming2025computational, title={Computational Higher-order Theories of Consciousness}, author={Fleming, Stephen M and Brown, Richard and Cleeremans, Axel}, year={2025}, url={https://philarchive.org/archive/FLECHT} } @article{mudrik2014information, title={Information Integration without Awareness}, author={Mudrik, Liad and Faivre, Nathan and Koch, Christof}, journal={Trends in cognitive sciences}, volume={18}, number={9}, pages={488--496}, year={2014}, publisher={Elsevier} } @book{kahneman2011thinking, title={Thinking, Fast and Slow}, author={Kahneman, Daniel}, year={2011}, publisher={macmillan} } @article{kihlstrom1987cognitive, title={The Cognitive Unconscious}, author={Kihlstrom, John F}, journal={Science}, volume={237}, number={4821}, pages={1445--1452}, year={1987}, publisher={American Association for the Advancement of Science} } @book{chalmers1996conscious, title={The Conscious Mind: In Search of a Theory of Conscious Experience}, author={Chalmers, David John}, year={1996}, publisher={Oxford University Press New York} } @article{lau2019consciousness, title={Consciousness, metacognition, \& perceptual reality monitoring}, author={Lau, Hakwan}, year={2019} } @article{fleming2020awareness, title={Awareness as inference in a higher-order state space}, author={Fleming, Stephen M}, journal={Neuroscience of consciousness}, volume={2020}, number={1}, pages={niz020}, year={2020}, publisher={Oxford University Press} } @article{searle1980minds, title={Minds, brains, and programs}, author={Searle, John R}, journal={Behavioral and brain sciences}, volume={3}, number={3}, pages={417--424}, year={1980}, publisher={Cambridge University Press} } @article{persaud2008blindsight, title={Blindsight is unlike normal conscious vision: evidence from an exclusion task}, author={Persaud, Navindra and Cowey, Alan}, journal={Consciousness and cognition}, volume={17}, number={3}, pages={1050--1055}, year={2008}, publisher={Elsevier} } @article{jacoby1991process, title={A process dissociation framework: Separating automatic from intentional uses of memory}, author={Jacoby, Larry L}, journal={Journal of memory and language}, volume={30}, number={5}, pages={513--541}, year={1991}, publisher={Elsevier} } @article{fang2025unsupervised, title={Unsupervised decoding of encoded reasoning using language model interpretability}, author={Fang, Ching and others}, journal={arXiv preprint arXiv:2512.01222}, year={2025}, url={https://arxiv.org/abs/2512.01222} } @article{krojer2026latentlens, title={LatentLens: Revealing Highly Interpretable Visual Tokens in LLMs}, author={Krojer, Benno and Nayak, Shravan and Ma{\~n}as, Oscar and Adlakha, Vaibhav and Elliott, Desmond and Reddy, Siva and Mosbach, Marius}, journal={arXiv preprint arXiv:2602.00462}, year={2026}, url={https://arxiv.org/abs/2602.00462} } @inproceedings{stolfo2024improving, title={Improving Instruction-Following in Language Models through Activation Steering}, author={Stolfo, Alessandro and others}, booktitle={arXiv preprint arXiv:2410.12877}, year={2024}, url={https://arxiv.org/abs/2410.12877} } @article{urbina2026synergistic, title={A Brain-like Synergistic Core in LLMs Drives Behaviour and Learning}, author={Urbina-Rodriguez, Pedro and Fountas, Zafeirios and Rosas, Fernando E. and Wang, Jun and Luppi, Andrea I. and Bou-Ammar, Haitham and Shanahan, Murray and Mediano, Pedro A. M.}, year={2026}, eprint={2601.06851}, archivePrefix={arXiv}, primaryClass={cs.AI}, url={https://arxiv.org/abs/2601.06851} } @inproceedings{skean2025layer, title={Layer by Layer: Uncovering Hidden Representations in Language Models}, author={Skean, Oscar and Arefin, Md Rifat and Zhao, Dan and Patel, Niket and Naghiyev, Jalal and LeCun, Yann and Shwartz-Ziv, Ravid}, booktitle={Proceedings of the 42nd International Conference on Machine Learning}, year={2025}, eprint={2502.02013}, archivePrefix={arXiv}, url={https://arxiv.org/abs/2502.02013} } @article{queipo2025mix, title={Attention Sinks and Compression Valleys in LLMs are Two Sides of the Same Coin}, author={Queipo-de-Llano, Enrique and Arroyo, {\'A}lvaro and Barbero, Federico and Dong, Xiaowen and Bronstein, Michael and LeCun, Yann and Shwartz-Ziv, Ravid}, year={2025}, eprint={2510.06477}, archivePrefix={arXiv}, primaryClass={cs.LG}, url={https://arxiv.org/abs/2510.06477} } @article{janiak2024stable, title={Characterizing Stable Regions in the Residual Stream of LLMs}, author={Janiak, Jett and Karwowski, Jacek and Mangat, Chatrik Singh and Giglemiani, Giorgi and Petrova, Nora and Heimersheim, Stefan}, year={2024}, eprint={2409.17113}, archivePrefix={arXiv}, primaryClass={cs.LG}, note={NeurIPS 2024 SciForDL Workshop}, url={https://arxiv.org/abs/2409.17113} } @article{gandikota2026gaze, title={Gaze Heads: How VLMs Look at What They Describe}, author={Gandikota, Rohit and Bau, David}, year={2026}, eprint={2606.14703}, archivePrefix={arXiv}, url={https://arxiv.org/abs/2606.14703} } @inproceedings{valeriani2023geometry, title={The Geometry of Hidden Representations of Large Transformer Models}, author={Valeriani, Lucrezia and Doimo, Diego and Cuturello, Francesca and Laio, Alessandro and Ansuini, Alessio and Cazzaniga, Alberto}, booktitle={Advances in Neural Information Processing Systems}, year={2023}, eprint={2302.00294}, archivePrefix={arXiv}, url={https://arxiv.org/abs/2302.00294} } @inproceedings{hendel2023context, title={In-context learning creates task vectors}, author={Hendel, Roee and Geva, Mor and Globerson, Amir}, booktitle={Findings of the Association for Computational Linguistics: EMNLP 2023}, pages={9318--9333}, year={2023} } @article{g2026simulation, title={From Simulation to Enaction: Post-trained language models recognize and react to their own generations}, author={G, Asvin and Lindsey, Jack}, journal={arXiv preprint arXiv:2605.25459}, year={2026} } @article{o2001sensorimotor, title={A sensorimotor account of vision and visual consciousness}, author={O'regan, J Kevin and No{\"e}, Alva}, journal={Behavioral and brain sciences}, volume={24}, number={5}, pages={939--973}, year={2001}, publisher={Cambridge University Press} } @article{bogdan2026slot, title={Slot Machines: How LLMs Keep Track of Multiple Entities}, author={Bogdan, Paul C and Lindsey, Jack}, journal={arXiv preprint arXiv:2604.21139}, year={2026} } @misc{olah2023weight, title={Weight Superposition}, author={Olah, Chris}, year={2023}, url={https://transformer-circuits.pub/2023/may-update/index.html#weight-superposition} } @misc{gao2025weightsparse, title={Weight-sparse transformers have interpretable circuits}, author={Gao, Leo and Rajaram, Achyuta and Coxon, Jacob and Govande, Soham V. and Baker, Bowen and Mossing, Dan}, year={2025}, note={arXiv:2511.13653}, url={https://arxiv.org/abs/2511.13653} } @misc{drori2025sparsely, title={Sparsely-connected Cross-layer Transcoders}, author={Drori, Jacob}, year={2025}, month={June}, howpublished={LessWrong}, url={https://www.lesswrong.com/posts/NAQpcNz9WGSJ8WH2A/sparsely-connected-cross-layer-transcoders} } @misc{drori2026unfaithful, title={Weight-Sparse Circuits May Be Interpretable Yet Unfaithful}, author={Drori, Jacob}, year={2026}, month={February}, howpublished={LessWrong}, url={https://www.lesswrong.com/posts/sHpZZnRDLg7ccX9aF/weight-sparse-circuits-may-be-interpretable-yet-unfaithful} }