@article {1799661,
	title = {The role of prospective contingency in the control of behavior and dopamine signals during associative learning},
	journal = {bioRxiv},
	year = {2024},
	abstract = {Associative learning depends on contingency, the degree to which a stimulus predicts an outcome. Despite its importance, the neural mechanisms linking contingency to behavior remain elusive. Here we examined the dopamine activity in the ventral striatum {\textemdash} a signal implicated in associative learning {\textemdash} in a Pavlovian contingency degradation task in mice. We show that both anticipatory licking and dopamine responses to a conditioned stimulus decreased when additional rewards were delivered uncued, but remained unchanged if additional rewards were cued. These results conflict with contingency-based accounts using a traditional definition of contingency or a novel causal learning model (ANCCR), but can be explained by temporal difference (TD) learning models equipped with an appropriate inter-trial-interval (ITI) state representation. Recurrent neural networks trained within a TD framework develop state representations like our best {\textquoteright}handcrafted{\textquoteright} model. Our findings suggest that the TD error can be a measure that describes both contingency and dopaminergic activity.},
	url = {https://www.biorxiv.org/content/10.1101/2024.02.05.578961v1},
	author = {Lechen Qian and Mark Burrell and Jay A. Hennig and Sara Matias and Venkatesh N. Murthy and Samuel J. Gershman and Uchida, Naoshige}
}
@article {1799656,
	title = {An opponent striatal circuit for distributional reinforcement learning},
	journal = {bioRxiv},
	year = {2024},
	abstract = {Machine learning research has achieved large performance gains on a wide range of tasks by expanding the learning target from mean rewards to entire probability distributions of rewards {\textemdash} an approach known as distributional reinforcement learning (RL)1. The mesolimbic dopamine system is thought to underlie RL in the mammalian brain by updating a representation of mean value in the striatum2,3, but little is known about whether, where, and how neurons in this circuit encode information about higher-order moments of reward distributions4. To fill this gap, we used high-density probes (Neuropixels) to acutely record striatal activity from well-trained, water-restricted mice performing a classical conditioning task in which reward mean, reward variance, and stimulus identity were independently manipulated. In contrast to traditional RL accounts, we found robust evidence for abstract encoding of variance in the striatum. Remarkably, chronic ablation of dopamine inputs disorganized these distributional representations in the striatum without interfering with mean value coding. Two-photon calcium imaging and optogenetics revealed that the two major classes of striatal medium spiny neurons {\textemdash} D1 and D2 MSNs {\textemdash} contributed to this code by preferentially encoding the right and left tails of the reward distribution, respectively. We synthesize these findings into a new model of the striatum and mesolimbic dopamine that harnesses the opponency between D1 and D2 MSNs5{\textendash}15\&nbsp;to reap the computational benefits of distributional RL.},
	url = {https://www.biorxiv.org/content/10.1101/2024.01.02.573966v1},
	author = {Adam S. Lowet and Qiao Zheng and Melissa Meng and Sara Matias and Jan Drugowitsch and Uchida, Naoshige}
}
@article {1799651,
	title = {Interpretable deep learning for deconvolutional analysis of neural signals},
	journal = {bioRxiv},
	year = {2024},
	abstract = {The widespread adoption of deep learning to build models that capture the dynamics of neural populations is typically based on {\textquotedblleft}black-box{\textquotedblright} approaches that lack an interpretable link between neural activity and function. Here, we propose to apply algorithm unrolling, a method for interpretable deep learning, to design the architecture of sparse deconvolutional neural networks and obtain a direct interpretation of network weights in relation to stimulus-driven single-neuron activity through a generative model. We characterize our method, referred to as deconvolutional unrolled neural learning (DUNL), and show its versatility by applying it to deconvolve single-trial local signals across multiple brain areas and recording modalities. To exemplify use cases of our decomposition method, we uncover multiplexed salience and reward prediction error signals from midbrain dopamine neurons in an unbiased manner, perform simultaneous event detection and characterization in somatosensory thalamus recordings, and characterize the responses of neurons in the piriform cortex. Our work leverages the advances in interpretable deep learning to gain a mechanistic understanding of neural dynamics.},
	url = {https://www.biorxiv.org/content/10.1101/2024.01.05.574379v2},
	author = {Tolooshams, Bahareh and Sara Matias and Wu, Hao and Temereanca, Simona and Uchida, Naoshige and Venkatesh N. Murthy and Paul Masset and Ba, Demba}
}
@article {1779371,
	title = {Glutamate inputs send prediction error of reward but not negative value of aversive stimuli to dopamine neurons},
	journal = {Neuron},
	year = {2024},
	abstract = {Midbrain dopamine neurons are thought to signal reward prediction errors (RPEs), but the mechanisms underlying RPE computation, particularly the contributions of different neurotransmitters, remain poorly understood. Here, we used a genetically encoded glutamate sensor to examine the pattern of glutamate inputs to dopamine neurons in mice. We found that glutamate inputs exhibit virtually all of the characteristics of RPE rather than conveying a specific component of RPE computation, such as reward or expectation. Notably, whereas glutamate inputs were transiently inhibited by reward omission, they were excited by aversive stimuli. Opioid analgesics altered dopamine negative responses to aversive stimuli into more positive responses, whereas excitatory responses of glutamate inputs remained unchanged. Our findings uncover previously unknown synaptic mechanisms underlying RPE computations; dopamine responses are shaped by both synergistic and competitive interactions between glutamatergic and GABAergic inputs to dopamine neurons depending on valences, with competitive interactions playing a role in responses to aversive stimuli.},
	url = {https://www.biorxiv.org/content/10.1101/2023.11.09.566472v1},
	author = {Ryunosuke Amo and Uchida, Naoshige and Watabe-Uchida, Mitsuko}
}
@article {1779381,
	title = {Multi-timescale reinforcement learning in the brain},
	journal = {bioRxiv},
	year = {2023},
	abstract = {To thrive in complex environments, animals and artificial agents must learn to act adaptively to maximize fitness and rewards. Such adaptive behavior can be learned through reinforcement learning1, a class of algorithms that has been successful at training artificial agents2{\textendash}6\&nbsp;and at characterizing the firing of dopamine neurons in the midbrain7{\textendash}9. In classical reinforcement learning, agents discount future rewards exponentially according to a single time scale, controlled by the discount factor. Here, we explore the presence of multiple timescales in biological reinforcement learning. We first show that reinforcement agents learning at a multitude of timescales possess distinct computational benefits. Next, we report that dopamine neurons in mice performing two behavioral tasks encode reward prediction error with a diversity of discount time constants. Our model explains the heterogeneity of temporal discounting in both cue-evoked transient responses and slower timescale fluctuations known as dopamine ramps. Crucially, the measured discount factor of individual neurons is correlated across the two tasks suggesting that it is a cell-specific property. Together, our results provide a new paradigm to understand functional heterogeneity in dopamine neurons, a mechanistic basis for the empirical observation that humans and animals use non-exponential discounts in many situations\&nbsp;10{\textendash}14, and open new avenues for the design of more efficient reinforcement learning algorithms.},
	url = {https://www.biorxiv.org/content/10.1101/2023.11.12.566754v1},
	author = {Paul Masset and Pablo Tano and HyungGoo R. Kim and Athar N. Malik and Alexandre Pouget and Uchida, Naoshige}
}
@article {1779376,
	title = {Tonic dopamine and biases in value learning linked through a biologically inspired reinforcement learning model},
	journal = {bioRxiv},
	year = {2023},
	abstract = {A hallmark of various psychiatric disorders is biased future predictions. Here we examined the mechanisms for biased value learning using reinforcement learning models incorporating recent findings on synaptic plasticity and opponent circuit mechanisms in the basal ganglia. We show that variations in tonic dopamine can alter the balance between learning from positive and negative reward prediction errors, leading to biased value predictions. This bias arises from the sigmoidal shapes of the dose-occupancy curves and distinct affinities of D1- and D2-type dopamine receptors: changes in tonic dopamine differentially alters the slope of the dose-occupancy curves of these receptors, thus sensitivities, at baseline dopamine concentrations. We show that this mechanism can explain biased value learning in both mice and humans and may also contribute to symptoms observed in psychiatric disorders. Our model provides a foundation for understanding the basal ganglia circuit and underscores the significance of tonic dopamine in modulating learning processes.},
	url = {https://www.biorxiv.org/content/10.1101/2023.11.10.566580v2},
	author = {Romero Pinto, Sandra A. and Uchida, Naoshige}
}
@article {1779366,
	title = {Competitive integration of time and reward explains value-sensitive foraging decisions and frontal cortex ramping dynamics},
	journal = {bioRxiv},
	year = {2023},
	pages = {2023.09.05.556267},
	abstract = {The ability to make advantageous decisions is critical for animals to ensure their survival. Patch foraging is a natural decision-making process in which animals decide when to leave a patch of depleting resources to search for a new one. To study the algorithmic and neural basis of patch foraging behavior in a controlled laboratory setting, we developed a virtual foraging task for head-fixed mice. Mouse behavior could be explained by ramp-to-threshold models integrating time and rewards antagonistically. Accurate behavioral modeling required inclusion of a slowly varying {\textquotedblleft}patience{\textquotedblright} variable, which modulated sensitivity to time. To investigate the neural basis of this decision-making process, we performed dense electrophysiological recordings with Neuropixels probes broadly throughout frontal cortex and underlying subcortical areas. We found that decision variables from the reward integrator model were represented in neural activity, most robustly in frontal cortical areas. Regression modeling followed by unsupervised clustering identified a subset of neurons with ramping activity. These neurons{\textquoteright} firing rates ramped up gradually in single trials over long time scales (up to tens of seconds), were inhibited by rewards, and were better described as being generated by a continuous ramp rather than a discrete stepping process. Together, these results identify reward integration via a continuous ramping process in frontal cortex as a likely candidate for the mechanism by which the mammalian brain solves patch foraging problems.},
	url = {https://www.biorxiv.org/content/10.1101/2023.09.05.556267v1},
	author = {Bukwich, Michael and Malcolm G. Campbell and David Zoltowski and Lyle Kingsbury and Momchil S. Tomov and Joshua Stern and HyungGoo R. Kim and Jan Drugowitsch and Scott W. Linderman and Uchida, Naoshige}
}
@article {1698816,
	title = {A Hypothalamic Circuit Underlying the Dynamic Control of Social Homeostasis},
	journal = {bioRxiv},
	year = {2023},
	pages = {2023.05.19.540391},
	abstract = {Social grouping increases survival in many species, including humans1,2. By contrast, social isolation generates an aversive state (loneliness) that motivates social seeking and heightens social interaction upon reunion3-5. The observed rebound in social interaction triggered by isolation suggests a homeostatic process underlying the control of social drive, similar to that observed for physiological needs such as hunger, thirst or sleep3,6. In this study, we assessed social responses in multiple mouse strains and identified the FVB/NJ line as exquisitely sensitive to social isolation. Using FVB/NJ mice, we uncovered two previously uncharacterized neuronal populations in the hypothalamic preoptic nucleus that are activated during social isolation and social rebound and that orchestrate the behavior display of social need and social satiety, respectively. We identified direct connectivity between these two populations of opposite function and with brain areas associated with social behavior, emotional state, reward, and physiological needs, and showed that animals require touch to assess the presence of others and fulfill their social need, thus revealing a brain-wide neural system underlying social homeostasis. These findings offer mechanistic insight into the nature and function of circuits controlling instinctive social need and for the understanding of healthy and diseased brain states associated with social context.},
	url = {https://www.biorxiv.org/content/10.1101/2023.05.19.540391v1.abstract},
	author = {Ding Liu and Mostafizur Rahman and Autumn Johnson and Iku Tsutsui-Kimura and Nicolai Pena and Mustafa Talay and Brandon L. Logeman and Samantha Finkbeiner and Choi, Seungwon and Athena CapoBattaglia and Ishmail Abdus-Saboor and David D. Ginty and Uchida, Naoshige and Watabe-Uchida, Mitsuko and Dulac, Catherine}
}
@article {1698811,
	title = {Spontaneous behaviour is structured by reinforcement without explicit reward},
	journal = {Nature},
	volume = {617},
	number = {7946},
	year = {2023},
	pages = {108-117},
	abstract = {Spontaneous animal behaviour is built from action modules that are concatenated by the brain into sequences1,2. However, the neural mechanisms that guide the composition of naturalistic, self-motivated behaviour remain unknown. Here we show that dopamine systematically fluctuates in the dorsolateral striatum (DLS) as mice spontaneously express sub-second behavioural modules, despite the absence of task structure, sensory cues or exogenous reward. Photometric recordings and calibrated closed-loop optogenetic manipulations during open field behaviour demonstrate that DLS dopamine fluctuations increase sequence variation over seconds, reinforce the use of associated behavioural modules over minutes, and modulate the vigour with which modules are expressed, without directly influencing movement initiation or moment-to-moment kinematics. Although the reinforcing effects of optogenetic DLS dopamine manipulations vary across behavioural modules and individual mice, these differences are well predicted by observed variation in the relationships between endogenous dopamine and module use. Consistent with the possibility that DLS dopamine fluctuations act as a teaching signal, mice build sequences during exploration as if to maximize dopamine. Together, these findings suggest a model in which the same circuits and computations that govern action choices in structured tasks have a key role in sculpting the content of unconstrained, high-dimensional, spontaneous behaviour.},
	url = {https://www-nature-com.ezp-prod1.hul.harvard.edu/articles/s41586-022-05611-2},
	author = {Markowitz, Jeffrey E and Gillis, Winthrop F and Maya Jay and Jeffrey Wood and Ryley W Harris and Robert Cieszkowski and Scott, Rebecca and Brann, David and Koveal, Dorothy and Tomasz Kula and Caleb Weinreb and Mohammed Abdal Monium Osman and Sandra Romero Pinto and Uchida, Naoshige and Linderman, Scott W and Sabatini, Bernardo L and Datta, Sandeep Robert}
}
@article {1698801,
	title = {Emergence of belief-like representations through reinforcement learning},
	journal = {PLoS Comput Biol.},
	volume = {19},
	number = {9},
	year = {2023},
	pages = {e1011067},
	abstract = {To behave adaptively, animals must learn to predict future reward, or value. To do this, animals are thought to learn reward predictions using reinforcement learning. However, in contrast to classical models, animals must learn to estimate value using only incomplete state information. Previous work suggests that animals estimate value in partially observable tasks by first forming "beliefs"-optimal Bayesian estimates of the hidden states in the task. Although this is one way to solve the problem of partial observability, it is not the only way, nor is it the most computationally scalable solution in complex, real-world environments. Here we show that a recurrent neural network (RNN) can learn to estimate value directly from observations, generating reward prediction errors that resemble those observed experimentally, without any explicit objective of estimating beliefs. We integrate statistical, functional, and dynamical systems perspectives on beliefs to show that the RNN{\textquoteright}s learned representation encodes belief information, but only when the RNN{\textquoteright}s capacity is sufficiently large. These results illustrate how animals can estimate value in tasks without explicitly estimating beliefs, yielding a representation useful for systems with limited capacity.},
	url = {https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1011067},
	author = {Jay A Hennig and Sandra A Romero Pinto and Takahiro Yamaguchi and Linderman, Scott W and Uchida, Naoshige and Gershman, Samuel J}
}
@article {1654687,
	title = {Striatal dopamine explains novelty-induced behavioral dynamics and individual variability in threat prediction},
	journal = {Neuron},
	volume = {(Advance online)},
	year = {2022},
	abstract = {Animals both explore and avoid novel objects in the environment, but the neural mechanisms that underlie these behaviors and their dynamics remain uncharacterized. Here, we used multi-point tracking (DeepLabCut) and behavioral segmentation (MoSeq) to characterize the behavior of mice freely interacting with a novel object. Novelty elicits a characteristic sequence of behavior, starting with investigatory approach and culminating in object engagement or avoidance. Dopamine in the tail of the striatum (TS) suppresses engagement, and dopamine responses were predictive of individual variability in behavior. Behavioral dynamics and individual variability are explained by a reinforcement-learning (RL) model of threat prediction in which behavior arises from a novelty-induced initial threat prediction (akin to {\textquotedblleft}shaping bonus{\textquotedblright}) and a threat prediction that is learned through dopamine-mediated threat prediction errors. These results uncover an algorithmic similarity between reward- and threat-related dopamine sub-systems.},
	url = {https://www-sciencedirect-com.ezp-prod1.hul.harvard.edu/science/article/pii/S0896627322007589?via\%3Dihub},
	author = {Korleki Akiti and Iku Tsutsui-Kimura and Yudi Xie and Mathis, Alexander and Jeffrey E. Markowitz and Rockwell Anyoha and Datta, Sandeep Robert and Mathis, Mackenzie Weygandt and Uchida, Naoshige and Watabe-Uchida, Mitsuko}
}
@article {1630206,
	title = {Dynamical management of potential threats regulated by dopamine and direct- and indirect-pathway neurons in the tail of the striatum},
	journal = {bioRxiv},
	year = {2022},
	abstract = {Avoiding potential threats before experiencing an actual outcome is critical to prevent a disaster. Here we examined roles of the tail of the striatum (TS) and its dopamine input in threat management. Mice were presented with a potential threat (a moving object) while pursuing rewards. Mice initially failed to obtain rewards, but gradually successfully obtained rewards in later trials. We show that the initial failures depended on dopamine and direct-pathway neurons in TS, and variability in failure rate across trials and individuals was positively correlated with the activity of these neurons. In contrast, indirect-pathway neurons in TS were critical for eventual improvement in reward acquisition, and their activity was positively correlated with successful reward acquisition. These results demonstrate that direct- and indirect-pathway TS neurons promote and suppress threat avoidance, respectively, at different stages, providing a mechanism for overcoming a potential threat while maintaining the threat estimates.},
	author = {Iku Tsutsui-Kimura and Uchida, Naoshige and Watabe-Uchida, Mitsuko}
}
@article {1625752,
	title = {The role of state uncertainty in the dynamics of dopamine},
	journal = {Curr Biol},
	volume = {32},
	number = {5},
	year = {2022},
	pages = {1077-1087},
	abstract = {Reinforcement learning models of the basal ganglia map the phasic dopamine signal to reward prediction errors (RPEs). Conventional models assert that, when a stimulus predicts a reward with fixed delay, dopamine activity during the delay should converge to baseline through learning. However, recent studies have found that dopamine ramps up before reward in certain conditions even after learning, thus challenging the conventional models. In this work, we show that sensory feedback causes an unbiased learner to produce RPE ramps. Our model predicts that when feedback gradually decreases during a trial, dopamine activity should resemble a {\textquotedblleft}bump,{\textquotedblright} whose ramp-up phase should, furthermore, be greater than that of conditions where the feedback stays high. We trained mice on a virtual navigation task with varying brightness, and both predictions were empirically observed. In sum, our theoretical and experimental results reconcile the seemingly conflicting data on dopamine behaviors under the RPE hypothesis.},
	url = {https://www-sciencedirect-com.ezp-prod1.hul.harvard.edu/science/article/pii/S0960982222000367?via\%3Dihub},
	author = {John G. Mikhael and HyungGoo R. Kim and Uchida, Naoshige and Samuel J. Gershman}
}
@article {1548566,
	title = {A gradual temporal shift of dopamine responses mirrors the progression of temporal difference error in machine learning},
	journal = {Nat. Neurosci.},
	volume = {25},
	year = {2022},
	pages = {1082-1092},
	abstract = {A large body of evidence has indicated that the phasic responses of midbrain dopamine neurons show a remarkable similarity to a type of teaching signal (temporal difference (TD) error) used in machine learning. However, previous studies failed to observe a key prediction of this algorithm: that when an agent associates a cue and a reward that are separated in time, the timing of dopamine signals should gradually move backward in time from the time of the reward to the time of the cue over multiple trials. Here we demonstrate that such a gradual shift occurs both at the level of dopaminergic cellular activity and dopamine release in the ventral striatum in mice. Our results establish a long-sought link between dopaminergic activity and the TD learning algorithm, providing fundamental insights into how the brain associates cues and rewards that are separated in time.},
	url = {https://www.nature.com/articles/s41593-022-01109-2},
	author = {Amo, R. and Sara Matias and Yamanaka, A. and Tanaka, K.F. and Uchida, N. and Watabe-Uchida, M.}
}
@article {1548564,
	title = {Dopamine signals as temporal difference errors: recent advances},
	journal = {Curr Opin Neurobiol},
	volume = {67},
	year = {2021},
	pages = {95-105},
	abstract = {In the brain, dopamine is thought to drive reward-based learning by signaling temporal difference reward prediction errors (TD errors), a {\textquoteleft}teaching signal{\textquoteright} used to train computers. Recent studies using optogenetic manipulations have provided multiple pieces of evidence supporting that phasic dopamine signals function as TD errors. Furthermore, novel experimental results have indicated that when the current state of the environment is uncertain, dopamine neurons compute TD errors using {\textquoteleft}belief states{\textquoteright} or a probability distribution over potential states. It remains unclear how belief states are computed but emerging evidence suggests involvement of the prefrontal cortex and the hippocampus. These results refine our understanding of the role of dopamine in learning and the algorithms by which dopamine functions in the brain.},
	url = {https://doi.org/10.1016/j.conb.2020.08.014},
	author = {Starkweather, C.K. and Uchida, N.}
}
@article {1548579,
	title = {A unified framework for dopamine signals across timescales},
	journal = {Cell},
	volume = {183},
	number = {6},
	year = {2020},
	pages = {1600-1616},
	abstract = {Rapid phasic activity of midbrain dopamine neurons is thought to signal reward prediction errors (RPEs), resembling temporal difference errors used in machine learning. However, recent studies describing slowly increasing dopamine signals have instead proposed that they represent state values and arise independent from somatic spiking activity. Here we developed experimental paradigms using virtual reality that disambiguate RPEs from values. We examined dopamine circuit activity at various stages, including somatic spiking, calcium signals at somata and axons, and striatal dopamine concentrations. Our results demonstrate that ramping dopamine signals are consistent with RPEs rather than value, and this ramping is observed at all stages examined. Ramping dopamine signals can be driven by a dynamic stimulus that indicates a gradual approach to a reward. We provide a unified computational understanding of rapid phasic and slowly ramping dopamine signals: dopamine neurons perform a derivative-like computation over values on a moment-by-moment basis.},
	url = {https://authors.elsevier.com/a/1c8r1L7PXcLwa},
	author = {Kim, H.R. and Malik, A.N. and Bech, P. and Tsutsui-Kimura, I. and F. Sun and Zhang, Y. and Y. Li and Watabe-Uchida, M. and S.J. Gershman and Uchida, N.}
}
@inbook {1548569,
	title = {Dopamine reward prediction errors: The interplay between experiments and theory},
	booktitle = {The Cognitive Neuroscience},
	year = {2020},
	publisher = {MIT Press},
	organization = {MIT Press},
	abstract = {Reinforcement-learning theories provide a normative perspective on learning and decision-making. In the 1990s, neurophysiology experiments revealed an exceptional correspondence between the activity of midbrain dopamine neurons and the reward prediction error (RPE) signal used to train computers in a reinforcement-learning algorithm called temporal difference (TD) learning. Studies of midbrain dopamine neurons play a pivotal role at the interface of empirical and theoretical studies. A theoretical framework for reinforcementlearning has facilitated the interpretation of neurophysiology data and has guided the design of future studies. Here we discuss recent developments in the interplay between experimental findings and theories of dopamine signaling. In particular, recent studies emphasize the importance of state uncertainty in the neurobiological implementation of reinforcement learning.},
	url = {https://mitpress.mit.edu/books/cognitive-neurosciences-sixth-edition},
	author = {Starkweather, C.K. and Uchida, N.}
}
@article {1548565,
	title = {Distinct temporal difference error signals in dopamine axons in three regions of the striatum in a decision-making task},
	journal = {eLife},
	volume = {9:e62390},
	year = {2020},
	abstract = {Different regions of the striatum regulate different types of behavior. However, how dopamine signals differ across striatal regions and how dopamine regulates different behaviors remain unclear. Here, we compared dopamine axon activity in the ventral, dorsomedial, and dorsolateral striatum, while mice performed a perceptual and value-based decision task. Surprisingly, dopamine axon activity was similar across all three areas. At a glance, the activity multiplexed different variables such as stimulus-associated values, confidence and reward feedback at different phases of the task. Our modeling demonstrates, however, that these modulations can be inclusively explained by moment-by-moment changes in the expected reward, i.e. the temporal difference error. A major difference between areas was the overall activity level of reward responses: reward responses in dorsolateral striatum were positively shifted, lacking inhibitory responses to negative prediction errors. The differences in dopamine signals put specific constraints on the properties of behaviors controlled by dopamine in these regions.},
	url = {https://elifesciences.org/articles/62390},
	author = {Tsutsui-Kimura, I. and Matsumoto, H. and Akiti, K and Yamada, M.M. and Uchida, N. and Watabe-Uchida, M.}
}
@article {1548563,
	title = {Distributional Reinforcement Learning in the Brain},
	journal = {Trends Neurosci.},
	volume = {43},
	year = {2020},
	pages = {980-997},
	abstract = {Learning about rewards and punishments is critical for survival. Classical studies have demonstrated an impressive correspondence between the firing of dopamine neurons in the mammalian midbrain and the reward prediction errors of reinforcement learning algorithms, which express the difference between actual reward and predicted mean reward. However, it may be advantageous to learn not only the mean but also the complete distribution of potential rewards. Recent advances in machine learning have revealed a biologically plausible set of algorithms for reconstructing this reward distribution from experience. Here, we review the mathematical foundations of these algorithms as well as initial evidence for their neurobiological implementation. We conclude by highlighting outstanding questions regarding the circuit computation and behavioral readout of these distributional codes.},
	url = {https://doi.org/10.1016/j.tins.2020.09.004},
	author = {Lowet, A.S. and Zheng, Q. and Matias, S. and Drugowitsch, J. and Uchida, N.}
}
@article {1548561,
	title = {Reinforcement biases subsequent perceptual decisions when confidence is low: a widespread behavioral phenomenon},
	journal = {eLife},
	volume = {9},
	year = {2020},
	abstract = {Learning from successes and failures often improves the quality of subsequent decisions. Past outcomes, however, should not influence purely perceptual decisions after task acquisition is complete since these are designed so that only sensory evidence determines the correct choice. Yet, numerous studies report that outcomes can bias perceptual decisions, causing spurious changes in choice behavior without improving accuracy. Here we show that the effects of reward on perceptual decisions are principled: past rewards bias future choices specifically when previous choice was difficult and hence decision confidence was low. We identified this phenomenon in six datasets from four laboratories, across mice, rats, and humans, and sensory modalities from olfaction and audition to vision. We show that this choice-updating strategy can be explained by reinforcement learning models incorporating statistical decision confidence into their teaching signals. Thus, reinforcement learning mechanisms are continually engaged to produce systematic adjustments of choices even in well-learned perceptual decisions in order to optimize behavior in an uncertain world.},
	url = {https://elifesciences.org/articles/49834},
	author = {Lak, A. and Hueske, E. and Hirokawa, J. and Masset, P. and Ott, T. and Urai, A.E. and Donner, T.H. and Carandini, M. and Tonegawa, S. and Uchida, N. and Kepecs, A.}
}
@article {1481172,
	title = {A distributional code for value in dopamine-based reinforcement learning},
	journal = {Nature},
	volume = {577},
	number = {7792},
	year = {2020},
	pages = {671-675},
	abstract = {Since its introduction, the reward prediction error theory of dopamine has explained a wealth of empirical phenomena, providing a unifying framework for understanding the representation of reward and value in the brain1,2,3. According to the now canonical theory, reward predictions are represented as a single scalar quantity, which supports learning about the expectation, or mean, of stochastic outcomes. Here we propose an account of dopamine-based reinforcement learning inspired by recent artificial intelligence research on distributional reinforcement learning4,5,6. We hypothesized that the brain represents possible future rewards not as a single mean, but instead as a probability distribution, effectively representing multiple future outcomes simultaneously and in parallel. This idea implies a set of empirical predictions, which we tested using single-unit recordings from mouse ventral tegmental area. Our findings provide strong evidence for a neural realization of distributional reinforcement learning.},
	url = {https://www-nature-com.ezp-prod1.hul.harvard.edu/articles/s41586-019-1924-6},
	author = {Dabney, W. and Kurth-Nelson, Z. and Uchida, N. and Starkweather, C.K. and Hassabis, D. and Munos, R. and Botvinick, M.}
}
@article {1481171,
	title = {Believing in dopamine},
	journal = {Nat Rev Neurosci. },
	volume = {20},
	number = {11},
	year = {2019},
	pages = {703-714},
	abstract = {Midbrain dopamine signals are widely thought to report reward prediction errors that drive learning in the basal ganglia. However, dopamine has also been implicated in various probabilistic computations, such as encoding uncertainty and controlling exploration. Here, we show how these different facets of dopamine signalling can be brought together under a common reinforcement learning framework. The key idea is that multiple sources of uncertainty impinge on reinforcement learning computations: uncertainty about the state of the environment, the parameters of the value function and the optimal action policy. Each of these sources plays a distinct role in the prefrontal cortex{\textendash}basal ganglia circuit for reinforcement learning and is ultimately reflected in dopamine activity. The view that dopamine plays a central role in the encoding and updating of beliefs brings the classical prediction error theory into alignment with more recent theories of Bayesian reinforcement learning.},
	url = {https://www-nature-com.ezp-prod1.hul.harvard.edu/articles/s41583-019-0220-7},
	author = {Uchida, N. and S.J. Gershman}
}
@article {1424084,
	title = {Multiple Dopamine Systems: Weal and Woe of Dopamine},
	journal = {Cold Spring Harb Symp Quant Biol},
	volume = {83},
	year = {2018},
	pages = {83-95},
	abstract = {The ability to predict future outcomes increases the fitness of the animal. Decades of research have shown that dopamine neurons broadcast reward prediction error (RPE) signals{\textemdash}the discrepancy between actual and predicted reward{\textemdash}to drive learning to predict future outcomes. Recent studies have begun to show, however, that dopamine neurons are more diverse than previously thought. In this review, we will summarize a series of our studies that have shown unique properties of dopamine neurons projecting to the posterior {\textquotedblleft}tail{\textquotedblright} of the striatum (TS) in terms of anatomy, activity, and function. Specifically, TS-projecting dopamine neurons are activated by a subset of negative events including threats from a novel object, send prediction errors for external threats, and reinforce avoidance behaviors. These results indicate that there are at least two axes of dopamine-mediated reinforcement learning in the brain{\textemdash}one learning from canonical RPEs and another learning from threat prediction errors. We argue that the existence of multiple learning systems is an adaptive strategy that makes possible each system optimized for its own needs. The compartmental organization in the mammalian striatum resembles that of a dopamine-recipient area in insects (mushroom body), pointing to a principle of dopamine function conserved across phyla.},
	url = {http://symposium.cshlp.org.ezp-prod1.hul.harvard.edu/content/early/2019/02/20/sqb.2018.83.037648.abstract},
	author = {Watabe-Uchida, Mitsuko and Uchida, Naoshige}
}
@article {1408064,
	title = {Editorial overview: Neurobiology of behavior},
	journal = {Current opinion in neurobiology},
	volume = {49},
	number = {April 2018},
	year = {2018},
	pages = {iv-ix},
	url = {https://www.sciencedirect.com/science/article/abs/pii/S0959438818300370},
	author = {Tye, Kay M and Uchida, Naoshige}
}
@article {1408063,
	title = {Belief state representation in the dopamine system},
	journal = {Nature communications},
	volume = {9},
	number = {1},
	year = {2018},
	pages = {1891},
	abstract = {Learning to predict future outcomes is critical for driving appropriate behaviors. Reinforcement learning (RL) models have successfully accounted for such learning, relying on reward prediction errors (RPEs) signaled by midbrain dopamine neurons. It has been proposed that when sensory data provide only ambiguous information about which state an animal is in, it can predict reward based on a set of probabilities assigned to hypothetical states (called\&nbsp;the belief state). Here we examine how dopamine RPEs and subsequent learning are regulated under state uncertainty. Mice are first trained in a task with two potential states defined by different reward amounts. During testing, intermediate-sized rewards are given in rare trials. Dopamine activity is a non-monotonic function of reward size, consistent with RL models operating on belief states. Furthermore, the magnitude of dopamine responses quantitatively predicts changes in behavior. These results establish the critical role of state inference in RL.},
	url = {https://www.nature.com/articles/s41467-018-04397-0},
	author = {Babayan, Benedicte M and Uchida, Naoshige and Gershman, Samuel J}
}
@article {1408061,
	title = {Dopamine neurons projecting to the posterior striatum reinforce avoidance of threatening stimuli},
	journal = {Nature neuroscience},
	volume = {21},
	number = {10},
	year = {2018},
	pages = {14-21},
	abstract = {Midbrain dopamine neurons are well known for their role in reward-based reinforcement learning. We found that the activity of dopamine axons in the posterior tail of the striatum (TS) scaled with the novelty and intensity of external stimuli, but did not encode reward value. We demonstrated that the ablation of TS-projecting dopamine neurons specifically inhibited avoidance of novel or high-intensity stimuli without affecting animals{\textquoteright} initial avoidance responses, suggesting a role in reinforcement rather than simply in avoidance itself. Furthermore, we found that animals avoided optogenetic activation of dopamine axons in TS during a choice task and that this stimulation could partially reinstate avoidance of a familiar object. These results suggest that TS-projecting dopamine neurons reinforce avoidance of threatening stimuli. More generally, our results indicate that there are at least two axes of reinforcement learning using dopamine in the striatum: one based on value and one based on external threat.},
	author = {Menegas, William and Korleki Akiti and Ryunosuke Amo and Uchida, Naoshige and Watabe-Uchida, Mitsuko}
}
@article {1309419,
	title = {The Medial Prefrontal Cortex Shapes Dopamine Reward Prediction Errors under State Uncertainty},
	journal = {Neuron},
	volume = {98},
	number = {3},
	year = {2018},
	pages = {616-629},
	abstract = {Animals make predictions based on currently available information. In natural settings, sensory cues may not reveal complete information, requiring the animal to infer the {\textquotedblleft}hidden state{\textquotedblright} of the environment. The brain structures important in hidden state\&nbsp;inference remain unknown. A previous study showed that\&nbsp;midbrain\&nbsp;dopamine\&nbsp;neurons exhibit distinct response patterns depending on whether reward is delivered in 100\% (task 1) or 90\% of trials (task 2) in a\&nbsp;classical conditioning\&nbsp;task. Here we found that inactivation of the\&nbsp;medial prefrontal cortex\&nbsp;(mPFC) affected\&nbsp;dopaminergic\&nbsp;signaling in task 2, in which the hidden state must be inferred ({\textquotedblleft}will reward come or not?{\textquotedblright}), but not in task 1, where the state was known with certainty. Computational modeling suggests that the effects of inactivation are best explained by a circuit in which the mPFC conveys inference over hidden states to the\&nbsp;dopamine system.},
	url = {https://www-sciencedirect-com.ezp-prod1.hul.harvard.edu/science/article/pii/S0896627318302423?via\%3Dihub},
	author = {Clara Kwan Starkweather and Samuel J. Gershman and Uchida, Naoshige}
}
@article {1309418,
	title = {Functional circuit architecture underlying parental behaviour},
	journal = {Nature},
	volume = {556},
	year = {2018},
	pages = {326-331},
	abstract = {Parenting is essential for the survival and wellbeing of mammalian offspring. However, we lack a circuit-level understanding of how distinct components of this behaviour are coordinated. Here we investigate how galanin-expressing neurons in the medial preoptic area (MPOAGal) of the hypothalamus coordinate motor, motivational, hormonal and social aspects of parenting in mice. These neurons integrate inputs from a large number of brain areas and the activation of these inputs depends on the animal{\textquoteright}s sex and reproductive state. Subsets of MPOAGal\&nbsp;neurons form discrete pools that are defined by their projection sites. While the MPOAGalpopulation is active during all episodes of parental behaviour, individual pools are tuned to characteristic aspects of parenting. Optogenetic manipulation of MPOAGal\&nbsp;projections mirrors this specificity, affecting discrete parenting components. This functional organization, reminiscent of the control of motor sequences by pools of spinal cord neurons, provides a new model for how discrete elements of a social behaviour are generated at the circuit level.},
	author = {Johannes Kohl and Benedicte M. Babayan and Nimrod D. Rubinstein and Anita E. Autry and Brenda Marin-Rodriguez and Kapoor, Vikrant and Kazunari Miyamichi and Larry S. Zweifel and Liqun Luo and Uchida, Naoshige and Dulac, Catherine}
}
@article {1408065,
	title = {Serotonin: Slow motion},
	journal = {eLife},
	volume = {6},
	year = {2017},
	pages = {e24792},
	abstract = {Optogenetic stimulation of serotonin neurons in the dorsal raphe causes mice to move more slowly without causing any apparent motor deficits or anxiety-like effects.},
	url = {https://elifesciences.org/articles/24792},
	author = {Jeremiah Y. Cohen and Uchida, Naoshige}
}
@article {1309417,
	title = {A Self-Killing Rabies Virus That Leaves a Trace on the DNA},
	journal = {Trends Neurosci.},
	volume = {40},
	number = {10},
	year = {2017},
	pages = {589-591},
	abstract = {Although modified\&nbsp;rabies viruses\&nbsp;have emerged as a powerful tool for tracing the inputs to genetically defined populations of neurons, the toxicity of the virus has limited its utility. A recent study employed a self-inactivating rabies (SiR) virus that enables recording or manipulation of targeted neurons for months.},
	url = {https://www-sciencedirect-com.ezp-prod1.hul.harvard.edu/science/article/pii/S0166223617301625?via\%3Dihub},
	author = {Menegas, William and Uchida, Naoshige and Watabe-Uchida, Mitsuko}
}
@article {1309416,
	title = {Neural circuitry of reward prediction error},
	journal = {Annu Rev Neurosci.},
	volume = {40},
	year = {2017},
	pages = {373-394},
	abstract = {Dopamine neurons facilitate learning by calculating reward prediction error, or the difference between expected and actual reward. Despite two decades of research, it remains unclear how dopamine neurons make this calculation. Here we review studies that tackle this problem from a diverse set of approaches, from anatomy to electrophysiology to computational modeling and behavior. Several patterns emerge from this synthesis: that dopamine neurons themselves calculate reward prediction error, rather than inherit it passively from upstream regions; that they combine multiple separate and redundant inputs, which are themselves interconnected in a dense recurrent network; and that despite the complexity of inputs, the output from dopamine neurons is remarkably homogeneous and robust. The more we study this simple arithmetic computation, the knottier it appears to be, suggesting a daunting (but stimulating) path ahead for neuroscience more generally.},
	url = {https://www-annualreviews-org.ezp-prod1.hul.harvard.edu/doi/full/10.1146/annurev-neuro-072116-031109?url_ver=Z39.88-2003\&rfr_id=ori\%3Arid\%3Acrossref.org\&rfr_dat=cr_pub\%3Dpubmed},
	author = {Watabe-Uchida, Mitsuko and Eshel, Neir and Uchida, Naoshige}
}
@article {1036881,
	title = {Slow motion},
	journal = {Elife},
	volume = {6},
	year = {2017},
	month = {2017 Feb 14},
	abstract = {Optogenetic stimulation of serotonin neurons in the dorsal raphe causes mice to move more slowly without causing any apparent motor deficits or anxiety-like effects.},
	issn = {2050-084X},
	doi = {10.7554/eLife.24792},
	author = {Uchida, Naoshige and Jeremiah Y. Cohen}
}
@article {1036871,
	title = {Dopamine reward prediction errors reflect hidden-state inference across time},
	journal = {Nat Neurosci},
	volume = {20},
	number = {4},
	year = {2017},
	month = {2017 Mar 06},
	pages = {581-589},
	abstract = {Midbrain dopamine neurons signal reward prediction error (RPE), or actual minus expected reward. The temporal difference (TD) learning model has been a cornerstone in understanding how dopamine RPEs could drive associative learning. Classically, TD learning imparts value to features that serially track elapsed time relative to observable stimuli. In the real world, however, sensory stimuli provide ambiguous information about the hidden state of the environment, leading to the proposal that TD learning might instead compute a value signal based on an inferred distribution of hidden states (a {\textquoteright}belief state{\textquoteright}). Here we asked whether dopaminergic signaling supports a TD learning framework that operates over hidden states. We found that dopamine signaling showed a notable difference between two tasks that differed only with respect to whether reward was delivered in a deterministic manner. Our results favor an associative learning rule that combines cached values with hidden-state inference.},
	issn = {1546-1726},
	doi = {10.1038/nn.4520},
	author = {Starkweather, Clara Kwon and Babayan, Benedicte M and Uchida, Naoshige and Gershman, Samuel J}
}
@article {1036861,
	title = {Somatosensory Cortex Plays an Essential Role in Forelimb Motor Adaptation in Mice},
	journal = {Neuron},
	volume = {93},
	number = {6},
	year = {2017},
	month = {2017 Mar 22},
	pages = {1493-1503.e6},
	abstract = {Our motor outputs are constantly re-calibrated to adapt to systematic perturbations. This motor adaptation is thought to depend on the ability to form a memory of a systematic perturbation, often called an internal model. However, the mechanisms underlying the formation, storage, and expression of such models remain unknown. Here, we developed a mouse model to study forelimb adaptation to force field perturbations. We found that temporally precise photoinhibition of somatosensory cortex (S1) applied concurrently with the force field abolished the ability to update subsequent motor commands needed to~reduce motor errors. This S1 photoinhibition did not impair basic motor patterns, post-perturbation completion of the action, or their performance in a reward-based learning task. Moreover, S1 photoinhibition after partial adaptation blocked further adaptation, but did not affect the expression of already-adapted motor commands. Thus, S1 is critically involved in updating the memory about the perturbation that is essential for forelimb motor adaptation.},
	issn = {1097-4199},
	doi = {10.1016/j.neuron.2017.02.049},
	author = {Mathis, Mackenzie Weygandt and Mathis, Alexander and Uchida, Naoshige}
}
@article {10.7554/eLife.21886,
	title = {Opposite initialization to novel cues in dopamine signaling in ventral and posterior striatum in mice},
	journal = {eLife},
	volume = {6},
	year = {2017},
	pages = {e21886},
	publisher = {eLife Sciences Publications, Ltd},
	abstract = {Dopamine neurons are thought to encode novelty in addition to reward prediction error (the discrepancy between actual and predicted values). In this study, we compared dopamine activity across the striatum using fiber fluorometry in mice. During classical conditioning, we observed opposite dynamics in dopamine axon signals in the ventral striatum ({\textquoteleft}VS dopamine{\textquoteright}) and the posterior tail of the striatum ({\textquoteleft}TS dopamine{\textquoteright}). TS dopamine showed strong excitation to novel cues, whereas VS dopamine showed no responses to novel cues until they had been paired with a reward. TS dopamine cue responses decreased over time, depending on what the cue predicted. Additionally, TS dopamine showed excitation to several types of stimuli including rewarding, aversive, and neutral stimuli whereas VS dopamine showed excitation only to reward or reward-predicting cues. Together, these results demonstrate that dopamine novelty signals are localized in TS along with general salience signals, while VS dopamine reliably encodes reward prediction error.},
	keywords = {Dopamine, Learning, novelty, Reward, striatum},
	issn = {2050-084X},
	doi = {10.7554/eLife.21886},
	url = {https://dx.doi.org/10.7554/eLife.21886},
	author = {Menegas, William and Babayan, Benedicte M and Uchida, Naoshige and Watabe-Uchida, Mitsuko}
}
@article {1005766,
	title = {Distributed and Mixed Information in Monosynaptic Inputs to Dopamine Neurons},
	journal = {Neuron},
	volume = {91},
	number = {6},
	year = {2016},
	pages = {1374-89},
	abstract = {Dopamine\&nbsp;neurons encode the difference between actual and predicted reward, or reward prediction error (RPE). Although many models have been proposed to account for this computation, it has been\&nbsp;difficult to test these models experimentally. Here we established an awake electrophysiological recording system, combined with\&nbsp;rabies virusand\&nbsp;optogenetic\&nbsp;cell-type identification, to characterize the firing patterns of monosynaptic inputs to dopamine neurons while mice performed\&nbsp;classical conditioningtasks. We found that each variable required to\&nbsp;compute RPE, including actual and predicted reward, was distributed in input neurons in multiple brain areas. Further, many input neurons across brain areas signaled combinations of these variables. These results demonstrate that even simple arithmetic computations such as RPE are not localized in specific brain areas but, rather, distributed across multiple nodes in a brain-wide network. Our systematic method to examine both activity and connectivity revealed unexpected redundancy for a simple computation in the brain.},
	url = {http://www.sciencedirect.com.ezp-prod1.hul.harvard.edu/science/article/pii/S0896627316305104},
	author = {Tian, J. and R. Huang and Cohen, J.Y. and Osakada, F. and Kobak, D. and Machens, C. and Callaway, E.M. and Uchida, N. and Watabe-Uchida, M.}
}
@article {10.7554/eLife.17328,
	title = {Midbrain dopamine neurons signal aversion in a reward-context-dependent manner},
	journal = {eLife},
	volume = {5},
	year = {2016},
	month = {oct},
	pages = {e17328},
	publisher = {eLife Sciences Publications, Ltd},
	abstract = {Dopamine is thought to regulate learning from appetitive and aversive events. Here we examined how optogenetically-identified dopamine neurons in the lateral ventral tegmental area of mice respond to aversive events in different conditions. In low reward contexts, most dopamine neurons were exclusively inhibited by aversive events, and expectation reduced dopamine neurons{\textquoteright} responses to reward and punishment. When a single odor predicted both reward and punishment, dopamine neurons{\textquoteright} responses to that odor reflected the integrated value of both outcomes. Thus, in low reward contexts, dopamine neurons signal value prediction errors (VPEs) integrating information about both reward and aversion in a common currency. In contrast, in high reward contexts, dopamine neurons acquired a short-latency excitation to aversive events that masked their VPE signaling. Our results demonstrate the importance of considering the contexts to examine the representation in dopamine neurons and uncover different modes of dopamine signaling, each of which may be adaptive for different environments.},
	keywords = {aversion, Context, Dopamine, eye blink, prediction error, value},
	issn = {2050-084X},
	doi = {10.7554/eLife.17328},
	url = {https://dx.doi.org/10.7554/eLife.17328},
	author = {Matsumoto, Hideyuki and Ju Tian and Uchida, Naoshige and Watabe-Uchida, Mitsuko}
}
@article {10.7554/eLife.10989,
	title = {Demixed principal component analysis of neural population data},
	journal = {eLife},
	volume = {5},
	year = {2016},
	month = {apr},
	pages = {e10989},
	publisher = {eLife Sciences Publications, Ltd},
	abstract = {Neurons in higher cortical areas, such as the prefrontal cortex, are often tuned to a variety of sensory and motor variables, and are therefore said to display mixed selectivity. This complexity of single neuron responses can obscure what information these areas represent and how it is represented. Here we demonstrate the advantages of a new dimensionality reduction technique, demixed principal component analysis (dPCA), that decomposes population activity into a few components. In addition to systematically capturing the majority of the variance of the data, dPCA also exposes the dependence of the neural representation on task parameters such as stimuli, decisions, or rewards. To illustrate our method we reanalyze population data from four datasets comprising different species, different cortical areas and different experimental tasks. In each case, dPCA provides a concise way of visualizing the data that summarizes the task-dependent features of the population response in a single figure.},
	keywords = {dimensionality reduction, population activity, Prefrontal Cortex, Principal Component Analysis},
	issn = {2050-084X},
	doi = {10.7554/eLife.10989},
	url = {https://dx.doi.org/10.7554/eLife.10989},
	author = {Kobak, Dmitry and Brendel, Wieland and Constantinidis, Christos and Feierstein, Claudia E and Kepecs, Adam and Mainen, Zachary F and Qi, Xue-Lian and Romo, Ranulfo and Uchida, Naoshige and Machens, Christian K},
	editor = {van Rossum, Mark CW}
}
@article {739701,
	title = {Dopamine neurons share common response function for reward prediction error.},
	journal = {Nat Neurosci},
	volume = {19},
	number = {3},
	year = {2016},
	month = {2016 Mar},
	pages = {479-86},
	abstract = {Dopamine neurons are thought to signal reward prediction error, or the difference between actual and predicted reward. How dopamine neurons jointly encode this information, however, remains unclear. One possibility is that different neurons specialize in different aspects of prediction error; another is that each neuron calculates prediction error in the same way. We recorded from optogenetically identified dopamine neurons in the lateral ventral tegmental area (VTA) while mice performed classical conditioning tasks. Our tasks allowed us to determine the full prediction error functions of dopamine neurons and compare them to each other. We found marked homogeneity among individual dopamine neurons: their responses to both unexpected and expected rewards followed the same function, just scaled up or down. As a result, we were able to describe both individual and population responses using just two parameters. Such uniformity ensures robust information coding, allowing each dopamine neuron to contribute fully to the prediction error signal.},
	issn = {1546-1726},
	doi = {10.1038/nn.4239},
	author = {Eshel, Neir and Ju Tian and Bukwich, Michael and Uchida, Naoshige}
}
@article {553926,
	title = {Arithmetic and local circuitry underlying dopamine prediction errors.},
	journal = {Nature},
	volume = {525},
	number = {7568},
	year = {2015},
	month = {2015 Sep 10},
	pages = {243-6},
	abstract = {Dopamine neurons are thought to facilitate learning by comparing actual and expected reward. Despite two decades of investigation, little is known about how this comparison is made. To determine how dopamine neurons calculate prediction error, we combined optogenetic manipulations with extracellular recordings in the ventral tegmental area while mice engaged in classical conditioning. Here we demonstrate, by manipulating the temporal expectation of reward, that dopamine neurons perform subtraction, a computation that is ideal for reinforcement learning but rarely observed in the brain. Furthermore, selectively exciting and inhibiting neighbouring GABA (γ-aminobutyric acid) neurons in the ventral tegmental area reveals that these neurons are a source of subtraction: they inhibit dopamine neurons when reward is expected, causally contributing to prediction-error calculations. Finally, bilaterally stimulating ventral tegmental area GABA neurons dramatically reduces anticipatory licking to conditioned odours, consistent with an important role for these neurons in reinforcement learning. Together, our results uncover the arithmetic and local circuitry underlying dopamine prediction errors.},
	issn = {1476-4687},
	doi = {10.1038/nature14855},
	author = {Eshel, Neir and Bukwich, Michael and Rao, Vinod and Hemmelder, Vivian and Ju Tian and Uchida, Naoshige}
}
@article {tian2015habenula,
	title = {Habenula Lesions Reveal that Multiple Mechanisms Underlie Dopamine Prediction Errors},
	journal = {Neuron},
	volume = {87},
	year = {2015},
	pages = {1304{\textendash}1316},
	publisher = {Cell Press},
	author = {Ju Tian and Uchida, Naoshige}
}
@article {menegas2015dopamine,
	title = {Dopamine neurons projecting to the posterior striatum form an anatomically distinct subclass},
	journal = {eLife},
	year = {2015},
	pages = {e10032},
	publisher = {eLife Sciences Publications Limited},
	author = {Menegas, William and Bergan, Joseph F and Ogawa, Sachie K and Isogai, Yoh and Venkataraju, Kannan Umadevi and Osten, Pavel and Uchida, Naoshige and Watabe-Uchida, Mitsuko}
}
@conference {eshel2015neural,
	title = {Neural Circuit Mechanism Underlying Dopamine Prediction Errors},
	booktitle = {BIOLOGICAL PSYCHIATRY},
	volume = {77},
	year = {2015},
	publisher = {ELSEVIER SCIENCE INC 360 PARK AVE SOUTH, NEW YORK, NY 10010-1710 USA},
	organization = {ELSEVIER SCIENCE INC 360 PARK AVE SOUTH, NEW YORK, NY 10010-1710 USA},
	author = {Eshel, Neir and Uchida, Naoshige}
}
@article {Tian20151046,
	title = {Monkeys in a Prisoner{\textquoteright}s Dilemma},
	journal = {Cell},
	volume = {160},
	year = {2015},
	pages = {1046 - 1048},
	abstract = {Haroush and Williams trained pairs of monkeys to play in a prisoner{\textquoteright}s dilemma game, a model of social interactions. Recording from the dorsal anterior cingulate cortex (dACC), they find neurons whose activity reflects the anticipation of the opponent{\textquoteright}s yet unknown choice, which may be important in guiding animals{\textquoteright} performance in the game.},
	issn = {0092-8674},
	doi = {http://dx.doi.org/10.1016/j.cell.2015.02.049},
	url = {http://www.sciencedirect.com/science/article/pii/S0092867415002500},
	author = {Ju Tian and Uchida, Naoshige}
}
@article {382871,
	title = {Serotonergic neurons signal reward and punishment on multiple timescales},
	journal = {eLife},
	year = {2015},
	abstract = {Serotonin{\textquoteright}s function in the brain is unclear. One challenge in testing the numerous hypotheses about serotonin{\textquoteright}s function has been observing the activity of identified serotonergic neurons in animals engaged in behavioral tasks. We recorded the activity of dorsal raphe neurons while mice experienced a task in which rewards and punishments varied across blocks of trials. We {\textquoteright}tagged{\textquoteright} serotonergic neurons with the light-sensitive protein channelrhodopsin-2 and identified them based on their responses to light. We found three main features of serotonergic neuron activity: (1) a large fraction of serotonergic neurons modulated their tonic firing rates over the course of minutes during reward versus punishment blocks; (2) most were phasically excited by punishments; and (3) a subset was phasically excited by reward-predicting cues. By contrast, dopaminergic neurons did not show firing rate changes across blocks of trials. These results suggest that serotonergic neurons signal information about reward and punishment on multiple timescales.},
	url = {http://elifesciences.org/content/early/2015/02/25/eLife.06346},
	author = {Jeremiah Y. Cohen and Mackenzie W. Amoroso and Uchida, Naoshige}
}
@article {382876,
	title = {Bilingual neurons release glutamate and GABA},
	journal = {Nature Neuroscience},
	volume = {2014},
	year = {2014},
	url = {http://www.nature.com.ezp-prod1.hul.harvard.edu/neuro/journal/v17/n11/full/nn.3840.html},
	author = {Uchida, Naoshige}
}
@article {242181,
	title = {Organization of Monosynaptic Inputs to the Serotonin and Dopamine Neuromodulatory Systems},
	journal = {Cell Reports},
	volume = {8},
	number = {4},
	year = {2014},
	note = {doi: 10.1016/j.celrep.2014.06.042},
	pages = {1105},
	publisher = {Elsevier},
	abstract = {Serotonin and dopamine are major neuromodulators. Here, we used a modified rabies virus to identify monosynaptic inputs to serotonin neurons in the dorsal and median raphe (DR and MR). We found that inputs to DR and MR serotonin neurons are spatially shifted in the forebrain, and MR serotonin neurons receive inputs from more medial structures. Then, we compared these data with inputs to dopamine neurons in the ventral tegmental area (VTA) and substantia nigra pars compacta (SNc). We found that DR serotonin neurons receive inputs from a remarkably similar set of areas as VTA dopamine neurons apart from the striatum, which preferentially targets dopamine neurons. Our results suggest three major input streams: a medial stream regulates MR serotonin neurons, an intermediate stream regulates DR serotonin and VTA dopamine neurons, and a lateral stream regulates SNc dopamine neurons. These results provide fundamental organizational principles of afferent control for serotonin and dopamine.},
	url = {http://www.sciencedirect.com.ezp-prod1.hul.harvard.edu/science/article/pii/S2211124714005269},
	author = {Ogawa, Sachie~K. and Cohen, Jeremiah~Y. and Hwang, Dabin and Uchida, Naoshige and Watabe-Uchida, Mitsuko}
}
@article {225606,
	title = {Coding and Transformations in the olfactory system},
	journal = {Annu Rev Neurosci.},
	volume = {37},
	year = {2014},
	pages = {363},
	abstract = {How is sensory information represented in the brain?Along-standing debate in neural coding is whether and how timing of spikes conveys information to downstream neurons. Although we know that neurons in the olfactory bulb (OB) exhibit rich temporal dynamics, the functional relevance of temporal coding remains hotly debated. Recent recording experiments in awake behaving animals have elucidated highly organized temporal structures of activity in the OB. In addition, the analysis of neural circuits in the piriform cortex (PC) demonstrated the importance of not only OB afferent inputs but also intrinsicPCneural circuits in shaping odor responses. Furthermore, new experiments involving stimulation of the OB with specific temporal patterns allowed for testing the relevance of temporal codes. Together, these studies suggest that the relative timing of neuronal activity in the OB conveys odor information and that neural circuits in the PC possess various mechanisms to decode temporal patterns of OB input.\&nbsp;},
	url = {http://www.annualreviews.org.ezp-prod1.hul.harvard.edu/doi/full/10.1146/annurev-neuro-071013-013941?url_ver=Z39.88-2003\&rfr_id=ori:rid:crossref.org\&rfr_dat=cr_pub\%3dpubmed},
	author = {N. Uchida and C. Poo and R. Haddad}
}
@article {225601,
	title = {An excitatory paraventricular nucleus to AgRP neuron circuit that drives hunger},
	journal = {Nature},
	volume = {507},
	number = {7491},
	year = {2014},
	pages = {238-242},
	abstract = {Hunger is a hard-wired motivational state essential for survival. Agouti-related peptide (AgRP)-expressing neurons in the arcuate nucleus (ARC) at the base of the hypothalamus are crucial to the control of hunger. They are activated by caloric deficiency and, when naturally or artificially stimulated, they potently induce intense hunger and subsequent food intake. Consistent with their obligatory role in regulating appetite, genetic ablation or chemogenetic inhibition of AgRP neurons decreases feeding. Excitatory input to AgRP neurons is important in caloric-deficiency-induced activation, and is notable for its remarkable degree of caloric-state-dependent synaptic plasticity. Despite the important role of excitatory input, its source(s) has been unknown. Here, through the use of Cre-recombinase-enabled, cell-specific neuron mapping techniques in mice, we have discovered strong excitatory drive that, unexpectedly, emanates from the hypothalamic paraventricular nucleus, specifically from subsets of neurons expressing thyrotropin-releasing hormone (TRH) and pituitary adenylate cyclase-activating polypeptide (PACAP, also known as ADCYAP1). Chemogenetic stimulation of these afferent neurons in sated mice markedly activates AgRP neurons and induces intense feeding. Conversely, acute inhibition in mice with caloric-deficiency-induced hunger decreases feeding. Discovery of these afferent neurons capable of triggering hunger advances understanding of how this intense motivational state is regulated.},
	url = {http://www.nature.com.ezp-prod1.hul.harvard.edu/nature/journal/v507/n7491/full/nature12956.html},
	author = {M.J. Krashes and B.P. Shah and D.P. Olson and D.E. Strochlic and A.S. Garfield and L. Vong and H. Pei and M. Watabe-Uchida and N. Uchida and S.D. Liberles and B.B. Lowell}
}
@article {225596,
	title = {Galanin neurons in the medial preoptic area govern parental behavior},
	journal = {Nature},
	volume = {509},
	number = {7500},
	year = {2014},
	pages = {325-330},
	abstract = {Mice display robust, stereotyped behaviours towards pups: virgin males typically attack pups, whereas virgin females and sexually experienced males and females display parental care. Here we show that virgin males genetically impaired in vomeronasal sensing do not attack pups and are parental. Furthermore, we uncover a subset of galanin-expressing neurons in the medial preoptic area (MPOA) that are specifically activated during male and female parenting, and a different subpopulation that is activated during mating. Genetic ablation of MPOA galanin neurons results in marked impairment of parental responses in males and females and affects male mating. Optogenetic activation of these neurons in virgin males suppresses inter-male and pup-directed aggression and induces pup grooming. Thus, MPOA galanin neurons emerge as an essential regulatory node of male and female parenting behaviour and other social responses. These results provide an entry point to a circuit-level dissection of parental behaviour and its modulation by social experience.},
	url = {http://www.nature.com.ezp-prod1.hul.harvard.edu/nature/journal/v509/n7500/full/nature13307.html},
	author = {Z. Wu and A.E. Autry and J.F. Bergan and M. Watabe-Uchida and C.G. Dulac}
}
@article {225611,
	title = {Division of labor for division: inhibitory interneurons with different spatial landscapes in the olfactory system},
	journal = {Neuron},
	volume = {80},
	number = {5},
	year = {2013},
	pages = {1106-1109},
	abstract = {Normalizing neural responses by the sum of population activity allows the nervous system to adjust its sensitivity according to task demands, facilitating intensity-invariant information processing. In this issue of Neuron, two studies, Kato et\&nbsp;al. (2013) and Miyamichi et\&nbsp;al. (2013), suggest that parvalbumin-positive interneurons in the olfactory bulb play a role in this process.},
	url = {http://www.sciencedirect.com.ezp-prod1.hul.harvard.edu/science/article/pii/S0896627313010817},
	author = {N. Uchida and N. Eshel and M. Watabe-Uchida}
}
@article {79736,
	title = {Opening the black box: dopamine, predictions, and learning},
	journal = {Trends in Cognitive Science},
	volume = {17},
	number = {9},
	year = {2013},
	pages = {430-431},
	abstract = {Dopamine neurons are thought to promote learning by signaling prediction errors, that is, the difference between actual and expected outcomes. Whether these signals are sufficient for associative learning, however, remains untested. A recent study used optogenetics in a classic behavioral paradigm to confirm the role of dopamine prediction errors in learning.},
	url = {http://www.sciencedirect.com.ezp-prod1.hul.harvard.edu/science/article/pii/S1364661313001460},
	author = {N. Eshel and J. Tian and N. Uchida}
}
@article {79741,
	title = {Olfactory cortical neurons read out a relative time code in the olfactory bulb},
	journal = {Nature Neuroscience},
	volume = {16},
	number = {7},
	year = {2013},
	pages = {949-957},
	abstract = {Odor stimulation evokes complex spatiotemporal activity in the olfactory bulb, suggesting that both the identity of activated neurons and the timing of their activity convey information about odors. However, whether and how downstream neurons decipher these temporal patterns remains unknown. We addressed this question by measuring the spiking activity of downstream neurons while optogenetically stimulating two foci in the olfactory bulb with varying relative timing in mice. We found that the overall spike rates of piriform cortex neurons (PCNs) were sensitive to the relative timing of activation. Posterior PCNs showed higher sensitivity to relative input times than neurons in the anterior piriform cortex. In contrast, olfactory bulb neurons rarely showed such sensitivity. Thus, the brain can transform a relative time code in the periphery into a firing rate{\textendash}based representation in central brain areas, providing evidence for the relevance of a relative time{\textendash}based code in the olfactory bulb.},
	url = {http://www.nature.com.ezp-prod1.hul.harvard.edu/neuro/journal/v16/n7/full/nn.3407.html},
	author = {R. Haddad and A. Lanjuin and L Madisen and H. Zeng and V.N. Murthy and N. Uchida}
}
@article {79731,
	title = {The dorsomedial striatum encodes net expected return, critical for energizing performance vigor},
	journal = {Nature Neuroscience},
	volume = {16},
	number = {5},
	year = {2013},
	pages = {639-647},
	abstract = {Decision making requires an actor to not only steer behavior toward specific goals but also determine the optimal vigor of performance. Current research and models have largely focused on the former problem of how actions are directed while overlooking the latter problem of how they are energized. Here we designed a self-paced decision-making paradigm, which showed that rats{\textquoteright} performance vigor globally fluctuates with the net value of their options, suggesting that they maintain long-term estimates of the value of their current state. Lesions of the dorsomedial striatum (DMS) and, to a lesser degree, in the ventral striatum impaired such state-dependent modulation of vigor, rendering vigor to depend more exclusively on the outcomes of immediately preceding trials. The lesions, however, spared choice biases. Neuronal recordings showed that the DMS is enriched in net value{\textendash}coding neurons. In sum, the DMS encodes one{\textquoteright}s net expected return, which drives the general motivation to perform.},
	url = {http://www.nature.com.ezp-prod1.hul.harvard.edu/neuro/journal/v16/n5/full/nn.3377.html},
	author = {A.Y. Wang and K. Miura and N. Uchida}
}
@article {1309422,
	title = {Neuron-type-specific signals for reward and punishment in the ventral tegmental area},
	journal = {Nature},
	volume = {482},
	year = {2012},
	pages = {85-88},
	author = {Jeremiah Y. Cohen and Sebastian Haesler and Linh Vong and Bradford B. Lowell and Uchida, Naoshige}
}
@article {1309421,
	title = {Whole-brain mapping of direct inputs to midbrain dopamine neurons},
	journal = {Neuron},
	volume = {74},
	number = {5},
	year = {2012},
	pages = {858-873},
	author = {Watabe-Uchida, Mitsuko and Lisa Zhu and Sachie K. Ogawa and Archana Vamanrao and Uchida, Naoshige}
}
@article {1309420,
	title = {Odor representations in olfactory cortex: distributed rate coding and decorrelated population activity},
	journal = {Neuron},
	volume = {74},
	number = {6},
	year = {2012},
	pages = {1087-1098},
	author = {Keiji Miura and Zachary F. Mainen and Uchida, Naoshige}
}
@article {1309423,
	title = {A wireless multi-channel neural amplifier for freely moving animals},
	journal = {Nature Neuroscience},
	volume = {14},
	year = {2011},
	pages = {263-269},
	abstract = {Conventional neural recording systems restrict behavioral experiments to a flat indoor environment compatible with the cable that tethers the subject to recording instruments. To overcome these constraints, we developed a wireless multi-channel system for recording neural signals from rats. The device takes up to 64 voltage signals from implanted electrodes, samples each at 20 kHz, time-division multiplexes them into one signal and transmits that output by radio frequency to a receiver up to 60 m away. The system introduces \&lt;4 μV of electrode-referred noise, comparable to wired recording systems, and outperforms existing rodent telemetry systems in channel count, weight and transmission range. This allows effective recording of brain signals in freely behaving animals. We report measurements of neural population activity taken outdoors and in tunnels. Neural firing in the visual cortex was relatively sparse, correlated even across large distances and was strongly influenced by locomotor activity.},
	author = {Tobi A Szuts and Vitaliy Fadeyev and Sergei Kachiguine and Alexander Sher and Matthew V. Grivich and Margarida Agroch{\~a}o and Pawel Hottowy and Wladyslaw Dabrowski and Evgueniy V Lubenov and Athanassios G Siapas and Uchida, Naoshige and Alan M Litke and Meister, Markus}
}
@article {79721,
	title = {Robust odor coding via inhalation-coupled transient activity in the mammalian olfactory bulb},
	journal = {Neuron},
	volume = {68},
	number = {3},
	year = {2010},
	pages = {570-585},
	author = {K.M. Cury and N. Uchida}
}
