%0 Conference Paper %1 Sutton:1999:PGM:3009657.3009806 %A Sutton, Richard S. %A McAllester, David %A Singh, Satinder %A Mansour, Yishay %B Proceedings of the 12th International Conference on Neural Information Processing Systems %C Cambridge, MA, USA %D 1999 %I MIT Press %K policy_gradient reinforcement_learning reserved thema %P 1057--1063 %T Policy Gradient Methods for Reinforcement Learning with Function Approximation %U http://dl.acm.org/citation.cfm?id=3009657.3009806 %X Function approximation is essential to reinforcement learning, but the standard approach of approximating a value function and determining a policy from it has so far proven theoretically intractable. In this paper we explore an alternative approach in which the policy is explicitly represented by its own function approximator, independent of the value function, and is updated according to the gradient of expected reward with respect to the policy parameters. Williams's REINFORCE method and actor-critic methods are examples of this approach. Our main new result is to show that the gradient can be written in a form suitable for estimation from experience aided by an approximate action-value or advantage function. Using this result, we prove for the first time that a version of policy iteration with arbitrary differentiable function approximation is convergent to a locally optimal policy.

@inproceedings{Sutton:1999:PGM:3009657.3009806, abstract = {Function approximation is essential to reinforcement learning, but the standard approach of approximating a value function and determining a policy from it has so far proven theoretically intractable. In this paper we explore an alternative approach in which the policy is explicitly represented by its own function approximator, independent of the value function, and is updated according to the gradient of expected reward with respect to the policy parameters. Williams's REINFORCE method and actor-critic methods are examples of this approach. Our main new result is to show that the gradient can be written in a form suitable for estimation from experience aided by an approximate action-value or advantage function. Using this result, we prove for the first time that a version of policy iteration with arbitrary differentiable function approximation is convergent to a locally optimal policy.}, acmid = {3009806}, added-at = {2019-09-16T00:13:50.000+0200}, address = {Cambridge, MA, USA}, author = {Sutton, Richard S. and McAllester, David and Singh, Satinder and Mansour, Yishay}, biburl = {https://www.bibsonomy.org/bibtex/24ca8cc04d8982aea21e8fd5ed719e89f/e.fischer}, booktitle = {Proceedings of the 12th International Conference on Neural Information Processing Systems}, description = {Policy gradient methods for reinforcement learning with function approximation}, interhash = {7db746ffbdad9f59d8382c7d5314ec4f}, intrahash = {4ca8cc04d8982aea21e8fd5ed719e89f}, keywords = {policy_gradient reinforcement_learning reserved thema}, location = {Denver, CO}, numpages = {7}, pages = {1057--1063}, publisher = {MIT Press}, series = {NIPS'99}, timestamp = {2020-04-14T12:11:11.000+0200}, title = {Policy Gradient Methods for Reinforcement Learning with Function Approximation}, url = {http://dl.acm.org/citation.cfm?id=3009657.3009806}, year = 1999 }