%0 Generic %1 sun2020global %A Sun, Ruoyu %A Li, Dawei %A Liang, Shiyu %A Ding, Tian %A Srikant, R %D 2020 %K 2020 deep-learning survey %T The Global Landscape of Neural Networks: An Overview %U http://arxiv.org/abs/2007.01429 %X One of the major concerns for neural network training is that the non-convexity of the associated loss functions may cause bad landscape. The recent success of neural networks suggests that their loss landscape is not too bad, but what specific results do we know about the landscape? In this article, we review recent findings and results on the global landscape of neural networks. First, we point out that wide neural nets may have sub-optimal local minima under certain assumptions. Second, we discuss a few rigorous results on the geometric properties of wide networks such as "no bad basin", and some modifications that eliminate sub-optimal local minima and/or decreasing paths to infinity. Third, we discuss visualization and empirical explorations of the landscape for practical neural nets. Finally, we briefly discuss some convergence results and their relation to landscape results.

@misc{sun2020global, abstract = {One of the major concerns for neural network training is that the non-convexity of the associated loss functions may cause bad landscape. The recent success of neural networks suggests that their loss landscape is not too bad, but what specific results do we know about the landscape? In this article, we review recent findings and results on the global landscape of neural networks. First, we point out that wide neural nets may have sub-optimal local minima under certain assumptions. Second, we discuss a few rigorous results on the geometric properties of wide networks such as "no bad basin", and some modifications that eliminate sub-optimal local minima and/or decreasing paths to infinity. Third, we discuss visualization and empirical explorations of the landscape for practical neural nets. Finally, we briefly discuss some convergence results and their relation to landscape results.}, added-at = {2020-07-07T12:19:47.000+0200}, author = {Sun, Ruoyu and Li, Dawei and Liang, Shiyu and Ding, Tian and Srikant, R}, biburl = {https://www.bibsonomy.org/bibtex/2b8181c46a959810633e0b6ee0559665c/analyst}, description = {[2007.01429] The Global Landscape of Neural Networks: An Overview}, interhash = {7e6cb93754795f59720966e4f8fe3f01}, intrahash = {b8181c46a959810633e0b6ee0559665c}, keywords = {2020 deep-learning survey}, note = {cite arxiv:2007.01429Comment: 16 pages. 8 figures}, timestamp = {2020-07-07T12:19:47.000+0200}, title = {The Global Landscape of Neural Networks: An Overview}, url = {http://arxiv.org/abs/2007.01429}, year = 2020 }