@inproceedings{9e60160bcdaf4629b5fde1b2aab37981,
title = "Understanding and Detecting Convergence for Stochastic Gradient Descent with Momentum",
abstract = "Convergence detection of iterative stochastic optimization methods is of great practical interest. This paper considers stochastic gradient descent (SGD) with a constant learning rate and momentum. We show that there exists a transient phase in which iterates move towards a region of interest, and a stationary phase in which iterates remain bounded in that region around a minimum point. We construct a statistical diagnostic test for convergence to the stationary phase using the inner product between successive gradients and demonstrate that the proposed diagnostic works well. We theoretically and empirically characterize how momentum can affect the test statistic of the diagnostic, and how the test statistic captures a relatively sparse signal within the gradients in convergence. Finally, we demonstrate an application to automatically tune the learning rate by reducing it each time stationarity is detected, and show the procedure is robust to mis-specified initial rates.",
author = "Jerry Chee and Ping Li",
note = "Publisher Copyright: {\textcopyright} 2020 IEEE.; 8th IEEE International Conference on Big Data, Big Data 2020 ; Conference date: 10-12-2020 Through 13-12-2020",
year = "2020",
month = dec,
day = "10",
doi = "10.1109/BigData50022.2020.9378129",
language = "English (US)",
series = "Proceedings - 2020 IEEE International Conference on Big Data, Big Data 2020",
publisher = "Institute of Electrical and Electronics Engineers Inc.",
pages = "133--140",
editor = "Xintao Wu and Chris Jermaine and Li Xiong and Hu, {Xiaohua Tony} and Olivera Kotevska and Siyuan Lu and Weijia Xu and Srinivas Aluru and Chengxiang Zhai and Eyhab Al-Masri and Zhiyuan Chen and Jeff Saltz",
booktitle = "Proceedings - 2020 IEEE International Conference on Big Data, Big Data 2020",
address = "United States",
}