%0 Generic %1 journals/corr/abs-1901-06829 %A He, Dongliang %A Zhao, Xiang %A Huang, Jizhou %A Li, Fu %A Liu, Xiao %A Wen, Shilei %D 2019 %K %T Read, Watch, and Move: Reinforcement Learning for Temporally Grounding Natural Language Descriptions in Videos. %U http://arxiv.org/abs/1901.06829 %X The task of video grounding, which temporally localizes a natural language description in a video, plays an important role in understanding videos. Existing studies have adopted strategies of sliding window over the entire video or exhaustively ranking all possible clip-sentence pairs in a pre-segmented video, which inevitably suffer from exhaustively enumerated candidates. To alleviate this problem, we formulate this task as a problem of sequential decision making by learning an agent which regulates the temporal grounding boundaries progressively based on its policy. Specifically, we propose a reinforcement learning based framework improved by multi-task learning and it shows steady performance gains by considering additional supervised boundary information during training. Our proposed framework achieves state-of-the-art performance on ActivityNet'18 DenseCaption dataset and Charades-STA dataset while observing only 10 or less clips per video.

@misc{journals/corr/abs-1901-06829, abstract = {The task of video grounding, which temporally localizes a natural language description in a video, plays an important role in understanding videos. Existing studies have adopted strategies of sliding window over the entire video or exhaustively ranking all possible clip-sentence pairs in a pre-segmented video, which inevitably suffer from exhaustively enumerated candidates. To alleviate this problem, we formulate this task as a problem of sequential decision making by learning an agent which regulates the temporal grounding boundaries progressively based on its policy. Specifically, we propose a reinforcement learning based framework improved by multi-task learning and it shows steady performance gains by considering additional supervised boundary information during training. Our proposed framework achieves state-of-the-art performance on ActivityNet'18 DenseCaption dataset and Charades-STA dataset while observing only 10 or less clips per video.}, added-at = {2019-02-28T21:19:51.000+0100}, author = {He, Dongliang and Zhao, Xiang and Huang, Jizhou and Li, Fu and Liu, Xiao and Wen, Shilei}, biburl = {https://www.bibsonomy.org/bibtex/2fdf669b2a5709efd0ef774d3c56987f8/nmatsuk}, interhash = {ef2d24deebf5fac16582b3e8ff5da755}, intrahash = {fdf669b2a5709efd0ef774d3c56987f8}, keywords = {}, note = {cite arxiv:1901.06829Comment: AAAI 2019}, timestamp = {2019-02-28T21:19:51.000+0100}, title = {Read, Watch, and Move: Reinforcement Learning for Temporally Grounding Natural Language Descriptions in Videos.}, url = {http://arxiv.org/abs/1901.06829}, year = 2019 }