%0 Conference Paper %1 BatlinerHackerEtAl07AVSP %A Batliner, Anton %A Hacker, Christian %A Kaiser, Moritz %A Mögele, Hannes %A Nöth, Elmar %B Proceedings of AVSP'07: International Conference on Auditory-Visual Speech Processing, Kasteel Groenendaal, Hilvarenbeek, The Netherlands %D 2007 %K v1205 isca paper ai multimodal user interface speech recognition image analysis smartweb %T Taking into Account the User's Focus of Attention with the Help of Audio-Visual Information: Towards less Artificial Human-Machine-Communication %U http://www.isca-speech.org/archive/archive_papers/avsp07/av07_P15.pdf %X In the German SmartWeb project, the user is interacting with the web via a PDA in order to get information on, for example, points of interest. To overcome the tedious use of devices such as push-to-talk, but still to be able to tell whether the user is addressing the system or talking to herself or to a third person, we developed a module that monitors speech and video in parallel. Our database (3.2 hours of speech, 2086 turns) has been recorded in a real-life setting, indoors as well as outdoors, with unfavourable acoustic and light conditions. With acoustic features, we classify up to 4 different types of addressing (talking to the system: On-Talk, reading from the display: Read Off-Talk, paraphrasing information presented on the screen: Paraphrasing Off-Talk, talking to a third person or to oneself: Spontaneous Off-Talk). With a camera integrated in the PDA, we record the user's face and decide whether she is looking onto the PDA or somewhere else. We use three different types of turn features based on classification scores of frame-based face detection and word-based analysis: 13 acoustic-prosodic features, 18 linguistic features, and 9 video features. The classification rate for acoustics only is up to 62 percent for the four-class problem, and up to 77 percent for the most important two-class problem user is focussing on interaction with the system or not. For video only, it is 45 percent and 71 percent, respectively. By combining the two modalities, and using linguistic information in addition, classification performance for the two-class problem so far rises up to 85 percent.

@inproceedings{BatlinerHackerEtAl07AVSP, abstract = {In the German SmartWeb project, the user is interacting with the web via a PDA in order to get information on, for example, points of interest. To overcome the tedious use of devices such as push-to-talk, but still to be able to tell whether the user is addressing the system or talking to herself or to a third person, we developed a module that monitors speech and video in parallel. Our database (3.2 hours of speech, 2086 turns) has been recorded in a real-life setting, indoors as well as outdoors, with unfavourable acoustic and light conditions. With acoustic features, we classify up to 4 different types of addressing (talking to the system: On-Talk, reading from the display: Read Off-Talk, paraphrasing information presented on the screen: Paraphrasing Off-Talk, talking to a third person or to oneself: Spontaneous Off-Talk). With a camera integrated in the PDA, we record the user's face and decide whether she is looking onto the PDA or somewhere else. We use three different types of turn features based on classification scores of frame-based face detection and word-based analysis: 13 acoustic-prosodic features, 18 linguistic features, and 9 video features. The classification rate for acoustics only is up to 62 percent for the four-class problem, and up to 77 percent for the most important two-class problem user is focussing on interaction with the system or not. For video only, it is 45 percent and 71 percent, respectively. By combining the two modalities, and using linguistic information in addition, classification performance for the two-class problem so far rises up to 85 percent.}, added-at = {2012-05-30T10:42:47.000+0200}, author = {Batliner, Anton and Hacker, Christian and Kaiser, Moritz and M\"{o}gele, Hannes and N\"{o}th, Elmar}, biburl = {https://www.bibsonomy.org/bibtex/2a9a6edcfe4abe4f797026e2bd3891fb0/flint63}, booktitle = {Proceedings of {AVSP'07}: International Conference on Auditory-Visual Speech Processing, Kasteel Groenendaal, Hilvarenbeek, The Netherlands}, file = {ISCA Archive:2007/BatlinerHackerEtAl07AVSP.pdf:PDF}, groups = {public}, interhash = {3e595b0bd5cdf233a31bb3ed3f9bbe0f}, intrahash = {a9a6edcfe4abe4f797026e2bd3891fb0}, keywords = {v1205 isca paper ai multimodal user interface speech recognition image analysis smartweb}, timestamp = {2018-04-16T11:34:11.000+0200}, title = {Taking into Account the User's Focus of Attention with the Help of Audio-Visual Information: Towards less Artificial Human-Machine-Communication}, url = {http://www.isca-speech.org/archive/archive_papers/avsp07/av07_P15.pdf}, username = {flint63}, year = 2007 }