@article {2021,
	title = {AttendAffectNet {\textendash} Emotion Prediction of Movie Viewers Using Multimodal Fusion with Self-attention},
	journal = {Sensors. Special issue on Intelligent Sensors: Sensor Based Multi-Modal Emotion Recognition},
	year = {2021},
	abstract = {In this paper, we tackle the problem of predicting the affective responses of movie viewers, based on the content of the movies. Current studies on this topic focus on video representation learning and fusion techniques to combine the extracted features for predicting affect. Yet, those typically , while ignoring the correlation between multiple modality inputs nor the correlation between temporal inputs (i.e., sequential features). To explore these correlations, a neural network architecture namely AttendAffectNet (AAN), which uses the self-attention mechanism for predicting emotions of movie viewers from different input modalities. Particularly, visual, audio, and text features are taken into account for predicting emotions (expressed in terms of valence and arousal). We analyze three variants of our proposed AAN: Feature AAN, Temporal AAN, and Mixed AAN. The Feature AAN applies the self-attention mechanism in an innovative way on the features extracted from the different modalities (including video, audio, and movie subtitles) of a whole movie excerpt, so as to capture the relationship between them. The Temporal AAN takes the time domain of the movies and the sequential dependency of affective responses into account. In the Temporal AAN, the self-attention is applied on the concatenated (multimodal) feature vectors representing different subsequent movie segments. In the Mixed AAN, we combine the strong points of the Feature AAN and the Temporal AAN, by applying the self-attention first on vectors of features obtained from different modalities in each movie segment, and then on the feature representations of all subsequent (temporal) movie segments. We extensively train and validate our proposed AAN on both the MediaEval 2016 dataset for the Emotional Impact of Movies Task and the extended COGNIMUSE dataset. Our experiments demonstrate that audio features play a more influential role than those extracted from video and movie subtitles when predicting emotions of movie viewers on these datasets. The models that use all visual, audio, and text features simultaneously as their inputs perform better than those using features extracted from each modality separately. In addition, the Feature AAN outperforms other AAN variants on the above-mentioned datasets, highlighting the importance of taking different features as context to one another when fusing them. The Feature AAN also performs better than the baseline models when predicting the valence dimension.},
	doi = {10.3390/s21248356},
	url = {https://www.mdpi.com/1424-8220/21/24/8356},
	author = {T. Ha Thi Phuong and Balamurali BT and G. Roig and D. Herremans}
}
@conference {2021,
	title = {AttendAffectNet: Self-Attention based Networks for Predicting Affective Responses from Movies},
	booktitle = {Proceedings of the International Conference on Pattern Recognition (ICPR2020)},
	year = {2021},
	month = {01/2021},
	address = {Milano, Italy (virtual)},
	author = {T. Ha Thi Phuong and Balamurali BT and D. Herremans and G. Roig}
}
@conference {2019,
	title = {Multimodal Deep Models for Predicting Affective Responses Evoked by Movies},
	booktitle = {The 2nd International Workshop on Computer Vision for Physiological Measurement as part of ICCV. Seoul, South Korea. 2019},
	year = {2019},
	month = {10/2019},
	address = {Seoul, South Korea},
	abstract = {The goal of this study is to develop and analyze multimodal models for predicting experienced affective responses of viewers watching movie clips. We develop hybrid multimodal prediction models based on both the video and audio of the clips.  For the video content, we hypothesize that both image content and motion are crucial features for evoked emotion prediction. To capture such information, we extract features from  RGB frames and optical flow using pre-trained neural networks. For the audio model, we compute an enhanced set of low-level descriptors including intensity, loudness, cepstrum, linear predictor coefficients, pitch and voice quality. Both visual and audio features are then concatenated to create audio-visual features, which are used to predict the evoked emotion. To classify the movie clips into the corresponding affective response categories, we propose two approaches based on deep neural network models. The first one is based on fully connected layers without memory on the time component, the second incorporates the sequential dependency with a long short-term memory recurrent neural network (LSTM). We perform a thorough analysis of the importance of each feature set. Our experiments reveal that in our set-up, predicting emotions at each time step independently gives slightly better accuracy performance than with the LSTM. Interestingly, we also observe that the optical flow is more informative than the RGB in videos, and overall, models using audio features are more accurate than those based on video features when making the final prediction of evoked emotions. },
	author = {T. Ha Thi Phuong and D. Herremans and G. Roig}
}
