@article {2022,
	title = {EmoMV: Affective Music-Video Correspondence Learning Datasets for Classification and Retrieval},
	journal = {Information Fusion},
	year = {2022},
	abstract = {Studies in affective audio-visual correspondence learning require ground-truth data to train, validate, and test models. The number of available datasets together with benchmarks, however, is still limited. In this paper, we create a collection of three datasets (called EmoMV) for affective correspondence learning between music and video modalities. The first two datasets (called EmoMVA, and EmoMV-B, respectively) are constructed by making use of music video segments from other available datasets. The third one called EmoMV-C is created from music videos that we self-collected from YouTube. The music-video pairs in our datasets are annotated as matched or mismatched in terms of the emotions they are conveying. The emotions are annotated by humans in the EmoMV-A dataset, while in the EmoMV-B and EmoMV-C datasets they are predicted using a pretrained deep neural network. A user study is carried out to evaluate the accuracy of the {\textquotedblleft}matched{\textquotedblright} and {\textquotedblleft}mismatched{\textquotedblright} labels offered in the EmoMV dataset collection. In addition to creating three new datasets, a benchmark deep neural network model for binary affective music-video correspondence classification is also proposed. This proposed benchmark model is then modified to adapt to affective music-video retrieval. Extensive experiments are carried out on all three datasets of the EmoMV collection. Experimental results demonstrate that our proposed model outperforms state-of-the-art approaches on both the binary classification and retrieval tasks. We envision that our newly created dataset collection together with the proposed benchmark models will facilitate advances in affective computing research.

Dataset available at: https://zenodo.org/record/7011072$\#$.YzpFoqTmgzZ},
	doi = {https://doi.org/10.1016/j.inffus.2022.10.002},
	url = {https://www.sciencedirect.com/science/article/abs/pii/S1566253522001725},
	author = {Quang-Hieu Pham and D. Herremans and G. Roig}
}
@article {2022,
	title = {Predicting emotion from music videos: exploring the relative contribution of visual and auditory information to affective responses},
	journal = {Arxiv preprint},
	year = {2022},
	abstract = {Although media content is increasingly produced, distributed, and consumed in multiple combinations of modalities, how individual modalities contribute to the perceived emotion of a media item remains poorly understood. In this paper we present MusicVideos (MuVi), a novel dataset for affective multimedia content analysis to study how the auditory and visual modalities contribute to the perceived emotion of media. The data were collected by presenting music videos to participants in three conditions: music, visual, and audiovisual. Participants annotated the music videos for valence and arousal over time, as well as the overall emotion conveyed. We present detailed descriptive statistics for key measures in the dataset and the results of feature importance analyses for each condition. Finally, we propose a novel transfer learning architecture to train Predictive models Augmented with Isolated modality Ratings (PAIR) and demonstrate the potential of isolated modality ratings for enhancing multimodal emotion recognition. Our results suggest that perceptions of arousal are influenced primarily by auditory information, while perceptions of valence are more subjective and can be influenced by both visual and auditory information. The dataset is made publicly available.},
	url = {https://arxiv.org/abs/2202.10453},
	author = {P. Chua and D. Makris and K. Agres and G. Roig and D. Herremans}
}
@article {2021,
	title = {AttendAffectNet {\textendash} Emotion Prediction of Movie Viewers Using Multimodal Fusion with Self-attention},
	journal = {Sensors. Special issue on Intelligent Sensors: Sensor Based Multi-Modal Emotion Recognition},
	year = {2021},
	abstract = {In this paper, we tackle the problem of predicting the affective responses of movie viewers, based on the content of the movies. Current studies on this topic focus on video representation learning and fusion techniques to combine the extracted features for predicting affect. Yet, those typically , while ignoring the correlation between multiple modality inputs nor the correlation between temporal inputs (i.e., sequential features). To explore these correlations, a neural network architecture namely AttendAffectNet (AAN), which uses the self-attention mechanism for predicting emotions of movie viewers from different input modalities. Particularly, visual, audio, and text features are taken into account for predicting emotions (expressed in terms of valence and arousal). We analyze three variants of our proposed AAN: Feature AAN, Temporal AAN, and Mixed AAN. The Feature AAN applies the self-attention mechanism in an innovative way on the features extracted from the different modalities (including video, audio, and movie subtitles) of a whole movie excerpt, so as to capture the relationship between them. The Temporal AAN takes the time domain of the movies and the sequential dependency of affective responses into account. In the Temporal AAN, the self-attention is applied on the concatenated (multimodal) feature vectors representing different subsequent movie segments. In the Mixed AAN, we combine the strong points of the Feature AAN and the Temporal AAN, by applying the self-attention first on vectors of features obtained from different modalities in each movie segment, and then on the feature representations of all subsequent (temporal) movie segments. We extensively train and validate our proposed AAN on both the MediaEval 2016 dataset for the Emotional Impact of Movies Task and the extended COGNIMUSE dataset. Our experiments demonstrate that audio features play a more influential role than those extracted from video and movie subtitles when predicting emotions of movie viewers on these datasets. The models that use all visual, audio, and text features simultaneously as their inputs perform better than those using features extracted from each modality separately. In addition, the Feature AAN outperforms other AAN variants on the above-mentioned datasets, highlighting the importance of taking different features as context to one another when fusing them. The Feature AAN also performs better than the baseline models when predicting the valence dimension.},
	doi = {10.3390/s21248356},
	url = {https://www.mdpi.com/1424-8220/21/24/8356},
	author = {T. Ha Thi Phuong and Balamurali BT and G. Roig and D. Herremans}
}
@conference {2021,
	title = {AttendAffectNet: Self-Attention based Networks for Predicting Affective Responses from Movies},
	booktitle = {Proceedings of the International Conference on Pattern Recognition (ICPR2020)},
	year = {2021},
	month = {01/2021},
	address = {Milano, Italy (virtual)},
	author = {T. Ha Thi Phuong and Balamurali BT and D. Herremans and G. Roig}
}
@conference {2020,
	title = {Regression-based music emotion prediction using triplet neural networks},
	booktitle = {Proceedings of the International Joint Conference on Neural Networks (IJCNN)},
	year = {2020},
	month = {07/2020},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Glasgow},
	abstract = {In this paper, we adapt triplet neural networks (TNNs) to a regression task, music emotion prediction. Since TNNs were initially introduced for classification, and not for regression, we propose a mechanism that allows them to provide meaningful low dimensional representations for regression tasks. We then use these new representations as the input for regression algorithms such as support vector machines and gradient boosting machines. To demonstrate the TNNs{\textquoteright} effectiveness at creating meaningful representations, we compare them to different dimensionality reduction methods on music emotion prediction, i.e., predicting valence and arousal values from musical audio signals. Our results on the DEAM dataset show that by using TNNs we achieve 90\% feature dimensionality reduction with a 9\% improvement in valence prediction and 4\% improvement in arousal prediction with respect to our baseline models (without TNN). Our TNN method outperforms other dimensionality reduction methods such as principal component analysis (PCA) and autoencoders (AE). This shows that, in addition to providing a compact latent space representation of audio features, the proposed approach has a relatively high performance over the baseline models.},
	url = {https://arxiv.org/abs/2001.09988},
	author = {K.W. Cheuk and Y.J. Luo and Balamurali BT and G. Roig and D. Herremans}
}
@conference {2019,
	title = {Latent space representation for multi-target speaker detection and identification with a sparse dataset using Triplet neural networks},
	booktitle = {IEEE Automatic Speech Recognition and Understanding Workshop (ASRU 2019)},
	year = {2019},
	month = {12/2019},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Sentosa, Singapore},
	author = {K.W. Cheuk and Balamurali BT and G. Roig and D. Herremans}
}
@conference {2019,
	title = {Multimodal Deep Models for Predicting Affective Responses Evoked by Movies},
	booktitle = {The 2nd International Workshop on Computer Vision for Physiological Measurement as part of ICCV. Seoul, South Korea. 2019},
	year = {2019},
	month = {10/2019},
	address = {Seoul, South Korea},
	abstract = {The goal of this study is to develop and analyze multimodal models for predicting experienced affective responses of viewers watching movie clips. We develop hybrid multimodal prediction models based on both the video and audio of the clips.  For the video content, we hypothesize that both image content and motion are crucial features for evoked emotion prediction. To capture such information, we extract features from  RGB frames and optical flow using pre-trained neural networks. For the audio model, we compute an enhanced set of low-level descriptors including intensity, loudness, cepstrum, linear predictor coefficients, pitch and voice quality. Both visual and audio features are then concatenated to create audio-visual features, which are used to predict the evoked emotion. To classify the movie clips into the corresponding affective response categories, we propose two approaches based on deep neural network models. The first one is based on fully connected layers without memory on the time component, the second incorporates the sequential dependency with a long short-term memory recurrent neural network (LSTM). We perform a thorough analysis of the importance of each feature set. Our experiments reveal that in our set-up, predicting emotions at each time step independently gives slightly better accuracy performance than with the LSTM. Interestingly, we also observe that the optical flow is more informative than the RGB in videos, and overall, models using audio features are more accurate than those based on video features when making the final prediction of evoked emotions. },
	author = {T. Ha Thi Phuong and D. Herremans and G. Roig}
}
@article {129,
	title = {Blacklisted speaker identification using triplet neural networks},
	year = {2018},
	url = {http://mce.csail.mit.edu/pdfs/SUTD_description.pdf},
	author = {K.W. Cheuk and Balamurali BT and G. Roig and D. Herremans}
}
