@conference {2026,
	title = {Aligning Generative Music AI with Human Preferences: Methods and Challenges},
	booktitle = {Proceedings of AAAI, senior member track},
	year = {2026},
	publisher = {AAAI},
	organization = {AAAI},
	address = {Singapore},
	author = {D. Herremans and A. Roy}
}
@conference {2026,
	title = {Emerging AI Technologies for Music: Towards Controllable, Collaborative, and Creative Systems},
	booktitle = {Proceedings of Machine Learning Research, PMLR 303:1-5, 2026},
	year = {2026},
	abstract = {The field of AI and music has witnessed rapid growth in recent years driving notable success across domains such as music information retrieval (MIR), generative modelling, representation learning, source separation, and transcription among several others. The emergence of large-scale music foundation models (e.g., MusicLM, Suno, Riffusion) and multimodal systems combining modalities such as text, audio and video has further expanded the creative possibilities of AI-driven music. However, despite these technical advances, there remains a significant gap between the capabilities of current AI music systems and their practical adoption by musicians, composers, producers, and educators. At the same time, there has been a growing consensus in the community that recognizes the limitations of fully autonomous generation and the need for interactive, interpretable, controllable, and human-centred approaches. In light of this emerging trend to maintain meaningful human agency over the creative process, the editors of this special issue organized the first workshop on Emerging AI Technologies for Music, as part of the Association for the Advancement of Artificial Intelligence (AAAI) conference in 2026 at Singapore.

Full proceedings: https://proceedings.mlr.press/v303/},
	author = {K. Bhandari and A. Roy and S. Colton and D. Herremans}
}
@conference {2026,
	title = {Generative AI in Education for SDG 4: Insights from Indonesia and Kazakhstan},
	booktitle = {Proceedings of the Pacific Asia Conference on Information Systems (PACIS).},
	year = {2026},
	author = {A. Mesha Putri and S. Saide and D. Kasim Riau and D. Herremans}
}
@conference {2026,
	title = {KARMA-MV: A Benchmark for Causal Question Answering on Music Videos},
	booktitle = {arXiv:2605.08175},
	year = {2026},
	abstract = {While significant progress has been made in Video Question Answering and cross-modal understanding, causal reasoning about how visual dynamics drive musical structure in music videos remains under-explored. We introduce KARMA-MV, a large-scale multiple-choice QA dataset derived from 2,682 YouTube music videos, designed to test models{\textquoteright} ability to integrate temporal audio-visual cues and reason about visual-to-musical influence across reasoning, prediction, and counterfactual questions. Unlike traditional datasets requiring manual annotation, KARMA-MV leverages LLM reasoning for scalable generation and validation, yielding 37,737 MCQs. We propose a causal knowledge graph (CKG) approach that augments vision-language models (VLMs) with structured retrieval of cross-modal dependencies. Experiments on state-of-the-art VLMs and LLMs show consistent gains from CKG grounding -- especially for smaller models -- establishing the value of explicit causal structure for music-video reasoning. KARMA-MV provides a new benchmark for advancing causal audio-visual understanding beyond correlation.},
	url = {https://arxiv.org/abs/2605.08175},
	author = {A. Ghosh and A. Roy and D. Herremans}
}
@conference {2025,
	title = {Leveraging LLM Embeddings for Cross Dataset Label Alignment and Zero Shot Music Emotion Prediction},
	booktitle = {Conference on AI Music Creativity (AIMC)},
	year = {2026},
	address = {Berlin, Germany},
	abstract = {In this work, we present a novel method for music emotion recognition that leverages Large Language Model (LLM) embeddings for label alignment across multiple datasets and zero-shot prediction on novel categories. First, we compute LLM embeddings for emotion labels and apply non-parametric clustering to group similar labels, across multiple datasets containing disjoint labels. We use these cluster centers to map music features (MERT) to the LLM embedding space. To further enhance the model, we introduce an alignment regularization that enables dissociation of MERT embeddings from different clusters. This further enhances the model{\textquoteright}s ability to better adaptation to unseen datasets. We demonstrate the effectiveness of our approach by performing zero-shot inference on a new dataset, showcasing its ability to generalize to unseen labels without additional training.},
	url = {https://arxiv.org/abs/2410.11522},
	author = {R. Liu and A. Roy and D. Herremans}
}
@conference {2025,
	title = {MelodySim: Measuring Melody-aware Music Similarity for Plagiarism Detection},
	booktitle = {IEEE Tencon},
	year = {2026},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Bali, Indonesia},
	abstract = {We propose MelodySim, a melody-aware music similarity model and dataset for plagiarism detection. First, we introduce a novel method to construct a dataset focused on melodic similarity. By augmenting Slakh2100, an existing MIDI dataset, we generate variations of each piece while preserving the melody through modifications such as note splitting, arpeggiation, minor track dropout, and re-instrumentation. A user study confirms that positive pairs indeed contain similar melodies, while other musical tracks are significantly changed. Second, we develop a segment-wise melodic-similarity detection model that uses a MERT encoder and applies a triplet neural network to capture melodic similarity. The resulting decision matrix highlights where plagiarism might occur. The experiments show that our model is able to outperform baseline models in detecting similar melodic fragments on the MelodySim test set.},
	url = {https://arxiv.org/abs/2505.20979},
	author = {T. Lu and Geist, C-M and J. Melechovsky and A. Roy and D. Herremans}
}
@conference {2026,
	title = {nnAudio 2: Overcoming Dynamic Compilation Barriers and Transform Inconsistencies},
	booktitle = {Conference on AI Music Creativity (AIMC)},
	year = {2026},
	address = {Berlin, Germany},
	abstract = {nnAudio is an open-source audio feature extraction toolbox for deep learning, but its use in current environments is hindered by TorchScript incompatibilities, inverse-transform edge cases, and dependency drift. We present a targeted modernization for modern PyTorch and scientific Python. We resolve TorchScript compilation failures in STFT and iSTFT by removing dynamic state mutation and module construction from scripted code paths and tightening argument handling in inverse-related helpers. We clarify inverse-STFT behavior by restricting reliable inversion to the uniform-bin setting (freq_scale={\textquoteleft}no{\textquoteright}) and raising explicit runtime errors for unsupported frequency scales, preventing silently degraded reconstructions. We restore CFP compatibility with modern SciPy and ensure VQT reduces to CQT when gamma = 0. Regression tests cover the new STFT/iSTFT behaviors, and the updated codebase passes the full repository test suite in a modern Python environment. These improvements provide a more robust foundation for differentiable audio analysis in research and deployment.},
	url = {https://arxiv.org/abs/2606.05394},
	author = {Roy, Abhinaba and Liang, Junyi and Herremans, Dorien}
}
@proceedings {2025,
	title = {SonicMaster: Towards Controllable All-in-One Music Restoration and Mastering},
	year = {2026},
	address = {South Korea},
	abstract = {Music recordings often suffer from audio quality issues such as excessive reverberation, distortion, clipping, tonal imbalances, and a narrowed stereo image, especially when created in non-professional settings without specialized equipment or expertise. These problems are typically corrected using separate specialized tools and manual adjustments. In this paper, we introduce SonicMaster, the first unified generative model for music restoration and mastering that addresses a broad spectrum of audio artifacts with text-based control. SonicMaster is conditioned on natural language instructions to apply targeted enhancements, or can operate in an automatic mode for general restoration. To train this model, we construct the SonicMaster dataset, a large dataset of paired degraded and high-quality tracks by simulating common degradation types with nineteen degradation functions belonging to five enhancements groups: equalization, dynamics, reverb, amplitude, and stereo. Our approach leverages a flow-matching generative training paradigm to learn an audio transformation that maps degraded inputs to their cleaned, mastered versions guided by text prompts. Objective audio quality metrics demonstrate that SonicMaster significantly improves sound quality across all artifact categories. Furthermore, subjective listening tests confirm that listeners prefer SonicMaster{\textquoteright}s enhanced outputs over other baselines. The model and demo samples are available through https://github.com/AMAAI-Lab/SonicMaster.},
	url = {https://arxiv.org/abs/2508.03448},
	author = {J. Melechovsky and A. Mehrish and A. Roy and D. Herremans}
}
@conference {2025,
	title = {Text2midi-InferAlign: Improving Symbolic Music Generation with Inference-Time Alignment},
	booktitle = {ICASSP},
	year = {2026},
	address = {Barcelona, Spain},
	abstract = {We present Text2midi-InferAlign, a novel technique for improving symbolic music generation at inference time. Our method leverages text-to-audio alignment and music structural alignment rewards during inference to encourage the generated music to be consistent with the input caption. Specifically, we introduce two objectives scores: a text-audio consistency score that measures rhythmic alignment between the generated music and the original text caption, and a harmonic consistency score that penalizes generated music containing notes inconsistent with the key. By optimizing these alignment-based objectives during the generation process, our model produces symbolic music that is more closely tied to the input captions, thereby improving the overall quality and coherence of the generated compositions. Our approach can extend any existing autoregressive model without requiring further training or fine-tuning. We evaluate our work on top of Text2midi - an existing text-to-midi generation model, demonstrating significant improvements in both objective and subjective evaluation metrics.},
	url = {https://arxiv.org/abs/2505.12669},
	author = {A. Roy and G. Puri and D. Herremans}
}
@conference {2026,
	title = {Text2Score: Generating Sheet Music From Textual Prompts},
	booktitle = {arXiv:2605.13431},
	year = {2026},
	abstract = {Developing text-driven symbolic music generation models remains challenging due to the scarcity of aligned text-music datasets and the unreliability of automated captioning pipelines. While most efforts have focused on MIDI, sheet music representations are largely underexplored in text-driven generation. We present Text2Score, a two-stage framework comprising a planning stage and an execution stage for generating sheet music from natural language prompts. By deriving supervision signals directly from symbolic XML data, we propose an alternative training paradigm that bypasses noisy or scarce text-music pairs. In the planning stage, an LLM orchestrator translates a natural language prompt into a structured measure-wise plan defining musical attributes such as instruments, key, time signatures, harmony, etc. This plan is then consumed by a generative model in the execution stage to produce interleaved ABC notation conditioned on the plan{\textquoteright}s structural constraints. To assess output quality, we introduce an evaluation framework covering playability, readability, instrument utilization, structural complexity, and prompt adherence, validated by expert musicians. Text2Score consistently outperforms both a pure LLM-based agentic framework and three end-to-end baselines across objective and subjective dimensions. We open-source the dataset, code, evaluation set and LLM prompts used in this work; a demo is available on our project page (https://keshavbhandari.github.io/portfolio/text2score).},
	url = {https://arxiv.org/abs/2605.13431},
	author = {K. Bhandari and S. Chang and A. Roy and F. Ronchini and E. Benetos and D. Herremans and S. Colton}
}
@conference {2025,
	title = {JamendoMaxCaps: A Large Scale Music-caption Dataset with Imputed Metadata},
	booktitle = {Proceedings of IJCNN, Rome, Italy},
	year = {2025},
	abstract = {We introduce JamendoMaxCaps, a large-scale music-caption dataset featuring over 200,000 freely licensed instrumental tracks from the renowned Jamendo platform. The dataset includes captions generated by a state-of-the-art captioning model, enhanced with imputed metadata. We also introduce a retrieval system that leverages both musical features and metadata to identify similar songs, which are then used to fill in missing metadata using a local large language model (LLLM). This approach allows us to provide a more comprehensive and informative dataset for researchers working on music-language understanding tasks. We validate this approach quantitatively with five different measurements. By making the JamendoMaxCaps dataset publicly available, we provide a high-quality resource to advance research in music-language understanding tasks such as music retrieval, multimodal representation learning, and generative music models.},
	doi = {10.1109/IJCNN64981.2025.11228034},
	url = {https://arxiv.org/abs/2502.07461},
	author = {A. Roy and R. Liu and T. Lu and D. Herremans}
}
@conference {2025,
	title = {SonicVerse: Multi-Task Learning for Music Feature-Informed Captioning},
	booktitle = {Proceedings of the 6th Conference on AI Music Creativity (AIMC 2025), Brussels, Belgium, September 10th - 12th, 2025},
	year = {2025},
	abstract = {Detailed captions that accurately reflect the characteristics of a music piece can enrich music databases and drive forward research in music AI. This paper introduces a multi-task music captioning model, SonicVerse, that integrates caption generation with auxiliary music feature detection tasks such as key detection, vocals detection, and more, so as to directly capture both low-level acoustic details as well as high-level musical attributes. The key contribution is a projection-based architecture that transforms audio input into language tokens, while simultaneously detecting music features through dedicated auxiliary heads. The outputs of these heads are also projected into language tokens, to enhance the captioning input. This framework not only produces rich, descriptive captions for short music fragments but also directly enables the generation of detailed time-informed descriptions for longer music pieces, by chaining the outputs using a large-language model. To train the model, we extended the MusicBench dataset by annotating it with music features using MIRFLEX, a modular music feature extractor, resulting in paired audio, captions and music feature data. Experimental results show that incorporating features in this way improves the quality and detail of the generated captions.},
	keywords = {captioning, music, nlp},
	url = {https://arxiv.org/abs/2506.15154},
	author = {A. Chopra and A. Roy and D. Herremans}
}
@article {2025,
	title = {Text2midi: Generating Symbolic Music from Captions},
	journal = {Proceedings of AAAI, Philadelphia},
	year = {2025},
	abstract = {This paper introduces text2midi, an end-to-end model to generate MIDI files from textual descriptions. Leveraging the growing popularity of multimodal generative approaches, text2midi capitalizes on the extensive availability of textual data and the success of large language models (LLMs). Our end-to-end system harnesses the power of LLMs to generate symbolic music in the form of MIDI files. Specifically, we utilize a pretrained LLM encoder to process captions, which then condition an autoregressive transformer decoder to produce MIDI sequences that accurately reflect the provided descriptions. This intuitive and user-friendly method significantly streamlines the music creation process by allowing users to generate music pieces using text prompts. We conduct comprehensive empirical evaluations, incorporating both automated and human studies, that show our model generates MIDI files of high quality that are indeed controllable by text captions that may include music theory terms such as chords, keys, and tempo. We release the code and music samples on our demo page (this https URL) for users to interact with text2midi.},
	url = {https://www.arxiv.org/abs/2412.16526},
	author = {K. Bhandari and A. Roy and K. Wang and G. Puri and S. Colton and D. Herremans}
}
@conference {2024,
	title = {MidiCaps {\textemdash} A large-scale MIDI dataset with text captions},
	booktitle = {ISMIR},
	year = {2024},
	address = {San Francisco},
	abstract = {Generative models guided by text prompts are increasingly becoming more popular. However, no text-to-MIDI models currently exist, mostly due to the lack of a captioned MIDI dataset. This work aims to enable research that combines LLMs with symbolic music by presenting the first large-scale MIDI dataset with text captions that is openly available: MidiCaps. MIDI (Musical Instrument Digital Interface) files are a widely used format for encoding musical information. Their structured format captures the nuances of musical composition and has practical applications by music producers, composers, musicologists, as well as performers. Inspired by recent advancements in captioning techniques applied to various domains, we present a large-scale curated dataset of over 168k MIDI files accompanied by textual descriptions. Each MIDI caption succinctly describes the musical content, encompassing tempo, chord progression, time signature, instruments present, genre and mood; thereby facilitating multi-modal exploration and analysis. The dataset contains a mix of various genres, styles, and complexities, offering a rich source for training and evaluating models for tasks such as music information retrieval, music understanding and cross-modal translation. We provide detailed statistics about the dataset and have assessed the quality of the captions in an extensive listening study. We anticipate that this resource will stimulate further research in the intersection of music and natural language processing, fostering advancements in both fields.},
	url = {https://arxiv.org/abs/2406.02255},
	author = {J. Melechovsky and A. Roy and D. Herremans}
}
@conference {2024,
	title = {MIRFLEX: Music Information Retrieval Feature Library for Extraction},
	booktitle = {ISMIR, Late Breaking Demos},
	year = {2024},
	address = {San Francisco},
	abstract = {This paper introduces an extendable modular system that compiles a range of music feature extraction models to aid music information retrieval research. The features include musical elements like key, downbeats, and genre, as well as audio characteristics like instrument recognition, vocals/instrumental classification, and vocals gender detection. The integrated models are state-of-the-art or latest open-source. The features can be extracted as latent or post-processed labels, enabling integration into music applications such as generative music, recommendation, and playlist generation. The modular design allows easy integration of newly developed systems, making it a good benchmarking and comparison tool. This versatile toolkit supports the research community in developing innovative solutions by providing concrete musical features.
},
	url = {https://arxiv.org/abs/2411.00469},
	author = {A. Chopra and A. Roy and D. Herremans}
}
@article {2022,
	title = {EmoMV: Affective Music-Video Correspondence Learning Datasets for Classification and Retrieval},
	journal = {Information Fusion},
	year = {2022},
	abstract = {Studies in affective audio-visual correspondence learning require ground-truth data to train, validate, and test models. The number of available datasets together with benchmarks, however, is still limited. In this paper, we create a collection of three datasets (called EmoMV) for affective correspondence learning between music and video modalities. The first two datasets (called EmoMVA, and EmoMV-B, respectively) are constructed by making use of music video segments from other available datasets. The third one called EmoMV-C is created from music videos that we self-collected from YouTube. The music-video pairs in our datasets are annotated as matched or mismatched in terms of the emotions they are conveying. The emotions are annotated by humans in the EmoMV-A dataset, while in the EmoMV-B and EmoMV-C datasets they are predicted using a pretrained deep neural network. A user study is carried out to evaluate the accuracy of the {\textquotedblleft}matched{\textquotedblright} and {\textquotedblleft}mismatched{\textquotedblright} labels offered in the EmoMV dataset collection. In addition to creating three new datasets, a benchmark deep neural network model for binary affective music-video correspondence classification is also proposed. This proposed benchmark model is then modified to adapt to affective music-video retrieval. Extensive experiments are carried out on all three datasets of the EmoMV collection. Experimental results demonstrate that our proposed model outperforms state-of-the-art approaches on both the binary classification and retrieval tasks. We envision that our newly created dataset collection together with the proposed benchmark models will facilitate advances in affective computing research.

Dataset available at: https://zenodo.org/record/7011072$\#$.YzpFoqTmgzZ},
	doi = {https://doi.org/10.1016/j.inffus.2022.10.002},
	url = {https://www.sciencedirect.com/science/article/abs/pii/S1566253522001725},
	author = {Quang-Hieu Pham and D. Herremans and G. Roig}
}
@conference {2022,
	title = {HEAR 2021: Holistic Evaluation of Audio Representations},
	booktitle = {Proceedings of Machine Learning Research (PMLR): NeurIPS 2021 Competition Track},
	year = {2022},
	abstract = {What audio embedding approach generalizes best to a wide range of downstream tasks across a variety of everyday domains without fine-tuning? The aim of the HEAR 2021 NeurIPS challenge is to develop a general-purpose audio representation that provides a strong basis for learning in a wide variety of tasks and scenarios. HEAR 2021 evaluates audio representations using a benchmark suite across a variety of domains, including speech, environmental sound, and music. In the spirit of shared exchange, each participant submitted an audio embedding model following a common API that is general-purpose, open-source, and freely available to use. Twenty-nine models by thirteen external teams were evaluated on nineteen diverse downstream tasks derived from sixteen datasets. Open evaluation code, submitted models and datasets are key contributions, enabling comprehensive and reproducible evaluation, as well as previously impossible longitudinal studies. It still remains an open question whether one single general-purpose audio representation can perform as holistically as the human ear.},
	url = {https://arxiv.org/abs/2203.03022},
	author = {Joseph Turian and Jordie Shier and Humair Raj Khan and Bhiksha Raj and Bj{\"o}rn W. Schuller and Christian J. Steinmetz and Colin Malloy and George Tzanetakis and Gissel Velarde and Kirk McNally and Max Henry and Nicolas Pinto and Camille Noufi and Christian Clough and D. Herremans and Eduardo Fonseca and Jesse Engel and Justin Salamon and Philippe Esling and Pranay Manocha and Shinji Watanabe and Zeyu Jin and Yonatan Bisk}
}
@article {2022,
	title = {Predicting emotion from music videos: exploring the relative contribution of visual and auditory information to affective responses},
	journal = {Arxiv preprint},
	year = {2022},
	abstract = {Although media content is increasingly produced, distributed, and consumed in multiple combinations of modalities, how individual modalities contribute to the perceived emotion of a media item remains poorly understood. In this paper we present MusicVideos (MuVi), a novel dataset for affective multimedia content analysis to study how the auditory and visual modalities contribute to the perceived emotion of media. The data were collected by presenting music videos to participants in three conditions: music, visual, and audiovisual. Participants annotated the music videos for valence and arousal over time, as well as the overall emotion conveyed. We present detailed descriptive statistics for key measures in the dataset and the results of feature importance analyses for each condition. Finally, we propose a novel transfer learning architecture to train Predictive models Augmented with Isolated modality Ratings (PAIR) and demonstrate the potential of isolated modality ratings for enhancing multimodal emotion recognition. Our results suggest that perceptions of arousal are influenced primarily by auditory information, while perceptions of valence are more subjective and can be influenced by both visual and auditory information. The dataset is made publicly available.},
	url = {https://arxiv.org/abs/2202.10453},
	author = {P. Chua and D. Makris and K. Agres and G. Roig and D. Herremans}
}
@article {2022,
	title = {Single Image Video Prediction with Auto-Regressive GANs},
	journal = {Sensors},
	volume = {22},
	year = {2022},
	pages = {3533},
	abstract = {In this paper, we introduce an approach for future frames prediction based on a single input image. Our method is able to generate an entire video sequence based on the information contained in the input frame. We adopt an autoregressive approach in our generation process, i.e., the output from each time step is fed as the input to the next step. Unlike other video prediction methods that use \&ldquo;one shot\&rdquo; generation, our method is able to preserve much more details from the input image, while also capturing the critical pixel-level changes between the frames. We overcome the problem of generation quality degradation by introducing a \&ldquo;complementary mask\&rdquo; module in our architecture, and we show that this allows the model to only focus on the generation of the pixels that need to be changed, and to reuse those that should remain static from its previous frame. We empirically validate our methods against various video prediction models on the UT Dallas Dataset, and show that our approach is able to generate high quality realistic video sequences from one static input image. In addition, we also validate the robustness of our method by testing a pre-trained model on the unseen ADFES facial expression dataset. We also provide qualitative results of our model tested on a human action dataset: The Weizmann Action database.},
	issn = {1424-8220},
	doi = {10.3390/s22093533},
	url = {https://www.mdpi.com/1424-8220/22/9/3533},
	author = {Huang, Jiahui and Chia, Yew Ken and Yu, Samson and Yee, Kevin and K{\"u}ster, Dennis and Krumhuber, Eva G. and Herremans, D and Roig, G.}
}
@article {2022,
	title = {A white paper on cyberphysical learning},
	journal = {White paper, Singapore University of Technology and Design},
	year = {2022},
	url = {https://www.sutd.edu.sg/SUTD/media/SUTD/LSL_WhitePaper_Cyber-physical-Campus-Higher-Education.pdf},
	author = {N. Sockalingam and K. Lo and KurniawaO. n and D. Herremans and N. Raghunath and H. G.C. Cancion and H. Kejun and H. Leong and J. Tan and K. Nizharzharudin and K.L. Pey}
}
@article {2021,
	title = {AttendAffectNet {\textendash} Emotion Prediction of Movie Viewers Using Multimodal Fusion with Self-attention},
	journal = {Sensors. Special issue on Intelligent Sensors: Sensor Based Multi-Modal Emotion Recognition},
	year = {2021},
	abstract = {In this paper, we tackle the problem of predicting the affective responses of movie viewers, based on the content of the movies. Current studies on this topic focus on video representation learning and fusion techniques to combine the extracted features for predicting affect. Yet, those typically , while ignoring the correlation between multiple modality inputs nor the correlation between temporal inputs (i.e., sequential features). To explore these correlations, a neural network architecture namely AttendAffectNet (AAN), which uses the self-attention mechanism for predicting emotions of movie viewers from different input modalities. Particularly, visual, audio, and text features are taken into account for predicting emotions (expressed in terms of valence and arousal). We analyze three variants of our proposed AAN: Feature AAN, Temporal AAN, and Mixed AAN. The Feature AAN applies the self-attention mechanism in an innovative way on the features extracted from the different modalities (including video, audio, and movie subtitles) of a whole movie excerpt, so as to capture the relationship between them. The Temporal AAN takes the time domain of the movies and the sequential dependency of affective responses into account. In the Temporal AAN, the self-attention is applied on the concatenated (multimodal) feature vectors representing different subsequent movie segments. In the Mixed AAN, we combine the strong points of the Feature AAN and the Temporal AAN, by applying the self-attention first on vectors of features obtained from different modalities in each movie segment, and then on the feature representations of all subsequent (temporal) movie segments. We extensively train and validate our proposed AAN on both the MediaEval 2016 dataset for the Emotional Impact of Movies Task and the extended COGNIMUSE dataset. Our experiments demonstrate that audio features play a more influential role than those extracted from video and movie subtitles when predicting emotions of movie viewers on these datasets. The models that use all visual, audio, and text features simultaneously as their inputs perform better than those using features extracted from each modality separately. In addition, the Feature AAN outperforms other AAN variants on the above-mentioned datasets, highlighting the importance of taking different features as context to one another when fusing them. The Feature AAN also performs better than the baseline models when predicting the valence dimension.},
	doi = {10.3390/s21248356},
	url = {https://www.mdpi.com/1424-8220/21/24/8356},
	author = {T. Ha Thi Phuong and Balamurali BT and G. Roig and D. Herremans}
}
@conference {2021,
	title = {AttendAffectNet: Self-Attention based Networks for Predicting Affective Responses from Movies},
	booktitle = {Proceedings of the International Conference on Pattern Recognition (ICPR2020)},
	year = {2021},
	month = {01/2021},
	address = {Milano, Italy (virtual)},
	author = {T. Ha Thi Phuong and Balamurali BT and D. Herremans and G. Roig}
}
@article {2021,
	title = {Music, Computing, and Health: A roadmap for the current and future roles of music technology for healthcare and well-being},
	journal = {Music \& Science},
	year = {2021},
	abstract = {The fields of music, health, and technology have seen significant interactions in recent years in developing music technology for health care and well-being. In an effort to strengthen the collaboration between the involved disciplines, the workshop {\textquotedblleft}Music, Computing, and Health{\textquotedblright} was held to discuss best practices and state-of-the-art at the intersection of these areas with researchers from music psychology and neuroscience, music therapy, music information retrieval, music technology, medical technology (medtech), and robotics. Following the discussions at the workshop, this article provides an overview of the different methods of the involved disciplines and their potential contributions to developing music technology for health and well-being. Furthermore, the article summarizes the state of the art in music technology that can be applied in various health scenarios and provides a perspective on challenges and opportunities for developing music technology that (1) supports person-centered care and evidence-based treatments, and (2) contributes to developing standardized, large-scale research on music-based interventions in an interdisciplinary manner. The article provides a resource for those seeking to engage in interdisciplinary research using music-based computational methods to develop technology for health care, and aims to inspire future research directions by evaluating the state of the art with respect to the challenges facing each field.},
	doi = {https://doi.org/10.1177/205920432199770},
	url = {https://journals.sagepub.com/doi/full/10.1177/2059204321997709},
	author = {K. Agres and Schaefer, Rebecca and Volk, Anja and Van Hooren, Susan and Holzapfel, Andr{\'e} and Dalla Bella, Simone and M{\"u}ller, Meinard and de Witte, Martina and D. Herremans and Ramirez Melendez, Rafael and Neerincx, Mark and Ruiz, Sebastian and Meredith, David and Dimitriadis, Theo and Magee, Wendy}
}
@conference {2020,
	title = {Regression-based music emotion prediction using triplet neural networks},
	booktitle = {Proceedings of the International Joint Conference on Neural Networks (IJCNN)},
	year = {2020},
	month = {07/2020},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Glasgow},
	abstract = {In this paper, we adapt triplet neural networks (TNNs) to a regression task, music emotion prediction. Since TNNs were initially introduced for classification, and not for regression, we propose a mechanism that allows them to provide meaningful low dimensional representations for regression tasks. We then use these new representations as the input for regression algorithms such as support vector machines and gradient boosting machines. To demonstrate the TNNs{\textquoteright} effectiveness at creating meaningful representations, we compare them to different dimensionality reduction methods on music emotion prediction, i.e., predicting valence and arousal values from musical audio signals. Our results on the DEAM dataset show that by using TNNs we achieve 90\% feature dimensionality reduction with a 9\% improvement in valence prediction and 4\% improvement in arousal prediction with respect to our baseline models (without TNN). Our TNN method outperforms other dimensionality reduction methods such as principal component analysis (PCA) and autoencoders (AE). This shows that, in addition to providing a compact latent space representation of audio features, the proposed approach has a relatively high performance over the baseline models.},
	url = {https://arxiv.org/abs/2001.09988},
	author = {K.W. Cheuk and Y.J. Luo and Balamurali BT and G. Roig and D. Herremans}
}
@conference {2019,
	title = {Latent space representation for multi-target speaker detection and identification with a sparse dataset using Triplet neural networks},
	booktitle = {IEEE Automatic Speech Recognition and Understanding Workshop (ASRU 2019)},
	year = {2019},
	month = {12/2019},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Sentosa, Singapore},
	author = {K.W. Cheuk and Balamurali BT and G. Roig and D. Herremans}
}
@conference {2019,
	title = {Multimodal Deep Models for Predicting Affective Responses Evoked by Movies},
	booktitle = {The 2nd International Workshop on Computer Vision for Physiological Measurement as part of ICCV. Seoul, South Korea. 2019},
	year = {2019},
	month = {10/2019},
	address = {Seoul, South Korea},
	abstract = {The goal of this study is to develop and analyze multimodal models for predicting experienced affective responses of viewers watching movie clips. We develop hybrid multimodal prediction models based on both the video and audio of the clips.  For the video content, we hypothesize that both image content and motion are crucial features for evoked emotion prediction. To capture such information, we extract features from  RGB frames and optical flow using pre-trained neural networks. For the audio model, we compute an enhanced set of low-level descriptors including intensity, loudness, cepstrum, linear predictor coefficients, pitch and voice quality. Both visual and audio features are then concatenated to create audio-visual features, which are used to predict the evoked emotion. To classify the movie clips into the corresponding affective response categories, we propose two approaches based on deep neural network models. The first one is based on fully connected layers without memory on the time component, the second incorporates the sequential dependency with a long short-term memory recurrent neural network (LSTM). We perform a thorough analysis of the importance of each feature set. Our experiments reveal that in our set-up, predicting emotions at each time step independently gives slightly better accuracy performance than with the LSTM. Interestingly, we also observe that the optical flow is more informative than the RGB in videos, and overall, models using audio features are more accurate than those based on video features when making the final prediction of evoked emotions. },
	author = {T. Ha Thi Phuong and D. Herremans and G. Roig}
}
@article {129,
	title = {Blacklisted speaker identification using triplet neural networks},
	year = {2018},
	url = {http://mce.csail.mit.edu/pdfs/SUTD_description.pdf},
	author = {K.W. Cheuk and Balamurali BT and G. Roig and D. Herremans}
}
