@conference {2026,
	title = {JAM: A Tiny Flow-based Song Generator with Fine-grained Controllability and Aesthetic Alignment},
	booktitle = {Empirical Methods in Natural Language Processing (EMNLP)},
	year = {2026},
	address = {Budapest, Hungary},
	abstract = {Diffusion and flow-matching models have revolutionized automatic text-to-audio generation, increasingly capable of generating high-quality audio outputs for speech and acoustic events. However, creative audio generation involving music and songs still has room for improvement. Recent open lyrics-to-song models like DiffRhythm, ACE-Step, and LeVo have set an acceptable standard for recreational use but lack the fine-grained word-level controllability desired by musicians. To our knowledge, our flow-matching-based JAM is the first to provide word-level timing and duration control in song generation, enabling fine-grained vocal control. To better align generated songs with human preferences, we implement aesthetic alignment through Direct Preference Optimization, iteratively refining the model using a synthetic dataset without manual annotations. Additionally, we aim to standardize evaluation of lyrics-to-song models through our public evaluation dataset JAME. We demonstrate that JAM outperforms open models on music-specific attributes.},
	url = {https://openreview.net/forum?id=3i1qFuRTYV},
	author = {R. Liu and C.Y. Hung and N. Majumder and D. Herremans and S. Poria}
}
@conference {2025,
	title = {Leveraging LLM Embeddings for Cross Dataset Label Alignment and Zero Shot Music Emotion Prediction},
	booktitle = {Conference on AI Music Creativity (AIMC)},
	year = {2026},
	address = {Berlin, Germany},
	abstract = {In this work, we present a novel method for music emotion recognition that leverages Large Language Model (LLM) embeddings for label alignment across multiple datasets and zero-shot prediction on novel categories. First, we compute LLM embeddings for emotion labels and apply non-parametric clustering to group similar labels, across multiple datasets containing disjoint labels. We use these cluster centers to map music features (MERT) to the LLM embedding space. To further enhance the model, we introduce an alignment regularization that enables dissociation of MERT embeddings from different clusters. This further enhances the model{\textquoteright}s ability to better adaptation to unseen datasets. We demonstrate the effectiveness of our approach by performing zero-shot inference on a new dataset, showcasing its ability to generalize to unseen labels without additional training.},
	url = {https://arxiv.org/abs/2410.11522},
	author = {R. Liu and A. Roy and D. Herremans}
}
@conference {2025,
	title = {Measuring and Mitigating Rapport Bias of Large Language Models under Multi-Agent Social Interactions},
	booktitle = {Proceedings of ICLR},
	year = {2026},
	address = {Rio de Janeiro, Brazil},
	abstract = {Large language models (LLMs) are increasingly deployed in multi-agent systems (MAS) as components of collaborative intelligence, where peer interactions dynamically shape individual decision-making. Although prior work has focused on conformity bias, we extend the analysis to examine how LLMs form trust from previous impressions, resist misinformation, and integrate peer input during interaction, key factors for achieving collective intelligence under complex social dynamics. We present KAIROS, a benchmark simulating quiz contests with peer agents of varying reliability, offering fine-grained control over conditions such as expert-novice roles, noisy crowds, and adversarial peers. LLMs receive both historical interactions and current peer responses, allowing systematic investigation into how trust, peer action, and self-confidence influence decisions. As for mitigation strategies, we evaluate prompting, supervised fine-tuning, and reinforcement learning, Group Relative Policy Optimisation (GRPO), across multiple models. Our results reveal that GRPO with multi-agent context combined with outcome-based rewards and unconstrained reasoning achieves the best overall performance, but also decreases the robustness to social influence compared to Base models. The code and datasets are available at: https://github.com/declare-lab/KAIROS},
	url = {https://arxiv.org/abs/2508.18321},
	author = {M. Song and T. D. Pala and W. Jin and A. Zadeh and C. Li and D. Herremans and S. Poria}
}
@conference {2025,
	title = {MelodySim: Measuring Melody-aware Music Similarity for Plagiarism Detection},
	booktitle = {IEEE Tencon},
	year = {2026},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Bali, Indonesia},
	abstract = {We propose MelodySim, a melody-aware music similarity model and dataset for plagiarism detection. First, we introduce a novel method to construct a dataset focused on melodic similarity. By augmenting Slakh2100, an existing MIDI dataset, we generate variations of each piece while preserving the melody through modifications such as note splitting, arpeggiation, minor track dropout, and re-instrumentation. A user study confirms that positive pairs indeed contain similar melodies, while other musical tracks are significantly changed. Second, we develop a segment-wise melodic-similarity detection model that uses a MERT encoder and applies a triplet neural network to capture melodic similarity. The resulting decision matrix highlights where plagiarism might occur. The experiments show that our model is able to outperform baseline models in detecting similar melodic fragments on the MelodySim test set.},
	url = {https://arxiv.org/abs/2505.20979},
	author = {T. Lu and Geist, C-M and J. Melechovsky and A. Roy and D. Herremans}
}
@conference {2026,
	title = {nnAudio 2: Overcoming Dynamic Compilation Barriers and Transform Inconsistencies},
	booktitle = {Conference on AI Music Creativity (AIMC)},
	year = {2026},
	address = {Berlin, Germany},
	abstract = {nnAudio is an open-source audio feature extraction toolbox for deep learning, but its use in current environments is hindered by TorchScript incompatibilities, inverse-transform edge cases, and dependency drift. We present a targeted modernization for modern PyTorch and scientific Python. We resolve TorchScript compilation failures in STFT and iSTFT by removing dynamic state mutation and module construction from scripted code paths and tightening argument handling in inverse-related helpers. We clarify inverse-STFT behavior by restricting reliable inversion to the uniform-bin setting (freq_scale={\textquoteleft}no{\textquoteright}) and raising explicit runtime errors for unsupported frequency scales, preventing silently degraded reconstructions. We restore CFP compatibility with modern SciPy and ensure VQT reduces to CQT when gamma = 0. Regression tests cover the new STFT/iSTFT behaviors, and the updated codebase passes the full repository test suite in a modern Python environment. These improvements provide a more robust foundation for differentiable audio analysis in research and deployment.},
	url = {https://arxiv.org/abs/2606.05394},
	author = {Roy, Abhinaba and Liang, Junyi and Herremans, Dorien}
}
@article {2024,
	title = {BandCondiNet: Parallel Transformers-based Conditional Popular Music Generation with Multi-View Features},
	journal = {Expert Systems with Applications},
	volume = {130059},
	year = {2025},
	abstract = {Conditional music generation offers significant advantages in terms of user convenience and control, presenting great potential in AI-generated content research. However, building conditional generative systems for multitrack popular songs presents three primary challenges: insufficient fidelity of input conditions, poor structural modeling, and inadequate inter-track harmony learning in generative models. To address these issues, we propose BandCondiNet, a conditional model based on parallel Transformers, designed to process the multiple music sequences and generate high-quality multitrack samples. Specifically, we propose multi-view features across time and instruments as high-fidelity conditions. Moreover, we propose two specialized modules for BandCondiNet: Structure Enhanced Attention (SEA) to strengthen the musical structure, and Cross-Track Transformer (CTT) to enhance inter-track harmony. We conducted both objective and subjective evaluations on two popular music datasets with different sequence lengths. Objective results on the shorter dataset show that BandCondiNet outperforms other conditional models in 9 out of 10 metrics related to fidelity and inference speed, with the exception of Chord Accuracy. On the longer dataset, BandCondiNet surpasses all conditional models across all 10 metrics. Subjective evaluations across four criteria reveal that BandCondiNet trained on the shorter dataset performs best in Richness and performs comparably to state-of-the-art models in the other three criteria, while significantly outperforming them across all criteria when trained on the longer dataset. To further expand the application scope of BandCondiNet, future work should focus on developing an advanced conditional model capable of adapting to more user-friendly input conditions and supporting flexible instrumentation.},
	url = {https://arxiv.org/abs/2407.10462},
	author = {J. Luo and X. Yang and D. Herremans}
}
@conference {2025,
	title = {Coarse-to-Fine Text-to-Music Latent Diffusion},
	booktitle = {Proceedings of ICASSP},
	year = {2025},
	publisher = {IEEE},
	organization = {IEEE},
	abstract = {We introduce DiscoDiff, a text-to-music generative model that utilizes two latent diffusion models to produce high-fidelity 44.1kHz music hierarchically. Our approach significantly enhances audio quality through a coarse-to-fine generation strategy, leveraging residual vector quantization from the Descript Audio Codec. We consolidate this coarse-to-fine design through an important observation that the audio latent representation of can be split into primary and secondary part, controlling music contents and details accordingly. We validate the effectiveness of our approach and text-audio alignment through various objective metrics. Furthermore, we provide access to high-quality synthetic captions for the MTG-Jamendo and FMA datasets, as well as open-sourcing DiscoDiff{\textquoteright}s codebase and model checkpoints.},
	url = {https://openreview.net/pdf/b3dcd6d5d6c26679621a2e6c176455d59658c0a8.pdf},
	author = {L.A. Lanzend{\"o}rfer and T. Lu and N. Perraudin and D. Herremans and R. Wattenhofer}
}
@article {2025,
	title = {Demystifying deep search: a holistic evaluation with hint-free multi-hop questions and factorised metrics},
	journal = {arXiv:2510.05137},
	year = {2025},
	url = {https://arxiv.org/abs/2510.05137},
	author = {M. Song and R. Liu and Wang, Xinyu and Jiang, Yong and Xie, Pengjun and Huang, Fei and Zhou, Jingren and D. Herremans and S. Poria}
}
@article {2022,
	title = {Forecasting Bitcoin Volatility Spikes from Whale Transactions and Cryptoquant Data Using Synthesizer Transformer Models},
	journal = {IEEE Access},
	volume = {13},
	year = {2025},
	pages = {117788-117807},
	abstract = {The cryptocurrency market is highly volatile compared to traditional financial markets. Hence, forecasting its volatility is crucial for risk management. In this paper, we investigate CryptoQuant data (e.g. on-chain analytics, exchange and miner data) and whale-alert tweets, and explore their relationship to Bitcoin{\textquoteright}s next-day volatility, with a focus on extreme volatility spikes. We propose a deep learning Synthesizer Transformer model for forecasting volatility. Our results show that the model outperforms existing state-of-the-art models when forecasting extreme volatility spikes for Bitcoin using CryptoQuant data as well as whale-alert tweets. We analysed our model with the Captum XAI library to investigate which features are most important. We also backtested our prediction results with different baseline trading strategies and the results show that we are able to minimize drawdown while keeping steady profits. Our findings underscore that the proposed method is a useful tool for forecasting extreme volatility movements in the Bitcoin market.},
	issn = {2169-3536},
	doi = {10.1109/ACCESS.2025.3584243},
	url = {https://ieeexplore.ieee.org/document/11058926},
	author = {D. Herremans and K.W. Low}
}
@conference {2025,
	title = {ImprovNet: Generating Controllable Musical Improvisations with Iterative Corruption Refinement},
	booktitle = {Proceedings of IJCNN},
	year = {2025},
	address = {Rome, Italy},
	abstract = {Deep learning has enabled remarkable advances in style transfer across various domains, offering new possibilities for creative content generation. However, in the realm of symbolic music, generating controllable and expressive performance-level style transfers for complete musical works remains challenging due to limited datasets, especially for genres such as jazz, and the lack of unified models that can handle multiple music generation tasks. This paper presents ImprovNet, a transformer-based architecture that generates expressive and controllable musical improvisations through a self-supervised corruption-refinement training strategy. ImprovNet unifies multiple capabilities within a single model: it can perform cross-genre and intra-genre improvisations, harmonize melodies with genre-specific styles, and execute short prompt continuation and infilling tasks. The model{\textquoteright}s iterative generation framework allows users to control the degree of style transfer and structural similarity to the original composition. Objective and subjective evaluations demonstrate ImprovNet{\textquoteright}s effectiveness in generating musically coherent improvisations while maintaining structural relationships with the original pieces. The model outperforms Anticipatory Music Transformer in short continuation and infilling tasks and successfully achieves recognizable genre conversion, with 79\% of participants correctly identifying jazz-style improvisations. Our code and demo page can be found at this https URL (https://github.com/keshavbhandari/improvnet).},
	doi = {10.1109/IJCNN64981.2025.11229312},
	url = {https://arxiv.org/abs/2502.04522},
	author = {K. Bhandari and S. Chang and T. Lu and F. R. Enus and L. B. Bradshaw and D. Herremans and S. Colton}
}
@conference {2025,
	title = {JamendoMaxCaps: A Large Scale Music-caption Dataset with Imputed Metadata},
	booktitle = {Proceedings of IJCNN, Rome, Italy},
	year = {2025},
	abstract = {We introduce JamendoMaxCaps, a large-scale music-caption dataset featuring over 200,000 freely licensed instrumental tracks from the renowned Jamendo platform. The dataset includes captions generated by a state-of-the-art captioning model, enhanced with imputed metadata. We also introduce a retrieval system that leverages both musical features and metadata to identify similar songs, which are then used to fill in missing metadata using a local large language model (LLLM). This approach allows us to provide a more comprehensive and informative dataset for researchers working on music-language understanding tasks. We validate this approach quantitatively with five different measurements. By making the JamendoMaxCaps dataset publicly available, we provide a high-quality resource to advance research in music-language understanding tasks such as music retrieval, multimodal representation learning, and generative music models.},
	doi = {10.1109/IJCNN64981.2025.11228034},
	url = {https://arxiv.org/abs/2502.07461},
	author = {A. Roy and R. Liu and T. Lu and D. Herremans}
}
@article {2024,
	title = {Natural Language Processing Methods for Symbolic Music Generation and Information Retrieval: a Survey},
	journal = {ACM Computing Surveys},
	year = {2025},
	abstract = {Several adaptations of Transformers models have been developed in various domains since its breakthrough in Natural Language Processing (NLP). This trend has spread into the field of Music Information Retrieval (MIR), including studies processing music data. However, the practice of leveraging NLP tools for symbolic music data is not novel in MIR. Music has been frequently compared to language, as they share several similarities, including sequential representations of text and music. These analogies are also reflected through similar tasks in MIR and NLP. This survey reviews NLP methods applied to symbolic music generation and information retrieval studies following two axes. We first propose an overview of representations of symbolic music adapted from natural language sequential representations. Such representations are designed by considering the specificities of symbolic music. These representations are then processed by models. Such models, possibly originally developed for text and adapted for symbolic music, are trained on various tasks. We describe these models, in particular deep learning models, through different prisms, highlighting music-specialized mechanisms. We finally present a discussion surrounding the effective use of NLP tools for symbolic music data. This includes technical issues regarding NLP methods and fundamental differences between text and music, which may open several doors for further research into more effectively adapting NLP tools to symbolic MIR.},
	doi = {https://arxiv.org/abs/2402.17467},
	url = {https://arxiv.org/abs/2402.17467},
	author = {Dinh-Viet-Toan Le and L. Bigo and M. Keller and D. Herremans}
}
@article {2025,
	title = {PRESENT: Zero-Shot Text-to-Prosody Control},
	journal = {IEEE Signal Processing Letters},
	year = {2025},
	abstract = {Current strategies for achieving fine-grained prosody control in speech synthesis entail extracting additional style embeddings or adopting more complex architectures. To enable zero-shot application of pretrained text-to-speech (TTS) models, we present PRESENT (PRosody Editing without Style Embeddings or New Training), which exploits explicit prosody prediction in FastSpeech2-based models by modifying the inference process directly. We apply our text-to-prosody framework to zero-shot language transfer using a JETS model exclusively trained on English LJSpeech data. We obtain character error rates (CER) of 12.8\%, 18.7\% and 5.9\% for German, Hungarian and Spanish respectively, beating the previous state-of-the-art CER by over 2x for all three languages. Furthermore, we allow subphoneme-level control, a first in this field. To evaluate its effectiveness, we show that PRESENT can improve the prosody of questions, and use it to generate Mandarin, a tonal language where vowel pitch varies at subphoneme level. We attain 25.3\% hanzi CER and 13.0\% pinyin CER with the JETS model. All our code and audio samples are available online.},
	doi = {10.1109/LSP.2025.3528359},
	url = {https://ieeexplore.ieee.org/document/10838710},
	author = {P. Lam and H. Zhang and N. F. Chen and B. Sisman and D. Herremans}
}
@article {2025,
	title = {Towards the future of education: cyber-physical learning},
	journal = {Discover Education},
	volume = {4},
	year = {2025},
	pages = {1{\textendash}16},
	abstract = {Singapore University of Technology and Design (SUTD) is embarking on an educational innovation program called SUTD campusX to support its future of education. SUTD campusX aims to innovate new educational models, technology tools, and pedagogies for a new form of learning called {\textquotedblleft}Cyber-Physical Learning{\textquotedblright}, where the concept is that both remote cyber students and face-to-face physical students can learn and interact effectively, seamlessly, and synchronously in the same class. SUTD campusX focuses on various streams of emerging technologies such as learning analytics, immersive technologies (e.g., AR/VR/MR, metaverse learning, gamification), telepresence robotics, and personalized learning. SUTD campusX supports and innovates the signature interdisciplinary, student-centric, and team-based problem-solving active learning model of SUTD to transform it for the cyber-physical learning environment. The various innovation streams of campusX not only generate insightful learning data to create new educational frameworks and practices but also enable a learning analytics and AI-assisted platform to deliver personalized adaptive learning. In this perspective article, we describe and reflect on the forward-looking nature of the various innovation themes and how these innovations act as the front-facing technologies to collect and generate learning insights to power an AI-driven cyber-physical learning platform to realize the future of Cyber-Physical Learning at SUTD.},
	doi = {https://doi.org/10.1007/s44217-025-00474-x},
	url = {https://link.springer.com/article/10.1007/s44217-025-00474-x},
	author = {Sockalingam, N. and Lo, K. and Teo, Ju. and Wei, C.C. and Chow, D. and Herremans, D. and Jun, M.L.M. and Kurniawan, O. and Wang, Y. and Leong, P. K.}
}
@conference {2024,
	title = {Coarse-to-Fine Text-to-Music Latent Diffusion},
	booktitle = {Audio Imagination: NeurIPS 2024 Workshop},
	year = {2024},
	address = {Vancouver},
	abstract = {We introduce DiscoDiff, a text-to-music generative model that utilizes two latent diffusion models to produce high-fidelity 44.1kHz music hierarchically. Our approach significantly enhances audio quality through a coarse-to-fine generation strategy, leveraging residual vector quantization from the Descript Audio Codec.  We consolidate this coarse-to-fine design through an important observation that the audio latent representation of can be split into primary and secondary part, controlling music contents and details accordingly. We validate the effectiveness of our approach and text-audio alignment through various objective metrics. Furthermore, we provide access to high-quality synthetic captions for the MTG-Jamendo and FMA datasets, as well as open-sourcing DiscoDiff{\textquoteright}s codebase and model checkpoints.},
	author = {L.A. Lanzend{\"o}rfer and T. Lu and N. Perraudin and D. Herremans and R. Wattenhofer}
}
@conference {2022,
	title = {SNIPER Training: Variable Sparsity Rate Training For Text-To-Speech},
	booktitle = {Proc. of IEEE Tencon, Singapore},
	year = {2024},
	abstract = {Text-to-speech (TTS) models have achieved remarkable naturalness in recent years, yet like most deep neural models, they have more parameters than necessary. Sparse TTS models can improve on dense models via pruning and extra retraining, or converge faster than dense models with some performance loss. Inspired by these results, we propose training TTS models using a decaying sparsity rate, i.e. a high initial sparsity to accelerate training first, followed by a progressive rate reduction to obtain better eventual performance. This decremental approach differs from current methods of incrementing sparsity to a desired target, which costs significantly more time than dense training. We call our method SNIPER training: Single-shot Initialization Pruning Evolving-Rate training. Our experiments on FastSpeech2 show that although we were only able to obtain better losses in the first few epochs before being overtaken by the baseline, the final SNIPER-trained models beat constant-sparsity models and pip dense models in performance.},
	url = {https://arxiv.org/abs/2211.07283},
	author = {P. Lam and H. Zhang and N. F. Chen and B. Sisman and D. Herremans}
}
@conference {2022,
	title = {Computationally Efficient Physics Approximating Neural Networks for Highly Nonlinear Maps},
	booktitle = {2022 International Conference on Research in Adaptive and Convergent Systems},
	year = {2022},
	publisher = {2022},
	organization = {2022},
	address = {Virtual},
	doi = {10.1145/3538641.3561501},
	url = {https://dl.acm.org/doi/abs/10.1145/3538641.3561501},
	author = {C.J. Clarke and J. Chowdhury and Balamurali BT and P. Priyadarshinee and C.M. Ying Lim and I. Fu Xing Tan and D. Herremans and J.M. Chen}
}
@article {2022,
	title = {Downscaling using Deep Convolutional Autoencoders, a case study for South East Asia},
	journal = {Egusphere preprint},
	year = {2022},
	abstract = {Inspired by recent advancements in the field of computer vision, specifically models for generating higher-resolution images from low-resolution images, we investigate the utility of a deep convolutional autoencoder for downscaling and bias correcting climate projections for South East Asia (SEA). Downscaled projections of 2 m surface temperature are generated, using autoencoders trained with data from the Coupled Model Intercomparison Project Phase 5 (CMIP5) and data from the fifth generation ECMWF atmospheric reanalysis (ERA5) project. Using CMIP5 projections as an input, three sets of downscaled data are generated using three methods of autoencoder training, which allow us to determine how autoencoder downscaling and bias correction modify temperature values. Where possible, the downscaled outputs are compared against the Southeast Asia Regional Climate Downscaling/Coordinated Regional Climate Downscaling Experiment{\textendash}Southeast Asia (SEACLID/CORDEX{\textendash}SEA) project and outputs from available CMIP6 experiments, to evaluate performance. The autoencoders are found to excel at the rapid generation of highly spatially-resolved climate projections for surface temperature. Realistic spatial features due to coastal and topographic variation are generated by the autoencoder, which are not present in the CMIP5 projections. Additionally, the autoencoders are capable of generating forecast data with regional temperature profiles exceeding that of those appearing in the training set (out-of-sample extrapolation). Seasonal temperature cycles are retained after downscaling throughout the region, despite the absence of temporal information provided to the model. However, autoencoders trained to carry out bias correction display a tendency to smooth daily average temperatures and reduce daily highs and lows beyond that which can be expected to be realistic. Without bias correction, downscaled outputs have a reduced improvement in spatial resolution but the daily temperature profiles of the CMIP5 input forecasts are maintained. Autoencoders rely on the presence of structural features in the datasets to carry out downscaling, and so performance over the oceans is reduced as strong temperature gradients are absent. For this reason, ocean warming is not well represented, an artefact which is not immediately clear in the downscaled outputs. This study demonstrates the importance of rigorous analysis of {\textquoteright}black-box{\textquoteright} methods, which can generate non-obvious artefacts that could potentially create misleading results. Despite these limitations, Autoencoders are clearly capable of generating much needed high-resolution climate projections, and strategies to improve upon shortcomings are numerous and well established. },
	doi = {https://doi.org/10.5194/egusphere-2022-234},
	url = {https://egusphere.copernicus.org/preprints/2022/egusphere-2022-234/},
	author = {O. D. Levers and D. Herremans and A. Dipankar and L. Blessing}
}
@conference {2022,
	title = {A Gaussian mixture classifier model to differentiate respiratory symptoms using phonated /ɑ:/ sounds},
	booktitle = {The 18th Australasian International Conference on Speech Science and Technology (SST)},
	year = {2022},
	month = {12/2022},
	publisher = {ASSTA},
	organization = {ASSTA},
	address = {Canberra, Australia},
	abstract = {An audio-based classification model that differentiates between healthy vs pathological respiratory symptoms using acoustic features extracted from phonated /ɑ:/ sounds is presented. For this, a new dataset of phonated /ɑ:/ sounds, together with a clinician{\textquoteright}s diagnosis, was compiled and a Gaussian Mixture Model (GMM) using Mel-Frequency Cepstral Coefficients (MFCCs) classifier was used. Despite no significant differences in mean values of the fundamental and formant frequency (F0, F1, F2, and F3) distribution for /ɑ:/ sounds retrieved from healthy vs pathological populations, our /ɑ:/ sound model trained using MFCCs resulted in an accuracy of 81.92\% when compared against clinician{\textquoteright}s diagnosis.},
	url = {https://sst2022.com/a-gaussian-mixture-classifier-model-to-differentiate-respiratory-symptoms-using-phonated-a\%cb\%90-sounds/},
	author = {Balamurali BT and H.I. Hee and C. Ming and Y. Lin and P. Priyadarshinee and C.J. Clarke and D. Herremans and J.M. Chen}
}
@conference {2022,
	title = {Jointist: Joint Learning for Multi-instrument Transcription and Its Applications},
	year = {2022},
	publisher = {Arxiv},
	organization = {Arxiv},
	abstract = {In this paper, we introduce Jointist, an instrument-aware multi-instrument framework that is capable of transcribing, recognizing, and separating multiple musical instruments from an audio clip. Jointist consists of the instrument recognition module that conditions the other modules: the transcription module that outputs instrument-specific piano rolls, and the source separation module that utilizes instrument information and transcription results.
The instrument conditioning is designed for an explicit multi-instrument functionality while the connection between the transcription and source separation modules is for better transcription performance. Our challenging problem formulation makes the model highly useful in the real world given that modern popular music typically consists of multiple instruments. However, its novelty necessitates a new perspective on how to evaluate such a model. During the experiment, we assess the model from various aspects, providing a new evaluation perspective for multi-instrument transcription. We also argue that transcription models can be utilized as a preprocessing module for other music analysis tasks. In the experiment on several downstream tasks, the symbolic representation provided by our transcription model turned out to be helpful to spectrograms in solving downbeat detection, chord recognition, and key estimation.},
	doi = { https://doi.org/10.48550/arXiv.2206.10805},
	url = {https://arxiv.org/abs/2206.10805},
	author = {K.W. Cheuk and K. Choi and Q. Kong and B. Li and M. Won and A. Hung and J.-C. Wang and D. Herremans}
}
@article {2022,
	title = {A white paper on cyberphysical learning},
	journal = {White paper, Singapore University of Technology and Design},
	year = {2022},
	url = {https://www.sutd.edu.sg/SUTD/media/SUTD/LSL_WhitePaper_Cyber-physical-Campus-Higher-Education.pdf},
	author = {N. Sockalingam and K. Lo and KurniawaO. n and D. Herremans and N. Raghunath and H. G.C. Cancion and H. Kejun and H. Leong and J. Tan and K. Nizharzharudin and K.L. Pey}
}
@article {2021,
	title = {Deep Neural Network Based Respiratory Pathology Classification Using Cough Sounds},
	journal = {Sensors},
	volume = {21},
	year = {2021},
	pages = {5555},
	abstract = {Intelligent systems are transforming the world, as well as our healthcare system. We propose a deep learning-based cough sound classification model that can distinguish between children with healthy versus pathological coughs such as asthma, upper respiratory tract infection (URTI), and lower respiratory tract infection (LRTI). In order to train a deep neural network model, we collected a new dataset of cough sounds, labelled with clinician{\textquoteright}s diagnosis. The chosen model is a bidirectional long-short term memory network (BiLSTM) based on Mel Frequency Cepstral Coefficients (MFCCs) features. The resulting trained model when trained for classifying two classes of coughs -- healthy or pathology (in general or belonging to a specific respiratory pathology), reaches accuracy exceeding 84\% when classifying cough to the label provided by the physicians{\textquoteright} diagnosis. In order to classify subject{\textquoteright}s respiratory pathology condition, results of multiple cough epochs per subject were combined. The resulting prediction accuracy exceeds 91\% for all three respiratory pathologies. However, when the model is trained to classify and discriminate among the four classes of coughs, overall accuracy dropped: one class of pathological coughs are often misclassified as other. However, if one consider the healthy cough classified as healthy and pathological cough classified to have some kind of pathologies, then the overall accuracy of four class model is above 84\%. A longitudinal study of MFCC feature space when comparing pathological and recovered coughs collected from the same subjects revealed the fact that pathological cough irrespective of the underlying conditions occupy the same feature space making it harder to differentiate only using MFCC features.},
	doi = {10.3390/s21165555},
	url = {https://www.mdpi.com/1424-8220/21/16/5555/htm},
	author = {Balamurali B T and Hwan Ing Hee and Saumitra Kapoor and Oon Hoe Teoh and Sung Shin Teng and Khai Pin Lee and Dorien Herremans and Jer Ming Chen}
}
@conference {2020,
	title = {The Effect of Spectrogram Reconstructions on Automatic Music Transcription:An Alternative Approach to Improve Transcription Accuracy},
	booktitle = {Proceedings of the International Conference on Pattern Recognition (ICPR2020)},
	year = {2021},
	month = {01/2021},
	address = {Milano, Italy (virtual)},
	author = {K.W. Cheuk and Y.J. Luo and E. Benetos and D. Herremans}
}
@conference {2021,
	title = {Revisiting the Onsets and Frames Model with Additive Attention},
	booktitle = {Proceedings of the International Joint Conference on Neural Networks (IJCNN)},
	year = {2021},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Virtual},
	author = {K.W. Cheuk and Y.J. Luo and E. Benetos and D. Herremans}
}
@article {2021,
	title = {Underwater Acoustic Communication Receiver Using Deep Belief Network},
	journal = {IEEE Transactions on Communications},
	year = {2021},
	pages = {1-1},
	abstract = {Underwater environments create a challenging channel for communications. In this paper, we design a novel receiver system by exploring the machine learning technique{\textendash}Deep Belief Network (DBN){\textendash} to combat the signal distortion caused by the Doppler effect and multi-path propagation. We evaluate the performance of the proposed receiver system in both simulation experiments and sea trials. Our proposed receiver system comprises of DBN based de-noising and classification of the received signal. First, the received signal is segmented into frames before the each of these frames is individually pre-processed using a novel pixelization algorithm. Then, using the DBN based de-noising algorithm, features are extracted from these frames and used to reconstruct the received signal. Finally, DBN based classification of the reconstructed signal occurs. Our proposed DBN based receiver system does show better performance in channels influenced by the Doppler effect and multi-path propagation with a performance improvement of 13.2dB at 10-3 Bit Error Rate (BER).},
	doi = {10.1109/TCOMM.2021.3063353},
	url = {https://ieeexplore.ieee.org/document/9367188},
	author = {A. Lee-Leon and C. Yuen and D. Herremans}
}
@article {2020,
	title = {Asthmatic versus healthy child classification based on cough and vocalised /a:/ sounds},
	journal = {The Journal of the Acoustical Society of America (JASA)},
	volume = {148, EL253},
	year = {2020},
	month = {09/2020},
	type = {Jasa Express Letters},
	abstract = {Cough is a common symptom presenting in asthmatic children. In this investigation, an audio-based classification model is presented that can differentiate between healthy and asthmatic children, based on the combination of cough and vocalised /ɑ:/ sounds. A Gaussian mixture model using mel-frequency cepstral coefficients and constant-Q cepstral coefficients was trained. When comparing the predicted labels with the clinician{\textquoteright}s diagnosis, this cough sound model reaches an overall accuracy of 95.3\%. The vocalised /ɑ:/ model reaches an accuracy of 72.2\%, which is still significant because the dataset contains only 333 /ɑ:/ sounds versus 2029 cough sounds.
},
	doi = {10.1121/10.0001933},
	url = {https://asa.scitation.org/doi/10.1121/10.0001933},
	author = {Balamurali BT and H.I. Hee and O.H. Teoh and K.P. Lee and S. Kapoor and D. Herremans and J.M. Chen}
}
@proceedings {2020,
	title = {Generative Modelling for Controllable Audio Synthesis of Expressive Piano Performance},
	year = {2020},
	month = {07/2020},
	publisher = {arXiv:2006.09833},
	abstract = {We present a controllable neural audio synthesizer based on Gaussian Mixture Variational Autoencoders (GM-VAE), which can generate realistic piano performances in the audio domain that closely follows temporal conditions of two essential style features for piano performances: articulation and dynamics. We demonstrate how the model is able to apply fine-grained style morphing over the course of synthesizing the audio. This is based on conditions which are latent variables that can be sampled from the prior or inferred from other pieces. One of the envisioned use cases is to inspire creative and brand new interpretations for existing pieces of piano music.},
	url = {https://arxiv.org/abs/2006.09833},
	author = {Hao Hao Tan and Y.J. Luo and D. Herremans}
}
@conference {2020,
	title = {PerceptionGAN: Real-world image construction from provided text through perceptual understanding},
	booktitle = {4th Int. Conf. on Imaging, Vision and Pattern Recognition (IVPR), and 9th Int. Conf. on Informatics, Electronics \& Vision (ICIEV)},
	year = {2020},
	month = {08/2020},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Kitakyushu, Japan},
	abstract = {Generating an image from a provided descriptive text is quite a challenging task because of the difficulty in incorporating perceptual information (object shapes, colors, and their interactions) along with providing high relevancy related to the provided text. Current methods first generate an initial low-resolution image, which typically has irregular object shapes, colors, and interaction between objects. This initial image is then improved by conditioning on the text. However, these methods mainly address the problem of using text representation efficiently in the refinement of the initially generated image, while the success of this refinement process depends heavily on the quality of the initially generated image, as pointed out in the Dynamic Memory Generative Adversarial Network (DM-GAN) paper. Hence, we propose a method to provide good initialized images by incorporating perceptual understanding in the discriminator module. We improve the perceptual information at the first stage itself, which results in significant improvement in the final generated image. In this paper, we have applied our approach to the novel StackGAN architecture. We then show that the perceptual information included in the initial image is improved while modeling image distribution at multiple stages. Finally, we generated realistic multi-colored images conditioned by text. These images have good quality along with containing improved basic perceptual information. More importantly, the proposed method can be integrated into the pipeline of other state-of-the-art text-based-image-generation models such as DM-GAN and AttnGAN to generate initial low-resolution images. We also worked on improving the refinement process in StackGAN by augmenting the third stage of the generator-discriminator pair in the StackGAN architecture. Our experimental analysis and comparison with the state-of-the-art on a large but sparse dataset MS COCO further validate the usefulness of our proposed approach.},
	author = {K. Garg and A. Singh and D. Herremans and B. Lall}
}
@conference {2020,
	title = {Regression-based music emotion prediction using triplet neural networks},
	booktitle = {Proceedings of the International Joint Conference on Neural Networks (IJCNN)},
	year = {2020},
	month = {07/2020},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Glasgow},
	abstract = {In this paper, we adapt triplet neural networks (TNNs) to a regression task, music emotion prediction. Since TNNs were initially introduced for classification, and not for regression, we propose a mechanism that allows them to provide meaningful low dimensional representations for regression tasks. We then use these new representations as the input for regression algorithms such as support vector machines and gradient boosting machines. To demonstrate the TNNs{\textquoteright} effectiveness at creating meaningful representations, we compare them to different dimensionality reduction methods on music emotion prediction, i.e., predicting valence and arousal values from musical audio signals. Our results on the DEAM dataset show that by using TNNs we achieve 90\% feature dimensionality reduction with a 9\% improvement in valence prediction and 4\% improvement in arousal prediction with respect to our baseline models (without TNN). Our TNN method outperforms other dimensionality reduction methods such as principal component analysis (PCA) and autoencoders (AE). This shows that, in addition to providing a compact latent space representation of audio features, the proposed approach has a relatively high performance over the baseline models.},
	url = {https://arxiv.org/abs/2001.09988},
	author = {K.W. Cheuk and Y.J. Luo and Balamurali BT and G. Roig and D. Herremans}
}
@conference {2019,
	title = {Singing voice conversion with disentangled representations of singer and vocal technique using variational autoencoders},
	booktitle = {IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP)},
	year = {2020},
	month = {05/2020},
	address = {Barcelona, Spain},
	url = {https://arxiv.org/abs/1912.02613},
	author = {Y.J. Luo and C.-C. Hsu and K. Agres and D. Herremans}
}
@conference {2020,
	title = {Unsupervised disentanglement of pitch and timbre for isolated musical instrument sounds},
	booktitle = {Proceedings of the International Society of Music Information Retrieval (ISMIR)},
	year = {2020},
	month = {10/2020},
	abstract = {Disentangling factors of variation aims to uncover latent variables that underlie the process of data generation. In this paper, we propose a framework that achieves unsupervised pitch and timbre disentanglement for isolated musical instrument sounds without relying on data annotations or pre-trained neural networks. Our framework, based on variational auto-encoders, takes as input a spectral frame, and encodes pitch and timbre as categorical and continuous variables, respectively. The input is then reconstructed by combining those variables. Under an unsupervised training setting, a major challenge is that encoders are tasked to capture factors of interest with distinct latent representations, without access to the corresponding ground-truth labels. We therefore introduce auxiliary tasks and objectives which leverage pitch shifting as a strategy to create surrogate labels, thereby encouraging the disentanglement of pitch and timbre. Through an ablation study we analyze the impact of the proposed objectives. The evaluation shows the efficacy of the proposed framework for learning disentangled representations, and verifies its applicability to unsupervised pitch classification and conditional spectral synthesis.},
	url = {https://arxiv.org/abs/1906.08152},
	author = {Y.J. Luo and K.W. Cheuk and T. Nakano and M. Goto and D. Herremans}
}
@article {9998,
	title = {Development of Machine Learning for asthmatic and healthy voluntary  cough - a proof of concept study},
	journal = {Applied Sciences},
	volume = {9},
	year = {2019},
	month = {07/2019},
	chapter = {2833},
	abstract = {(1) Background: Cough is a major presentation in childhood asthma. Here, we aim to develop a machine-learning based cough sound classifier for asthmatic and healthy children. (2) Methods: Children less than 16 years old were randomly recruited in a Children{\textquoteright}s Hospital, from February 2017 to April 2018, and were divided into 2 cohorts{\textemdash}healthy children and children with acute asthma presenting with cough. Children with other concurrent respiratory conditions were excluded in the asthmatic cohort. Demographic data, duration of cough, and history of respiratory status were obtained. Children were instructed to produce voluntary cough sounds. These clinically labeled cough sounds were randomly divided into training and testing sets. Audio features such as Mel-Frequency Cepstral Coefficients and Constant-Q Cepstral Coefficients were extracted. Using a training set, a classification model was developed with Gaussian Mixture Model{\textendash}Universal Background Model (GMM-UBM). Its predictive performance was tested using the test set against the physicians{\textquoteright} labels. (3) Results: Asthmatic cough sounds from 89 children (totaling 1192 cough sounds) and healthy coughs from 89 children (totaling 1140 cough sounds) were analyzed. The sensitivity and specificity of the audio-based classification model was 82.81\% and 84.76\%, respectively, when differentiating coughs from asthmatic children versus coughs from {\textquoteleft}healthy{\textquoteright} children. (4) Conclusion: Audio-based classification using machine learning is a potentially useful technique in assisting the differentiation of asthmatic cough sounds from healthy voluntary cough sounds in children.},
	doi = {10.3390/app9142833},
	url = {https://www.mdpi.com/2076-3417/9/14/2833},
	author = {H.I. Hee and Balamurali BT and A. Karunakaran and D. Herremans and O.H. Teoh and K.P. Lee and S.S. Teng and S. Lui and J.M. Chen}
}
@conference {2019,
	title = {Doppler Invariant Demodulation for Shallow Water Acoustic Communications Using Deep Belief Networks},
	booktitle = {16th IEEE Asia Pacific Wireless Communications Symposium (APWCS)},
	year = {2019},
	abstract = {Shallow water environments create a challenging channel for communications. In this paper, we focus on the challenges posed by the frequency-selective signal distortion called the Doppler effect. We explore the design and performance of machine learning (ML) based demodulation methods {\textemdash} (1) Deep Belief Network-feed forward Neural Network (DBN-NN) and (2) Deep Belief Network-Convolutional Neural Network (DBN-CNN) in the physical layer of Shallow Water Acoustic Communication (SWAC). The proposed method comprises of a ML based feature extraction method and classification technique. First, the feature extraction converts the received signals to feature images. Next, the classification model correlates the images to a corresponding binary representative. An analysis of the ML based proposed demodulation shows that despite the presence of instantaneous frequencies, the performance of the algorithm shows an invariance with a small 2dB error margin in terms of bit error rate (BER).},
	author = {A. Lee-Leon and C. Yuen and D. Herremans}
}
@conference {2019,
	title = {A Hybrid Fuzzy Logic-Neural Network Approach For Multi-path Separation Of Underwater Acoustic Signals},
	booktitle = {89th IEEE Vehicular Technology Conference},
	year = {2019},
	month = {04/2019},
	address = {Kuala Lumpur, Malaysia},
	abstract = {Abstract{\textemdash}Underwater acoustic channels are generally recognized as one of the most difficult communication media in use today. One of the most important constraints of underwater communications is the acoustic propagation of the signals. The aim of a multi-path separator is to extract individual correlated signals from their mixtures. We present an algorithm, Tag Receiver, that is similar to the RAKE receiver, but interpreted from the neural network viewpoint. First, the received signal is split into frames. Next, the frames are {\textquotedblleft}tagged{\textquotedblright} with a predicted number of multi-path in the framed segment via fuzzy logic and feature extraction. Finally, the frame and {\textquotedblleft}tag{\textquotedblright} number are inputted into a neural network, which separates the signals. The analysis of the proposed algorithm shows an improvement from the conventional RAKE receiver in terms of BER.},
	doi = {10.1109/VTCSpring.2019.8746614},
	url = {https://ieeexplore.ieee.org/abstract/document/8746614},
	author = {A. Lee-Leon and C. Yuen and D. Herremans}
}
@conference {2019,
	title = {Learning Disentangled Representations of Timbre and Pitch for Musical Instrument Sounds Using Gaussian Mixture Variational Autoencoders},
	booktitle = {ISMIR},
	year = {2019},
	publisher = {ISMIR},
	organization = {ISMIR},
	address = {Delft, The Netherlands},
	url = {https://arxiv.org/abs/1906.08152},
	author = {Y.J. Luo and K. Agres and D. Herremans}
}
@conference {2019,
	title = {A novel music-based game with motion capture to support cognitive and motor function in the elderly},
	booktitle = {IEEE Conference on Games},
	year = {2019},
	publisher = {IEEE },
	organization = {IEEE },
	address = {London, UK},
	abstract = {This paper presents a novel game prototype that uses music and motion detection as preventive medicine for the elderly.
Given the aging populations around the globe, and the limited resources and staff able to care for these populations, eHealth solutions are becoming increasingly important, if not crucial, additions to modern healthcare and preventive medicine. Furthermore, because compliance rates for performing physical exercises are often quite low in the elderly, systems able to motivate and engage this population are a necessity. Our prototype uses music not only to engage listeners, but also to leverage the efficacy of music to improve mental and physical wellness. The game is based on a memory task to stimulate cognitive function, and requires users to perform physical gestures to mimic the playing of different musical instruments. To this end, the Microsoft Kinect sensor is used together with a newly developed gesture detection module in order to process users{\textquoteright} gestures. The resulting prototype system supports both cognitive functioning and physical strengthening in the elderly. },
	author = {K. Agres and S. Lui and D. Herremans}
}
@article {2019,
	title = {Towards robust audio spoofing detection: a detailed comparison of traditional and learned features},
	journal = {IEEE Access},
	volume = {7},
	year = {2019},
	pages = {84229 - 84241},
	doi = {10.1109/ACCESS.2019.2923806},
	url = {https://arxiv.org/abs/1905.12439},
	author = {Balamurali BT and K.W.E. Lin and S. Lui and J.M. Chen and D. Herremans}
}
@article {119,
	title = {Minimally Simple Binaural Room Modelling Using a Single Feedback Delay Network},
	journal = {Journal of the Audio Engineering Society},
	volume = {66},
	year = {2018},
	month = {10/2018},
	pages = {791-807},
	abstract = {The most efficient binaural acoustic modeling systems use a multi-tap delay to generate accurately modeled early reflections, combined with a feedback delay network that produces generic late reverberation. We present a method of binaural acoustic simulation that uses one feedback delay network to simultaneously model both first-order reflections and late reverberation. The advantages are simplicity and efficiency. We compare the proposed method against the existing method of modeling binaural early reflections using a multi-tap delay line. Measurements of ISO standard evaluators including interaural correlation coefficient, decay time, clarity, definition, and center time, indicate that the proposed method achieves comparable level of accuracy as less-efficient existing methods. This method is implemented as an iOS application, and is able to auralize input signal directly without convolution and update in real time.},
	doi = {https://doi.org/10.17743/jaes.2018.0045},
	author = {N. Agus and H. Anderson and J.M. Chen and S. Lui and D. Herremans}
}
@article {117,
	title = {Perceptual evaluation of measures of spectral variance},
	journal = {Journal of the Acoustical Society of America},
	volume = {143},
	year = {2018},
	pages = {3300{\textendash}3311},
	abstract = {In many applications, it is desirable to achieve a signal that is as close as possible to ideal white noise. One example is in the design of an artificial reverberator, whereby there is a need for its lossless prototype output from an impulse input to be perceptually white as much as possible. The Ljung-Box test, the Drouiche test, and the Wiener Entropy{\textemdash}also called the Spectral Flatness Measure{\textemdash}are three well-known methods for quantifying the similarity of a given signal to ideal
white noise. In this paper, listening tests are conducted to measure the Just Noticeable Difference (JND) on the perception of white noise, which is the JND between ideal Gaussian white noise and noise with a specified deviation from the flat spectrum. This paper reports the JND values using one of these measures of whiteness, which is the Ljung-Box test. This paper finds considerable disagreement between the Ljung-Box test and the other two methods and shows that none of the methods is a significantly better predictor of listeners{\textquoteright} perception of whiteness. This suggests a need for a whiteness test that is more closely correlated to human perception.},
	doi = {10.1121/1.5040484},
	url = {https://asa.scitation.org/doi/10.1121/1.5040484},
	author = {N. Agus and H. Anderson and J.M. Chen and S. Lui and D. Herremans}
}
@article {134,
	title = {Singing Voice Separation Using a Deep Convolutional Neural Network Trained by Ideal Binary Mask and Cross Entropy},
	journal = {Neural Computing and Applications},
	year = {2018},
	abstract = {Separating a singing voice from its music accompaniment remains an important challenge in the field of music information retrieval. We present a unique neural network approach inspired by a technique that has revolutionized the field of vision: pixel-wise image classification, which we combine with cross entropy loss and pretraining of the CNN as an autoencoder on singing voice spectrograms. The pixel-wise classification technique directly estimates the sound source label for each time-frequency (T-F) bin in our spectrogram image, thus eliminating common pre-and postprocessing tasks. The proposed network is trained by using the Ideal Binary Mask (IBM) as the target output label. The IBM identifies the dominant sound source in each T-F bin of the magnitude spectrogram of a mixture signal, by considering each T-F bin as a pixel with a multi-label (for each sound source). Cross entropy is used as the training objective, so as to minimize the average probability error between the target and predicted label for each pixel. By treating the singing voice separation problem as a pixel-wise classification task, we additionally eliminate one of the commonly used, yet not easy to comprehend, postprocessing steps: the Wiener filter postprocessing. The proposed CNN outperforms the first runner up in the Music Information Retrieval Evaluation eXchange (MIREX) 2016 and the winner of MIREX 2014 with a gain of 2.2702 \~{} 5.9563 dB global normalized source to distortion ratio (GNSDR) when applied to the iKala dataset. An experiment with the DSD100 dataset on the full-tracks song evaluation task also shows that our model is able to compete with cutting-edge singing voice separation systems which use multichannel modeling, data augmentation, and model blending.},
	doi = {10.1007/s00521-018-3933-z},
	url = {https://rdcu.be/bdoTY},
	author = {K.W.E. Lin and Balamurali BT and E. Koh and S. Lui and D. Herremans}
}
@proceedings {100,
	title = {Visualizing the evolution of alternative hit charts},
	year = {2017},
	month = {10/2017},
	address = {Suzhou, China},
	author = {D. Herremans and W. Lauwers}
}
