@conference {2022,
	title = {DiffRoll: Diffusion-based Generative Music Transcription with Unsupervised Pretraining Capability},
	booktitle = {ICASSP},
	year = {2023},
	abstract = {In this paper we propose a novel generative approach, DiffRoll, to tackle automatic music transcription (AMT). Instead of treating AMT as a discriminative task in which the model is trained to convert spectrograms into piano rolls, we think of it as a conditional generative task where we train our model to generate realistic looking piano rolls from pure Gaussian noise conditioned on spectrograms. This new AMT formulation enables DiffRoll to transcribe, generate and even inpaint music. Due to the classifier-free nature, DiffRoll is also able to be trained on unpaired datasets where only piano rolls are available. Our experiments show that DiffRoll outperforms its discriminative counterpart by 19 percentage points (ppt.) and our ablation studies also indicate that it outperforms similar existing methods by 4.8 ppt.
Source code and demonstration are available <a href="https://sony.github.io/DiffRoll/">here</a>.},
	author = {K.W. Cheuk and Sawata, Ryosuke and Uesaka, Toshimitsu and Murata, Naoki and Takahashi, Naoya and Takahashi, Shusuke and D. Herremans and Mitsufuji, Yuki}
}
@article {2022,
	title = {MERP: A Music Dataset with Emotion Ratings and Raters{\textquoteright} Profile Information},
	journal = {Sensors - Intelligent Sensors},
	volume = {23},
	year = {2023},
	month = {01/2023},
	chapter = {382},
	abstract = {Music is capable of conveying many emotions. The level and type of emotion of the music perceived by a listener, however, is highly subjective. In this study, we present the Music Emotion Recognition with Profile information dataset (MERP). This database was collected through Amazon Mechanical Turk (MTurk) and features dynamical valence and arousal ratings of 54 selected full-length songs. The dataset contains music features, as well as user profile information of the annotators. The songs were selected from the Free Music Archive using an innovative method (a Triple Neural Network with the OpenSmile toolkit) to identify 50 songs with the most distinctive emotions. Specifically, the songs were chosen to fully cover the four quadrants of the valence arousal space. Four additional songs were selected from DEAM to act as a benchmark in this study and filter out low quality ratings. A total of 277 participants participated in annotating the dataset, and their demographic information, listening preferences, and musical background were recorded. We offer an extensive analysis of the resulting dataset, together with a baseline emotion prediction model based on a fully connected model and an LSTM model, for our newly proposed MERP dataset.},
	doi = {10.3390/s23010382},
	url = {https://www.mdpi.com/1424-8220/23/1/382},
	author = {E. Koh and K.W. Cheuk and K.Y. Heung and K. Agres and D. Herremans}
}
@conference {2022,
	title = {Jointist: Joint Learning for Multi-instrument Transcription and Its Applications},
	year = {2022},
	publisher = {Arxiv},
	organization = {Arxiv},
	abstract = {In this paper, we introduce Jointist, an instrument-aware multi-instrument framework that is capable of transcribing, recognizing, and separating multiple musical instruments from an audio clip. Jointist consists of the instrument recognition module that conditions the other modules: the transcription module that outputs instrument-specific piano rolls, and the source separation module that utilizes instrument information and transcription results.
The instrument conditioning is designed for an explicit multi-instrument functionality while the connection between the transcription and source separation modules is for better transcription performance. Our challenging problem formulation makes the model highly useful in the real world given that modern popular music typically consists of multiple instruments. However, its novelty necessitates a new perspective on how to evaluate such a model. During the experiment, we assess the model from various aspects, providing a new evaluation perspective for multi-instrument transcription. We also argue that transcription models can be utilized as a preprocessing module for other music analysis tasks. In the experiment on several downstream tasks, the symbolic representation provided by our transcription model turned out to be helpful to spectrograms in solving downbeat detection, chord recognition, and key estimation.},
	doi = { https://doi.org/10.48550/arXiv.2206.10805},
	url = {https://arxiv.org/abs/2206.10805},
	author = {K.W. Cheuk and K. Choi and Q. Kong and B. Li and M. Won and A. Hung and J.-C. Wang and D. Herremans}
}
@conference {2022,
	title = {Understanding Audio Features via Trainable Basis Functions},
	booktitle = {Arxiv preprint},
	year = {2022},
	abstract = {In this paper we explore the possibility of maximizing the information represented in spectrograms by making the spectrogram basis functions trainable. We experiment with two different tasks, namely keyword spotting (KWS) and automatic speech recognition (ASR). For most neural network models, the architecture and hyperparameters are typically fine-tuned and optimized in experiments. Input features, however, are often treated as fixed. In the case of audio, signals can be mainly expressed in two main ways: raw waveforms (time-domain) or spectrograms (time-frequency-domain). In addition, different spectrogram types are often used and tailored to fit different applications. In our experiments, we allow for this tailoring directly as part of the network.
Our experimental results show that using trainable basis functions can boost the accuracy of Keyword Spotting (KWS) by 14.2 percentage points, and lower the Phone Error Rate (PER) by 9.5 percentage points. Although models using trainable basis functions become less effective as the model complexity increases, the trained filter shapes could still provide us with insights on which frequency bins are important for that specific task. From our experiments, we can conclude that trainable basis functions are a useful tool to boost the performance when the model complexity is limited.},
	url = {https://arxiv.org/abs/2204.11437},
	author = {Y.H. Kwan and K.W. Cheuk and D. Herremans}
}
@conference {2020,
	title = {The Effect of Spectrogram Reconstructions on Automatic Music Transcription:An Alternative Approach to Improve Transcription Accuracy},
	booktitle = {Proceedings of the International Conference on Pattern Recognition (ICPR2020)},
	year = {2021},
	month = {01/2021},
	address = {Milano, Italy (virtual)},
	author = {K.W. Cheuk and Y.J. Luo and E. Benetos and D. Herremans}
}
@conference {2021,
	title = {ReconVAT: A Semi-Supervised Automatic Music Transcription Framework for Low-Resource Real-World Data},
	booktitle = {ACM Multimedia},
	year = {2021},
	publisher = {ACM},
	organization = {ACM},
	abstract = {Most of the current supervised automatic music transcription (AMT) models lack the ability to generalize. This means that they have trouble transcribing real-world music recordings from diverse musical genres which are not presented in the labelled training data. In this paper, we propose a semi-supervised framework, ReconVAT, which solves this issue by leveraging the huge amount of available unlabelled music recordings. The proposed ReconVAT uses reconstruction loss and virtual adversarial training. When combined with existing U-net models for AMT, ReconVAT shows competitive performance on the common benchmark datasets such as MAPS and MusicNet. For example, in the few-shot setting for the string part version of MusicNet, ReconVAT achieves F1-scores of 61.0\% and 41.6\% for the note-wise and note-with-offset-wise metrics respectively, which translate to improvements of 22.2\% and 62.5\% over the supervised baseline model. Our proposed framework also demonstrates the potential of continuous learning on new data, which could be used in real-world applications such as online training and transcribing instrumental covers of pop music.},
	url = {https://dl.acm.org/doi/10.1145/3474085.3475405},
	author = {K.W. Cheuk and L. Su and D. Herremans}
}
@conference {2021,
	title = {Revisiting the Onsets and Frames Model with Additive Attention},
	booktitle = {Proceedings of the International Joint Conference on Neural Networks (IJCNN)},
	year = {2021},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Virtual},
	author = {K.W. Cheuk and Y.J. Luo and E. Benetos and D. Herremans}
}
@conference {2020,
	title = {The impact of Audio input representations on neural network based music transcription},
	booktitle = {Proceedings of the International Joint Conference on Neural Networks (IJCNN)},
	year = {2020},
	month = {07/2020},
	address = {Glasgow},
	abstract = {This paper thoroughly analyses the effect of different input representations on polyphonic multi-instrument music transcription. We use our own GPU based spectrogram extraction tool, nnAudio, to investigate the influence of using a linear-frequency spectrogram, log-frequency spectrogram, Mel spectrogram, and constant-Q transform (CQT). Our results show that a 8.33\% increase in transcription accuracy and a 9.39\% reduction in error can be obtained by choosing the appropriate input representation (log-frequency spectrogram with STFT window length 4,096 and 2,048 frequency bins in the spectrogram) without changing the neural network design (single layer fully connected). Our experiments also show that Mel spectrogram is a compact representation for which we can reduce the number of frequency bins to only 512 while still keeping a relatively high music transcription accuracy.},
	url = {https://arxiv.org/abs/2001.09989},
	author = {K.W. Cheuk and K. Agres and D. Herremans}
}
@article {2019,
	title = {nnAudio: An on-the-fly GPU Audio to Spectrogram Conversion Toolbox Using 1D Convolution Neural Networks},
	journal = {IEEE Access},
	year = {2020},
	abstract = {Converting time domain waveforms to frequency domain spectrograms is typically considered to be a prepossessing step done before model training. This approach, however, has several drawbacks. First, it takes a lot of hard disk space to store different frequency domain representations. This is especially true during the model development and tuning process, when exploring various types of spectrograms for optimal performance. Second, if another dataset is used, one must process all the audio clips again before the network can be retrained. In this paper, we integrate the time domain to frequency domain conversion as part of the model structure, and propose a neural network based toolbox, nnAudio, which leverages 1D convolutional neural networks to perform time domain to frequency domain conversion during feed-forward. It allows on-the-fly spectrogram generation without the need to store any spectrograms on the disk. This approach also allows back-propagation on the waveforms-to-spectrograms transformation layer, which implies that this transformation process can be made trainable, and hence further optimized by gradient descent. nnAudio reduces the waveforms-to-spectrograms conversion time for 1,770 waveforms (from the MAPS dataset) from 10.64 seconds with librosa to only 0.001 seconds for Short-Time Fourier Transform (STFT), 18.3 seconds to 0.015 seconds for Mel spectrogram, 103.4 seconds to 0.258 for constant-Q transform (CQT), when using GPU on our DGX work station with CPU: Intel(R) Xeon(R) CPU E5-2698 v4 @ 2.20GHz Tesla v100 32Gb GPUs. (Only 1 GPU is being used for all the experiments.) We also further optimize the existing CQT algorithm, so that the CQT spectrogram can be obtained without aliasing in a much faster computation time (from 0.258 seconds to only 0.001 seconds).},
	doi = {10.1109/ACCESS.2020.3019084},
	url = {https://ieeexplore.ieee.org/document/9174990},
	author = {K.W. Cheuk and H. Anderson and K. Agres and D. Herremans}
}
@conference {2020,
	title = {Regression-based music emotion prediction using triplet neural networks},
	booktitle = {Proceedings of the International Joint Conference on Neural Networks (IJCNN)},
	year = {2020},
	month = {07/2020},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Glasgow},
	abstract = {In this paper, we adapt triplet neural networks (TNNs) to a regression task, music emotion prediction. Since TNNs were initially introduced for classification, and not for regression, we propose a mechanism that allows them to provide meaningful low dimensional representations for regression tasks. We then use these new representations as the input for regression algorithms such as support vector machines and gradient boosting machines. To demonstrate the TNNs{\textquoteright} effectiveness at creating meaningful representations, we compare them to different dimensionality reduction methods on music emotion prediction, i.e., predicting valence and arousal values from musical audio signals. Our results on the DEAM dataset show that by using TNNs we achieve 90\% feature dimensionality reduction with a 9\% improvement in valence prediction and 4\% improvement in arousal prediction with respect to our baseline models (without TNN). Our TNN method outperforms other dimensionality reduction methods such as principal component analysis (PCA) and autoencoders (AE). This shows that, in addition to providing a compact latent space representation of audio features, the proposed approach has a relatively high performance over the baseline models.},
	url = {https://arxiv.org/abs/2001.09988},
	author = {K.W. Cheuk and Y.J. Luo and Balamurali BT and G. Roig and D. Herremans}
}
@conference {2020,
	title = {Unsupervised disentanglement of pitch and timbre for isolated musical instrument sounds},
	booktitle = {Proceedings of the International Society of Music Information Retrieval (ISMIR)},
	year = {2020},
	month = {10/2020},
	abstract = {Disentangling factors of variation aims to uncover latent variables that underlie the process of data generation. In this paper, we propose a framework that achieves unsupervised pitch and timbre disentanglement for isolated musical instrument sounds without relying on data annotations or pre-trained neural networks. Our framework, based on variational auto-encoders, takes as input a spectral frame, and encodes pitch and timbre as categorical and continuous variables, respectively. The input is then reconstructed by combining those variables. Under an unsupervised training setting, a major challenge is that encoders are tasked to capture factors of interest with distinct latent representations, without access to the corresponding ground-truth labels. We therefore introduce auxiliary tasks and objectives which leverage pitch shifting as a strategy to create surrogate labels, thereby encouraging the disentanglement of pitch and timbre. Through an ablation study we analyze the impact of the proposed objectives. The evaluation shows the efficacy of the proposed framework for learning disentangled representations, and verifies its applicability to unsupervised pitch classification and conditional spectral synthesis.},
	url = {https://arxiv.org/abs/1906.08152},
	author = {Y.J. Luo and K.W. Cheuk and T. Nakano and M. Goto and D. Herremans}
}
@conference {2019,
	title = {Latent space representation for multi-target speaker detection and identification with a sparse dataset using Triplet neural networks},
	booktitle = {IEEE Automatic Speech Recognition and Understanding Workshop (ASRU 2019)},
	year = {2019},
	month = {12/2019},
	publisher = {IEEE},
	organization = {IEEE},
	address = {Sentosa, Singapore},
	author = {K.W. Cheuk and Balamurali BT and G. Roig and D. Herremans}
}
@conference {2019,
	title = {nnAudio: A PyTorch Audio Processing Tool Using 1D Convolution neural networks},
	booktitle = {ISMIR - Late Breaking Demo},
	year = {2019},
	month = {10/2019},
	address = {Delft, The Netherlands},
	author = {K.W. Cheuk and K. Agres and D. Herremans}
}
@article {129,
	title = {Blacklisted speaker identification using triplet neural networks},
	year = {2018},
	url = {http://mce.csail.mit.edu/pdfs/SUTD_description.pdf},
	author = {K.W. Cheuk and Balamurali BT and G. Roig and D. Herremans}
}
