Publications

@conference{nokey,

title = {MM-AU: Towards Multimodal Understanding of Advertisement Videos},

author = {Digbalay Bose and Rajat Hebbar and Tiantian Feng and Krishna Somandepalli and Anfeng Xu and Shrikanth Narayanan },

year  = {2023},

date = {2023-10-29},

urldate = {2023-10-29},

publisher = {ACM Multimedia },

keywords = {advertisements, computational media understanding, content analysis, multimedia understanding, multimodal},

pubstate = {published},

tppubtype = {conference}

}

@conference{bose-etal-2023-emotion-recognition,

title = {Contextually-rich human affect perception using multimodal scene information},

author = {Digbalay Bose and Rajat Hebbar and Krishna Somandepalli and Shrikanth Narayanan },

doi = {https://arxiv.org/abs/2303.06904},

year  = {2023},

date = {2023-03-13},

urldate = {2023-03-13},

publisher = {IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP) },

abstract = {The process of human affect understanding involves the ability to infer person specific emotional states from various sources including images, speech, and language. Affect perception from images has predominantly focused on expressions extracted from salient face crops. However, emotions perceived by humans rely on multiple contextual cues including social settings, foreground interactions, and ambient visual scenes. In this work, we leverage pretrained vision-language (VLN) models to extract descriptions of foreground context from images. Further, we propose a multimodal context fusion (MCF) module to combine foreground cues with the visual scene and person-based contextual information for emotion prediction. We show the effectiveness of our proposed modular design on two datasets associated with natural scenes and TV shows.},

keywords = {emotion recognition, multimedia understanding, multimodal},

pubstate = {published},

tppubtype = {conference}

}

@conference{avramidi-etal-vcmr,

title = {On the Role of Visual Context in Enriching Music Representations},

author = {Kleanthis Avramidis and Shanti Stewart and Shrikanth Narayanan},

doi = {https://arxiv.org/abs/2210.15828},

year  = {2023},

date = {2023-02-15},

urldate = {2023-02-15},

publisher = {IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP) },

abstract = {Human perception and experience of music is highly context-dependent. Contextual variability contributes to differences in how we interpret and interact with music, challenging the design of robust models for information retrieval. Incorporating multimodal context from diverse sources provides a promising approach toward modeling this variability. Music presented in media such as movies and music videos provide rich multimodal context that modulates underlying human experiences. However, such context modeling is underexplored, as it requires large amounts of multimodal data along with relevant annotations. Self-supervised learning can help address these challenges by automatically extracting rich, high-level correspondences between different modalities, hence alleviating the need for fine-grained annotations at scale. In this study, we propose VCMR -- Video-Conditioned Music Representations, a contrastive learning framework that learns music representations from audio and the accompanying music videos. The contextual visual information enhances representations of music audio, as evaluated on the downstream task of music tagging. Experimental results show that the proposed framework can contribute additive robustness to audio representations and indicates to what extent musical elements are affected or determined by visual context.},

keywords = {multimedia understanding, multimodal, music representations},

pubstate = {published},

tppubtype = {conference}

}