Publications

@article{nokey,

title = {Audio-Visual Activity Guided Cross-Modal Identity Association for Active Speaker Detection},

author = {Rahul Sharma and Shrikanth Narayanan},

doi = {10.1109/OJSP.2023.3267269},

year  = {2023},

date = {2023-04-14},

urldate = {2023-04-14},

journal = {IEEE Open Journal of Signal Processing },

pages = {225-232},

abstract = {Active speaker detection in videos addresses associating a source face, visible in the video frames, with the underlying speech in the audio modality. The two primary sources of information to derive such a speech-face relationship are i) visual activity and its interaction with the speech signal and ii) co-occurrences of speakers' identities across modalities in the form of face and speech. The two approaches have their limitations: the audio-visual activity models get confused with other frequently occurring vocal activities, such as laughing and chewing, while the speakers' identity-based methods are limited to videos having enough disambiguating information to establish a speech-face association. Since the two approaches are independent, we investigate their complementary nature in this work. We propose a novel unsupervised framework to guide the speakers' cross-modal identity association with the audio-visual activity for active speaker detection. Through experiments on entertainment media videos from two benchmark datasets–the AVA active speaker (movies) and Visual Person Clustering Dataset (TV shows)–we show that a simple late fusion of the two approaches enhances the active speaker detection performance.},

keywords = {active speaker localization, computational media understanding, cross-modal learning, multimedia understanding},

pubstate = {published},

tppubtype = {article}

}

@article{greer2023creating,

title = {Creating musical features using multi-faceted, multi-task encoders based on transformers},

author = {Timothy Greer and Xuan Shi and Benjamin Ma and Shrikanth Narayanan},

year  = {2023},

date = {2023-01-01},

urldate = {2023-01-01},

journal = {Scientific Reports},

volume = {13},

number = {1},

pages = {10713},

publisher = {Nature Publishing Group UK London},

abstract = {Computational machine intelligence approaches have enabled a variety of music-centric technologies in support of creating, sharing and interacting with music content. A strong performance on specific downstream application tasks, such as music genre detection and music emotion recognition, is paramount to ensuring broad capabilities for computational music understanding and Music Information Retrieval. Traditional approaches have relied on supervised learning to train models to support these music-related tasks. However, such approaches require copious annotated data and still may only provide insight into one view of music—namely, that related to the specific task at hand. We present a new model for generating audio-musical features that support music understanding, leveraging self-supervision and cross-domain learning. After pre-training using masked reconstruction of musical input features using self-attention bidirectional transformers, output representations are fine-tuned using several downstream music understanding tasks. Results show that the features generated by our multi-faceted, multi-task, music transformer model, which we call M3BERT, tend to outperform other audio and music embeddings on several diverse music-related tasks, indicating the potential of self-supervised and semi-supervised learning approaches toward a more generalized and robust computational approach to modeling music. Our work can offer a starting point for many music-related modeling tasks, with potential applications in learning deep representations and enabling robust technology applications.},

keywords = {autoencoders, music representations, self-supervision},

pubstate = {published},

tppubtype = {article}

}

@article{Martinez2022,

title = {Boys don’t cry (or kiss or dance): A computational linguistic lens into gendered actions in film},

author = {Victor Martinez and Krishna Somandepalli and Shrikanth Narayanan},

year  = {2022},

date = {2022-12-20},

urldate = {2022-12-20},

journal = {PLoS One},

abstract = { },

keywords = {gendered analysis, multimedia understanding, semantic role labeling},

pubstate = {published},

tppubtype = {article}

}

@article{Sharma2022,

title = {Cross modal video representations for weakly supervised active speaker localization},

author = {Rahul Sharma and Krishna Somandepalli and Shrikanth Narayanan},

url = {https://ieeexplore.ieee.org/document/9991097},

doi = {10.1109/TMM.2022.3229975},

year  = {2022},

date = {2022-12-16},

urldate = {2022-12-16},

journal = {IEEE Transactions on Multimedia},

volume = {Early Access},

pages = {1-12},

abstract = {An objective understanding of media depictions, such as inclusive portrayals of how much someone is heard and seen on screen such as in film and television, requires the machines to discern automatically who, when, how, and where someone is talking, and not. Speaker activity can be automatically discerned from the rich multimodal information present in the media content. This is however a challenging problem due to the vast variety and contextual variability in media content, and the lack of labeled data. In this work, we present a cross-modal neural network for learning visual representations, which have implicit information pertaining to the spatial location of a speaker in the visual frames. Avoiding the need for manual annotations for active speakers in visual frames, acquiring of which is very expensive, we present a weakly supervised system for the task of localizing active speakers in movie content. We use the learned cross-modal visual representations, and provide weak supervision from movie subtitles acting as a proxy for voice activity, thus requiring no manual annotations. Furthermore, we propose an audio-assisted post-processing formulation for the task of active speaker detection. We evaluate the performance of the proposed system on three benchmark datasets: i) AVA active speaker dataset, ii) Visual person clustering dataset, and iii) Columbia datset, and demonstrate the effectiveness of the cross-modal embeddings for localizing active speakers in comparison to fully supervised systems.},

keywords = {active speaker localization, cross-modal learning, multiple instance learning, weakly supervised learning},

pubstate = {published},

tppubtype = {article}

}

@article{Tóth2022,

title = {Studying Large-Scale Behavioral Differences in Auschwitz-Birkenau with Simulation of Gendered Narratives},

author = {Gábor Mihály Tóth and Tim Hempel and Krishna Somandepalli and Shrikanth Narayanan},

url = {http://www.digitalhumanities.org/dhq/vol/16/3/000622/000622.html},

year  = {2022},

date = {2022-08-29},

urldate = {2022-08-29},

journal = {Digital Humanities Quarterly},

volume = {16},

number = {3},

abstract = {In Auschwitz-Birkenau men and women were detained separately; anecdotal evidence suggests that they behaved differently. However, producing evidence based insights into victims' behavior is challenging. Perpetrators frequently destroyed camp documentations; victims' perspective remains dispersed in thousands of oral history interviews with survivors. Listening to, watching, or reading these thousands of interviews is not viable, and there is no established computational approach to gather systematic evidence from a large number of interviews. In this study, by applying methods and concepts of molecular physics, we developed a conceptual framework and computational approach to study thousands of human stories and we investigated 6628 interviews by survivors of the Auschwitz-Birkenau death camp. We applied the concept of state space and the Markov State Model to model the ensemble of 6628 testimonies. The Markov State Model along with the Transition Path Theory allowed us to compare the way women and men remember their time in the camp. We found that acts of solidarity and social bonds are the most important topics in their testimonies. However, we found that women are much more likely to address these topics. We provide systematic evidence that not only were women more likely to recall solidarity and social relations in their belated testimonies but they were also more likely to perform acts of solidarity and form social bonds in Auschwitz-Birkenau. Oral history interviews with Holocaust survivors constitute an important digital cultural heritage that documents one of the darkest moments in human history; generally, oral history collections are ubiquitous sources of modern history and significant assets of libraries and archives. We anticipate that our conceptual and computational framework will contribute not only to the understanding of gender behavior but also to the exploration of oral history as a cultural heritage, as well as to the computational study of narratives. This paper presents novel synergies between history, computer science, and physics, and it aims to stimulate further collaborations between these fields.},

keywords = {Auschwitz, computational narrative modeling, survivor testimonies},

pubstate = {published},

tppubtype = {article}

}

@article{baruah2021representation,

title = {Representation of professions in entertainment media: Insights into frequency and sentiment trends through computational text analysis},

author = {Sabyasachee Baruah and Krishna Somandepalli and Shrikanth Narayanan},

url = {https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0267812},

year  = {2022},

date = {2022-05-18},

journal = {PLoS ONE},

abstract = {Societal ideas and trends dictate media narratives and cinematic depictions which in turn influence people’s beliefs and perceptions of the real world. Media portrayal of individuals and social institutions related to culture, education, government, religion, and family affect their function and evolution over time as people perceive and incorporate the representations from portrayals into their everyday lives. It is important to study media depictions of social structures so that they do not propagate or reinforce negative stereotypes, or discriminate against a particular section of the society. In this work, we examine media representation of different professions and provide computational insights into their incidence, and sentiment expressed, in entertainment media content. We create a searchable taxonomy of professional groups, synsets, and titles to facilitate their retrieval from short-context speaker-agnostic text passages like movie and television (TV) show subtitles. We leverage this taxonomy and relevant natural language processing models to create a corpus of professional mentions in media content, spanning more than 136,000 IMDb titles over seven decades (1950-2017). We analyze the frequency and sentiment trends of different occupations, study the effect of media attributes such as genre, country of production, and title type on these trends, and investigate whether the incidence of professions in media subtitles correlate with their real-world employment statistics. We observe increased media mentions over time of STEM, arts, sports, and entertainment occupations in the analyzed subtitles, and a decreased frequency of manual labor jobs and military occupations. The sentiment expressed toward lawyers, police, and doctors showed increasing negative trends over time, whereas the mentions about astronauts, musicians, singers, and engineers appear more favorably. We found that genre is a good predictor of the type of professions mentioned in movies and TV shows. Professions that employ more people showed increased media frequency.},

keywords = {Media, Professions},

pubstate = {published},

tppubtype = {article}

}

@article{Somandepalli2020MultiFaceSM,

title = {Multi-Face: Self-supervised Multiview Adaptation for Robust Face Clustering in Videos},

author = {Krishna Somandepalli and Rajat Hebbar and Shrikanth S Narayanan},

doi = {10.1109/TMM.2021.3096155},

issn = {1520-9210},

year  = {2021},

date = {2021-07-09},

journal = {IEEE Transactions on Multimedia},

abstract = {Robust face clustering is a vital step in enabling a computational understanding of visual character portrayal in media. Face clustering for long-form content such as movies is challenging because of variations in appearance and lack of large-scale labeled data resources. Our work focuses on two key aspects of this problem: the lack of domain-specific training or benchmark datasets and adapting face embeddings learned on web images to the domain of movie videos. First, we curated over 169,000 face tracks from 240 Hollywood movies with weak labels on whether a pair of face tracks belong to the same or different characters. We proposed an offline nearest-neighbor search in the embedding space to mine hard-examples from these tracks. We then explored triplet-loss and multiview correlation-based methods for adapting face image embeddings to hard-examples from movie videos. We also developed SAIL-Movie Character Benchmark corpus to augment existing benchmarks with more racially diverse characters and provided face-quality labels for subsequent error analysis. Our experimental results highlight the use of weakly labeled data for domain-specific feature adaptation. Overall, we found that multiview correlation-based adaptation yielded robust and more discriminative face embeddings. Its performance on downstream face verification and clustering tasks was comparable to that of the state-of-the-art results in this domain. We hope that the large-scale datasets developed in this work can further advance automatic character labeling in videos. All resources are available at https://sail.usc.edu/~ccmi/multiface.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{Somandepalli2021b,

title = {Robust Character Labeling in Movie Videos: Data Resources and Self-supervised Feature Adaptation.},

author = {Krishna Somandepalli and Rajat Hebbar and Shrikanth Narayanan},

url = {https://sail.usc.edu/publications/files/Somandepalli-TMM2021.pdf},

doi = {10.1109/TMM.2021.3096155},

year  = {2021},

date = {2021-07-09},

urldate = {2021-07-09},

journal = {IEEE Transactions on Multimedia},

volume = {24},

pages = {3355 - 3368},

abstract = {Robust face clustering is a vital step in enabling computational understanding of visual character portrayal in media. Face clustering for long-form content is challenging because of variations in appearance and lack of supporting large-scale labeled data. Our work in this paper focuses on two key aspects of this problem: the lack of domain-specific training or benchmark datasets, and adapting face embeddings learned on web images to long-form content, specifically movies. First, we present a dataset of over 169000 face tracks curated from 240 Hollywood movies with weak labels on whether a pair of face tracks belong to the same or a different character. We propose an offline algorithm based on nearest-neighbor search in the embedding space to mine hard-examples from these tracks. We then investigate triplet-loss and multiview correlation-based methods for adapting face embeddings to hard-examples. Our experimental results highlight the usefulness of weakly labeled data for domain-specific feature adaptation. Overall, we find that multiview correlation-based adaptation yields more discriminative and robust face embeddings. Its performance on downstream face verification and clustering tasks is comparable to that of the state-of-the-art results in this domain. We also present the SAIL-Movie Character Benchmark corpus developed to augment existing benchmarks. It consists of racially diverse actors and provides face-quality labels for subsequent error analysis. We hope that the large-scale datasets developed in this work can further advance automatic character labeling in videos. All resources are available freely at https://sail.usc.edu/~ccmi/multiface .

},

keywords = {computational media understanding, face clustering, face diarization, multiview correlation, self-supervision, triplet loss, video character labeling},

pubstate = {published},

tppubtype = {article}

}

@article{ma2021computational,

title = {A computational lens into how music characterizes genre in film},

author = {Benjamin Ma and Timothy Greer and Dillon Knox and Shrikanth Narayanan},

year  = {2021},

date = {2021-01-01},

journal = {PloS one},

volume = {16},

number = {4},

pages = {e0249957},

publisher = {Public Library of Science San Francisco, CA USA},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{somandepalli2021computational,

title = {Computational media intelligence: human-centered machine analysis of media},

author = {Krishna Somandepalli and Tanaya Guha and Victor R Martinez and Naveen Kumar and Hartwig Adam and Shrikanth Narayanan},

year  = {2021},

date = {2021-01-01},

journal = {Proceedings of the IEEE},

publisher = {IEEE},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{ramakrishna2020joint,

title = {Joint Multi-Dimensional Model for Global and Time-Series Annotations},

author = {Anil Kumar Ramakrishna and Rahul Gupta and Shrikanth Narayanan},

year  = {2020},

date = {2020-01-01},

journal = {IEEE Transactions on Affective Computing},

publisher = {IEEE},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{NarayananMadni-Bridge2020,

title = {Inclusive Human centered Machine Intelligence},

author = {Shrikanth S Narayanan and Asad M Madni},

year  = {2020},

date = {2020-01-01},

journal = {The Bridge},

volume = {50},

pages = {113-116},

publisher = {National Academy of Engineering},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{somandepalli2019multimodal,

title = {Multimodal representation learning using deep multiset canonical correlation},

author = {Krishna Somandepalli and Naveen Kumar and Ruchir Travadi and Shrikanth Narayanan},

year  = {2019},

date = {2019-01-01},

journal = {arXiv preprint arXiv:1904.01775},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{8017484,

title = {Unsupervised Discovery of Character Dictionaries in Animation Movies},

author = {Krishna Somandepalli and Naveen Kumar and Tanaya Guha and Shrikanth S Narayanan},

doi = {10.1109/TMM.2017.2745712},

year  = {2018},

date = {2018-01-01},

journal = {IEEE Transactions on Multimedia},

volume = {20},

number = {3},

pages = {539-551},

keywords = {},

pubstate = {published},

tppubtype = {article}

}