@article{

    wang2023codet5plus,

    title={CodeT5+: Open Code Large Language Models for Code Understanding and Generation},

    author={Wang, Yue and Le, Hung and Gotmare, Akhilesh Deepak and Bui, Nghi D.Q. and Li, Junnan and Hoi, Steven C. H.},

    journal={arXiv preprint},

    year={2023}

}

@inproceedings{le-etal-2022-multimodal,

    title = "Multimodal Dialogue State Tracking",

    author = "Le, Hung  and

      Chen, Nancy  and

      Hoi, Steven",

    booktitle = "Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies",

    month = jul,

    year = "2022",

    address = "Seattle, United States",

    publisher = "Association for Computational Linguistics",

    url = "https://aclanthology.org/2022.naacl-main.248",

    pages = "3394--3415",

    abstract = "Designed for tracking user goals in dialogues, a dialogue state tracker is an essential component in a dialogue system. However, the research of dialogue state tracking has largely been limited to unimodality, in which slots and slot values are limited by knowledge domains (e.g. restaurant domain with slots of restaurant name and price range) and are defined by specific database schema. In this paper, we propose to extend the definition of dialogue state tracking to multimodality. Specifically, we introduce a novel dialogue state tracking task to track the information of visual objects that are mentioned in video-grounded dialogues. Each new dialogue utterance may introduce a new video segment, new visual objects, or new object attributes and a state tracker is required to update these information slots accordingly. We created a new synthetic benchmark and designed a novel baseline, Video-Dialogue Transformer Network (VDTN), for this task. VDTN combines both object-level features and segment-level features and learns contextual dependencies between videos and dialogues to generate multimodal dialogue states. We optimized VDTN for a state generation task as well as a self-supervised video understanding task which recovers video segment or object representations. Finally, we trained VDTN to use the decoded states in a response prediction task. Together with comprehensive ablation and qualitative analysis, we discovered interesting insights towards building more capable multimodal dialogue systems.",

}

@inproceedings{le-etal-2021-dvd,

    title = "{DVD}: A Diagnostic Dataset for Multi-step Reasoning in Video Grounded Dialogue",

    author = "Le, Hung  and

      Sankar, Chinnadhurai  and

      Moon, Seungwhan  and

      Beirami, Ahmad  and

      Geramifard, Alborz  and

      Kottur, Satwik",

    booktitle = "Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)",

    month = aug,

    year = "2021",

    address = "Online",

    publisher = "Association for Computational Linguistics",

    url = "https://aclanthology.org/2021.acl-long.439",

    doi = "10.18653/v1/2021.acl-long.439",

    pages = "5651--5665",

    abstract = "A video-grounded dialogue system is required to understand both dialogue, which contains semantic dependencies from turn to turn, and video, which contains visual cues of spatial and temporal scene variations. Building such dialogue systems is a challenging problem, involving various reasoning types on both visual and language inputs. Existing benchmarks do not have enough annotations to thoroughly analyze dialogue systems and understand their capabilities and limitations in isolation. These benchmarks are also not explicitly designed to minimise biases that models can exploit without actual reasoning. To address these limitations, in this paper, we present DVD, a Diagnostic Dataset for Video-grounded Dialogue. The dataset is designed to contain minimal biases and has detailed annotations for the different types of reasoning over the spatio-temporal space of video. Dialogues are synthesized over multiple question turns, each of which is injected with a set of cross-turn semantic relationships. We use DVD to analyze existing approaches, providing interesting insights into their abilities and limitations. In total, DVD is built from 11k CATER synthetic videos and contains 10 instances of 10-round dialogues for each video, resulting in more than 100k dialogues and 1M question-answer pairs. Our code and dataset are publicly available.",

}

@inproceedings{le-etal-2020-bist,

    title = "{B}i{ST}: Bi-directional Spatio-Temporal Reasoning for Video-Grounded Dialogues",

    author = "Le, Hung  and

      Sahoo, Doyen  and

      Chen, Nancy  and

      Hoi, Steven C.H.",

    booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)",

    month = nov,

    year = "2020",

    address = "Online",

    publisher = "Association for Computational Linguistics",

    url = "https://www.aclweb.org/anthology/2020.emnlp-main.145",

    doi = "10.18653/v1/2020.emnlp-main.145",

    pages = "1846--1859",

    abstract = "Video-grounded dialogues are very challenging due to (i) the complexity of videos which contain both spatial and temporal variations, and (ii) the complexity of user utterances which query different segments and/or different objects in videos over multiple dialogue turns. However, existing approaches to video-grounded dialogues often focus on superficial temporal-level visual cues, but neglect more fine-grained spatial signals from videos. To address this drawback, we proposed Bi-directional Spatio-Temporal Learning (BiST), a vision-language neural framework for high-resolution queries in videos based on textual cues. Specifically, our approach not only exploits both spatial and temporal-level information, but also learns dynamic information diffusion between the two feature spaces through spatial-to-temporal and temporal-to-spatial reasoning. The bidirectional strategy aims to tackle the evolving semantics of user queries in the dialogue setting. The retrieved visual cues are used as contextual information to construct relevant responses to the users. Our empirical results and comprehensive qualitative analysis show that BiST achieves competitive performance and generates reasonable responses on a large-scale AVSD benchmark. We also adapt our BiST models to the Video QA setting, and substantially outperform prior approaches on the TGIF-QA benchmark.",

}

@inproceedings{

Le2020Non-Autoregressive,

title={Non-Autoregressive Dialog State Tracking},

author={Hung Le and Richard Socher and Steven C.H. Hoi},

booktitle={International Conference on Learning Representations},

year={2020},

url={https://openreview.net/forum?id=H1e_cC4twS}

}

@article{le2018urlnet,

  title={URLNet: Learning a URL representation with deep learning for malicious URL detection},

  author={Le, Hung and Pham, Quang and Sahoo, Doyen and Hoi, Steven CH},

  journal={arXiv preprint arXiv:1802.03162},

  year={2018}

}