From ac6a8510a4678b4c7238d4e598a4e75604c6744d Mon Sep 17 00:00:00 2001 From: Colin Leong <122366389+cleong110@users.noreply.github.com> Date: Thu, 16 May 2024 16:55:39 -0400 Subject: [PATCH 01/11] CDL: add costerQueryingSignLanguage2023 to references.bib --- src/references.bib | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/references.bib b/src/references.bib index 2975c301..24ab0f86 100644 --- a/src/references.bib +++ b/src/references.bib @@ -2175,3 +2175,15 @@ @inproceedings{chenSimpleMultiModalityTransfer2022a urldate = {2024-05-13}, langid = {english} } + +@inproceedings{costerQueryingSignLanguage2023, + title = {Querying {{A Sign Language Dictionary}} with {{Videos Using Dense Vector Search}}}, + booktitle = {2023 {{IEEE International Conference}} on {{Acoustics}}, {{Speech}}, and {{Signal Processing Workshops}} ({{ICASSPW}})}, + author = {Coster, Mathieu De and Dambre, Joni}, + year = {2023}, + month = jun, + pages = {1--5}, + doi = {10.1109/ICASSPW59220.2023.10193531}, + url = {https://ieeexplore.ieee.org/document/10193531}, + urldate = {2024-03-13} +} From 1fa827e447cbd2311767495b59e774c90d6b481a Mon Sep 17 00:00:00 2001 From: Colin Leong <122366389+cleong110@users.noreply.github.com> Date: Thu, 16 May 2024 17:29:58 -0400 Subject: [PATCH 02/11] CDL: add SL Retrieval section and summary of costerQueryingSignLanguage2023 --- src/index.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/index.md b/src/index.md index 02ff123a..0f7760e6 100644 --- a/src/index.md +++ b/src/index.md @@ -299,6 +299,7 @@ encoding subtitles by BERT [@devlin-etal-2019-bert] and videos by CNN video repr + ### Sign Language Recognition, Translation, and Production Sign language translation (SLT) commonly refers to the translation of signed language to spoken language [@de2022machine;@muller-etal-2022-findings]. @@ -851,6 +852,25 @@ TODO ``` +### Sign Language Retrieval + +Sign Language Retrieval is the task of finding a particular data item, given some input. In contrast to translation or generation or production tasks, there exists a correct corresponding piece of data already, the task is simply to find the correct one out of many. + + + + + +#### Sign Video to Sign Video + +@costerQueryingSignLanguage2023 describe a proof of concept system which would allow users of an online dictionary of signs to search _by signing_, a more user-friendly method. +That is, by recording and uploading a video of themselves signing, one can query the dictionary that way. +They do this by essentially by finding the video with the most similar embeddings. +They start by first pretraining a Sign Language Recognition (Pose sequence to Gloss) model, thus training their transformer encoder to embed sign inputs. + +Once the encoder is trained, then they can use it to generate embeddings for all the signs in the dictionary. +When a user input query video arrives they can then compare the embeddings of the input to those of dictionary entries using Euclidean distance. +[A prototype repository is provided](https://github.com/m-decoster/VGT-SL-Dictionary), including scripts to generate a database for a dictionary as well as documentation for installation or training of new models. + ### Fingerspelling From 27291cca0f4238dfc5db6f588b74731bd3598820 Mon Sep 17 00:00:00 2001 From: Colin Leong <122366389+cleong110@users.noreply.github.com> Date: Thu, 16 May 2024 17:34:00 -0400 Subject: [PATCH 03/11] CDL: ChatGPT rewrite for conciseness, and link to SLR subheading --- src/index.md | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/index.md b/src/index.md index 0f7760e6..fdef7042 100644 --- a/src/index.md +++ b/src/index.md @@ -862,15 +862,7 @@ Sign Language Retrieval is the task of finding a particular data item, given som #### Sign Video to Sign Video -@costerQueryingSignLanguage2023 describe a proof of concept system which would allow users of an online dictionary of signs to search _by signing_, a more user-friendly method. -That is, by recording and uploading a video of themselves signing, one can query the dictionary that way. -They do this by essentially by finding the video with the most similar embeddings. -They start by first pretraining a Sign Language Recognition (Pose sequence to Gloss) model, thus training their transformer encoder to embed sign inputs. - -Once the encoder is trained, then they can use it to generate embeddings for all the signs in the dictionary. -When a user input query video arrives they can then compare the embeddings of the input to those of dictionary entries using Euclidean distance. -[A prototype repository is provided](https://github.com/m-decoster/VGT-SL-Dictionary), including scripts to generate a database for a dictionary as well as documentation for installation or training of new models. - +@costerQueryingSignLanguage2023 present a proof of concept for a sign language dictionary that allows users to search by signing. Users can record a video of themselves signing and upload it to query the dictionary. This system works by finding the video with the most similar embeddings. The process begins with pretraining a [Sign Language Recognition model](#pose-to-gloss) to embed sign inputs. Once the encoder is trained, it generates embeddings for all dictionary signs. When a user submits a query video, the system compares the input embeddings with those of the dictionary entries using Euclidean distance. Tests on a Flemish Sign Language dictionary show that the system can successfully retrieve a limited vocabulary of signs, including some not in the training set. A prototype repository is available, including scripts for database generation and documentation for model installation or training. ### Fingerspelling From 0aa63985ffa4cf8f9befd313144eba099abac4d2 Mon Sep 17 00:00:00 2001 From: Colin Leong <122366389+cleong110@users.noreply.github.com> Date: Mon, 20 May 2024 12:46:29 -0400 Subject: [PATCH 04/11] CDL: update citation for costerQueryingSignLanguage2023, based on official IEEE BibTex --- src/references.bib | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/references.bib b/src/references.bib index 24ab0f86..202a4897 100644 --- a/src/references.bib +++ b/src/references.bib @@ -2176,14 +2176,13 @@ @inproceedings{chenSimpleMultiModalityTransfer2022a langid = {english} } -@inproceedings{costerQueryingSignLanguage2023, - title = {Querying {{A Sign Language Dictionary}} with {{Videos Using Dense Vector Search}}}, - booktitle = {2023 {{IEEE International Conference}} on {{Acoustics}}, {{Speech}}, and {{Signal Processing Workshops}} ({{ICASSPW}})}, - author = {Coster, Mathieu De and Dambre, Joni}, - year = {2023}, - month = jun, - pages = {1--5}, - doi = {10.1109/ICASSPW59220.2023.10193531}, - url = {https://ieeexplore.ieee.org/document/10193531}, - urldate = {2024-03-13} -} +@INPROCEEDINGS{costerQueryingSignLanguage2023, + author={Coster, Mathieu De and Dambre, Joni}, + booktitle={2023 IEEE International Conference on Acoustics, Speech, and Signal Processing Workshops (ICASSPW)}, + title={Querying A Sign Language Dictionary with Videos Using Dense Vector Search}, + year={2023}, + volume={}, + number={}, + pages={1-5}, + doi={10.1109/ICASSPW59220.2023.10193531}} + From c01a74daa387c9e76994d8e7bdf3f086b56e0769 Mon Sep 17 00:00:00 2001 From: Colin Leong <122366389+cleong110@users.noreply.github.com> Date: Mon, 20 May 2024 12:50:31 -0400 Subject: [PATCH 05/11] CDL: newlines and remove redundant heading --- src/index.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/index.md b/src/index.md index fdef7042..f9d06815 100644 --- a/src/index.md +++ b/src/index.md @@ -859,10 +859,14 @@ Sign Language Retrieval is the task of finding a particular data item, given som - -#### Sign Video to Sign Video - -@costerQueryingSignLanguage2023 present a proof of concept for a sign language dictionary that allows users to search by signing. Users can record a video of themselves signing and upload it to query the dictionary. This system works by finding the video with the most similar embeddings. The process begins with pretraining a [Sign Language Recognition model](#pose-to-gloss) to embed sign inputs. Once the encoder is trained, it generates embeddings for all dictionary signs. When a user submits a query video, the system compares the input embeddings with those of the dictionary entries using Euclidean distance. Tests on a Flemish Sign Language dictionary show that the system can successfully retrieve a limited vocabulary of signs, including some not in the training set. A prototype repository is available, including scripts for database generation and documentation for model installation or training. +@costerQueryingSignLanguage2023 present a proof of concept for a sign language dictionary that allows users to search by signing. +Users can record a video of themselves signing and upload it to query the dictionary. +This system works by finding the video with the most similar embeddings. +The process begins with pretraining a [Sign Language Recognition model](#pose-to-gloss) to embed sign inputs. +Once the encoder is trained, it generates embeddings for all dictionary signs. +When a user submits a query video, the system compares the input embeddings with those of the dictionary entries using Euclidean distance. +Tests on a Flemish Sign Language dictionary show that the system can successfully retrieve a limited vocabulary of signs, including some not in the training set. +[A prototype repository](https://github.com/m-decoster/VGT-SL-Dictionary) is available, including scripts for database generation and documentation for model installation or training. ### Fingerspelling From aa09ef533940488974e338cfafbcef181c2552a5 Mon Sep 17 00:00:00 2001 From: Colin Leong <122366389+cleong110@users.noreply.github.com> Date: Tue, 21 May 2024 16:04:36 -0400 Subject: [PATCH 06/11] CDL: minor bibtex linting --- src/references.bib | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/references.bib b/src/references.bib index 202a4897..b412285b 100644 --- a/src/references.bib +++ b/src/references.bib @@ -2176,13 +2176,12 @@ @inproceedings{chenSimpleMultiModalityTransfer2022a langid = {english} } -@INPROCEEDINGS{costerQueryingSignLanguage2023, +@inproceedings{costerQueryingSignLanguage2023, author={Coster, Mathieu De and Dambre, Joni}, booktitle={2023 IEEE International Conference on Acoustics, Speech, and Signal Processing Workshops (ICASSPW)}, title={Querying A Sign Language Dictionary with Videos Using Dense Vector Search}, year={2023}, - volume={}, - number={}, pages={1-5}, - doi={10.1109/ICASSPW59220.2023.10193531}} + doi={10.1109/ICASSPW59220.2023.10193531} +} From 03d4f4693fe67b49e24414c72538a72b2efe9713 Mon Sep 17 00:00:00 2001 From: Colin Leong <122366389+cleong110@users.noreply.github.com> Date: Tue, 21 May 2024 16:16:20 -0400 Subject: [PATCH 07/11] CDL: making requested changes to costerQueryingSignLanguage2023 summary --- src/index.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/index.md b/src/index.md index f9d06815..f69f7c49 100644 --- a/src/index.md +++ b/src/index.md @@ -859,14 +859,11 @@ Sign Language Retrieval is the task of finding a particular data item, given som -@costerQueryingSignLanguage2023 present a proof of concept for a sign language dictionary that allows users to search by signing. -Users can record a video of themselves signing and upload it to query the dictionary. -This system works by finding the video with the most similar embeddings. -The process begins with pretraining a [Sign Language Recognition model](#pose-to-gloss) to embed sign inputs. -Once the encoder is trained, it generates embeddings for all dictionary signs. +@costerQueryingSignLanguage2023 present a method to query sign language dictionaries using dense vector search. +They pretrain a [Sign Language Recognition model](#pose-to-gloss) on a subset of the VGT corpus to embed sign inputs. +Once the encoder is trained, they use it to generate embeddings for all dictionary signs. When a user submits a query video, the system compares the input embeddings with those of the dictionary entries using Euclidean distance. -Tests on a Flemish Sign Language dictionary show that the system can successfully retrieve a limited vocabulary of signs, including some not in the training set. -[A prototype repository](https://github.com/m-decoster/VGT-SL-Dictionary) is available, including scripts for database generation and documentation for model installation or training. +Tests on a [proof-of-concept Flemish Sign Language dictionary](https://github.com/m-decoster/VGT-SL-Dictionary) show that the system can successfully retrieve a limited vocabulary of signs, including some not in the training set. ### Fingerspelling From 032e9393e9083acd6569ca1ccab26bca2d969beb Mon Sep 17 00:00:00 2001 From: Colin Leong <122366389+cleong110@users.noreply.github.com> Date: Tue, 21 May 2024 16:19:03 -0400 Subject: [PATCH 08/11] CDL: a few notes on the VGT Corpus --- src/index.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/index.md b/src/index.md index f69f7c49..d3e177de 100644 --- a/src/index.md +++ b/src/index.md @@ -860,10 +860,11 @@ Sign Language Retrieval is the task of finding a particular data item, given som @costerQueryingSignLanguage2023 present a method to query sign language dictionaries using dense vector search. -They pretrain a [Sign Language Recognition model](#pose-to-gloss) on a subset of the VGT corpus to embed sign inputs. +They pretrain a [Sign Language Recognition model](#pose-to-gloss) on a subset of the [VGT corpus](https://corpusvgt.be/) to embed sign inputs. Once the encoder is trained, they use it to generate embeddings for all dictionary signs. When a user submits a query video, the system compares the input embeddings with those of the dictionary entries using Euclidean distance. Tests on a [proof-of-concept Flemish Sign Language dictionary](https://github.com/m-decoster/VGT-SL-Dictionary) show that the system can successfully retrieve a limited vocabulary of signs, including some not in the training set. + ### Fingerspelling From 849288790cf95f0d94afb1927663f8f0b7c8fa74 Mon Sep 17 00:00:00 2001 From: Colin Leong <122366389+cleong110@users.noreply.github.com> Date: Wed, 22 May 2024 14:14:07 -0400 Subject: [PATCH 09/11] CDL: redraft Sign Language Retrieval intro --- src/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.md b/src/index.md index d3e177de..be855593 100644 --- a/src/index.md +++ b/src/index.md @@ -854,7 +854,7 @@ TODO ### Sign Language Retrieval -Sign Language Retrieval is the task of finding a particular data item, given some input. In contrast to translation or generation or production tasks, there exists a correct corresponding piece of data already, the task is simply to find the correct one out of many. +Sign Language Retrieval is the task of finding a particular data item, given some input. In contrast to translation, generation or production tasks, there can exist a correct corresponding piece of data already, and the task is to find it out of many, if it exists. From aba40b21bdfbd5a711282f69714352fd1f9d58e7 Mon Sep 17 00:00:00 2001 From: Colin Leong <122366389+cleong110@users.noreply.github.com> Date: Wed, 22 May 2024 14:17:55 -0400 Subject: [PATCH 10/11] CDL: cite VGT Corpus --- src/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/index.md b/src/index.md index be855593..0abe2b67 100644 --- a/src/index.md +++ b/src/index.md @@ -860,11 +860,11 @@ Sign Language Retrieval is the task of finding a particular data item, given som @costerQueryingSignLanguage2023 present a method to query sign language dictionaries using dense vector search. -They pretrain a [Sign Language Recognition model](#pose-to-gloss) on a subset of the [VGT corpus](https://corpusvgt.be/) to embed sign inputs. +They pretrain a [Sign Language Recognition model](#pose-to-gloss) on a subset of the VGT corpus [@dataset:herreweghe2015VGTCorpus] to embed sign inputs. Once the encoder is trained, they use it to generate embeddings for all dictionary signs. When a user submits a query video, the system compares the input embeddings with those of the dictionary entries using Euclidean distance. Tests on a [proof-of-concept Flemish Sign Language dictionary](https://github.com/m-decoster/VGT-SL-Dictionary) show that the system can successfully retrieve a limited vocabulary of signs, including some not in the training set. - + ### Fingerspelling From 5acf7aa5c051c7408eef5033e357556626425cde Mon Sep 17 00:00:00 2001 From: Colin Leong <122366389+cleong110@users.noreply.github.com> Date: Wed, 22 May 2024 14:18:22 -0400 Subject: [PATCH 11/11] CDL: add VGT Corpus to references.bib --- src/references.bib | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/references.bib b/src/references.bib index b412285b..567afad5 100644 --- a/src/references.bib +++ b/src/references.bib @@ -2185,3 +2185,11 @@ @inproceedings{costerQueryingSignLanguage2023 doi={10.1109/ICASSPW59220.2023.10193531} } +@misc{dataset:herreweghe2015VGTCorpus, + author = {{Van Herreweghe, Mieke and Vermeerbergen, Myriam and Demey, Eline and De Durpel, Hannes and Nyffels, Hilde and Verstraete, Sam}}, + keywords = {{Vlaamse Gebarentaal,Corpuslinguïstiek}}, + language = {{dut}}, + title = {{Het Corpus VGT. Een digitaal open access corpus van videos and annotaties van Vlaamse Gebarentaal, ontwikkeld aan de Universiteit Gent ism KU Leuven. }}, + url = {{http://www.corpusvgt.ugent.be/}}, + year = {{2015}}, +} \ No newline at end of file