diff --git a/Jenkinsfile b/Jenkinsfile index 38e05bb00..825f78044 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -30,6 +30,7 @@ pipeline { JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-26-5' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' + TA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-21-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { @@ -113,6 +114,25 @@ pipeline { } } + stage('L0: Create TA TN Grammars') { + when { + anyOf { + branch 'main' + branch 'staging/**' + branch 'staging_*' + changeRequest target: 'main' + } + } + failFast true + parallel { + stage('L0: Ta TN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=ta --text="௧" --cache_dir ${TA_TN_CACHE}' + } + } + + } + } stage('L0: Create DE/ES TN/ITN Grammars') { when { anyOf { @@ -408,6 +428,11 @@ pipeline { sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/hi/ -m "not pleasefixme" --cpu --tn_cache_dir ${HI_TN_CACHE}' } } + stage('L1: Run all TA TN tests (restore grammars from cache)') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/ta/ -m "not pleasefixme" --cpu --tn_cache_dir ${TA_TN_CACHE}' + } + } stage('L1: Run all Codeswitched ES/EN TN/ITN tests (restore grammars from cache)') { steps { sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/es_en/ -m "not pleasefixme" --cpu --tn_cache_dir ${ES_EN_TN_CACHE}' diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index d8ebf2f4d..a7e001652 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -166,6 +166,7 @@ def __init__( if post_process: self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) + elif lang == 'it': from nemo_text_processing.text_normalization.it.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.it.verbalizers.verbalize_final import VerbalizeFinalFst @@ -191,6 +192,10 @@ def __init__( elif lang == 'ko': from nemo_text_processing.text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.ko.verbalizers.verbalize_final import VerbalizeFinalFst + elif lang == 'ta': + from nemo_text_processing.text_normalization.ta.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.ta.verbalizers.verbalize_final import VerbalizeFinalFst + else: raise NotImplementedError(f"Language {lang} has not been supported yet.") @@ -737,7 +742,7 @@ def parse_args(): parser.add_argument( "--language", help="language", - choices=["en", "de", "es", "fr", "hu", "sv", "zh", "ar", "it", "hy", "ja", "hi", "ko", "vi", "pt"], + choices=["en", "de", "es", "fr", "hu", "sv", "zh", "ar", "it", "hy", "ja", "hi", "ta", "ko", "vi", "pt"], default="en", type=str, ) diff --git a/nemo_text_processing/text_normalization/run_evaluate.py b/nemo_text_processing/text_normalization/run_evaluate.py index 3a1964bbd..cf0b6efb2 100644 --- a/nemo_text_processing/text_normalization/run_evaluate.py +++ b/nemo_text_processing/text_normalization/run_evaluate.py @@ -35,7 +35,7 @@ def parse_args(): parser.add_argument( "--lang", help="language", - choices=['ar', 'de', 'en', 'es', 'fr', 'hu', 'it', 'ru', 'sv', 'zh', 'hy', 'hi', 'ko', 'vi', 'pt'], + choices=['ar', 'de', 'en', 'es', 'fr', 'hu', 'it', 'ru', 'sv', 'zh', 'hy', 'hi', 'ta', 'ko', 'vi', 'pt'], default="en", type=str, ) diff --git a/nemo_text_processing/text_normalization/ta/__init__.py b/nemo_text_processing/text_normalization/ta/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/data/__init__.py b/nemo_text_processing/text_normalization/ta/data/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/data/numbers/crore.tsv b/nemo_text_processing/text_normalization/ta/data/numbers/crore.tsv new file mode 100644 index 000000000..2c92d20d7 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/numbers/crore.tsv @@ -0,0 +1,2 @@ +கோடி +கோடியே \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ta/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/ta/data/numbers/digit.tsv new file mode 100644 index 000000000..92111ede3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/numbers/digit.tsv @@ -0,0 +1,18 @@ +௧ ஒன்று +௨ இரண்டு +௩ மூன்று +௪ நான்கு +௫ ஐந்து +௬ ஆறு +௭ ஏழு +௮ எட்டு +௯ ஒன்பது +1 ஒன்று +2 இரண்டு +3 மூன்று +4 நான்கு +5 ஐந்து +6 ஆறு +7 ஏழு +8 எட்டு +9 ஒன்பது \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ta/data/numbers/hundred_ta.tsv b/nemo_text_processing/text_normalization/ta/data/numbers/hundred_ta.tsv new file mode 100644 index 000000000..873a94569 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/numbers/hundred_ta.tsv @@ -0,0 +1,36 @@ +100 நூறு +200 இருநூறு +300 முந்நூறு +400 நானூறு +500 ஐந்நூறு +600 அறுநூறு +700 எழுநூறு +800 எட்டுநூறு +900 ஒன்பதுநூறு +௧௦௦ நூறு +௨௦௦ இருநூறு +௩௦௦ முந்நூறு +௪௦௦ நானூறு +௫௦௦ ஐந்நூறு +௬௦௦ அறுநூறு +௭௦௦ எழுநூறு +௮௦௦ எட்டுநூறு +௯௦௦ ஒன்பதுநூறு +1 நூற்று +2 இருநூற்று +3 முந்நூற்று +4 நானூற்று +5 ஐநூற்று +6 அறுநூற்று +7 எழுநூற்று +8 எட்டுநூற்று +9 ஒன்பதுநூற்று +௧ நூற்று +௨ இருநூற்று +௩ முந்நூற்று +௪ நானூற்று +௫ ஐநூற்று +௬ அறுநூற்று +௭ எழுநூற்று +௮ எட்டுநூற்று +௯ ஒன்பதுநூற்று \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ta/data/numbers/teens_and_ties.tsv b/nemo_text_processing/text_normalization/ta/data/numbers/teens_and_ties.tsv new file mode 100644 index 000000000..058253fa1 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/numbers/teens_and_ties.tsv @@ -0,0 +1,90 @@ +௧௦ பத்து +௧௧ பதினொன்று +௧௨ பன்னிரண்டு +௧௩ பதிமூன்று +௧௪ பதினான்கு +௧௫ பதினைந்து +௧௬ பதினாறு +௧௭ பதினேழு +௧௮ பதினெட்டு +௧௯ பத்தொன்பது +௨௦ இருபது +௨௧ இருபத்தொன்று +௨௨ இருபத்திரண்டு +௨௩ இருபத்திமூன்று +௨௪ இருபத்திநான்கு +௨௫ இருபத்திஐந்து +௨௬ இருபத்திஆறு +௨௭ இருபத்திஏழு +௨௮ இருபத்திஎட்டு +௨௯ இருபத்திஒன்பது +௩௦ முப்பது +௩௧ முப்பத்தொன்று +௩௨ முப்பத்திரண்டு +௩௩ முப்பத்திமூன்று +௩௪ முப்பத்திநான்கு +௩௫ முப்பத்திஐந்து +௩௬ முப்பத்திஆறு +௩௭ முப்பத்திஏழு +௩௮ முப்பத்திஎட்டு +௩௯ முப்பத்திஒன்பது +௪௦ நாற்பது +௪௧ நாற்பத்தொன்று +௪௨ நாற்பத்திரண்டு +௪௩ நாற்பத்திமூன்று +௪௪ நாற்பத்திநான்கு +௪௫ நாற்பத்திஐந்து +௪௬ நாற்பத்திஆறு +௪௭ நாற்பத்திஏழு +௪௮ நாற்பத்திஎட்டு +௪௯ நாற்பத்திஒன்பது +௫௦ ஐம்பது +௫௧ ஐம்பத்தொன்று +௫௨ ஐம்பத்திரண்டு +௫௩ ஐம்பத்திமூன்று +௫௪ ஐம்பத்திநான்கு +௫௫ ஐம்பத்திஐந்து +௫௬ ஐம்பத்திஆறு +௫௭ ஐம்பத்திஏழு +௫௮ ஐம்பத்திஎட்டு +௫௯ ஐம்பத்திஒன்பது +௬௦ அறுபது +௬௧ அறுபத்தொன்று +௬௨ அறுபத்திரண்டு +௬௩ அறுபத்திமூன்று +௬௪ அறுபத்திநான்கு +௬௫ அறுபத்திஐந்து +௬௬ அறுபத்திஆறு +௬௭ அறுபத்திஏழு +௬௮ அறுபத்திஎட்டு +௬௯ அறுபத்திஒன்பது +௭௦ எழுபது +௭௧ எழுபத்தொன்று +௭௨ எழுபத்திரண்டு +௭௩ எழுபத்திமூன்று +௭௪ எழுபத்திநான்கு +௭௫ எழுபத்திஐந்து +௭௬ எழுபத்திஆறு +௭௭ எழுபத்திஏழு +௭௮ எழுபத்திஎட்டு +௭௯ எழுபத்திஒன்பது +௮௦ எண்பது +௮௧ எண்பத்தொன்று +௮௨ எண்பத்திரண்டு +௮௩ எண்பத்திமூன்று +௮௪ எண்பத்திநான்கு +௮௫ எண்பத்திஐந்து +௮௬ எண்பத்திஆறு +௮௭ எண்பத்திஏழு +௮௮ எண்பத்திஎட்டு +௮௯ எண்பத்திஒன்பது +௯௦ தொண்ணூறு +௯௧ தொண்ணூற்றொன்று +௯௨ தொண்ணூற்றிரண்டு +௯௩ தொண்ணூற்றிமூன்று +௯௪ தொண்ணூற்றிநான்கு +௯௫ தொண்ணூற்றிஐந்து +௯௬ தொண்ணூற்றிஆறு +௯௭ தொண்ணூற்றிஏழு +௯௮ தொண்ணூற்றிஎட்டு +௯௯ தொண்ணூற்றிஒன்பது \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ta/data/numbers/teens_and_ties_en.tsv b/nemo_text_processing/text_normalization/ta/data/numbers/teens_and_ties_en.tsv new file mode 100644 index 000000000..b796a9028 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/numbers/teens_and_ties_en.tsv @@ -0,0 +1,90 @@ +10 பத்து +11 பதினொன்று +12 பன்னிரண்டு +13 பதிமூன்று +14 பதினான்கு +15 பதினைந்து +16 பதினாறு +17 பதினேழு +18 பதினெட்டு +19 பத்தொன்பது +20 இருபது +21 இருபத்தொன்று +22 இருபத்திரண்டு +23 இருபத்திமூன்று +24 இருபத்திநான்கு +25 இருபத்திஐந்து +26 இருபத்திஆறு +27 இருபத்திஏழு +28 இருபத்திஎட்டு +29 இருபத்திஒன்பது +30 முப்பது +31 முப்பத்தொன்று +32 முப்பத்திரண்டு +33 முப்பத்திமூன்று +34 முப்பத்திநான்கு +35 முப்பத்திஐந்து +36 முப்பத்திஆறு +37 முப்பத்திஏழு +38 முப்பத்திஎட்டு +39 முப்பத்திஒன்பது +40 நாற்பது +41 நாற்பத்தொன்று +42 நாற்பத்திரண்டு +43 நாற்பத்திமூன்று +44 நாற்பத்திநான்கு +45 நாற்பத்திஐந்து +46 நாற்பத்திஆறு +47 நாற்பத்திஏழு +48 நாற்பத்திஎட்டு +49 நாற்பத்திஒன்பது +50 ஐம்பது +51 ஐம்பத்தொன்று +52 ஐம்பத்திரண்டு +53 ஐம்பத்திமூன்று +54 ஐம்பத்திநான்கு +55 ஐம்பத்திஐந்து +56 ஐம்பத்திஆறு +57 ஐம்பத்திஏழு +58 ஐம்பத்திஎட்டு +59 ஐம்பத்திஒன்பது +60 அறுபது +61 அறுபத்தொன்று +62 அறுபத்திரண்டு +63 அறுபத்திமூன்று +64 அறுபத்திநான்கு +65 அறுபத்திஐந்து +66 அறுபத்திஆறு +67 அறுபத்திஏழு +68 அறுபத்திஎட்டு +69 அறுபத்திஒன்பது +70 எழுபது +71 எழுபத்தொன்று +72 எழுபத்திரண்டு +73 எழுபத்திமூன்று +74 எழுபத்திநான்கு +75 எழுபத்திஐந்து +76 எழுபத்திஆறு +77 எழுபத்திஏழு +78 எழுபத்திஎட்டு +79 எழுபத்திஒன்பது +80 எண்பது +81 எண்பத்தொன்று +82 எண்பத்திரண்டு +83 எண்பத்திமூன்று +84 எண்பத்திநான்கு +85 எண்பத்திஐந்து +86 எண்பத்திஆறு +87 எண்பத்திஏழு +88 எண்பத்திஎட்டு +89 எண்பத்திஒன்பது +90 தொண்ணூறு +91 தொண்ணூற்றொன்று +92 தொண்ணூற்றிரண்டு +93 தொண்ணூற்றிமூன்று +94 தொண்ணூற்றிநான்கு +95 தொண்ணூற்றிஐந்து +96 தொண்ணூற்றிஆறு +97 தொண்ணூற்றிஏழு +98 தொண்ணூற்றிஎட்டு +99 தொண்ணூற்றிஒன்பது \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ta/data/numbers/thousand.tsv b/nemo_text_processing/text_normalization/ta/data/numbers/thousand.tsv new file mode 100644 index 000000000..1a16b1271 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/numbers/thousand.tsv @@ -0,0 +1,18 @@ +1 ஆயிர +2 இரண்டாயிர +3 மூவாயிர +4 நான்காயிர +5 ஐந்தாயிர +6 ஆறாயிர +7 ஏழாயிர +8 எட்டாயிர +9 ஒன்பதாயிர +௧ ஆயிர +௨ இரண்டாயிர +௩ மூவாயிர +௪ நான்காயிர +௫ ஐந்தாயிர +௬ ஆறாயிர +௭ ஏழாயிர +௮ எட்டாயிர +௯ ஒன்பதாயிர \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ta/data/numbers/zero.tsv b/nemo_text_processing/text_normalization/ta/data/numbers/zero.tsv new file mode 100644 index 000000000..c5011b29f --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/data/numbers/zero.tsv @@ -0,0 +1,2 @@ +௦ சுழியம் +0 சுழியம் \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ta/graph_utils.py b/nemo_text_processing/text_normalization/ta/graph_utils.py new file mode 100644 index 000000000..2598f4415 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/graph_utils.py @@ -0,0 +1,129 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from nemo_text_processing.text_normalization.ta.utils import get_abs_path + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +ta_digit_map = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) +ta_zero_map = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + +ta_digit_only = pynini.project(ta_digit_map, "input").optimize() +NEMO_ALL_ZERO = pynini.project(ta_zero_map, "input").optimize() +NEMO_ALL_DIGIT = pynini.union(ta_digit_only, NEMO_ALL_ZERO).optimize() + +NEMO_NON_BREAKING_SPACE = u"\u00a0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() +NEMO_SIGMA = pynini.closure(NEMO_CHAR) + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/text_normalization/ta/taggers/__init__.py b/nemo_text_processing/text_normalization/ta/taggers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/taggers/cardinal.py b/nemo_text_processing/text_normalization/ta/taggers/cardinal.py new file mode 100644 index 000000000..b25ba48f6 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/cardinal.py @@ -0,0 +1,149 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ta.graph_utils import ( + NEMO_ALL_DIGIT, + NEMO_ALL_ZERO, + GraphFst, + insert_space, +) +from nemo_text_processing.text_normalization.ta.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying cardinals + e.g. 23 -> cardinal { integer: "இருபத்திமூன்று" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True, lm: bool = False): + super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + + digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + teens_ties = pynini.union( + pynini.string_file(get_abs_path("data/numbers/teens_and_ties.tsv")), + pynini.string_file(get_abs_path("data/numbers/teens_and_ties_en.tsv")), + ) + teens_and_ties = teens_ties + + # digit_oru + one_oru = pynini.cross("1", "ஒரு") | pynini.cross("௧", "ஒரு") + digit_oru = ( + one_oru | pynini.compose(pynini.difference(NEMO_ALL_DIGIT, pynini.union("1", "௧")), digit) + ).optimize() + + hundred = pynini.string_file(get_abs_path("data/numbers/hundred_ta.tsv")) + hundred_ta = pynini.compose(NEMO_ALL_DIGIT**3, hundred).optimize() + hundred_prefix = pynini.compose(NEMO_ALL_DIGIT, hundred).optimize() + + # ஆயிரம் (exact) and ஆயிரத்து (combining) share the same stem + thousand_stem = pynini.string_file(get_abs_path("data/numbers/thousand.tsv")) + thousand_exact = thousand_stem + pynutil.insert("ம்") + thousand_prefix = thousand_stem + pynutil.insert("த்து") + + self.digit = digit + self.zero = zero + self.hundred_ta = hundred_ta + self.teens_and_ties = teens_and_ties + + single_digit = digit | zero + self.single_digits_graph = single_digit + pynini.closure(insert_space + single_digit) + zero_del = pynutil.add_weight(pynutil.delete(NEMO_ALL_ZERO), -0.1) + + def zdel(k): + return zero_del**k if k > 0 else pynini.accep("") + + def scale(head_exact, head_tail, n, tails): + graph = head_exact + for i, sub in enumerate(tails): + graph |= head_tail + zdel(n - 1 - i) + insert_space + sub + return graph.optimize() + + def band(base, exact_word, tail_word, n, tails): + return scale(base + pynutil.insert(exact_word) + zdel(n), base + pynutil.insert(tail_word), n, tails) + + # HUNDREDS (100-999): நூறு / நூற்று forms + graph_hundreds = scale(pynutil.add_weight(hundred_ta, -1.0), hundred_prefix, 2, [single_digit, teens_ties]) + self.graph_hundreds = graph_hundreds + + # THOUSANDS (1000-9999): ஆயிரம் / ஆயிரத்து forms + graph_thousands = scale( + thousand_exact + zdel(3), thousand_prefix, 3, [single_digit, teens_ties, graph_hundreds] + ) + self.graph_thousands = graph_thousands + tails = [single_digit, teens_ties, graph_hundreds, graph_thousands] + + # TEN-THOUSANDS (10^4): stem + ஆயிரம் + graph_ten_thousands = band(teens_and_ties, "ஆயிரம்", "ஆயிரத்து", 3, tails[:3]) + self.graph_ten_thousands = graph_ten_thousands + tails.append(graph_ten_thousands) + + # LAKHS / TEN-LAKHS (10^5, 10^6): stem + லட்சம் + graph_lakhs = band(digit_oru, " லட்சம்", " லட்சத்து", 5, tails[:5]) + self.graph_lakhs = graph_lakhs + graph_ten_lakhs = band(teens_and_ties, " லட்சம்", " லட்சத்து", 5, tails[:5]) + self.graph_ten_lakhs = graph_ten_lakhs + tails += [graph_lakhs, graph_ten_lakhs] + + # CRORES and higher (10^7 .. 10^15): stem + கோடி + crore_bases = [ + digit_oru, # crores + teens_and_ties, # ten-crores + graph_hundreds, # hundreds of crores + graph_thousands, # thousands of crores + graph_ten_thousands, # ten-thousands of crores + graph_lakhs, # lakhs of crores + graph_ten_lakhs, # ten-lakhs of crores + ] + with open(get_abs_path("data/numbers/crore.tsv"), encoding="utf-8") as f: + crore_exact_raw, crore_prefix_raw = [line.strip() for line in f if line.strip()] + crore_word = " " + crore_exact_raw + crore_prefix_word = " " + crore_prefix_raw + crore_graphs = [band(b, crore_word, crore_prefix_word, 7, tails) for b in crore_bases] + graph_crores, graph_ten_crores = crore_graphs[0], crore_graphs[1] + crore_graphs += [ + band(graph_crores, crore_word, crore_prefix_word, 7, tails), # crores of crores + band(graph_ten_crores, crore_word, crore_prefix_word, 7, tails), # ten-crores of crores + ] + + # FINAL GRAPH + graph_without_leading_zeros = pynini.union( + digit, + zero, + teens_and_ties, + graph_hundreds, + graph_thousands, + graph_ten_thousands, + graph_lakhs, + graph_ten_lakhs, + *crore_graphs, + ) + self.graph_without_leading_zeros = graph_without_leading_zeros.optimize() + + cardinal_with_leading_zeros = pynutil.add_weight( + pynini.compose(NEMO_ALL_ZERO + pynini.closure(NEMO_ALL_DIGIT), self.single_digits_graph), 0.5 + ) + self.final_graph = (self.graph_without_leading_zeros | cardinal_with_leading_zeros).optimize() + + optional_minus_graph = pynini.closure(pynutil.insert("negative: ") + pynini.cross("-", "\"true\" "), 0, 1) + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.final_graph + pynutil.insert("\"") + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ta/taggers/punctuation.py b/nemo_text_processing/text_normalization/ta/taggers/punctuation.py new file mode 100644 index 000000000..82aa82c93 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/punctuation.py @@ -0,0 +1,61 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys +from unicodedata import category + +import pynini +from pynini.examples import plurals +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ta.graph_utils import NEMO_NOT_SPACE, NEMO_SIGMA, GraphFst + + +class PunctuationFst(GraphFst): + """ + Finite state transducer for classifying punctuation + e.g. a, -> tokens { name: "a" } tokens { name: "," } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transductions are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + s = "!#%&\'()*+,-./:;<=>?@^_`{|}~\"" + + punct_symbols_to_exclude = ["[", "]"] + punct_unicode = [ + chr(i) + for i in range(sys.maxunicode) + if category(chr(i)).startswith("P") and chr(i) not in punct_symbols_to_exclude + ] + + self.punct_marks = [p for p in punct_unicode + list(s)] + + punct = pynini.union(*self.punct_marks) + punct = pynini.closure(punct, 1) + + emphasis = ( + pynini.accep("<") + + pynini.union( + (pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1) + pynini.closure(pynini.accep("/"), 0, 1)), + (pynini.accep("/") + pynini.closure(NEMO_NOT_SPACE - pynini.union("<", ">"), 1)), + ) + + pynini.accep(">") + ) + punct = plurals._priority_union(emphasis, punct, NEMO_SIGMA) + + self.graph = punct + self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() diff --git a/nemo_text_processing/text_normalization/ta/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ta/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..af2adf3ba --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/tokenize_and_classify.py @@ -0,0 +1,115 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ta.graph_utils import ( + NEMO_SPACE, + NEMO_WHITE_SPACE, + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.ta.taggers.cardinal import CardinalFst +from nemo_text_processing.text_normalization.ta.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.ta.taggers.word import WordFst + + +class ClassifyFst(GraphFst): + """ + Final class that composes all other classification grammars. This class can process an entire sentence including punctuation. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + input_case: accepting either "lower_cased" or "cased" input. + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + whitelist: path to a file with whitelist replacements + """ + + def __init__( + self, + input_case: str, + deterministic: bool = True, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + whitelist_file = os.path.basename(whitelist) if whitelist else "" + far_file = os.path.join( + cache_dir, + f"ta_tn_{deterministic}_deterministic_{input_case}_{whitelist_file}_tokenize.far", + ) + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info(f"Creating ClassifyFst grammars.") + + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + punctuation = PunctuationFst(deterministic=deterministic) + punct_graph = punctuation.fst + + classify = pynutil.add_weight(cardinal_graph, 1.1) + + word_graph = WordFst().fst + + punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=2.1) + pynutil.insert(" }") + punct = pynini.closure( + pynini.union( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space), + (pynutil.insert(NEMO_SPACE) + punct), + ), + 1, + ) + + classify = pynini.union(classify, pynutil.add_weight(word_graph, 100)) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + token_plus_punct = ( + pynini.closure(punct + pynutil.insert(NEMO_SPACE)) + + token + + pynini.closure(pynutil.insert(NEMO_SPACE) + punct) + ) + + graph = token_plus_punct + pynini.closure( + pynini.union( + pynini.compose(pynini.closure(NEMO_WHITE_SPACE, 1), delete_extra_space), + (pynutil.insert(NEMO_SPACE) + punct + pynutil.insert(NEMO_SPACE)), + ) + + token_plus_punct + ) + + graph = delete_space + graph + delete_space + graph = pynini.union(graph, punct) + + self.fst = graph.optimize() + + if far_file: + generator_main(far_file, {"tokenize_and_classify": self.fst}) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/ta/taggers/word.py b/nemo_text_processing/text_normalization/ta/taggers/word.py new file mode 100644 index 000000000..c36141a92 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/taggers/word.py @@ -0,0 +1,33 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil +from nemo_text_processing.text_normalization.ta.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying Tamil words. + e.g. சோனா -> tokens { name: "சோனா" } + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transductions are generated (used for audio-based normalization) + + """ + + def __init__(self): + super().__init__(name="word", kind="classify") + word = pynutil.insert("name: \"") + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/nemo_text_processing/text_normalization/ta/utils.py b/nemo_text_processing/text_normalization/ta/utils.py new file mode 100644 index 000000000..2bcba780d --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/utils.py @@ -0,0 +1,27 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + '/' + rel_path diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/__init__.py b/nemo_text_processing/text_normalization/ta/verbalizers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/cardinal.py b/nemo_text_processing/text_normalization/ta/verbalizers/cardinal.py new file mode 100644 index 000000000..9065cf2a5 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/cardinal.py @@ -0,0 +1,49 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ta.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal, e.g. + cardinal { negative: "true" integer: "23" } -> minus twenty three + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) + + self.optional_sign = pynini.cross("negative: \"true\"", "minus ") + if not deterministic: + self.optional_sign |= pynini.cross("negative: \"true\"", "negative ") + self.optional_sign |= pynini.cross("negative: \"true\"", "dash ") + + self.optional_sign = pynini.closure(self.optional_sign + delete_space, 0, 1) + + integer = pynini.closure(NEMO_NOT_QUOTE) + + self.integer = delete_space + pynutil.delete("\"") + integer + pynutil.delete("\"") + integer = pynutil.delete("integer:") + self.integer + + self.numbers = self.optional_sign + integer + delete_tokens = self.delete_tokens(self.numbers) + + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/ta/verbalizers/verbalize.py new file mode 100644 index 000000000..67c87718b --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/verbalize.py @@ -0,0 +1,38 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.text_normalization.ta.graph_utils import GraphFst +from nemo_text_processing.text_normalization.ta.verbalizers.cardinal import CardinalFst + + +class VerbalizeFst(GraphFst): + """ + Composes other verbalizer grammars. + For deployment, this grammar will be compiled and exported to OpenFst Finite State Archive (FAR) File. + More details to deployment at NeMo/tools/text_processing_deployment. + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) + + cardinal = CardinalFst(deterministic=deterministic) + cardinal_graph = cardinal.fst + + graph = cardinal_graph + + self.fst = graph diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/ta/verbalizers/verbalize_final.py new file mode 100644 index 000000000..f0d78ea9e --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/verbalize_final.py @@ -0,0 +1,76 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ta.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.ta.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.text_normalization.ta.verbalizers.word import WordFst + + +class VerbalizeFinalFst(GraphFst): + """ + Finite state transducer that verbalizes an entire sentence, e.g. + tokens { name: "its" } tokens { time { hours: "twelve" minutes: "thirty" } } tokens { name: "now" } tokens { name: "." } -> its twelve thirty now . + + Args: + deterministic: if True will provide a single transduction option, + for False multiple options (used for audio-based normalization) + cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. + overwrite_cache: set to True to overwrite .far files + """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"ta_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logging.info(f'VerbalizeFinalFst graph was restored from {far_file}.') + else: + verbalize = VerbalizeFst(deterministic=deterministic).fst + word = WordFst(deterministic=deterministic).fst + types = verbalize | word + + if deterministic: + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + types + + delete_space + + pynutil.delete("}") + ) + else: + graph = delete_space + types + delete_space + + graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space + + self.fst = graph.optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) + logging.info(f"VerbalizeFinalFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/text_normalization/ta/verbalizers/word.py b/nemo_text_processing/text_normalization/ta/verbalizers/word.py new file mode 100644 index 000000000..054edbdd9 --- /dev/null +++ b/nemo_text_processing/text_normalization/ta/verbalizers/word.py @@ -0,0 +1,34 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pynini +from pynini.lib import pynutil +from nemo_text_processing.text_normalization.ta.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing Tamil words. + e.g. tokens { name: "சின்ன" } -> சின்ன + + Args: + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="verbalize", deterministic=deterministic) + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete("\"") + chars + pynutil.delete("\"") + + self.fst = char.optimize() diff --git a/tests/nemo_text_processing/ta/__init__.py b/tests/nemo_text_processing/ta/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/tests/nemo_text_processing/ta/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/ta/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..299a5754f --- /dev/null +++ b/tests/nemo_text_processing/ta/data_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,53 @@ +4 நான்குகள்~நான்கு நான்குகள் +௬ வீரர்கள் அவுட்~ஆறு வீரர்கள் அவுட் +4 ஓவர்களில் 17 ரன்கள்~நான்கு ஓவர்களில் பதினேழு ரன்கள் +௫ சாக்லெட்டுகள் ௯ டாஃபிகள்~ஐந்து சாக்லெட்டுகள் ஒன்பது டாஃபிகள் +10099~பத்துஆயிரத்து தொண்ணூற்றிஒன்பது +100001~ஒரு லட்சத்து ஒன்று +௫ நான்குகள்~ஐந்து நான்குகள் +4 பேர் உயிரிழப்பு 18 பேருக்கு காயம்~நான்கு பேர் உயிரிழப்பு பதினெட்டு பேருக்கு காயம் +51022345567~ஐந்தாயிரத்து நூற்று இரண்டு கோடியே இருபத்திமூன்று லட்சத்து நாற்பத்திஐந்துஆயிரத்து ஐநூற்று அறுபத்திஏழு +அத்தியாயம் ௧௦ ஸ்லோகம் ௨~அத்தியாயம் பத்து ஸ்லோகம் இரண்டு +101~நூற்று ஒன்று +100~நூறு +௧௦௦~நூறு +௧௦௫~நூற்று ஐந்து +200~இருநூறு +௨௦௦~இருநூறு +300~முந்நூறு +௩௦௦~முந்நூறு +203~இருநூற்று மூன்று +௫௦௨~ஐநூற்று இரண்டு +111~நூற்று பதினொன்று +௧௧௯~நூற்று பத்தொன்பது +130~நூற்று முப்பது +௧௫௦~நூற்று ஐம்பது +312~முந்நூற்று பன்னிரண்டு +௫௨௫~ஐநூற்று இருபத்திஐந்து +425~நானூற்று இருபத்திஐந்து +௯௧௫~ஒன்பதுநூற்று பதினைந்து +1001~ஆயிரத்து ஒன்று +5551 வாழைப்பழங்கள்~ஐந்தாயிரத்து ஐநூற்று ஐம்பத்தொன்று வாழைப்பழங்கள் +௫௫௫௧ வாழைப்பழங்கள்~ஐந்தாயிரத்து ஐநூற்று ஐம்பத்தொன்று வாழைப்பழங்கள் +85~எண்பத்திஐந்து +௯௧~தொண்ணூற்றொன்று +௧௩௨௩~ஆயிரத்து முந்நூற்று இருபத்திமூன்று +1345~ஆயிரத்து முந்நூற்று நாற்பத்திஐந்து +௧௩௪௫௬~பதிமூன்றுஆயிரத்து நானூற்று ஐம்பத்திஆறு +12346~பன்னிரண்டுஆயிரத்து முந்நூற்று நாற்பத்திஆறு +௧௨௩௪௫௬~ஒரு லட்சத்து இருபத்திமூன்றுஆயிரத்து நானூற்று ஐம்பத்திஆறு +872987~எட்டு லட்சத்து எழுபத்திரண்டுஆயிரத்து ஒன்பதுநூற்று எண்பத்திஏழு +௯௮௭௬௦௯~ஒன்பது லட்சத்து எண்பத்திஏழுஆயிரத்து அறுநூற்று ஒன்பது +9876789~தொண்ணூற்றிஎட்டு லட்சத்து எழுபத்திஆறுஆயிரத்து எழுநூற்று எண்பத்திஒன்பது +௨௩௪௫௫௬௭~இருபத்திமூன்று லட்சத்து நாற்பத்திஐந்துஆயிரத்து ஐநூற்று அறுபத்திஏழு +12345567~ஒரு கோடியே இருபத்திமூன்று லட்சத்து நாற்பத்திஐந்துஆயிரத்து ஐநூற்று அறுபத்திஏழு +௧௨௧௨௧௨௧௨~ஒரு கோடியே இருபத்தொன்று லட்சத்து இருபத்தொன்றுஆயிரத்து இருநூற்று பன்னிரண்டு +1122345567~நூற்று பன்னிரண்டு கோடியே இருபத்திமூன்று லட்சத்து நாற்பத்திஐந்துஆயிரத்து ஐநூற்று அறுபத்திஏழு +௧௦௨௨௩௪௫௫௬௭~நூற்று இரண்டு கோடியே இருபத்திமூன்று லட்சத்து நாற்பத்திஐந்துஆயிரத்து ஐநூற்று அறுபத்திஏழு +11022345567~ஆயிரத்து நூற்று இரண்டு கோடியே இருபத்திமூன்று லட்சத்து நாற்பத்திஐந்துஆயிரத்து ஐநூற்று அறுபத்திஏழு +௫௧௦௨௨௩௪௫௫௬௭~ஐந்தாயிரத்து நூற்று இரண்டு கோடியே இருபத்திமூன்று லட்சத்து நாற்பத்திஐந்துஆயிரத்து ஐநூற்று அறுபத்திஏழு +௦௫~சுழியம் ஐந்து +01~சுழியம் ஒன்று +௦௭௩~சுழியம் ஏழு மூன்று +0001~சுழியம் சுழியம் சுழியம் ஒன்று +௦௦௦~சுழியம் சுழியம் சுழியம் \ No newline at end of file diff --git a/tests/nemo_text_processing/ta/test_cardinal.py b/tests/nemo_text_processing/ta/test_cardinal.py new file mode 100644 index 000000000..f0944c24c --- /dev/null +++ b/tests/nemo_text_processing/ta/test_cardinal.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized +from nemo_text_processing.text_normalization.normalize import Normalizer +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestCardinal: + normalizer = Normalizer( + input_case='cased', lang='ta', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + + @parameterized.expand(parse_test_case_file('ta/data_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred == expected diff --git a/tests/nemo_text_processing/ta/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/ta/test_sparrowhawk_normalization.sh new file mode 100644 index 000000000..fca2affec --- /dev/null +++ b/tests/nemo_text_processing/ta/test_sparrowhawk_normalization.sh @@ -0,0 +1,41 @@ +#! /bin/sh + +PROJECT_DIR=/workspace/tests + +runtest () { + input=$1 + cd /workspace/sparrowhawk/documentation/grammars + + # read test file + while IFS= read -r testcase; do + IFS='~' read -r written spoken <<< "$testcase" + + # Escape backslashes and replace non breaking space with breaking space + escaped_written=$(printf '%s' "$written" | sed 's/\\/\\\\/g') + denorm_pred=$(echo "$escaped_written" | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1 | sed 's/\xC2\xA0/ /g') + + # trim white space + # spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + # denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # trim white space and remove space before punctuation + spoken="$(echo -e "${spoken}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's/ \([!?.]\)/\1/g')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's/ \([!?.]\)/\1/g')" + + # input expected actual + assertEquals "$written" "$spoken" "$denorm_pred" + done < "$input" +} + +#testTNSpecialText() { +# input=$PROJECT_DIR/ta/data_text_normalization/test_cases_special_text.txt +# runtest $input +#} + +testTNCardinal() { + input=$PROJECT_DIR/ta/data_text_normalization/test_cases_cardinal.txt + runtest $input +} + +# Load shUnit2 +. $PROJECT_DIR/../shunit2/shunit2 \ No newline at end of file diff --git a/tools/text_processing_deployment/pynini_export.py b/tools/text_processing_deployment/pynini_export.py index 73a4fc138..be5742d9c 100644 --- a/tools/text_processing_deployment/pynini_export.py +++ b/tools/text_processing_deployment/pynini_export.py @@ -103,6 +103,7 @@ def parse_args(): 'es_en', 'he', 'hi', + 'ta', 'hi_en', 'hy', 'mr', @@ -140,7 +141,7 @@ def parse_args(): if __name__ == '__main__': args = parse_args() - if args.language in ['ru', 'es_en', 'hi_en', 'mr'] and args.grammars == 'tn_grammars': + if args.language in ['ru', 'es_en', 'hi_en', 'mr', 'ta'] and args.grammars == 'tn_grammars': raise ValueError('Only ITN grammars could be deployed in Sparrowhawk for the selected languages.') TNPostProcessingFst = None ITNPostProcessingFst = None @@ -224,6 +225,11 @@ def parse_args(): PostProcessingFst as TNPostProcessingFst, ) from nemo_text_processing.text_normalization.hi.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst + elif args.language == 'ta': + from nemo_text_processing.text_normalization.ta.taggers.tokenize_and_classify import ( + ClassifyFst as TNClassifyFst, + ) + from nemo_text_processing.text_normalization.ta.verbalizers.verbalize import VerbalizeFst as TNVerbalizeFst elif args.language == 'hu': from nemo_text_processing.text_normalization.hu.taggers.tokenize_and_classify import ( ClassifyFst as TNClassifyFst,