From 7f2796ac54d14debbe319f9bd0130bd18caeb204 Mon Sep 17 00:00:00 2001 From: Ryan Leary Date: Wed, 19 Aug 2026 20:26:44 +0000 Subject: [PATCH] Replace setup.py packaging with pyproject.toml and add uv Move to a standard PEP 621 build, removing setup.py, setup.cfg, MANIFEST.in, reinstall.sh, the requirements/ directory and the stray repository-root __init__.py. Dependencies, extras, classifiers, package data and the black, isort and pytest settings all move into pyproject.toml unchanged; the version still comes from nemo_text_processing/package_info.py, now via [tool.setuptools.dynamic]. Three latent defects surface as a result of declaring things properly: * The wheel was shipping the test suite. packages=find_packages() picked up tests/ because it has an __init__.py, and the exclude=[...] argument was passed to setup() where it is not a parameter and did nothing. The built wheel drops 247 files under tests/ and is otherwise identical, file for file, to the one setup.py produced. * requires-python was never declared while the classifiers claimed 3.8. The test extra pins black==25.1.0, which needs >=3.9, so [all] has been unsatisfiable on 3.8. Declared >=3.9 to match what the dependencies actually allow. * isort[requirements] asks for an extra isort does not have, so the marker silently did nothing. Dropped. Jenkins now builds a wheel with uv and installs that, in place of reinstall.sh. No uv.lock is committed: this is a library, so the resolution that matters is the consumer's, and pynini publishes manylinux x86_64 wheels only. pyproject.toml also documents the two-line change that adds a native runtime package to a [tool.uv.workspace] later, and why that has to be a separate distribution rather than an extra of this one. Signed-off-by: Ryan Leary --- .gitignore | 3 + Jenkinsfile | 4 +- MANIFEST.in | 3 - README.md | 29 +++- __init__.py | 13 -- pyproject.toml | 154 ++++++++++++++++++ reinstall.sh | 24 --- requirements/requirements.txt | 13 -- requirements/requirements_test.txt | 13 -- setup.cfg | 38 ----- setup.py | 251 ----------------------------- 11 files changed, 187 insertions(+), 358 deletions(-) delete mode 100644 MANIFEST.in delete mode 100644 __init__.py create mode 100644 pyproject.toml delete mode 100755 reinstall.sh delete mode 100644 requirements/requirements.txt delete mode 100644 requirements/requirements_test.txt delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 5f72de53c..8450cc735 100644 --- a/.gitignore +++ b/.gitignore @@ -175,3 +175,6 @@ examples/neural_graphs/*.yml .hydra/ nemo_experiments/ *.swp + +# uv: this is a library, so no lockfile is committed (see pyproject.toml) +uv.lock diff --git a/Jenkinsfile b/Jenkinsfile index 39972c461..80185cd0c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,7 +44,9 @@ pipeline { stage('NeMo Installation') { steps { - sh './reinstall.sh release' + sh 'pip install -U uv' + sh 'uv build --wheel --out-dir dist' + sh 'uv pip install --system "$(ls dist/*.whl)[all]"' } } diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 11c42f612..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -include LICENSE -include NEWS -include requirements/* diff --git a/README.md b/README.md index 8bbb5c617..0c3171e1d 100644 --- a/README.md +++ b/README.md @@ -71,13 +71,38 @@ python -m pip install git+https://github.com/NVIDIA/NeMo-text-processing.git@{BR Use this installation mode if you are contributing to NeMo-text-processing. +We use [uv](https://docs.astral.sh/uv/) for development environments. + ``` git clone https://github.com/NVIDIA/NeMo-text-processing cd NeMo-text-processing -./reinstall.sh +uv sync +``` + +`uv sync` creates `.venv`, installs the project in editable mode and adds the +test and style tooling. Run things through it with `uv run`: + +``` +uv run pytest --cpu +uv run python nemo_text_processing/text_normalization/normalize.py --text="1" +``` + +**_NOTE:_** uv is a convenience, not a requirement — the project is a standard +PEP 621 package. `pip install -e ".[all]"` from the repository root does the same +thing. + +**_NOTE:_** No `uv.lock` is committed. This is a library rather than an +application, so the resolution that matters is the consumer's; run `uv lock` +locally if you want one. + + +### Building + +``` +uv build ``` -**_NOTE:_** If you only want the toolkit without additional conda-based dependencies, you may replace ``reinstall.sh`` with ``pip install -e .`` with the NeMo-text-processing root directory as your current working director. +writes a wheel and an sdist to `dist/`. `python -m build` does the same. Contributing diff --git a/__init__.py b/__init__.py deleted file mode 100644 index bc443be41..000000000 --- a/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..c05cd24cc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,154 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[project] +name = "nemo_text_processing" +description = "NeMo text processing for ASR and TTS" +readme = "README.md" +license = { text = "Apache-2.0" } +authors = [{ name = "NVIDIA", email = "nemo-toolkit@nvidia.com" }] +maintainers = [{ name = "NVIDIA", email = "nemo-toolkit@nvidia.com" }] +requires-python = ">=3.9" +keywords = [ + "NeMo", + "nvidia", + "tts", + "asr", + "text processing", + "text normalization", + "inverse text normalization", + "language", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Intended Audience :: Information Technology", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Scientific/Engineering :: Image Recognition", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Utilities", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Environment :: Console", + "Natural Language :: English", + "Operating System :: OS Independent", +] +dependencies = [ + "cdifflib", + "editdistance", + "inflect", + "joblib", + "pandas", + "pynini==2.1.6.post1", + "regex", + "sacremoses>=0.0.43", + "setuptools>=65.5.1", + "tqdm>=4.41.0", + "transformers", + "wget", + "wrapt", +] +dynamic = ["version"] + +[project.urls] +Homepage = "https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/stable/" +Repository = "https://github.com/nvidia/nemo-text-processing" +Download = "https://github.com/NVIDIA/NeMo-text-processing/releases" + +[project.optional-dependencies] +test = [ + "black==25.1.0", + "click>=8.0.2", + "isort>5.1.0,<=6.0.1", + "parameterized", + "pynini==2.1.6.post1", + "pytest", + "pytest-runner", + "ruamel.yaml", + "sphinx", + "sphinxcontrib-bibtex", + "wandb", + "wget", + "wrapt", +] +all = ["nemo_text_processing[test]"] + +[dependency-groups] +dev = ["nemo_text_processing[test]"] + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.dynamic] +version = { attr = "nemo_text_processing.package_info.__version__" } + +[tool.setuptools.packages.find] +include = ["nemo_text_processing*"] + +[tool.setuptools.package-data] +"*" = ["*.tsv", "*.far", "*.fst"] + +[tool.black] +skip-string-normalization = true +line-length = 119 + +[tool.isort] +known_localfolder = ["nemo", "tests"] +sections = ["FUTURE", "STDLIB", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"] +default_section = "THIRDPARTY" +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +line_length = 119 + +[tool.pytest.ini_options] +addopts = "--verbose --pyargs --durations=0" +testpaths = ["tests"] +norecursedirs = [ + "nemo", + "nemo_text_processing", + "external", + "examples", + "docs", + "scripts", + "tools", + "tutorials", + "*.egg", + ".*", + "_darcs", + "build", + "CVS", + "dist", + "venv", + "{arch}", +] +markers = [ + "unit: marks unit test, i.e. testing a single, well isolated functionality (deselect with '-m \"not unit\"')", + "integration: marks test checking the elements when integrated into subsystems (deselect with '-m \"not integration\"')", + "system: marks test working at the highest integration level (deselect with '-m \"not system\"')", + "acceptance: marks test checking whether the developed product/model passes the user defined acceptance criteria (deselect with '-m \"not acceptance\"')", + "docs: mark tests related to documentation (deselect with '-m \"not docs\"')", + "skipduringci: marks tests that are skipped ci as they are addressed by Jenkins jobs but should be run to test user setups", + "pleasefixme: marks tests that are broken and need fixing", +] diff --git a/reinstall.sh b/reinstall.sh deleted file mode 100755 index 2f1aecd67..000000000 --- a/reinstall.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -e - -INSTALL_OPTION=${1:-"dev"} - -PIP=pip - -echo 'Uninstalling stuff' -${PIP} uninstall -y nemo_text_processing - -${PIP} install -U setuptools - -echo 'Installing nemo' -if [[ "$INSTALL_OPTION" == "dev" ]]; then - ${PIP} install --editable ".[all]" -else - rm -rf dist/ - ${PIP} install build - python -m build --no-isolation --wheel - DIST_FILE=$(find ./dist -name "*.whl" | head -n 1) - ${PIP} install "${DIST_FILE}[all]" -fi - -echo 'All done!' diff --git a/requirements/requirements.txt b/requirements/requirements.txt deleted file mode 100644 index 6622e2d49..000000000 --- a/requirements/requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -cdifflib -editdistance -inflect -joblib -pandas -pynini==2.1.6.post1 -regex -sacremoses>=0.0.43 -setuptools>=65.5.1 -tqdm>=4.41.0 -transformers -wget -wrapt diff --git a/requirements/requirements_test.txt b/requirements/requirements_test.txt deleted file mode 100644 index aacfde319..000000000 --- a/requirements/requirements_test.txt +++ /dev/null @@ -1,13 +0,0 @@ -black==25.1.0 -click>=8.0.2 -isort[requirements]>5.1.0,<=6.0.1 -parameterized -pynini==2.1.6.post1 -pytest -pytest-runner -ruamel.yaml -sphinx -sphinxcontrib-bibtex -wandb -wget -wrapt diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 5b0dd345a..000000000 --- a/setup.cfg +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[aliases] -test=pytest - -# durations=0 will display all tests execution time, sorted in ascending order starting from from the slowest one. -# -vv will also display tests with durration = 0.00s -[tool:pytest] -addopts = --verbose --pyargs --durations=0 -testpaths = tests -norecursedirs = nemo nemo_text_processing external examples docs scripts tools tutorials *.egg .* _darcs build CVS dist venv {arch} -markers = - unit: marks unit test, i.e. testing a single, well isolated functionality (deselect with '-m "not unit"') - integration: marks test checking the elements when integrated into subsystems (deselect with '-m "not integration"') - system: marks test working at the highest integration level (deselect with '-m "not system"') - acceptance: marks test checking whether the developed product/model passes the user defined acceptance criteria (deselect with '-m "not acceptance"') - docs: mark tests related to documentation (deselect with '-m "not docs"') - skipduringci: marks tests that are skipped ci as they are addressed by Jenkins jobs but should be run to test user setups - pleasefixme: marks tests that are broken and need fixing - -[isort] -known_localfolder = nemo,tests -sections = FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER -default_section = THIRDPARTY - -skip = setup.py diff --git a/setup.py b/setup.py deleted file mode 100644 index e22afbab3..000000000 --- a/setup.py +++ /dev/null @@ -1,251 +0,0 @@ -# ! /usr/bin/python -# -*- coding: utf-8 -*- - -# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Setup for pip package.""" - -import codecs -import importlib.util -import os -import subprocess -from distutils import cmd as distutils_cmd -from distutils import log as distutils_log -from itertools import chain - -import setuptools - -spec = importlib.util.spec_from_file_location('package_info', 'nemo_text_processing/package_info.py') -package_info = importlib.util.module_from_spec(spec) -spec.loader.exec_module(package_info) - - -__contact_emails__ = package_info.__contact_emails__ -__contact_names__ = package_info.__contact_names__ -__description__ = package_info.__description__ -__download_url__ = package_info.__download_url__ -__homepage__ = package_info.__homepage__ -__keywords__ = package_info.__keywords__ -__license__ = package_info.__license__ -__package_name__ = package_info.__package_name__ -__repository_url__ = package_info.__repository_url__ -__version__ = package_info.__version__ - - -if os.path.exists('README.md'): - with open("README.md", "r", encoding='utf-8') as fh: - long_description = fh.read() - long_description_content_type = "text/markdown" - -elif os.path.exists('README.rst'): - # codec is used for consistent encoding - long_description = codecs.open( - os.path.join(os.path.abspath(os.path.dirname(__file__)), 'README.rst'), - 'r', - encoding='utf-8', - ).read() - long_description_content_type = "text/x-rst" - -else: - long_description = 'See ' + __homepage__ - - -############################################################################### -# Dependency Loading # -# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% # - - -def req_file(filename, folder="requirements"): - with open(os.path.join(folder, filename), encoding='utf-8') as f: - content = f.readlines() - # you may also want to remove whitespace characters - # Example: `\n` at the end of each line - return [x.strip() for x in content] - - -install_requires = req_file("requirements.txt") - -extras_require = { - # User packages - 'test': req_file("requirements_test.txt") -} - - -extras_require['all'] = list(chain(extras_require.values())) - -# Add lightning requirements as needed -# extras_require['nemo_text_processing'] = list(chain([extras_require['nemo_text_processing']])) -# extras_require['test'] = list( -# chain( -# [ -# extras_require['nemo_text_processing'], -# ] -# ) -# ) - -tests_requirements = extras_require["test"] - - -############################################################################### -# Code style checkers # -# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% # - - -class StyleCommand(distutils_cmd.Command): - __LINE_WIDTH = 119 - __ISORT_BASE = ( - 'isort ' - # These two lines makes isort compatible with black. - '--multi-line=3 --trailing-comma --force-grid-wrap=0 ' - f'--use-parentheses --line-width={__LINE_WIDTH} -rc -ws' - ) - __BLACK_BASE = f'black --skip-string-normalization --line-length={__LINE_WIDTH}' - description = 'Checks overall project code style.' - user_options = [ - ('scope=', None, 'Folder of file to operate within.'), - ('fix', None, 'True if tries to fix issues in-place.'), - ] - - def __call_checker(self, base_command, scope, check): - command = list(base_command) - - command.append(scope) - - if check: - command.extend(['--check', '--diff']) - - self.announce( - msg='Running command: %s' % str(' '.join(command)), - level=distutils_log.INFO, - ) - - return_code = subprocess.call(command) - - return return_code - - def _isort(self, scope, check): - return self.__call_checker( - base_command=self.__ISORT_BASE.split(), - scope=scope, - check=check, - ) - - def _black(self, scope, check): - return self.__call_checker( - base_command=self.__BLACK_BASE.split(), - scope=scope, - check=check, - ) - - def _pass(self): - self.announce(msg='\033[32mPASS\x1b[0m', level=distutils_log.INFO) - - def _fail(self): - self.announce(msg='\033[31mFAIL\x1b[0m', level=distutils_log.INFO) - - # noinspection PyAttributeOutsideInit - def initialize_options(self): - self.scope = '.' - self.fix = '' - - def run(self): - scope, check = self.scope, not self.fix - isort_return = self._isort(scope=scope, check=check) - black_return = self._black(scope=scope, check=check) - - if isort_return == 0 and black_return == 0: - self._pass() - else: - self._fail() - exit(isort_return if isort_return != 0 else black_return) - - def finalize_options(self): - pass - - -############################################################################### - -setuptools.setup( - name=__package_name__, - # Versions should comply with PEP440. For a discussion on single-sourcing - # the version across setup.py and the project code, see - # https://packaging.python.org/en/latest/single_source_version.html - version=__version__, - description=__description__, - long_description=long_description, - long_description_content_type=long_description_content_type, - # The project's main homepage. - url=__repository_url__, - download_url=__download_url__, - # Author details - author=__contact_names__, - author_email=__contact_emails__, - # maintainer Details - maintainer=__contact_names__, - maintainer_email=__contact_emails__, - # The licence under which the project is released - license=__license__, - classifiers=[ - # How mature is this project? Common values are - # 1 - Planning - # 2 - Pre-Alpha - # 3 - Alpha - # 4 - Beta - # 5 - Production/Stable - # 6 - Mature - # 7 - Inactive - 'Development Status :: 5 - Production/Stable', - # Indicate who your project is intended for - 'Intended Audience :: Developers', - 'Intended Audience :: Science/Research', - 'Intended Audience :: Information Technology', - # Indicate what your project relates to - 'Topic :: Scientific/Engineering', - 'Topic :: Scientific/Engineering :: Mathematics', - 'Topic :: Scientific/Engineering :: Image Recognition', - 'Topic :: Scientific/Engineering :: Artificial Intelligence', - 'Topic :: Software Development :: Libraries', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: Utilities', - # Pick your license as you wish (should match "license" above) - 'License :: OSI Approved :: Apache Software License', - # Supported python versions - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - # Additional Setting - 'Environment :: Console', - 'Natural Language :: English', - 'Operating System :: OS Independent', - ], - packages=setuptools.find_packages(), - install_requires=install_requires, - # setup_requires=['pytest-runner'], - tests_require=tests_requirements, - # List additional groups of dependencies here (e.g. development - # dependencies). You can install these using the following syntax, - # $ pip install -e ".[all]" - # $ pip install nemo_toolkit[all] - extras_require=extras_require, - # Add in any packaged data. - include_package_data=True, - exclude=['tools', 'tests', 'data'], - package_data={'': ['*.tsv', '*.far', '*.fst']}, - zip_safe=False, - # PyPI package information. - keywords=__keywords__, - # Custom commands. - cmdclass={'style': StyleCommand}, -)