diff --git a/.github/workflows/check-coverage.yml b/.github/workflows/check-coverage.yml index cb83fa36..0bfaa03a 100644 --- a/.github/workflows/check-coverage.yml +++ b/.github/workflows/check-coverage.yml @@ -28,10 +28,11 @@ jobs: run: | python --version python -m pip install --upgrade pip - pip install -e .[build] - pip install -e .[test] + python -m pip install --upgrade build + pip install . + pip install .[test] doit version - python setup.py build + python -m build # https://github.com/marketplace/actions/pytest-coverage-comment - name: Generate coverage report diff --git a/.github/workflows/generate-metadata.yml b/.github/workflows/generate-metadata.yml index 213b564a..7f4abb2c 100644 --- a/.github/workflows/generate-metadata.yml +++ b/.github/workflows/generate-metadata.yml @@ -20,12 +20,15 @@ jobs: - name: Install App and Extras run: | + python --version python -m pip install --upgrade pip - python -m pip install -e .[build] + python -m pip install --upgrade build + pip install . + pip install .[test] doit version - python setup.py build + python -m build - - name: Generate dependencies list + - name: Run license check run: python bin/license-checker.py - name: Type-check @@ -35,7 +38,10 @@ jobs: run: black . --check - name: Generate pyinstaller specs - run: echo Using locally generated specs - # pyi-makespec tabcmd\tabcmd.py -n tabcmd-windows -i res\tabcmd.ico -F -c --version-file versionfile.txt --onefile --collect-data tabcmd.locales && - # pyi-makespec tabcmd\tabcmd.py -n tabcmd-mac -i res\tabcmd.icns -F -c --version-file versionfile.txt --onefile --collect-data tabcmd.locales --osx-bundle-identifier com.tableau.tabcmd - # 3. Unix + run: | + echo "Using manually generated specs" + + # todo: add hidden dependencies to these command lines + # pyi-makespec tabcmd\tabcmd.py -n tabcmd-windows -i res\tabcmd.ico -F -c --version-file versionfile.txt --onefile --collect-data tabcmd.locales + # pyi-makespec tabcmd\tabcmd.py -n tabcmd-mac -i res\tabcmd.icns -F -c --version-file versionfile.txt --onefile --collect-data tabcmd.locales --osx-bundle-identifier com.tableau.tabcmd + # 3. Unix diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index cf6ba4fc..2b46a3e6 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -26,13 +26,12 @@ jobs: TARGET: windows CMD_BUILD: > pyinstaller tabcmd-windows.spec --clean --noconfirm --distpath ./dist/windows - OUT_FILE_NAME: tabcmd.exe + OUT_FILE_NAME: tabcmd-windows.exe ASSET_MIME: application/vnd.microsoft.portable-executable - os: macos-latest TARGET: macos CMD_BUILD: > - pyinstaller tabcmd-mac.spec --clean --noconfirm --distpath ./dist/macos && - cwd && ls && ls dist + pyinstaller tabcmd-mac.spec --clean --noconfirm --distpath ./dist/macos && ls && ls dist # zip -r9 mac tabcmd-mac* OUT_FILE_NAME: tabcmd-mac.app # tabcmd.zip ASSET_MIME: application/zip @@ -54,10 +53,13 @@ jobs: - name: Install dependencies and build run: | - pip install -e .[build] - pip install -e .[package] + python --version + python -m pip install --upgrade pip + python -m pip install --upgrade build + pip install . + pip install .[test] doit version - python setup.py build + python -m build - name: Package with pyinstaller for ${{matrix.TARGET}} run: ${{matrix.CMD_BUILD}} @@ -66,7 +68,7 @@ jobs: uses: WebFreak001/upload-asset@v1.0.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # automatically provided by github actions - OS: windows # just a variable we use in the name pattern + OS: windows # a variable we use in the name pattern? with: file: ./dist/${{ matrix.TARGET }}/tabcmd/${{ matrix.OUT_FILE_NAME}} mime: ${{ matrix.ASSET_MIME}} # required by GitHub API diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index f0a36b53..42c234e3 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -23,9 +23,13 @@ jobs: python-version: 3.8 - name: Build dist files run: | - pip install -e .[build] + python --version + python -m pip install --upgrade pip + python -m pip install --upgrade build + pip install . + pip install .[package] doit version - python setup.py sdist --formats=gztar + python -m build - name: Publish distribution 📦 to Test PyPI if: ${{ inputs.is_draft }} uses: pypa/gh-action-pypi-publish@release/v1 # license BSD-2 diff --git a/.github/workflows/run-e2-tests.yml b/.github/workflows/run-e2-tests.yml index 26f2222c..ec1262af 100644 --- a/.github/workflows/run-e2-tests.yml +++ b/.github/workflows/run-e2-tests.yml @@ -34,10 +34,11 @@ jobs: run: | python --version python -m pip install --upgrade pip - pip install -e .[build] - pip install -e .[test] + python -m pip install --upgrade build + pip install . + pip install .[test] doit version - python setup.py build + python -m build - name: Run e2e tests run: | diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 37266837..bad6a73b 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -31,10 +31,11 @@ jobs: run: | python --version python -m pip install --upgrade pip - pip install -e .[build] - pip install -e .[test] + python -m pip install --upgrade build + pip install . + pip install .[test] doit version - python setup.py build + python -m build - name: Test with pytest run: pytest diff --git a/pyproject.toml b/pyproject.toml index 485decd9..77c00308 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,73 @@ [build-system] requires = ["build", "setuptools>=62", "wheel", "setuptools_scm>=6.2"] +build-backend = "setuptools.build_meta" [tool.setuptools_scm] -"local_scheme"= "no-local-version" # require pypi supported versions always +local_scheme = "no-local-version" # require pypi supported versions always +[tool.setuptools] +packages = ["tabcmd"] +[tool.setuptools.package-data] +tabcmd = ["tabcmd.locales/**/*.mo"] [tool.black] line-length = 120 +target-version = ['py37', 'py38', 'py39', 'py310'] extend-exclude = '^/bin/*' +[tool.mypy] +disable_error_code = [ + 'misc', + 'import' +] +files = ["tabcmd", "tests"] +show_error_codes = true +ignore_missing_imports = true +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "--junitxml=./test.junit.xml" + +[project] +name="tabcmd" +dynamic = ["version"] +description="A command line client for working with Tableau Server." +authors = [{name="Tableau", email="github@tableau.com"}] +license = {file = "LICENSE"} +readme = "README.md" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10" +] +dependencies = [ + 'argparse', + "appdirs", + "doit", + "ftfy", + "polling2", + "pyinstaller_versionfile", + "requests>=2.11,<3.0", + "setuptools_scm", + "types-appdirs", + "types-mock", + "types-requests", + "types-setuptools", + "tableauserverclient>=0.19", + "urllib3>=1.24.3,<2.0", +] +[project.optional-dependencies] +test = [ + "black", + "mock", + "mypy", + "pytest>=7.0", + "pytest-cov", + "pytest-order", + "pytest-runner", + "requests-mock>=1.0,<2.0"] +localize = ["doit", "ftfy"] +package = ["pyinstaller>=5.1"] +[project.urls] +repository = "https://github.com/tableau/tabcmd" +[project.scripts] +tabcmd = "tabcmd.tabcmd:main" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index d4230e64..00000000 --- a/setup.cfg +++ /dev/null @@ -1,10 +0,0 @@ -[pycodestyle] -max_line_length = 120 -[mypy] -exclude = tests/e2e -ignore_missing_imports = True -[tool:pytest] -# useful settings for debugging -# log_cli = True -# log_cli_level = DEBUG - diff --git a/setup.py b/setup.py index 8d834b82..914e6b5d 100644 --- a/setup.py +++ b/setup.py @@ -1,63 +1,4 @@ -from setuptools import setup, find_packages +# legacy stub setup.py, do not add configuration here +from setuptools import setup -setup( - name="tabcmd", - author="Tableau", - author_email="github@tableau.com", - description="A command line client for working with Tableau Server.", - long_description="A command line client for working with Tableau Server.", - license="MIT", - url="https://github.com/tableau/tabcmd", - python_requires=">=3.7", - packages=find_packages(), - package_data={"tabcmd": ["tabcmd.locales/**/*.mo"]}, - include_package_data=True, - entry_points={"console_scripts": ["tabcmd = tabcmd.tabcmd:main"]}, - setup_requires=[ - # copy of pyproject.toml for back compat - "build", - "setuptools>=62", - "setuptools_scm>=6.2", - "wheel", - ], - install_requires=[ - "polling2", - "requests>=2.11,<3.0", - "tableauserverclient>=0.19", - "urllib3>=1.24.3,<2.0", - ], - extras_require={ - "localize": [ - "doit", - "ftfy", - ], - "build": [ - "appdirs", - "black", - "doit", - "ftfy", - "mypy", - "pyinstaller_versionfile", - "setuptools>=62", - "setuptools_scm", - "types-appdirs", - "types-mock", - "types-requests", - "types-setuptools", - ], - "package": [ - "pyinstaller>=5.1", - "pyinstaller-versionfile", - ], - "test": [ - "mock", - "pytest", - "pytest-cov", - "pytest-order", - "pytest-runner", - "requests-mock>=1.0,<2.0", - ], - }, - test_suite="tests", - zip_safe=False, -) +setup() diff --git a/tabcmd-linux.spec b/tabcmd-linux.spec index d71fd83e..c5e367fc 100644 --- a/tabcmd-linux.spec +++ b/tabcmd-linux.spec @@ -12,7 +12,7 @@ a = Analysis( pathex=[], binaries=[], datas=datas, - hiddenimports=[], + hiddenimports=['tableauserverclient', 'requests.packages.urllib3', 'pkg_resources'], hookspath=[], hooksconfig={}, runtime_hooks=[], diff --git a/tabcmd-mac.spec b/tabcmd-mac.spec index fcce4bfe..fee41fa8 100644 --- a/tabcmd-mac.spec +++ b/tabcmd-mac.spec @@ -13,7 +13,7 @@ a = Analysis( pathex=[], binaries=[], datas=datas, - hiddenimports=[], + hiddenimports=['tableauserverclient', 'requests.packages.urllib3', 'pkg_resources'], hookspath=[], hooksconfig={}, runtime_hooks=[], diff --git a/tabcmd-windows.spec b/tabcmd-windows.spec index d71fd83e..03f1dfd1 100644 --- a/tabcmd-windows.spec +++ b/tabcmd-windows.spec @@ -12,7 +12,7 @@ a = Analysis( pathex=[], binaries=[], datas=datas, - hiddenimports=[], + hiddenimports=['tableauserverclient', 'requests', 'pkg_resources'], hookspath=[], hooksconfig={}, runtime_hooks=[], diff --git a/tabcmd.spec b/tabcmd.spec index 650b541a..a1e0421c 100644 --- a/tabcmd.spec +++ b/tabcmd.spec @@ -9,7 +9,7 @@ a = Analysis( pathex=[], binaries=[], datas=[], - hiddenimports=[], + hiddenimports=['tableauserverclient', 'requests.packages.urllib3', 'pkg_resources'], hookspath=[], hooksconfig={}, runtime_hooks=[], diff --git a/tabcmd/commands/auth/session.py b/tabcmd/commands/auth/session.py index 866efbc8..805ad387 100644 --- a/tabcmd/commands/auth/session.py +++ b/tabcmd/commands/auth/session.py @@ -214,7 +214,7 @@ def create_session(self, args): self._read_existing_state() self._update_session_data(args) self.logging_level = args.logging_level or self.logging_level - self.logger = log(__name__, self.logging_level) + self.logger = self.logger or log(__class__.__name__, self.logging_level) credentials = None if args.password: @@ -245,8 +245,8 @@ def create_session(self, args): signed_in_object = self._sign_in(credentials) if not signed_in_object: - missing_var = _("editdomain.errors.requires_nickname_name").format("username", "token") - Errors.exit_with_error(self.logger, _("session.errors.missing_arguments").format(missing_var)) + message = "Run 'tabcmd login -h' for details on required arguments" + Errors.exit_with_error(self.logger, _("session.errors.missing_arguments").format(message)) if args.no_cookie: self._remove_json() else: diff --git a/tabcmd/commands/datasources_and_workbooks/datasources_and_workbooks_command.py b/tabcmd/commands/datasources_and_workbooks/datasources_and_workbooks_command.py index af54d65a..1eadb3a5 100644 --- a/tabcmd/commands/datasources_and_workbooks/datasources_and_workbooks_command.py +++ b/tabcmd/commands/datasources_and_workbooks/datasources_and_workbooks_command.py @@ -25,7 +25,7 @@ def get_view_by_content_url(logger, server, view_content_url) -> TSC.ViewItem: req_option.filter.add(TSC.Filter("contentUrl", TSC.RequestOptions.Operator.Equals, view_content_url)) matching_views, paging = server.views.get(req_option) except TSC.ServerResponseError as e: - Errors.exit_with_error(logger, _("publish.errors.unexpected_server_response").format("")) + Errors.exit_with_error(logger, _("publish.errors.unexpected_server_response").format(e)) if len(matching_views) < 1: Errors.exit_with_error(logger, message=_("errors.xmlapi.not_found")) return matching_views[0] @@ -42,3 +42,83 @@ def get_wb_by_content_url(logger, server, workbook_content_url) -> TSC.WorkbookI if len(matching_workbooks) < 1: Errors.exit_with_error(logger, message=_("dataalerts.failure.error.workbookNotFound")) return matching_workbooks[0] + + @staticmethod + def apply_values_from_url_params(request_options: TSC.PDFRequestOptions, url, logger) -> None: + # should be able to replace this with request_options._append_view_filters(params) + logger.debug(url) + try: + if "?" in url: + query = url.split("?")[1] + logger.trace("Query parameters: {}".format(query)) + else: + logger.debug("No query parameters present in url") + return + + params = query.split("&") + logger.trace(params) + for value in params: + if value.startswith(":"): + DatasourcesAndWorkbooks.apply_option_value(request_options, value, logger) + else: # it must be a filter + DatasourcesAndWorkbooks.apply_filter_value(request_options, value, logger) + + except BaseException as e: + logger.warn("Error building filter params", e) + # ExportCommand.log_stack(logger) # type: ignore + + @staticmethod + def apply_filter_value(request_options: TSC.PDFRequestOptions, value: str, logger) -> None: + # todo: do we need to strip Parameters.x -> x? + logger.trace("handling filter param {}".format(value)) + data_filter = value.split("=") + request_options.vf(data_filter[0], data_filter[1]) + + @staticmethod + def apply_option_value(request_options: TSC.PDFRequestOptions, value: str, logger) -> None: + logger.trace("handling url option {}".format(value)) + setting = value.split("=") + if ":iid" == setting[0]: + logger.debug(":iid value ignored in url") + elif ":refresh" == setting[0] and DatasourcesAndWorkbooks.is_truthy(setting[1]): + # mypy is worried that this is readonly + request_options.max_age = 0 # type:ignore + logger.debug("Set max age to {} from {}".format(request_options.max_age, value)) + elif ":size" == setting[0]: + height, width = setting[1].split(",") + logger.warn("Height/weight parameters not yet implemented ({})".format(value)) + else: + logger.debug("Parameter[s] not recognized: {}".format(value)) + + @staticmethod + def is_truthy(value: str): + return value.lower() in ["yes", "y", "1", "true"] + + @staticmethod + def apply_png_options(request_options: TSC.ImageRequestOptions, args, logger): + if args.height or args.width: + # only applicable for png + logger.warn("Height/width arguments not yet implemented in export") + # Always request high-res images + request_options.image_resolution = "high" + + @staticmethod + def apply_pdf_options(request_options: TSC.PDFRequestOptions, args, logger): + request_options.page_type = args.pagesize + if args.pagelayout: + logger.debug("Setting page layout to: {}".format(args.pagelayout)) + request_options.orientation = args.pagelayout + + @staticmethod + def save_to_data_file(logger, output, filename): + logger.info(_("httputils.found_attachment").format(filename)) + with open(filename, "wb") as f: + f.writelines(output) + logger.info(_("export.success").format("", filename)) + + @staticmethod + def save_to_file(logger, output, filename): + logger.info(_("httputils.found_attachment").format(filename)) + with open(filename, "wb") as f: + f.write(output) + logger.info(_("export.success").format("", filename)) diff --git a/tabcmd/commands/datasources_and_workbooks/export_command.py b/tabcmd/commands/datasources_and_workbooks/export_command.py index f05a88ce..428c530f 100644 --- a/tabcmd/commands/datasources_and_workbooks/export_command.py +++ b/tabcmd/commands/datasources_and_workbooks/export_command.py @@ -6,6 +6,8 @@ from tabcmd.execution.logger_config import log from .datasources_and_workbooks_command import DatasourcesAndWorkbooks +pagesize = TSC.PDFRequestOptions.PageType # type alias for brevity + class ExportCommand(DatasourcesAndWorkbooks): @@ -24,16 +26,38 @@ def define_args(export_parser): export_parser.add_argument( "--pagelayout", choices=["landscape", "portrait"], - default="landscape", + default=None, help="page orientation (landscape or portrait) of the exported PDF", ) - export_parser.add_argument("--pagesize", default="letter", help="Set the page size of the exported PDF") - export_parser.add_argument("--width", default=800, help="Set the width in pixels. Default is 800 px") + export_parser.add_argument( + "--pagesize", + choices=[ + pagesize.A3, + pagesize.A4, + pagesize.A5, + pagesize.B4, + pagesize.B5, + pagesize.Executive, + pagesize.Folio, + pagesize.Ledger, + pagesize.Legal, + pagesize.Letter, + pagesize.Note, + pagesize.Quarto, + pagesize.Tabloid, + pagesize.Unspecified, + ], + default="letter", + help="Set the page size of the exported PDF", + ) + + export_parser.add_argument( + "--width", default=800, help="Set the width of the image in pixels. Default is 800 px" + ) export_parser.add_argument("--filename", "-f", help="filename to store the exported data") export_parser.add_argument("--height", default=600, help=_("export.options.height")) export_parser.add_argument( "--filter", - "-vf", metavar="COLUMN:VALUE", help="View filter to apply to the view", ) @@ -52,14 +76,16 @@ def run_command(args): view_content_url, wb_content_url = ExportCommand.parse_export_url_to_workbook_and_view(logger, args.url) logger.debug([view_content_url, wb_content_url]) if not view_content_url and not wb_content_url: - Errors.exit_with_error(logger, _("export.errors.requires_workbook_view_param").format(ExportCommand)) - - logger.debug(args.pagelayout, args.pagesize, args.filename, args.width, args.height, args.filter) + view_example = "/workbook_name/view_name" + message = "{} [{}]".format( + _("export.errors.requires_workbook_view_param").format(__class__.__name__), view_example + ) + Errors.exit_with_error(logger, message) try: if args.fullpdf: # it's a workbook workbook_item = ExportCommand.get_wb_by_content_url(logger, server, wb_content_url) - output = ExportCommand.download_wb_pdf(server, workbook_item, args.url, logger) + output = ExportCommand.download_wb_pdf(server, workbook_item, args, logger) default_filename = "{}.pdf".format(workbook_item.name) @@ -67,13 +93,13 @@ def run_command(args): view_item = ExportCommand.get_view_by_content_url(logger, server, view_content_url) if args.pdf: - output = ExportCommand.download_view_pdf(server, view_item, args.url, logger) + output = ExportCommand.download_view_pdf(server, view_item, args, logger) default_filename = "{}.pdf".format(view_item.name) elif args.csv: - output = ExportCommand.download_csv(server, view_item, args.url, logger) + output = ExportCommand.download_csv(server, view_item, args, logger) default_filename = "{}.csv".format(view_item.name) elif args.png: - output = ExportCommand.download_png(server, view_item, args.url, logger) + output = ExportCommand.download_png(server, view_item, args, logger) default_filename = "{}.png".format(view_item.name) @@ -92,59 +118,59 @@ def run_command(args): Errors.exit_with_error(logger, "Error saving to file", e) @staticmethod - def extract_filter_values_from_url_params(request_options: TSC.PDFRequestOptions, url, logger=None) -> None: - try: - # todo make logging better - logger = logger or log(ExportCommand.__class__.__name__, "DEBUG") - logger.debug(url) - - if "?" in url: - query = url.split("?")[1] - else: - return - - params = query.split("&") - logger.trace(params) + def apply_values_from_args(request_options: TSC.PDFRequestOptions, args, logger=None) -> None: + logger.debug( + "Args: {}, {}, {}, {}, {}".format(args.pagelayout, args.pagesize, args.width, args.height, args.filter) + ) + if args.pagelayout: + request_options.orientation = args.pagelayout + if args.pagesize: + request_options.page_type = args.pagesize + if args.filter: + params = args.filter.split("&") for value in params: - data_filter = value.split("=") - request_options.vf(data_filter[0], data_filter[1]) - except BaseException as e: - logger.error("Error building filter params", e) - ExportCommand.log_stack(logger) # type: ignore + ExportCommand.apply_filter_value(request_options, value, logger) @staticmethod - def download_wb_pdf(server, workbook_item, url, logger): - logger.trace(url) - pdf = TSC.PDFRequestOptions(maxage=1) - ExportCommand.extract_filter_values_from_url_params(pdf, url) - - server.workbooks.populate_pdf(workbook_item, pdf) + def download_wb_pdf(server, workbook_item, args, logger): + logger.debug(args.url) + pdf_options = TSC.PDFRequestOptions(maxage=1) + ExportCommand.apply_values_from_url_params(pdf_options, args.url, logger) + ExportCommand.apply_values_from_args(pdf_options, args, logger) + logger.debug(pdf_options.get_query_params()) + server.workbooks.populate_pdf(workbook_item, pdf_options) return workbook_item.pdf @staticmethod - def download_view_pdf(server, view_item, url, logger): - logger.trace(url) - pdf = TSC.PDFRequestOptions(maxage=1) - ExportCommand.extract_filter_values_from_url_params(pdf, url) - logger.trace(pdf.view_filters) - server.views.populate_pdf(view_item, pdf) + def download_view_pdf(server, view_item, args, logger): + logger.debug(args.url) + pdf_options = TSC.PDFRequestOptions(maxage=1) + ExportCommand.apply_values_from_url_params(pdf_options, args.url, logger) + ExportCommand.apply_values_from_args(pdf_options, args, logger) + logger.debug(pdf_options.get_query_params()) + server.views.populate_pdf(view_item, pdf_options) return view_item.pdf @staticmethod - def download_csv(server, view_item, url, logger): - logger.trace(url) - csv = TSC.CSVRequestOptions(maxage=1) - ExportCommand.extract_filter_values_from_url_params(csv, url) - server.views.populate_csv(view_item, csv) + def download_csv(server, view_item, args, logger): + logger.debug(args.url) + csv_options = TSC.CSVRequestOptions(maxage=1) + ExportCommand.apply_values_from_url_params(csv_options, args.url, logger) + ExportCommand.apply_values_from_args(csv_options, args, logger) + logger.debug(csv_options.get_query_params()) + server.views.populate_csv(view_item, csv_options) return view_item.csv @staticmethod - def download_png(server, view_item, url, logger): - logger.trace(url) - req_option_image = TSC.ImageRequestOptions(maxage=1) - ExportCommand.extract_filter_values_from_url_params(req_option_image, url) - server.views.populate_image(view_item, req_option_image) - return view_item.png + def download_png(server, view_item, args, logger): + logger.debug(args.url) + image_options = TSC.ImageRequestOptions(maxage=1) + ExportCommand.apply_values_from_url_params(image_options, args.url, logger) + ExportCommand.apply_values_from_args(image_options, args, logger) + DatasourcesAndWorkbooks.apply_png_options(image_options, args, logger) + logger.debug(image_options.get_query_params()) + server.views.populate_image(view_item, image_options) + return view_item.image @staticmethod def parse_export_url_to_workbook_and_view(logger, url): @@ -163,17 +189,3 @@ def parse_export_url_to_workbook_and_view(logger, url): workbook = name_parts[0] view = "{}/sheets/{}".format(workbook, name_parts[1]) return view, workbook - - @staticmethod - def save_to_data_file(logger, output, filename): - logger.info(_("httputils.found_attachment").format(filename)) - with open(filename, "wb") as f: - f.writelines(output) - logger.info(_("export.success").format("", filename)) - - @staticmethod - def save_to_file(logger, output, filename): - logger.info(_("httputils.found_attachment").format(filename)) - with open(filename, "wb") as f: - f.write(output) - logger.info(_("export.success").format("", filename)) diff --git a/tabcmd/commands/datasources_and_workbooks/get_url_command.py b/tabcmd/commands/datasources_and_workbooks/get_url_command.py index 4e1ba4b8..be0d96a0 100644 --- a/tabcmd/commands/datasources_and_workbooks/get_url_command.py +++ b/tabcmd/commands/datasources_and_workbooks/get_url_command.py @@ -37,48 +37,51 @@ def run_command(args): if " " in args.url: Errors.exit_with_error(logger, _("export.errors.white_space_workbook_view")) - if not args.url.startswith("/"): - args.url = "/" + args.url - logger.trace("helpfully fix format of url: " + args.url) + url = args.url.lstrip("/") # strip opening / if present + + file_type = GetUrl.get_file_type_from_filename(logger, args.filename, url) + content_type = GetUrl.evaluate_content_type(logger, url) - file_type = GetUrl.get_file_type_from_filename(logger, args.filename, args.url) - content_type = GetUrl.evaluate_content_type(logger, args.url) if content_type == "workbook": - if file_type == "twbx" or file_type == "twb": - GetUrl.generate_twb(logger, server, args, file_type) + if file_type in ["twbx", "twb"]: + GetUrl.generate_twb(logger, server, args, file_type, url) else: Errors.exit_with_error( logger, message=_("publish.errors.mutually_exclusive_option").format("twb", "twbx") ) else: # content type = view + view_url = GetUrl.get_view_url(url, logger) if file_type == "pdf": - GetUrl.generate_pdf(logger, server, args) + GetUrl.generate_pdf(logger, server, args, view_url) elif file_type == "png": - GetUrl.generate_png(logger, server, args) + GetUrl.generate_png(logger, server, args, view_url) elif file_type == "csv": - GetUrl.generate_csv(logger, server, args) + GetUrl.generate_csv(logger, server, args, view_url) else: Errors.exit_with_error(logger, message=_("tabcmd.get.extension.not_found")) @staticmethod def evaluate_content_type(logger, url): - # specify a view to get using "/views//." - # specify a workbook to get using "/workbooks/.". - if url.find("/views/") == 0: + # specify a view to get using "views//." + # specify a workbook to get using "workbooks/.". + if url.startswith("views/"): return "view" - elif url.find("/workbooks/") == 0: + elif url.startswith("workbooks/"): return "workbook" else: - view_example = "/views//." - message = "{} [{}]".format( - _("export.errors.requires_workbook_view_param").format(__class__.__name__), view_example - ) - Errors.exit_with_error(logger, message) + GetUrl.explain_expected_url(logger) + + @staticmethod + def explain_expected_url(logger): + view_example = "/views//." + message = "{} [{}]".format( + _("export.errors.requires_workbook_view_param").format(__class__.__name__), view_example + ) + Errors.exit_with_error(logger, message) @staticmethod def get_file_type_from_filename(logger, file_name, url): - type_of_file = None file_name = file_name or url logger.debug(_("get.options.file") + ": {}".format(file_name)) type_of_file = GetUrl.get_file_extension(file_name) @@ -114,24 +117,24 @@ def get_name_without_possible_extension(filename): return filename @staticmethod - def get_workbook_name(logger, url): # /workbooks/wb-name" -> "wb-name" + def get_workbook_name(url, logger): # workbooks/wb-name" -> "wb-name" name_parts = url.split("/") - if len(name_parts) != 3: - raise ValueError(_("export.errors.requires_workbook_view_param").format(GetUrl.name)) + if len(name_parts) != 2: + GetUrl.explain_expected_url(logger) workbook_name = name_parts[::-1][0] # last part workbook_name = GetUrl.strip_query_params(workbook_name) workbook_name = GetUrl.get_name_without_possible_extension(workbook_name) return workbook_name @staticmethod - def get_view_url(url): # "/views/wb-name/view-name" -> wb-name/sheets/view-name - name_parts = url.split("/") # ['', 'views', 'wb-name', 'view-name'] - if len(name_parts) != 4: - raise ValueError(_("export.errors.requires_workbook_view_param").format(GetUrl.name)) + def get_view_url(url, logger): # "views/wb-name/view-name" -> wb-name/sheets/view-name + name_parts = url.split("/") # ['views', 'wb-name', 'view-name'] + if len(name_parts) != 3: + GetUrl.explain_expected_url(logger) + workbook_name = name_parts[1] view_name = name_parts[::-1][0] view_name = GetUrl.strip_query_params(view_name) view_name = GetUrl.get_name_without_possible_extension(view_name) - workbook_name = name_parts[2] return DatasourcesAndWorkbooks.get_view_url_from_names(workbook_name, view_name) @staticmethod @@ -141,55 +144,49 @@ def filename_from_args(file_argument, item_name, filetype): return file_argument @staticmethod - def generate_pdf(logger, server, args): - view_url = GetUrl.get_view_url(args.url) + def generate_pdf(logger, server, args, view_url): try: view_item: TSC.ViewItem = GetUrl.get_view_by_content_url(logger, server, view_url) logger.debug(_("content_type.view") + ": {}".format(view_item.name)) req_option_pdf = TSC.PDFRequestOptions(maxage=1) + DatasourcesAndWorkbooks.apply_values_from_url_params(req_option_pdf, args.url, logger) server.views.populate_pdf(view_item, req_option_pdf) filename = GetUrl.filename_from_args(args.filename, view_item.name, "pdf") - with open(filename, "wb") as f: - f.write(view_item.pdf) - logger.info(_("export.success").format(view_item.name, filename)) + DatasourcesAndWorkbooks.save_to_file(logger, view_item.pdf, filename) except TSC.ServerResponseError as e: Errors.exit_with_error(logger, _("publish.errors.unexpected_server_response"), e) @staticmethod - def generate_png(logger, server, args): - view = GetUrl.get_view_url(args.url) + def generate_png(logger, server, args, view_url): try: - view_item: TSC.ViewItem = GetUrl.get_view_by_content_url(logger, server, view) + view_item: TSC.ViewItem = GetUrl.get_view_by_content_url(logger, server, view_url) logger.debug(_("content_type.view") + ": {}".format(view_item.name)) - req_option_csv = TSC.CSVRequestOptions(maxage=1) # same as png + req_option_csv = TSC.ImageRequestOptions(maxage=1) + DatasourcesAndWorkbooks.apply_values_from_url_params(req_option_csv, args.url, logger) server.views.populate_image(view_item, req_option_csv) filename = GetUrl.filename_from_args(args.filename, view_item.name, "png") - with open(filename, "wb") as f: - f.write(view_item.image) - logger.info(_("export.success").format(view_item.name, filename)) + DatasourcesAndWorkbooks.save_to_file(logger, view_item.image, filename) except TSC.ServerResponseError as e: Errors.exit_with_error(logger, _("publish.errors.unexpected_server_response"), e) @staticmethod - def generate_csv(logger, server, args): - view_url = GetUrl.get_view_url(args.url) + def generate_csv(logger, server, args, view_url): try: view_item: TSC.ViewItem = GetUrl.get_view_by_content_url(logger, server, view_url) logger.debug(_("content_type.view") + ": {}".format(view_item.name)) req_option_csv = TSC.CSVRequestOptions(maxage=1) + DatasourcesAndWorkbooks.apply_values_from_url_params(req_option_csv, args.url, logger) server.views.populate_csv(view_item, req_option_csv) file_name_with_path = GetUrl.filename_from_args(args.filename, view_item.name, "csv") - with open(file_name_with_path, "wb") as f: - f.writelines(view_item.csv) - logger.info(_("export.success").format(view_item.name, file_name_with_path)) + DatasourcesAndWorkbooks.save_to_data_file(logger, view_item.csv, file_name_with_path) except TSC.ServerResponseError as e: Errors.exit_with_error(logger, _("publish.errors.unexpected_server_response"), e) except Exception as e: Errors.exit_with_error(logger, exception=e) @staticmethod - def generate_twb(logger, server, args, file_extension): - workbook_name = GetUrl.get_workbook_name(logger, args.url) + def generate_twb(logger, server, args, file_extension, url): + workbook_name = GetUrl.get_workbook_name(url, logger) try: target_workbook = GetUrl.get_wb_by_content_url(logger, server, workbook_name) diff --git a/tabcmd/commands/datasources_and_workbooks/publish_command.py b/tabcmd/commands/datasources_and_workbooks/publish_command.py index ec607755..59432ee1 100644 --- a/tabcmd/commands/datasources_and_workbooks/publish_command.py +++ b/tabcmd/commands/datasources_and_workbooks/publish_command.py @@ -65,6 +65,7 @@ def run_command(args): elif source in ["tds", "tdsx", "hyper"]: new_datasource = TSC.DatasourceItem(project_id, name=args.name) + new_datasource.use_remote_query_agent = args.use_tableau_bridge try: new_datasource = server.datasources.publish(new_datasource, args.filename, publish_mode) except IOError as ioe: diff --git a/tabcmd/commands/server.py b/tabcmd/commands/server.py index cf3700cf..fbec528e 100644 --- a/tabcmd/commands/server.py +++ b/tabcmd/commands/server.py @@ -93,7 +93,8 @@ def get_site_for_command_or_throw(logger, server, site_name): @staticmethod def get_site_by_name(logger, server, site_name): try: - site_item = Server.get_items_by_name(logger, server.sites, site_name)[0] + # sites don't use the normal filter + site_item = server.sites.get_by_name(site_name) except Exception as e: Errors.exit_with_error(logger, exception=e) return site_item diff --git a/tabcmd/commands/user/user_data.py b/tabcmd/commands/user/user_data.py index 8e4caaeb..f8c088f7 100644 --- a/tabcmd/commands/user/user_data.py +++ b/tabcmd/commands/user/user_data.py @@ -273,4 +273,3 @@ def act_on_users( if number_of_errors > max_printing: logger.info(_("importcsvsummary.error.too_many_errors")) logger.info(_("importcsvsummary.remainingerrors")) - diff --git a/tabcmd/execution/global_options.py b/tabcmd/execution/global_options.py index 34672806..a1f76c02 100644 --- a/tabcmd/execution/global_options.py +++ b/tabcmd/execution/global_options.py @@ -134,7 +134,7 @@ def set_encryption_option(parser): parser.add_argument( "--encrypt", dest="encrypt", - action="store_false", + action="store_true", # set to true IF user passes in option --encrypt help="Encrypt the newly created extract.", ) return parser @@ -316,8 +316,12 @@ def set_publish_args(parser): ) parser.add_argument("--oauth-username", help="The email address of a preconfigured OAuth connection") parser.add_argument("--save-oauth", action="store_true", help="Save embedded OAuth credentials in the datasource") - parser.add_argument("--thumbnail-username", help="Not yet implemented") - parser.add_argument("--thumbnail-group", help="Not yet implemented") # not implemented in the REST API + + thumbnails = parser.add_mutually_exclusive_group() + thumbnails.add_argument("--thumbnail-username", help="Not yet implemented") + thumbnails.add_argument("--thumbnail-group", help="Not yet implemented") # not implemented in the REST API + + parser.add_argument("--use-tableau-bridge", help="Refresh datasource through Tableau Bridge") def set_overwrite_option(parser): diff --git a/tabcmd/execution/localize.py b/tabcmd/execution/localize.py index a53e43f1..d6eed6dd 100644 --- a/tabcmd/execution/localize.py +++ b/tabcmd/execution/localize.py @@ -3,6 +3,7 @@ import logging import os import sys +from os import listdir from typing import Any from typing import Callable @@ -20,7 +21,7 @@ def _(string_key: str) -> str: def _identity_func(x: Any) -> Any: - return x + return "++" + x + "++" # The client should present text in the OS language, or english if not present. @@ -41,7 +42,7 @@ def set_client_locale(lang: str = None, logger=None) -> Callable: for lang in locale_options: try: if lang: - translate = _load_language(lang, domain) + translate = _load_language(lang, domain, logger) break except Exception as e: print("Failed to load language '", lang, "':", e) @@ -49,23 +50,32 @@ def set_client_locale(lang: str = None, logger=None) -> Callable: return translate or _identity_func -# Handling file locations in unbundled (e.g dev) layout and when bundled by pyinstaller +"""Get absolute path to resource, works for unbundled (e.g dev) and when bundled by PyInstaller""" # https://stackoverflow.com/questions/7674790/bundling-data-files-with-pyinstaller-onefile/13790741#13790741 -def resource_path(relative_path): - """Get absolute path to resource, works for dev and for PyInstaller""" - base_path = getattr(sys, "_MEIPASS", os.path.dirname(os.path.abspath(__file__))) - return os.path.join(base_path, relative_path) - - -def _load_language(current_locale, domain): - locale_path = os.path.join("..", "locales") - - # fallback=True means if loading the translated files fails, strings will be returned - # we use the identity function above instead - locale_dir = resource_path(locale_path) - language: gettext.NullTranslations = gettext.translation( - domain, locale_dir, languages=[current_locale], fallback=False - ) +def define_locale_dir(logger): + try: + base_path = getattr(sys, "_MEIPASS") + except AttributeError: # sys._MEIPASS will only exist in bundled pyinstaller exe, + # in unbundled src code we take the location of the current file + # and go 2 dirs up so that the relative path /tabcmd/locales is still correct + base_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..") + relative_path = os.path.join(".", "tabcmd", "locales") + locale_dir = os.path.join(base_path, relative_path) + logger.debug("Checking for language resources at " + locale_dir) + """ to debug pyinstaller file bundling, try something like this example debug line + try: + logger.debug(listdir(sys._MEIPASS)) + except AttributeError as e: + logger.debug(e) + """ + logger.debug(locale_dir) + logger.debug(listdir(locale_dir)) + return locale_dir + + +def _load_language(current_locale, domain, logger): + locale_dir = define_locale_dir(logger) + language: gettext.NullTranslations = gettext.translation(domain, locale_dir, languages=[current_locale]) language.install() # I believe this is the expensive call _ = language.gettext return _ diff --git a/tabcmd/execution/logger_config.py b/tabcmd/execution/logger_config.py index c57de868..8a42e2cb 100644 --- a/tabcmd/execution/logger_config.py +++ b/tabcmd/execution/logger_config.py @@ -5,9 +5,9 @@ FORMATS = { logging.ERROR: "%(asctime)s %(levelname)-5s:(%(name)-10s %(filename)-10s: %(lineno)d): %(message)-30s", - logging.WARN: "%(asctime)s %(levelname)s : (%(name)-10s %(filename)-10s: %(lineno)d): %(message)-30s", - logging.INFO: "%(message)-30s", - logging.DEBUG: "%(asctime)s %(levelname)s : (%(name)-10s %(filename)-10s: %(lineno)d): %(message)-30s", + logging.WARN: "%(asctime)s %(levelname)-5s: (%(name)-10s %(filename)-10s: %(lineno)d): %(message)-30s", + logging.INFO: "%(filename)-10s: %(message)-30s", + logging.DEBUG: "%(asctime)s %(levelname)-5s: (%(name)-10s %(filename)-10s: %(lineno)d): %(message)-30s", } # https://stackoverflow.com/questions/2183233/how-to-add-a-custom-loglevel-to-pythons-logging-facility @@ -41,7 +41,6 @@ def add_trace_level(): FORMATS[trace_level] = FORMATS[logging.ERROR] - def configure_log(name: str, logging_level_input: str): """function for logging statements to console and logfile""" logging_level = getattr(logging, logging_level_input.upper()) diff --git a/tests/commands/test_execution.py b/tests/commands/test_execution.py index f737829c..364ce456 100644 --- a/tests/commands/test_execution.py +++ b/tests/commands/test_execution.py @@ -11,26 +11,25 @@ class ExecutionTests(unittest.TestCase): def test_launch(self): parser = TabcmdController.initialize() TabcmdController.run(parser, ["help"]) - # check exit code = 0? def test_initialize(self): TabcmdController.initialize() @mock.patch("sys.argv", "") - def test_inputs(self): + def test_no_inputs_exits(self): fake_parser = mock.MagicMock(argparse.ArgumentParser) with self.assertRaises(SystemExit): TabcmdController.run(fake_parser, None) - def test_invalid_command(self): + def test_invalid_command_exits(self): parser = TabcmdController.initialize() - # crashes out during parse_args + # alerts and exits during parse_args with self.assertRaises(SystemExit): TabcmdController.run(parser, ["boo", "--language", "fr"]) - def test_launch_languages(self): + def test_launch_languages_succeeds(self): parser = TabcmdController.initialize() - # crashes out from list_sites.cmd + # fails to connect to server... with self.assertRaises(SystemExit): TabcmdController.run(parser, ["listsites", "--language", "fr"]) diff --git a/tests/commands/test_geturl_utils.py b/tests/commands/test_geturl_utils.py index 0bdb4555..fb354141 100644 --- a/tests/commands/test_geturl_utils.py +++ b/tests/commands/test_geturl_utils.py @@ -8,6 +8,15 @@ from tabcmd.commands.datasources_and_workbooks.export_command import * from tabcmd.commands.server import Server +mock_args = argparse.Namespace() +mock_args.pagelayout = None +mock_args.pagesize = None +mock_args.image_resolution = None +mock_args.width = None +mock_args.height = None +mock_args.filename = None +mock_args.filter = None + mock_logger = mock.MagicMock() fake_item = mock.MagicMock(TSC.ViewItem) @@ -51,10 +60,13 @@ def test_get_view_without_extension_that_doesnt_have_one(self): assert GetUrl.get_name_without_possible_extension(filename) == filename def test_get_workbook_name(self): - assert GetUrl.get_workbook_name(mock_logger, "/workbooks/wbname") == "wbname" + assert GetUrl.get_workbook_name("workbooks/wbname", mock_logger) == "wbname" def test_view_name(self): - assert GetUrl.get_view_url("/views/wb-name/view-name") == "wb-name/sheets/view-name" + assert GetUrl.get_view_url("views/wb-name/view-name", None) == "wb-name/sheets/view-name" + + def test_view_name_with_url_params(self): + assert GetUrl.get_view_url("views/wb-name/view-name?:refresh=y", None) == "wb-name/sheets/view-name" """ GetUrl.get_view_without_extension(view_name) @@ -93,22 +105,15 @@ def test_parse_export_url_to_workbook_and_view_bad_url(self): assert view is None assert wb is None - def test_extract_query_params(self): - url = "wb-name/view-name?param1=value1" - options = TSC.PDFRequestOptions() - assert options.view_filters is not None - assert len(options.view_filters) is 0 - ExportCommand.extract_filter_values_from_url_params(options, url) - assert len(options.view_filters) == 1 - assert options.view_filters[0] == ("param1", "value1") - @mock.patch("tableauserverclient.Server") def test_download_csv(self, mock_server): mock_server.views = mock.MagicMock() mock_server.views.csv = mock.MagicMock() mock_view = tableauserverclient.ViewItem() url = "wb-name/view-name?param1=value1" - ExportCommand.download_csv(mock_server, mock_view, url, mock_logger) + mock_args.url = url + ExportCommand.download_csv(mock_server, mock_view, mock_args, mock_logger) + @mock.patch("tableauserverclient.Server") def test_download_image(self, mock_server): @@ -116,7 +121,8 @@ def test_download_image(self, mock_server): mock_server.views.png = mock.MagicMock() mock_view = tableauserverclient.ViewItem() url = "wb-name/view-name?param1=value1" - ExportCommand.download_png(mock_server, mock_view, url, mock_logger) + mock_args.url = url + ExportCommand.download_png(mock_server, mock_view, mock_args, mock_logger) @mock.patch("tableauserverclient.Server") def test_download_view_pdf(self, mock_server): @@ -124,7 +130,8 @@ def test_download_view_pdf(self, mock_server): mock_server.views.pdf = mock.MagicMock() mock_view = tableauserverclient.ViewItem() url = "wb-name/view-name?param1=value1" - ExportCommand.download_view_pdf(mock_server, mock_view, url, mock_logger) + mock_args.url = url + ExportCommand.download_view_pdf(mock_server, mock_view, mock_args, mock_logger) @mock.patch("tableauserverclient.Server") def test_download_wb_pdf(self, mock_server): @@ -132,7 +139,51 @@ def test_download_wb_pdf(self, mock_server): mock_server.workbooks.pdf = mock.MagicMock() mock_view = tableauserverclient.ViewItem() url = "wb-name/view-name?param1=value1" - ExportCommand.download_wb_pdf(mock_server, mock_view, url, mock_logger) + mock_args.url = url + ExportCommand.download_wb_pdf(mock_server, mock_view, mock_args, mock_logger) + + +@mock.patch("tableauserverclient.ViewItem", fake_item) +class DS_WB_Tests(unittest.TestCase): + def test_apply_filter(self): + url = "wb-name/view-name?param1=value1" + options = TSC.PDFRequestOptions() + assert options.view_filters is not None + assert len(options.view_filters) is 0 + ExportCommand.apply_filter_value(options, "param1=value1", mock_logger) + assert len(options.view_filters) == 1 + assert options.view_filters[0] == ("param1", "value1") + + def test_extract_query_params(self): + url = "wb-name/view-name?param1=value1" + options = TSC.PDFRequestOptions() + assert options.view_filters is not None + assert len(options.view_filters) is 0 + ExportCommand.apply_values_from_url_params(options, url, mock_logger) + assert len(options.view_filters) == 1 + assert options.view_filters[0] == ("param1", "value1") + + def test_refresh_true(self): + url = "wb-name/view-name?:refresh=TRUE" + options = TSC.PDFRequestOptions() + assert options.max_age == -1 + ExportCommand.apply_values_from_url_params(options, url, mock_logger) + assert options.max_age == 0 + + def test_refresh_yes(self): + url = "wb-name/view-name?:refresh=yes" + options = TSC.PDFRequestOptions() + assert options.max_age == -1 + ExportCommand.apply_values_from_url_params(options, url, mock_logger) + assert options.max_age == 0 + + def test_refresh_y(self): + url = "wb-name/view-name?:refresh=y" + options = TSC.PDFRequestOptions() + assert options.max_age == -1 + ExportCommand.apply_values_from_url_params(options, url, mock_logger) + assert options.max_age == 0 + def test_save_to_binary_file(self): mock_content = bytes() diff --git a/tests/e2e/online_tests.py b/tests/e2e/online_tests.py index bce557ef..f7da3677 100644 --- a/tests/e2e/online_tests.py +++ b/tests/e2e/online_tests.py @@ -51,7 +51,7 @@ def setup_class(cls): def _create_project(self, project_name, parent_path=None): command = "createproject" arguments = [command, "--name", project_name] - if parent_path or parent_location: + if parent_path: arguments.append("--parent-project-path") arguments.append(parent_path) _test_command(arguments) @@ -59,9 +59,9 @@ def _create_project(self, project_name, parent_path=None): def _delete_project(self, project_name, parent_path=None): command = "deleteproject" arguments = [command, project_name] - if parent_path or parent_location: + if parent_path: arguments.append("--parent-project-path") - arguments.append(parent_path or parent_location) + arguments.append(parent_path) _test_command(arguments) def _publish_samples(self, project_name): diff --git a/tests/e2e/setup_e2e.py b/tests/e2e/setup_e2e.py index ddbe1573..4edecf7c 100644 --- a/tests/e2e/setup_e2e.py +++ b/tests/e2e/setup_e2e.py @@ -1,9 +1,8 @@ import subprocess import os -import pytest try: - from tests.e2e import credentials + from tests.e2e import credentials # type: ignore except ImportError: credentials = None # type: ignore diff --git a/tests/e2e/tests_integration.py b/tests/e2e/tests_integration.py index d41070a1..e18c38b6 100644 --- a/tests/e2e/tests_integration.py +++ b/tests/e2e/tests_integration.py @@ -8,7 +8,7 @@ try: - from tests.e2e import credentials + from tests.e2e import credentials # type: ignore except ImportError: credentials = None # type: ignore