From 033f03fc0489bbb4b62c970a3acc45b43e0e0b6c Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Wed, 24 Aug 2022 16:50:23 -0700 Subject: [PATCH 1/4] make export use filters, add tests --- .../export_command.py | 84 ++++++++++++------- tabcmd/execution/logger_config.py | 2 + tests/commands/test_geturl_utils.py | 71 +++++++++++++++- tests/commands/test_run_commands.py | 9 ++ 4 files changed, 137 insertions(+), 29 deletions(-) diff --git a/tabcmd/commands/datasources_and_workbooks/export_command.py b/tabcmd/commands/datasources_and_workbooks/export_command.py index 72f7375a..5a6452fb 100644 --- a/tabcmd/commands/datasources_and_workbooks/export_command.py +++ b/tabcmd/commands/datasources_and_workbooks/export_command.py @@ -40,23 +40,6 @@ def define_args(export_parser): # TODO: ARGUMENT --COMPLETE - @staticmethod - def get_content_url_for_workbook(url): - # check the size of list - separated_list = url.split("/") - reversed_list = separated_list[::-1] - return reversed_list[1] - - @staticmethod - def get_content_url_for_view(url): - # check the size of list - separated_list = url.split("/") - if len(separated_list) > 2: - print("error") - workbook_name = separated_list[0] - view_name = separated_list[1] - return DatasourcesAndWorkbooks.get_view_url_from_names(workbook_name, view_name) - """ Command to Export a view_name or workbook from Tableau Server and save it to a file. This command can also export just the data used for a view_name @@ -73,24 +56,25 @@ def run_command(args): if not view_content_url and not wb_content_url: Errors.exit_with_error(logger, _("export.errors.requires_workbook_view_param").format(ExportCommand)) - try: + print(args.pagelayout, args.pagesize, args.filename, args.width, args.height, args.filter) + try: if args.fullpdf: # it's a workbook workbook_item = ExportCommand.get_wb_by_content_url(logger, server, wb_content_url) - output = ExportCommand.download_wb_pdf(server, workbook_item) + output = ExportCommand.download_wb_pdf(server, workbook_item, args.url, logger) default_filename = "{}.pdf".format(workbook_item.name) elif args.pdf or args.png or args.csv: # it's a view view_item = ExportCommand.get_view_by_content_url(logger, server, view_content_url) if args.pdf: - output = ExportCommand.download_view_pdf(server, view_item) + output = ExportCommand.download_view_pdf(server, view_item, args.url, logger) default_filename = "{}.pdf".format(view_item.name) elif args.csv: - output = ExportCommand.download_csv(server, view_item) + output = ExportCommand.download_csv(server, view_item, args.url, logger) default_filename = "{}.csv".format(view_item.name) elif args.png: - output = ExportCommand.download_png(server, view_item) + output = ExportCommand.download_png(server, view_item, args.url, logger) default_filename = "{}.png".format(view_item.name) except Exception as e: @@ -98,40 +82,77 @@ def run_command(args): try: save_name = args.filename or default_filename - ExportCommand.save_to_file(logger, output, save_name) + if args.csv: + ExportCommand.save_to_data_file(logger, output, save_name) + else: + ExportCommand.save_to_file(logger, output, save_name) except Exception as e: Errors.exit_with_error(logger, "Error saving to file", e) @staticmethod - def download_wb_pdf(server, workbook_item): + def extract_query_params(request_options: TSC.PDFRequestOptions, url, logger=None): + try: + # todo make logging better + logger = logger or log(ExportCommand.__class__.__name__, "DEBUG") + logger.debug(url) + + if "?" in url: + query = url.split("?")[1] + else: + logger.trace("No params?") + return None + params = query.split("&") + logger.trace(params) + for value in params: + filter = value.split("=") + request_options.vf(filter[0], filter[1]) + except BaseException as e: + logger.error("Error building filter params", e) + ExportCommand.log_stack(logger) # type: ignore + return request_options + + @staticmethod + def download_wb_pdf(server, workbook_item, url, logger): + logger.trace(url) pdf = TSC.PDFRequestOptions(maxage=1) + pdf = ExportCommand.extract_query_params(pdf, url) server.workbooks.populate_pdf(workbook_item, pdf) return workbook_item.pdf @staticmethod - def download_view_pdf(server, view_item): + def download_view_pdf(server, view_item, url, logger): + logger.trace(url) pdf = TSC.PDFRequestOptions(maxage=1) + pdf = ExportCommand.extract_query_params(pdf, url) + logger.trace(pdf.view_filters) server.views.populate_pdf(view_item, pdf) return view_item.pdf @staticmethod - def download_csv(server, view_item): + def download_csv(server, view_item, url, logger): + logger.trace(url) csv = TSC.CSVRequestOptions(maxage=1) + csv = ExportCommand.extract_query_params(csv, url) server.views.populate_csv(view_item, csv) return view_item.csv @staticmethod - def download_png(server, view_item): + def download_png(server, view_item, url, logger): + logger.trace(url) req_option_image = TSC.ImageRequestOptions(maxage=1) + req_option_image = ExportCommand.extract_query_params(req_option_image, url) server.views.populate_image(view_item, req_option_image) return view_item.png @staticmethod def parse_export_url_to_workbook_and_view(logger, url): + filters = "" logger.info(_("export.status").format(url)) if " " in url: Errors.exit_with_error(logger, _("export.errors.white_space_workbook_view")) + if "?" in url: + url = url.split("?")[0] # input should be workbook_name/view_name if not url.find("/"): return None, None @@ -142,9 +163,16 @@ def parse_export_url_to_workbook_and_view(logger, url): view = "{}/sheets/{}".format(workbook, name_parts[1]) return view, workbook + @staticmethod + def save_to_data_file(logger, output, filename): + logger.info(_("httputils.found_attachment").format(filename)) + with open(filename, "wb") as f: + f.writelines(output) + logger.info(_("export.success").format("", filename)) + @staticmethod def save_to_file(logger, output, filename): logger.info(_("httputils.found_attachment").format(filename)) with open(filename, "wb") as f: f.write(output) - logger.info(_("export.success").format(filename, "")) + logger.info(_("export.success").format("", filename)) diff --git a/tabcmd/execution/logger_config.py b/tabcmd/execution/logger_config.py index 1837e951..b478909e 100644 --- a/tabcmd/execution/logger_config.py +++ b/tabcmd/execution/logger_config.py @@ -29,4 +29,6 @@ def configure_log(name: str, logging_level_input: str): def log(file_name, logging_level): logger = configure_log(file_name, logging_level) + if not hasattr(logger, "trace"): + logger.trace = logger.debug return logger diff --git a/tests/commands/test_geturl_utils.py b/tests/commands/test_geturl_utils.py index 2430bf28..9bf0224a 100644 --- a/tests/commands/test_geturl_utils.py +++ b/tests/commands/test_geturl_utils.py @@ -1,11 +1,19 @@ import unittest +from typing import Iterator from unittest import mock + +import tableauserverclient + from tabcmd.commands.datasources_and_workbooks.get_url_command import * from tabcmd.commands.datasources_and_workbooks.export_command import * from tabcmd.commands.server import Server mock_logger = mock.MagicMock() +fake_item = mock.MagicMock(TSC.ViewItem) +fake_item.name = "fake-name" +fake_item.id = "fake-id" + class GeturlTests(unittest.TestCase): def test_evaluate_file_name_pdf(self): @@ -59,9 +67,70 @@ def test_view_name(self): """ +@mock.patch("tableauserverclient.ViewItem", fake_item) class ExportTests(unittest.TestCase): - def test_parse_export_url_to_workbook(self): + + mock_logger = mock.MagicMock("logger") + fake_item.csv = mock.MagicMock("bytes[]") + fake_item.pdf = mock.MagicMock("bytes") + fake_item.png = mock.MagicMock("bytes") + + def test_parse_export_url_to_workbook_and_view(self): wb_url = "wb-name/view-name" view, wb = ExportCommand.parse_export_url_to_workbook_and_view(mock_logger, wb_url) assert view == "wb-name/sheets/view-name" assert wb == "wb-name" + + def test_parse_export_url_to_workbook_and_view_bad_url(self): + wb_url = "wb-name/view-name/kitty" + view, wb = ExportCommand.parse_export_url_to_workbook_and_view(mock_logger, wb_url) + assert view is None + assert wb is None + + def test_extract_query_params(self): + url = "wb-name/view-name?param1=value1" + options = TSC.PDFRequestOptions() + updated_options = ExportCommand.extract_query_params(options, url) + assert updated_options is not None + + @mock.patch("tableauserverclient.Server") + def test_download_csv(self, mock_server): + mock_server.views = mock.MagicMock() + mock_server.views.csv = mock.MagicMock() + mock_view = tableauserverclient.ViewItem() + url = "wb-name/view-name?param1=value1" + ExportCommand.download_csv(mock_server, mock_view, url, mock_logger) + + @mock.patch("tableauserverclient.Server") + def test_download_image(self, mock_server): + mock_server.views = mock.MagicMock() + mock_server.views.png = mock.MagicMock() + mock_view = tableauserverclient.ViewItem() + url = "wb-name/view-name?param1=value1" + ExportCommand.download_png(mock_server, mock_view, url, mock_logger) + + @mock.patch("tableauserverclient.Server") + def test_download_view_pdf(self, mock_server): + mock_server.views = mock.MagicMock() + mock_server.views.pdf = mock.MagicMock() + mock_view = tableauserverclient.ViewItem() + url = "wb-name/view-name?param1=value1" + ExportCommand.download_view_pdf(mock_server, mock_view, url, mock_logger) + + @mock.patch("tableauserverclient.Server") + def test_download_wb_pdf(self, mock_server): + mock_server.workbooks = mock.MagicMock() + mock_server.workbooks.pdf = mock.MagicMock() + mock_view = tableauserverclient.ViewItem() + url = "wb-name/view-name?param1=value1" + ExportCommand.download_wb_pdf(mock_server, mock_view, url, mock_logger) + + def test_save_to_binary_file(self): + mock_content = bytes() + filename = "test_out.pdf" + ExportCommand.save_to_file(mock_logger, mock_content, filename) + + def test_save_to_data_file(self): + mock_content = mock.MagicMock() + filename = "test_out.csv" + ExportCommand.save_to_data_file(mock_logger, mock_content, filename) diff --git a/tests/commands/test_run_commands.py b/tests/commands/test_run_commands.py index 7e7e3d45..22b9dda0 100644 --- a/tests/commands/test_run_commands.py +++ b/tests/commands/test_run_commands.py @@ -103,6 +103,15 @@ def test_export(self, mock_session, mock_server): mock_args.fullpdf = True mock_args.filename = "filename.pdf" mock_args.url = "workbook-name/view-name" + mock_args.csv = None + mock_args.image = None + mock_args.pdf = None + mock_args.pagelayout = None + mock_args.pagesize = None + mock_args.size = None + mock_args.height = None + mock_args.width = None + mock_args.filter = None export_command.ExportCommand.run_command(mock_args) mock_session.assert_called() From 0642be3e3fdcd0aac9a083f4e6179a9c8eaf836f Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Wed, 31 Aug 2022 18:33:05 -0700 Subject: [PATCH 2/4] respond to feedback rename method, remove println, also add smoother handling for unnecessary / character --- .../datasources_and_workbooks/export_command.py | 16 ++++++++-------- tests/commands/test_geturl_utils.py | 13 +++++++++++-- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/tabcmd/commands/datasources_and_workbooks/export_command.py b/tabcmd/commands/datasources_and_workbooks/export_command.py index 5a6452fb..ac9cf1a8 100644 --- a/tabcmd/commands/datasources_and_workbooks/export_command.py +++ b/tabcmd/commands/datasources_and_workbooks/export_command.py @@ -56,7 +56,7 @@ def run_command(args): if not view_content_url and not wb_content_url: Errors.exit_with_error(logger, _("export.errors.requires_workbook_view_param").format(ExportCommand)) - print(args.pagelayout, args.pagesize, args.filename, args.width, args.height, args.filter) + logger.debug(args.pagelayout, args.pagesize, args.filename, args.width, args.height, args.filter) try: if args.fullpdf: # it's a workbook @@ -91,7 +91,7 @@ def run_command(args): Errors.exit_with_error(logger, "Error saving to file", e) @staticmethod - def extract_query_params(request_options: TSC.PDFRequestOptions, url, logger=None): + def extract_filter_values_from_url_params(request_options: TSC.PDFRequestOptions, url, logger=None): try: # todo make logging better logger = logger or log(ExportCommand.__class__.__name__, "DEBUG") @@ -116,7 +116,7 @@ def extract_query_params(request_options: TSC.PDFRequestOptions, url, logger=Non def download_wb_pdf(server, workbook_item, url, logger): logger.trace(url) pdf = TSC.PDFRequestOptions(maxage=1) - pdf = ExportCommand.extract_query_params(pdf, url) + pdf = ExportCommand.extract_filter_values_from_url_params(pdf, url) server.workbooks.populate_pdf(workbook_item, pdf) return workbook_item.pdf @@ -124,7 +124,7 @@ def download_wb_pdf(server, workbook_item, url, logger): def download_view_pdf(server, view_item, url, logger): logger.trace(url) pdf = TSC.PDFRequestOptions(maxage=1) - pdf = ExportCommand.extract_query_params(pdf, url) + ExportCommand.extract_filter_values_from_url_params(pdf, url) logger.trace(pdf.view_filters) server.views.populate_pdf(view_item, pdf) return view_item.pdf @@ -133,7 +133,7 @@ def download_view_pdf(server, view_item, url, logger): def download_csv(server, view_item, url, logger): logger.trace(url) csv = TSC.CSVRequestOptions(maxage=1) - csv = ExportCommand.extract_query_params(csv, url) + ExportCommand.extract_filter_values_from_url_params(csv, url) server.views.populate_csv(view_item, csv) return view_item.csv @@ -141,19 +141,19 @@ def download_csv(server, view_item, url, logger): def download_png(server, view_item, url, logger): logger.trace(url) req_option_image = TSC.ImageRequestOptions(maxage=1) - req_option_image = ExportCommand.extract_query_params(req_option_image, url) + ExportCommand.extract_filter_values_from_url_params(req_option_image, url) server.views.populate_image(view_item, req_option_image) return view_item.png @staticmethod def parse_export_url_to_workbook_and_view(logger, url): - filters = "" logger.info(_("export.status").format(url)) if " " in url: Errors.exit_with_error(logger, _("export.errors.white_space_workbook_view")) if "?" in url: url = url.split("?")[0] - # input should be workbook_name/view_name + # input should be workbook_name/view_name or /workbook_name/view_name + url = url.lstrip("/") # strip opening / if present if not url.find("/"): return None, None name_parts = url.split("/") diff --git a/tests/commands/test_geturl_utils.py b/tests/commands/test_geturl_utils.py index 9bf0224a..3f2a826a 100644 --- a/tests/commands/test_geturl_utils.py +++ b/tests/commands/test_geturl_utils.py @@ -81,6 +81,12 @@ def test_parse_export_url_to_workbook_and_view(self): assert view == "wb-name/sheets/view-name" assert wb == "wb-name" + def test_parse_export_url_to_workbook_and_view(self): + wb_url = "/wb-name/view-name" + view, wb = ExportCommand.parse_export_url_to_workbook_and_view(mock_logger, wb_url) + assert view == "wb-name/sheets/view-name" + assert wb == "wb-name" + def test_parse_export_url_to_workbook_and_view_bad_url(self): wb_url = "wb-name/view-name/kitty" view, wb = ExportCommand.parse_export_url_to_workbook_and_view(mock_logger, wb_url) @@ -90,8 +96,11 @@ def test_parse_export_url_to_workbook_and_view_bad_url(self): def test_extract_query_params(self): url = "wb-name/view-name?param1=value1" options = TSC.PDFRequestOptions() - updated_options = ExportCommand.extract_query_params(options, url) - assert updated_options is not None + assert options.view_filters is not None + assert len(options.view_filters) is 0 + ExportCommand.extract_filter_values_from_url_params(options, url) + assert len(options.view_filters) == 1 + assert options.view_filters[0] == ('param1', 'value1') @mock.patch("tableauserverclient.Server") def test_download_csv(self, mock_server): From bc8cb3c703a8033320ff60e7a069d78a22dac83e Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Wed, 31 Aug 2022 23:09:30 -0700 Subject: [PATCH 3/4] fix duplicated test name --- tests/commands/test_geturl_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/commands/test_geturl_utils.py b/tests/commands/test_geturl_utils.py index 3f2a826a..0bdb4555 100644 --- a/tests/commands/test_geturl_utils.py +++ b/tests/commands/test_geturl_utils.py @@ -81,7 +81,7 @@ def test_parse_export_url_to_workbook_and_view(self): assert view == "wb-name/sheets/view-name" assert wb == "wb-name" - def test_parse_export_url_to_workbook_and_view(self): + def test_parse_export_url_to_workbook_and_view_with_start_slash(self): wb_url = "/wb-name/view-name" view, wb = ExportCommand.parse_export_url_to_workbook_and_view(mock_logger, wb_url) assert view == "wb-name/sheets/view-name" @@ -100,7 +100,7 @@ def test_extract_query_params(self): assert len(options.view_filters) is 0 ExportCommand.extract_filter_values_from_url_params(options, url) assert len(options.view_filters) == 1 - assert options.view_filters[0] == ('param1', 'value1') + assert options.view_filters[0] == ("param1", "value1") @mock.patch("tableauserverclient.Server") def test_download_csv(self, mock_server): From b67f08280e7fb7093505b091bde955eef8d34292 Mon Sep 17 00:00:00 2001 From: Jac Fitzgerald Date: Thu, 1 Sep 2022 11:26:24 -0700 Subject: [PATCH 4/4] actually remove return value --- .../datasources_and_workbooks/export_command.py | 15 ++++++--------- tests/e2e/online_tests.py | 2 +- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/tabcmd/commands/datasources_and_workbooks/export_command.py b/tabcmd/commands/datasources_and_workbooks/export_command.py index ac9cf1a8..11cf2b82 100644 --- a/tabcmd/commands/datasources_and_workbooks/export_command.py +++ b/tabcmd/commands/datasources_and_workbooks/export_command.py @@ -38,8 +38,6 @@ def define_args(export_parser): help="View filter to apply to the view", ) - # TODO: ARGUMENT --COMPLETE - """ Command to Export a view_name or workbook from Tableau Server and save it to a file. This command can also export just the data used for a view_name @@ -91,7 +89,7 @@ def run_command(args): Errors.exit_with_error(logger, "Error saving to file", e) @staticmethod - def extract_filter_values_from_url_params(request_options: TSC.PDFRequestOptions, url, logger=None): + def extract_filter_values_from_url_params(request_options: TSC.PDFRequestOptions, url, logger=None) -> None: try: # todo make logging better logger = logger or log(ExportCommand.__class__.__name__, "DEBUG") @@ -100,23 +98,22 @@ def extract_filter_values_from_url_params(request_options: TSC.PDFRequestOptions if "?" in url: query = url.split("?")[1] else: - logger.trace("No params?") - return None + return + params = query.split("&") logger.trace(params) for value in params: - filter = value.split("=") - request_options.vf(filter[0], filter[1]) + data_filter = value.split("=") + request_options.vf(data_filter[0], data_filter[1]) except BaseException as e: logger.error("Error building filter params", e) ExportCommand.log_stack(logger) # type: ignore - return request_options @staticmethod def download_wb_pdf(server, workbook_item, url, logger): logger.trace(url) pdf = TSC.PDFRequestOptions(maxage=1) - pdf = ExportCommand.extract_filter_values_from_url_params(pdf, url) + ExportCommand.extract_filter_values_from_url_params(pdf, url) server.workbooks.populate_pdf(workbook_item, pdf) return workbook_item.pdf diff --git a/tests/e2e/online_tests.py b/tests/e2e/online_tests.py index cb391e29..73a0e5dc 100644 --- a/tests/e2e/online_tests.py +++ b/tests/e2e/online_tests.py @@ -313,7 +313,7 @@ def test_export_view(self): file = os.path.join("tests", "assets", OnlineCommandTest.TWBX_FILE_WITH_EXTRACT) self._publish_wb(file, name_on_server) command = "export" - friendly_name = name_on_server + "/" + OnlineCommandTest.TWBX_WITH_EXTRACT_SHEET + friendly_name = name_on_server + "/" + OnlineCommandTest.TWBX_WITH_EXTRACT_SHEET + "?param1=3" arguments = [command, friendly_name, "--pdf", "-f", "exported_view.pdf"] _test_command(arguments)