From 7d78e0da9695628d8751b755de69ff779f955d68 Mon Sep 17 00:00:00 2001 From: Erwan RIVOAL Date: Mon, 21 Feb 2022 16:58:06 +0100 Subject: [PATCH 1/6] [sc-80919] new PAPI - dataset get full info --- dataikuapi/dss/dataset.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dataikuapi/dss/dataset.py b/dataikuapi/dss/dataset.py index bf8500196..3a3599c78 100644 --- a/dataikuapi/dss/dataset.py +++ b/dataikuapi/dss/dataset.py @@ -511,7 +511,6 @@ def get_last_metric_values(self, partition=''): return ComputedMetrics(self.client._perform_json( "GET", "/projects/%s/datasets/%s/metrics/last/%s" % (self.project_key, self.dataset_name, 'NP' if len(partition) == 0 else partition))) - def get_metric_history(self, metric, partition=''): """ Get the history of the values of the metric on this dataset @@ -523,6 +522,17 @@ def get_metric_history(self, metric, partition=''): "GET", "/projects/%s/datasets/%s/metrics/history/%s" % (self.project_key, self.dataset_name, 'NP' if len(partition) == 0 else partition), params={'metricLookup' : metric if isinstance(metric, str) or isinstance(metric, unicode) else json.dumps(metric)}) + def get_full_info(self): + """ + Retrieve all the information about a dataset + + Returns: + a complex JSON object containing all the information on a dataset, such as params, schema, lastbuild infos, status, etc. + """ + return self.client._perform_json( + "GET", "/projects/%s/datasets/%s/getFullInfo" % (self.project_key, self.dataset_name) + ) + ######################################################## # Misc ######################################################## @@ -542,7 +552,7 @@ def move_to_zone(self, zone): :param object zone: a :class:`dataikuapi.dss.flow.DSSFlowZone` where to move the object """ if isinstance(zone, basestring): - zone = self.project.get_flow().get_zone(zone) + zone = self.project.get_flow().get_zone(zone) zone.add_item(self) def share_to_zone(self, zone): From 5c842547784468907ad251e657819a26d3f2a12b Mon Sep 17 00:00:00 2001 From: Erwan RIVOAL Date: Wed, 23 Feb 2022 11:20:46 +0100 Subject: [PATCH 2/6] endpoint simplified name --- dataikuapi/dss/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/dss/dataset.py b/dataikuapi/dss/dataset.py index 3a3599c78..de084ffdc 100644 --- a/dataikuapi/dss/dataset.py +++ b/dataikuapi/dss/dataset.py @@ -530,7 +530,7 @@ def get_full_info(self): a complex JSON object containing all the information on a dataset, such as params, schema, lastbuild infos, status, etc. """ return self.client._perform_json( - "GET", "/projects/%s/datasets/%s/getFullInfo" % (self.project_key, self.dataset_name) + "GET", "/projects/%s/datasets/%s/info" % (self.project_key, self.dataset_name) ) ######################################################## From ca3dcea370a816b5897bd6f5a35fe9664ed1a983 Mon Sep 17 00:00:00 2001 From: Erwan RIVOAL Date: Wed, 23 Feb 2022 16:29:11 +0100 Subject: [PATCH 3/6] docstring fix --- dataikuapi/dss/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataikuapi/dss/dataset.py b/dataikuapi/dss/dataset.py index de084ffdc..6c25a5afb 100644 --- a/dataikuapi/dss/dataset.py +++ b/dataikuapi/dss/dataset.py @@ -527,7 +527,7 @@ def get_full_info(self): Retrieve all the information about a dataset Returns: - a complex JSON object containing all the information on a dataset, such as params, schema, lastbuild infos, status, etc. + a complex JSON object containing all the information about a dataset, such as params, schema, last build infos, status, etc. """ return self.client._perform_json( "GET", "/projects/%s/datasets/%s/info" % (self.project_key, self.dataset_name) From 241029d1faef4efc81ce4ddee56f523698b55dfd Mon Sep 17 00:00:00 2001 From: Erwan RIVOAL Date: Mon, 14 Mar 2022 12:52:18 +0100 Subject: [PATCH 4/6] [sc-80919] new DSSDatasetInfo class --- dataikuapi/dss/dataset.py | 88 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 6 deletions(-) diff --git a/dataikuapi/dss/dataset.py b/dataikuapi/dss/dataset.py index 6c25a5afb..0d45f7f25 100644 --- a/dataikuapi/dss/dataset.py +++ b/dataikuapi/dss/dataset.py @@ -1,3 +1,5 @@ +import datetime + from ..utils import DataikuException from ..utils import DataikuUTF8CSVReader from ..utils import DataikuStreamedHttpUTF8CSVReader @@ -522,16 +524,15 @@ def get_metric_history(self, metric, partition=''): "GET", "/projects/%s/datasets/%s/metrics/history/%s" % (self.project_key, self.dataset_name, 'NP' if len(partition) == 0 else partition), params={'metricLookup' : metric if isinstance(metric, str) or isinstance(metric, unicode) else json.dumps(metric)}) - def get_full_info(self): + def get_info(self): """ Retrieve all the information about a dataset - Returns: - a complex JSON object containing all the information about a dataset, such as params, schema, last build infos, status, etc. + :returns: a :class:`DSSDatasetInfo` containing all the information about a dataset, such as params, schema, last build infos, status, etc. + :rtype: :class:`DSSDatasetInfo` """ - return self.client._perform_json( - "GET", "/projects/%s/datasets/%s/info" % (self.project_key, self.dataset_name) - ) + data = self.client._perform_json("GET", "/projects/%s/datasets/%s/info" % (self.project_key, self.dataset_name)) + return DSSDatasetInfo(self, data) ######################################################## # Misc @@ -869,3 +870,78 @@ def already_exists(self): return True except Exception as e: return False + + +class DSSDatasetInfo(object): + """ + Info class for a DSS dataset (Read-Only). + Do not instantiate this class directly, use :meth:`DSSDataset.get_info` + """ + + def __init__(self, dataset, info): + """ + :type dataset: :class:`DSSDataset` + :type info: dict + """ + self.dataset = dataset + self.info = info + + def get_raw(self): + """ + Get the raw dataset full information as a dict + + :return: the raw dataset full information + :rtype: dict + """ + return self.info + + def get_raw_last_build(self): + """ + Get the last build information of the dataset, as a raw dict, or None if there is no last build information. + + :return: the last build information + :rtype: dict or None + """ + return self.info.get("lastBuild", None) + + def get_last_build_start_time(self, as_date=False): + """ + Get the last build start time of the dataset as a timestamp or as a :class:`datetime.datetime`, or None if there + is no last build information. + + :return: the last build start time + :rtype: int or :class:`datetime.datetime` or None + """ + last_build_info = self.info.get("lastBuild", dict()) + timestamp = last_build_info.get("buildStartTime", None) + + if as_date and timestamp is not None: + return datetime.datetime.fromtimestamp(timestamp / 1000) + return timestamp + + def get_last_build_end_time(self, as_date=False): + """ + Get the last build end time of the dataset as a timestamp or as a :class:`datetime.datetime`, or None if there + is no last build information. + + :return: the last build end time + :rtype: int or :class:`datetime.datetime` or None + """ + last_build_info = self.info.get("lastBuild", dict()) + timestamp = last_build_info.get("buildEndTime", None) + + if as_date and timestamp is not None: + return datetime.datetime.fromtimestamp(timestamp / 1000) + return timestamp + + def is_last_build_successful(self): + """ + Get whether the last build of the dataset is successful. + + :return: True if the last build is successful, False otherwise + :rtype: bool + """ + last_build_info = self.info.get("lastBuild", dict()) + success = last_build_info.get("buildSuccess", False) + + return success From d06ac0372de8a037b3731d5b13eb983a2b62f317 Mon Sep 17 00:00:00 2001 From: Erwan RIVOAL Date: Wed, 16 Mar 2022 09:25:20 +0100 Subject: [PATCH 5/6] comments --- dataikuapi/dss/dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataikuapi/dss/dataset.py b/dataikuapi/dss/dataset.py index 0d45f7f25..26b310a8e 100644 --- a/dataikuapi/dss/dataset.py +++ b/dataikuapi/dss/dataset.py @@ -930,7 +930,7 @@ def get_last_build_end_time(self, as_date=False): last_build_info = self.info.get("lastBuild", dict()) timestamp = last_build_info.get("buildEndTime", None) - if as_date and timestamp is not None: + if as_date and timestamp: return datetime.datetime.fromtimestamp(timestamp / 1000) return timestamp @@ -938,7 +938,7 @@ def is_last_build_successful(self): """ Get whether the last build of the dataset is successful. - :return: True if the last build is successful, False otherwise + :return: True if the last build is successful :rtype: bool """ last_build_info = self.info.get("lastBuild", dict()) From bc580bdad2a2f5f61bbc0fdaf4d6527d01dbc5e9 Mon Sep 17 00:00:00 2001 From: Erwan RIVOAL Date: Fri, 22 Apr 2022 17:15:57 +0200 Subject: [PATCH 6/6] methods as properties and only dates --- dataikuapi/dss/dataset.py | 43 +++++++++++---------------------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/dataikuapi/dss/dataset.py b/dataikuapi/dss/dataset.py index 26b310a8e..ca20bc325 100644 --- a/dataikuapi/dss/dataset.py +++ b/dataikuapi/dss/dataset.py @@ -528,7 +528,7 @@ def get_info(self): """ Retrieve all the information about a dataset - :returns: a :class:`DSSDatasetInfo` containing all the information about a dataset, such as params, schema, last build infos, status, etc. + :returns: a :class:`DSSDatasetInfo` containing all the information about a dataset. :rtype: :class:`DSSDatasetInfo` """ data = self.client._perform_json("GET", "/projects/%s/datasets/%s/info" % (self.project_key, self.dataset_name)) @@ -879,10 +879,6 @@ class DSSDatasetInfo(object): """ def __init__(self, dataset, info): - """ - :type dataset: :class:`DSSDataset` - :type info: dict - """ self.dataset = dataset self.info = info @@ -895,45 +891,31 @@ def get_raw(self): """ return self.info - def get_raw_last_build(self): - """ - Get the last build information of the dataset, as a raw dict, or None if there is no last build information. - - :return: the last build information - :rtype: dict or None - """ - return self.info.get("lastBuild", None) - - def get_last_build_start_time(self, as_date=False): + @property + def last_build_start_time(self): """ - Get the last build start time of the dataset as a timestamp or as a :class:`datetime.datetime`, or None if there - is no last build information. + The last build start time of the dataset as a :class:`datetime.datetime` or None if there is no last build information. :return: the last build start time - :rtype: int or :class:`datetime.datetime` or None + :rtype: :class:`datetime.datetime` or None """ last_build_info = self.info.get("lastBuild", dict()) timestamp = last_build_info.get("buildStartTime", None) + return datetime.datetime.fromtimestamp(timestamp / 1000) if timestamp is not None else None - if as_date and timestamp is not None: - return datetime.datetime.fromtimestamp(timestamp / 1000) - return timestamp - - def get_last_build_end_time(self, as_date=False): + @property + def last_build_end_time(self): """ - Get the last build end time of the dataset as a timestamp or as a :class:`datetime.datetime`, or None if there - is no last build information. + The last build end time of the dataset as a :class:`datetime.datetime` or None if there is no last build information. :return: the last build end time - :rtype: int or :class:`datetime.datetime` or None + :rtype: :class:`datetime.datetime` or None """ last_build_info = self.info.get("lastBuild", dict()) timestamp = last_build_info.get("buildEndTime", None) + return datetime.datetime.fromtimestamp(timestamp / 1000) if timestamp is not None else None - if as_date and timestamp: - return datetime.datetime.fromtimestamp(timestamp / 1000) - return timestamp - + @property def is_last_build_successful(self): """ Get whether the last build of the dataset is successful. @@ -943,5 +925,4 @@ def is_last_build_successful(self): """ last_build_info = self.info.get("lastBuild", dict()) success = last_build_info.get("buildSuccess", False) - return success