From a58e94fbb1a5d771fa3fe814fd1cd38d500d8b38 Mon Sep 17 00:00:00 2001 From: nimasarajpoor Date: Sat, 6 May 2023 14:46:47 -0400 Subject: [PATCH 01/12] add test function to expose loss of precision in self-match distance --- tests/test_precision.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/test_precision.py b/tests/test_precision.py index 30a90aebc..d55d6583e 100644 --- a/tests/test_precision.py +++ b/tests/test_precision.py @@ -3,7 +3,7 @@ import numpy.testing as npt import stumpy -from stumpy import config +from stumpy import config, core def test_mpdist_snippets_s(): @@ -38,3 +38,20 @@ def test_mpdist_snippets_s(): npt.assert_almost_equal( ref_fractions, cmp_fractions, decimal=config.STUMPY_TEST_PRECISION ) + + +def test_distace_profile(): + # This test function raises an error when the distance profile between + # the query `Q = T[i: i+m]` and `T` becomes non-zero at index `i`. + T = np.random.rand(64) + m = 3 + T, M_T, Σ_T, T_subseq_isconstant = core.preprocess(T, m) + + for i in range(len(T) - m + 1): + Q = T[i : i + m] + D_ref = naive.distance_profile(Q, T, m) + D_comp = core.mass( + Q, T, M_T=M_T, Σ_T=Σ_T, T_subseq_isconstant=T_subseq_isconstant + ) + + npt.assert_almost_equal(D_ref, D_comp) From 343c7a0c9119dba5331a7315cf21b64d1ff8efa6 Mon Sep 17 00:00:00 2001 From: nimasarajpoor Date: Sat, 6 May 2023 15:44:33 -0400 Subject: [PATCH 02/12] add new param to force set self distance to 0.0 --- stumpy/core.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/stumpy/core.py b/stumpy/core.py index 714a4f403..f758be202 100644 --- a/stumpy/core.py +++ b/stumpy/core.py @@ -1460,6 +1460,7 @@ def mass( T_subseq_isfinite=None, T_subseq_isconstant=None, Q_subseq_isconstant=None, + query_idx=None, ): """ Compute the distance profile using the MASS algorithm @@ -1514,6 +1515,13 @@ def mass( subsequence with at least one np.nan/np.inf will automatically have its corresponding value set to False in this boolean array. + query_idx : int, default None + This is the index position along the time series, `T`, where the query + subsequence, `Q`, is located. `query_idx` should be set to None if `Q` + is not a subsequence of `T`. If `Q` is a subsequence of `T`, provding + this argument is optional. If provided, the precision of computation + can be slightly improved. + Returns ------- distance_profile : numpy.ndarray @@ -1605,6 +1613,11 @@ def mass( T_subseq_isconstant, ) + if query_idx is not None: + query_idx = int(query_idx) + if np.isfinite(M_T[query_idx]) and np.isfinite(μ_Q[0]): + distance_profile[query_idx] = 0 + return distance_profile From 5d550516866518abba734132f291d584deac00d0 Mon Sep 17 00:00:00 2001 From: nimasarajpoor Date: Sat, 6 May 2023 15:45:21 -0400 Subject: [PATCH 03/12] fix loss of precision by passing the query_idx to core.mass func --- tests/test_precision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_precision.py b/tests/test_precision.py index d55d6583e..c819c7e22 100644 --- a/tests/test_precision.py +++ b/tests/test_precision.py @@ -51,7 +51,7 @@ def test_distace_profile(): Q = T[i : i + m] D_ref = naive.distance_profile(Q, T, m) D_comp = core.mass( - Q, T, M_T=M_T, Σ_T=Σ_T, T_subseq_isconstant=T_subseq_isconstant + Q, T, M_T=M_T, Σ_T=Σ_T, T_subseq_isconstant=T_subseq_isconstant, query_idx=i ) npt.assert_almost_equal(D_ref, D_comp) From be005093334e935503697550295661e4c045c418 Mon Sep 17 00:00:00 2001 From: nimasarajpoor Date: Sat, 6 May 2023 16:00:43 -0400 Subject: [PATCH 04/12] add new paramto non-normalized function for consistency --- stumpy/core.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/stumpy/core.py b/stumpy/core.py index f758be202..fe6b5796f 100644 --- a/stumpy/core.py +++ b/stumpy/core.py @@ -1196,7 +1196,7 @@ def _mass_absolute(Q, T, p=2.0): ).flatten() -def mass_absolute(Q, T, T_subseq_isfinite=None, p=2.0): +def mass_absolute(Q, T, T_subseq_isfinite=None, p=2.0, query_idx=None): """ Compute the non-normalized distance profile (i.e., without z-normalization) using the "MASS absolute" algorithm. This is a convenience wrapper around the Numba JIT @@ -1217,6 +1217,13 @@ def mass_absolute(Q, T, T_subseq_isfinite=None, p=2.0): p : float, default 2.0 The p-norm to apply for computing the Minkowski distance. + query_idx : int, default None + This is the index position along the time series, `T`, where the query + subsequence, `Q`, is located. `query_idx` should be set to None if `Q` + is not a subsequence of `T`. If `Q` is a subsequence of `T`, provding + this argument is optional. If provided, the precision of computation + can be slightly improved. + Returns ------- output : numpy.ndarray @@ -1261,6 +1268,10 @@ def mass_absolute(Q, T, T_subseq_isfinite=None, p=2.0): if T_subseq_isfinite is None: T, T_subseq_isfinite = preprocess_non_normalized(T, m) distance_profile[:] = _mass_absolute(Q, T, p) + if query_idx is not None: + query_idx = int(query_idx) + distance_profile[query_idx] = 0.0 + distance_profile[~T_subseq_isfinite] = np.inf return distance_profile From 8f6f0d810f858e7ae970e9620950cfb69c27ceb0 Mon Sep 17 00:00:00 2001 From: nimasarajpoor Date: Sat, 6 May 2023 16:27:25 -0400 Subject: [PATCH 05/12] fix missing coverage --- stumpy/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stumpy/core.py b/stumpy/core.py index fe6b5796f..2f65300cf 100644 --- a/stumpy/core.py +++ b/stumpy/core.py @@ -1268,7 +1268,7 @@ def mass_absolute(Q, T, T_subseq_isfinite=None, p=2.0, query_idx=None): if T_subseq_isfinite is None: T, T_subseq_isfinite = preprocess_non_normalized(T, m) distance_profile[:] = _mass_absolute(Q, T, p) - if query_idx is not None: + if query_idx is not None: # pragma: no cover query_idx = int(query_idx) distance_profile[query_idx] = 0.0 From 78b87d71e560d5f289b8d781e88216dea882a82a Mon Sep 17 00:00:00 2001 From: nimasarajpoor Date: Sat, 6 May 2023 20:24:59 -0400 Subject: [PATCH 06/12] add warning for param query_idx --- stumpy/core.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/stumpy/core.py b/stumpy/core.py index 2f65300cf..f3748bc4a 100644 --- a/stumpy/core.py +++ b/stumpy/core.py @@ -1265,11 +1265,20 @@ def mass_absolute(Q, T, T_subseq_isfinite=None, p=2.0, query_idx=None): if np.any(~np.isfinite(Q)): distance_profile[:] = np.inf else: + if query_idx is not None: + query_idx = int(query_idx) + if not np.allclose(Q, T[query_idx : query_idx + m]): + msg = ( + "Subsequences `Q` and `T[query_idx:query_idx+m]` are " + + "different but were expected to be identical. Please " + + "verify that `query_idx` is correct." + ) + warnings.warn(msg) + if T_subseq_isfinite is None: T, T_subseq_isfinite = preprocess_non_normalized(T, m) distance_profile[:] = _mass_absolute(Q, T, p) if query_idx is not None: # pragma: no cover - query_idx = int(query_idx) distance_profile[query_idx] = 0.0 distance_profile[~T_subseq_isfinite] = np.inf @@ -1595,6 +1604,16 @@ def mass( if np.any(~np.isfinite(Q)): distance_profile[:] = np.inf else: + if query_idx is not None: + query_idx = int(query_idx) + if not np.allclose(Q, T[query_idx : query_idx + m]): + msg = ( + "Subsequences `Q` and `T[query_idx:query_idx+m]` are " + + "different but were expected to be identical. Please " + + "verify that `query_idx` is correct." + ) + warnings.warn(msg) + T, M_T, Σ_T, T_subseq_isconstant = preprocess( T, m, @@ -1625,9 +1644,7 @@ def mass( ) if query_idx is not None: - query_idx = int(query_idx) - if np.isfinite(M_T[query_idx]) and np.isfinite(μ_Q[0]): - distance_profile[query_idx] = 0 + distance_profile[query_idx] = 0 return distance_profile From 132a2149254abda0790ed148b65c50df74fbee0c Mon Sep 17 00:00:00 2001 From: nimasarajpoor Date: Sat, 6 May 2023 20:54:03 -0400 Subject: [PATCH 07/12] fix missing coverage --- stumpy/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stumpy/core.py b/stumpy/core.py index f3748bc4a..d1f026457 100644 --- a/stumpy/core.py +++ b/stumpy/core.py @@ -1265,7 +1265,7 @@ def mass_absolute(Q, T, T_subseq_isfinite=None, p=2.0, query_idx=None): if np.any(~np.isfinite(Q)): distance_profile[:] = np.inf else: - if query_idx is not None: + if query_idx is not None: # pragma: no cover query_idx = int(query_idx) if not np.allclose(Q, T[query_idx : query_idx + m]): msg = ( @@ -1606,7 +1606,7 @@ def mass( else: if query_idx is not None: query_idx = int(query_idx) - if not np.allclose(Q, T[query_idx : query_idx + m]): + if not np.allclose(Q, T[query_idx : query_idx + m]): # pragma: no cover msg = ( "Subsequences `Q` and `T[query_idx:query_idx+m]` are " + "different but were expected to be identical. Please " From 354bf7a11436977ce30b13b1500252bcefba3c28 Mon Sep 17 00:00:00 2001 From: nimasarajpoor Date: Sun, 7 May 2023 00:14:40 -0400 Subject: [PATCH 08/12] minor change --- stumpy/core.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/stumpy/core.py b/stumpy/core.py index d1f026457..bb686f206 100644 --- a/stumpy/core.py +++ b/stumpy/core.py @@ -1245,6 +1245,9 @@ def mass_absolute(Q, T, T_subseq_isfinite=None, p=2.0, query_idx=None): raise ValueError(f"`Q` is {Q.ndim}-dimensional and must be 1-dimensional. ") check_window_size(m, max_size=Q.shape[-1]) + if query_idx is not None: + query_idx = int(query_idx) + QT_query_idx_allclose = np.allclose(Q, T[query_idx : query_idx + m]) T = _preprocess(T) n = T.shape[0] @@ -1265,20 +1268,17 @@ def mass_absolute(Q, T, T_subseq_isfinite=None, p=2.0, query_idx=None): if np.any(~np.isfinite(Q)): distance_profile[:] = np.inf else: + if T_subseq_isfinite is None: + T, T_subseq_isfinite = preprocess_non_normalized(T, m) + distance_profile[:] = _mass_absolute(Q, T, p) if query_idx is not None: # pragma: no cover - query_idx = int(query_idx) - if not np.allclose(Q, T[query_idx : query_idx + m]): + if not QT_query_idx_allclose: msg = ( "Subsequences `Q` and `T[query_idx:query_idx+m]` are " + "different but were expected to be identical. Please " + "verify that `query_idx` is correct." ) warnings.warn(msg) - - if T_subseq_isfinite is None: - T, T_subseq_isfinite = preprocess_non_normalized(T, m) - distance_profile[:] = _mass_absolute(Q, T, p) - if query_idx is not None: # pragma: no cover distance_profile[query_idx] = 0.0 distance_profile[~T_subseq_isfinite] = np.inf @@ -1584,6 +1584,9 @@ def mass( raise ValueError(f"Q is {Q.ndim}-dimensional and must be 1-dimensional. ") check_window_size(m, max_size=Q.shape[-1]) + if query_idx is not None: + query_idx = int(query_idx) + QT_query_idx_allclose = np.allclose(Q, T[query_idx : query_idx + m]) T = _preprocess(T) n = T.shape[0] @@ -1604,16 +1607,6 @@ def mass( if np.any(~np.isfinite(Q)): distance_profile[:] = np.inf else: - if query_idx is not None: - query_idx = int(query_idx) - if not np.allclose(Q, T[query_idx : query_idx + m]): # pragma: no cover - msg = ( - "Subsequences `Q` and `T[query_idx:query_idx+m]` are " - + "different but were expected to be identical. Please " - + "verify that `query_idx` is correct." - ) - warnings.warn(msg) - T, M_T, Σ_T, T_subseq_isconstant = preprocess( T, m, @@ -1644,6 +1637,13 @@ def mass( ) if query_idx is not None: + if not QT_query_idx_allclose: # pragma: no cover + msg = ( + "Subsequences `Q` and `T[query_idx:query_idx+m]` are " + + "different but were expected to be identical. Please " + + "verify that `query_idx` is correct." + ) + warnings.warn(msg) distance_profile[query_idx] = 0 return distance_profile From 7f5c9917960ee6c472ecdafb92df935338fcf5bc Mon Sep 17 00:00:00 2001 From: nimasarajpoor Date: Sun, 7 May 2023 01:37:45 -0400 Subject: [PATCH 09/12] fix missing coverage --- stumpy/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stumpy/core.py b/stumpy/core.py index bb686f206..9184e8edc 100644 --- a/stumpy/core.py +++ b/stumpy/core.py @@ -1245,7 +1245,7 @@ def mass_absolute(Q, T, T_subseq_isfinite=None, p=2.0, query_idx=None): raise ValueError(f"`Q` is {Q.ndim}-dimensional and must be 1-dimensional. ") check_window_size(m, max_size=Q.shape[-1]) - if query_idx is not None: + if query_idx is not None: # pragma: no cover query_idx = int(query_idx) QT_query_idx_allclose = np.allclose(Q, T[query_idx : query_idx + m]) From 1dbdff496ddd711faa69b79fe2237fccf51a192f Mon Sep 17 00:00:00 2001 From: nimasarajpoor Date: Sun, 7 May 2023 20:17:54 -0400 Subject: [PATCH 10/12] move warning to the earlier stage of process --- stumpy/core.py | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/stumpy/core.py b/stumpy/core.py index 9184e8edc..310a270e9 100644 --- a/stumpy/core.py +++ b/stumpy/core.py @@ -1243,11 +1243,21 @@ def mass_absolute(Q, T, T_subseq_isfinite=None, p=2.0, query_idx=None): if Q.ndim != 1: # pragma: no cover raise ValueError(f"`Q` is {Q.ndim}-dimensional and must be 1-dimensional. ") + Q_isfinite = np.isfinite(Q) check_window_size(m, max_size=Q.shape[-1]) if query_idx is not None: # pragma: no cover query_idx = int(query_idx) - QT_query_idx_allclose = np.allclose(Q, T[query_idx : query_idx + m]) + T_isfinite_idx = np.isfinite(T[query_idx : query_idx + m]) + if not np.all(Q_isfinite == T_isfinite_idx) or not np.allclose( + Q[Q_isfinite], T[query_idx : query_idx + m][T_isfinite_idx] + ): + msg = ( + "Subsequences `Q` and `T[query_idx:query_idx+m]` are " + + "different but were expected to be identical. Please " + + "verify that `query_idx` is correct." + ) + warnings.warn(msg) T = _preprocess(T) n = T.shape[0] @@ -1265,20 +1275,13 @@ def mass_absolute(Q, T, T_subseq_isfinite=None, p=2.0, query_idx=None): ) distance_profile = np.empty(n - m + 1, dtype=np.float64) - if np.any(~np.isfinite(Q)): + if np.any(~Q_isfinite): distance_profile[:] = np.inf else: if T_subseq_isfinite is None: T, T_subseq_isfinite = preprocess_non_normalized(T, m) distance_profile[:] = _mass_absolute(Q, T, p) if query_idx is not None: # pragma: no cover - if not QT_query_idx_allclose: - msg = ( - "Subsequences `Q` and `T[query_idx:query_idx+m]` are " - + "different but were expected to be identical. Please " - + "verify that `query_idx` is correct." - ) - warnings.warn(msg) distance_profile[query_idx] = 0.0 distance_profile[~T_subseq_isfinite] = np.inf @@ -1582,11 +1585,22 @@ def mass( if Q.ndim != 1: # pragma: no cover raise ValueError(f"Q is {Q.ndim}-dimensional and must be 1-dimensional. ") + Q_isfinite = np.isfinite(Q) check_window_size(m, max_size=Q.shape[-1]) if query_idx is not None: query_idx = int(query_idx) - QT_query_idx_allclose = np.allclose(Q, T[query_idx : query_idx + m]) + T_isfinite_idx = np.isfinite(T[query_idx : query_idx + m]) + + if not np.all(Q_isfinite == T_isfinite_idx) or not np.allclose( + Q[Q_isfinite], T[query_idx : query_idx + m][T_isfinite_idx] + ): # pragma: no cover + msg = ( + "Subsequences `Q` and `T[query_idx:query_idx+m]` are " + + "different but were expected to be identical. Please " + + "verify that `query_idx` is correct." + ) + warnings.warn(msg) T = _preprocess(T) n = T.shape[0] @@ -1604,7 +1618,7 @@ def mass( ) distance_profile = np.empty(n - m + 1, dtype=np.float64) - if np.any(~np.isfinite(Q)): + if np.any(~Q_isfinite): distance_profile[:] = np.inf else: T, M_T, Σ_T, T_subseq_isconstant = preprocess( @@ -1637,13 +1651,6 @@ def mass( ) if query_idx is not None: - if not QT_query_idx_allclose: # pragma: no cover - msg = ( - "Subsequences `Q` and `T[query_idx:query_idx+m]` are " - + "different but were expected to be identical. Please " - + "verify that `query_idx` is correct." - ) - warnings.warn(msg) distance_profile[query_idx] = 0 return distance_profile From 4648601a4689cab041dded984d7772811b7e05ae Mon Sep 17 00:00:00 2001 From: nimasarajpoor Date: Sun, 7 May 2023 20:56:15 -0400 Subject: [PATCH 11/12] revise description of new param in docstrings --- stumpy/core.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/stumpy/core.py b/stumpy/core.py index 310a270e9..651b1253c 100644 --- a/stumpy/core.py +++ b/stumpy/core.py @@ -1221,8 +1221,8 @@ def mass_absolute(Q, T, T_subseq_isfinite=None, p=2.0, query_idx=None): This is the index position along the time series, `T`, where the query subsequence, `Q`, is located. `query_idx` should be set to None if `Q` is not a subsequence of `T`. If `Q` is a subsequence of `T`, provding - this argument is optional. If provided, the precision of computation - can be slightly improved. + this argument is optional. If query_idx is provided, the distance between + Q and T[query_idx : query_idx + m] will automatically be set to zero. Returns ------- @@ -1542,8 +1542,9 @@ def mass( This is the index position along the time series, `T`, where the query subsequence, `Q`, is located. `query_idx` should be set to None if `Q` is not a subsequence of `T`. If `Q` is a subsequence of `T`, provding - this argument is optional. If provided, the precision of computation - can be slightly improved. + this argument is optional. If query_idx is provided, the distance + between Q and `T[query_idx : query_idx + m]` will automatically be set to + zero. Returns ------- From 1c4b1586292534d781c849ce99d361016244d8e2 Mon Sep 17 00:00:00 2001 From: nimasarajpoor Date: Sun, 7 May 2023 22:05:06 -0400 Subject: [PATCH 12/12] remove/add blank lines to improve readability --- stumpy/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stumpy/core.py b/stumpy/core.py index 651b1253c..4acd6d22a 100644 --- a/stumpy/core.py +++ b/stumpy/core.py @@ -1246,6 +1246,7 @@ def mass_absolute(Q, T, T_subseq_isfinite=None, p=2.0, query_idx=None): Q_isfinite = np.isfinite(Q) check_window_size(m, max_size=Q.shape[-1]) + if query_idx is not None: # pragma: no cover query_idx = int(query_idx) T_isfinite_idx = np.isfinite(T[query_idx : query_idx + m]) @@ -1589,10 +1590,10 @@ def mass( Q_isfinite = np.isfinite(Q) check_window_size(m, max_size=Q.shape[-1]) + if query_idx is not None: query_idx = int(query_idx) T_isfinite_idx = np.isfinite(T[query_idx : query_idx + m]) - if not np.all(Q_isfinite == T_isfinite_idx) or not np.allclose( Q[Q_isfinite], T[query_idx : query_idx + m][T_isfinite_idx] ): # pragma: no cover