From d1598493a2e01c8313f4ca467d504af6898a51d0 Mon Sep 17 00:00:00 2001 From: Luke Bloy Date: Sun, 3 Nov 2019 20:33:33 -0500 Subject: [PATCH 01/11] BUG: anonymize annotations --- mne/channels/channels.py | 4 +++- mne/io/tests/test_meas_info.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mne/channels/channels.py b/mne/channels/channels.py index 63dc74683f8..2827dc56d7a 100644 --- a/mne/channels/channels.py +++ b/mne/channels/channels.py @@ -22,6 +22,7 @@ from ..io.pick import (channel_type, pick_info, pick_types, _picks_by_type, _check_excludes_includes, _contains_ch_type, channel_indices_by_type, pick_channels, _picks_to_idx) +from ..annotations import _handle_meas_date DEPRECATED_PARAM = object() @@ -582,7 +583,8 @@ def anonymize(self, daysback=None, keep_his=False): """ anonymize_info(self.info, daysback=daysback, keep_his=keep_his) if hasattr(self, 'annotations'): - self.annotations.orig_time = self.info['meas_date'] + self.annotations.orig_time = \ + _handle_meas_date(self.info['meas_date']) self.annotations.onset -= self._first_time return self diff --git a/mne/io/tests/test_meas_info.py b/mne/io/tests/test_meas_info.py index 9663d9e662e..b64db0699c7 100644 --- a/mne/io/tests/test_meas_info.py +++ b/mne/io/tests/test_meas_info.py @@ -539,7 +539,8 @@ def test_anonymize(tmpdir): # test that annotations are correctly zeroed raw.anonymize() - assert(raw.annotations.orig_time is raw.info['meas_date']) + assert(raw.annotations.orig_time == (raw.info['meas_date'][0] + + raw.info['meas_date'][1] / 1000000.)) @testing.requires_testing_data From 817516d012725d60bcaee91412e2e0914b5c3b42 Mon Sep 17 00:00:00 2001 From: Luke Bloy Date: Tue, 5 Nov 2019 16:45:30 -0500 Subject: [PATCH 02/11] shift all secs/usecs in block,meas,file _ids\nImprove testing --- mne/io/meas_info.py | 62 ++++++++++++++++++++++------- mne/io/tests/test_meas_info.py | 72 +++++++++++++++++++++++++++++----- 2 files changed, 109 insertions(+), 25 deletions(-) diff --git a/mne/io/meas_info.py b/mne/io/meas_info.py index 1124e19d6fc..ce1e3c5f2a9 100644 --- a/mne/io/meas_info.py +++ b/mne/io/meas_info.py @@ -1866,6 +1866,30 @@ def _force_update_info(info_base, info_target): i_targ[key] = val +def _add_timedelta_to_meas_date(meas_date, dt): + """Add a timedelta to a meas_date tuple. + + Parameters + ---------- + meas_date : tuple | None + The Info object you want to use for overwriting values + in target Info objects. + dt : datetime.timedelta + The time difference that is added to the meas_date timestamp + + Returns + ------- + new_meas_date : tuple | none + The new meas_date tuple. + """ + + if meas_date is None: + new_meas_date = None + else: + new_meas_date = _dt_to_stamp(_stamp_to_dt(meas_date) + dt) + return new_meas_date + + def anonymize_info(info, daysback=None, keep_his=False): """Anonymize measurement information in place. @@ -1917,25 +1941,31 @@ def anonymize_info(info, daysback=None, keep_his=False): default_desc = ("Anonymized using a time shift" " to preserve age at acquisition") - # datetime object representing meas_date - meas_date_datetime = _stamp_to_dt(info['meas_date']) - - if daysback is None: - delta_t = meas_date_datetime - default_anon_dos + # compute timeshift delta + if daysback is None and info['meas_date'] is None: + delta_t = datetime.timedelta(days=np.random.randint(365, 45*365)) + elif daysback is None: + delta_t = _stamp_to_dt(info['meas_date']) - default_anon_dos else: delta_t = datetime.timedelta(days=daysback) # adjust meas_date - info['meas_date'] = _dt_to_stamp(meas_date_datetime - delta_t) + info['meas_date'] = _add_timedelta_to_meas_date(info['meas_date'], + -delta_t) # file_id and meas_id for key in ('file_id', 'meas_id'): value = info.get(key) if value is not None: assert 'msecs' not in value - value['secs'] = info['meas_date'][0] - value['usecs'] = info['meas_date'][1] - value['machid'][:] = 0 + tmp = _add_timedelta_to_meas_date((value['secs'], value['usecs']), + -delta_t) + value['secs'] = tmp[0] + value['usecs'] = tmp[1] + # this is needed for a test CTF dataset + _tmp = value['machid'].copy() + _tmp[:] = 0 + value['machid'] = _tmp # subject info subject_info = info.get('subject_info') @@ -1975,10 +2005,13 @@ def anonymize_info(info, daysback=None, keep_his=False): proc_hist = info.get('proc_history') if proc_hist is not None: for record in proc_hist: - record['block_id']['secs'] = info['meas_date'][0] - record['block_id']['usecs'] = info['meas_date'][1] + this_t0 = (record['block_id']['secs'], record['block_id']['usecs']) + this_t1 = _add_timedelta_to_meas_date(this_t0, -delta_t) + record['block_id']['secs'] = this_t1[0] + record['block_id']['usecs'] = this_t1[1] record['block_id']['machid'][:] = 0 - record['date'] = info['meas_date'] + record['date'] = _add_timedelta_to_meas_date(record['date'], + -delta_t) record['experimenter'] = default_str hi = info.get('helium_info') @@ -1986,15 +2019,14 @@ def anonymize_info(info, daysback=None, keep_his=False): if hi.get('orig_file_guid') is not None: hi['orig_file_guid'] = default_str if hi.get('meas_date') is not None: - hi['meas_date'] = [info['meas_date'][0], - info['meas_date'][1]] + hi['meas_date'] = _add_timedelta_to_meas_date(hi['meas_date'], + -delta_t) di = info.get('device_info') if di is not None: for k in ('serial', 'site'): if di.get(k) is not None: di[k] = default_str - return info diff --git a/mne/io/tests/test_meas_info.py b/mne/io/tests/test_meas_info.py index b64db0699c7..bc8ec74fb0b 100644 --- a/mne/io/tests/test_meas_info.py +++ b/mne/io/tests/test_meas_info.py @@ -23,7 +23,8 @@ from mne.io.meas_info import (Info, create_info, _merge_info, _force_update_info, RAW_INFO_FIELDS, _bad_chans_comp, _get_valid_units, - anonymize_info, _stamp_to_dt, _dt_to_stamp) + anonymize_info, _stamp_to_dt, _dt_to_stamp, + _add_timedelta_to_meas_date) from mne.io._digitization import (_write_dig_points, _read_dig_points, _make_dig_points,) from mne.io import read_raw_ctf @@ -457,7 +458,7 @@ def _test_anonymize_info(base_info): exp_info['description'] = default_desc exp_info['experimenter'] = default_str exp_info['proj_name'] = default_str - exp_info['proj_id'][:] = 0 + exp_info['proj_id'] = np.array([0]) exp_info['subject_info']['first_name'] = default_str exp_info['subject_info']['last_name'] = default_str exp_info['subject_info']['id'] = default_subject_id @@ -467,12 +468,21 @@ def _test_anonymize_info(base_info): # 2010 and 2000. exp_info['subject_info']['birthday'] = (1977, 4, 7) exp_info['meas_date'] = _dt_to_stamp(default_anon_dos) + + # make copies + exp_info_3 = exp_info.copy() + exp_info_4 = exp_info.copy() + + # adjust each expected outcome + dt = timedelta(days=3653) for key in ('file_id', 'meas_id'): value = exp_info.get(key) if value is not None: assert 'msecs' not in value - value['secs'] = exp_info['meas_date'][0] - value['usecs'] = exp_info['meas_date'][1] + tmp = _add_timedelta_to_meas_date((value['secs'], value['usecs']), + -dt) + value['secs'] = tmp[0] + value['usecs'] = tmp[1] value['machid'][:] = 0 # exp 2 tests the keep_his option @@ -480,16 +490,31 @@ def _test_anonymize_info(base_info): exp_info_2['subject_info']['his_id'] = 'foobar' # exp 3 tests is a supplied daysback - dt = timedelta(days=43) - exp_info_3 = exp_info.copy() + dt_1 = timedelta(days=43) exp_info_3['subject_info']['birthday'] = (1987, 2, 24) - exp_info_3['meas_date'] = _dt_to_stamp(meas_date - dt) + exp_info_3['meas_date'] = _dt_to_stamp(meas_date - dt_1) for key in ('file_id', 'meas_id'): value = exp_info_3.get(key) if value is not None: assert 'msecs' not in value - value['secs'] = exp_info_3['meas_date'][0] - value['usecs'] = exp_info_3['meas_date'][1] + tmp = _add_timedelta_to_meas_date((value['secs'], value['usecs']), + -dt_1) + value['secs'] = tmp[0] + value['usecs'] = tmp[1] + value['machid'][:] = 0 + + # exp 4 tests is a supplied daysback + dt_2 = timedelta(days=223+364*500) + exp_info_4['subject_info']['birthday'] = (1488, 5, 10) + exp_info_4['meas_date'] = _dt_to_stamp(meas_date - dt_2) + for key in ('file_id', 'meas_id'): + value = exp_info_4.get(key) + if value is not None: + assert 'msecs' not in value + tmp = _add_timedelta_to_meas_date((value['secs'], value['usecs']), + -dt_2) + value['secs'] = tmp[0] + value['usecs'] = tmp[1] value['machid'][:] = 0 new_info = anonymize_info(base_info.copy()) @@ -498,9 +523,24 @@ def _test_anonymize_info(base_info): new_info = anonymize_info(base_info.copy(), keep_his=True) assert_object_equal(new_info, exp_info_2) - new_info = anonymize_info(base_info.copy(), daysback=dt.days) + new_info = anonymize_info(base_info.copy(), daysback=dt_1.days) assert_object_equal(new_info, exp_info_3) + new_info = anonymize_info(base_info.copy(), daysback=dt_2.days) + assert_object_equal(new_info, exp_info_4) + + # test with a non meas_date + base_info['meas_date'] = None + exp_info_3['meas_date'] = None + new_info = anonymize_info(base_info.copy(), daysback=dt_1.days) + assert_object_equal(new_info, exp_info_3) + + # smoke test not providing days back + # note this picks a random shift + new_info = anonymize_info(base_info.copy()) + assert(new_info['subject_info']['birthday'] != + base_info['subject_info']['birthday']) + def test_meas_date_convert(tmpdir): """Test conversions of meas_date to datetime objects.""" @@ -541,6 +581,18 @@ def test_anonymize(tmpdir): raw.anonymize() assert(raw.annotations.orig_time == (raw.info['meas_date'][0] + raw.info['meas_date'][1] / 1000000.)) + raw.info['meas_date'] = None + raw.anonymize() + assert(raw.annotations.orig_time == 0) + + # smoke test CTF dataset. + raw = read_raw_ctf(ctf_fname) + raw.set_annotations(Annotations(onset=[0, 1], + duration=[1, 1], + description='dummy', + orig_time=None)) + raw.anonymize() + assert(raw.annotations.orig_time == 0) @testing.requires_testing_data From 7e2db16be80a9072232cbec500097bb961845936 Mon Sep 17 00:00:00 2001 From: Luke Bloy Date: Tue, 5 Nov 2019 16:53:21 -0500 Subject: [PATCH 03/11] Fix Pep --- mne/io/meas_info.py | 3 +-- mne/io/tests/test_meas_info.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mne/io/meas_info.py b/mne/io/meas_info.py index ce1e3c5f2a9..35a3ef96a54 100644 --- a/mne/io/meas_info.py +++ b/mne/io/meas_info.py @@ -1882,7 +1882,6 @@ def _add_timedelta_to_meas_date(meas_date, dt): new_meas_date : tuple | none The new meas_date tuple. """ - if meas_date is None: new_meas_date = None else: @@ -1943,7 +1942,7 @@ def anonymize_info(info, daysback=None, keep_his=False): # compute timeshift delta if daysback is None and info['meas_date'] is None: - delta_t = datetime.timedelta(days=np.random.randint(365, 45*365)) + delta_t = datetime.timedelta(days=np.random.randint(365, 45 * 365)) elif daysback is None: delta_t = _stamp_to_dt(info['meas_date']) - default_anon_dos else: diff --git a/mne/io/tests/test_meas_info.py b/mne/io/tests/test_meas_info.py index bc8ec74fb0b..6f321959b1a 100644 --- a/mne/io/tests/test_meas_info.py +++ b/mne/io/tests/test_meas_info.py @@ -504,7 +504,7 @@ def _test_anonymize_info(base_info): value['machid'][:] = 0 # exp 4 tests is a supplied daysback - dt_2 = timedelta(days=223+364*500) + dt_2 = timedelta(days=223 + 364 * 500) exp_info_4['subject_info']['birthday'] = (1488, 5, 10) exp_info_4['meas_date'] = _dt_to_stamp(meas_date - dt_2) for key in ('file_id', 'meas_id'): From 117287b058a97c668441d2e66305df7b99d00d41 Mon Sep 17 00:00:00 2001 From: Luke Bloy Date: Tue, 5 Nov 2019 22:36:15 -0500 Subject: [PATCH 04/11] Test for int32 limits --- mne/io/meas_info.py | 40 +++++++++++++++++++++++++++++----- mne/io/tests/test_meas_info.py | 17 +++------------ 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/mne/io/meas_info.py b/mne/io/meas_info.py index 35a3ef96a54..29e1aaf7990 100644 --- a/mne/io/meas_info.py +++ b/mne/io/meas_info.py @@ -589,12 +589,33 @@ def _check_consistency(self): raise RuntimeError('bad channel(s) %s marked do not exist in info' % (missing,)) meas_date = self.get('meas_date') - if meas_date is not None and ( - not isinstance(self['meas_date'], tuple) or - len(self['meas_date']) != 2): - raise RuntimeError('info["meas_date"] must be a tuple of length ' - '2 or None, got "%r"' - % (repr(self['meas_date']),)) + if meas_date is not None: + if (not isinstance(self['meas_date'], tuple) or + len(self['meas_date']) != 2): + raise RuntimeError('info["meas_date"] must be a tuple ' + 'of length 2 or None, got "%r"' + % (repr(self['meas_date']),)) + if (meas_date[0] < np.iinfo('>i4').min or + meas_date[0] > np.iinfo('>i4').max): + raise RuntimeError('info["meas_date"] must be between "%r" ' + 'and "%r", got "%r"' + % (repr((np.iinfo('>i4').min, 0)), + repr((np.iinfo('>i4').max, 0)), + repr(self['meas_date']),)) + + for key in ('file_id', 'meas_id'): + value = self.get(key) + if value is not None: + assert 'msecs' not in value + for key_2 in ('secs', 'usecs'): + if (value[key_2] < np.iinfo('>i4').min or + value[key_2] > np.iinfo('>i4').max): + raise RuntimeError('info[%s][%s] must be between ' + '"%r" and "%r", got "%r"' + % (key, key_2, + repr(np.iinfo('>i4').min), + repr(np.iinfo('>i4').max), + repr(value[key_2]),)) chs = [ch['ch_name'] for ch in self['chs']] if len(self['ch_names']) != len(chs) or any( @@ -2026,6 +2047,13 @@ def anonymize_info(info, daysback=None, keep_his=False): for k in ('serial', 'site'): if di.get(k) is not None: di[k] = default_str + try: + info._check_consistency() + except RuntimeError as e: + raise RuntimeError('anonymize_info generated an inconsistant info ' + 'object. Most often this is because daysback ' + 'parameter was too large.\nUnderlying Error: "%r"' + % str(e)) return info diff --git a/mne/io/tests/test_meas_info.py b/mne/io/tests/test_meas_info.py index 6f321959b1a..bd9f96af9cd 100644 --- a/mne/io/tests/test_meas_info.py +++ b/mne/io/tests/test_meas_info.py @@ -471,7 +471,6 @@ def _test_anonymize_info(base_info): # make copies exp_info_3 = exp_info.copy() - exp_info_4 = exp_info.copy() # adjust each expected outcome dt = timedelta(days=3653) @@ -505,17 +504,6 @@ def _test_anonymize_info(base_info): # exp 4 tests is a supplied daysback dt_2 = timedelta(days=223 + 364 * 500) - exp_info_4['subject_info']['birthday'] = (1488, 5, 10) - exp_info_4['meas_date'] = _dt_to_stamp(meas_date - dt_2) - for key in ('file_id', 'meas_id'): - value = exp_info_4.get(key) - if value is not None: - assert 'msecs' not in value - tmp = _add_timedelta_to_meas_date((value['secs'], value['usecs']), - -dt_2) - value['secs'] = tmp[0] - value['usecs'] = tmp[1] - value['machid'][:] = 0 new_info = anonymize_info(base_info.copy()) assert_object_equal(new_info, exp_info) @@ -526,8 +514,9 @@ def _test_anonymize_info(base_info): new_info = anonymize_info(base_info.copy(), daysback=dt_1.days) assert_object_equal(new_info, exp_info_3) - new_info = anonymize_info(base_info.copy(), daysback=dt_2.days) - assert_object_equal(new_info, exp_info_4) + with pytest.raises(RuntimeError, match='anonymize_info generated'): + anonymize_info(base_info.copy(), daysback=dt_2.days) + # assert_object_equal(new_info, exp_info_4) # test with a non meas_date base_info['meas_date'] = None From 9bfc2cdeaa972e05447ba823ad0d367eeaa4ae61 Mon Sep 17 00:00:00 2001 From: Luke Bloy Date: Tue, 5 Nov 2019 22:41:26 -0500 Subject: [PATCH 05/11] fix spelling --- mne/io/meas_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne/io/meas_info.py b/mne/io/meas_info.py index 29e1aaf7990..5a83072d62d 100644 --- a/mne/io/meas_info.py +++ b/mne/io/meas_info.py @@ -2050,7 +2050,7 @@ def anonymize_info(info, daysback=None, keep_his=False): try: info._check_consistency() except RuntimeError as e: - raise RuntimeError('anonymize_info generated an inconsistant info ' + raise RuntimeError('anonymize_info generated an inconsistent info ' 'object. Most often this is because daysback ' 'parameter was too large.\nUnderlying Error: "%r"' % str(e)) From 65fad80902039a83e12fe5abdb06724647be6c01 Mon Sep 17 00:00:00 2001 From: Luke Bloy Date: Mon, 11 Nov 2019 09:23:52 -0500 Subject: [PATCH 06/11] Response to Review --- doc/changes/latest.inc | 4 +++- mne/io/meas_info.py | 51 +++++++++++++++++++++--------------------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/doc/changes/latest.inc b/doc/changes/latest.inc index bdc4e448ba3..de18a2a3b7c 100644 --- a/doc/changes/latest.inc +++ b/doc/changes/latest.inc @@ -63,6 +63,8 @@ Changelog Bug ~~~ +- Fix :meth:`mne.io.Raw.anonymize` correctly reset ``raw.annotations.orig_time`` by `Luke Bloy`_. + - Fix date reading before Unix time zero (1970 Jan 1) on Windows by `Alex Rockhill`_. - Fix :meth:`mne.Epochs.shift_time` and :meth:`mne.Evoked.shift_time` to return the modified :class:`~mne.Epochs` or :class:`~mne.Evoked` instance (instead of ``None``) by `Daniel McCloy`_. @@ -75,7 +77,7 @@ Bug - Fix bug in :func:`mne.write_evokeds` where ``evoked.nave`` was not saved properly when multiple :class:`~mne.Evoked` instances were written to a single file, by `Eric Larson`_ -- Fix bug in :func:`mne.preprocessing.mark_flat` where acquisition skips were not handled proeprly, by `Eric Larson`_ +- Fix bug in :func:`mne.preprocessing.mark_flat` where acquisition skips were not handled properly, by `Eric Larson`_ - Fix bug in :func:`mne.viz.plot_bem` where some sources were not plotted by `Jean-Remi King`_ and `Eric Larson`_ diff --git a/mne/io/meas_info.py b/mne/io/meas_info.py index 5a83072d62d..cf83dfd0316 100644 --- a/mne/io/meas_info.py +++ b/mne/io/meas_info.py @@ -582,47 +582,49 @@ def __repr__(self): st %= non_empty return st - def _check_consistency(self): + def _check_consistency(self, prepend_error=''): """Do some self-consistency checks and datatype tweaks.""" missing = [bad for bad in self['bads'] if bad not in self['ch_names']] if len(missing) > 0: - raise RuntimeError('bad channel(s) %s marked do not exist in info' - % (missing,)) + msg = '%sbad channel(s) %s marked do not exist in info' + raise RuntimeError(msg % (prepend_error, missing,)) meas_date = self.get('meas_date') if meas_date is not None: if (not isinstance(self['meas_date'], tuple) or len(self['meas_date']) != 2): - raise RuntimeError('info["meas_date"] must be a tuple ' + raise RuntimeError('%sinfo["meas_date"] must be a tuple ' 'of length 2 or None, got "%r"' - % (repr(self['meas_date']),)) + % (prepend_error, repr(self['meas_date']),)) if (meas_date[0] < np.iinfo('>i4').min or meas_date[0] > np.iinfo('>i4').max): - raise RuntimeError('info["meas_date"] must be between "%r" ' + raise RuntimeError('%sinfo["meas_date"] must be between "%r" ' 'and "%r", got "%r"' - % (repr((np.iinfo('>i4').min, 0)), - repr((np.iinfo('>i4').max, 0)), - repr(self['meas_date']),)) + % (prepend_error, + (np.iinfo('>i4').min, 0), + (np.iinfo('>i4').max, 0), + self['meas_date'],)) for key in ('file_id', 'meas_id'): - value = self.get(key) - if value is not None: + if key in self: + value = self.get(key) assert 'msecs' not in value for key_2 in ('secs', 'usecs'): if (value[key_2] < np.iinfo('>i4').min or value[key_2] > np.iinfo('>i4').max): - raise RuntimeError('info[%s][%s] must be between ' + raise RuntimeError('%sinfo[%s][%s] must be between ' '"%r" and "%r", got "%r"' - % (key, key_2, - repr(np.iinfo('>i4').min), - repr(np.iinfo('>i4').max), - repr(value[key_2]),)) + % (prepend_error, key, key_2, + np.iinfo('>i4').min, + np.iinfo('>i4').max, + value[key_2]),) chs = [ch['ch_name'] for ch in self['chs']] if len(self['ch_names']) != len(chs) or any( ch_1 != ch_2 for ch_1, ch_2 in zip(self['ch_names'], chs)) or \ self['nchan'] != len(chs): - raise RuntimeError('info channel name inconsistency detected, ' - 'please notify mne-python developers') + raise RuntimeError('%sinfo channel name inconsistency detected, ' + 'please notify mne-python developers' + % (prepend_error,)) # make sure we have the proper datatypes for key in ('sfreq', 'highpass', 'lowpass'): @@ -2047,13 +2049,12 @@ def anonymize_info(info, daysback=None, keep_his=False): for k in ('serial', 'site'): if di.get(k) is not None: di[k] = default_str - try: - info._check_consistency() - except RuntimeError as e: - raise RuntimeError('anonymize_info generated an inconsistent info ' - 'object. Most often this is because daysback ' - 'parameter was too large.\nUnderlying Error: "%r"' - % str(e)) + + err_mesg = ('anonymize_info generated an inconsistent info object. Most ' + + 'often this is because daysback parameter was too large.\n' + + 'Underlying Error:') + info._check_consistency(prepend_error=err_mesg) + return info From 46256b6eba62dd402e8588d5824ccc30c62d0450 Mon Sep 17 00:00:00 2001 From: Luke Bloy Date: Mon, 11 Nov 2019 11:13:16 -0500 Subject: [PATCH 07/11] roll back check --- mne/io/meas_info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mne/io/meas_info.py b/mne/io/meas_info.py index cf83dfd0316..7b5af3d750d 100644 --- a/mne/io/meas_info.py +++ b/mne/io/meas_info.py @@ -605,8 +605,8 @@ def _check_consistency(self, prepend_error=''): self['meas_date'],)) for key in ('file_id', 'meas_id'): - if key in self: - value = self.get(key) + value = self.get(key) + if value is not None: assert 'msecs' not in value for key_2 in ('secs', 'usecs'): if (value[key_2] < np.iinfo('>i4').min or From a69c8d535dcc5510b7631678c217a455b19de8cb Mon Sep 17 00:00:00 2001 From: Luke Bloy Date: Tue, 12 Nov 2019 11:16:21 -0500 Subject: [PATCH 08/11] remove all date/time info (no timeshifting) if info['meas_date'] is None --- mne/io/meas_info.py | 62 ++++++++++++++++++++++------------ mne/io/tests/test_meas_info.py | 42 +++++++++++------------ 2 files changed, 60 insertions(+), 44 deletions(-) diff --git a/mne/io/meas_info.py b/mne/io/meas_info.py index 7b5af3d750d..8ecc8d09b94 100644 --- a/mne/io/meas_info.py +++ b/mne/io/meas_info.py @@ -1963,28 +1963,37 @@ def anonymize_info(info, daysback=None, keep_his=False): default_desc = ("Anonymized using a time shift" " to preserve age at acquisition") - # compute timeshift delta - if daysback is None and info['meas_date'] is None: - delta_t = datetime.timedelta(days=np.random.randint(365, 45 * 365)) - elif daysback is None: - delta_t = _stamp_to_dt(info['meas_date']) - default_anon_dos - else: - delta_t = datetime.timedelta(days=daysback) + none_meas_date = info['meas_date'] is None - # adjust meas_date - info['meas_date'] = _add_timedelta_to_meas_date(info['meas_date'], - -delta_t) + if none_meas_date: + logger.warning('Input info has \'meas_date\' set to None.' + + ' Removing all information from time/date structures.' + + ' *NOT* performing any time shifts') + info['meas_date'] = None + else: + # compute timeshift delta + if daysback is None: + delta_t = _stamp_to_dt(info['meas_date']) - default_anon_dos + else: + delta_t = datetime.timedelta(days=daysback) + # adjust meas_date + info['meas_date'] = _add_timedelta_to_meas_date(info['meas_date'], + -delta_t) # file_id and meas_id for key in ('file_id', 'meas_id'): value = info.get(key) if value is not None: assert 'msecs' not in value - tmp = _add_timedelta_to_meas_date((value['secs'], value['usecs']), - -delta_t) + if none_meas_date: + tmp = DATE_NONE + else: + tmp = _add_timedelta_to_meas_date((value['secs'], + value['usecs']), -delta_t) value['secs'] = tmp[0] value['usecs'] = tmp[1] - # this is needed for a test CTF dataset + # The following copy is needed for a test CTF dataset + # otherwise value['machid'][:] = 0 would suffice _tmp = value['machid'].copy() _tmp[:] = 0 value['machid'] = _tmp @@ -2003,7 +2012,10 @@ def anonymize_info(info, daysback=None, keep_his=False): if subject_info.get(key) is not None: subject_info[key] = default_str - if subject_info.get('birthday') is not None: + # anonymize the subject birthday + if none_meas_date: + subject_info.pop('birthday', None) + elif subject_info.get('birthday') is not None: dob = datetime.datetime(subject_info['birthday'][0], subject_info['birthday'][1], subject_info['birthday'][2]) @@ -2027,20 +2039,28 @@ def anonymize_info(info, daysback=None, keep_his=False): proc_hist = info.get('proc_history') if proc_hist is not None: for record in proc_hist: - this_t0 = (record['block_id']['secs'], record['block_id']['usecs']) - this_t1 = _add_timedelta_to_meas_date(this_t0, -delta_t) - record['block_id']['secs'] = this_t1[0] - record['block_id']['usecs'] = this_t1[1] record['block_id']['machid'][:] = 0 - record['date'] = _add_timedelta_to_meas_date(record['date'], - -delta_t) record['experimenter'] = default_str + if none_meas_date: + record['block_id']['secs'] = DATE_NONE[0] + record['block_id']['usecs'] = DATE_NONE[1] + record['date'] = DATE_NONE + else: + this_t0 = (record['block_id']['secs'], + record['block_id']['usecs']) + this_t1 = _add_timedelta_to_meas_date(this_t0, -delta_t) + record['block_id']['secs'] = this_t1[0] + record['block_id']['usecs'] = this_t1[1] + record['date'] = _add_timedelta_to_meas_date(record['date'], + -delta_t) hi = info.get('helium_info') if hi is not None: if hi.get('orig_file_guid') is not None: hi['orig_file_guid'] = default_str - if hi.get('meas_date') is not None: + if none_meas_date and hi.get('meas_date') is not None: + hi['meas_date'] = DATE_NONE + elif hi.get('meas_date') is not None: hi['meas_date'] = _add_timedelta_to_meas_date(hi['meas_date'], -delta_t) diff --git a/mne/io/tests/test_meas_info.py b/mne/io/tests/test_meas_info.py index bd9f96af9cd..d5a6ed83abb 100644 --- a/mne/io/tests/test_meas_info.py +++ b/mne/io/tests/test_meas_info.py @@ -19,7 +19,7 @@ from mne.io import (read_fiducials, write_fiducials, _coil_trans_to_loc, _loc_to_coil_trans, read_raw_fif, read_info, write_info) from mne.io.constants import FIFF -from mne.io.write import _generate_meas_id +from mne.io.write import _generate_meas_id, DATE_NONE from mne.io.meas_info import (Info, create_info, _merge_info, _force_update_info, RAW_INFO_FIELDS, _bad_chans_comp, _get_valid_units, @@ -473,13 +473,13 @@ def _test_anonymize_info(base_info): exp_info_3 = exp_info.copy() # adjust each expected outcome - dt = timedelta(days=3653) + delta_t = timedelta(days=3653) for key in ('file_id', 'meas_id'): value = exp_info.get(key) if value is not None: assert 'msecs' not in value tmp = _add_timedelta_to_meas_date((value['secs'], value['usecs']), - -dt) + -delta_t) value['secs'] = tmp[0] value['usecs'] = tmp[1] value['machid'][:] = 0 @@ -489,21 +489,21 @@ def _test_anonymize_info(base_info): exp_info_2['subject_info']['his_id'] = 'foobar' # exp 3 tests is a supplied daysback - dt_1 = timedelta(days=43) + delta_t_2 = timedelta(days=43) exp_info_3['subject_info']['birthday'] = (1987, 2, 24) - exp_info_3['meas_date'] = _dt_to_stamp(meas_date - dt_1) + exp_info_3['meas_date'] = _dt_to_stamp(meas_date - delta_t_2) for key in ('file_id', 'meas_id'): value = exp_info_3.get(key) if value is not None: assert 'msecs' not in value tmp = _add_timedelta_to_meas_date((value['secs'], value['usecs']), - -dt_1) + -delta_t_2) value['secs'] = tmp[0] value['usecs'] = tmp[1] value['machid'][:] = 0 # exp 4 tests is a supplied daysback - dt_2 = timedelta(days=223 + 364 * 500) + delta_t_3 = timedelta(days=223 + 364 * 500) new_info = anonymize_info(base_info.copy()) assert_object_equal(new_info, exp_info) @@ -511,24 +511,27 @@ def _test_anonymize_info(base_info): new_info = anonymize_info(base_info.copy(), keep_his=True) assert_object_equal(new_info, exp_info_2) - new_info = anonymize_info(base_info.copy(), daysback=dt_1.days) + new_info = anonymize_info(base_info.copy(), daysback=delta_t_2.days) assert_object_equal(new_info, exp_info_3) with pytest.raises(RuntimeError, match='anonymize_info generated'): - anonymize_info(base_info.copy(), daysback=dt_2.days) + anonymize_info(base_info.copy(), daysback=delta_t_3.days) # assert_object_equal(new_info, exp_info_4) - # test with a non meas_date + # test with meas_date = None base_info['meas_date'] = None exp_info_3['meas_date'] = None - new_info = anonymize_info(base_info.copy(), daysback=dt_1.days) + exp_info_3['file_id']['secs'] = DATE_NONE[0] + exp_info_3['file_id']['usecs'] = DATE_NONE[1] + exp_info_3['meas_id']['secs'] = DATE_NONE[0] + exp_info_3['meas_id']['usecs'] = DATE_NONE[1] + exp_info_3['subject_info'].pop('birthday', None) + + new_info = anonymize_info(base_info.copy(), daysback=delta_t_2.days) assert_object_equal(new_info, exp_info_3) - # smoke test not providing days back - # note this picks a random shift new_info = anonymize_info(base_info.copy()) - assert(new_info['subject_info']['birthday'] != - base_info['subject_info']['birthday']) + assert_object_equal(new_info, exp_info_3) def test_meas_date_convert(tmpdir): @@ -574,14 +577,7 @@ def test_anonymize(tmpdir): raw.anonymize() assert(raw.annotations.orig_time == 0) - # smoke test CTF dataset. - raw = read_raw_ctf(ctf_fname) - raw.set_annotations(Annotations(onset=[0, 1], - duration=[1, 1], - description='dummy', - orig_time=None)) - raw.anonymize() - assert(raw.annotations.orig_time == 0) + @testing.requires_testing_data From 57367b5d4be6f3c2dcca27edb632f5c8c433346b Mon Sep 17 00:00:00 2001 From: Luke Bloy Date: Tue, 12 Nov 2019 11:31:52 -0500 Subject: [PATCH 09/11] flake --- mne/io/tests/test_meas_info.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mne/io/tests/test_meas_info.py b/mne/io/tests/test_meas_info.py index d5a6ed83abb..835e32c6197 100644 --- a/mne/io/tests/test_meas_info.py +++ b/mne/io/tests/test_meas_info.py @@ -578,8 +578,6 @@ def test_anonymize(tmpdir): assert(raw.annotations.orig_time == 0) - - @testing.requires_testing_data def test_csr_csc(tmpdir): """Test CSR and CSC.""" From 124144b84c3e7872a4eec5840c0e8eb1ecec9c9d Mon Sep 17 00:00:00 2001 From: Luke Bloy Date: Tue, 12 Nov 2019 12:33:02 -0500 Subject: [PATCH 10/11] use delta_t instead of dt --- mne/io/meas_info.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mne/io/meas_info.py b/mne/io/meas_info.py index 8ecc8d09b94..698745683eb 100644 --- a/mne/io/meas_info.py +++ b/mne/io/meas_info.py @@ -1889,7 +1889,7 @@ def _force_update_info(info_base, info_target): i_targ[key] = val -def _add_timedelta_to_meas_date(meas_date, dt): +def _add_timedelta_to_meas_date(meas_date, delta_t): """Add a timedelta to a meas_date tuple. Parameters @@ -1897,7 +1897,7 @@ def _add_timedelta_to_meas_date(meas_date, dt): meas_date : tuple | None The Info object you want to use for overwriting values in target Info objects. - dt : datetime.timedelta + delta_t : datetime.timedelta The time difference that is added to the meas_date timestamp Returns @@ -1908,7 +1908,7 @@ def _add_timedelta_to_meas_date(meas_date, dt): if meas_date is None: new_meas_date = None else: - new_meas_date = _dt_to_stamp(_stamp_to_dt(meas_date) + dt) + new_meas_date = _dt_to_stamp(_stamp_to_dt(meas_date) + delta_t) return new_meas_date From 3bda9bb8d4e180c68ab7dd44aa4866247b1caf87 Mon Sep 17 00:00:00 2001 From: Luke Bloy Date: Tue, 12 Nov 2019 12:39:09 -0500 Subject: [PATCH 11/11] Apply suggestions from code review remove str additions Co-Authored-By: Alexandre Gramfort --- mne/io/meas_info.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mne/io/meas_info.py b/mne/io/meas_info.py index 698745683eb..ba3e42e02cc 100644 --- a/mne/io/meas_info.py +++ b/mne/io/meas_info.py @@ -1966,8 +1966,8 @@ def anonymize_info(info, daysback=None, keep_his=False): none_meas_date = info['meas_date'] is None if none_meas_date: - logger.warning('Input info has \'meas_date\' set to None.' + - ' Removing all information from time/date structures.' + + logger.warning('Input info has \'meas_date\' set to None.' + ' Removing all information from time/date structures.' ' *NOT* performing any time shifts') info['meas_date'] = None else: @@ -2070,8 +2070,8 @@ def anonymize_info(info, daysback=None, keep_his=False): if di.get(k) is not None: di[k] = default_str - err_mesg = ('anonymize_info generated an inconsistent info object. Most ' + - 'often this is because daysback parameter was too large.\n' + + err_mesg = ('anonymize_info generated an inconsistent info object. Most ' + 'often this is because daysback parameter was too large.\n' 'Underlying Error:') info._check_consistency(prepend_error=err_mesg)