diff --git a/doc/changes/latest.inc b/doc/changes/latest.inc index bdc4e448ba3..de18a2a3b7c 100644 --- a/doc/changes/latest.inc +++ b/doc/changes/latest.inc @@ -63,6 +63,8 @@ Changelog Bug ~~~ +- Fix :meth:`mne.io.Raw.anonymize` correctly reset ``raw.annotations.orig_time`` by `Luke Bloy`_. + - Fix date reading before Unix time zero (1970 Jan 1) on Windows by `Alex Rockhill`_. - Fix :meth:`mne.Epochs.shift_time` and :meth:`mne.Evoked.shift_time` to return the modified :class:`~mne.Epochs` or :class:`~mne.Evoked` instance (instead of ``None``) by `Daniel McCloy`_. @@ -75,7 +77,7 @@ Bug - Fix bug in :func:`mne.write_evokeds` where ``evoked.nave`` was not saved properly when multiple :class:`~mne.Evoked` instances were written to a single file, by `Eric Larson`_ -- Fix bug in :func:`mne.preprocessing.mark_flat` where acquisition skips were not handled proeprly, by `Eric Larson`_ +- Fix bug in :func:`mne.preprocessing.mark_flat` where acquisition skips were not handled properly, by `Eric Larson`_ - Fix bug in :func:`mne.viz.plot_bem` where some sources were not plotted by `Jean-Remi King`_ and `Eric Larson`_ diff --git a/mne/channels/channels.py b/mne/channels/channels.py index 63dc74683f8..2827dc56d7a 100644 --- a/mne/channels/channels.py +++ b/mne/channels/channels.py @@ -22,6 +22,7 @@ from ..io.pick import (channel_type, pick_info, pick_types, _picks_by_type, _check_excludes_includes, _contains_ch_type, channel_indices_by_type, pick_channels, _picks_to_idx) +from ..annotations import _handle_meas_date DEPRECATED_PARAM = object() @@ -582,7 +583,8 @@ def anonymize(self, daysback=None, keep_his=False): """ anonymize_info(self.info, daysback=daysback, keep_his=keep_his) if hasattr(self, 'annotations'): - self.annotations.orig_time = self.info['meas_date'] + self.annotations.orig_time = \ + _handle_meas_date(self.info['meas_date']) self.annotations.onset -= self._first_time return self diff --git a/mne/io/meas_info.py b/mne/io/meas_info.py index 1124e19d6fc..ba3e42e02cc 100644 --- a/mne/io/meas_info.py +++ b/mne/io/meas_info.py @@ -582,26 +582,49 @@ def __repr__(self): st %= non_empty return st - def _check_consistency(self): + def _check_consistency(self, prepend_error=''): """Do some self-consistency checks and datatype tweaks.""" missing = [bad for bad in self['bads'] if bad not in self['ch_names']] if len(missing) > 0: - raise RuntimeError('bad channel(s) %s marked do not exist in info' - % (missing,)) + msg = '%sbad channel(s) %s marked do not exist in info' + raise RuntimeError(msg % (prepend_error, missing,)) meas_date = self.get('meas_date') - if meas_date is not None and ( - not isinstance(self['meas_date'], tuple) or - len(self['meas_date']) != 2): - raise RuntimeError('info["meas_date"] must be a tuple of length ' - '2 or None, got "%r"' - % (repr(self['meas_date']),)) + if meas_date is not None: + if (not isinstance(self['meas_date'], tuple) or + len(self['meas_date']) != 2): + raise RuntimeError('%sinfo["meas_date"] must be a tuple ' + 'of length 2 or None, got "%r"' + % (prepend_error, repr(self['meas_date']),)) + if (meas_date[0] < np.iinfo('>i4').min or + meas_date[0] > np.iinfo('>i4').max): + raise RuntimeError('%sinfo["meas_date"] must be between "%r" ' + 'and "%r", got "%r"' + % (prepend_error, + (np.iinfo('>i4').min, 0), + (np.iinfo('>i4').max, 0), + self['meas_date'],)) + + for key in ('file_id', 'meas_id'): + value = self.get(key) + if value is not None: + assert 'msecs' not in value + for key_2 in ('secs', 'usecs'): + if (value[key_2] < np.iinfo('>i4').min or + value[key_2] > np.iinfo('>i4').max): + raise RuntimeError('%sinfo[%s][%s] must be between ' + '"%r" and "%r", got "%r"' + % (prepend_error, key, key_2, + np.iinfo('>i4').min, + np.iinfo('>i4').max, + value[key_2]),) chs = [ch['ch_name'] for ch in self['chs']] if len(self['ch_names']) != len(chs) or any( ch_1 != ch_2 for ch_1, ch_2 in zip(self['ch_names'], chs)) or \ self['nchan'] != len(chs): - raise RuntimeError('info channel name inconsistency detected, ' - 'please notify mne-python developers') + raise RuntimeError('%sinfo channel name inconsistency detected, ' + 'please notify mne-python developers' + % (prepend_error,)) # make sure we have the proper datatypes for key in ('sfreq', 'highpass', 'lowpass'): @@ -1866,6 +1889,29 @@ def _force_update_info(info_base, info_target): i_targ[key] = val +def _add_timedelta_to_meas_date(meas_date, delta_t): + """Add a timedelta to a meas_date tuple. + + Parameters + ---------- + meas_date : tuple | None + The Info object you want to use for overwriting values + in target Info objects. + delta_t : datetime.timedelta + The time difference that is added to the meas_date timestamp + + Returns + ------- + new_meas_date : tuple | none + The new meas_date tuple. + """ + if meas_date is None: + new_meas_date = None + else: + new_meas_date = _dt_to_stamp(_stamp_to_dt(meas_date) + delta_t) + return new_meas_date + + def anonymize_info(info, daysback=None, keep_his=False): """Anonymize measurement information in place. @@ -1917,25 +1963,40 @@ def anonymize_info(info, daysback=None, keep_his=False): default_desc = ("Anonymized using a time shift" " to preserve age at acquisition") - # datetime object representing meas_date - meas_date_datetime = _stamp_to_dt(info['meas_date']) + none_meas_date = info['meas_date'] is None - if daysback is None: - delta_t = meas_date_datetime - default_anon_dos + if none_meas_date: + logger.warning('Input info has \'meas_date\' set to None.' + ' Removing all information from time/date structures.' + ' *NOT* performing any time shifts') + info['meas_date'] = None else: - delta_t = datetime.timedelta(days=daysback) - - # adjust meas_date - info['meas_date'] = _dt_to_stamp(meas_date_datetime - delta_t) + # compute timeshift delta + if daysback is None: + delta_t = _stamp_to_dt(info['meas_date']) - default_anon_dos + else: + delta_t = datetime.timedelta(days=daysback) + # adjust meas_date + info['meas_date'] = _add_timedelta_to_meas_date(info['meas_date'], + -delta_t) # file_id and meas_id for key in ('file_id', 'meas_id'): value = info.get(key) if value is not None: assert 'msecs' not in value - value['secs'] = info['meas_date'][0] - value['usecs'] = info['meas_date'][1] - value['machid'][:] = 0 + if none_meas_date: + tmp = DATE_NONE + else: + tmp = _add_timedelta_to_meas_date((value['secs'], + value['usecs']), -delta_t) + value['secs'] = tmp[0] + value['usecs'] = tmp[1] + # The following copy is needed for a test CTF dataset + # otherwise value['machid'][:] = 0 would suffice + _tmp = value['machid'].copy() + _tmp[:] = 0 + value['machid'] = _tmp # subject info subject_info = info.get('subject_info') @@ -1951,7 +2012,10 @@ def anonymize_info(info, daysback=None, keep_his=False): if subject_info.get(key) is not None: subject_info[key] = default_str - if subject_info.get('birthday') is not None: + # anonymize the subject birthday + if none_meas_date: + subject_info.pop('birthday', None) + elif subject_info.get('birthday') is not None: dob = datetime.datetime(subject_info['birthday'][0], subject_info['birthday'][1], subject_info['birthday'][2]) @@ -1975,19 +2039,30 @@ def anonymize_info(info, daysback=None, keep_his=False): proc_hist = info.get('proc_history') if proc_hist is not None: for record in proc_hist: - record['block_id']['secs'] = info['meas_date'][0] - record['block_id']['usecs'] = info['meas_date'][1] record['block_id']['machid'][:] = 0 - record['date'] = info['meas_date'] record['experimenter'] = default_str + if none_meas_date: + record['block_id']['secs'] = DATE_NONE[0] + record['block_id']['usecs'] = DATE_NONE[1] + record['date'] = DATE_NONE + else: + this_t0 = (record['block_id']['secs'], + record['block_id']['usecs']) + this_t1 = _add_timedelta_to_meas_date(this_t0, -delta_t) + record['block_id']['secs'] = this_t1[0] + record['block_id']['usecs'] = this_t1[1] + record['date'] = _add_timedelta_to_meas_date(record['date'], + -delta_t) hi = info.get('helium_info') if hi is not None: if hi.get('orig_file_guid') is not None: hi['orig_file_guid'] = default_str - if hi.get('meas_date') is not None: - hi['meas_date'] = [info['meas_date'][0], - info['meas_date'][1]] + if none_meas_date and hi.get('meas_date') is not None: + hi['meas_date'] = DATE_NONE + elif hi.get('meas_date') is not None: + hi['meas_date'] = _add_timedelta_to_meas_date(hi['meas_date'], + -delta_t) di = info.get('device_info') if di is not None: @@ -1995,6 +2070,11 @@ def anonymize_info(info, daysback=None, keep_his=False): if di.get(k) is not None: di[k] = default_str + err_mesg = ('anonymize_info generated an inconsistent info object. Most ' + 'often this is because daysback parameter was too large.\n' + 'Underlying Error:') + info._check_consistency(prepend_error=err_mesg) + return info diff --git a/mne/io/tests/test_meas_info.py b/mne/io/tests/test_meas_info.py index 9663d9e662e..835e32c6197 100644 --- a/mne/io/tests/test_meas_info.py +++ b/mne/io/tests/test_meas_info.py @@ -19,11 +19,12 @@ from mne.io import (read_fiducials, write_fiducials, _coil_trans_to_loc, _loc_to_coil_trans, read_raw_fif, read_info, write_info) from mne.io.constants import FIFF -from mne.io.write import _generate_meas_id +from mne.io.write import _generate_meas_id, DATE_NONE from mne.io.meas_info import (Info, create_info, _merge_info, _force_update_info, RAW_INFO_FIELDS, _bad_chans_comp, _get_valid_units, - anonymize_info, _stamp_to_dt, _dt_to_stamp) + anonymize_info, _stamp_to_dt, _dt_to_stamp, + _add_timedelta_to_meas_date) from mne.io._digitization import (_write_dig_points, _read_dig_points, _make_dig_points,) from mne.io import read_raw_ctf @@ -457,7 +458,7 @@ def _test_anonymize_info(base_info): exp_info['description'] = default_desc exp_info['experimenter'] = default_str exp_info['proj_name'] = default_str - exp_info['proj_id'][:] = 0 + exp_info['proj_id'] = np.array([0]) exp_info['subject_info']['first_name'] = default_str exp_info['subject_info']['last_name'] = default_str exp_info['subject_info']['id'] = default_subject_id @@ -467,12 +468,20 @@ def _test_anonymize_info(base_info): # 2010 and 2000. exp_info['subject_info']['birthday'] = (1977, 4, 7) exp_info['meas_date'] = _dt_to_stamp(default_anon_dos) + + # make copies + exp_info_3 = exp_info.copy() + + # adjust each expected outcome + delta_t = timedelta(days=3653) for key in ('file_id', 'meas_id'): value = exp_info.get(key) if value is not None: assert 'msecs' not in value - value['secs'] = exp_info['meas_date'][0] - value['usecs'] = exp_info['meas_date'][1] + tmp = _add_timedelta_to_meas_date((value['secs'], value['usecs']), + -delta_t) + value['secs'] = tmp[0] + value['usecs'] = tmp[1] value['machid'][:] = 0 # exp 2 tests the keep_his option @@ -480,25 +489,48 @@ def _test_anonymize_info(base_info): exp_info_2['subject_info']['his_id'] = 'foobar' # exp 3 tests is a supplied daysback - dt = timedelta(days=43) - exp_info_3 = exp_info.copy() + delta_t_2 = timedelta(days=43) exp_info_3['subject_info']['birthday'] = (1987, 2, 24) - exp_info_3['meas_date'] = _dt_to_stamp(meas_date - dt) + exp_info_3['meas_date'] = _dt_to_stamp(meas_date - delta_t_2) for key in ('file_id', 'meas_id'): value = exp_info_3.get(key) if value is not None: assert 'msecs' not in value - value['secs'] = exp_info_3['meas_date'][0] - value['usecs'] = exp_info_3['meas_date'][1] + tmp = _add_timedelta_to_meas_date((value['secs'], value['usecs']), + -delta_t_2) + value['secs'] = tmp[0] + value['usecs'] = tmp[1] value['machid'][:] = 0 + # exp 4 tests is a supplied daysback + delta_t_3 = timedelta(days=223 + 364 * 500) + new_info = anonymize_info(base_info.copy()) assert_object_equal(new_info, exp_info) new_info = anonymize_info(base_info.copy(), keep_his=True) assert_object_equal(new_info, exp_info_2) - new_info = anonymize_info(base_info.copy(), daysback=dt.days) + new_info = anonymize_info(base_info.copy(), daysback=delta_t_2.days) + assert_object_equal(new_info, exp_info_3) + + with pytest.raises(RuntimeError, match='anonymize_info generated'): + anonymize_info(base_info.copy(), daysback=delta_t_3.days) + # assert_object_equal(new_info, exp_info_4) + + # test with meas_date = None + base_info['meas_date'] = None + exp_info_3['meas_date'] = None + exp_info_3['file_id']['secs'] = DATE_NONE[0] + exp_info_3['file_id']['usecs'] = DATE_NONE[1] + exp_info_3['meas_id']['secs'] = DATE_NONE[0] + exp_info_3['meas_id']['usecs'] = DATE_NONE[1] + exp_info_3['subject_info'].pop('birthday', None) + + new_info = anonymize_info(base_info.copy(), daysback=delta_t_2.days) + assert_object_equal(new_info, exp_info_3) + + new_info = anonymize_info(base_info.copy()) assert_object_equal(new_info, exp_info_3) @@ -539,7 +571,11 @@ def test_anonymize(tmpdir): # test that annotations are correctly zeroed raw.anonymize() - assert(raw.annotations.orig_time is raw.info['meas_date']) + assert(raw.annotations.orig_time == (raw.info['meas_date'][0] + + raw.info['meas_date'][1] / 1000000.)) + raw.info['meas_date'] = None + raw.anonymize() + assert(raw.annotations.orig_time == 0) @testing.requires_testing_data