FIX: anonymize annotations

doc/changes/latest.inc

@@ -63,6 +63,8 @@ Changelog
 Bug
 ~~~
 
+- Fix :meth:`mne.io.Raw.anonymize` correctly reset ``raw.annotations.orig_time`` by `Luke Bloy`_.
+
 - Fix date reading before Unix time zero (1970 Jan 1) on Windows by `Alex Rockhill`_.
 
 - Fix :meth:`mne.Epochs.shift_time` and :meth:`mne.Evoked.shift_time` to return the modified :class:`~mne.Epochs` or :class:`~mne.Evoked` instance (instead of ``None``) by `Daniel McCloy`_.
@@ -75,7 +77,7 @@ Bug
 
 - Fix bug in :func:`mne.write_evokeds` where ``evoked.nave`` was not saved properly when multiple :class:`~mne.Evoked` instances were written to a single file, by `Eric Larson`_
 
-- Fix bug in :func:`mne.preprocessing.mark_flat` where acquisition skips were not handled proeprly, by `Eric Larson`_
+- Fix bug in :func:`mne.preprocessing.mark_flat` where acquisition skips were not handled properly, by `Eric Larson`_
 
 - Fix bug in :func:`mne.viz.plot_bem` where some sources were not plotted by `Jean-Remi King`_ and `Eric Larson`_
 

mne/channels/channels.py

@@ -22,6 +22,7 @@
 from ..io.pick import (channel_type, pick_info, pick_types, _picks_by_type,
                        _check_excludes_includes, _contains_ch_type,
                        channel_indices_by_type, pick_channels, _picks_to_idx)
+from ..annotations import _handle_meas_date
 
 
 DEPRECATED_PARAM = object()
@@ -582,7 +583,8 @@ def anonymize(self, daysback=None, keep_his=False):
         """
         anonymize_info(self.info, daysback=daysback, keep_his=keep_his)
         if hasattr(self, 'annotations'):
-            self.annotations.orig_time = self.info['meas_date']
+            self.annotations.orig_time = \
+                _handle_meas_date(self.info['meas_date'])
             self.annotations.onset -= self._first_time
         return self
 

mne/io/meas_info.py

@@ -582,26 +582,49 @@ def __repr__(self):
         st %= non_empty
         return st
 
-    def _check_consistency(self):
+    def _check_consistency(self, prepend_error=''):
         """Do some self-consistency checks and datatype tweaks."""
         missing = [bad for bad in self['bads'] if bad not in self['ch_names']]
         if len(missing) > 0:
-            raise RuntimeError('bad channel(s) %s marked do not exist in info'
-                               % (missing,))
+            msg = '%sbad channel(s) %s marked do not exist in info'
+            raise RuntimeError(msg % (prepend_error, missing,))
         meas_date = self.get('meas_date')
-        if meas_date is not None and (
-                not isinstance(self['meas_date'], tuple) or
-                len(self['meas_date']) != 2):
-            raise RuntimeError('info["meas_date"] must be a tuple of length '
-                               '2 or None, got "%r"'
-                               % (repr(self['meas_date']),))
+        if meas_date is not None:
+            if (not isinstance(self['meas_date'], tuple) or
+                    len(self['meas_date']) != 2):
+                raise RuntimeError('%sinfo["meas_date"] must be a tuple '
+                                   'of length 2 or None, got "%r"'
+                                   % (prepend_error, repr(self['meas_date']),))
+            if (meas_date[0] < np.iinfo('>i4').min or
+                    meas_date[0] > np.iinfo('>i4').max):
+                raise RuntimeError('%sinfo["meas_date"] must be between "%r" '
+                                   'and "%r", got "%r"'
+                                   % (prepend_error,
+                                      (np.iinfo('>i4').min, 0),
+                                      (np.iinfo('>i4').max, 0),
+                                      self['meas_date'],))
+
+        for key in ('file_id', 'meas_id'):
+            value = self.get(key)
+            if value is not None:
+                assert 'msecs' not in value
+                for key_2 in ('secs', 'usecs'):
+                    if (value[key_2] < np.iinfo('>i4').min or
+                            value[key_2] > np.iinfo('>i4').max):
+                        raise RuntimeError('%sinfo[%s][%s] must be between '
+                                           '"%r" and "%r", got "%r"'
+                                           % (prepend_error, key, key_2,
+                                              np.iinfo('>i4').min,
+                                              np.iinfo('>i4').max,
+                                              value[key_2]),)
 
         chs = [ch['ch_name'] for ch in self['chs']]
         if len(self['ch_names']) != len(chs) or any(
                 ch_1 != ch_2 for ch_1, ch_2 in zip(self['ch_names'], chs)) or \
                 self['nchan'] != len(chs):
-            raise RuntimeError('info channel name inconsistency detected, '
-                               'please notify mne-python developers')
+            raise RuntimeError('%sinfo channel name inconsistency detected, '
+                               'please notify mne-python developers'
+                               % (prepend_error,))
 
         # make sure we have the proper datatypes
         for key in ('sfreq', 'highpass', 'lowpass'):
@@ -1866,6 +1889,29 @@ def _force_update_info(info_base, info_target):
             i_targ[key] = val
 
 
+def _add_timedelta_to_meas_date(meas_date, delta_t):
+    """Add a timedelta to a meas_date tuple.
+
+    Parameters
+    ----------
+    meas_date : tuple | None
+        The Info object you want to use for overwriting values
+        in target Info objects.
+    delta_t : datetime.timedelta
+        The time difference that is added to the meas_date timestamp
+
+    Returns
+    -------
+    new_meas_date : tuple | none
+        The new meas_date tuple.
+    """
+    if meas_date is None:
+        new_meas_date = None
+    else:
+        new_meas_date = _dt_to_stamp(_stamp_to_dt(meas_date) + delta_t)
+    return new_meas_date
+
+
 def anonymize_info(info, daysback=None, keep_his=False):
     """Anonymize measurement information in place.
 
@@ -1917,25 +1963,40 @@ def anonymize_info(info, daysback=None, keep_his=False):
     default_desc = ("Anonymized using a time shift"
                     " to preserve age at acquisition")
 
-    # datetime object representing meas_date
-    meas_date_datetime = _stamp_to_dt(info['meas_date'])
+    none_meas_date = info['meas_date'] is None
 
-    if daysback is None:
-        delta_t = meas_date_datetime - default_anon_dos
+    if none_meas_date:
+        logger.warning('Input info has \'meas_date\' set to None.'
+                       ' Removing all information from time/date structures.'
+                       ' *NOT* performing any time shifts')
+        info['meas_date'] = None
     else:
-        delta_t = datetime.timedelta(days=daysback)
-
-    # adjust meas_date
-    info['meas_date'] = _dt_to_stamp(meas_date_datetime - delta_t)
+        # compute timeshift delta
+        if daysback is None:
+            delta_t = _stamp_to_dt(info['meas_date']) - default_anon_dos
+        else:
+            delta_t = datetime.timedelta(days=daysback)
+        # adjust meas_date
+        info['meas_date'] = _add_timedelta_to_meas_date(info['meas_date'],
+                                                        -delta_t)
 
     # file_id and meas_id
     for key in ('file_id', 'meas_id'):
         value = info.get(key)
         if value is not None:
             assert 'msecs' not in value
-            value['secs'] = info['meas_date'][0]
-            value['usecs'] = info['meas_date'][1]
-            value['machid'][:] = 0
+            if none_meas_date:
+                tmp = DATE_NONE
+            else:
+                tmp = _add_timedelta_to_meas_date((value['secs'],
+                                                   value['usecs']), -delta_t)
+            value['secs'] = tmp[0]
+            value['usecs'] = tmp[1]
+            # The following copy is needed for a test CTF dataset
+            # otherwise value['machid'][:] = 0 would suffice
+            _tmp = value['machid'].copy()
+            _tmp[:] = 0
+            value['machid'] = _tmp
 
     # subject info
     subject_info = info.get('subject_info')
@@ -1951,7 +2012,10 @@ def anonymize_info(info, daysback=None, keep_his=False):
             if subject_info.get(key) is not None:
                 subject_info[key] = default_str
 
-        if subject_info.get('birthday') is not None:
+        # anonymize the subject birthday
+        if none_meas_date:
+            subject_info.pop('birthday', None)
+        elif subject_info.get('birthday') is not None:
             dob = datetime.datetime(subject_info['birthday'][0],
                                     subject_info['birthday'][1],
                                     subject_info['birthday'][2])
@@ -1975,26 +2039,42 @@ def anonymize_info(info, daysback=None, keep_his=False):
     proc_hist = info.get('proc_history')
     if proc_hist is not None:
         for record in proc_hist:
-            record['block_id']['secs'] = info['meas_date'][0]
-            record['block_id']['usecs'] = info['meas_date'][1]
             record['block_id']['machid'][:] = 0
-            record['date'] = info['meas_date']
             record['experimenter'] = default_str
+            if none_meas_date:
+                record['block_id']['secs'] = DATE_NONE[0]
+                record['block_id']['usecs'] = DATE_NONE[1]
+                record['date'] = DATE_NONE
+            else:
+                this_t0 = (record['block_id']['secs'],
+                           record['block_id']['usecs'])
+                this_t1 = _add_timedelta_to_meas_date(this_t0, -delta_t)
+                record['block_id']['secs'] = this_t1[0]
+                record['block_id']['usecs'] = this_t1[1]
+                record['date'] = _add_timedelta_to_meas_date(record['date'],
+                                                             -delta_t)
 
     hi = info.get('helium_info')
     if hi is not None:
         if hi.get('orig_file_guid') is not None:
             hi['orig_file_guid'] = default_str
-        if hi.get('meas_date') is not None:
-            hi['meas_date'] = [info['meas_date'][0],
-                               info['meas_date'][1]]
+        if none_meas_date and hi.get('meas_date') is not None:
+            hi['meas_date'] = DATE_NONE
+        elif hi.get('meas_date') is not None:
+            hi['meas_date'] = _add_timedelta_to_meas_date(hi['meas_date'],
+                                                          -delta_t)
 
     di = info.get('device_info')
     if di is not None:
         for k in ('serial', 'site'):
             if di.get(k) is not None:
                 di[k] = default_str
 
+    err_mesg = ('anonymize_info generated an inconsistent info object. Most '
+                'often this is because daysback parameter was too large.\n'
+                'Underlying Error:')
+    info._check_consistency(prepend_error=err_mesg)
+
     return info
 
 

mne/io/tests/test_meas_info.py

@@ -19,11 +19,12 @@
 from mne.io import (read_fiducials, write_fiducials, _coil_trans_to_loc,
                     _loc_to_coil_trans, read_raw_fif, read_info, write_info)
 from mne.io.constants import FIFF
-from mne.io.write import _generate_meas_id
+from mne.io.write import _generate_meas_id, DATE_NONE
 from mne.io.meas_info import (Info, create_info, _merge_info,
                               _force_update_info, RAW_INFO_FIELDS,
                               _bad_chans_comp, _get_valid_units,
-                              anonymize_info, _stamp_to_dt, _dt_to_stamp)
+                              anonymize_info, _stamp_to_dt, _dt_to_stamp,
+                              _add_timedelta_to_meas_date)
 from mne.io._digitization import (_write_dig_points, _read_dig_points,
                                   _make_dig_points,)
 from mne.io import read_raw_ctf
@@ -457,7 +458,7 @@ def _test_anonymize_info(base_info):
     exp_info['description'] = default_desc
     exp_info['experimenter'] = default_str
     exp_info['proj_name'] = default_str
-    exp_info['proj_id'][:] = 0
+    exp_info['proj_id'] = np.array([0])
     exp_info['subject_info']['first_name'] = default_str
     exp_info['subject_info']['last_name'] = default_str
     exp_info['subject_info']['id'] = default_subject_id
@@ -467,38 +468,69 @@ def _test_anonymize_info(base_info):
     # 2010 and 2000.
     exp_info['subject_info']['birthday'] = (1977, 4, 7)
     exp_info['meas_date'] = _dt_to_stamp(default_anon_dos)
+
+    # make copies
+    exp_info_3 = exp_info.copy()
+
+    # adjust each expected outcome
+    delta_t = timedelta(days=3653)
     for key in ('file_id', 'meas_id'):
         value = exp_info.get(key)
         if value is not None:
             assert 'msecs' not in value
-            value['secs'] = exp_info['meas_date'][0]
-            value['usecs'] = exp_info['meas_date'][1]
+            tmp = _add_timedelta_to_meas_date((value['secs'], value['usecs']),
+                                              -delta_t)
+            value['secs'] = tmp[0]
+            value['usecs'] = tmp[1]
             value['machid'][:] = 0
 
     # exp 2 tests the keep_his option
     exp_info_2 = exp_info.copy()
     exp_info_2['subject_info']['his_id'] = 'foobar'
 
     # exp 3 tests is a supplied daysback
-    dt = timedelta(days=43)
-    exp_info_3 = exp_info.copy()
+    delta_t_2 = timedelta(days=43)
     exp_info_3['subject_info']['birthday'] = (1987, 2, 24)
-    exp_info_3['meas_date'] = _dt_to_stamp(meas_date - dt)
+    exp_info_3['meas_date'] = _dt_to_stamp(meas_date - delta_t_2)
     for key in ('file_id', 'meas_id'):
         value = exp_info_3.get(key)
         if value is not None:
             assert 'msecs' not in value
-            value['secs'] = exp_info_3['meas_date'][0]
-            value['usecs'] = exp_info_3['meas_date'][1]
+            tmp = _add_timedelta_to_meas_date((value['secs'], value['usecs']),
+                                              -delta_t_2)
+            value['secs'] = tmp[0]
+            value['usecs'] = tmp[1]
             value['machid'][:] = 0
 
+    # exp 4 tests is a supplied daysback
+    delta_t_3 = timedelta(days=223 + 364 * 500)
+
     new_info = anonymize_info(base_info.copy())
     assert_object_equal(new_info, exp_info)
 
     new_info = anonymize_info(base_info.copy(), keep_his=True)
     assert_object_equal(new_info, exp_info_2)
 
-    new_info = anonymize_info(base_info.copy(), daysback=dt.days)
+    new_info = anonymize_info(base_info.copy(), daysback=delta_t_2.days)
+    assert_object_equal(new_info, exp_info_3)
+
+    with pytest.raises(RuntimeError, match='anonymize_info generated'):
+        anonymize_info(base_info.copy(), daysback=delta_t_3.days)
+    # assert_object_equal(new_info, exp_info_4)
+
+    # test with meas_date = None
+    base_info['meas_date'] = None
+    exp_info_3['meas_date'] = None
+    exp_info_3['file_id']['secs'] = DATE_NONE[0]
+    exp_info_3['file_id']['usecs'] = DATE_NONE[1]
+    exp_info_3['meas_id']['secs'] = DATE_NONE[0]
+    exp_info_3['meas_id']['usecs'] = DATE_NONE[1]
+    exp_info_3['subject_info'].pop('birthday', None)
+
+    new_info = anonymize_info(base_info.copy(), daysback=delta_t_2.days)
+    assert_object_equal(new_info, exp_info_3)
+
+    new_info = anonymize_info(base_info.copy())
     assert_object_equal(new_info, exp_info_3)
 
 
@@ -539,7 +571,11 @@ def test_anonymize(tmpdir):
 
     # test that annotations are correctly zeroed
     raw.anonymize()
-    assert(raw.annotations.orig_time is raw.info['meas_date'])
+    assert(raw.annotations.orig_time == (raw.info['meas_date'][0] +
+                                         raw.info['meas_date'][1] / 1000000.))
+    raw.info['meas_date'] = None
+    raw.anonymize()
+    assert(raw.annotations.orig_time == 0)
 
 
 @testing.requires_testing_data