The tracks say what they are: motiongrams from the motion frame, videograms from the picture

CHANGELOG.md

@@ -8,6 +8,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Fixed
+- **The `videogram_*` tracks of `extract_tracks` were motiongrams** (#383): they averaged the motion
+  frame, not the picture. They are now written as `motiongram_v.u1` / `motiongram_h.u1` under
+  `motiongram_*` keys, and true videograms come from one extra ffmpeg filter graph
+  (`extract_videograms`, on by default, `videograms=False` to skip), trimmed so that column *j*
+  lines up with motion frame *j*. `read_columns`, `build_pyramid` and `render_timeline` take both
+  names; an analysis folder written before the rename serves its motion means under
+  `motiongram_*` and, under `videogram_*`, with a warning that says what they are.
+- `_mocap.read_c3d` imported `MgError` from `_utils`, where it does not live; mypy on master failed on it.
 - **`_tracks.read_columns` failed on any recording long enough to need a pyramid level.**
   `extract_tracks` and `extract_tracks_parallel` write the base videograms only, and
   `read_columns` memory-mapped `videogram_v.L<k>.u1` without checking it existed, so the first

docs/user-guide/people-and-camera.md

@@ -58,5 +58,9 @@ A 36 GB 1080p50 file decodes on the GPU by passing ffmpeg input options through:
 `detect_people(video, ffmpeg_input_args=["-hwaccel", "cuda"])` and
 `camera_motion(video, ffmpeg_input_args=["-hwaccel", "cuda"])`.
 
+For the picture itself over a whole recording, `extract_tracks` now also writes true videograms
+(`videogram_v.u1`, one column per frame) next to the motiongrams; `read_columns(dir, which="videogram_v")`
+reads any span at any width.
+
 Both analyses cache well: keep the proxy and the detections next to the recording and the
 counts for any span are instant.

musicalgestures/_mocap.py

@@ -237,7 +237,7 @@ def read_c3d(path: str) -> tuple[list[str], np.ndarray, float]:
     try:
         import ezc3d
     except ImportError as exc:
-        from musicalgestures._utils import MgError
+        from musicalgestures._exceptions import MgError
         raise MgError(
             "Reading C3D files requires the 'ezc3d' package. Install it with: "
             "pip install musicalgestures[c3d] (or: pip install ezc3d)"

musicalgestures/_timeline.py

@@ -144,15 +144,15 @@ def render_timeline(analysis_dir, start_s: float = 0.0, end_s=None,
     i1 = max(min(int(end_s * fps), n_frames), i0 + 1)
 
     for ax, panel in zip(axes, drawn):
-        if panel in ("videogram_v", "videogram_h"):
+        if panel in ("videogram_v", "videogram_h", "motiongram_v", "motiongram_h"):
             from musicalgestures._tracks import read_columns
             cols, spc = read_columns(d, start_s, end_s, max_columns=n_columns,
                                      which=panel)
             if cols.size:
                 ax.imshow(cols.T, aspect="auto", origin="lower", cmap="magma",
                           extent=(start_s, end_s, 0, cols.shape[1]))
                 factor = max(factor, int(round(spc * fps)))
-            ax.set_ylabel(panel.replace("videogram_", "videogram "))
+            ax.set_ylabel(panel.replace("_", " "))
             ax.set_yticks([])
 
         elif panel == "qom":

musicalgestures/_tracks.py

@@ -12,7 +12,7 @@
 
 The motiongrams are 71 per cent of it and the area of motion another 12. This module
 does the one pass those numbers argue for: **convert each motion frame to greyscale
-once, and take everything from that** --- the quantity of motion, and both videogram
+once, and take everything from that** --- the quantity of motion, both motiongram
 columns. `centroid()` converts to greyscale internally and then throws the conversion
 away; doing it once and reusing it is most of the saving, and working on one channel
 rather than three is the rest.
@@ -144,8 +144,8 @@ def _chunk_worker(args) -> int:
         cmd += ["-t", f"{dur:.6f}"]
 
     qom = _np.memmap(d / "qom.f4", dtype=_np.float32, mode="r+", shape=(n_total,))
-    vg = _np.memmap(d / "videogram_v.u1", dtype=_np.uint8, mode="r+", shape=(n_total, H))
-    hg = _np.memmap(d / "videogram_h.u1", dtype=_np.uint8, mode="r+", shape=(n_total, W))
+    vg = _np.memmap(d / "motiongram_v.u1", dtype=_np.uint8, mode="r+", shape=(n_total, H))
+    hg = _np.memmap(d / "motiongram_h.u1", dtype=_np.uint8, mode="r+", shape=(n_total, W))
 
     #: total_time drives the progress bar's arithmetic, so it must be a number
     #: even when no bar is wanted --- None makes it subtract from nothing.
@@ -180,7 +180,7 @@ def _chunk_worker(args) -> int:
     return written
 
 
-def extract_tracks(video, out_dir=None, filtertype="Regular", threshold=0.05,
+def extract_tracks(video, out_dir=None, filtertype="Regular", threshold=0.05, videograms=True,
                    blur="None", use_median=False, kernel_size=5,
                    plate_every=None, progress=True) -> dict:
     """Quantity of motion and both videogram bases, in one pass over the video.
@@ -208,8 +208,8 @@ def extract_tracks(video, out_dir=None, filtertype="Regular", threshold=0.05,
 
     d = _analysis_dir(video, out_dir)
     qom_path = d / "qom.f4"
-    vgram_path = d / "videogram_v.u1"      # one column per frame, height H
-    hgram_path = d / "videogram_h.u1"      # one row per frame, width W
+    vgram_path = d / "motiongram_v.u1"     # one column per motion frame, height H
+    hgram_path = d / "motiongram_h.u1"     # one row per motion frame, width W
 
     qom = np.memmap(qom_path, dtype=np.float32, mode="w+", shape=(n_max,))
     vg = np.memmap(vgram_path, dtype=np.uint8, mode="w+", shape=(n_max, H))
@@ -267,10 +267,10 @@ def extract_tracks(video, out_dir=None, filtertype="Regular", threshold=0.05,
         "duration_s": n / fps,
         "filtertype": filtertype, "threshold": threshold, "blur": blur,
         "use_median": use_median, "kernel_size": kernel_size,
-        "qom": qom_path.name, "videogram_v": vgram_path.name,
-        "videogram_h": hgram_path.name,
+        "qom": qom_path.name, "motiongram_v": vgram_path.name,
+        "motiongram_h": hgram_path.name,
         "note": ("qom is the sum of the greyscale motion frame, the same quantity "
-                 "mg_motion writes as QomRaw. The videogram bases hold one column "
+                 "mg_motion writes as QomRaw. The motiongram bases hold one column "
                  "per frame; read them through pyramid levels rather than whole."),
     }
     if plates:
@@ -283,16 +283,81 @@ def extract_tracks(video, out_dir=None, filtertype="Regular", threshold=0.05,
                               "everywhere they stood; a median removes them, because at "
                               "any pixel they are a minority of the samples.")
     meta["analysis_dir"] = str(d)
+    if videograms:
+        meta.update(extract_videograms(video, d, frames=meta["frames"], width=W, height=H))
     (d / "tracks.json").write_text(json.dumps(meta, indent=1) + "\n")
     return meta
 
 
+def extract_videograms(video, analysis_dir, frames=None, width=None, height=None,
+                       ffmpeg_input_args=None) -> dict:
+    """True videograms of the whole video, one column (row) per frame, as memmap bases.
+
+    A videogram averages the *picture* across one axis; the motiongram averages the *motion
+    frame*. `extract_tracks` computes the latter in its pass over the filtered stream, so the
+    videogram needs one more decode, which this does with a single ffmpeg filter graph that
+    writes both axes straight to disk (``videogram_v.u1``: frames × height, ``videogram_h.u1``:
+    frames × width, uint8 grey). When `frames` is given the bases are trimmed at the front to
+    that many rows, so column *j* lines up with motion frame *j* (the motion frame is the
+    difference to the previous picture, hence one fewer). Returns the meta keys to merge.
+    """
+    d = Path(analysis_dir)
+    if width is None or height is None:
+        from musicalgestures._utils import get_widthheight
+        width, height = get_widthheight(str(video))
+    vpath, hpath = d / "videogram_v.u1", d / "videogram_h.u1"
+    graph = (f"[0:v]format=gray,split=2[a][b];[a]scale=1:{height}:flags=area[va];"
+             f"[b]scale={width}:1:flags=area[vb]")
+    cmd = ["ffmpeg", "-v", "error", "-y", *(ffmpeg_input_args or []), "-i", str(video), "-filter_complex", graph,
+           "-map", "[va]", "-f", "rawvideo", "-pix_fmt", "gray", str(vpath),
+           "-map", "[vb]", "-f", "rawvideo", "-pix_fmt", "gray", str(hpath)]
+    import subprocess
+    subprocess.run(cmd, check=True, capture_output=True)
+    n_v = vpath.stat().st_size // height
+    n_h = hpath.stat().st_size // width
+    n = min(n_v, n_h)
+    if frames is not None and n > frames:
+        # keep the last `frames` columns: motion frame j is picture j+1 minus picture j
+        for path, span, count in ((vpath, height, n_v), (hpath, width, n_h)):
+            arr = np.memmap(path, dtype=np.uint8, mode="r", shape=(count, span))
+            tail = np.array(arr[count - frames:])
+            del arr
+            tail.tofile(path)
+        n = frames
+    elif frames is not None and n < frames:
+        for path, span, count in ((vpath, height, n_v), (hpath, width, n_h)):
+            arr = np.memmap(path, dtype=np.uint8, mode="r", shape=(count, span))
+            padded = np.concatenate([np.array(arr), np.zeros((frames - count, span), np.uint8)])
+            del arr
+            padded.tofile(path)
+        n = frames
+    return {"videogram_v": vpath.name, "videogram_h": hpath.name, "videogram_frames": int(n),
+            "videogram_note": "row and column means of the grey picture itself (motiongram_* are those of the motion frame)"}
+
+
 def _truncate(path: Path, nbytes: int) -> None:
     """Cut a memmap file back to the rows that were actually written."""
     with open(path, "r+b") as fh:
         fh.truncate(nbytes)
 
 
+def _track_file(meta: dict, which: str) -> str:
+    """The base file for a track name. Analysis folders written before the motiongram rename hold the
+    motion-frame means under the videogram_* keys; those are served for motiongram_* requests, and a
+    videogram_* request on such a folder is served with a warning that says what it is."""
+    if which in meta:
+        if which.startswith("videogram") and "motiongram_v" not in meta and "videogram_frames" not in meta:
+            import warnings
+            warnings.warn(f"{which} in this analysis folder was written before the motiongram rename and holds "
+                          "motion-frame means; re-run extract_tracks (or extract_videograms) for a true videogram",
+                          stacklevel=3)
+        return str(meta[which])
+    legacy = which.replace("motiongram", "videogram")
+    if which.startswith("motiongram") and legacy in meta and "motiongram_v" not in meta:
+        return str(meta[legacy])
+    raise KeyError(f"{which} not in tracks.json (have {sorted(k for k in meta if k.endswith(('_v', '_h')))})")
+
+
 def build_pyramid(analysis_dir, which="videogram_v") -> list[Path]:
     """Halve a videogram base repeatedly, keeping extremes rather than means.
 
@@ -306,8 +371,8 @@ def build_pyramid(analysis_dir, which="videogram_v") -> list[Path]:
     d = Path(analysis_dir)
     meta = json.loads((d / "tracks.json").read_text())
     n, H, W = meta["frames"], meta["height"], meta["width"]
-    span = H if which == "videogram_v" else W
-    base = np.memmap(d / meta[which], dtype=np.uint8, mode="r", shape=(n, span))
+    span = H if which.endswith("_v") else W
+    base = np.memmap(d / _track_file(meta, which), dtype=np.uint8, mode="r", shape=(n, span))
 
     out, level, cur = [], 0, np.asarray(base)
     while cur.shape[0] > MIN_LEVEL_COLUMNS:
@@ -337,7 +402,7 @@ def read_columns(analysis_dir, start_s=0.0, end_s=None, max_columns=2000,
     d = Path(analysis_dir)
     meta = json.loads((d / "tracks.json").read_text())
     n, fps = meta["frames"], meta["fps"]
-    span = meta["height"] if which == "videogram_v" else meta["width"]
+    span = meta["height"] if which.endswith("_v") else meta["width"]
     end_s = meta["duration_s"] if end_s is None else end_s
     want = max(1, int((end_s - start_s) * fps))
 
@@ -349,7 +414,7 @@ def read_columns(analysis_dir, start_s=0.0, end_s=None, max_columns=2000,
         stride *= 2
         level += 1
     if level == 0:
-        arr = np.memmap(d / meta[which], dtype=np.uint8, mode="r", shape=(n, span))
+        arr = np.memmap(d / _track_file(meta, which), dtype=np.uint8, mode="r", shape=(n, span))
     else:
         name = f"{which}.L{level}.u1"
         if not (d / name).exists():
@@ -371,7 +436,7 @@ def read_columns(analysis_dir, start_s=0.0, end_s=None, max_columns=2000,
 def extract_tracks_parallel(video, out_dir=None, workers=None, chunk_s=120.0,
                             filtertype="Regular", threshold=0.05, blur="None",
                             use_median=False, kernel_size=5, plate_every=None,
-                            resume=True) -> dict:
+                            resume=True, videograms=True) -> dict:
     """The same pass, split over processes by time. Resumable.
 
     The work is embarrassingly parallel because each frame's motion depends only on
@@ -390,8 +455,8 @@ def extract_tracks_parallel(video, out_dir=None, workers=None, chunk_s=120.0,
     d = _analysis_dir(video, out_dir)
 
     for name, dt, shape in (("qom.f4", np.float32, (n_total,)),
-                            ("videogram_v.u1", np.uint8, (n_total, H)),
-                            ("videogram_h.u1", np.uint8, (n_total, W))):
+                            ("motiongram_v.u1", np.uint8, (n_total, H)),
+                            ("motiongram_h.u1", np.uint8, (n_total, W))):
         if not (d / name).exists() or not resume:
             m = np.memmap(d / name, dtype=dt, mode="w+", shape=shape)
             m.flush(); del m
@@ -419,17 +484,18 @@ def extract_tracks_parallel(video, out_dir=None, workers=None, chunk_s=120.0,
     n = written or n_total
 
     _truncate(d / "qom.f4", n * 4)
-    _truncate(d / "videogram_v.u1", n * H)
-    _truncate(d / "videogram_h.u1", n * W)
+    _truncate(d / "motiongram_v.u1", n * H)
+    _truncate(d / "motiongram_h.u1", n * W)
 
     meta = {"video": str(video), "frames": n, "fps": fps, "width": W, "height": H,
             "duration_s": n / fps, "filtertype": filtertype, "threshold": threshold,
             "blur": blur, "use_median": use_median, "kernel_size": kernel_size,
             "workers": workers, "chunk_s": chunk_s,
-            "qom": "qom.f4", "videogram_v": "videogram_v.u1",
-            "videogram_h": "videogram_h.u1",
+            "qom": "qom.f4", "motiongram_v": "motiongram_v.u1",
+            "motiongram_h": "motiongram_h.u1",
             "note": ("qom is the sum of the greyscale motion frame, the quantity "
-                     "mg_motion writes as QomRaw. Chunks overlap by one frame and "
+                     "mg_motion writes as QomRaw; the motiongram bases are the row and "
+                     "column means of that motion frame. Chunks overlap by one frame and "
                      "discard it, because the first frame after a seek has no "
                      "predecessor to differ from.")}
 

tests/test_tracks.py

@@ -169,8 +169,8 @@ def test_chunk_worker_reaps_its_decoder(tmp_path):
     v = _synth(tmp_path / "leak.mp4", seconds=8)
     fps, W, H, n_total = 25.0, 320, 240, 208
     for name, dt, shape in (("qom.f4", np.float32, (n_total,)),
-                            ("videogram_v.u1", np.uint8, (n_total, H)),
-                            ("videogram_h.u1", np.uint8, (n_total, W))):
+                            ("motiongram_v.u1", np.uint8, (n_total, H)),
+                            ("motiongram_h.u1", np.uint8, (n_total, W))):
         m = np.memmap(tmp_path / name, dtype=dt, mode="w+", shape=shape)
         m.flush()
         del m

tests/test_true_videograms.py

@@ -0,0 +1,62 @@
+"""The videogram tracks are videograms (issue #383): means of the picture, not of the motion frame."""
+import json
+import subprocess
+
+import numpy as np
+import pytest
+
+from musicalgestures._tracks import extract_tracks, read_columns, build_pyramid
+
+cv2 = pytest.importorskip("cv2")
+
+
+def _static_then_moving(path, size=(160, 120), fps=25):
+    """4 s of a bright bar that never moves, then 4 s where a small square moves across it."""
+    W, H = size
+    out = cv2.VideoWriter(str(path), cv2.VideoWriter_fourcc(*"mp4v"), fps, (W, H))
+    for k in range(8 * fps):
+        f = np.zeros((H, W, 3), np.uint8)
+        f[40:60, :, :] = 200                         # the bar: rows 40..59 bright in every frame
+        if k >= 4 * fps:
+            x = (k - 4 * fps) * 3 % (W - 20)
+            f[90:110, x:x + 20, :] = 255              # the mover, lower in the frame
+        out.write(f)
+    out.release()
+    return str(path)
+
+
+def test_videogram_sees_the_static_bar_and_the_motiongram_does_not(tmp_path):
+    v = _static_then_moving(tmp_path / "v.mp4")
+    meta = extract_tracks(v, out_dir=tmp_path / "out", progress=False)
+    d = tmp_path / "out" / "v"
+    assert {"motiongram_v", "motiongram_h", "videogram_v", "videogram_h"} <= set(meta)
+    n, H = meta["frames"], meta["height"]
+    vg = np.memmap(d / meta["videogram_v"], dtype=np.uint8, mode="r", shape=(n, H))
+    mg = np.memmap(d / meta["motiongram_v"], dtype=np.uint8, mode="r", shape=(n, H))
+    early = slice(5, 3 * 25)                         # static seconds
+    assert vg[early, 40:60].mean() > 150             # the bar is in the picture
+    assert vg[early, 0:30].mean() < 10               # and the background is dark
+    assert mg[early, 40:60].mean() < 5               # nothing moves there, so the motiongram is dark
+    late = slice(4 * 25 + 5, n - 2)
+    assert mg[late, 90:110].mean() > mg[late, 40:60].mean() + 5   # motion shows where the square moves
+    # readers take both names, and the pyramid is per track
+    cols, spc = read_columns(d, max_columns=50, which="videogram_v")
+    assert cols.shape[1] == H and cols[:, 40:60].max() > 150
+    mcols, _ = read_columns(d, max_columns=50, which="motiongram_v")
+    assert mcols.shape[1] == H
+    pyr = json.loads((d / "tracks.json").read_text())["pyramid"]
+    assert "videogram_v" in pyr and "motiongram_v" in pyr
+
+
+def test_legacy_folder_serves_motion_means_under_the_old_name(tmp_path):
+    d = tmp_path / "old"; d.mkdir()
+    n, H, W = 300, 8, 6
+    np.full((n, H), 7, np.uint8).tofile(d / "videogram_v.u1")
+    np.zeros((n, W), np.uint8).tofile(d / "videogram_h.u1")
+    np.zeros(n, np.float32).tofile(d / "qom.f4")
+    (d / "tracks.json").write_text(json.dumps({"frames": n, "fps": 25.0, "width": W, "height": H, "duration_s": 12.0,
+                                               "qom": "qom.f4", "videogram_v": "videogram_v.u1", "videogram_h": "videogram_h.u1"}))
+    cols, _ = read_columns(d, max_columns=40, which="motiongram_v")     # the honest name works on old data
+    assert cols.max() == 7
+    with pytest.warns(UserWarning, match="motion-frame means"):
+        read_columns(d, max_columns=40, which="videogram_v")            # the old name still works, and says what it is