From 5b744971fc73b6d7991de119198b32f822a9bba6 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Sat, 1 Aug 2026 14:57:43 +0200 Subject: [PATCH 1/5] feat(cursor): carry editable motion regions from the scene into the compositor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Groundwork for the cursor choreography proposed in #113, whose editor was written against the pre-1.8.0 tree and cannot be rebased onto it. This is the half that is shared infrastructure rather than feature UI: the JS→Rust contract, and the sampling. `SceneCursor` gains a `motion` list. Anchors arrive RESOLVED -- the editor owns discovering rests, clicks and manual splits, and hands over absolute normalised points -- so the compositor holds pure geometry and the same list always yields the same path. The field is `#[serde(default)]` on the Rust side: every existing scene predates it and must stay readable. `CursorTrack::with_motion` applies the presets by rewriting the sample list ONCE, the way `smoothed` already does, rather than per frame. That is what keeps the track a pure function of `t`, so a seek lands on the same image as linear playback and the preview matches the export -- the constraint `smooth_follow_samples` documents. Regions resample at 240 Hz internally because the curve is analytic while telemetry may hold only a handful of points across a region; reusing those would draw an arc as a polyline. Motion is applied BEFORE smoothing at both call sites. The preset defines the trajectory and smoothing is a render filter over whatever trajectory exists; the other order would leave edited stretches crisp inside a damped track and make the slider inert on them. In the live preview the regeneration moved below the scene block, since the regions come from `full_scene` -- read above it they would be one iteration stale, and an edit would surface a frame late or not at all if nothing else changed. The geometry is a direct port of `sampleCursorMotionRegion` from #113, including the speed rounding, which feeds an exponent and is therefore not cosmetic. What is NOT here: the editor UI (draggable control points, timeline segments, inspector) and the anchor discovery that feeds these regions. That is the feature, and it belongs to its author. Refs #113 --- crates/compositor/src/cursor.rs | 297 +++++++++++++++++++++++++ crates/compositor/src/live.rs | 30 ++- crates/compositor/src/scene.rs | 122 ++++++++++ crates/compositor/src/timeline_walk.rs | 11 +- src/native/sceneDescription.ts | 46 ++++ 5 files changed, 497 insertions(+), 9 deletions(-) diff --git a/crates/compositor/src/cursor.rs b/crates/compositor/src/cursor.rs index 621309e67..caef46287 100644 --- a/crates/compositor/src/cursor.rs +++ b/crates/compositor/src/cursor.rs @@ -219,6 +219,204 @@ impl CursorTrack { // déplace la trajectoire, pas la chronologie de ce que faisait l'utilisateur. CursorTrack::new(samples, self.clicks.clone(), self.types.clone()) } + + /// Piste dont les portions couvertes par une région éditée suivent la géométrie du preset + /// au lieu de la télémétrie. + /// + /// Appliqué UNE fois aux échantillons, comme `smoothed` et pour la même raison : la piste + /// reste une pure fonction de `t`, donc la preview et l'export rendent le même tracé, et un + /// seek retombe sur l'image de la lecture linéaire. + /// + /// La courbe est ré-échantillonnée à 240 Hz à l'intérieur des régions, indépendamment de la + /// densité de la télémétrie : le tracé est analytique, alors qu'une capture peut n'avoir que + /// quelques points sur la durée d'une région — les reprendre tels quels rendrait un arc en + /// ligne brisée. Hors régions, les échantillons bruts passent intacts. + /// + /// Les clics et les états gardent leurs instants (même principe que `smoothed`) : rien ici + /// ne retime ce que faisait l'utilisateur, seule la position change. + pub fn with_motion(&self, regions: &[CursorMotionRegion]) -> CursorTrack { + // `Recorded` est inerte : une région qui ne change rien ne doit pas déclencher le + // ré-échantillonnage, sinon elle remplacerait la télémétrie par sa propre relecture. + let active: Vec<&CursorMotionRegion> = regions + .iter() + .filter(|r| r.preset != CursorMotionPreset::Recorded && r.end_s > r.start_s) + .collect(); + if active.is_empty() { + return CursorTrack::new(self.samples.clone(), self.clicks.clone(), self.types.clone()); + } + + // Recouvrement : la DERNIÈRE région gagne, parité `findCursorMotionRegionAtTime` (TS) + // qui parcourt la liste à l'envers. + let covering = |t: f32| -> Option<&CursorMotionRegion> { + active.iter().rev().find(|r| t >= r.start_s && t <= r.end_s).copied() + }; + + const STEP_S: f32 = 1.0 / 240.0; + let mut samples: Vec<(f32, f32, f32)> = + self.samples.iter().filter(|(t, _, _)| covering(*t).is_none()).copied().collect(); + + for region in &active { + let span = region.end_s - region.start_s; + let steps = ((span / STEP_S).round() as usize).max(1); + for i in 0..=steps { + // Le dernier point est posé sur `end_s` exact plutôt que sur la grille : la + // couture avec l'échantillon brut suivant doit être franche, sinon + // l'interpolation de `sample_at` traverse un trou et coupe le raccord. + let t = if i == steps { region.end_s } else { region.start_s + i as f32 * STEP_S }; + let (x, y) = sample_region(region, t); + samples.push((t, x, y)); + } + } + + samples.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); + CursorTrack::new(samples, self.clicks.clone(), self.types.clone()) + } +} + +/// Forme dessinée entre les deux ancres d'une région. `Recorded` est inerte : la région +/// existe pour porter une sélection dans l'éditeur sans toucher au tracé. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum CursorMotionPreset { + Recorded, + Straight, + Arc, + Wave, + Loop, + Overshoot, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum CursorMotionEasing { + Linear, + EaseInOut, + EaseIn, + EaseOut, +} + +/// Une portion de trajectoire éditée, qui remplace la télémétrie entre `start_s` et `end_s`. +/// +/// Les ancres arrivent RÉSOLUES (l'éditeur possède la détection des rests, clics et coupes +/// manuelles) : ce module ne fait que de la géométrie, donc la même liste redonne toujours +/// le même tracé. `control` est un point absolu, pas un décalage — le sampler en prend +/// l'écart au milieu de [start, end], si bien qu'un point de contrôle laissé au milieu est +/// neutre pour tous les presets. +#[derive(Clone, Debug, PartialEq)] +pub struct CursorMotionRegion { + pub start_s: f32, + pub end_s: f32, + pub start: (f32, f32), + pub end: (f32, f32), + pub control: (f32, f32), + pub preset: CursorMotionPreset, + pub cycles: u32, + pub speed: f32, + pub easing: CursorMotionEasing, +} + +fn lerp(a: f32, b: f32, p: f32) -> f32 { + a + (b - a) * p +} + +/// Parité `clampCursorMotionCycles` (TS) : entier, borné 1..6. +fn clamp_cycles(cycles: u32) -> f32 { + cycles.clamp(1, 6) as f32 +} + +/// Parité `clampCursorMotionSpeed` (TS) : borné 1..4 PUIS arrondi au dixième. L'arrondi +/// n'est pas cosmétique — il entre dans l'exposant ci-dessous, donc l'omettre ferait +/// diverger le tracé de ce que l'éditeur a affiché. +fn clamp_speed(speed: f32) -> f32 { + let v = if speed.is_finite() { speed } else { 1.0 }; + (v.clamp(1.0, 4.0) * 10.0).round() / 10.0 +} + +/// Parité `applyCursorMotionSpeed` (TS) : `1 - (1-t)^vitesse`. Reprofile la progression, +/// ne retime pas la région — et démarre au premier instant, sans palier gelé en tête (un +/// palier avait rendu invisibles la plupart des courts déplacements à 2x). +fn apply_speed(progress: f32, speed: f32) -> f32 { + let t = progress.clamp(0.0, 1.0); + (1.0 - (1.0 - t).powf(clamp_speed(speed))).clamp(0.0, 1.0) +} + +/// Parité `easeProgress` (TS). +fn ease_progress(progress: f32, easing: CursorMotionEasing) -> f32 { + let t = progress.clamp(0.0, 1.0); + match easing { + CursorMotionEasing::EaseIn => t * t * t, + CursorMotionEasing::EaseOut => 1.0 - (1.0 - t).powi(3), + CursorMotionEasing::EaseInOut => { + if t < 0.5 { + 4.0 * t * t * t + } else { + 1.0 - (-2.0 * t + 2.0).powi(3) / 2.0 + } + } + CursorMotionEasing::Linear => t, + } +} + +/// Parité `easeOutBack` (TS) : dépasse la cible puis revient. +fn ease_out_back(progress: f32) -> f32 { + const C1: f32 = 1.70158; + const C3: f32 = C1 + 1.0; + let t = progress.clamp(0.0, 1.0) - 1.0; + 1.0 + C3 * t.powi(3) + C1 * t.powi(2) +} + +/// Position dans une région à l'instant `t` — port direct de `sampleCursorMotionRegion` +/// (TS). Fonction pure de `t` : c'est ce qui autorise à l'appliquer une fois aux +/// échantillons plutôt que par frame, donc à faire coïncider preview et export. +fn sample_region(region: &CursorMotionRegion, t: f32) -> (f32, f32) { + let duration = (region.end_s - region.start_s).max(0.001); + let raw = ((t - region.start_s) / duration).clamp(0.0, 1.0); + if raw <= 0.0 { + return region.start; + } + if raw >= 1.0 { + return region.end; + } + let motion = apply_speed(raw, region.speed); + if motion == 0.0 { + return region.start; + } + let progress = ease_progress(motion, region.easing); + + let mid = ((region.start.0 + region.end.0) / 2.0, (region.start.1 + region.end.1) / 2.0); + let offset = (region.control.0 - mid.0, region.control.1 - mid.1); + let envelope = (std::f32::consts::PI * motion).sin(); + + if region.preset == CursorMotionPreset::Overshoot { + let p = ease_out_back(progress); + return ( + lerp(region.start.0, region.end.0, p) + offset.0 * envelope * 0.35, + lerp(region.start.1, region.end.1, p) + offset.1 * envelope * 0.35, + ); + } + + let base = ( + lerp(region.start.0, region.end.0, progress), + lerp(region.start.1, region.end.1, progress), + ); + let cycles = clamp_cycles(region.cycles); + + match region.preset { + CursorMotionPreset::Arc => (base.0 + offset.0 * envelope, base.1 + offset.1 * envelope), + CursorMotionPreset::Wave => { + let wave = (std::f32::consts::PI * 2.0 * cycles * motion).sin() * envelope; + (base.0 + offset.0 * wave, base.1 + offset.1 * wave) + } + CursorMotionPreset::Loop => { + let phase = std::f32::consts::PI * 2.0 * cycles * motion; + let tangent = phase.sin() * envelope; + let normal = (1.0 - phase.cos()) * 0.5 * envelope; + ( + base.0 + offset.0 * tangent - offset.1 * normal, + base.1 + offset.1 * tangent + offset.0 * normal, + ) + } + // `Straight` suit la droite ; `Recorded` n'arrive jamais ici (filtré en amont). + _ => base, + } } /// Ressort-amortisseur, intégration semi-implicite (symplectique) d'Euler — stable pour ces @@ -277,6 +475,105 @@ mod tests { assert_eq!(track.type_at(99.0), Some("pointer"), "la dernière tient jusqu'à la fin"); } + fn region(preset: CursorMotionPreset) -> CursorMotionRegion { + CursorMotionRegion { + start_s: 1.0, + end_s: 2.0, + start: (0.0, 0.0), + end: (1.0, 0.0), + // Écarté du milieu (0.5, 0.0) : de quoi rendre les presets courbes visibles. + control: (0.5, 0.4), + preset, + cycles: 1, + speed: 1.0, + easing: CursorMotionEasing::Linear, + } + } + + /// Une piste qui fait un détour par le haut entre t=1 et t=2. + fn detour_track() -> CursorTrack { + CursorTrack::new( + vec![ + (0.0, 0.0, 0.0), + (1.0, 0.0, 0.0), + (1.5, 0.5, -0.9), // le détour que l'édition doit effacer + (2.0, 1.0, 0.0), + (3.0, 1.0, 0.0), + ], + vec![1.5], + vec![(0.0, "arrow".into())], + ) + } + + /// `Straight` remplace la trajectoire enregistrée par la droite entre les deux ancres : + /// au milieu de la région on doit être sur la corde, pas sur le détour capturé. + #[test] + fn straight_region_replaces_the_recorded_detour() { + let edited = detour_track().with_motion(&[region(CursorMotionPreset::Straight)]); + let (x, y) = edited.at(1.5).expect("position au milieu de la région"); + assert!((x - 0.5).abs() < 0.01, "x devrait être à mi-corde, obtenu {x}"); + assert!(y.abs() < 0.01, "le détour vertical devrait avoir disparu, obtenu {y}"); + } + + /// `Recorded` est inerte — c'est le préréglage par défaut, et il ne doit surtout pas + /// ré-échantillonner la piste : le tracé capturé passe intact. + #[test] + fn recorded_region_leaves_the_track_untouched() { + let raw = detour_track(); + let edited = raw.with_motion(&[region(CursorMotionPreset::Recorded)]); + assert_eq!(edited.sample_count(), raw.sample_count()); + assert_eq!(edited.at(1.5), raw.at(1.5), "le détour doit survivre"); + } + + /// Les bornes sont exactes : une région ne doit pas décaler la position aux instants où + /// elle se raccorde au reste de la piste, sinon la couture saute à l'image près. + #[test] + fn region_endpoints_land_exactly_on_their_anchors() { + let edited = detour_track().with_motion(&[region(CursorMotionPreset::Arc)]); + let (sx, sy) = edited.at(1.0).unwrap(); + let (ex, ey) = edited.at(2.0).unwrap(); + assert!(sx.abs() < 1e-4 && sy.abs() < 1e-4, "début ({sx}, {sy})"); + assert!((ex - 1.0).abs() < 1e-4 && ey.abs() < 1e-4, "fin ({ex}, {ey})"); + } + + /// Hors région, la télémétrie brute n'est pas touchée. + #[test] + fn samples_outside_the_region_survive() { + let edited = detour_track().with_motion(&[region(CursorMotionPreset::Straight)]); + assert_eq!(edited.at(0.0), Some((0.0, 0.0))); + assert_eq!(edited.at(3.0), Some((1.0, 0.0))); + } + + /// `Arc` s'écarte de la corde du côté du point de contrôle, au maximum en milieu de course. + #[test] + fn arc_bends_toward_its_control_point() { + let edited = detour_track().with_motion(&[region(CursorMotionPreset::Arc)]); + let (_, y) = edited.at(1.5).unwrap(); + assert!(y > 0.3, "l'arc devrait culminer vers le point de contrôle, obtenu {y}"); + } + + /// Recouvrement : la dernière région gagne, comme `findCursorMotionRegionAtTime` en TS. + #[test] + fn overlapping_regions_resolve_to_the_last_one() { + let mut second = region(CursorMotionPreset::Straight); + second.end = (1.0, 0.8); // destination distincte, pour trancher sans ambiguïté + let edited = + detour_track().with_motion(&[region(CursorMotionPreset::Arc), second]); + let (_, y) = edited.at(2.0).unwrap(); + assert!((y - 0.8).abs() < 1e-3, "la seconde région devrait l'emporter, obtenu {y}"); + } + + /// L'édition déplace la trajectoire, pas la chronologie : clics et états gardent leurs + /// instants, exactement comme sous `smoothed`. + #[test] + fn motion_preserves_clicks_and_types() { + let edited = detour_track().with_motion(&[region(CursorMotionPreset::Loop)]); + assert_eq!(edited.type_at(1.5), Some("arrow")); + // Échantillonné DANS la fenêtre d'animation, pas sur son bord : à l'instant même du + // clic la pression commence tout juste et vaut encore 1.0. + assert!(edited.bounce(1.55) < 1.0, "le clic à 1.5 s doit toujours produire son bounce"); + } + /// Le lissage déplace la trajectoire, pas la chronologie : les états doivent survivre /// intacts à `smoothed()`, comme les clics. #[test] diff --git a/crates/compositor/src/live.rs b/crates/compositor/src/live.rs index 49199a699..042f846b5 100644 --- a/crates/compositor/src/live.rs +++ b/crates/compositor/src/live.rs @@ -1228,6 +1228,9 @@ unsafe fn render_thread( let mut last_preview_size: (u32, u32) = (0, 0); let mut last_ip: Option = None; let mut last_smoothing: f32 = -1.0; // force la 1re application (0.0 est une valeur valide) + // Dernières régions appliquées. La liste vide est l'état de repos ET une valeur valide : + // c'est `last_smoothing = -1.0` qui garantit la première application, pas ce champ. + let mut last_motion: Vec = Vec::new(); // La vue live est TOUJOURS pilotée par la scène de l'app. Tant qu'aucune scène n'a été // appliquée, on refuse de jouer le layout fixture (POC) : un fallback fixture ne ferait que // MASQUER un scene-push cassé. On attend la scène avant de produire le 1er frame. @@ -1364,14 +1367,6 @@ unsafe fn render_thread( cursor_motion_blur: ip.cursor_motion_blur, has_webcam: has_real_webcam, }); - // Lissage ressort-amortisseur : re-génère la piste (240 Hz) uniquement quand la valeur - // change (pas à chaque frame — le resample+ressort parcourt tout l'enregistrement). - if let Some(raw) = &raw_cursor { - if ip.cursor_smoothing != last_smoothing { - comp.set_cursor(raw.smoothed(ip.cursor_smoothing)); - last_smoothing = ip.cursor_smoothing; - } - } // un changement de param doit se voir même en pause (édition live des sliders) : // on recompose la frame courante dans la branche pause ci-dessous. let ip_changed = last_ip != Some(ip); @@ -1401,6 +1396,25 @@ unsafe fn render_thread( comp.set_scene(scene); } + // Piste curseur : trajectoire éditée puis lissage ressort-amortisseur. Re-générée + // uniquement quand l'un des deux change — le resample 240 Hz parcourt tout + // l'enregistrement, c'est trop cher par frame. + // + // Placé APRÈS le bloc de scène, pas avant : les régions viennent de `full_scene`, et + // les lire au-dessus les prendrait à l'état du tour précédent — une trajectoire + // éditée n'apparaîtrait qu'à la frame suivante, ou jamais si rien d'autre ne bouge. + if let Some(raw) = &raw_cursor { + let motion: Vec = full_scene + .as_ref() + .map(|s| s.cursor.motion.iter().map(Into::into).collect()) + .unwrap_or_default(); + if ip.cursor_smoothing != last_smoothing || motion != last_motion { + comp.set_cursor(raw.with_motion(&motion).smoothed(ip.cursor_smoothing)); + last_smoothing = ip.cursor_smoothing; + last_motion = motion; + } + } + // résolution cible du preview (le canvas Electron) → force le recadrage des // ressources GPU si elle change. BUG évité : sans ce suivi, redimensionner le // panneau preview PENDANT une pause ne redéclenchait ni recompose ni readback diff --git a/crates/compositor/src/scene.rs b/crates/compositor/src/scene.rs index 2de569510..f9baab583 100644 --- a/crates/compositor/src/scene.rs +++ b/crates/compositor/src/scene.rs @@ -336,6 +336,87 @@ pub struct SceneCursor { /// `#[serde(default)]` : champ ajouté après coup, absent des JSON de test existants. #[serde(default)] pub cursor_sprites: std::collections::HashMap, + /// Portions de trajectoire éditées, dans l'ordre de la timeline. Absent ou vide → la + /// télémétrie enregistrée joue telle quelle, ce que produit tout projet sans l'éditeur + /// de choréographie. `#[serde(default)]` pour cette raison : les scènes existantes ne + /// portent pas ce champ et doivent rester lisibles. + #[serde(default)] + pub motion: Vec, +} + +/// Point normalisé dans le cadre screen. +#[derive(Debug, Clone, Copy, Deserialize)] +pub struct ScenePoint { + pub cx: f32, + pub cy: f32, +} + +/// Une région de trajectoire éditée. Miroir de `SceneCursorMotionRegion` (TS, +/// `src/native/sceneDescription.ts`) — les ancres arrivent résolues, ce module ne fait que +/// de la géométrie. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneCursorMotionRegion { + #[allow(dead_code)] + pub id: String, + pub start_sec: f32, + pub end_sec: f32, + pub start_point: ScenePoint, + pub end_point: ScenePoint, + pub control_point: ScenePoint, + pub preset: SceneCursorMotionPreset, + pub cycles: u32, + pub speed: f32, + pub easing: SceneCursorMotionEasing, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SceneCursorMotionPreset { + Recorded, + Straight, + Arc, + Wave, + Loop, + Overshoot, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum SceneCursorMotionEasing { + Linear, + EaseInOut, + EaseIn, + EaseOut, +} + +impl From<&SceneCursorMotionRegion> for crate::cursor::CursorMotionRegion { + fn from(r: &SceneCursorMotionRegion) -> Self { + use crate::cursor::{CursorMotionEasing as E, CursorMotionPreset as P}; + crate::cursor::CursorMotionRegion { + start_s: r.start_sec, + end_s: r.end_sec, + start: (r.start_point.cx, r.start_point.cy), + end: (r.end_point.cx, r.end_point.cy), + control: (r.control_point.cx, r.control_point.cy), + preset: match r.preset { + SceneCursorMotionPreset::Recorded => P::Recorded, + SceneCursorMotionPreset::Straight => P::Straight, + SceneCursorMotionPreset::Arc => P::Arc, + SceneCursorMotionPreset::Wave => P::Wave, + SceneCursorMotionPreset::Loop => P::Loop, + SceneCursorMotionPreset::Overshoot => P::Overshoot, + }, + cycles: r.cycles, + speed: r.speed, + easing: match r.easing { + SceneCursorMotionEasing::Linear => E::Linear, + SceneCursorMotionEasing::EaseInOut => E::EaseInOut, + SceneCursorMotionEasing::EaseIn => E::EaseIn, + SceneCursorMotionEasing::EaseOut => E::EaseOut, + }, + } + } } /// Un sprite de curseur : image + point de pivot. @@ -611,4 +692,45 @@ mod annotation_tests { vec!["keep"] ); } + + /// Le contrat JS→Rust, dans les deux sens où il dérive en silence : le camelCase des + /// champs et l'orthographe des variantes. `easing` est en kebab-case (`ease-in-out`) + /// alors que `preset` est en minuscules collées — deux conventions dans le même objet, + /// donc exactement le genre de détail qu'un test doit tenir. + #[test] + fn a_cursor_motion_region_parses_from_the_typescript_shape() { + let json = r#"{ + "show": true, "size": 1.0, "smoothing": 0.5, "motionBlur": 0.0, + "clickBounce": 1.0, "clipToBounds": false, "theme": "system", + "motion": [{ + "id": "r1", + "startSec": 1.0, "endSec": 2.5, + "startPoint": { "cx": 0.1, "cy": 0.2 }, + "endPoint": { "cx": 0.8, "cy": 0.4 }, + "controlPoint": { "cx": 0.5, "cy": 0.9 }, + "preset": "overshoot", "cycles": 3, "speed": 2.5, + "easing": "ease-in-out" + }] + }"#; + let cursor: SceneCursor = serde_json::from_str(json).expect("SceneCursor doit parser"); + assert_eq!(cursor.motion.len(), 1); + let r = &cursor.motion[0]; + assert_eq!(r.preset, SceneCursorMotionPreset::Overshoot); + assert_eq!(r.easing, SceneCursorMotionEasing::EaseInOut); + assert_eq!(r.start_sec, 1.0); + assert_eq!(r.control_point.cy, 0.9); + assert_eq!(r.cycles, 3); + } + + /// Une scène d'avant ce champ doit rester lisible : `motion` absent → aucune région, + /// pas une erreur de parse. C'est le cas de tout projet existant. + #[test] + fn a_cursor_without_motion_still_parses() { + let json = r#"{ + "show": true, "size": 1.0, "smoothing": 0.0, "motionBlur": 0.0, + "clickBounce": 1.0, "clipToBounds": false, "theme": "system" + }"#; + let cursor: SceneCursor = serde_json::from_str(json).expect("scène héritée"); + assert!(cursor.motion.is_empty()); + } } diff --git a/crates/compositor/src/timeline_walk.rs b/crates/compositor/src/timeline_walk.rs index e5146e6cd..7e0dfe1c9 100644 --- a/crates/compositor/src/timeline_walk.rs +++ b/crates/compositor/src/timeline_walk.rs @@ -73,6 +73,10 @@ pub(crate) unsafe fn walk_composited_timeline( ) -> Result { let cursor_enabled = scene.as_ref().map(|s| s.cursor.show).unwrap_or(false); let cursor_smoothing = scene.as_ref().map(|s| s.cursor.smoothing).unwrap_or(0.0); + // Régions éditées converties une fois : elles sont les mêmes pour tous les clips, et la + // conversion n'a pas à être refaite à chaque piste chargée. + let cursor_motion: Vec = + scene.as_ref().map(|s| s.cursor.motion.iter().map(Into::into).collect()).unwrap_or_default(); let mut cursor_tracks: HashMap = HashMap::new(); let mut cursor_active_path: Option = None; @@ -161,7 +165,12 @@ pub(crate) unsafe fn walk_composited_timeline( if !cursor_tracks.contains_key(&clip.screen) { let path = format!("{}.cursor.json", clip.screen); if let Ok(raw) = CursorTrack::load(&path, 0.0, 24.0 * 3600.0) { - cursor_tracks.insert(clip.screen.clone(), raw.smoothed(cursor_smoothing)); + // Trajectoire éditée d'ABORD, lissage ensuite : le preset définit le tracé, + // le lissage est un filtre de rendu qui s'applique à celui qu'on a. Dans + // l'autre ordre les portions éditées resteraient nettes au milieu d'une + // piste amortie, et le slider n'aurait plus d'effet sur elles. + let edited = raw.with_motion(&cursor_motion); + cursor_tracks.insert(clip.screen.clone(), edited.smoothed(cursor_smoothing)); } // absente/illisible → pas d'entrée : ce clip s'exporte sans curseur (visible, // pas masqué en un curseur fantôme d'un autre clip). diff --git a/src/native/sceneDescription.ts b/src/native/sceneDescription.ts index 4aa13987c..b3bc9caf4 100644 --- a/src/native/sceneDescription.ts +++ b/src/native/sceneDescription.ts @@ -306,6 +306,48 @@ export interface SceneEffects { motionBlur: number; } +/** Shape a cursor motion region draws between its two anchors. `recorded` keeps the + * captured trajectory, i.e. the region is inert — it exists so the editor can hold a + * selection without altering the path. */ +export type SceneCursorMotionPreset = + | "recorded" + | "straight" + | "arc" + | "wave" + | "loop" + | "overshoot"; + +export type SceneCursorMotionEasing = "linear" | "ease-in-out" | "ease-in" | "ease-out"; + +/** One editable stretch of cursor trajectory, overriding the recorded telemetry between + * `startSec` and `endSec`. + * + * Anchors arrive RESOLVED: the editor owns anchor discovery (rests, clicks, manual + * splits) and hands over absolute normalised points, so the compositor never has to + * re-derive them from telemetry. That keeps this contract a pure geometry description — + * the same list yields the same path whatever produced it. + * + * `controlPoint` is absolute too, not an offset: the sampler takes its displacement from + * the start/end midpoint, so a control point left at the midpoint is a no-op for every + * preset. */ +export interface SceneCursorMotionRegion { + id: string; + startSec: number; + endSec: number; + /** Normalised screen-frame position the region starts from. */ + startPoint: { cx: number; cy: number }; + endPoint: { cx: number; cy: number }; + /** Absolute normalised point steering the curve; midpoint = straight line. */ + controlPoint: { cx: number; cy: number }; + preset: SceneCursorMotionPreset; + /** Oscillations for `wave` and `loop`, 1..6. Ignored by the other presets. */ + cycles: number; + /** 1..4. Higher settles near the destination sooner; it reshapes progress, it does + * not retime the region. */ + speed: number; + easing: SceneCursorMotionEasing; +} + /** Cursor rendering, from the editor settings. */ export interface SceneCursor { show: boolean; @@ -318,6 +360,10 @@ export interface SceneCursor { clipToBounds: boolean; /** Cursor theme id (sprite set). */ theme: string; + /** Editable motion regions, in timeline order. Absent or empty means the recorded + * trajectory plays untouched — which is what every project without the cursor + * choreography editor produces. */ + motion?: SceneCursorMotionRegion[]; } /** Everything native needs to compose the scene, serialized from one document. */ From 00d7018fad57b5b216367ca1be3b192691b33652 Mon Sep 17 00:00:00 2001 From: YoneRai12 <183966348+YoneRai12@users.noreply.github.com> Date: Sat, 1 Aug 2026 15:24:45 +0200 Subject: [PATCH 2/5] feat(cursor): the editable cursor motion model Lifted verbatim from #116, which cannot be rebased: its base is 428 commits behind and the first commit alone conflicts on 56 files. The module itself needs none of that history -- it lands on main untouched, with its 17 tests passing and no new type errors. This is the pure model: presets, easing, speed, anchor kinds, rest and click detection, and the sampling that turns a region into a position. No rendering, no editor, no dependency on anything the 1.8.0 merge removed. Taken from #116 rather than #113. The two carry different lineages of this module, not an old and a new one: #116's knows about source time and crop projection, which is what its editor and preview call into. #113's is the shape the deleted web editor wanted. Not wired to anything yet. The timeline lane and inspector controls need an `AxcutCursorMotionRegion` on the document schema first, and #116's preview layer is built on Pixi, which main dropped when the preview moved to the native compositor -- that part needs rebuilding on the native overlay rather than porting. The compositor already samples these presets for preview and export (see `feat/cursor-motion-contract`), so the rendering half is done. Co-authored-by: Etienne Lescot Refs #116, #113 --- src/lib/cursor/cursorMotion.test.ts | 330 ++++++++++++++++ src/lib/cursor/cursorMotion.ts | 561 ++++++++++++++++++++++++++++ 2 files changed, 891 insertions(+) create mode 100644 src/lib/cursor/cursorMotion.test.ts create mode 100644 src/lib/cursor/cursorMotion.ts diff --git a/src/lib/cursor/cursorMotion.test.ts b/src/lib/cursor/cursorMotion.test.ts new file mode 100644 index 000000000..2c1dc2138 --- /dev/null +++ b/src/lib/cursor/cursorMotion.test.ts @@ -0,0 +1,330 @@ +import { describe, expect, it } from "vitest"; +import { + applyCursorMotionSpeed, + buildCursorMotionRegionDrafts, + type CursorMotionPath, + type CursorMotionPreset, + type CursorMotionRegion, + clampCursorMotionCycles, + clampCursorMotionSpeed, + findCursorMotionRegionAtSourceTime, + projectCursorMotionPointToCrop, + sampleCursorMotion, + sampleCursorMotionRegion, + unprojectCursorMotionPointFromCrop, +} from "./cursorMotion"; + +const owner = { clipId: "clip-1", assetId: "asset-1" }; +const start = { cx: 0.1, cy: 0.5 }; +const end = { cx: 0.8, cy: 0.5 }; + +function region(overrides: Partial = {}): CursorMotionRegion { + return { + id: "motion-1", + ...owner, + startMs: 2000, + endMs: 3000, + sourceStartMs: 100, + sourceEndMs: 1100, + startPoint: start, + endPoint: end, + controlPoints: [{ cx: 0.45, cy: 0.2 }], + startAnchor: "manual", + endAnchor: "click", + segmentKind: "move", + preset: "arc", + speed: 1, + cycles: 2, + easing: "linear", + ...overrides, + }; +} + +function linearPath(): CursorMotionPath { + return { + sampleAtSourceTime(sourceTimeMs) { + const progress = Math.max(0, Math.min(1, (sourceTimeMs - 100) / 1000)); + return { + cx: start.cx + (end.cx - start.cx) * progress, + cy: start.cy, + }; + }, + }; +} + +describe("cursor motion sampling", () => { + it("preserves identity crop coordinates by reference", () => { + const point = { cx: 0.25, cy: 0.75 }; + expect(projectCursorMotionPointToCrop(point, { x: 0, y: 0, width: 1, height: 1 })).toBe(point); + }); + + it("projects crop coordinates and keeps inclusive edges stable", () => { + const crop = { x: 0.2, y: 0.25, width: 0.5, height: 0.5 }; + expect(projectCursorMotionPointToCrop({ cx: 0.45, cy: 0.5 }, crop)).toEqual({ + cx: 0.5, + cy: 0.5, + }); + expect(projectCursorMotionPointToCrop({ cx: 0.2, cy: 0.25 }, crop)).toEqual({ + cx: 0, + cy: 0, + }); + expect(projectCursorMotionPointToCrop({ cx: 0.7, cy: 0.75 }, crop)).toEqual({ + cx: 1, + cy: 1, + }); + }); + + it("hides points outside the crop and rejects invalid crop spans", () => { + const crop = { x: 0.2, y: 0.25, width: 0.5, height: 0.5 }; + expect(projectCursorMotionPointToCrop({ cx: 0.199, cy: 0.5 }, crop)).toBeNull(); + expect(projectCursorMotionPointToCrop({ cx: 0.45, cy: 0.751 }, crop)).toBeNull(); + expect(projectCursorMotionPointToCrop({ cx: 0.45, cy: 0.5 }, { ...crop, width: 0 })).toBeNull(); + }); + + it("maps dragged crop-space controls back to source normalized coordinates", () => { + const crop = { x: 0.2, y: 0.25, width: 0.5, height: 0.5 }; + expect(unprojectCursorMotionPointFromCrop({ cx: 0.5, cy: 0.5 }, crop)).toEqual({ + cx: 0.45, + cy: 0.5, + }); + expect(unprojectCursorMotionPointFromCrop({ cx: -1, cy: 2 }, crop)).toEqual({ + cx: 0.2, + cy: 0.75, + }); + }); + + it("keeps every preset's normalized endpoints exact", () => { + const presets: CursorMotionPreset[] = [ + "recorded", + "straight", + "arc", + "wave", + "loop", + "overshoot", + ]; + for (const preset of presets) { + const motion = region({ preset, speed: 4, cycles: 6, easing: "ease-in-out" }); + expect(sampleCursorMotionRegion(motion, motion.sourceStartMs)).toBe(start); + expect(sampleCursorMotionRegion(motion, motion.sourceEndMs)).toBe(end); + } + }); + + it("preserves the recorded sample by reference until a creative preset is selected", () => { + const recorded = { cx: 0.37, cy: 0.61 }; + const path: CursorMotionPath = { sampleAtSourceTime: () => recorded }; + const result = sampleCursorMotion({ + path, + regions: [region({ preset: "recorded", speed: 4 })], + owner, + sourceTimeMs: 400, + }); + expect(result).toBe(recorded); + }); + + it("clamps and rounds speed, and accelerates without a frozen prefix", () => { + expect(clampCursorMotionSpeed(Number.NaN)).toBe(1); + expect(clampCursorMotionSpeed(Number.POSITIVE_INFINITY)).toBe(1); + expect(clampCursorMotionSpeed(0.2)).toBe(1); + expect(clampCursorMotionSpeed(2.26)).toBe(2.3); + expect(clampCursorMotionSpeed(99)).toBe(4); + expect(applyCursorMotionSpeed(0.25, 2)).toBeCloseTo(0.4375); + expect(applyCursorMotionSpeed(0, 4)).toBe(0); + expect(applyCursorMotionSpeed(1, 4)).toBe(1); + }); + + it("clamps wave and loop cycles to whole values from one through six", () => { + expect(clampCursorMotionCycles(Number.NaN)).toBe(1); + expect(clampCursorMotionCycles(0)).toBe(1); + expect(clampCursorMotionCycles(3.6)).toBe(4); + expect(clampCursorMotionCycles(20)).toBe(6); + }); + + it("uses a source-time half-open interval at adjacent boundaries", () => { + const first = region({ id: "first", sourceStartMs: 100, sourceEndMs: 500 }); + const second = region({ + id: "second", + sourceStartMs: 500, + sourceEndMs: 900, + startPoint: { cx: 0.45, cy: 0.5 }, + preset: "straight", + }); + expect(findCursorMotionRegionAtSourceTime([first, second], owner, 499.999)?.id).toBe("first"); + expect(findCursorMotionRegionAtSourceTime([first, second], owner, 500)?.id).toBe("second"); + expect(findCursorMotionRegionAtSourceTime([first, second], owner, 900)).toBeNull(); + }); + + it("does not apply a region owned by another clip or asset", () => { + const recorded = { cx: 0.42, cy: 0.42 }; + const path: CursorMotionPath = { sampleAtSourceTime: () => recorded }; + const motion = region({ preset: "wave" }); + expect( + sampleCursorMotion({ + path, + regions: [motion], + owner: { clipId: "clip-2", assetId: "asset-1" }, + sourceTimeMs: 500, + }), + ).toBe(recorded); + expect( + sampleCursorMotion({ + path, + regions: [motion], + owner: { clipId: "clip-1", assetId: "asset-2" }, + sourceTimeMs: 500, + }), + ).toBe(recorded); + }); + + it("produces distinct arc, wave, loop, and overshoot shapes", () => { + const arc = sampleCursorMotionRegion(region({ preset: "arc" }), 600); + const wave = sampleCursorMotionRegion(region({ preset: "wave" }), 300); + const loop = sampleCursorMotionRegion(region({ preset: "loop" }), 475); + const overshoot = Array.from({ length: 80 }, (_, index) => + sampleCursorMotionRegion( + region({ preset: "overshoot", controlPoints: [{ cx: 0.45, cy: 0.5 }] }), + 100 + (1000 * index) / 79, + ), + ); + expect(arc.cy).toBeLessThan(0.5); + expect(wave.cy).not.toBeCloseTo(0.5); + expect(loop.cy).not.toBeCloseTo(0.5); + expect(overshoot.some((point) => point.cx > end.cx)).toBe(true); + }); +}); + +describe("cursor motion draft builder", () => { + it("builds only through the next click and keeps virtual and source time separate", () => { + const drafts = buildCursorMotionRegionDrafts({ + owner, + currentSourceTimeMs: 500, + currentVirtualTimeMs: 4500, + clipSourceEndMs: 2000, + path: linearPath(), + samples: [ + { timeMs: 500, cx: 0.38, cy: 0.5 }, + { timeMs: 900, cx: 0.66, cy: 0.5, interactionType: "click" }, + { timeMs: 1500, cx: 0.8, cy: 0.5, interactionType: "click" }, + ], + }); + + expect(drafts).toHaveLength(1); + expect(drafts[0]).toMatchObject({ + ...owner, + startMs: 4500, + endMs: 4900, + sourceStartMs: 500, + sourceEndMs: 900, + startAnchor: "manual", + endAnchor: "click", + segmentKind: "move", + preset: "recorded", + speed: 1, + easing: "ease-in-out", + }); + }); + + it("splits a recorded stop into move, hold, and move drafts", () => { + const drafts = buildCursorMotionRegionDrafts({ + owner, + currentSourceTimeMs: 0, + currentVirtualTimeMs: 2000, + clipSourceEndMs: 1000, + samples: [ + { timeMs: 0, cx: 0.1, cy: 0.5 }, + { timeMs: 100, cx: 0.25, cy: 0.5 }, + { timeMs: 200, cx: 0.4, cy: 0.5 }, + { timeMs: 350, cx: 0.4, cy: 0.5 }, + { timeMs: 500, cx: 0.401, cy: 0.5 }, + { timeMs: 650, cx: 0.4, cy: 0.5 }, + { timeMs: 800, cx: 0.7, cy: 0.5 }, + { timeMs: 1000, cx: 0.9, cy: 0.5, interactionType: "click" }, + ], + }); + + expect( + drafts.map((draft) => [ + draft.sourceStartMs, + draft.sourceEndMs, + draft.segmentKind, + draft.startAnchor, + draft.endAnchor, + ]), + ).toEqual([ + [0, 200, "move", "manual", "rest"], + [200, 650, "hold", "rest", "rest"], + [650, 1000, "move", "rest", "click"], + ]); + expect(drafts.every((draft) => draft.preset === "recorded" && draft.speed === 1)).toBe(true); + }); + + it("does not treat a telemetry gap longer than 150ms as a stop", () => { + const drafts = buildCursorMotionRegionDrafts({ + owner, + currentSourceTimeMs: 0, + currentVirtualTimeMs: 0, + clipSourceEndMs: 1000, + samples: [ + { timeMs: 0, cx: 0.1, cy: 0.5 }, + { timeMs: 100, cx: 0.4, cy: 0.5 }, + { timeMs: 700, cx: 0.4, cy: 0.5 }, + { timeMs: 1000, cx: 0.8, cy: 0.5, interactionType: "click" }, + ], + }); + + expect(drafts).toHaveLength(1); + expect(drafts[0].segmentKind).toBe("move"); + }); + + it("does not confuse a native cursor atlas asset id with media ownership", () => { + const activeSamples = [ + { timeMs: 0, cx: 0.1, cy: 0.5, assetId: "cursor-arrow" }, + { + timeMs: 800, + cx: 0.7, + cy: 0.5, + interactionType: "click", + assetId: "cursor-pointer", + }, + ]; + const drafts = buildCursorMotionRegionDrafts({ + owner, + currentSourceTimeMs: 0, + currentVirtualTimeMs: 0, + clipSourceEndMs: 1000, + samples: activeSamples, + }); + + expect(drafts).toHaveLength(1); + expect(drafts[0].sourceEndMs).toBe(800); + }); + + it("returns no draft when the active clip has no following click", () => { + expect( + buildCursorMotionRegionDrafts({ + owner, + currentSourceTimeMs: 500, + currentVirtualTimeMs: 2500, + clipSourceEndMs: 1000, + samples: [ + { timeMs: 500, cx: 0.4, cy: 0.5, interactionType: "click" }, + { timeMs: 800, cx: 0.7, cy: 0.5 }, + ], + }), + ).toEqual([]); + }); + + it("does not create a microscopic segment for anchors within one millisecond", () => { + expect( + buildCursorMotionRegionDrafts({ + owner, + currentSourceTimeMs: 0, + currentVirtualTimeMs: 0, + clipSourceEndMs: 100, + samples: [ + { timeMs: 0, cx: 0.4, cy: 0.5 }, + { timeMs: 0.5, cx: 0.4, cy: 0.5, interactionType: "click" }, + ], + }), + ).toEqual([]); + }); +}); diff --git a/src/lib/cursor/cursorMotion.ts b/src/lib/cursor/cursorMotion.ts new file mode 100644 index 000000000..7137b3f13 --- /dev/null +++ b/src/lib/cursor/cursorMotion.ts @@ -0,0 +1,561 @@ +export const CURSOR_MOTION_PRESETS = [ + "recorded", + "straight", + "arc", + "wave", + "loop", + "overshoot", +] as const; + +export type CursorMotionPreset = (typeof CURSOR_MOTION_PRESETS)[number]; + +export const CURSOR_MOTION_EASINGS = ["linear", "ease-in", "ease-out", "ease-in-out"] as const; + +export type CursorMotionEasing = (typeof CURSOR_MOTION_EASINGS)[number]; + +export const CURSOR_MOTION_ANCHORS = ["manual", "rest", "click"] as const; + +export type CursorMotionAnchor = (typeof CURSOR_MOTION_ANCHORS)[number]; + +export const CURSOR_MOTION_SEGMENT_KINDS = ["move", "hold"] as const; + +export type CursorMotionSegmentKind = (typeof CURSOR_MOTION_SEGMENT_KINDS)[number]; + +export interface CursorMotionPoint { + cx: number; + cy: number; +} + +export interface CursorMotionCropRegion { + x: number; + y: number; + width: number; + height: number; +} + +export interface CursorMotionOwner { + clipId: string; + assetId: string; +} + +export interface CursorMotionTelemetrySample extends CursorMotionPoint { + timeMs: number; + visible?: boolean; + interactionType?: string | null; +} + +export interface CursorMotionPath { + sampleAtSourceTime(sourceTimeMs: number): CursorMotionPoint | null; +} + +export interface CursorMotionRegion extends CursorMotionOwner { + id: string; + startMs: number; + endMs: number; + sourceStartMs: number; + sourceEndMs: number; + startPoint: CursorMotionPoint; + endPoint: CursorMotionPoint; + controlPoints: CursorMotionPoint[]; + startAnchor: CursorMotionAnchor; + endAnchor: CursorMotionAnchor; + segmentKind: CursorMotionSegmentKind; + preset: CursorMotionPreset; + speed: number; + cycles: number; + easing: CursorMotionEasing; +} + +export type CursorMotionRegionDraft = Omit; + +export const CURSOR_MOTION_SPEED_MIN = 1; +export const CURSOR_MOTION_SPEED_MAX = 4; +export const DEFAULT_CURSOR_MOTION_SPEED = 1; +export const CURSOR_MOTION_CYCLES_MIN = 1; +export const CURSOR_MOTION_CYCLES_MAX = 6; + +const REST_MIN_DURATION_MS = 300; +const REST_MAX_DIAMETER = 0.009; +const REST_MAX_SAMPLE_GAP_MS = 150; +const MIN_SEGMENT_DURATION_MS = 2; + +function clamp(value: number, min: number, max: number): number { + return Math.min(max, Math.max(min, value)); +} + +export function clampCursorMotionPoint(point: CursorMotionPoint): CursorMotionPoint { + return { + cx: clamp(Number.isFinite(point.cx) ? point.cx : 0.5, 0, 1), + cy: clamp(Number.isFinite(point.cy) ? point.cy : 0.5, 0, 1), + }; +} + +export function projectCursorMotionPointToCrop( + point: CursorMotionPoint, + crop: CursorMotionCropRegion, +): CursorMotionPoint | null { + if ( + !Number.isFinite(point.cx) || + !Number.isFinite(point.cy) || + !Number.isFinite(crop.x) || + !Number.isFinite(crop.y) || + !Number.isFinite(crop.width) || + !Number.isFinite(crop.height) || + crop.width <= 0 || + crop.height <= 0 + ) { + return null; + } + const right = crop.x + crop.width; + const bottom = crop.y + crop.height; + if (point.cx < crop.x || point.cx > right || point.cy < crop.y || point.cy > bottom) { + return null; + } + if (crop.x === 0 && crop.y === 0 && crop.width === 1 && crop.height === 1) return point; + return { + cx: point.cx === crop.x ? 0 : point.cx === right ? 1 : (point.cx - crop.x) / crop.width, + cy: point.cy === crop.y ? 0 : point.cy === bottom ? 1 : (point.cy - crop.y) / crop.height, + }; +} + +export function unprojectCursorMotionPointFromCrop( + point: CursorMotionPoint, + crop: CursorMotionCropRegion, +): CursorMotionPoint | null { + if ( + !Number.isFinite(point.cx) || + !Number.isFinite(point.cy) || + !Number.isFinite(crop.x) || + !Number.isFinite(crop.y) || + !Number.isFinite(crop.width) || + !Number.isFinite(crop.height) || + crop.width <= 0 || + crop.height <= 0 + ) { + return null; + } + if (crop.x === 0 && crop.y === 0 && crop.width === 1 && crop.height === 1) { + return clampCursorMotionPoint(point); + } + return clampCursorMotionPoint({ + cx: crop.x + clamp(point.cx, 0, 1) * crop.width, + cy: crop.y + clamp(point.cy, 0, 1) * crop.height, + }); +} + +export function clampCursorMotionSpeed(speed: number | null | undefined): number { + if (!Number.isFinite(speed)) return DEFAULT_CURSOR_MOTION_SPEED; + return ( + Math.round(clamp(Number(speed), CURSOR_MOTION_SPEED_MIN, CURSOR_MOTION_SPEED_MAX) * 10) / 10 + ); +} + +export function clampCursorMotionCycles(cycles: number | null | undefined): number { + if (!Number.isFinite(cycles)) return CURSOR_MOTION_CYCLES_MIN; + return clamp(Math.round(Number(cycles)), CURSOR_MOTION_CYCLES_MIN, CURSOR_MOTION_CYCLES_MAX); +} + +export function applyCursorMotionSpeed(progress: number, speed: number | null | undefined): number { + const t = clamp(progress, 0, 1); + return 1 - (1 - t) ** clampCursorMotionSpeed(speed); +} + +function applyCursorMotionEasing(progress: number, easing: CursorMotionEasing): number { + const t = clamp(progress, 0, 1); + switch (easing) { + case "ease-in": + return t ** 3; + case "ease-out": + return 1 - (1 - t) ** 3; + case "ease-in-out": + return t < 0.5 ? 4 * t ** 3 : 1 - (-2 * t + 2) ** 3 / 2; + default: + return t; + } +} + +function lerp(a: number, b: number, progress: number): number { + return a + (b - a) * progress; +} + +function cubicBezier( + start: number, + firstControl: number, + secondControl: number, + end: number, + progress: number, +): number { + const inverse = 1 - progress; + return ( + inverse ** 3 * start + + 3 * inverse ** 2 * progress * firstControl + + 3 * inverse * progress ** 2 * secondControl + + progress ** 3 * end + ); +} + +function easeOutBack(progress: number): number { + const overshoot = 1.70158; + const t = clamp(progress, 0, 1) - 1; + return 1 + (overshoot + 1) * t ** 3 + overshoot * t ** 2; +} + +export function createDefaultCursorMotionControlPoint( + start: CursorMotionPoint, + end: CursorMotionPoint, +): CursorMotionPoint { + const dx = end.cx - start.cx; + const dy = end.cy - start.cy; + const distance = Math.hypot(dx, dy); + const normalX = distance > 0.0001 ? dy / distance : 0; + const normalY = distance > 0.0001 ? -dx / distance : -1; + const amplitude = clamp(distance * 0.35, 0.06, 0.18); + return clampCursorMotionPoint({ + cx: (start.cx + end.cx) / 2 + normalX * amplitude, + cy: (start.cy + end.cy) / 2 + normalY * amplitude, + }); +} + +function cursorMotionControlOffset(region: CursorMotionRegion): CursorMotionPoint { + const fallback = createDefaultCursorMotionControlPoint(region.startPoint, region.endPoint); + const controls = region.controlPoints.length > 0 ? region.controlPoints : [fallback]; + const control = controls.reduce( + (sum, point) => ({ cx: sum.cx + point.cx, cy: sum.cy + point.cy }), + { cx: 0, cy: 0 }, + ); + const midpoint = { + cx: (region.startPoint.cx + region.endPoint.cx) / 2, + cy: (region.startPoint.cy + region.endPoint.cy) / 2, + }; + return { + cx: control.cx / controls.length - midpoint.cx, + cy: control.cy / controls.length - midpoint.cy, + }; +} + +export function sampleCursorMotionRegion( + region: CursorMotionRegion, + sourceTimeMs: number, +): CursorMotionPoint { + const durationMs = Math.max(Number.EPSILON, region.sourceEndMs - region.sourceStartMs); + const rawProgress = clamp((sourceTimeMs - region.sourceStartMs) / durationMs, 0, 1); + if (rawProgress === 0) return region.startPoint; + if (rawProgress === 1) return region.endPoint; + + const motionProgress = applyCursorMotionSpeed(rawProgress, region.speed); + const progress = applyCursorMotionEasing(motionProgress, region.easing); + const offset = cursorMotionControlOffset(region); + const base = { + cx: lerp(region.startPoint.cx, region.endPoint.cx, progress), + cy: lerp(region.startPoint.cy, region.endPoint.cy, progress), + }; + const envelope = Math.sin(Math.PI * motionProgress); + const cycles = clampCursorMotionCycles(region.cycles); + + let point: CursorMotionPoint; + switch (region.preset) { + case "arc": { + const first = region.controlPoints[0]; + const second = region.controlPoints[1]; + point = + first && second + ? { + cx: cubicBezier( + region.startPoint.cx, + first.cx, + second.cx, + region.endPoint.cx, + progress, + ), + cy: cubicBezier( + region.startPoint.cy, + first.cy, + second.cy, + region.endPoint.cy, + progress, + ), + } + : { cx: base.cx + offset.cx * envelope, cy: base.cy + offset.cy * envelope }; + break; + } + case "wave": { + const wave = Math.sin(Math.PI * 2 * cycles * motionProgress) * envelope; + point = { cx: base.cx + offset.cx * wave, cy: base.cy + offset.cy * wave }; + break; + } + case "loop": { + const phase = Math.PI * 2 * cycles * motionProgress; + const tangentOffset = Math.sin(phase) * envelope; + const normalOffset = ((1 - Math.cos(phase)) / 2) * envelope; + point = { + cx: base.cx + offset.cx * tangentOffset - offset.cy * normalOffset, + cy: base.cy + offset.cy * tangentOffset + offset.cx * normalOffset, + }; + break; + } + case "overshoot": { + const overshootProgress = easeOutBack(progress); + point = { + cx: + lerp(region.startPoint.cx, region.endPoint.cx, overshootProgress) + + offset.cx * envelope * 0.35, + cy: + lerp(region.startPoint.cy, region.endPoint.cy, overshootProgress) + + offset.cy * envelope * 0.35, + }; + break; + } + default: + point = base; + } + return clampCursorMotionPoint(point); +} + +export function findCursorMotionRegionAtSourceTime( + regions: readonly CursorMotionRegion[], + owner: CursorMotionOwner, + sourceTimeMs: number, +): CursorMotionRegion | null { + for (let index = regions.length - 1; index >= 0; index -= 1) { + const region = regions[index]; + if ( + region.clipId === owner.clipId && + region.assetId === owner.assetId && + sourceTimeMs >= region.sourceStartMs && + sourceTimeMs < region.sourceEndMs + ) { + return region; + } + } + return null; +} + +export function sampleCursorMotion(options: { + path: CursorMotionPath | null | undefined; + regions: readonly CursorMotionRegion[]; + owner: CursorMotionOwner; + sourceTimeMs: number; +}): CursorMotionPoint | null { + const recorded = options.path?.sampleAtSourceTime(options.sourceTimeMs) ?? null; + if (!recorded) return null; + const region = findCursorMotionRegionAtSourceTime( + options.regions, + options.owner, + options.sourceTimeMs, + ); + if (!region || region.preset === "recorded") return recorded; + return sampleCursorMotionRegion(region, options.sourceTimeMs); +} + +interface CursorMotionRestSpan { + startMs: number; + endMs: number; + point: CursorMotionPoint; +} + +interface CursorMotionBuilderAnchor { + timeMs: number; + point: CursorMotionPoint; + kind: CursorMotionAnchor; +} + +function isClickInteraction(interactionType: string | null | undefined): boolean { + return ( + interactionType === "click" || + interactionType === "double-click" || + interactionType === "right-click" || + interactionType === "middle-click" + ); +} + +function normalizeTelemetrySamples( + samples: readonly CursorMotionTelemetrySample[], +): CursorMotionTelemetrySample[] { + return samples + .filter( + (sample) => + sample.visible !== false && + Number.isFinite(sample.timeMs) && + Number.isFinite(sample.cx) && + Number.isFinite(sample.cy), + ) + .map((sample) => ({ ...sample, ...clampCursorMotionPoint(sample) })) + .sort((a, b) => a.timeMs - b.timeMs); +} + +function detectCursorMotionRestSpans( + samples: readonly CursorMotionTelemetrySample[], +): CursorMotionRestSpan[] { + const rests: CursorMotionRestSpan[] = []; + let startIndex = 0; + while (startIndex < samples.length - 1) { + let endIndex = startIndex + 1; + let minCx = samples[startIndex].cx; + let maxCx = minCx; + let minCy = samples[startIndex].cy; + let maxCy = minCy; + while (endIndex < samples.length) { + const sample = samples[endIndex]; + if (sample.timeMs - samples[endIndex - 1].timeMs > REST_MAX_SAMPLE_GAP_MS) break; + const nextMinCx = Math.min(minCx, sample.cx); + const nextMaxCx = Math.max(maxCx, sample.cx); + const nextMinCy = Math.min(minCy, sample.cy); + const nextMaxCy = Math.max(maxCy, sample.cy); + if (Math.hypot(nextMaxCx - nextMinCx, nextMaxCy - nextMinCy) > REST_MAX_DIAMETER) { + break; + } + minCx = nextMinCx; + maxCx = nextMaxCx; + minCy = nextMinCy; + maxCy = nextMaxCy; + endIndex += 1; + } + + const run = samples.slice(startIndex, endIndex); + if (run.at(-1)!.timeMs - run[0].timeMs >= REST_MIN_DURATION_MS) { + const click = run.find((sample) => isClickInteraction(sample.interactionType)); + const point = click + ? clampCursorMotionPoint(click) + : clampCursorMotionPoint({ + cx: run.reduce((total, sample) => total + sample.cx, 0) / run.length, + cy: run.reduce((total, sample) => total + sample.cy, 0) / run.length, + }); + rests.push({ startMs: run[0].timeMs, endMs: run.at(-1)!.timeMs, point }); + startIndex = endIndex; + } else { + startIndex += 1; + } + } + return rests; +} + +function nearestSamplePoint( + samples: readonly CursorMotionTelemetrySample[], + timeMs: number, +): CursorMotionPoint | null { + let nearest: CursorMotionTelemetrySample | null = null; + let distance = Number.POSITIVE_INFINITY; + for (const sample of samples) { + const candidateDistance = Math.abs(sample.timeMs - timeMs); + if (candidateDistance < distance) { + nearest = sample; + distance = candidateDistance; + } + } + return nearest ? clampCursorMotionPoint(nearest) : null; +} + +function anchorPriority(anchor: CursorMotionAnchor): number { + if (anchor === "click") return 3; + if (anchor === "rest") return 2; + return 1; +} + +function normalizeBuilderAnchors( + anchors: CursorMotionBuilderAnchor[], +): CursorMotionBuilderAnchor[] { + const sorted = [...anchors].sort( + (a, b) => a.timeMs - b.timeMs || anchorPriority(b.kind) - anchorPriority(a.kind), + ); + const normalized: CursorMotionBuilderAnchor[] = []; + for (const anchor of sorted) { + const previous = normalized.at(-1); + if (previous && Math.abs(previous.timeMs - anchor.timeMs) <= 1) { + if (anchorPriority(anchor.kind) > anchorPriority(previous.kind)) { + normalized[normalized.length - 1] = anchor; + } + continue; + } + normalized.push(anchor); + } + return normalized; +} + +function getCursorMotionSegmentKind( + start: CursorMotionBuilderAnchor, + end: CursorMotionBuilderAnchor, +): CursorMotionSegmentKind { + if (start.kind === "rest" && end.kind === "rest") return "hold"; + return Math.hypot(end.point.cx - start.point.cx, end.point.cy - start.point.cy) <= + REST_MAX_DIAMETER + ? "hold" + : "move"; +} + +export function buildCursorMotionRegionDrafts(options: { + owner: CursorMotionOwner; + currentSourceTimeMs: number; + currentVirtualTimeMs: number; + clipSourceEndMs: number; + samples: readonly CursorMotionTelemetrySample[]; + path?: CursorMotionPath | null; +}): CursorMotionRegionDraft[] { + if ( + !Number.isFinite(options.currentSourceTimeMs) || + !Number.isFinite(options.currentVirtualTimeMs) || + !Number.isFinite(options.clipSourceEndMs) + ) { + return []; + } + const sourceStartMs = Math.max(0, options.currentSourceTimeMs); + const clipSourceEndMs = Math.max(sourceStartMs, options.clipSourceEndMs); + if (sourceStartMs >= clipSourceEndMs) return []; + + const ownerSamples = normalizeTelemetrySamples(options.samples); + const endClick = ownerSamples.find( + (sample) => + sample.timeMs > sourceStartMs && + sample.timeMs <= clipSourceEndMs && + isClickInteraction(sample.interactionType), + ); + if (!endClick) return []; + + const sourceEndMs = endClick.timeMs; + const samples = ownerSamples.filter( + (sample) => sample.timeMs >= sourceStartMs && sample.timeMs <= sourceEndMs, + ); + const startPoint = + options.path?.sampleAtSourceTime(sourceStartMs) ?? nearestSamplePoint(samples, sourceStartMs); + if (!startPoint) return []; + + const rests = detectCursorMotionRestSpans(samples); + const anchors: CursorMotionBuilderAnchor[] = [ + { timeMs: sourceStartMs, point: clampCursorMotionPoint(startPoint), kind: "manual" }, + { timeMs: sourceEndMs, point: clampCursorMotionPoint(endClick), kind: "click" }, + ]; + for (const rest of rests) { + anchors.push( + { timeMs: rest.startMs, point: rest.point, kind: "rest" }, + { timeMs: rest.endMs, point: rest.point, kind: "rest" }, + ); + } + + const normalizedAnchors = normalizeBuilderAnchors(anchors).filter( + (anchor) => anchor.timeMs >= sourceStartMs && anchor.timeMs <= sourceEndMs, + ); + const drafts: CursorMotionRegionDraft[] = []; + for (let index = 1; index < normalizedAnchors.length; index += 1) { + const start = normalizedAnchors[index - 1]; + const end = normalizedAnchors[index]; + if (end.timeMs - start.timeMs < MIN_SEGMENT_DURATION_MS) continue; + const virtualOffset = options.currentVirtualTimeMs - sourceStartMs; + drafts.push({ + ...options.owner, + startMs: start.timeMs + virtualOffset, + endMs: end.timeMs + virtualOffset, + sourceStartMs: start.timeMs, + sourceEndMs: end.timeMs, + startPoint: start.point, + endPoint: end.point, + controlPoints: [createDefaultCursorMotionControlPoint(start.point, end.point)], + startAnchor: start.kind, + endAnchor: end.kind, + segmentKind: getCursorMotionSegmentKind(start, end), + preset: "recorded", + speed: DEFAULT_CURSOR_MOTION_SPEED, + cycles: CURSOR_MOTION_CYCLES_MIN, + easing: "ease-in-out", + }); + } + return drafts; +} From 51cfab869806a84adc81a7a243dee5145c3d6239 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Mon, 31 Aug 2026 23:00:51 +0200 Subject: [PATCH 3/5] feat(cursor): the choreography lane, its inspector, and the handle on the preview MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds the editor half of #548 on the v4 shell. The model (#116, already on main as `contrib/cursor-choreography`) and the compositor half (`feat/cursor-motion-contract`) were both written; nothing connected them to an editor, so the feature was invisible. The interaction model is @YoneRai12's, from #113, rebuilt rather than ported: every file that PR touches was deleted from main on 1 Aug when the v4 editor became the only editor. Sections split at rests and clicks, `recorded` at 1x by default so creating them changes nothing, speed reshaping progress inside a section that still ends on the recorded click at its original time. Two faults in #113 are fixed rather than reproduced: its panel had no label for the `recorded` preset, so the first button in the grid rendered blank, and its speed help text described a leading pause its own code comment says was reverted for looking broken. `cursorMotionRegions` mirrors `zoomRanges` — a document-level array with the shared clip anchor, additive with a `[]` default, so old projects load unchanged and no schema version bump is needed. What does NOT mirror the zoom lane: these pills never coalesce and never drag. Two touching sections with the same preset are still two sections, and being individually selectable is the feature; their boundaries are anchors the recording placed, not a span to stretch. `sceneDescription` emits the SOURCE span, deliberately not projected the way zoom and annotation regions are: the compositor applies these once to the cursor track, which is already loaded in that clock. Known gap, commented at the site: `Scene.cursor.motion` carries no owner, so in a project whose clips draw on two different recordings the second one's regions land on the first one's track. The fix belongs in the contract. Also unwired: auto-split of an existing section, GIF export, the agent tools and the CLI. Eleven locales carry English placeholders; French is translated. Co-authored-by: YoneRai12 <183966348+YoneRai12@users.noreply.github.com> Co-authored-by: Claude Opus 5 Refs #548, #113, #116 --- .../ai-edition/CursorMotionPathOverlay.tsx | 274 ++++++++++++++ src/components/ai-edition/NewEditorShell.tsx | 16 + src/components/ai-edition/Preview.tsx | 13 + src/components/ai-edition/PreviewCanvas.tsx | 33 ++ .../ai-edition/v4/EditorShellV4.module.css | 14 + .../ai-edition/v4/FloatingInspector.tsx | 345 +++++++++++++++++ .../v4/V4Timeline.geometry.test.tsx | 1 + src/components/ai-edition/v4/V4Timeline.tsx | 162 +++++++- .../v4/V4Timeline.waveform.test.tsx | 1 + src/i18n/locales/ar/settings.json | 32 ++ src/i18n/locales/ar/timeline.json | 24 +- src/i18n/locales/en/settings.json | 32 ++ src/i18n/locales/en/timeline.json | 24 +- src/i18n/locales/es/settings.json | 32 ++ src/i18n/locales/es/timeline.json | 24 +- src/i18n/locales/fr/settings.json | 32 ++ src/i18n/locales/fr/timeline.json | 24 +- src/i18n/locales/it/settings.json | 32 ++ src/i18n/locales/it/timeline.json | 24 +- src/i18n/locales/ja-JP/settings.json | 32 ++ src/i18n/locales/ja-JP/timeline.json | 24 +- src/i18n/locales/ko-KR/settings.json | 32 ++ src/i18n/locales/ko-KR/timeline.json | 24 +- src/i18n/locales/pt-BR/settings.json | 32 ++ src/i18n/locales/pt-BR/timeline.json | 24 +- src/i18n/locales/ru/settings.json | 32 ++ src/i18n/locales/ru/timeline.json | 24 +- src/i18n/locales/tr/settings.json | 32 ++ src/i18n/locales/tr/timeline.json | 24 +- src/i18n/locales/vi/settings.json | 32 ++ src/i18n/locales/vi/timeline.json | 24 +- src/i18n/locales/zh-CN/settings.json | 32 ++ src/i18n/locales/zh-CN/timeline.json | 24 +- src/i18n/locales/zh-TW/settings.json | 32 ++ src/i18n/locales/zh-TW/timeline.json | 24 +- src/lib/ai-edition/document/timeline.ts | 18 +- src/lib/ai-edition/schema/index.ts | 60 +++ .../store/documentWriteAudit.test.ts | 21 ++ .../store/useTimeline.cursorMotion.test.ts | 348 ++++++++++++++++++ src/lib/ai-edition/store/useTimeline.ts | 257 ++++++++++++- .../timeline/cursorMotionRegions.test.ts | 100 +++++ .../timeline/cursorMotionRegions.ts | 69 ++++ src/native/sceneDescription.test.ts | 60 +++ src/native/sceneDescription.ts | 30 ++ 44 files changed, 2505 insertions(+), 45 deletions(-) create mode 100644 src/components/ai-edition/CursorMotionPathOverlay.tsx create mode 100644 src/lib/ai-edition/store/useTimeline.cursorMotion.test.ts create mode 100644 src/lib/ai-edition/timeline/cursorMotionRegions.test.ts create mode 100644 src/lib/ai-edition/timeline/cursorMotionRegions.ts diff --git a/src/components/ai-edition/CursorMotionPathOverlay.tsx b/src/components/ai-edition/CursorMotionPathOverlay.tsx new file mode 100644 index 000000000..fa8ce1cc8 --- /dev/null +++ b/src/components/ai-edition/CursorMotionPathOverlay.tsx @@ -0,0 +1,274 @@ +// The edited cursor path drawn over the preview, with the one handle that shapes +// it. Sibling of `ZoomFocusOverlay` and built the same way — a plain CSS/SVG layer +// over the screen stage, not a Pixi one; #116's version was built on Pixi, which +// `main` dropped when the preview moved to the native compositor. +// +// Two paths are drawn, and the pair is the point: the recorded trajectory says +// where the cursor actually went, the edited one says where it will go. A single +// path would leave "is this better than what I recorded" unanswerable without +// scrubbing back and forth. +// +// Coordinates are normalised (0..1) against the SCREEN RECT, which is what the +// motion model stores and what the compositor samples, so nothing here converts +// between spaces. The viewBox is 0..100 with `preserveAspectRatio="none"`: the +// overlay stretches with the stage, and every stroke carries +// `vector-effect="non-scaling-stroke"` so that stretch never thickens a line. + +import type { PointerEvent as ReactPointerEvent } from "react"; +import { useCallback, useMemo, useRef } from "react"; +import type { AxcutCursorMotionRegion } from "@/lib/ai-edition/schema"; +import { toModelCursorMotionRegion } from "@/lib/ai-edition/timeline/cursorMotionRegions"; +import type { CursorMotionPoint } from "@/lib/cursor/cursorMotion"; +import { sampleCursorMotionRegion } from "@/lib/cursor/cursorMotion"; +import { clamp01 } from "@/utils/math"; + +/** Enough segments that an arc reads as a curve at any stage size, few enough that + * re-sampling on every drag frame stays free. The compositor re-samples the same + * curve at 240 Hz; this is only what the eye needs. */ +const PATH_STEPS = 64; + +interface CursorMotionPathOverlayProps { + region: AxcutCursorMotionRegion; + /** Recorded cursor positions for the owning asset, in SOURCE ms. Only the ones + * inside the region's span are drawn. Absent when the recording carries no + * telemetry, which just means the comparison line is absent. + * + * Nullable, not just optional: `useCursorTelemetry` declares an array but hands + * through whatever the bridge returns, and a bridge with no cursor endpoint — + * the browser-mode shim, a platform without the sampler helper — returns null. + * It reached here as a crash on `.filter`, so the null is handled where it + * arrives rather than assumed away. */ + telemetry?: readonly { timeMs: number; cx: number; cy: number }[] | null; + onControlPointChange: (id: string, point: CursorMotionPoint) => void; + onControlPointCommit?: () => void; +} + +function toPolyline(points: readonly CursorMotionPoint[]): string { + return points.map((p) => `${(p.cx * 100).toFixed(3)},${(p.cy * 100).toFixed(3)}`).join(" "); +} + +function AnchorMarker({ + point, + kind, +}: { + point: CursorMotionPoint; + kind: AxcutCursorMotionRegion["startAnchor"]; +}) { + const x = point.cx * 100; + const y = point.cy * 100; + // A rest is a square, a click is a ringed dot, a manual cut is a plain dot. + // Three shapes rather than three colours: the anchors sit on footage, and a + // colour-only distinction disappears over the wrong frame. + if (kind === "rest") { + return ( + + ); + } + return ( + <> + + {kind === "click" ? : null} + + ); +} + +export function CursorMotionPathOverlay({ + region, + telemetry, + onControlPointChange, + onControlPointCommit, +}: CursorMotionPathOverlayProps) { + const hostRef = useRef(null); + const draggingRef = useRef(false); + + const model = useMemo(() => toModelCursorMotionRegion(region), [region]); + + const editedPath = useMemo(() => { + // A hold has no path — its two anchors are the same point, and drawing a + // zero-length line there would put a stray dot on the frame. + if (region.segmentKind === "hold") return []; + const span = model.sourceEndMs - model.sourceStartMs; + return Array.from({ length: PATH_STEPS + 1 }, (_, i) => + sampleCursorMotionRegion(model, model.sourceStartMs + (span * i) / PATH_STEPS), + ); + }, [model, region.segmentKind]); + + const recordedPath = useMemo( + () => + (telemetry ?? []) + .filter((s) => s.timeMs >= model.sourceStartMs && s.timeMs <= model.sourceEndMs) + .map((s) => ({ cx: s.cx, cy: s.cy })), + [telemetry, model.sourceStartMs, model.sourceEndMs], + ); + + const updateFromClientPoint = useCallback( + (clientX: number, clientY: number) => { + const el = hostRef.current; + if (!el) return; + const rect = el.getBoundingClientRect(); + if (!rect.width || !rect.height) return; + onControlPointChange(region.id, { + cx: clamp01((clientX - rect.left) / rect.width), + cy: clamp01((clientY - rect.top) / rect.height), + }); + }, + [onControlPointChange, region.id], + ); + + const onPointerDown = useCallback( + (e: ReactPointerEvent) => { + e.preventDefault(); + e.stopPropagation(); + draggingRef.current = true; + (e.target as Element).setPointerCapture?.(e.pointerId); + updateFromClientPoint(e.clientX, e.clientY); + }, + [updateFromClientPoint], + ); + + const onPointerMove = useCallback( + (e: ReactPointerEvent) => { + if (!draggingRef.current) return; + updateFromClientPoint(e.clientX, e.clientY); + }, + [updateFromClientPoint], + ); + + const endDrag = useCallback(() => { + if (!draggingRef.current) return; + draggingRef.current = false; + onControlPointCommit?.(); + }, [onControlPointCommit]); + + const controlX = region.controlPoint.cx * 100; + const controlY = region.controlPoint.cy * 100; + const shapeable = region.segmentKind === "move" && region.preset !== "recorded"; + + return ( +
+ + {shapeable ? ( + // The handle is a DOM node, not an SVG circle: the overlay stretches with + // `preserveAspectRatio="none"`, which would squash a circle into an + // ellipse on any non-square stage, and a grab target should not change + // shape with the aspect ratio of the footage. +
+ ) : null} +
+ ); +} diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index b371d100c..49d23b1af 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -870,6 +870,16 @@ export function NewEditorShell() { return; } + // A cursor motion region is a path between two RESOLVED points — the cursor's + // actual position at a rest or a recorded click. Pasting it at the playhead + // would draw that path between two places the cursor never was. The inspector's + // "apply to all move sections" is the operation that means what copy would mean + // here, and it copies the styling without the anchors. + if (sel.kind === "cursorMotion") { + toast.info("Cursor motion is tied to its anchors — use Apply to all moves instead"); + return; + } + const source = sel.kind === "zoom" ? tl.zoomRegions @@ -1243,6 +1253,12 @@ export function NewEditorShell() { selectedZoomRegionId={tl.selection?.kind === "zoom" ? tl.selection.id : null} onZoomFocusChange={tl.updateZoomFocusLive} onZoomFocusCommit={() => void tl.commitZoomFocus()} + cursorMotionRegions={tl.cursorMotionRegions} + selectedCursorMotionId={ + tl.selection?.kind === "cursorMotion" ? tl.selection.id : null + } + onCursorMotionControlPointChange={tl.updateCursorMotionControlPointLive} + onCursorMotionControlPointCommit={() => void tl.commitCursorMotionControlPoint()} annotationRegions={tl.annotationRegions} selectedAnnotationId={ tl.selection?.kind === "annotation" ? tl.selection.id : null diff --git a/src/components/ai-edition/Preview.tsx b/src/components/ai-edition/Preview.tsx index aaf04f2db..1c4b19398 100644 --- a/src/components/ai-edition/Preview.tsx +++ b/src/components/ai-edition/Preview.tsx @@ -4,6 +4,7 @@ import { useScopedT } from "@/contexts/I18nContext"; import type { AxcutAnnotationRegion, AxcutClip, + AxcutCursorMotionRegion, AxcutTrimRange, AxcutZoomRegion, } from "@/lib/ai-edition/schema"; @@ -29,6 +30,10 @@ interface PreviewProps { selectedZoomRegionId?: string | null; onZoomFocusChange?: (id: string, focus: ZoomFocus) => void; onZoomFocusCommit?: () => void; + cursorMotionRegions?: AxcutCursorMotionRegion[]; + selectedCursorMotionId?: string | null; + onCursorMotionControlPointChange?: (id: string, point: { cx: number; cy: number }) => void; + onCursorMotionControlPointCommit?: () => void; annotationRegions?: AxcutAnnotationRegion[]; selectedAnnotationId?: string | null; onSelectAnnotation?: (id: string) => void; @@ -59,6 +64,10 @@ export function Preview({ trimRanges, selectedZoomRegionId, onZoomFocusChange, + cursorMotionRegions, + selectedCursorMotionId, + onCursorMotionControlPointChange, + onCursorMotionControlPointCommit, onZoomFocusCommit, annotationRegions, selectedAnnotationId, @@ -185,6 +194,10 @@ export function Preview({ trimRanges={trimRanges} selectedZoomRegionId={selectedZoomRegionId} onZoomFocusChange={onZoomFocusChange} + cursorMotionRegions={cursorMotionRegions} + selectedCursorMotionId={selectedCursorMotionId} + onCursorMotionControlPointChange={onCursorMotionControlPointChange} + onCursorMotionControlPointCommit={onCursorMotionControlPointCommit} onZoomFocusCommit={onZoomFocusCommit} annotationRegions={annotationRegions} selectedAnnotationId={selectedAnnotationId} diff --git a/src/components/ai-edition/PreviewCanvas.tsx b/src/components/ai-edition/PreviewCanvas.tsx index 6f59e066f..829b6033f 100644 --- a/src/components/ai-edition/PreviewCanvas.tsx +++ b/src/components/ai-edition/PreviewCanvas.tsx @@ -22,6 +22,7 @@ import type { PointerEvent as ReactPointerEvent } from "react"; import { useEffect, useMemo, useRef, useState } from "react"; +import { fromFileUrl } from "@/components/video-editor/projectPersistence"; import { type CameraFullscreenRegion, type CropRegion, @@ -35,6 +36,7 @@ import { resolveAspectRatioValue } from "@/lib/ai-edition/document/outputFormat" import type { AxcutAnnotationRegion, AxcutClip, + AxcutCursorMotionRegion, AxcutTrimRange, AxcutZoomRegion, } from "@/lib/ai-edition/schema"; @@ -53,8 +55,10 @@ import { import { classifyWallpaper, resolveImageWallpaperUrl } from "@/lib/wallpaper"; import { getCssClipPath } from "@/lib/webcamMaskShapes"; import { computeCameraFullscreenProgress } from "@/lib/zoomMath/cameraFullscreenUtils"; +import { useCursorTelemetry } from "@/native/hooks/useCursorTelemetry"; import { clamp, clamp01 } from "@/utils/math"; import { AnnotationLayer } from "./AnnotationLayer"; +import { CursorMotionPathOverlay } from "./CursorMotionPathOverlay"; import { NativeCompositorOverlay } from "./NativeCompositorOverlay"; import styles from "./NewEditorShell.module.css"; import { type VideoSource, VirtualPreview } from "./VirtualPreview"; @@ -73,6 +77,10 @@ interface PreviewCanvasProps { selectedZoomRegionId?: string | null; onZoomFocusChange?: (id: string, focus: ZoomFocus) => void; onZoomFocusCommit?: () => void; + cursorMotionRegions?: AxcutCursorMotionRegion[]; + selectedCursorMotionId?: string | null; + onCursorMotionControlPointChange?: (id: string, point: { cx: number; cy: number }) => void; + onCursorMotionControlPointCommit?: () => void; annotationRegions?: AxcutAnnotationRegion[]; selectedAnnotationId?: string | null; onSelectAnnotation?: (id: string) => void; @@ -387,6 +395,23 @@ export function PreviewCanvas(props: PreviewCanvasProps) { const isPipGrab = settings.webcamLayoutPreset === "picture-in-picture"; + // Telemetry for the recorded-trace comparison line. Loaded from the SELECTED + // section's own asset, not the primary one: in a multi-clip project those differ, + // and comparing an edited path against another recording's trace would be worse + // than showing no comparison at all. + const selectedCursorMotion = props.selectedCursorMotionId + ? (props.cursorMotionRegions?.find((r) => r.id === props.selectedCursorMotionId) ?? null) + : null; + // The recorded trace is loaded through the same hook the cursor preview layer + // uses, keyed by the SELECTED section's asset. `useCursorTelemetry` takes null + // as "nothing to load" and returns an empty list, so nothing is fetched while no + // section is selected — which is the common case. + const cursorMotionSource = selectedCursorMotion?.assetId + ? (props.videoSources.find((v) => v.id === selectedCursorMotion.assetId) ?? null) + : null; + const { samples: cursorMotionTelemetry } = useCursorTelemetry( + cursorMotionSource ? fromFileUrl(cursorMotionSource.src) : null, + ); const selectedZoomRegion = props.selectedZoomRegionId ? (props.zoomRegions?.find((z) => z.id === props.selectedZoomRegionId) ?? null) : null; @@ -429,6 +454,14 @@ export function PreviewCanvas(props: PreviewCanvasProps) { /> ); })()} + {selectedCursorMotion && props.onCursorMotionControlPointChange ? ( + + ) : null} {selectedZoomRegion && props.onZoomFocusChange ? ( = { + recorded: "M4 18 C12 8 18 21 27 11 S38 4 44 9", + straight: "M4 18 L44 6", + arc: "M4 18 Q24 1 44 18", + wave: "M4 13 C10 2 16 2 22 13 S34 24 44 13", + loop: "M4 16 C14 2 34 2 32 15 C30 25 14 23 18 13 C22 5 36 8 44 16", + overshoot: "M4 18 C24 18 37 5 46 8 C42 8 40 10 44 14", +}; +const CURSOR_MOTION_PREVIEW_END_Y: Record = { + recorded: 9, + straight: 6, + arc: 18, + wave: 13, + loop: 16, + overshoot: 14, +}; +// The speeds worth one click. The slider covers everything between them; these are +// the values an editor actually reaches for, and hunting for "2.0" on a 0.1-step +// slider is a worse way to get there. +const CURSOR_MOTION_SPEED_PRESETS = [1, 1.5, 2, 3, 4] as const; + +function CursorMotionPresetGlyph({ preset }: { preset: CursorMotionPreset }) { + return ( + + ); +} + type AnnotationKind = AxcutAnnotationRegion["type"]; type ArrowDirectionKind = NonNullable["arrowDirection"]; @@ -610,6 +670,291 @@ function SelectionPane({ tl, onClose }: { tl: TimelineApi; onClose: () => void } ); } + if (selection.kind === "cursorMotion") { + const regions = tl.cursorMotionRegions ?? []; + const region = regions.find((r) => r.id === selection.id); + if (!region) return null; + // Sections in timeline order, so "previous / next" walks the recording rather + // than the order they happen to sit in the document. + const ordered = [...regions].sort((a, b) => a.startMs - b.startMs); + const index = ordered.findIndex((r) => r.id === region.id); + const step = (delta: number) => { + const target = ordered[index + delta]; + if (target) tl.selectRegion("cursorMotion", target.id); + }; + const isHold = region.segmentKind === "hold"; + return ( +
+ {paneHeader(, ts("cursorMotion.title"), onClose, tc("actions.close"))} +
+
+ + + {ts("cursorMotion.sectionCounter", { + current: index + 1, + total: ordered.length, + })} + + +
+ + {isHold ? ( + // A hold has no path, so every control below would be inert on it. Say + // what the section IS instead of showing six shapes that cannot apply — + // this is the section that keeps the cursor still, and that is the whole + // reason the rests were split out in the first place. +

+ {ts("cursorMotion.holdDescription")} +

+ ) : ( + <> +
+ + {ts("cursorMotion.preset")} + +
+ {CURSOR_MOTION_PRESETS.map((preset) => { + const active = region.preset === preset; + return ( + + ); + })} +
+
+ + {/* `recorded` is inert: it plays the captured trace, so speed, turns and + timing have nothing to reshape. Hiding them is the honest reading of + "this section is unedited" — greying six controls says the same thing + at four times the visual weight. */} + {region.preset !== "recorded" ? ( + <> +
+ {paneRow( + ts("cursorMotion.speed"), + + {region.speed.toFixed(1)}× + , + )} +
+ {CURSOR_MOTION_SPEED_PRESETS.map((speed) => ( + + ))} +
+ + void tl.updateCursorMotionSettings(region.id, { + speed: Number(e.target.value), + }) + } + style={{ width: "100%", accentColor: "var(--accent)" }} + /> +

+ {ts("cursorMotion.speedHint")} +

+
+ + {region.preset === "wave" || region.preset === "loop" ? ( +
+ {paneRow( + ts("cursorMotion.cycles"), + + {region.cycles} + , + )} + + void tl.updateCursorMotionSettings(region.id, { + cycles: Number(e.target.value), + }) + } + style={{ width: "100%", accentColor: "var(--accent)" }} + /> +
+ ) : null} + + {paneRow( + ts("cursorMotion.easing"), + , + )} + + + + ) : null} + + )} + +

+ {ts("cursorMotion.anchorHint")} +

+ + + + +
+
+ ); + } + if (selection.kind === "speed") { const region = tl.speedRegions.find((s) => s.id === selection.id); if (!region) return null; diff --git a/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx b/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx index 8e515568c..eb20f7e21 100644 --- a/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx +++ b/src/components/ai-edition/v4/V4Timeline.geometry.test.tsx @@ -80,6 +80,7 @@ function renderTimeline( speedRegions: [], cameraFullscreenRegions: [], zoomRegions: [], + cursorMotionRegions: [], trimRanges: [], selection: null, multiSelection: [], diff --git a/src/components/ai-edition/v4/V4Timeline.tsx b/src/components/ai-edition/v4/V4Timeline.tsx index f6c94e2ca..35942b739 100644 --- a/src/components/ai-edition/v4/V4Timeline.tsx +++ b/src/components/ai-edition/v4/V4Timeline.tsx @@ -4,9 +4,11 @@ import { Loader2, Maximize2, MessageSquare, + Pause, Pencil, Scissors, Sparkles, + Spline, SplitSquareHorizontal, Trash2, Wand2, @@ -37,13 +39,17 @@ import { useChatPromptBus } from "@/lib/ai-edition/store/useChatPromptBus"; import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; import type { useTimeline } from "@/lib/ai-edition/store/useTimeline"; import { hasAnyClipWithCamera } from "@/lib/ai-edition/timeline/camera"; +import { toCursorMotionSamples } from "@/lib/ai-edition/timeline/cursorMotionRegions"; import { formatSec } from "@/lib/ai-edition/timeline/format"; import { newRegionDurationSec, setTimelineScale, } from "@/lib/ai-edition/timeline/newRegionDuration"; import { ventilateSpanAcrossClips } from "@/lib/ai-edition/timeline/region-ventilation"; -import { coalesceRegionsForRuler } from "@/lib/ai-edition/timeline/timelineMap"; +import { + coalesceRegionsForRuler, + resolveNativePosition, +} from "@/lib/ai-edition/timeline/timelineMap"; import { coalescedTrimGroups, resolveTimelineSpanToTrim, @@ -54,6 +60,7 @@ import { buildAutoZoomSuggestionsForClips, } from "@/lib/ai-edition/timeline/zoom-suggestions"; import { nativeBridgeClient } from "@/native/client"; +import { resolveVisibleClips } from "@/native/sceneDescription"; import { TransportBar } from "../TransportBar"; import type { VideoSource } from "../VirtualPreview"; import styles from "./EditorShellV4.module.css"; @@ -337,12 +344,15 @@ const ClipWaveform = memo(function ClipWaveform({ interface LanePill { id: string; - kind: "annotation" | "speed" | "trim" | "zoom" | "cameraFullscreen"; + kind: "annotation" | "speed" | "trim" | "zoom" | "cameraFullscreen" | "cursorMotion"; start: number; end: number; label: string; /** Underlying row ids this pill represents — >1 for a coalesced trim group. */ sourceIds: string[]; + /** A cursor-motion HOLD: the pointer stays where it stopped. Drawn quieter than + * a move because there is no path in it to direct — see `laneCursorHold`. */ + quiet?: boolean; } export function V4Timeline({ @@ -509,6 +519,30 @@ export function V4Timeline({ label: `${(p.member.customScale ?? ZOOM_DEPTH_SCALES[p.member.depth]).toFixed(2)}×`, sourceIds: p.ids, })); + // Cursor motion is the one lane that does NOT coalesce. Everywhere else, two + // touching regions with equal properties are the same effect and merging them + // is what makes a ventilated region read as one pill. Here they are two + // SECTIONS, and being individually selectable is the whole feature: an editor + // tunes one move, walks to the next, tunes that one. Merging the two moves + // that happen to share a preset would silently take that away — and it is the + // state a fresh auto-split is always in, since every section starts + // `recorded` at 1x. + const cursorMotionPills: LanePill[] = (tl.cursorMotionRegions ?? []).map((region) => ({ + id: region.id, + kind: "cursorMotion" as const, + start: region.startMs / 1000, + end: region.endMs / 1000, + // The speed rides on the label because it is the property with no other + // visible tell: a preset shows itself in the preview path, a 3x does not. + label: + region.segmentKind === "hold" + ? t("cursorMotion.segmentKinds.hold") + : `${t(`cursorMotion.presets.${region.preset}`)}${ + region.speed === 1 ? "" : ` ${region.speed.toFixed(1)}×` + }`, + sourceIds: [region.id], + quiet: region.segmentKind === "hold", + })); // trims: content-free (no per-instance text/settings), so touching rows — // inevitable once a trim is ventilated across a clip boundary — are // coalesced into one pill. This is what makes growing a trim across a @@ -665,6 +699,12 @@ export function V4Timeline({ e.preventDefault(); e.stopPropagation(); tl.selectRegion(pill.kind, pill.id, { additive: e.shiftKey }); + // A cursor motion section selects but does not drag. Its two ends are not + // a span someone chose — they are a rest, a click, or a cut, each pinned to + // a POINT the recording puts the pointer at. Sliding the section would keep + // the anchors and move the times, so the path would start where the cursor + // no longer is. Re-splitting is how you change these boundaries. + if (pill.kind === "cursorMotion") return; // Scale drag deltas against the canvas (full zoomed timeline) width, so a // drag tracks the cursor exactly regardless of padding, scrollbar or zoom. const el = canvasRef.current; @@ -892,25 +932,37 @@ export function V4Timeline({ transform: `translateX(${(-nav.start * 100).toFixed(3)}%)`, } as const; - const laneOf = (kind: LanePill["kind"]) => - kind === "annotation" + const laneOf = (pill: LanePill) => + pill.kind === "annotation" ? styles.laneAnnotation - : kind === "speed" + : pill.kind === "speed" ? styles.laneSpeed - : kind === "trim" + : pill.kind === "trim" ? styles.laneTrim - : kind === "cameraFullscreen" + : pill.kind === "cameraFullscreen" ? styles.laneCameraFullscreen - : styles.laneZoom; - const pillIcon = (kind: LanePill["kind"]) => - kind === "annotation" ? ( + : pill.kind === "cursorMotion" + ? pill.quiet + ? styles.laneCursorHold + : styles.laneCursorMotion + : styles.laneZoom; + const pillIcon = (pill: LanePill) => + pill.kind === "annotation" ? ( - ) : kind === "speed" ? ( + ) : pill.kind === "speed" ? ( - ) : kind === "trim" ? ( + ) : pill.kind === "trim" ? ( - ) : kind === "cameraFullscreen" ? ( + ) : pill.kind === "cameraFullscreen" ? ( + ) : pill.kind === "cursorMotion" ? ( + // A hold is the absence of movement, so it gets the pause glyph rather + // than a dimmer copy of the path one — the lane already dims it. + pill.quiet ? ( + + ) : ( + + ) ) : ( ); @@ -1081,6 +1133,56 @@ export function V4Timeline({ } }, [videoSources, clips, tl, t]); + // Cursor motion — build the sections from the playhead to the next recorded + // click. Unlike auto-zoom above, this reads the RECORDING data rather than the + // position telemetry: the split points are clicks, and only the native cursor + // sampler records those. A capture with position-only telemetry (the fallback + // adapter, or a platform without the helper) therefore has nothing to split on, + // which is one of the three ways this can legitimately find nothing. + const [cursorMotionBusy, setCursorMotionBusy] = useState(false); + const runCursorMotion = useCallback(async () => { + const state = useProjectStore.getState(); + const doc = state.document; + // Which recording's telemetry to read is decided by the clip UNDER THE + // PLAYHEAD, not by the primary asset: in a multi-clip project those are + // different recordings, and reading the wrong one produces sections whose + // anchors sit where that other capture's cursor was. + const position = doc + ? resolveNativePosition(state.currentTimeSec ?? 0, resolveVisibleClips(doc), clips) + : null; + const source = position + ? videoSources.find((s) => s.id === position.clip.assetId) + : videoSources[0]; + if (!source) { + toast.error(t("toolbar.importRecordingFirst")); + return; + } + setCursorMotionBusy(true); + try { + const data = await nativeBridgeClient.cursor.getRecordingData(fromFileUrl(source.src)); + const samples = data?.samples ?? []; + if (samples.length === 0) { + toast.info(t("cursorMotion.noCursorData")); + return; + } + const added = await tl.addCursorMotion(toCursorMotionSamples(samples)); + // Nothing added has exactly one cause worth naming: there is no click left + // to reach. Saying "no cursor data" there would send the user looking for a + // recording problem they do not have. + if (added === 0) { + toast.info(t("cursorMotion.noFollowingClick")); + return; + } + toast.success(t("cursorMotion.createdSegments", { count: added })); + } catch (err) { + toast.error(t("cursorMotion.failed"), { + description: err instanceof Error ? err.message : String(err), + }); + } finally { + setCursorMotionBusy(false); + } + }, [clips, videoSources, tl, t]); + // Auto-enhance option 2 — hand a generic prompt to the AI agent (smart // zooms + cuts) via the chat prompt-bus. The chat panel owns the outcome // toast: submitting is not the same as being accepted (no usable provider @@ -1133,12 +1235,16 @@ export function V4Timeline({ // The box is exactly as long as the effect is; only what fits INSIDE it // varies with the zoom. const { compact, roomForLabel } = pillAffordance(durSec, pxPerSec); + // No edge handles on a cursor motion section: its boundaries are anchors the + // recording placed, not a span to stretch (see `startPillDrag`). Showing grab + // strips that refuse to grab is worse than showing none. + const resizable = p.kind !== "cursorMotion"; return (
startPillDrag(e, p, "move") : undefined} title={p.label} > - {seg.interactive ? ( + {seg.interactive && resizable ? ( - {pillIcon(p.kind)} + {pillIcon(p)} {p.label} ) : null} - {seg.interactive ? ( + {seg.interactive && resizable ? ( +