From 8db54f6c9d63abe6e6c07f1592e6f6cf9af122ce Mon Sep 17 00:00:00 2001 From: Nika Siradze Date: Sun, 6 Sep 2026 23:07:31 +0400 Subject: [PATCH 1/3] Let a caller hand in the face map, and stop a lost crop looking deliberate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A clip that asks to follow the speaker and cannot gets the whole wide frame shrunk into a band with a blurred copy of itself behind it. That is the right answer for a source with nothing to crop to and the wrong one for a two-person recording, and it has been reached in silence. The ladder for `speaker` is five rungs, and the four above the fallback all need either speaker labels on the words or a face map. Both come from the diarizing transcriber, so a run against a transcript that has neither — any whisper run, and every imported transcript — could only ever land on the last rung. The caller often knows where the faces are and had no way to say so. `--face-map` takes a path to one, on both `process` and `studio`, and it wins over the map the transcriber found: whoever passed it scanned this video, rather than inferring the layout from who was speaking. `--crop-keyframes` does the same for a frame somebody placed by hand, and `manual` joins the `--crop` choices it has needed since the strategy was written — the renderer has understood it for months while the parser refused it, so the only way in was the MCP path. Manual with no keyframes used to fall past every branch and reach the return with no filter built, which is an UnboundLocalError rather than a clip. It became reachable the moment the flag existed, so it now degrades to a face crop and says so. The fallback reports itself. `[crop] chose=center-blur-bg` carries what was asked for, and asking for anything but centre and getting this logs an `uncropped` warning, so a caller watching the render can tell the difference between a centre crop that was the plan and a face crop that gave up. --- backend/cli.py | 51 +++++++++++++++++++++++++++-- backend/clip_studio.py | 27 ++++++++++++++- backend/services/video_processor.py | 25 +++++++++++++- 3 files changed, 99 insertions(+), 4 deletions(-) diff --git a/backend/cli.py b/backend/cli.py index 0764f47..deec7e5 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -84,6 +84,23 @@ def _parse_json_transcript(raw_text): return data.get("words", []), data.get("segments", []), data +def _json_file_arg(path: str | None, name: str): + """A JSON file named on the command line, or nothing. + + A path rather than the JSON itself: a face map is a per-second record of + every face in the video, which an argument list will not carry. Missing or + malformed loses the framing hint, never the run. + """ + if not path: + return None + try: + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + except (OSError, ValueError) as exc: + print(f" Warning: {name} could not be read ({exc}); ignoring it", file=sys.stderr) + return None + + def _cached_face_map(video_path: str): """Face maps are keyed by video content, not by transcript, so an imported transcript can still borrow the map from an earlier run on the same file.""" @@ -470,6 +487,16 @@ def cmd_studio(args): "--caption-scale", str(getattr(args, "caption_scale", 1.0)), "--crop", args.crop, "--format", getattr(args, "format", None) or "vertical", + ] + # Where the faces are, and where somebody put the frame by hand. Both are + # what the crop needs and neither could be handed to it before, so a caller + # that already knew — the cloud worker scans the window before it renders — + # had no way to say so and watched `speaker` fall through to a letterbox. + if getattr(args, "face_map", None): + cmd += ["--face-map", os.path.abspath(args.face_map)] + if getattr(args, "crop_keyframes", None): + cmd += ["--crop-keyframes", os.path.abspath(args.crop_keyframes)] + cmd += [ "--logo-position", getattr(args, "logo_position", "top-left"), "--logo-scale", str(getattr(args, "logo_scale", 1.0)), "--intro-seconds", str(args.intro_seconds), @@ -979,6 +1006,16 @@ def _transcribe_progress(pct, msg): # Extract face_map before result gets overwritten in clip loop face_map = result.get("face_map") + # A map handed in on the command line wins over one the transcriber found, + # because the caller who bothered to pass it scanned this video rather than + # inferring the layout from who was speaking. It is also the only map there + # is when the transcript came from an engine that does not diarize, which is + # the case where `speaker` framing used to fall through to a letterbox. + given = _json_file_arg(getattr(args, "face_map", None), "--face-map") + if given: + face_map = given + print(" Using the face map passed in (speaker framing preserved)") + # Check speaker data availability (needed for smart cropping) speakers_in_words = set(w.get("speaker") for w in words if w.get("speaker")) diarization_warning = result.get("diarization_warning") @@ -4399,7 +4436,12 @@ def main(): help="Caption placement (default: follows the chosen style)") proc.add_argument("--caption-scale", type=float, choices=[0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 1.0, 1.05, 1.1, 1.15, 1.2, 1.25, 1.3, 1.35, 1.4, 1.45, 1.5], help="Caption size multiplier (default: 1)") - proc.add_argument("--crop", choices=["center", "face", "speaker", "speaker-hardcut"]) + proc.add_argument("--crop", choices=["center", "face", "speaker", "speaker-hardcut", "manual"]) + proc.add_argument("--face-map", dest="face_map", default=None, + help="Where the faces sit in this video, as JSON. Skips detection and lets " + "speaker framing work on a transcript that carries no speaker labels.") + proc.add_argument("--crop-keyframes", dest="crop_keyframes", default=None, + help="Hand-placed crop positions, as JSON. Used by --crop manual.") proc.add_argument("--format", choices=["vertical", "horizontal", "square"], help="Output aspect ratio (default: vertical)") proc.add_argument("--profile", choices=["podcast", "party", "action"], help="Detection profile: podcast (transcript-first, default), party/action (laughter/energy highlights)") proc.add_argument("--logo", help="Logo image (asset name or path)") @@ -4485,7 +4527,12 @@ def main(): studio.add_argument("--caption-style", choices=["hormozi", "karaoke", "subtle", "branded"], default="hormozi") studio.add_argument("--caption-position", choices=["auto", "upper", "center", "lower"], default="auto") studio.add_argument("--caption-scale", type=float, default=1.0) - studio.add_argument("--crop", choices=["center", "face", "speaker", "speaker-hardcut"], default="face") + studio.add_argument("--crop", choices=["center", "face", "speaker", "speaker-hardcut", "manual"], default="face") + studio.add_argument("--face-map", dest="face_map", default=None, + help="Where the faces sit in this video, as JSON. Skips detection and lets " + "speaker framing work on a transcript that carries no speaker labels.") + studio.add_argument("--crop-keyframes", dest="crop_keyframes", default=None, + help="Hand-placed crop positions, as JSON. Used by --crop manual.") studio.add_argument("--format", choices=["vertical", "horizontal", "square"], default="vertical", help="Output aspect ratio (default: vertical)") studio.add_argument("--template", help="Cut in a saved look (podcli Pro). Name or id.") diff --git a/backend/clip_studio.py b/backend/clip_studio.py index 8e79196..bba2d1c 100644 --- a/backend/clip_studio.py +++ b/backend/clip_studio.py @@ -161,11 +161,28 @@ def _json_arg(raw, name): return None +def _json_file(path, name): + """A JSON file, or nothing. + + A path rather than the JSON itself, because a face map is a per-second + record of every face in the video and an argument list has a limit. + """ + if not path: + return None + try: + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + except (OSError, ValueError) as exc: + print(f" Warning: {name} could not be read ({exc}); ignoring it", + file=sys.stderr, flush=True) + return None + + def _render_fragment(video, start, end, words, style, crop, title, out_dir, fmt="vertical", logo=None, name_card=None, motion=None, caption_position="auto", caption_scale=1.0, logo_position="top-left", logo_scale=1.0, topic=None, progress=None, cards=None, brand=None, font_family=None, - captions=True): + captions=True, face_map=None, crop_keyframes=None): """Render the fragment with face-crop + captions via the existing engine.""" from services.clip_generator import generate_clip print(f" [fragment] rendering {start:.1f}s–{end:.1f}s ({style}, crop={crop}, {fmt})", flush=True) @@ -175,6 +192,7 @@ def _render_fragment(video, start, end, words, style, crop, title, out_dir, fmt= caption_font_scale=round(caption_scale * 100), logo_position=logo_position, logo_scale=logo_scale, crop_strategy=crop, format=fmt, + face_map=face_map, crop_keyframes=crop_keyframes, transcript_words=words, title=title, output_dir=out_dir, logo_path=logo, name_card=name_card, motion=motion, topic=topic, progress=progress, cards=cards, brand=brand, font_family=font_family, @@ -296,6 +314,11 @@ def main(): ap.add_argument("--progress", action="store_true", help="Draw how much of the clip is left along the bottom edge") ap.add_argument("--progress-color", default=None) + ap.add_argument("--face-map", dest="face_map", default=None, + help="Path to a JSON face map for this video. Lets speaker framing work " + "on a transcript that carries no speaker labels.") + ap.add_argument("--crop-keyframes", dest="crop_keyframes", default=None, + help="Path to hand-placed crop positions as JSON. Used by --crop manual.") ap.add_argument("--cards", default=None, help="On-screen cards as JSON, each with kind/start/end") ap.add_argument("--brand", default=None, @@ -410,6 +433,8 @@ def media_path(value: str | None, kind: str) -> str | None: brand=_json_arg(args.brand, "--brand"), font_family=args.font_family, captions=not args.no_captions, + face_map=_json_file(args.face_map, "--face-map"), + crop_keyframes=_json_file(args.crop_keyframes, "--crop-keyframes"), ) platforms = [p.strip() for p in platforms_str.split(",") if p.strip()] diff --git a/backend/services/video_processor.py b/backend/services/video_processor.py index 8f10461..eace9c7 100644 --- a/backend/services/video_processor.py +++ b/backend/services/video_processor.py @@ -247,12 +247,26 @@ def crop_to_vertical( clip_start: The start time of this clip in the original video (for timestamp alignment). """ width, height = get_dimensions(input_path) + # What the caller asked for, kept because `strategy` is reassigned on the + # way down the ladder. A clip that asked to follow the speaker and ended up + # letterboxed should say so in those words, not report a centre crop as + # though centre was the plan. + wanted = strategy face_map = _sane_speaker_mappings(face_map) target_w, target_h = target_dims target_ratio = target_w / target_h # 0.5625 for the vertical default source_ratio = width / height + # Manual with nothing to place the frame from used to fall past every + # branch below and reach the return with no filter built at all, which is + # an UnboundLocalError rather than a clip. It became reachable the moment + # a caller could pass --crop manual, so it answers for itself: no + # keyframes is no hand placement, and the crop it wanted is a face. + if strategy == "manual" and not crop_keyframes: + log_event("crop", "fallback", reason="manual_without_keyframes", to="face") + strategy = "face" + if strategy == "manual" and crop_keyframes: log_event("crop", "chose=manual", keyframes=len(crop_keyframes), source=f"{width}x{height}") crop_h = height @@ -417,7 +431,16 @@ def crop_to_vertical( if strategy == "center": if source_ratio > target_ratio: - log_event("crop", "chose=center-blur-bg", source=f"{width}x{height}") + log_event("crop", "chose=center-blur-bg", source=f"{width}x{height}", asked=wanted) + # The whole wide frame, shrunk into a band with a blur behind it. + # It is the honest answer for a source with nothing to crop to and + # the wrong one for a clip that asked to follow a face, so it is + # reported at warn rather than left to look deliberate. + if wanted != "center": + log_event( + "crop", "uncropped", level="warn", asked=wanted, + reason="no face map, no speaker labels and no face found in this window", + ) # Wide source with no face detected: blurred background + sharp center. # Scales source to fill 9:16 height → blur → overlay sharp fit-to-width. vf_complex = ( From 53051e063745f69bccca239c5fb84fb114686ad4 Mon Sep 17 00:00:00 2001 From: Nika Siradze Date: Sun, 6 Sep 2026 23:08:36 +0400 Subject: [PATCH 2/3] Take the face map and the keyframes inline or as a file The keyframes have always been passed inline by the one caller that sends them, and a face map is far too big for an argument list. Rather than a flag each, both read what they were given: JSON if it opens with a brace, a path otherwise. --- backend/cli.py | 28 +++++++++++++++++++--------- backend/clip_studio.py | 17 +++++++++++------ 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/backend/cli.py b/backend/cli.py index deec7e5..ab490e9 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -84,17 +84,21 @@ def _parse_json_transcript(raw_text): return data.get("words", []), data.get("segments", []), data -def _json_file_arg(path: str | None, name: str): - """A JSON file named on the command line, or nothing. +def _json_file_arg(raw: str | None, name: str): + """JSON given inline or named as a file, or nothing. - A path rather than the JSON itself: a face map is a per-second record of - every face in the video, which an argument list will not carry. Missing or - malformed loses the framing hint, never the run. + Both, because the two callers differ: keyframes are a handful of numbers + and have always been passed inline, while a face map is a per-second record + of every face in the video, which an argument list will not carry. Missing + or malformed loses the framing hint, never the run. """ - if not path: + if not raw: return None + text = raw.strip() try: - with open(path, "r", encoding="utf-8") as handle: + if text.startswith("{") or text.startswith("["): + return json.loads(text) + with open(text, "r", encoding="utf-8") as handle: return json.load(handle) except (OSError, ValueError) as exc: print(f" Warning: {name} could not be read ({exc}); ignoring it", file=sys.stderr) @@ -492,10 +496,16 @@ def cmd_studio(args): # what the crop needs and neither could be handed to it before, so a caller # that already knew — the cloud worker scans the window before it renders — # had no way to say so and watched `speaker` fall through to a letterbox. + # A path is made absolute because the script runs from elsewhere; JSON + # given inline is handed straight through. + def _pass_through(value): + text = str(value).strip() + return text if text.startswith(("{", "[")) else os.path.abspath(text) + if getattr(args, "face_map", None): - cmd += ["--face-map", os.path.abspath(args.face_map)] + cmd += ["--face-map", _pass_through(args.face_map)] if getattr(args, "crop_keyframes", None): - cmd += ["--crop-keyframes", os.path.abspath(args.crop_keyframes)] + cmd += ["--crop-keyframes", _pass_through(args.crop_keyframes)] cmd += [ "--logo-position", getattr(args, "logo_position", "top-left"), "--logo-scale", str(getattr(args, "logo_scale", 1.0)), diff --git a/backend/clip_studio.py b/backend/clip_studio.py index bba2d1c..a1ef7c2 100644 --- a/backend/clip_studio.py +++ b/backend/clip_studio.py @@ -161,16 +161,21 @@ def _json_arg(raw, name): return None -def _json_file(path, name): - """A JSON file, or nothing. +def _json_file(raw, name): + """JSON given inline or named as a file, or nothing. - A path rather than the JSON itself, because a face map is a per-second - record of every face in the video and an argument list has a limit. + Both, because the two callers differ: keyframes are a handful of numbers + and have always been passed inline, while a face map is a per-second record + of every face in the video and an argument list will not carry one. Telling + them apart by looking is cheaper than a second flag. """ - if not path: + if not raw: return None + text = raw.strip() + if text.startswith("{") or text.startswith("["): + return _json_arg(text, name) try: - with open(path, "r", encoding="utf-8") as handle: + with open(text, "r", encoding="utf-8") as handle: return json.load(handle) except (OSError, ValueError) as exc: print(f" Warning: {name} could not be read ({exc}); ignoring it", From 162fa67b46abbc3ca63f9a3f91dca7a8e55c9565 Mon Sep 17 00:00:00 2001 From: Nika Siradze Date: Sun, 6 Sep 2026 23:11:53 +0400 Subject: [PATCH 3/3] Scan for faces when the crop needs a map and nothing supplied one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flag lets a caller hand a face map in. This is the answer for everyone who has none to hand: build it. A map only ever arrived from the diarizing transcriber. Any whisper run, any imported transcript and every cloud render that transcribes locally therefore had none, and `speaker` and `face` skipped all four rungs that could place a frame and landed on the last one — the whole wide source shrunk into a band with a blurred copy behind it, logged as a centre crop. The scan already exists and is what the diarizing path calls. It clusters faces off the frames themselves; speaker segments only decide which cluster belongs to whom, so it still finds where the faces are and whether the recording is a split screen when there are none. That is what a crop needs. `process` scans once for the episode, before the clip loop. `studio` scans the fragment it was handed, which for a cloud recut is the window already cut. It runs only for a crop that needs it, and a failed or empty scan falls back the way it did before rather than losing the clip. --- backend/cli.py | 29 ++++++++++++++++++++++ backend/clip_studio.py | 55 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/backend/cli.py b/backend/cli.py index ab490e9..98fec7c 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -1026,6 +1026,35 @@ def _transcribe_progress(pct, msg): face_map = given print(" Using the face map passed in (speaker framing preserved)") + # Still nothing, and the crop about to run needs one. Scanning costs a + # minute on a long episode and buys back every clip in it: without a map, + # `speaker` and `face` skip every rung that could place a frame and land on + # the whole wide source letterboxed into the cut. + if not face_map and config.get("crop_strategy") in ("face", "speaker", "speaker-hardcut"): + try: + from services.face_analysis import analyze_faces + print(" No face map yet; scanning the episode for faces") + # The last word's end, because the scan only uses this to decide + # how many frames to sample; it reads the real frame count off the + # file itself and spreads the samples across all of it. + spoken = max((float(w.get("end") or 0) for w in words), default=0.0) + found = analyze_faces( + video_path, + [{"speaker": w["speaker"], "start": w["start"], "end": w["end"]} + for w in words if w.get("speaker")], + spoken, + progress_callback=lambda p, m: None, + ) + except Exception as exc: + print(f" Face scan failed ({type(exc).__name__}: {exc}); " + "the crop will fall back", file=sys.stderr) + found = None + if found and found.get("clusters"): + face_map = found + print(f" Found {len(found['clusters'])} face position(s)") + else: + print(" No faces found; the crop will fall back") + # Check speaker data availability (needed for smart cropping) speakers_in_words = set(w.get("speaker") for w in words if w.get("speaker")) diarization_warning = result.get("diarization_warning") diff --git a/backend/clip_studio.py b/backend/clip_studio.py index a1ef7c2..a6fe152 100644 --- a/backend/clip_studio.py +++ b/backend/clip_studio.py @@ -183,6 +183,55 @@ def _json_file(raw, name): return None +# The crops that cannot place a frame without knowing where the faces are. +_WANTS_FACES = ("face", "speaker", "speaker-hardcut") + + +def _face_map_for(video, crop, start, end): + """Where the faces are, scanned now, when the crop needs it and nobody said. + + A face map used to arrive only from the diarizing transcriber, so a cut + made against a whisper transcript or an imported one had none, and every + rung of the crop ladder that needs one was skipped. What the caller got + instead was the whole wide frame letterboxed into the cut, reported as a + centre crop and looking deliberate. + + Scanning here costs a few seconds on the window actually being cut, which + is the trade every one of those clips would have taken. + """ + if crop not in _WANTS_FACES: + return None + try: + from services.face_analysis import analyze_faces + except ImportError: + return None + + # The whole file's length rather than the fragment's: the scan spreads its + # samples across the file it is given, and sizing the count to a 40-second + # window would take twenty samples across an hour-long episode. + duration = _probe_duration(video) or max(0.0, (end or 0) - (start or 0)) + if duration <= 0: + return None + + print(" [fragment] no face map for this cut; scanning it", flush=True) + try: + # No speaker segments: those come from diarization, and the whole point + # of scanning here is that there was none. Clusters and the split + # screen are still found, which is what a crop needs to place a frame. + found = analyze_faces(video, [], duration) + except Exception as exc: + print(f" Warning: face scan failed ({type(exc).__name__}: {exc}); " + "the crop will fall back", file=sys.stderr, flush=True) + return None + + if not found or not found.get("clusters"): + print(" [fragment] no faces found in this cut", flush=True) + return None + print(f" [fragment] found {len(found['clusters'])} face position(s)" + f"{' , split screen' if found.get('is_split_screen') else ''}", flush=True) + return found + + def _render_fragment(video, start, end, words, style, crop, title, out_dir, fmt="vertical", logo=None, name_card=None, motion=None, caption_position="auto", caption_scale=1.0, logo_position="top-left", logo_scale=1.0, @@ -419,6 +468,10 @@ def media_path(value: str | None, kind: str) -> str | None: print(" Warning: --motion is not valid JSON; using each style's own motion", flush=True) + face_map = _json_file(args.face_map, "--face-map") + if face_map is None: + face_map = _face_map_for(video, args.crop, start, end) + fragment = _render_fragment( video, start, end, words, args.caption_style, args.crop, "fragment", out_dir, fmt=args.format, @@ -438,7 +491,7 @@ def media_path(value: str | None, kind: str) -> str | None: brand=_json_arg(args.brand, "--brand"), font_family=args.font_family, captions=not args.no_captions, - face_map=_json_file(args.face_map, "--face-map"), + face_map=face_map, crop_keyframes=_json_file(args.crop_keyframes, "--crop-keyframes"), )