From 044474715f6970b859e05bc602f3bc61ada39289 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Tue, 18 Aug 2026 05:53:50 +0000
Subject: [PATCH] plain arm: callee-name residual detail + the drill's proposer
 stage

Two pieces of increment 2, both scoped to what could actually be measured
this round.

1. non_literal_assign's detail now carries the callee name for a resolvable
   call (`field(...)` -> "call:field"), falling back to the generic "call"
   tag when the callee is itself unresolvable (a call, lambda, subscript -
   never guessed). Measured payoff on dismech: the 451-row "call" bucket
   from increment 1 decomposes losslessly into three named LinkML schema
   factories - call:PermissibleValue 355, call:EnumDefinition 91,
   call:ConfigDict 5 (355+91+5=451, exact) - which is exactly the kind of
   drill target increment 1's own histogram predicted without being able
   to name.

2. New drill.rs: the proposer stage. propose() groups a residual ledger by
   (reason, detail) for the four reasons dense enough to carry one, and
   thresholds on support. classify_across_corpora() runs propose() over
   2+ labeled corpora and splits candidates into Generic (cleared the bar
   in EVERY supplied corpus) vs CorpusScoped (some but not all) - an
   exact, falsifiable rule, not a similarity heuristic.

   Scope boundary, stated in the module doc and here: this builds
   grouping + classification ONLY. It does not build a promotion gate
   that re-runs extraction with a candidate row "active" and checks the
   coverage delta - that needs a config-consuming extractor (a trie the
   plain arm reads before deciding a site's classification) that does not
   exist yet. Claiming this stage ratifies rows would overstate what it
   measures.

Measured, min_support=2, three real corpora (dismech / A2UI SDK /
ruff/scripts): 18 candidate rows, ZERO generic. This falsifies my own
prediction from the increment-1 handoff that some row would prove
cross-corpus generic - worth recording plainly rather than quietly
dropping. unresolved_annotation/binop comes closest (96 in dismech, 12 in
ruff/scripts) but is silent on the A2UI corpus, so by the exact "fires in
ALL supplied corpora" rule it stays CorpusScoped. That is the strict rule
working as designed, not a bug: three modest corpora is a small n for
"generic," and the alternative (a fuzzy magnitude-similarity threshold)
would have manufactured a Generic verdict this data does not support.

plain_propose new example: runs classify_across_corpora over N labeled
corpora and prints candidates as TOML - data, not Rust, per the earlier
design note that the proposer must never write code.

Falsifier discipline: every new grouping/threshold/classification site
verified by a disable run (mutation asserted present, test observed red,
restored, full suite green) - the min_support filter, the Generic/
CorpusScoped split, and the callee-name fallback.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AGVLyRZNEKKBSfBDJfbY3V
---
 .../ruff_python_spo/examples/plain_propose.rs |  73 +++++
 crates/ruff_python_spo/src/drill.rs           | 273 ++++++++++++++++++
 crates/ruff_python_spo/src/lib.rs             |   2 +
 crates/ruff_python_spo/src/plain.rs           |  45 ++-
 4 files changed, 392 insertions(+), 1 deletion(-)
 create mode 100644 crates/ruff_python_spo/examples/plain_propose.rs
 create mode 100644 crates/ruff_python_spo/src/drill.rs

diff --git a/crates/ruff_python_spo/examples/plain_propose.rs b/crates/ruff_python_spo/examples/plain_propose.rs
new file mode 100644
index 00000000000000..e66376f149a13c
--- /dev/null
+++ b/crates/ruff_python_spo/examples/plain_propose.rs
@@ -0,0 +1,73 @@
+//! Dev probe: run the plain-Python residual proposer over 2+ corpora and
+//! print candidate config rows as TOML — data, not Rust — with each row's
+//! `scope` (`generic` iff it cleared `min_support` in EVERY supplied
+//! corpus, `corpus_scoped` otherwise). See `drill.rs`'s module doc for
+//! exactly what this claims (grouping + classification) and what it does
+//! NOT claim (a promotion gate — that needs a config-consuming
+//! extractor this probe does not build).
+//!
+//! Usage: `cargo run -p ruff_python_spo --example plain_propose -- \
+//!     <min_support> <label1>=<root1> [<label2>=<root2> ...]`
+//! (2+ corpora required — see `classify_across_corpora`'s doc comment.)
+
+#![expect(
+    clippy::print_stdout,
+    reason = "a dev probe's stdout report IS its deliverable"
+)]
+
+use std::path::Path;
+
+use ruff_python_spo::{RowScope, classify_across_corpora, extract_plain_with_residuals};
+
+fn main() {
+    let mut args = std::env::args().skip(1);
+    let min_support: usize = args
+        .next()
+        .expect("usage: plain_propose <min_support> <label>=<root> [<label>=<root> ...]")
+        .parse()
+        .expect("min_support must be a positive integer");
+
+    let labeled: Vec<(String, Vec<ruff_python_spo::PlainResidual>)> = args
+        .map(|arg| {
+            let (label, root) = arg
+                .split_once('=')
+                .unwrap_or_else(|| panic!("expected <label>=<root>, got {arg:?}"));
+            let (_, residuals) = extract_plain_with_residuals(Path::new(root), "py");
+            (label.to_string(), residuals)
+        })
+        .collect();
+
+    assert!(
+        labeled.len() >= 2,
+        "need 2+ corpora for a meaningful generic/corpus_scoped split; got {}",
+        labeled.len()
+    );
+
+    let rows = classify_across_corpora(&labeled, min_support);
+    let generic = rows.iter().filter(|r| r.scope == RowScope::Generic).count();
+    let scoped = rows.len() - generic;
+    println!(
+        "# {} candidate rows ({generic} generic, {scoped} corpus_scoped) from {} corpora, min_support={min_support}",
+        rows.len(),
+        labeled.len()
+    );
+    for row in &rows {
+        let scope = match row.scope {
+            RowScope::Generic => "generic",
+            RowScope::CorpusScoped => "corpus_scoped",
+        };
+        println!("[[candidate]]");
+        println!("reason = \"{}\"", row.reason.as_str());
+        println!("detail = \"{}\"", row.detail);
+        println!("scope = \"{scope}\"");
+        print!("per_corpus = [");
+        for (i, (label, support)) in row.per_corpus.iter().enumerate() {
+            if i > 0 {
+                print!(", ");
+            }
+            print!("{{ corpus = \"{label}\", support = {support} }}");
+        }
+        println!("]");
+        println!();
+    }
+}
diff --git a/crates/ruff_python_spo/src/drill.rs b/crates/ruff_python_spo/src/drill.rs
new file mode 100644
index 00000000000000..6104708f3f7f49
--- /dev/null
+++ b/crates/ruff_python_spo/src/drill.rs
@@ -0,0 +1,273 @@
+//! The self-adaptive drill's proposer — turns a concentrated residual
+//! ledger ([`crate::plain::PlainResidual`]) into candidate config rows.
+//!
+//! # Scope boundary (read before extending this module)
+//!
+//! This module builds the **grouping and cross-corpus classification**
+//! stage only. It does NOT build a promotion gate that re-runs extraction
+//! with a candidate row "active" and checks the coverage delta is exactly
+//! the row's support — that check needs a config-consuming extractor
+//! (a trie the plain arm reads before deciding a site's classification),
+//! which does not exist yet. Building that engine is the next increment;
+//! claiming this module ratifies rows would overstate what it measures.
+//!
+//! What this module DOES do, honestly: group residual rows by
+//! `(reason, detail)`, threshold on support, and — when residuals from
+//! more than one corpus are supplied — classify each candidate as
+//! [`RowScope::Generic`] (fires in every corpus measured) or
+//! [`RowScope::CorpusScoped`] (fires in some but not all). That
+//! generic/scoped split is itself a measured, falsifiable fact: a row is
+//! `Generic` only because it was observed above `min_support` in EVERY
+//! supplied corpus, not because of a similarity heuristic.
+//!
+//! A candidate row is data-shaped by design (`reason` + `detail` string +
+//! per-corpus support) so a downstream session can serialise it straight
+//! to TOML without inventing a second representation.
+
+use std::collections::BTreeMap;
+
+use crate::plain::{PlainResidual, PlainResidualReason};
+
+/// Reasons whose `detail` field is dense enough to drill on. The other
+/// four reasons ([`PlainResidualReason::UnparsableSource`],
+/// [`PlainResidualReason::ModuleConstant`] without a CURIE shape,
+/// [`PlainResidualReason::NestedClass`],
+/// [`PlainResidualReason::NonNameAssignTarget`]) never carry a `detail`
+/// today, so grouping by `(reason, detail)` would only ever produce a
+/// `None` bucket for them — filtered out rather than proposed as an
+/// empty-detail row.
+const DRILLABLE_REASONS: &[PlainResidualReason] = &[
+    PlainResidualReason::CurieConstant,
+    PlainResidualReason::NonLiteralAssign,
+    PlainResidualReason::UnresolvedAnnotation,
+    PlainResidualReason::UnresolvedBase,
+];
+
+/// One proposed config row: a `(reason, detail)` pair that recurred at
+/// least `min_support` times in a residual ledger. Never auto-applied —
+/// see the module doc's scope boundary.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CandidateRow {
+    pub reason: PlainResidualReason,
+    pub detail: String,
+    pub support: usize,
+}
+
+/// Group `residuals` by `(reason, detail)` for the `DRILLABLE_REASONS`,
+/// keeping only groups with `support >= min_support`. Sorted by support
+/// descending, then `(reason, detail)` for determinism.
+#[must_use]
+pub fn propose(residuals: &[PlainResidual], min_support: usize) -> Vec<CandidateRow> {
+    let mut counts: BTreeMap<(PlainResidualReason, &str), usize> = BTreeMap::new();
+    for r in residuals {
+        if !DRILLABLE_REASONS.contains(&r.reason) {
+            continue;
+        }
+        let Some(detail) = r.detail.as_deref() else {
+            continue;
+        };
+        *counts.entry((r.reason, detail)).or_insert(0) += 1;
+    }
+    let mut rows: Vec<CandidateRow> = counts
+        .into_iter()
+        .filter(|&(_, support)| support >= min_support)
+        .map(|((reason, detail), support)| CandidateRow {
+            reason,
+            detail: detail.to_string(),
+            support,
+        })
+        .collect();
+    rows.sort_by(|a, b| {
+        b.support
+            .cmp(&a.support)
+            .then_with(|| a.reason.as_str().cmp(b.reason.as_str()))
+            .then_with(|| a.detail.cmp(&b.detail))
+    });
+    rows
+}
+
+/// Whether a cross-corpus candidate fired everywhere it was measured, or
+/// only in a subset — see the module doc for exactly what this claims
+/// and what it deliberately does not.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum RowScope {
+    /// Cleared `min_support` in every supplied corpus.
+    Generic,
+    /// Cleared `min_support` in at least one, but not all, supplied
+    /// corpora.
+    CorpusScoped,
+}
+
+/// A [`CandidateRow`] classified across the corpora it was measured
+/// against, with the per-corpus support that produced the classification.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CrossCorpusRow {
+    pub reason: PlainResidualReason,
+    pub detail: String,
+    pub scope: RowScope,
+    /// `(corpus label, support)`, one entry per corpus the row cleared
+    /// `min_support` in — sorted by label for determinism. A corpus the
+    /// row did NOT clear the bar in is simply absent, not zero-padded:
+    /// absence here means "not measured as support", the same posture
+    /// the residual ledger itself takes (a reason that never fired gets
+    /// no row, not a row with `support: 0`).
+    pub per_corpus: Vec<(String, usize)>,
+}
+
+/// Propose candidates independently over each `(corpus_label, residuals)`
+/// pair, then classify each `(reason, detail)` that appears in at least
+/// one corpus's proposal by how many of the SUPPLIED corpora it cleared
+/// `min_support` in. Requires 2+ corpora to be meaningful — with one
+/// corpus every row is trivially cross-corpus-generic, which the calling
+/// convention (never call this with fewer than two) exists to avoid.
+#[must_use]
+pub fn classify_across_corpora(
+    labeled: &[(String, Vec<PlainResidual>)],
+    min_support: usize,
+) -> Vec<CrossCorpusRow> {
+    let total_corpora = labeled.len();
+    let per_corpus_candidates: Vec<(&str, Vec<CandidateRow>)> = labeled
+        .iter()
+        .map(|(label, residuals)| (label.as_str(), propose(residuals, min_support)))
+        .collect();
+
+    let mut merged: BTreeMap<(PlainResidualReason, String), Vec<(String, usize)>> = BTreeMap::new();
+    for (label, rows) in &per_corpus_candidates {
+        for row in rows {
+            merged
+                .entry((row.reason, row.detail.clone()))
+                .or_default()
+                .push(((*label).to_string(), row.support));
+        }
+    }
+
+    merged
+        .into_iter()
+        .map(|((reason, detail), mut per_corpus)| {
+            per_corpus.sort_by(|a, b| a.0.cmp(&b.0));
+            let scope = if per_corpus.len() == total_corpora {
+                RowScope::Generic
+            } else {
+                RowScope::CorpusScoped
+            };
+            CrossCorpusRow {
+                reason,
+                detail,
+                scope,
+                per_corpus,
+            }
+        })
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::plain::extract_plain_from_source_with_residuals;
+
+    fn residuals_of(src: &str) -> Vec<PlainResidual> {
+        extract_plain_from_source_with_residuals(src, "mod").1
+    }
+
+    #[test]
+    fn propose_groups_by_reason_and_detail_and_thresholds_on_support() {
+        let src = r#"
+class A:
+    x = field()
+    y = field()
+    z = field()
+    w = other()
+"#;
+        let residuals = residuals_of(src);
+        let rows = propose(&residuals, 2);
+        // "call:field" support=3 clears; "call:other" support=1 doesn't.
+        assert_eq!(rows.len(), 1);
+        assert_eq!(rows[0].reason, PlainResidualReason::NonLiteralAssign);
+        assert_eq!(rows[0].detail, "call:field");
+        assert_eq!(rows[0].support, 3);
+    }
+
+    #[test]
+    fn propose_never_emits_a_row_for_a_detail_free_reason() {
+        // ModuleConstant (non-CURIE) and NestedClass never carry `detail`
+        // — propose() must not synthesise an empty-detail row for them.
+        let src = r#"
+NAME = "x"
+COUNT = 3
+class Outer:
+    class Inner:
+        pass
+    a, b = 1, 2
+"#;
+        let residuals = residuals_of(src);
+        assert!(!residuals.is_empty(), "fixture must produce residue");
+        let rows = propose(&residuals, 1);
+        assert!(rows.is_empty());
+    }
+
+    #[test]
+    fn classify_across_corpora_splits_generic_from_corpus_scoped() {
+        // "call:field" fires in BOTH corpora -> Generic.
+        // "call:only_a" fires ONLY in corpus a -> CorpusScoped.
+        let corpus_a = r#"
+class A:
+    x = field()
+    y = field()
+    z = only_a()
+    w = only_a()
+"#;
+        let corpus_b = r#"
+class B:
+    x = field()
+    y = field()
+"#;
+        let labeled = vec![
+            ("a".to_string(), residuals_of(corpus_a)),
+            ("b".to_string(), residuals_of(corpus_b)),
+        ];
+        let rows = classify_across_corpora(&labeled, 2);
+        assert_eq!(rows.len(), 2);
+
+        let field_row = rows
+            .iter()
+            .find(|r| r.detail == "call:field")
+            .expect("call:field row");
+        assert_eq!(field_row.scope, RowScope::Generic);
+        assert_eq!(field_row.per_corpus.len(), 2);
+        assert_eq!(
+            field_row.per_corpus,
+            vec![("a".to_string(), 2), ("b".to_string(), 2)]
+        );
+
+        let only_a_row = rows
+            .iter()
+            .find(|r| r.detail == "call:only_a")
+            .expect("call:only_a row");
+        assert_eq!(only_a_row.scope, RowScope::CorpusScoped);
+        assert_eq!(only_a_row.per_corpus, vec![("a".to_string(), 2)]);
+    }
+
+    #[test]
+    fn classify_across_corpora_with_curie_prefixes_matches_the_measured_shape() {
+        // Mirrors the real measurement: a CURIE prefix that only appears
+        // in an ontology-bearing corpus must classify CorpusScoped, never
+        // Generic, when a sibling corpus has zero CURIE constants.
+        let ontology_corpus = r#"
+A = "dismech:a"
+B = "dismech:b"
+"#;
+        let plain_corpus = r#"
+NAME = "x"
+"#;
+        let labeled = vec![
+            ("ontology".to_string(), residuals_of(ontology_corpus)),
+            ("plain".to_string(), residuals_of(plain_corpus)),
+        ];
+        let rows = classify_across_corpora(&labeled, 2);
+        assert_eq!(rows.len(), 1);
+        assert_eq!(rows[0].reason, PlainResidualReason::CurieConstant);
+        assert_eq!(rows[0].detail, "dismech");
+        assert_eq!(rows[0].scope, RowScope::CorpusScoped);
+        assert_eq!(rows[0].per_corpus, vec![("ontology".to_string(), 2)]);
+    }
+}
diff --git a/crates/ruff_python_spo/src/lib.rs b/crates/ruff_python_spo/src/lib.rs
index e4241300f42621..916db004a379ef 100644
--- a/crates/ruff_python_spo/src/lib.rs
+++ b/crates/ruff_python_spo/src/lib.rs
@@ -45,6 +45,7 @@ use std::path::Path;
 use ruff_python_ast::Expr;
 use ruff_spo_triplet::{Field, Function, Model, ModelGraph};
 
+mod drill;
 mod functions;
 mod navigation;
 mod odoo_nav;
@@ -56,6 +57,7 @@ mod plain;
 mod templates;
 mod walk;
 
+pub use drill::{CandidateRow, CrossCorpusRow, RowScope, classify_across_corpora, propose};
 pub use navigation::{
     NavScanReport, NavVocab, PyNavEdge, extract_nav_edges, extract_nav_edges_with_report,
 };
diff --git a/crates/ruff_python_spo/src/plain.rs b/crates/ruff_python_spo/src/plain.rs
index 3230e3788495ee..093760eb502bcc 100644
--- a/crates/ruff_python_spo/src/plain.rs
+++ b/crates/ruff_python_spo/src/plain.rs
@@ -334,6 +334,24 @@ fn curie_prefix_of_literal(value: &Expr) -> Option<String> {
     (prefix_ok && rest_ok).then(|| prefix.to_string())
 }
 
+/// The detail for a [`PlainResidualReason::NonLiteralAssign`] row: for a
+/// call expression whose callee resolves to a terminal name, `"call:<name>"`
+/// (`field(default_factory=list)` -> `"call:field"`, `a.b.factory()` ->
+/// `"call:factory"`) so the drill signal can distinguish which factory
+/// functions dominate a corpus's non-literal assigns instead of lumping
+/// every call under one bucket; anything else falls back to
+/// [`expr_kind`] unchanged (a call whose own callee is itself a call,
+/// subscript, etc. stays the generic `"call"` — never guessed).
+fn non_literal_assign_detail(expr: &Expr) -> String {
+    if let Expr::Call(call) = expr
+        && let Some(callee) = base_name(&call.func)
+    {
+        format!("call:{callee}")
+    } else {
+        expr_kind(expr).to_string()
+    }
+}
+
 /// Compact kind tag for an expression, used as residual `detail`. The
 /// closed reason taxonomy lives in [`PlainResidualReason`]; this tag is
 /// free-text granularity underneath it, so the trailing arm is acceptable
@@ -437,7 +455,7 @@ fn walk_plain_class(
                         residuals.push(residual(
                             PlainResidualReason::NonLiteralAssign,
                             Some(name.to_string()),
-                            Some(expr_kind(&assign.value).to_string()),
+                            Some(non_literal_assign_detail(&assign.value)),
                         ));
                     }
                 } else {
@@ -823,6 +841,31 @@ class Sample(Base, Generic[T]):
         assert_eq!(rows.len(), 7);
     }
 
+    #[test]
+    fn non_literal_assign_detail_names_the_call_and_falls_back_when_unresolvable() {
+        let src = r#"
+class Row:
+    a = field(default_factory=list)
+    b = some.module.factory()
+    c = get_default()()
+    d = (lambda: 1)()
+"#;
+        let (_, rows) = extract_plain_from_source_with_residuals(src, "mod");
+        let details: Vec<&str> = rows
+            .iter()
+            .filter(|r| r.reason == PlainResidualReason::NonLiteralAssign)
+            .filter_map(|r| r.detail.as_deref())
+            .collect();
+        assert_eq!(details.len(), 4);
+        assert!(details.contains(&"call:field"));
+        assert!(details.contains(&"call:factory"));
+        // `get_default()()` — the callee is itself a call, not a
+        // Name/Attribute, so it stays the generic tag rather than guessing.
+        assert!(details.contains(&"call"));
+        // `(lambda: 1)()` — same: the callee is a lambda expression.
+        assert_eq!(details.iter().filter(|&&d| d == "call").count(), 2);
+    }
+
     #[test]
     fn unparsable_source_lands_one_addressed_ledger_row() {
         let (graph, rows) =