Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions src/keboola_agent_cli/commands/lineage.py
Original file line number Diff line number Diff line change
Expand Up @@ -397,11 +397,15 @@ def _output_mermaid_or_html(
edges = query_result.get("edges", [])

if output_format == "er":
er_code = DeepLineageService.render_er_diagram(edges, graph, node_fqn)
er_code = DeepLineageService.render_er_diagram(
edges, graph, node_fqn, warnings=query_result.get("warnings")
)
typer.echo(er_code)
return

mermaid_code = DeepLineageService.render_mermaid(edges, graph, direction, node_fqn)
mermaid_code = DeepLineageService.render_mermaid(
edges, graph, direction, node_fqn, warnings=query_result.get("warnings")
)

if output_format == "mermaid":
typer.echo(mermaid_code)
Expand Down Expand Up @@ -1360,6 +1364,7 @@ def _handle_mermaid(self, parsed) -> None:
self.graph,
node,
show_columns=show_cols,
warnings=result.get("warnings"),
)
else:
mermaid_code = DeepLineageService.render_mermaid(
Expand All @@ -1368,6 +1373,7 @@ def _handle_mermaid(self, parsed) -> None:
direction,
node,
show_columns=show_cols,
warnings=result.get("warnings"),
Comment thread
padak marked this conversation as resolved.
)
self._serve(mermaid_code, "text/plain")

Expand Down
6 changes: 4 additions & 2 deletions src/keboola_agent_cli/server/routers/lineage.py
Original file line number Diff line number Diff line change
Expand Up @@ -218,9 +218,11 @@ def mermaid(
edges = result.get("edges", [])
show_cols = columns == "true"
if view == "er":
code = DeepLineageService.render_er_diagram(edges, graph, node, show_columns=show_cols)
code = DeepLineageService.render_er_diagram(
edges, graph, node, show_columns=show_cols, warnings=result.get("warnings")
)
else:
code = DeepLineageService.render_mermaid(
edges, graph, direction, node, show_columns=show_cols
edges, graph, direction, node, show_columns=show_cols, warnings=result.get("warnings")
)
return PlainTextResponse(code)
46 changes: 40 additions & 6 deletions src/keboola_agent_cli/services/deep_lineage_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -393,8 +393,15 @@ def _ambiguity_warning(identifier: str, candidates: list[str]) -> str:
``_find_node_candidates``, which matches a bare table name across buckets:
those candidates can share a single project, so counting them as projects
would tell the user a table "exists in 2 projects (alpha, alpha)", and
``<project>:<identifier>`` would not resolve because the real node ids
``PROJECT:<identifier>`` would not resolve because the real node ids
carry a bucket. That case gets the full ids instead.

The placeholder is spelled ``PROJECT``, not ``<project>``: this text also
reaches the mermaid diagrams, where it is rendered into SVG. Mermaid turns
the escaped entity back into a literal ``<project>``, which the browser
then swallows as an unknown tag -- the remedy silently loses the very part
that makes it actionable. Escaping is correct and holds; the loss happens
one layer further on, so keep angle brackets out of warning text (#584).
"""
shown = candidates[0]
if all(fqn.partition(":")[2] == identifier for fqn in candidates):
Expand All @@ -403,7 +410,7 @@ def _ambiguity_warning(identifier: str, candidates: list[str]) -> str:
f"'{identifier}' exists in {len(projects)} projects "
f"({_format_candidates(projects)}); showing '{shown}' only. "
f"Query a specific one with '--upstream/--downstream "
f"<project>:{identifier}' or scope with --project."
f"PROJECT:{identifier}' or scope with --project."
)
return (
f"'{identifier}' matches {len(candidates)} nodes "
Expand Down Expand Up @@ -1063,10 +1070,6 @@ def _graph_from_dict(data: dict) -> LineageGraph:
)
return graph

def _find_node(self, graph: LineageGraph, identifier: str, project: str = "") -> str | None:
candidates = self._find_node_candidates(graph, identifier, project)
return candidates[0] if candidates else None

def _find_node_candidates(
self, graph: LineageGraph, identifier: str, project: str = ""
) -> list[str]:
Expand Down Expand Up @@ -1120,6 +1123,7 @@ def render_mermaid(
direction: str,
node_fqn: str,
show_columns: bool = False,
warnings: list[str] | None = None,
) -> str:
"""Render lineage edges as a mermaid flowchart.

Expand All @@ -1129,6 +1133,12 @@ def render_mermaid(
direction: "upstream" or "downstream".
node_fqn: The FQN of the queried node.
show_columns: If True, include column names in table labels.
warnings: Non-fatal notes to render into the diagram itself,
typically the ambiguity warning from an unqualified id (#568).
A diagram carries no metadata channel the way the JSON shapes
do, so a caller that has warnings and drops them leaves the
viewer with one project's answer looking like the whole
picture -- which is the bug, not a cosmetic omission.

Returns:
Mermaid flowchart source code.
Expand All @@ -1139,6 +1149,13 @@ def render_mermaid(
graph_dir = "RL" if direction == "upstream" else "LR"
lines: list[str] = [f"graph {graph_dir}"]

# Standalone nodes -- deliberately unconnected, so they read as a note
# on the diagram rather than as part of the dependency graph.
for index, warning in enumerate(warnings or []):
warning_id = f"kbagentNote{index}"
lines.append(f' {warning_id}["⚠ {escape(warning)}"]')
lines.append(f" style {warning_id} fill:#fff3cd,stroke:#856404,color:#856404")

# Determine the root node's project for cross-project detection
root_project = node_fqn.split(":")[0] if ":" in node_fqn else ""

Expand Down Expand Up @@ -1202,14 +1219,31 @@ def render_er_diagram(
graph: LineageGraph,
node_fqn: str,
show_columns: bool = False,
warnings: list[str] | None = None,
) -> str:
"""Render lineage as a mermaid ER diagram.

Without show_columns: entities with name/row count + relationships.
With show_columns: full column list with PK markers and AI mappings.

``warnings`` carries non-fatal notes -- the ambiguity warning of #568,
typically. ``erDiagram`` has no free-standing annotation the way a
flowchart does, and the HTTP surfaces return this as plain text with
no metadata channel, so the notes ride as one relationship-less entity.
Dropping them would leave an ER viewer with one project's answer
looking like the whole picture, which is the bug itself (#584).
"""
lines: list[str] = ["erDiagram"]

if warnings:
lines.append(' "⚠ note" {')
for index, warning in enumerate(warnings, start=1):
# html.escape(quote=True) also neutralises the double quotes
# that would otherwise terminate the attribute comment.
attribute = "warning" if len(warnings) == 1 else f"warning_{index}"
lines.append(f' string {attribute} "{html.escape(warning, quote=True)}"')
lines.append(" }")

# Collect all table FQNs and their column mappings
table_fqns: set[str] = set()
config_fqns: set[str] = set()
Expand Down
102 changes: 101 additions & 1 deletion tests/test_deep_lineage_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -676,13 +676,113 @@ def test_name_only_match_suggests_a_retry_that_resolves(self, tmp_path: Path) ->

warning = service.query_downstream(graph, "orders")["warnings"][0]

assert "<project>:orders" not in warning
assert "PROJECT:orders" not in warning
suggested = "alpha:in.c-a.orders"
assert suggested in warning
retry = service.query_downstream(graph, suggested)
assert "error" not in retry
assert retry["node"] == suggested

def test_mermaid_carries_the_warning_into_the_diagram(self, tmp_path: Path) -> None:
"""A diagram has no metadata channel, so the note has to be a node (#584)."""
service = self._service(tmp_path)
graph = _shared_table_graph()
result = service.query_downstream(graph, SHARED_TABLE)

code = DeepLineageService.render_mermaid(
result["edges"],
graph,
"downstream",
result["node"],
warnings=result.get("warnings"),
)

assert "kbagentNote0" in code
assert "2 projects" in code
# Standalone: the note must not become part of the dependency graph.
assert "kbagentNote0 --" not in code and "--> kbagentNote0" not in code

def test_mermaid_without_warnings_gains_no_note(self, tmp_path: Path) -> None:
service = self._service(tmp_path)
graph = _shared_table_graph()
result = service.query_downstream(graph, f"beta:{SHARED_TABLE}")

code = DeepLineageService.render_mermaid(
result["edges"], graph, "downstream", result["node"], warnings=result.get("warnings")
)

assert "kbagentNote" not in code

def test_mermaid_warning_is_escaped(self) -> None:
"""Warnings reach the diagram as text; sec-05 escaping must apply to them too."""
code = DeepLineageService.render_mermaid(
[], LineageGraph(), "downstream", "alpha:in.c-a.t", warnings=['<script>"x"</script>']
)

assert "<script>" not in code
assert "&lt;script&gt;" in code
assert "&quot;x&quot;" in code

def test_warning_text_carries_no_angle_brackets(self, tmp_path: Path) -> None:
"""Angle brackets survive escaping but not the browser (#584).

The warning also reaches the mermaid diagrams. Mermaid renders the
escaped entity back to a literal ``<project>`` in SVG text, where the
browser drops it as an unknown tag -- so the remedy loses the part
that makes it usable, while looking correct in a terminal and in every
string assertion. Keep the placeholder bracket-free.
"""
service = self._service(tmp_path)

cross_project = service.query_downstream(_shared_table_graph(), SHARED_TABLE)
same_project = service.query_downstream(self._one_project_two_buckets(), "orders")

for result in (cross_project, same_project):
warning = result["warnings"][0]
assert "<" not in warning and ">" not in warning, warning

def test_er_diagram_carries_the_warning_too(self, tmp_path: Path) -> None:
"""The ER view is a separate renderer and was dropping it (#584)."""
service = self._service(tmp_path)
graph = _shared_table_graph()
result = service.query_downstream(graph, SHARED_TABLE)

code = DeepLineageService.render_er_diagram(
result["edges"], graph, result["node"], warnings=result.get("warnings")
)

assert '"⚠ note" {' in code
assert "2 projects" in code

def test_er_diagram_without_warnings_gains_no_note(self, tmp_path: Path) -> None:
service = self._service(tmp_path)
graph = _shared_table_graph()
result = service.query_downstream(graph, f"beta:{SHARED_TABLE}")

code = DeepLineageService.render_er_diagram(
result["edges"], graph, result["node"], warnings=result.get("warnings")
)

assert "⚠ note" not in code

def test_er_diagram_warning_cannot_break_out_of_its_comment(self) -> None:
"""A raw double quote would terminate the attribute comment (sec-05)."""
code = DeepLineageService.render_er_diagram(
[], LineageGraph(), "alpha:in.c-a.t", warnings=['say "hi" <script>alert(1)</script>']
)

assert '"hi"' not in code
assert "&quot;hi&quot;" in code
assert "<script>" not in code

def test_er_diagram_numbers_multiple_warnings(self) -> None:
code = DeepLineageService.render_er_diagram(
[], LineageGraph(), "alpha:in.c-a.t", warnings=["first", "second"]
)

assert "string warning_1 " in code
assert "string warning_2 " in code

def test_unknown_identifier_still_errors(self, tmp_path: Path) -> None:
service = self._service(tmp_path)
result = service.query_downstream(_shared_table_graph(), "in.c-nope.missing")
Expand Down