diff --git a/README.md b/README.md index e3d05b0..090a234 100644 --- a/README.md +++ b/README.md @@ -222,8 +222,13 @@ The alerts stream endpoint returns lightweight `StreamingWatcherAlert` objects t `fetch_alert_targets` resolves each alert's API link in parallel and pairs it with the fully fetched target object — a report, forum post, credential, indicator, or any other supported type. +When a target cannot be fetched (no link, no known SDK route, forbidden, or an unexpected error), +the alert is still returned by default with `target=None` and a `status` explaining why, so nothing +is silently lost. Pass `skip_missing_targets=True` to omit such alerts instead. Marketplace alerts +are always skipped silently. + ```python -from verity471.helpers import fetch_alert_targets, AlertTarget +from verity471.helpers import fetch_alert_targets, AlertTarget, AlertTargetStatus ``` ### Parameters @@ -232,7 +237,8 @@ from verity471.helpers import fetch_alert_targets, AlertTarget |---|---|---|---| | `alerts_response` | `StreamingAlertsResponse` | *(required)* | The page returned by `AlertsApi.get_alerts_stream()`. | | `api_client` | `ApiClient` | *(required)* | An active `ApiClient` instance (must share credentials with the alerts call). | -| `raise_on_error` | `bool` | `False` | When `True`, re-raise exceptions instead of logging and skipping the alert. | +| `raise_on_error` | `bool` | `False` | When `True`, re-raise unexpected errors (and the missing-link error) instead of recording them on the result. | +| `skip_missing_targets` | `bool` | `False` | When `True`, alerts whose target cannot be fetched are omitted from the result. When `False` (default), they are returned with `target=None` and a failure `status`. Marketplace alerts are always skipped regardless. | ### Returns @@ -243,8 +249,10 @@ Each `AlertTarget` exposes: | Attribute | Type | Description | |---|---|---| | `.alert` | `StreamingWatcherAlert` | The original alert object (status, watcher IDs, timestamps, highlights, etc.). | -| `.target` | model instance or `None` | The resolved API object (report, post, credential, …). `None` when the URL could not be mapped to a known SDK route. | -| `.target_summary` | `str \| None` | A compact, human-readable one-liner describing the target. | +| `.target` | model instance or `None` | The resolved API object (report, post, credential, …). `None` when the target could not be fetched (see `.status`). | +| `.status` | `AlertTargetStatus` | The fetch outcome: `OK` when the target was fetched, otherwise `NO_LINK`, `UNRESOLVABLE`, `FORBIDDEN`, or `ERROR`. | +| `.status_reason` | `str \| None` | A human-readable detail for a non-`OK` status (e.g. the URL or the underlying error message). `None` when `status` is `OK`. | +| `.target_summary` | `str \| None` | A compact, human-readable one-liner describing the target. Falls back to a summary built from the alert envelope (type, link, first highlight snippet) when the target is missing or not summarizable. | | `.watcher` | `GetWatcherResponse \| None` | The full watcher object that triggered this alert (name, DSL query, mute status, etc.). `None` if the watcher ID was not found in the user's watcher list. | | `.watcher_group` | `GetWatcherGroupResponse \| None` | The full watcher group object the watcher belongs to (name, description, etc.). `None` if not found. | @@ -256,7 +264,7 @@ share the same watcher. ```python import verity471 -from verity471.helpers import fetch_alert_targets +from verity471.helpers import fetch_alert_targets, AlertTargetStatus configuration = verity471.Configuration( username="your_username", @@ -271,6 +279,9 @@ with verity471.ApiClient(configuration) as api_client: for t in targets: watcher_name = t.watcher.name if t.watcher else None group_name = t.watcher_group.name if t.watcher_group else None + if t.status is not AlertTargetStatus.OK: + print(t.alert.source_type, t.alert.status, f"[{t.status.value}: {t.status_reason}]", t.target_summary) + continue print(t.alert.source_type, t.alert.status, watcher_name, group_name, t.target_summary) ``` @@ -280,6 +291,7 @@ with verity471.ApiClient(configuration) as api_client: fintel read threat_actor Ransomware actors [Fintel] Threat Landscape: Q1 2025 Summary | 2025-03-15T12:00:00Z | Key findings from the first quarter include… forum_post unread ddos_monitor My Watchers [Forum Post] Selling access to corporate VPN… | 2025-03-14T08:30:00Z credential_occurrence unread cred_watcher Credential Alerts [Credential Occurrence] https://example.com/login | email | 2025-03-13T10:00:00Z +forum_post unread [forbidden: Forbidden (403) fetching target] Forums Post: https://api.intel471.cloud/integrations/sources/v1/forums/posts/post--… | …matched snippet text… malware_report read malware_tracker My Watchers [Malware Report] New variant of Lumma Stealer | 2025-03-12T15:45:00Z | A new variant has been observed… ``` diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 6c1ca5d..98735f3 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -5,9 +5,11 @@ import pytest from tests.conftest import PREFIX, read_fixture -from verity471.helpers import fetch_alert_targets +from types import SimpleNamespace + +from verity471.helpers import fetch_alert_targets, AlertTargetStatus from verity471.exceptions import ForbiddenException -from verity471.helpers.alerts import _patch_portal_url +from verity471.helpers.alerts import _patch_portal_url, _summarize_alert from verity471.helpers.url_router import UnresolvableURL from verity471.models.get_watcher_response import GetWatcherResponse from verity471.models.get_watcher_group_response import GetWatcherGroupResponse @@ -227,6 +229,19 @@ def _alerts_response(*alerts): return resp +def _alert_like(source_type="forums_post", portal=None, api=_SPOT_URL, snippets=None, + source_id="src--test", watcher_id=1, watcher_group_id=1): + """A lightweight alert with real string attributes (for summary tests).""" + links = SimpleNamespace( + verity_portal=SimpleNamespace(href=portal) if portal else None, + verity_api=SimpleNamespace(href=api) if api else None, + ) + highlights = [SimpleNamespace(field_name="x", snippets=snippets)] if snippets else None + return SimpleNamespace( + source_type=source_type, links=links, highlights=highlights, + source_id=source_id, watcher_id=watcher_id, watcher_group_id=watcher_group_id) + + def _no_watchers(watchers_mock): watchers_mock.return_value.get_watchers.return_value.watchers = [] watchers_mock.return_value.get_watcher_groups.return_value.watchers_groups = [] @@ -253,7 +268,7 @@ def test_missing_link_is_skipped(self, _call_url_mock, _watchers_mock): alert = _mock_alert() alert.links = None with verity471.ApiClient(configuration) as api_client: - result = fetch_alert_targets(_alerts_response(alert), api_client) + result = fetch_alert_targets(_alerts_response(alert), api_client, skip_missing_targets=True) assert result == [] def test_missing_link_raises_when_requested(self, _call_url_mock, _watchers_mock): @@ -272,19 +287,86 @@ def test_unresolvable_url_yields_none_target(self, call_url_mock, watchers_mock) assert len(result) == 1 assert result[0].alert is alert assert result[0].target is None + assert result[0].status == AlertTargetStatus.UNRESOLVABLE + assert result[0].status_reason def test_forbidden_is_skipped(self, call_url_mock, _watchers_mock): call_url_mock.side_effect = ForbiddenException() with verity471.ApiClient(configuration) as api_client: - result = fetch_alert_targets(_alerts_response(_mock_alert()), api_client) + result = fetch_alert_targets(_alerts_response(_mock_alert()), api_client, skip_missing_targets=True) assert result == [] - def test_api_error_skipped_by_default(self, call_url_mock, _watchers_mock): + def test_api_error_skipped_when_requested(self, call_url_mock, _watchers_mock): + call_url_mock.side_effect = RuntimeError("boom") + with verity471.ApiClient(configuration) as api_client: + result = fetch_alert_targets(_alerts_response(_mock_alert()), api_client, skip_missing_targets=True) + assert result == [] + + # --- default (skip_missing_targets=False): failures kept with a status --- + + def test_missing_link_kept_with_status_by_default(self, _call_url_mock, watchers_mock): + _no_watchers(watchers_mock) + alert = _mock_alert() + alert.links = None + with verity471.ApiClient(configuration) as api_client: + result = fetch_alert_targets(_alerts_response(alert), api_client) + assert len(result) == 1 + assert result[0].target is None + assert result[0].status == AlertTargetStatus.NO_LINK + assert "link" in result[0].status_reason.lower() + + def test_forbidden_kept_with_status_by_default(self, call_url_mock, watchers_mock): + call_url_mock.side_effect = ForbiddenException() + _no_watchers(watchers_mock) + with verity471.ApiClient(configuration) as api_client: + result = fetch_alert_targets(_alerts_response(_mock_alert()), api_client) + assert len(result) == 1 + assert result[0].target is None + assert result[0].status == AlertTargetStatus.FORBIDDEN + + def test_api_error_kept_with_status_by_default(self, call_url_mock, watchers_mock): call_url_mock.side_effect = RuntimeError("boom") + _no_watchers(watchers_mock) with verity471.ApiClient(configuration) as api_client: result = fetch_alert_targets(_alerts_response(_mock_alert()), api_client) + assert len(result) == 1 + assert result[0].status == AlertTargetStatus.ERROR + assert "boom" in result[0].status_reason + + def test_unresolvable_skipped_when_skip_true(self, call_url_mock, _watchers_mock): + call_url_mock.side_effect = UnresolvableURL("no route") + with verity471.ApiClient(configuration) as api_client: + result = fetch_alert_targets(_alerts_response(_mock_alert()), api_client, skip_missing_targets=True) assert result == [] + def test_error_raise_takes_priority_over_skip_false(self, call_url_mock, _watchers_mock): + call_url_mock.side_effect = RuntimeError("boom") + with verity471.ApiClient(configuration) as api_client: + with pytest.raises(RuntimeError): + fetch_alert_targets(_alerts_response(_mock_alert()), api_client, + raise_on_error=True, skip_missing_targets=False) + + def test_success_has_ok_status(self, call_url_mock, watchers_mock): + _no_watchers(watchers_mock) + call_url_mock.return_value = MagicMock() + with verity471.ApiClient(configuration) as api_client: + result = fetch_alert_targets(_alerts_response(_mock_alert()), api_client) + assert len(result) == 1 + assert result[0].status == AlertTargetStatus.OK + assert result[0].status_reason is None + + def test_failure_result_still_enriched_with_watcher(self, call_url_mock, watchers_mock): + call_url_mock.side_effect = ForbiddenException() + watcher = MagicMock(spec=GetWatcherResponse) + watcher.id = 1 + watchers_mock.return_value.get_watchers.return_value.watchers = [watcher] + watchers_mock.return_value.get_watcher_groups.return_value.watchers_groups = [] + with verity471.ApiClient(configuration) as api_client: + result = fetch_alert_targets(_alerts_response(_mock_alert(watcher_id=1)), api_client) + assert len(result) == 1 + assert result[0].status == AlertTargetStatus.FORBIDDEN + assert result[0].watcher is watcher + def test_api_error_raises_when_requested(self, call_url_mock, _watchers_mock): call_url_mock.side_effect = RuntimeError("boom") with verity471.ApiClient(configuration) as api_client: @@ -314,4 +396,41 @@ def test_result_order_preserved(self, call_url_mock, watchers_mock): assert len(result) == 5 for i, r in enumerate(result): assert r.alert is alerts[i] - assert r.target is targets[i] \ No newline at end of file + assert r.target is targets[i] + + def test_alert_summary_used_when_target_missing(self, call_url_mock, watchers_mock): + call_url_mock.side_effect = ForbiddenException() + _no_watchers(watchers_mock) + alert = _alert_like(source_type="forums_post", api=_SPOT_URL, snippets=["hello world"]) + with verity471.ApiClient(configuration) as api_client: + result = fetch_alert_targets(_alerts_response(alert), api_client) + assert len(result) == 1 + summary = result[0].target_summary + assert summary.startswith("Forums Post: ") + assert _SPOT_URL in summary + assert "hello world" in summary + + +# --------------------------------------------------------------------------- +# Tests for the alert-envelope fallback summary (_summarize_alert). +# --------------------------------------------------------------------------- + +class TestSummarizeAlert: + + def test_uses_type_url_and_snippet(self): + summary = _summarize_alert(_alert_like(api=_SPOT_URL, snippets=["a snippet"])) + assert summary == f"Forums Post: {_SPOT_URL} | a snippet" + + def test_link_prefers_portal_over_api(self): + portal = "https://portal.intel471.com/forum/post/123" + summary = _summarize_alert(_alert_like(portal=portal, api=_SPOT_URL)) + assert portal in summary + assert _SPOT_URL not in summary + + def test_link_falls_back_to_api(self): + summary = _summarize_alert(_alert_like(portal=None, api=_SPOT_URL)) + assert _SPOT_URL in summary + + def test_without_highlights_has_no_snippet(self): + summary = _summarize_alert(_alert_like(api=_SPOT_URL, snippets=None)) + assert summary == f"Forums Post: {_SPOT_URL}" \ No newline at end of file diff --git a/verity471/helpers/__init__.py b/verity471/helpers/__init__.py index f94eff5..6e57f7b 100644 --- a/verity471/helpers/__init__.py +++ b/verity471/helpers/__init__.py @@ -1,5 +1,5 @@ -from verity471.helpers.alerts import AlertTarget, fetch_alert_targets +from verity471.helpers.alerts import AlertTarget, AlertTargetStatus, fetch_alert_targets from verity471.helpers.stream_latest import get_latest from verity471.helpers.url_router import call_url, resolve_url -__all__ = ["AlertTarget", "fetch_alert_targets", "get_latest", "resolve_url", "call_url"] +__all__ = ["AlertTarget", "AlertTargetStatus", "fetch_alert_targets", "get_latest", "resolve_url", "call_url"] diff --git a/verity471/helpers/alerts.py b/verity471/helpers/alerts.py index 7e91ca3..c6408fd 100644 --- a/verity471/helpers/alerts.py +++ b/verity471/helpers/alerts.py @@ -1,5 +1,6 @@ from __future__ import annotations +import enum import logging import re import concurrent.futures @@ -236,6 +237,41 @@ def _summarize_target(target: Any) -> str | None: return None +def _summarize_alert(alert: StreamingWatcherAlert) -> str | None: + """Best-effort one-liner from the alert envelope alone (no target needed). + + Used as a fallback when the target couldn't be fetched or isn't + summarizable: ``": "`` plus the first highlight snippet if + present. + """ + label = _type_label(alert.source_type) if alert.source_type else None + + url = None + if alert.links: + if alert.links.verity_portal and alert.links.verity_portal.href: + url = alert.links.verity_portal.href + elif alert.links.verity_api and alert.links.verity_api.href: + url = alert.links.verity_api.href + + head = f"{label}: {url}" if (label and url) else (label or url) + + snippet = None + if alert.highlights and alert.highlights[0].snippets: + snippet = _snippet(alert.highlights[0].snippets[0]) + + return _join([head, snippet]) + + +class AlertTargetStatus(str, enum.Enum): + """HTTP-style outcome of fetching an alert's target.""" + + OK = "ok" # target fetched + NO_LINK = "no_link" # alert had no links.verity_api.href + UNRESOLVABLE = "unresolvable" # URL matched no known SDK route (404-ish) + FORBIDDEN = "forbidden" # API returned 403 + ERROR = "error" # unexpected fetch error (5xx-ish) + + @dataclass class AlertTarget: """An alert paired with its fully fetched target object. @@ -245,9 +281,16 @@ class AlertTarget: object — a report, forum post, credential, or whatever the alert refers to. ``target`` is ``None`` when the URL could not be mapped to a known route. + ``status`` is the :class:`AlertTargetStatus` for the fetch — ``OK`` when the + target was fetched, otherwise the reason it could not be (``NO_LINK``, + ``UNRESOLVABLE``, ``FORBIDDEN``, ``ERROR``). ``target is None`` together + with ``status != OK`` means the fetch failed; ``status_reason`` carries the + human-readable detail (e.g. the URL or the underlying error message). + ``target_summary`` provides a compact, human-readable one-liner for the target (e.g. report title + date, indicator type + value, credential - login + domain). + login + domain). When the target is missing or not summarizable, it falls + back to a summary built from the alert envelope itself. ``watcher`` is the full :class:`GetWatcherResponse` for the watcher that triggered this alert, or ``None`` if not found in the fetched list. @@ -256,18 +299,21 @@ class AlertTarget: alert: StreamingWatcherAlert target: Any + status: AlertTargetStatus = AlertTargetStatus.OK + status_reason: str | None = None watcher: GetWatcherResponse | None = None watcher_group: GetWatcherGroupResponse | None = None @property def target_summary(self) -> str | None: - return _summarize_target(self.target) + return _summarize_target(self.target) or _summarize_alert(self.alert) def fetch_alert_targets( alerts_response: StreamingAlertsResponse, api_client: ApiClient, raise_on_error: bool = False, + skip_missing_targets: bool = False, ) -> list[AlertTarget]: """Fetch the full target object for every alert in *alerts_response*. @@ -276,18 +322,25 @@ def fetch_alert_targets( URL and returns :class:`AlertTarget` pairs so you can work with the actual content (report body, forum post text, etc.) alongside the alert metadata. - URLs that cannot be mapped to a known SDK route always produce an - :class:`AlertTarget` with ``target=None`` (and emit a warning). Other - errors (missing link, API call failure) follow *raise_on_error*: when - ``True`` the exception propagates; when ``False`` an error is logged and - the alert is omitted from the result. + When a target cannot be fetched (no link, no known SDK route, forbidden, or + an unexpected error), the behaviour depends on *skip_missing_targets*: by + default (``False``) the alert is still returned with ``target=None`` and a + non-``OK`` :class:`AlertTargetStatus` (plus a ``status_reason``) so callers + can see it failed and why; when ``True`` such alerts are omitted from the + result entirely. Marketplace alerts are always skipped silently, regardless + of either flag. Args: alerts_response: The page returned by :meth:`AlertsApi.get_alerts_stream`. api_client: An active :class:`ApiClient` (must share credentials with the alerts call). - raise_on_error: When ``True``, re-raise unexpected errors instead of - logging and skipping the alert. Defaults to ``False``. + raise_on_error: When ``True``, re-raise unexpected errors (and the + missing-link ``ValueError``) instead of recording them on the + result. Defaults to ``False``. + skip_missing_targets: When ``True``, alerts whose target cannot be + fetched are omitted from the result. When ``False`` (default) they + are returned with ``target=None`` and a failure ``status``. + Defaults to ``False``. Returns: A list of :class:`AlertTarget` objects in the same order as @@ -297,7 +350,7 @@ def fetch_alert_targets( alerts = alerts_api.get_alerts_stream(size=10) for r in fetch_alert_targets(alerts, api_client): - print(r.alert.source_type, r.alert.status, r.target) + print(r.alert.source_type, r.status, r.status_reason, r.target) """ def _fetch(alert: StreamingWatcherAlert) -> AlertTarget | None: url = alert.links.verity_api.href if (alert.links and alert.links.verity_api) else None @@ -307,20 +360,32 @@ def _fetch(alert: StreamingWatcherAlert) -> AlertTarget | None: if raise_on_error: raise ValueError("Alert %s has no verity_api link" % alert.source_id) log.error("Alert %s has no verity_api link", alert.source_id) - return None + if skip_missing_targets: + return None + return AlertTarget(alert=alert, target=None, status=AlertTargetStatus.NO_LINK, + status_reason="Alert has no verity_api link") try: target = call_url(api_client, url) except UnresolvableURL: log.warning("No SDK route for alert %s URL: %s", alert.source_id, url) - return AlertTarget(alert=alert, target=None) + if skip_missing_targets: + return None + return AlertTarget(alert=alert, target=None, status=AlertTargetStatus.UNRESOLVABLE, + status_reason=f"No SDK route for URL: {url}") except ForbiddenException: log.debug("Failed to fetch target for alert %s (%s) - Forbidden", alert.source_id, url) - return None - except Exception: + if skip_missing_targets: + return None + return AlertTarget(alert=alert, target=None, status=AlertTargetStatus.FORBIDDEN, + status_reason="Forbidden (403) fetching target") + except Exception as exc: if raise_on_error: raise log.error("Failed to fetch target for alert %s (%s)", alert.source_id, url, exc_info=True) - return None + if skip_missing_targets: + return None + return AlertTarget(alert=alert, target=None, status=AlertTargetStatus.ERROR, + status_reason=f"Error fetching target: {exc}") _patch_portal_url(alert, target) # TEMPORARY WORKAROUND — remove once API is fixed return AlertTarget(alert=alert, target=target)