Alerts helper - configurable skip mechanism

README.md

@@ -222,8 +222,13 @@ The alerts stream endpoint returns lightweight `StreamingWatcherAlert` objects t
 `fetch_alert_targets` resolves each alert's API link in parallel and pairs it with the fully
 fetched target object — a report, forum post, credential, indicator, or any other supported type.
 
+When a target cannot be fetched (no link, no known SDK route, forbidden, or an unexpected error),
+the alert is still returned by default with `target=None` and a `status` explaining why, so nothing
+is silently lost. Pass `skip_missing_targets=True` to omit such alerts instead. Marketplace alerts
+are always skipped silently.
+
 ```python
-from verity471.helpers import fetch_alert_targets, AlertTarget
+from verity471.helpers import fetch_alert_targets, AlertTarget, AlertTargetStatus
 ```
 
 ### Parameters
@@ -232,7 +237,8 @@ from verity471.helpers import fetch_alert_targets, AlertTarget
 |---|---|---|---|
 | `alerts_response` | `StreamingAlertsResponse` | *(required)* | The page returned by `AlertsApi.get_alerts_stream()`. |
 | `api_client` | `ApiClient` | *(required)* | An active `ApiClient` instance (must share credentials with the alerts call). |
-| `raise_on_error` | `bool` | `False` | When `True`, re-raise exceptions instead of logging and skipping the alert. |
+| `raise_on_error` | `bool` | `False` | When `True`, re-raise unexpected errors (and the missing-link error) instead of recording them on the result. |
+| `skip_missing_targets` | `bool` | `False` | When `True`, alerts whose target cannot be fetched are omitted from the result. When `False` (default), they are returned with `target=None` and a failure `status`. Marketplace alerts are always skipped regardless. |
 
 ### Returns
 
@@ -243,8 +249,10 @@ Each `AlertTarget` exposes:
 | Attribute | Type | Description |
 |---|---|---|
 | `.alert` | `StreamingWatcherAlert` | The original alert object (status, watcher IDs, timestamps, highlights, etc.). |
-| `.target` | model instance or `None` | The resolved API object (report, post, credential, …). `None` when the URL could not be mapped to a known SDK route. |
-| `.target_summary` | `str \| None` | A compact, human-readable one-liner describing the target. |
+| `.target` | model instance or `None` | The resolved API object (report, post, credential, …). `None` when the target could not be fetched (see `.status`). |
+| `.status` | `AlertTargetStatus` | The fetch outcome: `OK` when the target was fetched, otherwise `NO_LINK`, `UNRESOLVABLE`, `FORBIDDEN`, or `ERROR`. |
+| `.status_reason` | `str \| None` | A human-readable detail for a non-`OK` status (e.g. the URL or the underlying error message). `None` when `status` is `OK`. |
+| `.target_summary` | `str \| None` | A compact, human-readable one-liner describing the target. Falls back to a summary built from the alert envelope (type, link, first highlight snippet) when the target is missing or not summarizable. |
 | `.watcher` | `GetWatcherResponse \| None` | The full watcher object that triggered this alert (name, DSL query, mute status, etc.). `None` if the watcher ID was not found in the user's watcher list. |
 | `.watcher_group` | `GetWatcherGroupResponse \| None` | The full watcher group object the watcher belongs to (name, description, etc.). `None` if not found. |
 
@@ -256,7 +264,7 @@ share the same watcher.
 
 ```python
 import verity471
-from verity471.helpers import fetch_alert_targets
+from verity471.helpers import fetch_alert_targets, AlertTargetStatus
 
 configuration = verity471.Configuration(
     username="your_username",
@@ -271,6 +279,9 @@ with verity471.ApiClient(configuration) as api_client:
     for t in targets:
         watcher_name = t.watcher.name if t.watcher else None
         group_name = t.watcher_group.name if t.watcher_group else None
+        if t.status is not AlertTargetStatus.OK:
+            print(t.alert.source_type, t.alert.status, f"[{t.status.value}: {t.status_reason}]", t.target_summary)
+            continue
         print(t.alert.source_type, t.alert.status, watcher_name, group_name, t.target_summary)
 ```
 
@@ -280,6 +291,7 @@ with verity471.ApiClient(configuration) as api_client:
 fintel read threat_actor Ransomware actors [Fintel] Threat Landscape: Q1 2025 Summary | 2025-03-15T12:00:00Z | Key findings from the first quarter include…
 forum_post unread ddos_monitor My Watchers [Forum Post] Selling access to corporate VPN… | 2025-03-14T08:30:00Z
 credential_occurrence unread cred_watcher Credential Alerts [Credential Occurrence] https://example.com/login | email | 2025-03-13T10:00:00Z
+forum_post unread [forbidden: Forbidden (403) fetching target] Forums Post: https://api.intel471.cloud/integrations/sources/v1/forums/posts/post--… | …matched snippet text…
 malware_report read malware_tracker My Watchers [Malware Report] New variant of Lumma Stealer | 2025-03-12T15:45:00Z | A new variant has been observed…
 ```
 

tests/test_helpers.py

@@ -5,9 +5,11 @@
 import pytest
 
 from tests.conftest import PREFIX, read_fixture
-from verity471.helpers import fetch_alert_targets
+from types import SimpleNamespace
+
+from verity471.helpers import fetch_alert_targets, AlertTargetStatus
 from verity471.exceptions import ForbiddenException
-from verity471.helpers.alerts import _patch_portal_url
+from verity471.helpers.alerts import _patch_portal_url, _summarize_alert
 from verity471.helpers.url_router import UnresolvableURL
 from verity471.models.get_watcher_response import GetWatcherResponse
 from verity471.models.get_watcher_group_response import GetWatcherGroupResponse
@@ -227,6 +229,19 @@ def _alerts_response(*alerts):
     return resp
 
 
+def _alert_like(source_type="forums_post", portal=None, api=_SPOT_URL, snippets=None,
+                source_id="src--test", watcher_id=1, watcher_group_id=1):
+    """A lightweight alert with real string attributes (for summary tests)."""
+    links = SimpleNamespace(
+        verity_portal=SimpleNamespace(href=portal) if portal else None,
+        verity_api=SimpleNamespace(href=api) if api else None,
+    )
+    highlights = [SimpleNamespace(field_name="x", snippets=snippets)] if snippets else None
+    return SimpleNamespace(
+        source_type=source_type, links=links, highlights=highlights,
+        source_id=source_id, watcher_id=watcher_id, watcher_group_id=watcher_group_id)
+
+
 def _no_watchers(watchers_mock):
     watchers_mock.return_value.get_watchers.return_value.watchers = []
     watchers_mock.return_value.get_watcher_groups.return_value.watchers_groups = []
@@ -253,7 +268,7 @@ def test_missing_link_is_skipped(self, _call_url_mock, _watchers_mock):
         alert = _mock_alert()
         alert.links = None
         with verity471.ApiClient(configuration) as api_client:
-            result = fetch_alert_targets(_alerts_response(alert), api_client)
+            result = fetch_alert_targets(_alerts_response(alert), api_client, skip_missing_targets=True)
         assert result == []
 
     def test_missing_link_raises_when_requested(self, _call_url_mock, _watchers_mock):
@@ -272,19 +287,86 @@ def test_unresolvable_url_yields_none_target(self, call_url_mock, watchers_mock)
         assert len(result) == 1
         assert result[0].alert is alert
         assert result[0].target is None
+        assert result[0].status == AlertTargetStatus.UNRESOLVABLE
+        assert result[0].status_reason
 
     def test_forbidden_is_skipped(self, call_url_mock, _watchers_mock):
         call_url_mock.side_effect = ForbiddenException()
         with verity471.ApiClient(configuration) as api_client:
-            result = fetch_alert_targets(_alerts_response(_mock_alert()), api_client)
+            result = fetch_alert_targets(_alerts_response(_mock_alert()), api_client, skip_missing_targets=True)
         assert result == []
 
-    def test_api_error_skipped_by_default(self, call_url_mock, _watchers_mock):
+    def test_api_error_skipped_when_requested(self, call_url_mock, _watchers_mock):
+        call_url_mock.side_effect = RuntimeError("boom")
+        with verity471.ApiClient(configuration) as api_client:
+            result = fetch_alert_targets(_alerts_response(_mock_alert()), api_client, skip_missing_targets=True)
+        assert result == []
+
+    # --- default (skip_missing_targets=False): failures kept with a status ---
+
+    def test_missing_link_kept_with_status_by_default(self, _call_url_mock, watchers_mock):
+        _no_watchers(watchers_mock)
+        alert = _mock_alert()
+        alert.links = None
+        with verity471.ApiClient(configuration) as api_client:
+            result = fetch_alert_targets(_alerts_response(alert), api_client)
+        assert len(result) == 1
+        assert result[0].target is None
+        assert result[0].status == AlertTargetStatus.NO_LINK
+        assert "link" in result[0].status_reason.lower()
+
+    def test_forbidden_kept_with_status_by_default(self, call_url_mock, watchers_mock):
+        call_url_mock.side_effect = ForbiddenException()
+        _no_watchers(watchers_mock)
+        with verity471.ApiClient(configuration) as api_client:
+            result = fetch_alert_targets(_alerts_response(_mock_alert()), api_client)
+        assert len(result) == 1
+        assert result[0].target is None
+        assert result[0].status == AlertTargetStatus.FORBIDDEN
+
+    def test_api_error_kept_with_status_by_default(self, call_url_mock, watchers_mock):
         call_url_mock.side_effect = RuntimeError("boom")
+        _no_watchers(watchers_mock)
         with verity471.ApiClient(configuration) as api_client:
             result = fetch_alert_targets(_alerts_response(_mock_alert()), api_client)
+        assert len(result) == 1
+        assert result[0].status == AlertTargetStatus.ERROR
+        assert "boom" in result[0].status_reason
+
+    def test_unresolvable_skipped_when_skip_true(self, call_url_mock, _watchers_mock):
+        call_url_mock.side_effect = UnresolvableURL("no route")
+        with verity471.ApiClient(configuration) as api_client:
+            result = fetch_alert_targets(_alerts_response(_mock_alert()), api_client, skip_missing_targets=True)
         assert result == []
 
+    def test_error_raise_takes_priority_over_skip_false(self, call_url_mock, _watchers_mock):
+        call_url_mock.side_effect = RuntimeError("boom")
+        with verity471.ApiClient(configuration) as api_client:
+            with pytest.raises(RuntimeError):
+                fetch_alert_targets(_alerts_response(_mock_alert()), api_client,
+                                    raise_on_error=True, skip_missing_targets=False)
+
+    def test_success_has_ok_status(self, call_url_mock, watchers_mock):
+        _no_watchers(watchers_mock)
+        call_url_mock.return_value = MagicMock()
+        with verity471.ApiClient(configuration) as api_client:
+            result = fetch_alert_targets(_alerts_response(_mock_alert()), api_client)
+        assert len(result) == 1
+        assert result[0].status == AlertTargetStatus.OK
+        assert result[0].status_reason is None
+
+    def test_failure_result_still_enriched_with_watcher(self, call_url_mock, watchers_mock):
+        call_url_mock.side_effect = ForbiddenException()
+        watcher = MagicMock(spec=GetWatcherResponse)
+        watcher.id = 1
+        watchers_mock.return_value.get_watchers.return_value.watchers = [watcher]
+        watchers_mock.return_value.get_watcher_groups.return_value.watchers_groups = []
+        with verity471.ApiClient(configuration) as api_client:
+            result = fetch_alert_targets(_alerts_response(_mock_alert(watcher_id=1)), api_client)
+        assert len(result) == 1
+        assert result[0].status == AlertTargetStatus.FORBIDDEN
+        assert result[0].watcher is watcher
+
     def test_api_error_raises_when_requested(self, call_url_mock, _watchers_mock):
         call_url_mock.side_effect = RuntimeError("boom")
         with verity471.ApiClient(configuration) as api_client:
@@ -314,4 +396,41 @@ def test_result_order_preserved(self, call_url_mock, watchers_mock):
         assert len(result) == 5
         for i, r in enumerate(result):
             assert r.alert is alerts[i]
-            assert r.target is targets[i]
+            assert r.target is targets[i]
+
+    def test_alert_summary_used_when_target_missing(self, call_url_mock, watchers_mock):
+        call_url_mock.side_effect = ForbiddenException()
+        _no_watchers(watchers_mock)
+        alert = _alert_like(source_type="forums_post", api=_SPOT_URL, snippets=["hello world"])
+        with verity471.ApiClient(configuration) as api_client:
+            result = fetch_alert_targets(_alerts_response(alert), api_client)
+        assert len(result) == 1
+        summary = result[0].target_summary
+        assert summary.startswith("Forums Post: ")
+        assert _SPOT_URL in summary
+        assert "hello world" in summary
+
+
+# ---------------------------------------------------------------------------
+# Tests for the alert-envelope fallback summary (_summarize_alert).
+# ---------------------------------------------------------------------------
+
+class TestSummarizeAlert:
+
+    def test_uses_type_url_and_snippet(self):
+        summary = _summarize_alert(_alert_like(api=_SPOT_URL, snippets=["a snippet"]))
+        assert summary == f"Forums Post: {_SPOT_URL} | a snippet"
+
+    def test_link_prefers_portal_over_api(self):
+        portal = "https://portal.intel471.com/forum/post/123"
+        summary = _summarize_alert(_alert_like(portal=portal, api=_SPOT_URL))
+        assert portal in summary
+        assert _SPOT_URL not in summary
+
+    def test_link_falls_back_to_api(self):
+        summary = _summarize_alert(_alert_like(portal=None, api=_SPOT_URL))
+        assert _SPOT_URL in summary
+
+    def test_without_highlights_has_no_snippet(self):
+        summary = _summarize_alert(_alert_like(api=_SPOT_URL, snippets=None))
+        assert summary == f"Forums Post: {_SPOT_URL}"

verity471/helpers/__init__.py

@@ -1,5 +1,5 @@
-from verity471.helpers.alerts import AlertTarget, fetch_alert_targets
+from verity471.helpers.alerts import AlertTarget, AlertTargetStatus, fetch_alert_targets
 from verity471.helpers.stream_latest import get_latest
 from verity471.helpers.url_router import call_url, resolve_url
 
-__all__ = ["AlertTarget", "fetch_alert_targets", "get_latest", "resolve_url", "call_url"]
+__all__ = ["AlertTarget", "AlertTargetStatus", "fetch_alert_targets", "get_latest", "resolve_url", "call_url"]