fix(purl): expose fail-open batch params and harden dedupe

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "socketdev"
-version = "3.4.1"
+version = "3.4.2"
 requires-python = ">= 3.9"
 dependencies = [
     'requests',

socketdev/core/dedupe.py

@@ -11,8 +11,8 @@ def normalize_file_path(path: str) -> str:
     @staticmethod
     def alert_key(alert: dict) -> tuple:
         return (
-            alert["type"],
-            alert["severity"],
+            alert.get("type"),
+            alert.get("severity"),
             alert.get("category"),
             Dedupe.normalize_file_path(alert.get("file")),
             alert.get("start"),
@@ -23,8 +23,8 @@ def alert_key(alert: dict) -> tuple:
     def consolidate_and_merge_alerts(package_group: List[Dict[str, Any]]) -> Dict[str, Any]:
         def alert_identity(alert: dict) -> tuple:
             return (
-                alert["type"],
-                alert["severity"],
+                alert.get("type"),
+                alert.get("severity"),
                 alert.get("category"),
                 Dedupe.normalize_file_path(alert.get("file")),
                 alert.get("start"),
@@ -41,14 +41,17 @@ def alert_identity(alert: dict) -> tuple:
                 identity = alert_identity(alert)
 
                 if identity not in alert_map:
-                    # Build alert dict with only fields that exist in the original alert
+                    # Build alert dict with only fields that exist in the original alert.
+                    # Use .get() for key/type/severity/action so synthetic status rows
+                    # (e.g. pendingScan/notFound), which are built server-side from a
+                    # minimal {type, key} base, don't raise KeyError here.
                     consolidated_alert = {
-                        "key": alert["key"],  # keep the first key seen
-                        "type": alert["type"],
-                        "severity": alert["severity"],
+                        "key": alert.get("key"),  # keep the first key seen
+                        "type": alert.get("type"),
+                        "severity": alert.get("severity"),
                         "releases": [release],
                         "props": alert.get("props", []),
-                        "action": alert["action"]
+                        "action": alert.get("action")
                     }
 
                     # Only include optional fields if they exist in the original alert

socketdev/exceptions.py

@@ -78,3 +78,31 @@ class APIBadGateway(APIFailure):
 
     def __init__(self, *args):
         super().__init__(*args, status_code=502)
+
+
+class APIPartialResponse(APIFailure):
+    """Raised by ``purl.post(strict=True)`` when the batch response omits requested inputs.
+
+    The batch purl API is fail-open: input purls whose resolution/analysis has not
+    completed are silently dropped from the response unless the caller opts in via
+    ``alerts=True`` (synthetic ``pendingScan``/``notFound`` rows) or ``poll=True`` (a
+    bounded fail-closed wait). ``strict=True`` turns that silent omission into this
+    explicit error so callers get a first-class "partial batch" signal without having
+    to diff the response themselves.
+
+    The ``missing`` attribute holds the requested purls that were absent from the
+    response (the HTTP call itself succeeded, so there is no status code). A missing
+    row may reflect pending analysis, a malformed or unknown purl, or a response contract
+    failure, so blindly retrying is not guaranteed to succeed. Callers that need a bounded
+    wait should use ``poll=True``; callers that need omission reasons should request
+    ``alerts=True`` and/or ``purl_errors=True``.
+    """
+
+    def __init__(self, *args, missing=None):
+        super().__init__(*args)
+        self.missing = list(missing or [])
+
+    def is_transient_error(self) -> bool:
+        # The HTTP request completed successfully, and the omission reason may be
+        # permanent. Server-side polling is the explicit bounded retry mechanism.
+        return False

socketdev/purl/__init__.py

@@ -1,10 +1,19 @@
 import json
 import urllib.parse
 import warnings
+from typing import Optional
 from socketdev.log import log
+from socketdev.exceptions import APIPartialResponse
 from ..core.dedupe import Dedupe
 
 
+def _encode_bool_query_value(value) -> str:
+    """Encode typed bools while preserving legacy string query values."""
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    return str(value)
+
+
 class Purl:
     def __init__(self, api):
         self.api = api
@@ -14,8 +23,60 @@ def post(
         license: str = "false",
         components: list = None,
         org_slug: str = None,
+        poll: Optional[bool] = None,
+        timeout_sec: Optional[int] = None,
+        alerts: Optional[bool] = None,
+        purl_errors: Optional[bool] = None,
+        strict: bool = False,
         **kwargs,
     ) -> list:
+        """POST a batch of purls to the Socket batch purl endpoint and return deduped rows.
+
+        The batch purl API (``POST /v0/purl`` and ``POST /v0/orgs/{slug}/purl``) defaults
+        to **fail-open**: any input purl whose resolution/analysis has not finished is
+        **silently omitted** from the response. A naive caller therefore cannot tell
+        "this version is clean" apart from "this version was dropped from the response".
+        The parameters below opt into the server behaviors that make omissions visible.
+
+        Args:
+            license: ``"true"``/``"false"`` — request license information (stringly-typed
+                to match the query param the API expects).
+            components: list of component dicts to score, e.g. ``[{"purl": "pkg:npm/lodash@4.18.1"}]``.
+            org_slug: organization slug. When provided, routes to the org-scoped endpoint
+                ``POST /v0/orgs/{org_slug}/purl``; otherwise the deprecated ``POST /v0/purl``.
+            poll: opt into a fail-closed bounded wait for pending analysis (``poll=True`` →
+                ``poll=true`` query param). ``None`` omits the param (server default).
+            timeout_sec: bound in seconds for the ``poll`` wait (``→ timeoutSec``). The
+                server may cap this via a feature flag. ``None`` omits the param.
+            alerts: when ``True`` (``→ alerts=true``), the server emits synthetic
+                ``pendingScan``/``notFound`` status rows instead of silently omitting
+                unresolved inputs, so callers can distinguish "no data yet" from "clean".
+            purl_errors: when ``True`` (``→ purlErrors``), the server includes per-purl
+                error rows for malformed/unresolvable inputs. ``None`` omits the param.
+                For backward compatibility, legacy string values passed to the promoted
+                Boolean parameters are forwarded unchanged.
+            strict: client-side guard. When ``True``, compares the exact ``purl`` string
+                of each requested component against the returned ``inputPurl`` (or the
+                ``purl`` fallback). The API defines ``inputPurl`` as the original,
+                unmodified input before server normalization, so canonicalized ``purl``
+                values do not cause false omissions. Raises
+                :class:`~socketdev.exceptions.APIPartialResponse` (with a ``missing``
+                list) if any requested purl is absent from the response. This surfaces
+                partial batches even without ``alerts=True``. Only components that carry
+                a ``purl`` string are checked.
+            **kwargs: forwarded verbatim into the query string (back-compat passthrough for
+                any params not yet promoted to first-class arguments).
+
+        Returns:
+            A deduped list of result rows. When ``alerts=True``, unresolved inputs appear
+            as synthetic rows carrying ``pendingScan``/``notFound`` alerts rather than being
+            omitted. On a non-200 response, logs the error and returns ``[]`` (callers that
+            need to fail closed should treat ``[]`` as an error).
+
+        Raises:
+            APIPartialResponse: if ``strict=True`` and one or more requested component purls
+                are missing from the response.
+        """
         if org_slug is None:
             warnings.warn(
                 "Calling purl.post() without org_slug uses the deprecated POST /v0/purl endpoint. "
@@ -31,25 +92,85 @@ def post(
         query_args = {
             "license": license,
         }
+        # Promote the typed params into query args only when explicitly set, so existing
+        # callers keep the server's fail-open default (None => omit the param entirely).
+        if poll is not None:
+            query_args["poll"] = _encode_bool_query_value(poll)
+        if timeout_sec is not None:
+            query_args["timeoutSec"] = str(timeout_sec)
+        if alerts is not None:
+            query_args["alerts"] = _encode_bool_query_value(alerts)
+        if purl_errors is not None:
+            query_args["purlErrors"] = _encode_bool_query_value(purl_errors)
         if kwargs:
             query_args.update(kwargs)
         params = urllib.parse.urlencode(query_args)
         path += params
         response = self.api.do_request(path=path, payload=purls, method="POST")
         if response.status_code == 200:
-            purl = []
+            artifact_rows = []
+            stream_records = []
             result = response.text
             result = result.strip('"').strip()
             for line in result.split("\n"):
                 if line and line != '"':
                     try:
                         item = json.loads(line)
-                        purl.append(item)
+                        if isinstance(item, dict) and item.get("_type") in {
+                            "purlError",
+                            "summary",
+                        }:
+                            stream_records.append(item)
+                        else:
+                            artifact_rows.append(item)
                     except json.JSONDecodeError:
                         continue
-            purl_deduped = Dedupe.dedupe(purl, batched=True)
+            purl_deduped = Dedupe.dedupe(artifact_rows, batched=True)
+            purl_deduped.extend(stream_records)
+            if strict:
+                self._raise_on_missing(components, purl_deduped)
             return purl_deduped
 
         log.error(f"Error posting {components} to the Purl API: {response.status_code}")
         log.error(response.text)
         return []
+
+    @staticmethod
+    def _raise_on_missing(components: list, results: list) -> None:
+        """Raise APIPartialResponse if any requested component purl is absent from results.
+
+        Only components exposing a ``purl`` string are checked. The batch API contract
+        defines ``inputPurl`` as the original, unmodified input string before server-side
+        normalization, so matching it exactly preserves the caller's identity even when
+        the response's canonical ``purl`` differs. ``purl`` is retained as a fallback,
+        and typed ``purlError`` stream records carry ``inputPurl`` under ``value``.
+        """
+        requested = [
+            c["purl"]
+            for c in components
+            if isinstance(c, dict) and isinstance(c.get("purl"), str)
+        ]
+        if not requested:
+            return
+        returned = set()
+        for row in results:
+            if not isinstance(row, dict):
+                continue
+            for field in ("inputPurl", "purl"):
+                value = row.get(field)
+                if isinstance(value, str):
+                    returned.add(value)
+            record_value = row.get("value")
+            if isinstance(record_value, dict):
+                input_purl = record_value.get("inputPurl")
+                if isinstance(input_purl, str):
+                    returned.add(input_purl)
+        missing = [purl for purl in requested if purl not in returned]
+        if missing:
+            raise APIPartialResponse(
+                "purl.post(strict=True): the batch response omitted "
+                f"{len(missing)} of {len(requested)} requested purls "
+                "(fail-open: unresolved inputs are dropped unless alerts=True/poll=True): "
+                f"{missing}",
+                missing=missing,
+            )

socketdev/version.py

@@ -1 +1 @@
-__version__ = "3.4.1"
+__version__ = "3.4.2"