Add multi-dataset model run prediction uploads

CHANGELOG.md

@@ -5,6 +5,23 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.20.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.20.0) - 2026-08-03
+
+### Added
+- **Multi-dataset model runs.** `Dataset.upload_predictions_for_model_run(model_run_id, predictions, ...)` uploads predictions for an existing run against *this* dataset, adding the dataset to the run's set if it isn't there already. This is what lets a single model run be scored against a benchmark whose items span several datasets. Supports the same `update` / `asynchronous` / `batch_size` / file-batching / `trained_slice_id` arguments as `upload_predictions`.
+  - A run's dataset set only ever grows — a later upload never removes a dataset, so it cannot widen who can read the run.
+  - Access: write on this dataset **and** on every dataset the run already covers. Runs are visible only to users who can read all of their datasets, so adding one can remove the run from a collaborator's view.
+  - `Dataset.upload_predictions` is unchanged and still cannot widen a run: it identifies the run by `(dataset, model)`, so it finds the run already on this dataset or creates a new one.
+
+### Changed
+- **Benchmark evaluations no longer require the run to cover the benchmark's datasets.** `create_benchmark_evaluation_v2` previously failed when the benchmark contained items outside the model run's dataset. Those members are now scored as false negatives like any other uncovered item, so a partial run ranks comparably instead of being rejected. Docstrings on `create_benchmark_evaluation_v2` and `Benchmark.create_evaluation_v2` updated accordingly.
+- `PredictionUploader` accepts `dataset_id` together with `model_run_id` to select the new endpoint. Previously that combination was rejected by an assertion. The other two forms — `(dataset_id, model_id)` and `model_run_id` alone — route exactly as before.
+
+### Deprecated
+- `ModelRun.predict()` (already deprecated with the rest of `ModelRun`) infers its target dataset from the run, so it fails for a run spanning several datasets. Use `Dataset.upload_predictions_for_model_run` instead.
+
+> **Server dependency:** requires the `POST /nucleus/dataset/:datasetId/modelRun/:modelRunId/uploadPredictions` route and the multi-dataset model-run work in scaleapi. Unit tests pass regardless; live calls 404 until that deploys.
+
 ## [0.19.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.19.0) - 2026-07-10
 
 ### Added

nucleus/__init__.py

@@ -1264,10 +1264,16 @@ def create_benchmark_evaluation_v2(
         background — call :meth:`EvaluationV2.wait_for_completion`, then
         :meth:`EvaluationV2.charts` or :meth:`EvaluationV2.examples`.
 
+        The benchmark may span datasets the model run has no predictions in at
+        all. Those members are scored as false negatives like any other
+        uncovered item, so a partial run still ranks comparably rather than
+        being rejected. To give a run predictions across several datasets, use
+        :meth:`Dataset.upload_predictions_for_model_run`.
+
         Parameters:
             benchmark_id: Benchmark id (``bm_*``).
-            model_run_id: Model run id (``run_*``). Its predictions must
-                cover items from the benchmark's datasets.
+            model_run_id: Model run id (``run_*``). It need not cover the
+                benchmark's datasets — coverage may be partial, or empty.
             name: Optional display name.
             rollup_groups: Optional rollup classes (the primary label
                 configuration); each :class:`RollupGroup` maps raw labels

nucleus/annotation_uploader.py

@@ -236,6 +236,29 @@ def check_for_duplicate_ids(self, annotations: Iterable[Annotation]):
 
 
 class PredictionUploader(AnnotationUploader):
+    """Routes a prediction upload to one of three endpoints.
+
+    Which one depends on the identifiers supplied:
+
+    ``dataset_id`` + ``model_run_id``
+        ``dataset/{dataset_id}/modelRun/{model_run_id}/uploadPredictions``. The
+        only route that lets a run span more than one dataset — uploading here
+        adds ``dataset_id`` to the run's dataset set. Requires write access to
+        every dataset the run already covers, not just this one, because a run
+        is only visible to users who can read all of its datasets.
+
+    ``dataset_id`` + ``model_id``
+        ``dataset/{dataset_id}/model/{model_id}/uploadPredictions``. Resolves
+        (or creates) the run for that model on that dataset. Cannot widen an
+        existing run: passing a ``model_run_id`` belonging to a different
+        dataset is rejected server-side.
+
+    ``model_run_id`` alone
+        ``modelRun/{model_run_id}/predict``. Deprecated — the target dataset is
+        inferred from the run, so the server rejects it for a run spanning
+        several datasets. Prefer the first form.
+    """
+
     def __init__(
         self,
         client: "NucleusClient",
@@ -247,8 +270,13 @@ def __init__(
         super().__init__(dataset_id, client)
         self._client = client
         self.trained_slice_id = trained_slice_id
-        if model_run_id is not None:
-            assert model_id is None and dataset_id is None
+        if model_run_id is not None and dataset_id is not None:
+            assert (
+                model_id is None
+            ), "Pass either model_id or model_run_id, not both."
+            self._route = f"dataset/{dataset_id}/modelRun/{model_run_id}/uploadPredictions"
+        elif model_run_id is not None:
+            assert model_id is None
             self._route = f"modelRun/{model_run_id}/predict"
         else:
             assert (

nucleus/benchmark.py

@@ -152,6 +152,11 @@ def create_evaluation_v2(
     ) -> EvaluationV2:
         """Evaluate a model run against this benchmark.
 
+        The run need not cover this benchmark's datasets — uncovered members are
+        scored as false negatives, so a partial run still ranks comparably. To
+        give a run predictions across several datasets, use
+        :meth:`Dataset.upload_predictions_for_model_run`.
+
         See :meth:`NucleusClient.create_benchmark_evaluation_v2` for parameter
         details.
 

nucleus/dataset.py

@@ -2151,6 +2151,97 @@ def upload_predictions(
             trained_slice_id=trained_slice_id,
         )
 
+    def upload_predictions_for_model_run(
+        self,
+        model_run_id: str,
+        predictions: List[Prediction],
+        update: bool = False,
+        asynchronous: bool = False,
+        batch_size: int = 5000,
+        remote_files_per_upload_request: int = 20,
+        local_files_per_upload_request: int = 10,
+        trained_slice_id: Optional[str] = None,
+    ):
+        """Uploads predictions for an existing model run against **this** dataset.
+
+        Use this instead of :meth:`upload_predictions` when one model run should
+        hold predictions across several datasets — for example to evaluate the
+        run against a benchmark whose items span more than one dataset. The run
+        does not need to cover this dataset already; uploading here adds it to
+        the run's dataset set.
+
+        :meth:`upload_predictions` cannot do this. It identifies the run by
+        ``(dataset, model)``, so it either finds the run already on this dataset
+        or creates a new one — an existing run belonging to a different dataset
+        is rejected.
+
+        A run's dataset set only ever grows: a later upload never removes a
+        dataset, so it cannot widen who can read the run.
+
+        Access: you need write access to this dataset **and** to every dataset
+        the run already covers. Model runs are visible only to users who can
+        read all of their datasets, so adding a dataset to a run can remove it
+        from a collaborator's view — hence the stricter check.
+
+        Parameters:
+            model_run_id: Nucleus-generated model run ID (starts with ``run_``).
+            predictions: List of prediction objects to upload. Same types as
+              :meth:`upload_predictions`.
+            update: Whether or not to overwrite metadata or ignore on reference
+              ID collision. Default is False.
+            asynchronous: Whether or not to process the upload asynchronously
+              (and return an :class:`AsyncJob` object). Default is False.
+            batch_size: Number of predictions processed in each concurrent
+              batch. Default is 5000. Only relevant for asynchronous=False.
+            remote_files_per_upload_request: Number of remote files to upload in
+              each request. Only relevant for asynchronous=False.
+            local_files_per_upload_request: Number of local files to upload in
+              each request. Maximum is 10. Only relevant for asynchronous=False.
+            trained_slice_id: Nucleus-generated slice ID (starts with ``slc_``)
+              which was used to train the model. Must belong to this dataset.
+
+        Returns:
+            Payload describing the synchronous upload::
+
+                {
+                    "dataset_id": str,
+                    "model_run_id": str,
+                    "predictions_processed": int,
+                    "predictions_ignored": int,
+                }
+        """
+        uploader = PredictionUploader(
+            model_run_id=model_run_id,
+            dataset_id=self.id,
+            client=self._client,
+        )
+        uploader.check_for_duplicate_ids(predictions)
+
+        if asynchronous:
+            check_all_mask_paths_remote(predictions)
+
+            request_id = serialize_and_write_to_presigned_url(
+                predictions, self.id, self._client
+            )
+            response = self._client.make_request(
+                payload={
+                    REQUEST_ID_KEY: request_id,
+                    UPDATE_KEY: update,
+                    TRAINED_SLICE_ID_KEY: trained_slice_id,
+                },
+                route=f"dataset/{self.id}/modelRun/{model_run_id}/uploadPredictions?async=1",
+            )
+            return AsyncJob.from_json(response, self._client)
+
+        return uploader.upload(
+            annotations=predictions,
+            batch_size=batch_size,
+            update=update,
+            remote_files_per_upload_request=remote_files_per_upload_request,
+            local_files_per_upload_request=local_files_per_upload_request,
+            trained_slice_id=trained_slice_id,
+        )
+
     def predictions_iloc(self, model, index):
         """Fetches all predictions of a dataset item by its absolute index.
 

nucleus/model_run.py

@@ -137,6 +137,12 @@ def predict(
     ) -> Union[dict, AsyncJob]:
         """Uploads model outputs as predictions for a model_run.
 
+        Deprecated along with the rest of this class. The target dataset is
+        inferred from the run rather than named, so this fails for a run that
+        spans more than one dataset — there is no single dataset to infer. Use
+        :meth:`Dataset.upload_predictions_for_model_run` instead, which takes
+        both ids explicitly.
+
         Args:
             annotations: Predictions to upload for this model run.
             update: If True, existing predictions for the same (reference_id, annotation_id)

pyproject.toml

@@ -25,7 +25,7 @@ ignore = ["E501", "E741", "E731", "F401"]  # Easy ignore for getting it running
 
 [tool.poetry]
 name = "scale-nucleus"
-version = "0.19.0"
+version = "0.20.0"
 description = "The official Python client library for Nucleus, the Data Platform for AI"
 license =  "MIT"
 authors = ["Scale AI Nucleus Team <nucleusapi@scaleapi.com>"]

tests/test_multi_dataset_model_runs.py

@@ -0,0 +1,173 @@
+"""Unit tests for multi-dataset model runs (no live API).
+
+A model run used to declare exactly one dataset, and prediction uploads had to
+stay inside it. It now carries the *set* of datasets its predictions actually
+land in, which is what lets one run be scored against a benchmark whose items
+span several datasets.
+
+These tests pin the routing, because the route is the whole difference: only
+``dataset/{dataset_id}/modelRun/{model_run_id}/uploadPredictions`` can add a
+dataset to a run. The other two prediction routes deliberately cannot, and
+silently sending a widening upload to one of them would either create a second
+run or be rejected server-side.
+"""
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from nucleus import NucleusClient
+from nucleus.annotation_uploader import PredictionUploader
+from nucleus.dataset import Dataset
+from nucleus.prediction import BoxPrediction
+
+
+def _client():
+    return NucleusClient(api_key="test")
+
+
+def _predictions():
+    return [
+        BoxPrediction(
+            label="car",
+            x=0,
+            y=0,
+            width=10,
+            height=10,
+            reference_id="item_1",
+            confidence=0.9,
+        )
+    ]
+
+
+# --------------------------------------------------------------------------- #
+# PredictionUploader routing — the three forms
+# --------------------------------------------------------------------------- #
+def test_dataset_and_model_run_ids_route_to_the_widening_endpoint():
+    uploader = PredictionUploader(
+        client=_client(), dataset_id="ds_1", model_run_id="run_1"
+    )
+    assert (
+        uploader._route
+        == "dataset/ds_1/modelRun/run_1/uploadPredictions"  # noqa: SLF001
+    )
+
+
+def test_dataset_and_model_ids_route_to_the_model_endpoint():
+    """The (dataset, model) form is unchanged — it cannot widen a run."""
+    uploader = PredictionUploader(
+        client=_client(), dataset_id="ds_1", model_id="prj_1"
+    )
+    assert (
+        uploader._route
+        == "dataset/ds_1/model/prj_1/uploadPredictions"  # noqa: SLF001
+    )
+
+
+def test_model_run_id_alone_routes_to_the_deprecated_endpoint():
+    """Kept working for single-dataset runs; the server infers the dataset."""
+    uploader = PredictionUploader(client=_client(), model_run_id="run_1")
+    assert uploader._route == "modelRun/run_1/predict"  # noqa: SLF001
+
+
+def test_model_id_and_model_run_id_together_are_rejected():
+    with pytest.raises(AssertionError):
+        PredictionUploader(
+            client=_client(),
+            dataset_id="ds_1",
+            model_id="prj_1",
+            model_run_id="run_1",
+        )
+
+
+def test_neither_model_nor_model_run_is_rejected():
+    with pytest.raises(AssertionError):
+        PredictionUploader(client=_client(), dataset_id="ds_1")
+
+
+# --------------------------------------------------------------------------- #
+# Dataset.upload_predictions_for_model_run
+# --------------------------------------------------------------------------- #
+def test_upload_predictions_for_model_run_uses_the_widening_route():
+    client = _client()
+    dataset = Dataset("ds_1", client)
+    uploaded = {}
+
+    def _capture(**kwargs):
+        uploaded.update(kwargs)
+        return {"predictions_processed": 1, "predictions_ignored": 0}
+
+    with pytest.MonkeyPatch.context() as mp:
+        routes = []
+        original_init = PredictionUploader.__init__
+
+        def _spy_init(self, *args, **kwargs):
+            original_init(self, *args, **kwargs)
+            routes.append(self._route)  # noqa: SLF001
+
+        mp.setattr(PredictionUploader, "__init__", _spy_init)
+        mp.setattr(
+            PredictionUploader, "upload", lambda self, **kw: _capture(**kw)
+        )
+        dataset.upload_predictions_for_model_run("run_1", _predictions())
+
+    assert routes == ["dataset/ds_1/modelRun/run_1/uploadPredictions"]
+    assert uploaded["update"] is False
+
+
+def test_upload_predictions_for_model_run_async_hits_the_async_route():
+    client = _client()
+    dataset = Dataset("ds_1", client)
+    client.make_request = MagicMock(
+        return_value={
+            "job_id": "job_1",
+            "job_last_known_status": "Running",
+            "job_type": "uploadPredictions",
+            "job_creation_time": "2026-08-03T00:00:00.000Z",
+        }
+    )
+
+    with pytest.MonkeyPatch.context() as mp:
+        mp.setattr(
+            "nucleus.dataset.serialize_and_write_to_presigned_url",
+            lambda *args, **kwargs: "req_1",
+        )
+        dataset.upload_predictions_for_model_run(
+            "run_1", _predictions(), asynchronous=True
+        )
+
+    route = client.make_request.call_args[1]["route"]
+    assert route == "dataset/ds_1/modelRun/run_1/uploadPredictions?async=1"
+
+
+def test_upload_predictions_for_model_run_forwards_trained_slice_id():
+    client = _client()
+    dataset = Dataset("ds_1", client)
+    captured = {}
+
+    with pytest.MonkeyPatch.context() as mp:
+        mp.setattr(
+            PredictionUploader,
+            "upload",
+            lambda self, **kw: captured.update(kw) or {},
+        )
+        dataset.upload_predictions_for_model_run(
+            "run_1", _predictions(), trained_slice_id="slc_1", update=True
+        )
+
+    assert captured["trained_slice_id"] == "slc_1"
+    assert captured["update"] is True
+
+
+def test_upload_predictions_for_model_run_rejects_duplicate_ids():
+    """Inherited from PredictionUploader; asserted here so the new entry point
+    is known to run the check rather than bypass it."""
+    from nucleus.errors import DuplicateIDError
+
+    dataset = Dataset("ds_1", _client())
+    duplicate = _predictions() * 2
+    for pred in duplicate:
+        pred.annotation_id = "ann_1"
+
+    with pytest.raises(DuplicateIDError):
+        dataset.upload_predictions_for_model_run("run_1", duplicate)