diff --git a/CHANGELOG.md b/CHANGELOG.md index b4057930..7aeeb267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.20.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.20.0) - 2026-08-03 + +### Added +- **Multi-dataset model runs.** `Dataset.upload_predictions_for_model_run(model_run_id, predictions, ...)` uploads predictions for an existing run against *this* dataset, adding the dataset to the run's set if it isn't there already. This is what lets a single model run be scored against a benchmark whose items span several datasets. Supports the same `update` / `asynchronous` / `batch_size` / file-batching / `trained_slice_id` arguments as `upload_predictions`. + - A run's dataset set only ever grows — a later upload never removes a dataset, so it cannot widen who can read the run. + - Access: write on this dataset **and** on every dataset the run already covers. Runs are visible only to users who can read all of their datasets, so adding one can remove the run from a collaborator's view. + - `Dataset.upload_predictions` is unchanged and still cannot widen a run: it identifies the run by `(dataset, model)`, so it finds the run already on this dataset or creates a new one. + +### Changed +- **Benchmark evaluations no longer require the run to cover the benchmark's datasets.** `create_benchmark_evaluation_v2` previously failed when the benchmark contained items outside the model run's dataset. Those members are now scored as false negatives like any other uncovered item, so a partial run ranks comparably instead of being rejected. Docstrings on `create_benchmark_evaluation_v2` and `Benchmark.create_evaluation_v2` updated accordingly. +- `PredictionUploader` accepts `dataset_id` together with `model_run_id` to select the new endpoint. Previously that combination was rejected by an assertion. The other two forms — `(dataset_id, model_id)` and `model_run_id` alone — route exactly as before. + +### Deprecated +- `ModelRun.predict()` (already deprecated with the rest of `ModelRun`) infers its target dataset from the run, so it fails for a run spanning several datasets. Use `Dataset.upload_predictions_for_model_run` instead. + +> **Server dependency:** requires the `POST /nucleus/dataset/:datasetId/modelRun/:modelRunId/uploadPredictions` route and the multi-dataset model-run work in scaleapi. Unit tests pass regardless; live calls 404 until that deploys. + ## [0.19.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.19.0) - 2026-07-10 ### Added diff --git a/nucleus/__init__.py b/nucleus/__init__.py index 1f5c892a..060e0f39 100644 --- a/nucleus/__init__.py +++ b/nucleus/__init__.py @@ -1264,10 +1264,16 @@ def create_benchmark_evaluation_v2( background — call :meth:`EvaluationV2.wait_for_completion`, then :meth:`EvaluationV2.charts` or :meth:`EvaluationV2.examples`. + The benchmark may span datasets the model run has no predictions in at + all. Those members are scored as false negatives like any other + uncovered item, so a partial run still ranks comparably rather than + being rejected. To give a run predictions across several datasets, use + :meth:`Dataset.upload_predictions_for_model_run`. + Parameters: benchmark_id: Benchmark id (``bm_*``). - model_run_id: Model run id (``run_*``). Its predictions must - cover items from the benchmark's datasets. + model_run_id: Model run id (``run_*``). It need not cover the + benchmark's datasets — coverage may be partial, or empty. name: Optional display name. rollup_groups: Optional rollup classes (the primary label configuration); each :class:`RollupGroup` maps raw labels diff --git a/nucleus/annotation_uploader.py b/nucleus/annotation_uploader.py index fe97f085..be7a2c81 100644 --- a/nucleus/annotation_uploader.py +++ b/nucleus/annotation_uploader.py @@ -236,6 +236,29 @@ def check_for_duplicate_ids(self, annotations: Iterable[Annotation]): class PredictionUploader(AnnotationUploader): + """Routes a prediction upload to one of three endpoints. + + Which one depends on the identifiers supplied: + + ``dataset_id`` + ``model_run_id`` + ``dataset/{dataset_id}/modelRun/{model_run_id}/uploadPredictions``. The + only route that lets a run span more than one dataset — uploading here + adds ``dataset_id`` to the run's dataset set. Requires write access to + every dataset the run already covers, not just this one, because a run + is only visible to users who can read all of its datasets. + + ``dataset_id`` + ``model_id`` + ``dataset/{dataset_id}/model/{model_id}/uploadPredictions``. Resolves + (or creates) the run for that model on that dataset. Cannot widen an + existing run: passing a ``model_run_id`` belonging to a different + dataset is rejected server-side. + + ``model_run_id`` alone + ``modelRun/{model_run_id}/predict``. Deprecated — the target dataset is + inferred from the run, so the server rejects it for a run spanning + several datasets. Prefer the first form. + """ + def __init__( self, client: "NucleusClient", @@ -247,8 +270,13 @@ def __init__( super().__init__(dataset_id, client) self._client = client self.trained_slice_id = trained_slice_id - if model_run_id is not None: - assert model_id is None and dataset_id is None + if model_run_id is not None and dataset_id is not None: + assert ( + model_id is None + ), "Pass either model_id or model_run_id, not both." + self._route = f"dataset/{dataset_id}/modelRun/{model_run_id}/uploadPredictions" + elif model_run_id is not None: + assert model_id is None self._route = f"modelRun/{model_run_id}/predict" else: assert ( diff --git a/nucleus/benchmark.py b/nucleus/benchmark.py index d20d8e78..4e275c71 100644 --- a/nucleus/benchmark.py +++ b/nucleus/benchmark.py @@ -152,6 +152,11 @@ def create_evaluation_v2( ) -> EvaluationV2: """Evaluate a model run against this benchmark. + The run need not cover this benchmark's datasets — uncovered members are + scored as false negatives, so a partial run still ranks comparably. To + give a run predictions across several datasets, use + :meth:`Dataset.upload_predictions_for_model_run`. + See :meth:`NucleusClient.create_benchmark_evaluation_v2` for parameter details. diff --git a/nucleus/dataset.py b/nucleus/dataset.py index 7ece96ca..6a4b838d 100644 --- a/nucleus/dataset.py +++ b/nucleus/dataset.py @@ -2151,6 +2151,97 @@ def upload_predictions( trained_slice_id=trained_slice_id, ) + def upload_predictions_for_model_run( + self, + model_run_id: str, + predictions: List[Prediction], + update: bool = False, + asynchronous: bool = False, + batch_size: int = 5000, + remote_files_per_upload_request: int = 20, + local_files_per_upload_request: int = 10, + trained_slice_id: Optional[str] = None, + ): + """Uploads predictions for an existing model run against **this** dataset. + + Use this instead of :meth:`upload_predictions` when one model run should + hold predictions across several datasets — for example to evaluate the + run against a benchmark whose items span more than one dataset. The run + does not need to cover this dataset already; uploading here adds it to + the run's dataset set. + + :meth:`upload_predictions` cannot do this. It identifies the run by + ``(dataset, model)``, so it either finds the run already on this dataset + or creates a new one — an existing run belonging to a different dataset + is rejected. + + A run's dataset set only ever grows: a later upload never removes a + dataset, so it cannot widen who can read the run. + + Access: you need write access to this dataset **and** to every dataset + the run already covers. Model runs are visible only to users who can + read all of their datasets, so adding a dataset to a run can remove it + from a collaborator's view — hence the stricter check. + + Parameters: + model_run_id: Nucleus-generated model run ID (starts with ``run_``). + predictions: List of prediction objects to upload. Same types as + :meth:`upload_predictions`. + update: Whether or not to overwrite metadata or ignore on reference + ID collision. Default is False. + asynchronous: Whether or not to process the upload asynchronously + (and return an :class:`AsyncJob` object). Default is False. + batch_size: Number of predictions processed in each concurrent + batch. Default is 5000. Only relevant for asynchronous=False. + remote_files_per_upload_request: Number of remote files to upload in + each request. Only relevant for asynchronous=False. + local_files_per_upload_request: Number of local files to upload in + each request. Maximum is 10. Only relevant for asynchronous=False. + trained_slice_id: Nucleus-generated slice ID (starts with ``slc_``) + which was used to train the model. Must belong to this dataset. + + Returns: + Payload describing the synchronous upload:: + + { + "dataset_id": str, + "model_run_id": str, + "predictions_processed": int, + "predictions_ignored": int, + } + """ + uploader = PredictionUploader( + model_run_id=model_run_id, + dataset_id=self.id, + client=self._client, + ) + uploader.check_for_duplicate_ids(predictions) + + if asynchronous: + check_all_mask_paths_remote(predictions) + + request_id = serialize_and_write_to_presigned_url( + predictions, self.id, self._client + ) + response = self._client.make_request( + payload={ + REQUEST_ID_KEY: request_id, + UPDATE_KEY: update, + TRAINED_SLICE_ID_KEY: trained_slice_id, + }, + route=f"dataset/{self.id}/modelRun/{model_run_id}/uploadPredictions?async=1", + ) + return AsyncJob.from_json(response, self._client) + + return uploader.upload( + annotations=predictions, + batch_size=batch_size, + update=update, + remote_files_per_upload_request=remote_files_per_upload_request, + local_files_per_upload_request=local_files_per_upload_request, + trained_slice_id=trained_slice_id, + ) + def predictions_iloc(self, model, index): """Fetches all predictions of a dataset item by its absolute index. diff --git a/nucleus/model_run.py b/nucleus/model_run.py index ad722893..4689e2a6 100644 --- a/nucleus/model_run.py +++ b/nucleus/model_run.py @@ -137,6 +137,12 @@ def predict( ) -> Union[dict, AsyncJob]: """Uploads model outputs as predictions for a model_run. + Deprecated along with the rest of this class. The target dataset is + inferred from the run rather than named, so this fails for a run that + spans more than one dataset — there is no single dataset to infer. Use + :meth:`Dataset.upload_predictions_for_model_run` instead, which takes + both ids explicitly. + Args: annotations: Predictions to upload for this model run. update: If True, existing predictions for the same (reference_id, annotation_id) diff --git a/pyproject.toml b/pyproject.toml index e12a2c2f..6f6de6a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ ignore = ["E501", "E741", "E731", "F401"] # Easy ignore for getting it running [tool.poetry] name = "scale-nucleus" -version = "0.19.0" +version = "0.20.0" description = "The official Python client library for Nucleus, the Data Platform for AI" license = "MIT" authors = ["Scale AI Nucleus Team "] diff --git a/tests/test_multi_dataset_model_runs.py b/tests/test_multi_dataset_model_runs.py new file mode 100644 index 00000000..39d3c3db --- /dev/null +++ b/tests/test_multi_dataset_model_runs.py @@ -0,0 +1,173 @@ +"""Unit tests for multi-dataset model runs (no live API). + +A model run used to declare exactly one dataset, and prediction uploads had to +stay inside it. It now carries the *set* of datasets its predictions actually +land in, which is what lets one run be scored against a benchmark whose items +span several datasets. + +These tests pin the routing, because the route is the whole difference: only +``dataset/{dataset_id}/modelRun/{model_run_id}/uploadPredictions`` can add a +dataset to a run. The other two prediction routes deliberately cannot, and +silently sending a widening upload to one of them would either create a second +run or be rejected server-side. +""" + +from unittest.mock import MagicMock + +import pytest + +from nucleus import NucleusClient +from nucleus.annotation_uploader import PredictionUploader +from nucleus.dataset import Dataset +from nucleus.prediction import BoxPrediction + + +def _client(): + return NucleusClient(api_key="test") + + +def _predictions(): + return [ + BoxPrediction( + label="car", + x=0, + y=0, + width=10, + height=10, + reference_id="item_1", + confidence=0.9, + ) + ] + + +# --------------------------------------------------------------------------- # +# PredictionUploader routing — the three forms +# --------------------------------------------------------------------------- # +def test_dataset_and_model_run_ids_route_to_the_widening_endpoint(): + uploader = PredictionUploader( + client=_client(), dataset_id="ds_1", model_run_id="run_1" + ) + assert ( + uploader._route + == "dataset/ds_1/modelRun/run_1/uploadPredictions" # noqa: SLF001 + ) + + +def test_dataset_and_model_ids_route_to_the_model_endpoint(): + """The (dataset, model) form is unchanged — it cannot widen a run.""" + uploader = PredictionUploader( + client=_client(), dataset_id="ds_1", model_id="prj_1" + ) + assert ( + uploader._route + == "dataset/ds_1/model/prj_1/uploadPredictions" # noqa: SLF001 + ) + + +def test_model_run_id_alone_routes_to_the_deprecated_endpoint(): + """Kept working for single-dataset runs; the server infers the dataset.""" + uploader = PredictionUploader(client=_client(), model_run_id="run_1") + assert uploader._route == "modelRun/run_1/predict" # noqa: SLF001 + + +def test_model_id_and_model_run_id_together_are_rejected(): + with pytest.raises(AssertionError): + PredictionUploader( + client=_client(), + dataset_id="ds_1", + model_id="prj_1", + model_run_id="run_1", + ) + + +def test_neither_model_nor_model_run_is_rejected(): + with pytest.raises(AssertionError): + PredictionUploader(client=_client(), dataset_id="ds_1") + + +# --------------------------------------------------------------------------- # +# Dataset.upload_predictions_for_model_run +# --------------------------------------------------------------------------- # +def test_upload_predictions_for_model_run_uses_the_widening_route(): + client = _client() + dataset = Dataset("ds_1", client) + uploaded = {} + + def _capture(**kwargs): + uploaded.update(kwargs) + return {"predictions_processed": 1, "predictions_ignored": 0} + + with pytest.MonkeyPatch.context() as mp: + routes = [] + original_init = PredictionUploader.__init__ + + def _spy_init(self, *args, **kwargs): + original_init(self, *args, **kwargs) + routes.append(self._route) # noqa: SLF001 + + mp.setattr(PredictionUploader, "__init__", _spy_init) + mp.setattr( + PredictionUploader, "upload", lambda self, **kw: _capture(**kw) + ) + dataset.upload_predictions_for_model_run("run_1", _predictions()) + + assert routes == ["dataset/ds_1/modelRun/run_1/uploadPredictions"] + assert uploaded["update"] is False + + +def test_upload_predictions_for_model_run_async_hits_the_async_route(): + client = _client() + dataset = Dataset("ds_1", client) + client.make_request = MagicMock( + return_value={ + "job_id": "job_1", + "job_last_known_status": "Running", + "job_type": "uploadPredictions", + "job_creation_time": "2026-08-03T00:00:00.000Z", + } + ) + + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "nucleus.dataset.serialize_and_write_to_presigned_url", + lambda *args, **kwargs: "req_1", + ) + dataset.upload_predictions_for_model_run( + "run_1", _predictions(), asynchronous=True + ) + + route = client.make_request.call_args[1]["route"] + assert route == "dataset/ds_1/modelRun/run_1/uploadPredictions?async=1" + + +def test_upload_predictions_for_model_run_forwards_trained_slice_id(): + client = _client() + dataset = Dataset("ds_1", client) + captured = {} + + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + PredictionUploader, + "upload", + lambda self, **kw: captured.update(kw) or {}, + ) + dataset.upload_predictions_for_model_run( + "run_1", _predictions(), trained_slice_id="slc_1", update=True + ) + + assert captured["trained_slice_id"] == "slc_1" + assert captured["update"] is True + + +def test_upload_predictions_for_model_run_rejects_duplicate_ids(): + """Inherited from PredictionUploader; asserted here so the new entry point + is known to run the check rather than bypass it.""" + from nucleus.errors import DuplicateIDError + + dataset = Dataset("ds_1", _client()) + duplicate = _predictions() * 2 + for pred in duplicate: + pred.annotation_id = "ann_1" + + with pytest.raises(DuplicateIDError): + dataset.upload_predictions_for_model_run("run_1", duplicate)