From 42d6063909a8699565721ef1cc10e0fe7f81c6b2 Mon Sep 17 00:00:00 2001 From: Gunnar Atli Thoroddsen Date: Mon, 25 Apr 2022 17:16:10 +0200 Subject: [PATCH 1/5] Drop invalid polygons for IOU matching --- nucleus/metrics/polygon_utils.py | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/nucleus/metrics/polygon_utils.py b/nucleus/metrics/polygon_utils.py index c40e90ff..43ac9298 100644 --- a/nucleus/metrics/polygon_utils.py +++ b/nucleus/metrics/polygon_utils.py @@ -1,3 +1,4 @@ +import logging import sys from functools import wraps from typing import Dict, List, Tuple, TypeVar @@ -82,6 +83,39 @@ def _iou_assignments_for_same_reference_id( polygon_annotations = list(map(polygon_annotation_to_shape, annotations)) polygon_predictions = list(map(polygon_annotation_to_shape, predictions)) + invalid_anns = [ + ann + for ann, poly in zip(annotations, polygon_annotations) + if not poly.is_valid + ] + invalid_preds = [ + pred + for pred, poly in zip(predictions, polygon_predictions) + if not poly.is_valid + ] + + if invalid_anns or invalid_preds: + # Filter out invalid polys + polygon_annotations = [ + poly + for ann, poly in zip(annotations, polygon_annotations) + if poly.is_valid + ] + polygon_predictions = [ + poly + for pred, poly in zip(predictions, polygon_predictions) + if poly.is_valid + ] + invalid_dataset_ids = set( + ann.reference_id for ann in invalid_anns + ).union(set(pred.reference_id for pred in invalid_preds)) + logging.warning( + "Invalid polygons for dataset items: %s Annotations:%s, predictions: %s", + invalid_dataset_ids, + [a.annotation_id for a in invalid_anns], + [p.reference_id for p in invalid_preds], + ) + # Compute IoU matrix and set IoU values below the threshold to 0. iou_matrix = _iou_matrix(polygon_annotations, polygon_predictions) iou_matrix[iou_matrix < iou_threshold] = 0 From ff79b7a83a0b772fdf4edb6280fc3ff9f3be74b0 Mon Sep 17 00:00:00 2001 From: Gunnar Atli Thoroddsen Date: Tue, 26 Apr 2022 13:53:52 +0200 Subject: [PATCH 2/5] Change log for invalid itemsm --- nucleus/metrics/polygon_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nucleus/metrics/polygon_utils.py b/nucleus/metrics/polygon_utils.py index 43ac9298..ac37de7f 100644 --- a/nucleus/metrics/polygon_utils.py +++ b/nucleus/metrics/polygon_utils.py @@ -109,11 +109,12 @@ def _iou_assignments_for_same_reference_id( invalid_dataset_ids = set( ann.reference_id for ann in invalid_anns ).union(set(pred.reference_id for pred in invalid_preds)) + # TODO(gunnar): change to .id once field is surfaced) logging.warning( "Invalid polygons for dataset items: %s Annotations:%s, predictions: %s", invalid_dataset_ids, [a.annotation_id for a in invalid_anns], - [p.reference_id for p in invalid_preds], + [p.annotation_id for p in invalid_preds], ) # Compute IoU matrix and set IoU values below the threshold to 0. From 2f3e27d17e10df2be80b26cbedce140f93bc4130 Mon Sep 17 00:00:00 2001 From: Gunnar Atli Thoroddsen Date: Tue, 26 Apr 2022 14:17:04 +0200 Subject: [PATCH 3/5] Add id fields to annotations and predictions when loading from server --- nucleus/annotation.py | 18 ++++++++++++++++++ nucleus/metrics/polygon_utils.py | 4 ++-- nucleus/prediction.py | 20 ++++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/nucleus/annotation.py b/nucleus/annotation.py index 7a6e1eb2..1dc3ab99 100644 --- a/nucleus/annotation.py +++ b/nucleus/annotation.py @@ -15,6 +15,7 @@ EMBEDDING_VECTOR_KEY, GEOMETRY_KEY, HEIGHT_KEY, + ID_KEY, INDEX_KEY, KEYPOINTS_KEY, KEYPOINTS_NAMES_KEY, @@ -50,6 +51,7 @@ class Annotation: """ reference_id: str + id: Optional[str] @classmethod def from_json(cls, payload: dict): @@ -143,6 +145,7 @@ class BoxAnnotation(Annotation): # pylint: disable=R0902 annotation_id: Optional[str] = None metadata: Optional[Dict] = None embedding_vector: Optional[list] = None + id: Optional[str] = None def __post_init__(self): self.metadata = self.metadata if self.metadata else {} @@ -162,6 +165,7 @@ def from_json(cls, payload: dict): annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), embedding_vector=payload.get(EMBEDDING_VECTOR_KEY, None), + id=payload.get(ID_KEY, None), ) def to_payload(self) -> dict: @@ -239,6 +243,7 @@ class LineAnnotation(Annotation): reference_id: str annotation_id: Optional[str] = None metadata: Optional[Dict] = None + id: Optional[str] = None def __post_init__(self): self.metadata = self.metadata if self.metadata else {} @@ -267,6 +272,7 @@ def from_json(cls, payload: dict): reference_id=payload[REFERENCE_ID_KEY], annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), + id=payload.get(ID_KEY, None), ) def to_payload(self) -> dict: @@ -324,6 +330,7 @@ class PolygonAnnotation(Annotation): annotation_id: Optional[str] = None metadata: Optional[Dict] = None embedding_vector: Optional[list] = None + id: Optional[str] = None def __post_init__(self): self.metadata = self.metadata if self.metadata else {} @@ -353,6 +360,7 @@ def from_json(cls, payload: dict): annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), embedding_vector=payload.get(EMBEDDING_VECTOR_KEY, None), + id=payload.get(ID_KEY, None), ) def to_payload(self) -> dict: @@ -457,6 +465,7 @@ class KeypointsAnnotation(Annotation): reference_id: str annotation_id: Optional[str] = None metadata: Optional[Dict] = None + id: Optional[str] = None def __post_init__(self): self.metadata = self.metadata or {} @@ -483,6 +492,7 @@ def from_json(cls, payload: dict): reference_id=payload[REFERENCE_ID_KEY], annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), + id=payload.get(ID_KEY, None), ) def to_payload(self) -> dict: @@ -564,6 +574,7 @@ class CuboidAnnotation(Annotation): # pylint: disable=R0902 reference_id: str annotation_id: Optional[str] = None metadata: Optional[Dict] = None + id: Optional[str] = None def __post_init__(self): self.metadata = self.metadata if self.metadata else {} @@ -579,6 +590,7 @@ def from_json(cls, payload: dict): reference_id=payload[REFERENCE_ID_KEY], annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), + id=payload.get(ID_KEY, None), ) def to_payload(self) -> dict: @@ -709,6 +721,7 @@ class SegmentationAnnotation(Annotation): annotations: List[Segment] reference_id: str annotation_id: Optional[str] = None + id: Optional[str] = None # metadata: Optional[dict] = None # TODO(sc: 422637) def __post_init__(self): @@ -727,6 +740,7 @@ def from_json(cls, payload: dict): ], reference_id=payload[REFERENCE_ID_KEY], annotation_id=payload.get(ANNOTATION_ID_KEY, None), + id=payload.get(ID_KEY, None), # metadata=payload.get(METADATA_KEY, None), # TODO(sc: 422637) ) @@ -804,6 +818,7 @@ class CategoryAnnotation(Annotation): reference_id: str taxonomy_name: Optional[str] = None metadata: Optional[Dict] = None + id: Optional[str] = None def __post_init__(self): self.metadata = self.metadata if self.metadata else {} @@ -815,6 +830,7 @@ def from_json(cls, payload: dict): reference_id=payload[REFERENCE_ID_KEY], taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None), metadata=payload.get(METADATA_KEY, {}), + id=payload.get(ID_KEY, None), ) def to_payload(self) -> dict: @@ -838,6 +854,7 @@ class MultiCategoryAnnotation(Annotation): reference_id: str taxonomy_name: Optional[str] = None metadata: Optional[Dict] = None + id: Optional[str] = None def __post_init__(self): self.metadata = self.metadata if self.metadata else {} @@ -849,6 +866,7 @@ def from_json(cls, payload: dict): reference_id=payload[REFERENCE_ID_KEY], taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None), metadata=payload.get(METADATA_KEY, {}), + id=payload.get(ID_KEY, None), ) def to_payload(self) -> dict: diff --git a/nucleus/metrics/polygon_utils.py b/nucleus/metrics/polygon_utils.py index ac37de7f..051cb160 100644 --- a/nucleus/metrics/polygon_utils.py +++ b/nucleus/metrics/polygon_utils.py @@ -113,8 +113,8 @@ def _iou_assignments_for_same_reference_id( logging.warning( "Invalid polygons for dataset items: %s Annotations:%s, predictions: %s", invalid_dataset_ids, - [a.annotation_id for a in invalid_anns], - [p.annotation_id for p in invalid_preds], + [a.id for a in invalid_anns], + [p.id for p in invalid_preds], ) # Compute IoU matrix and set IoU values below the threshold to 0. diff --git a/nucleus/prediction.py b/nucleus/prediction.py index e2beee92..d69d58c0 100644 --- a/nucleus/prediction.py +++ b/nucleus/prediction.py @@ -31,6 +31,7 @@ EMBEDDING_VECTOR_KEY, GEOMETRY_KEY, HEIGHT_KEY, + ID_KEY, KEYPOINTS_KEY, KEYPOINTS_NAMES_KEY, KEYPOINTS_SKELETON_KEY, @@ -128,6 +129,7 @@ def from_json(cls, payload: dict): ], reference_id=payload[REFERENCE_ID_KEY], annotation_id=payload.get(ANNOTATION_ID_KEY, None), + id=payload.get(ID_KEY, None), # metadata=payload.get(METADATA_KEY, None), # TODO(sc: 422637) ) @@ -179,6 +181,7 @@ def __init__( metadata: Optional[Dict] = None, class_pdf: Optional[Dict] = None, embedding_vector: Optional[list] = None, + id: Optional[str] = None, # pylint: disable=redefined-builtin ): super().__init__( label=label, @@ -190,6 +193,7 @@ def __init__( annotation_id=annotation_id, metadata=metadata, embedding_vector=embedding_vector, + id=id, ) self.confidence = confidence self.class_pdf = class_pdf @@ -218,6 +222,7 @@ def from_json(cls, payload: dict): metadata=payload.get(METADATA_KEY, {}), class_pdf=payload.get(CLASS_PDF_KEY, None), embedding_vector=payload.get(EMBEDDING_VECTOR_KEY, None), + id=payload.get(ID_KEY, None), ) @@ -253,6 +258,7 @@ def __init__( annotation_id: Optional[str] = None, metadata: Optional[Dict] = None, class_pdf: Optional[Dict] = None, + id: Optional[str] = None, # pylint: disable=redefined-builtin ): super().__init__( label=label, @@ -260,6 +266,7 @@ def __init__( reference_id=reference_id, annotation_id=annotation_id, metadata=metadata, + id=id, ) self.confidence = confidence self.class_pdf = class_pdf @@ -286,6 +293,7 @@ def from_json(cls, payload: dict): annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), class_pdf=payload.get(CLASS_PDF_KEY, None), + id=payload.get(ID_KEY, None), ) @@ -325,6 +333,7 @@ def __init__( metadata: Optional[Dict] = None, class_pdf: Optional[Dict] = None, embedding_vector: Optional[list] = None, + id: Optional[str] = None, # pylint: disable=redefined-builtin ): super().__init__( label=label, @@ -333,6 +342,7 @@ def __init__( annotation_id=annotation_id, metadata=metadata, embedding_vector=embedding_vector, + id=id, ) self.confidence = confidence self.class_pdf = class_pdf @@ -360,6 +370,7 @@ def from_json(cls, payload: dict): metadata=payload.get(METADATA_KEY, {}), class_pdf=payload.get(CLASS_PDF_KEY, None), embedding_vector=payload.get(EMBEDDING_VECTOR_KEY, None), + id=payload.get(ID_KEY, None), ) @@ -400,6 +411,7 @@ def __init__( annotation_id: Optional[str] = None, metadata: Optional[Dict] = None, class_pdf: Optional[Dict] = None, + id: Optional[str] = None, # pylint: disable=redefined-builtin ): super().__init__( label=label, @@ -409,6 +421,7 @@ def __init__( reference_id=reference_id, annotation_id=annotation_id, metadata=metadata, + id=id, ) self.confidence = confidence self.class_pdf = class_pdf @@ -437,6 +450,7 @@ def from_json(cls, payload: dict): annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), class_pdf=payload.get(CLASS_PDF_KEY, None), + id=payload.get(ID_KEY, None), ) @@ -475,6 +489,7 @@ def __init__( annotation_id: Optional[str] = None, metadata: Optional[Dict] = None, class_pdf: Optional[Dict] = None, + id: Optional[str] = None, # pylint: disable=redefined-builtin ): super().__init__( label=label, @@ -484,6 +499,7 @@ def __init__( reference_id=reference_id, annotation_id=annotation_id, metadata=metadata, + id=id, ) self.confidence = confidence self.class_pdf = class_pdf @@ -510,6 +526,7 @@ def from_json(cls, payload: dict): annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), class_pdf=payload.get(CLASS_PDF_KEY, None), + id=payload.get(ID_KEY, None), ) @@ -540,12 +557,14 @@ def __init__( confidence: Optional[float] = None, metadata: Optional[Dict] = None, class_pdf: Optional[Dict] = None, + id: Optional[str] = None, # pylint: disable=redefined-builtin ): super().__init__( label=label, taxonomy_name=taxonomy_name, reference_id=reference_id, metadata=metadata, + id=id, ) self.confidence = confidence self.class_pdf = class_pdf @@ -568,6 +587,7 @@ def from_json(cls, payload: dict): confidence=payload.get(CONFIDENCE_KEY, None), metadata=payload.get(METADATA_KEY, {}), class_pdf=payload.get(CLASS_PDF_KEY, None), + id=payload.get(ID_KEY, None), ) From 9ad3bd39791277819ea85fe6907cc4d1abb9db81 Mon Sep 17 00:00:00 2001 From: Gunnar Atli Thoroddsen Date: Tue, 26 Apr 2022 14:23:44 +0200 Subject: [PATCH 4/5] Add id fields to docstrings --- nucleus/annotation.py | 8 ++++++++ nucleus/prediction.py | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/nucleus/annotation.py b/nucleus/annotation.py index 1dc3ab99..3d8959e9 100644 --- a/nucleus/annotation.py +++ b/nucleus/annotation.py @@ -134,6 +134,7 @@ class BoxAnnotation(Annotation): # pylint: disable=R0902 embedding_vector: Custom embedding vector for this object annotation. If any custom object embeddings have been uploaded previously to this dataset, this vector must match the dimensions of the previously ingested vectors. + id: Unique Nucleus generated ID. This field is populated when loading Annotations from the server. """ label: str @@ -236,6 +237,7 @@ class LineAnnotation(Annotation): attach to this annotation. Strings, floats and ints are supported best by querying and insights features within Nucleus. For more details see our `metadata guide `_. + id: Unique Nucleus generated ID. This field is populated when loading Annotations from the server. """ label: str @@ -322,6 +324,7 @@ class PolygonAnnotation(Annotation): embedding_vector: Custom embedding vector for this object annotation. If any custom object embeddings have been uploaded previously to this dataset, this vector must match the dimensions of the previously ingested vectors. + id: Unique Nucleus generated ID. This field is populated when loading Annotations from the server. """ label: str @@ -456,6 +459,7 @@ class KeypointsAnnotation(Annotation): attach to this annotation. Strings, floats and ints are supported best by querying and insights features within Nucleus. For more details see our `metadata guide `_. + id: Unique Nucleus generated ID. This field is populated when loading Annotations from the server. """ label: str @@ -565,6 +569,7 @@ class CuboidAnnotation(Annotation): # pylint: disable=R0902 annotation. Strings, floats and ints are supported best by querying and insights features within Nucleus. For more details see our `metadata guide `_. + id: Unique Nucleus generated ID. This field is populated when loading Annotations from the server. """ label: str @@ -632,6 +637,7 @@ class index and the string label. Strings, floats and ints are supported best by querying and insights features within Nucleus. For more details see our `metadata guide `_. + id: Unique Nucleus generated ID. This field is populated when loading Annotations from the server. """ label: str @@ -715,6 +721,7 @@ class SegmentationAnnotation(Annotation): is passed to :meth:`Dataset.annotate`, in which case it will be overwritten. Storing a custom ID here may be useful in order to tie this annotation to an external database, and its value will be returned for any export. + id: Unique Nucleus generated ID. This field is populated when loading Annotations from the server. """ mask_url: str @@ -812,6 +819,7 @@ class CategoryAnnotation(Annotation): Strings, floats and ints are supported best by querying and insights features within Nucleus. For more details see our `metadata guide `_. + id: Unique Nucleus generated ID. This field is populated when loading Annotations from the server. """ label: str diff --git a/nucleus/prediction.py b/nucleus/prediction.py index d69d58c0..7ffa46fa 100644 --- a/nucleus/prediction.py +++ b/nucleus/prediction.py @@ -117,6 +117,7 @@ class SegmentationPrediction(SegmentationAnnotation): is passed to :meth:`Dataset.annotate`, in which case it will be overwritten. Storing a custom ID here may be useful in order to tie this annotation to an external database, and its value will be returned for any export. + id: Unique Nucleus generated ID. This field is populated when loading Predictions from the server. """ @classmethod @@ -166,6 +167,7 @@ class BoxPrediction(BoxAnnotation): embedding_vector (Optional[List]): Custom embedding vector for this object annotation. If any custom object embeddings have been uploaded previously to this dataset, this vector must match the dimensions of the previously ingested vectors. + id: Unique Nucleus generated ID. This field is populated when loading Predictions from the server. """ def __init__( @@ -247,6 +249,7 @@ class LinePrediction(LineAnnotation): annotation. Each value should be between 0 and 1 (inclusive), and sum up to 1 as a complete distribution. This can be useful for computing entropy to surface places where the model is most uncertain. + id: Unique Nucleus generated ID. This field is populated when loading Predictions from the server. """ def __init__( @@ -321,6 +324,7 @@ class PolygonPrediction(PolygonAnnotation): embedding_vector: Custom embedding vector for this object annotation. If any custom object embeddings have been uploaded previously to this dataset, this vector must match the dimensions of the previously ingested vectors. + id: Unique Nucleus generated ID. This field is populated when loading Predictions from the server. """ def __init__( @@ -398,6 +402,7 @@ class KeypointsPrediction(KeypointsAnnotation): annotation. Each value should be between 0 and 1 (inclusive), and sum up to 1 as a complete distribution. This can be useful for computing entropy to surface places where the model is most uncertain. + id: Unique Nucleus generated ID. This field is populated when loading Predictions from the server. """ def __init__( @@ -476,6 +481,7 @@ class CuboidPrediction(CuboidAnnotation): annotation. Each value should be between 0 and 1 (inclusive), and sum up to 1 as a complete distribution. This can be useful for computing entropy to surface places where the model is most uncertain. + id: Unique Nucleus generated ID. This field is populated when loading Predictions from the server. """ def __init__( @@ -547,6 +553,7 @@ class CategoryPrediction(CategoryAnnotation): Strings, floats and ints are supported best by querying and insights features within Nucleus. For more details see our `metadata guide `_. + id: Unique Nucleus generated ID. This field is populated when loading Predictions from the server. """ def __init__( From 5906e6eec50de40b66b539a8924e15636dae82bb Mon Sep 17 00:00:00 2001 From: Gunnar Atli Thoroddsen Date: Wed, 27 Apr 2022 12:45:44 +0200 Subject: [PATCH 5/5] Change id to property and init in common Annotation.from_json --- nucleus/annotation.py | 91 ++++++--------------------- nucleus/prediction.py | 45 ------------- tests/metrics/test_polygon_metrics.py | 2 +- 3 files changed, 20 insertions(+), 118 deletions(-) diff --git a/nucleus/annotation.py b/nucleus/annotation.py index 3d8959e9..71da3290 100644 --- a/nucleus/annotation.py +++ b/nucleus/annotation.py @@ -45,13 +45,16 @@ class Annotation: """Internal base class, not to be used directly. - .. todo :: Inherit common constructor parameters from here """ reference_id: str - id: Optional[str] + _nucleus_id: Optional[str] = None + + @property + def id(self) -> Optional[str]: + return self._nucleus_id @classmethod def from_json(cls, payload: dict): @@ -67,7 +70,12 @@ def from_json(cls, payload: dict): } type_key = payload.get(TYPE_KEY, None) AnnotationCls = type_key_to_type.get(type_key, SegmentationAnnotation) - return AnnotationCls.from_json(payload) + instance = AnnotationCls.from_json(payload) + # NOTE: Accessing protected var of sub-class looks like the cleanest way for a common classmethod functionality + instance._nucleus_id = payload.get( # pylint: disable=protected-access + ID_KEY, None + ) + return instance def to_payload(self) -> dict: """Serializes annotation object to schematized JSON dict.""" @@ -82,7 +90,6 @@ def to_json(self) -> str: def has_local_files_to_upload(self) -> bool: """Returns True if annotation has local files that need to be uploaded. - Nearly all subclasses have no local files, so we default this to just return false. If the subclass has local files, it should override this method (but that is not the only thing required to get local upload of files to work.) @@ -93,11 +100,8 @@ def has_local_files_to_upload(self) -> bool: @dataclass # pylint: disable=R0902 class BoxAnnotation(Annotation): # pylint: disable=R0902 """A bounding box annotation. - :: - from nucleus import BoxAnnotation - box = BoxAnnotation( label="car", x=0, @@ -109,7 +113,6 @@ class BoxAnnotation(Annotation): # pylint: disable=R0902 metadata={"vehicle_color": "red"}, embedding_vector=[0.1423, 1.432, ...3.829], ) - Parameters: label (str): The label for this annotation. x (Union[float, int]): The distance, in pixels, between the left border @@ -134,7 +137,7 @@ class BoxAnnotation(Annotation): # pylint: disable=R0902 embedding_vector: Custom embedding vector for this object annotation. If any custom object embeddings have been uploaded previously to this dataset, this vector must match the dimensions of the previously ingested vectors. - id: Unique Nucleus generated ID. This field is populated when loading Annotations from the server. + """ label: str @@ -146,7 +149,6 @@ class BoxAnnotation(Annotation): # pylint: disable=R0902 annotation_id: Optional[str] = None metadata: Optional[Dict] = None embedding_vector: Optional[list] = None - id: Optional[str] = None def __post_init__(self): self.metadata = self.metadata if self.metadata else {} @@ -166,7 +168,6 @@ def from_json(cls, payload: dict): annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), embedding_vector=payload.get(EMBEDDING_VECTOR_KEY, None), - id=payload.get(ID_KEY, None), ) def to_payload(self) -> dict: @@ -189,7 +190,6 @@ def to_payload(self) -> dict: @dataclass class Point: """A point in 2D space. - Parameters: x (float): The x coordinate of the point. y (float): The y coordinate of the point. @@ -211,11 +211,8 @@ class LineAnnotation(Annotation): """A polyline annotation consisting of an ordered list of 2D points. A LineAnnotation differs from a PolygonAnnotation by not forming a closed loop, and by having zero area. - :: - from nucleus import LineAnnotation - line = LineAnnotation( label="face", vertices=[Point(100, 100), Point(200, 300), Point(300, 200)], @@ -223,7 +220,6 @@ class LineAnnotation(Annotation): annotation_id="person_image_1_line_1", metadata={"camera_mode": "portrait"}, ) - Parameters: label (str): The label for this annotation. vertices (List[:class:`Point`]): The list of points making up the line. @@ -237,7 +233,7 @@ class LineAnnotation(Annotation): attach to this annotation. Strings, floats and ints are supported best by querying and insights features within Nucleus. For more details see our `metadata guide `_. - id: Unique Nucleus generated ID. This field is populated when loading Annotations from the server. + """ label: str @@ -245,7 +241,6 @@ class LineAnnotation(Annotation): reference_id: str annotation_id: Optional[str] = None metadata: Optional[Dict] = None - id: Optional[str] = None def __post_init__(self): self.metadata = self.metadata if self.metadata else {} @@ -274,7 +269,6 @@ def from_json(cls, payload: dict): reference_id=payload[REFERENCE_ID_KEY], annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), - id=payload.get(ID_KEY, None), ) def to_payload(self) -> dict: @@ -294,11 +288,8 @@ def to_payload(self) -> dict: @dataclass class PolygonAnnotation(Annotation): """A polygon annotation consisting of an ordered list of 2D points. - :: - from nucleus import PolygonAnnotation - polygon = PolygonAnnotation( label="bus", vertices=[Point(100, 100), Point(150, 200), Point(200, 100)], @@ -307,7 +298,6 @@ class PolygonAnnotation(Annotation): metadata={"vehicle_color": "yellow"}, embedding_vector=[0.1423, 1.432, ...3.829], ) - Parameters: label (str): The label for this annotation. vertices (List[:class:`Point`]): The list of points making up the polygon. @@ -324,7 +314,7 @@ class PolygonAnnotation(Annotation): embedding_vector: Custom embedding vector for this object annotation. If any custom object embeddings have been uploaded previously to this dataset, this vector must match the dimensions of the previously ingested vectors. - id: Unique Nucleus generated ID. This field is populated when loading Annotations from the server. + """ label: str @@ -333,7 +323,6 @@ class PolygonAnnotation(Annotation): annotation_id: Optional[str] = None metadata: Optional[Dict] = None embedding_vector: Optional[list] = None - id: Optional[str] = None def __post_init__(self): self.metadata = self.metadata if self.metadata else {} @@ -363,7 +352,6 @@ def from_json(cls, payload: dict): annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), embedding_vector=payload.get(EMBEDDING_VECTOR_KEY, None), - id=payload.get(ID_KEY, None), ) def to_payload(self) -> dict: @@ -384,13 +372,11 @@ def to_payload(self) -> dict: @dataclass class Keypoint: """A 2D point that has an additional visibility flag. - Keypoints are intended to be part of a larger collection, and connected via a pre-defined skeleton. A keypoint in this skeleton may be visible or not-visible, and may be unlabeled and not visible. Because of this, the x, y coordinates may be optional, assuming that the keypoint is not visible, and would not be shown as part of the combined label. - Parameters: x (Optional[float]): The x coordinate of the point. y (Optional[float]): The y coordinate of the point. @@ -428,11 +414,8 @@ class KeypointsAnnotation(Annotation): """A keypoints annotation containing a list of keypoints and the structure of those keypoints: the naming of each point and the skeleton that connects those keypoints. - :: - from nucleus import KeypointsAnnotation - keypoints = KeypointsAnnotation( label="face", keypoints=[Keypoint(100, 100), Keypoint(120, 120), Keypoint(visible=False), Keypoint(0, 0)], @@ -442,7 +425,6 @@ class KeypointsAnnotation(Annotation): annotation_id="image_2_face_keypoints_1", metadata={"face_direction": "forward"}, ) - Parameters: label (str): The label for this annotation. keypoints (List[:class:`Keypoint`]): The list of keypoints objects. @@ -459,7 +441,7 @@ class KeypointsAnnotation(Annotation): attach to this annotation. Strings, floats and ints are supported best by querying and insights features within Nucleus. For more details see our `metadata guide `_. - id: Unique Nucleus generated ID. This field is populated when loading Annotations from the server. + """ label: str @@ -469,7 +451,6 @@ class KeypointsAnnotation(Annotation): reference_id: str annotation_id: Optional[str] = None metadata: Optional[Dict] = None - id: Optional[str] = None def __post_init__(self): self.metadata = self.metadata or {} @@ -496,7 +477,6 @@ def from_json(cls, payload: dict): reference_id=payload[REFERENCE_ID_KEY], annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), - id=payload.get(ID_KEY, None), ) def to_payload(self) -> dict: @@ -518,7 +498,6 @@ def to_payload(self) -> dict: @dataclass class Point3D: """A point in 3D space. - Parameters: x (float): The x coordinate of the point. y (float): The y coordinate of the point. @@ -540,11 +519,8 @@ def to_payload(self) -> dict: @dataclass # pylint: disable=R0902 class CuboidAnnotation(Annotation): # pylint: disable=R0902 """A 3D Cuboid annotation. - :: - from nucleus import CuboidAnnotation - cuboid = CuboidAnnotation( label="car", position=Point3D(100, 100, 10), @@ -554,7 +530,6 @@ class CuboidAnnotation(Annotation): # pylint: disable=R0902 annotation_id="pointcloud_1_car_cuboid_1", metadata={"vehicle_color": "green"} ) - Parameters: label (str): The label for this annotation. position (:class:`Point3D`): The point at the center of the cuboid @@ -569,7 +544,7 @@ class CuboidAnnotation(Annotation): # pylint: disable=R0902 annotation. Strings, floats and ints are supported best by querying and insights features within Nucleus. For more details see our `metadata guide `_. - id: Unique Nucleus generated ID. This field is populated when loading Annotations from the server. + """ label: str @@ -579,7 +554,6 @@ class CuboidAnnotation(Annotation): # pylint: disable=R0902 reference_id: str annotation_id: Optional[str] = None metadata: Optional[Dict] = None - id: Optional[str] = None def __post_init__(self): self.metadata = self.metadata if self.metadata else {} @@ -595,7 +569,6 @@ def from_json(cls, payload: dict): reference_id=payload[REFERENCE_ID_KEY], annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), - id=payload.get(ID_KEY, None), ) def to_payload(self) -> dict: @@ -613,22 +586,17 @@ def to_payload(self) -> dict: payload[ANNOTATION_ID_KEY] = self.annotation_id if self.metadata: payload[METADATA_KEY] = self.metadata - return payload @dataclass class Segment: """Segment represents either a class or an instance depending on the task type. - For semantic segmentation, this object should store the mapping between a single class index and the string label. - For instance segmentation, you can use this class to store the label of a single instance, whose extent in the image is represented by the value of ``index``. - In both cases, additional metadata can be attached to the segment. - Parameters: label (str): The label name of the class for the class or instance represented by index in the associated mask. @@ -637,7 +605,7 @@ class index and the string label. Strings, floats and ints are supported best by querying and insights features within Nucleus. For more details see our `metadata guide `_. - id: Unique Nucleus generated ID. This field is populated when loading Annotations from the server. + """ label: str @@ -665,21 +633,16 @@ def to_payload(self) -> dict: @dataclass class SegmentationAnnotation(Annotation): """A segmentation mask on a 2D image. - When uploading a mask annotation, Nucleus expects the mask file to be in PNG format with each pixel being a 0-255 uint8. Currently, Nucleus only supports uploading masks from URL. - Nucleus automatically enforces the constraint that each DatasetItem can have at most one ground truth segmentation mask. As a consequence, if during upload a duplicate mask is detected for a given image, by default it will be ignored. You can change this behavior by setting ``update = True``, which will replace the existing segmentation mask with the new mask. - :: - from nucleus import SegmentationAnnotation - segmentation = SegmentationAnnotation( mask_url="s3://your-bucket-name/segmentation-masks/image_2_mask_id1.png", annotations=[ @@ -691,19 +654,16 @@ class SegmentationAnnotation(Annotation): reference_id="image_2", annotation_id="image_2_mask_1", ) - Parameters: mask_url (str): A URL pointing to the segmentation prediction mask which is accessible to Scale, or a local path. The mask is an HxW int8 array saved in PNG format, with each pixel value ranging from [0, N), where N is the number of possible classes (for semantic segmentation) or instances (for instance segmentation). - The height and width of the mask must be the same as the original image. One example for semantic segmentation: the mask is 0 for pixels where there is background, 1 where there is a car, and 2 where there is a pedestrian. - Another example for instance segmentation: the mask is 0 for one car, 1 for another car, 2 for a motorcycle and 3 for another motorcycle. The class name for each value in the mask is stored in the list of @@ -721,14 +681,13 @@ class SegmentationAnnotation(Annotation): is passed to :meth:`Dataset.annotate`, in which case it will be overwritten. Storing a custom ID here may be useful in order to tie this annotation to an external database, and its value will be returned for any export. - id: Unique Nucleus generated ID. This field is populated when loading Annotations from the server. + """ mask_url: str annotations: List[Segment] reference_id: str annotation_id: Optional[str] = None - id: Optional[str] = None # metadata: Optional[dict] = None # TODO(sc: 422637) def __post_init__(self): @@ -747,7 +706,6 @@ def from_json(cls, payload: dict): ], reference_id=payload[REFERENCE_ID_KEY], annotation_id=payload.get(ANNOTATION_ID_KEY, None), - id=payload.get(ID_KEY, None), # metadata=payload.get(METADATA_KEY, None), # TODO(sc: 422637) ) @@ -759,9 +717,7 @@ def to_payload(self) -> dict: ANNOTATION_ID_KEY: self.annotation_id, # METADATA_KEY: self.metadata, # TODO(sc: 422637) } - payload[REFERENCE_ID_KEY] = self.reference_id - return payload def has_local_files_to_upload(self) -> bool: @@ -798,18 +754,14 @@ class AnnotationTypes(Enum): @dataclass class CategoryAnnotation(Annotation): """A category annotation. - :: - from nucleus import CategoryAnnotation - category = CategoryAnnotation( label="dress", reference_id="image_1", taxonomy_name="clothing_type", metadata={"dress_color": "navy"} ) - Parameters: label (str): The label for this annotation. reference_id (str): User-defined ID of the image to which to apply this annotation. @@ -819,14 +771,13 @@ class CategoryAnnotation(Annotation): Strings, floats and ints are supported best by querying and insights features within Nucleus. For more details see our `metadata guide `_. - id: Unique Nucleus generated ID. This field is populated when loading Annotations from the server. + """ label: str reference_id: str taxonomy_name: Optional[str] = None metadata: Optional[Dict] = None - id: Optional[str] = None def __post_init__(self): self.metadata = self.metadata if self.metadata else {} @@ -838,7 +789,6 @@ def from_json(cls, payload: dict): reference_id=payload[REFERENCE_ID_KEY], taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None), metadata=payload.get(METADATA_KEY, {}), - id=payload.get(ID_KEY, None), ) def to_payload(self) -> dict: @@ -862,7 +812,6 @@ class MultiCategoryAnnotation(Annotation): reference_id: str taxonomy_name: Optional[str] = None metadata: Optional[Dict] = None - id: Optional[str] = None def __post_init__(self): self.metadata = self.metadata if self.metadata else {} @@ -874,7 +823,6 @@ def from_json(cls, payload: dict): reference_id=payload[REFERENCE_ID_KEY], taxonomy_name=payload.get(TAXONOMY_NAME_KEY, None), metadata=payload.get(METADATA_KEY, {}), - id=payload.get(ID_KEY, None), ) def to_payload(self) -> dict: @@ -916,7 +864,6 @@ def add_annotations(self, annotations: List[Annotation]): assert isinstance( annotation, Annotation ), "Expected annotation to be of type 'Annotation" - if isinstance(annotation, BoxAnnotation): self.box_annotations.append(annotation) elif isinstance(annotation, LineAnnotation): diff --git a/nucleus/prediction.py b/nucleus/prediction.py index 7ffa46fa..5fd56961 100644 --- a/nucleus/prediction.py +++ b/nucleus/prediction.py @@ -31,7 +31,6 @@ EMBEDDING_VECTOR_KEY, GEOMETRY_KEY, HEIGHT_KEY, - ID_KEY, KEYPOINTS_KEY, KEYPOINTS_NAMES_KEY, KEYPOINTS_SKELETON_KEY, @@ -70,11 +69,8 @@ def from_json(payload: dict): class SegmentationPrediction(SegmentationAnnotation): """Predicted segmentation mask on a 2D image. - :: - from nucleus import SegmentationPrediction - segmentation = SegmentationPrediction( mask_url="s3://your-bucket-name/pred-seg-masks/image_2_pred_mask_id1.png", annotations=[ @@ -86,7 +82,6 @@ class SegmentationPrediction(SegmentationAnnotation): reference_id="image_2", annotation_id="image_2_pred_mask_1", ) - Parameters: mask_url (str): A URL pointing to the segmentation prediction mask which is accessible to Scale. This URL can be a path to a local file. @@ -94,12 +89,10 @@ class SegmentationPrediction(SegmentationAnnotation): with each pixel value ranging from [0, N), where N is the number of possible classes (for semantic segmentation) or instances (for instance segmentation). - The height and width of the mask must be the same as the original image. One example for semantic segmentation: the mask is 0 for pixels where there is background, 1 where there is a car, and 2 where there is a pedestrian. - Another example for instance segmentation: the mask is 0 for one car, 1 for another car, 2 for a motorcycle and 3 for another motorcycle. The class name for each value in the mask is stored in the list of @@ -117,7 +110,6 @@ class SegmentationPrediction(SegmentationAnnotation): is passed to :meth:`Dataset.annotate`, in which case it will be overwritten. Storing a custom ID here may be useful in order to tie this annotation to an external database, and its value will be returned for any export. - id: Unique Nucleus generated ID. This field is populated when loading Predictions from the server. """ @classmethod @@ -130,14 +122,12 @@ def from_json(cls, payload: dict): ], reference_id=payload[REFERENCE_ID_KEY], annotation_id=payload.get(ANNOTATION_ID_KEY, None), - id=payload.get(ID_KEY, None), # metadata=payload.get(METADATA_KEY, None), # TODO(sc: 422637) ) class BoxPrediction(BoxAnnotation): """Prediction of a bounding box. - Parameters: label (str): The label for this annotation (e.g. car, pedestrian, bicycle) x (Union[float, int]): The distance, in pixels, between the left border @@ -167,7 +157,6 @@ class BoxPrediction(BoxAnnotation): embedding_vector (Optional[List]): Custom embedding vector for this object annotation. If any custom object embeddings have been uploaded previously to this dataset, this vector must match the dimensions of the previously ingested vectors. - id: Unique Nucleus generated ID. This field is populated when loading Predictions from the server. """ def __init__( @@ -183,7 +172,6 @@ def __init__( metadata: Optional[Dict] = None, class_pdf: Optional[Dict] = None, embedding_vector: Optional[list] = None, - id: Optional[str] = None, # pylint: disable=redefined-builtin ): super().__init__( label=label, @@ -195,7 +183,6 @@ def __init__( annotation_id=annotation_id, metadata=metadata, embedding_vector=embedding_vector, - id=id, ) self.confidence = confidence self.class_pdf = class_pdf @@ -206,7 +193,6 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf - return payload @classmethod @@ -224,13 +210,11 @@ def from_json(cls, payload: dict): metadata=payload.get(METADATA_KEY, {}), class_pdf=payload.get(CLASS_PDF_KEY, None), embedding_vector=payload.get(EMBEDDING_VECTOR_KEY, None), - id=payload.get(ID_KEY, None), ) class LinePrediction(LineAnnotation): """Prediction of a line. - Parameters: label (str): The label for this prediction (e.g. car, pedestrian, bicycle). vertices List[:class:`Point`]: The list of points making up the line. @@ -249,7 +233,6 @@ class LinePrediction(LineAnnotation): annotation. Each value should be between 0 and 1 (inclusive), and sum up to 1 as a complete distribution. This can be useful for computing entropy to surface places where the model is most uncertain. - id: Unique Nucleus generated ID. This field is populated when loading Predictions from the server. """ def __init__( @@ -261,7 +244,6 @@ def __init__( annotation_id: Optional[str] = None, metadata: Optional[Dict] = None, class_pdf: Optional[Dict] = None, - id: Optional[str] = None, # pylint: disable=redefined-builtin ): super().__init__( label=label, @@ -269,7 +251,6 @@ def __init__( reference_id=reference_id, annotation_id=annotation_id, metadata=metadata, - id=id, ) self.confidence = confidence self.class_pdf = class_pdf @@ -280,7 +261,6 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf - return payload @classmethod @@ -296,13 +276,11 @@ def from_json(cls, payload: dict): annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), class_pdf=payload.get(CLASS_PDF_KEY, None), - id=payload.get(ID_KEY, None), ) class PolygonPrediction(PolygonAnnotation): """Prediction of a polygon. - Parameters: label (str): The label for this annotation (e.g. car, pedestrian, bicycle). vertices List[:class:`Point`]: The list of points making up the polygon. @@ -324,7 +302,6 @@ class PolygonPrediction(PolygonAnnotation): embedding_vector: Custom embedding vector for this object annotation. If any custom object embeddings have been uploaded previously to this dataset, this vector must match the dimensions of the previously ingested vectors. - id: Unique Nucleus generated ID. This field is populated when loading Predictions from the server. """ def __init__( @@ -337,7 +314,6 @@ def __init__( metadata: Optional[Dict] = None, class_pdf: Optional[Dict] = None, embedding_vector: Optional[list] = None, - id: Optional[str] = None, # pylint: disable=redefined-builtin ): super().__init__( label=label, @@ -346,7 +322,6 @@ def __init__( annotation_id=annotation_id, metadata=metadata, embedding_vector=embedding_vector, - id=id, ) self.confidence = confidence self.class_pdf = class_pdf @@ -357,7 +332,6 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf - return payload @classmethod @@ -374,13 +348,11 @@ def from_json(cls, payload: dict): metadata=payload.get(METADATA_KEY, {}), class_pdf=payload.get(CLASS_PDF_KEY, None), embedding_vector=payload.get(EMBEDDING_VECTOR_KEY, None), - id=payload.get(ID_KEY, None), ) class KeypointsPrediction(KeypointsAnnotation): """Prediction of keypoints. - Parameters: label (str): The label for this annotation (e.g. car, pedestrian, bicycle). keypoints (List[:class:`Keypoint`]): The list of keypoints objects. @@ -402,7 +374,6 @@ class KeypointsPrediction(KeypointsAnnotation): annotation. Each value should be between 0 and 1 (inclusive), and sum up to 1 as a complete distribution. This can be useful for computing entropy to surface places where the model is most uncertain. - id: Unique Nucleus generated ID. This field is populated when loading Predictions from the server. """ def __init__( @@ -416,7 +387,6 @@ def __init__( annotation_id: Optional[str] = None, metadata: Optional[Dict] = None, class_pdf: Optional[Dict] = None, - id: Optional[str] = None, # pylint: disable=redefined-builtin ): super().__init__( label=label, @@ -426,7 +396,6 @@ def __init__( reference_id=reference_id, annotation_id=annotation_id, metadata=metadata, - id=id, ) self.confidence = confidence self.class_pdf = class_pdf @@ -437,7 +406,6 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf - return payload @classmethod @@ -455,13 +423,11 @@ def from_json(cls, payload: dict): annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), class_pdf=payload.get(CLASS_PDF_KEY, None), - id=payload.get(ID_KEY, None), ) class CuboidPrediction(CuboidAnnotation): """A prediction of 3D cuboid. - Parameters: label (str): The label for this annotation (e.g. car, pedestrian, bicycle) position (:class:`Point3D`): The point at the center of the cuboid @@ -481,7 +447,6 @@ class CuboidPrediction(CuboidAnnotation): annotation. Each value should be between 0 and 1 (inclusive), and sum up to 1 as a complete distribution. This can be useful for computing entropy to surface places where the model is most uncertain. - id: Unique Nucleus generated ID. This field is populated when loading Predictions from the server. """ def __init__( @@ -495,7 +460,6 @@ def __init__( annotation_id: Optional[str] = None, metadata: Optional[Dict] = None, class_pdf: Optional[Dict] = None, - id: Optional[str] = None, # pylint: disable=redefined-builtin ): super().__init__( label=label, @@ -505,7 +469,6 @@ def __init__( reference_id=reference_id, annotation_id=annotation_id, metadata=metadata, - id=id, ) self.confidence = confidence self.class_pdf = class_pdf @@ -516,7 +479,6 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf - return payload @classmethod @@ -532,13 +494,11 @@ def from_json(cls, payload: dict): annotation_id=payload.get(ANNOTATION_ID_KEY, None), metadata=payload.get(METADATA_KEY, {}), class_pdf=payload.get(CLASS_PDF_KEY, None), - id=payload.get(ID_KEY, None), ) class CategoryPrediction(CategoryAnnotation): """A prediction of a category. - Parameters: label: The label for this annotation (e.g. car, pedestrian, bicycle). reference_id: The reference ID of the image you wish to apply this annotation to. @@ -553,7 +513,6 @@ class CategoryPrediction(CategoryAnnotation): Strings, floats and ints are supported best by querying and insights features within Nucleus. For more details see our `metadata guide `_. - id: Unique Nucleus generated ID. This field is populated when loading Predictions from the server. """ def __init__( @@ -564,14 +523,12 @@ def __init__( confidence: Optional[float] = None, metadata: Optional[Dict] = None, class_pdf: Optional[Dict] = None, - id: Optional[str] = None, # pylint: disable=redefined-builtin ): super().__init__( label=label, taxonomy_name=taxonomy_name, reference_id=reference_id, metadata=metadata, - id=id, ) self.confidence = confidence self.class_pdf = class_pdf @@ -582,7 +539,6 @@ def to_payload(self) -> dict: payload[CONFIDENCE_KEY] = self.confidence if self.class_pdf is not None: payload[CLASS_PDF_KEY] = self.class_pdf - return payload @classmethod @@ -594,7 +550,6 @@ def from_json(cls, payload: dict): confidence=payload.get(CONFIDENCE_KEY, None), metadata=payload.get(METADATA_KEY, {}), class_pdf=payload.get(CLASS_PDF_KEY, None), - id=payload.get(ID_KEY, None), ) diff --git a/tests/metrics/test_polygon_metrics.py b/tests/metrics/test_polygon_metrics.py index 6d7fc8fd..e6a19069 100644 --- a/tests/metrics/test_polygon_metrics.py +++ b/tests/metrics/test_polygon_metrics.py @@ -9,7 +9,7 @@ PolygonPrecision, PolygonRecall, ) -from nucleus.metrics.base import Metric, ScalarResult +from nucleus.metrics.base import ScalarResult from tests.metrics.helpers import ( TEST_ANNOTATION_LIST, TEST_BOX_ANNOTATION_LIST,