.gitignore

@@ -31,5 +31,9 @@ sdk/diffgram/utils/__pycache__/
 
 venv/*
 
+diffgram.egg-info
+
+playground.py
+
 sdk/build/*
 sdk/dist/

sdk/diffgram/brain/brain.py

@@ -2,21 +2,7 @@
 
 import tempfile
 
-try:
-    import cv2
-except:
-    print("Warning: Could not import cv2. Some SDK functions may not be available.")
-
-try:
-    import tensorflow as tf
-except:
-    print("Warning: Could not import tensorflow. Some SDK functions may not be available")
-
-import numpy as np
 import requests
-import scipy.misc
-
-import diffgram.utils.visualization_utils as vis_util
 
 
 class Brain():
@@ -37,6 +23,20 @@ def __init__(
         if local is true will perform additional setup work local_setup()
 
         """
+        try:
+            import cv2
+        except:
+            print("Info: Could not import cv2. Some SDK functions may not be available.")
+
+        try:
+            import tensorflow as tf
+        except:
+            print("Info: Could not import tensorflow. Some SDK functions may not be available")
+
+        import numpy as np
+        import scipy.misc
+        import diffgram.utils.visualization_utils as vis_util
+
 
         self.client = client
 

sdk/diffgram/core/core.py

@@ -16,6 +16,8 @@
 from diffgram.brain.train import Train
 from diffgram.export.export import Export
 from diffgram.task.task import Task
+from diffgram.schema.schema import Schema
+from diffgram.schema.attribute import Attribute
 from requests.auth import HTTPBasicAuth
 
 
@@ -86,6 +88,8 @@ def __init__(
 
         self.label_schema_list = self.get_label_schema_list()
 
+        self.schema = Schema(self)
+        self.attribute = Attribute(self)
 
 
     def get_member_list(self):

sdk/diffgram/core/diffgram_dataset_iterator.py

@@ -1,8 +1,5 @@
 import logging
 
-from PIL import Image, ImageDraw
-from imageio import imread
-import numpy as np
 import traceback
 import sys
 from threading import Thread
@@ -137,6 +134,7 @@ def set_custom_url_signer(self, signer_fn: Callable):
         self.custom_signer_fn = signer_fn
 
     def get_image_data(self, diffgram_file):
+        from imageio import imread
         MAX_RETRIES = 10
         image = None
         if hasattr(diffgram_file, 'image'):
@@ -187,13 +185,24 @@ def gen_tag_instances(self, instance_list):
             result.append(elm)
         return result
 
-    def get_file_instances(self, diffgram_file):
-        sample = {'diffgram_file': diffgram_file}
+    def get_file_instances(self, diffgram_file) -> dict:
+        if not diffgram_file:
+            return
+        sample = {'diffgram_file': diffgram_file, 'type': diffgram_file.type}
         if diffgram_file.type not in ['image', 'frame', 'compound']:
             logging.warning('File type "{}" is not supported yet'.format(diffgram_file.type))
             return sample
         if diffgram_file.type in ['image', 'frame']:
             sample['image'] = self.get_image_data(diffgram_file)
+        elif diffgram_file.type  is not None and diffgram_file.type.startswith('compound'):
+            from diffgram.file.compound_file import CompoundFile
+            compound_file: CompoundFile = diffgram_file
+            sample['children'] = []
+            child_files = compound_file.fetch_child_files(with_instances = True)
+            print('chsad', child_files)
+            for child in child_files:
+                result = self.get_file_instances(child)
+                sample['children'].append(result)
         instance_list = diffgram_file.instance_list
         instance_types_in_file = set([x['type'] for x in instance_list])
         # Process the instances of each file
@@ -243,6 +252,8 @@ def get_file_instances(self, diffgram_file):
         return sample
 
     def extract_masks_from_polygon(self, instance_list, diffgram_file, empty_value = 0):
+        from PIL import Image, ImageDraw
+        import numpy as np
         nx, ny = diffgram_file.image['width'], diffgram_file.image['height']
         mask_list = []
         if nx is None or ny is None:

sdk/diffgram/core/directory.py

@@ -119,7 +119,7 @@ def all_file_ids(self, query = None):
             page_num = page_num,
             file_view_mode = 'ids_only',
             query = query,
-            with_children_files = True)
+            with_children_files = False)
 
         if diffgram_ids is False:
             raise Exception('Error Fetching Files: Please check you are providing a valid query.')

sdk/diffgram/file/compound_file.py

@@ -6,6 +6,7 @@
 
 
 class CompoundChildFile:
+    id: int
     path: str
     url: str
     blob_path: str
@@ -24,8 +25,10 @@ class CompoundChildFile:
     video_split_duration: int
     local_id: str
     ordinal: int
+    root_file: 'CompoundFile'
 
     def __init__(self,
+                 root_file: 'CompoundFile',
                  child_file_type: str,
                  path: str = None,
                  url: str = None,
@@ -42,7 +45,9 @@ def __init__(self,
                  assume_new_instances_machine_made: bool = None,
                  convert_names_to_label_files: bool = None,
                  video_split_duration: int = None,
-                 ordinal: int = 0):
+                 ordinal: int = 0,
+                 id: int = None):
+        self.root_file = root_file
         self.child_file_type = child_file_type
         self.path = path
         self.url = url
@@ -61,6 +66,38 @@ def __init__(self,
         self.video_split_duration = video_split_duration
         self.local_id = str(uuid4())
         self.ordinal = ordinal
+        self.id = None
+
+    def __str__(self):
+        return f'<CompoundChildFile id={self.id} ordinal={self.ordinal} file_name={self.file_name} child_file_type={self.child_file_type}>'
+    def refresh_from_data_dict(self, data: dict):
+        if not data:
+            return
+        for key in data:
+            setattr(self, key, data[key])
+
+    def update(self):
+        """
+            Syncs child file data with backend API
+        :return:
+        """
+        payload = {
+            'ordinal': self.ordinal,
+        }
+        client = self.root_file.project
+        endpoint = f"/api/v1/project/{self.root_file.project.project_string_id}/file/{self.id}/update-metadata"
+
+        response = client.session.put(
+            client.host + endpoint,
+            json = payload)
+
+        client.handle_errors(response)
+
+        data = response.json()
+        new_file_data = data['file']
+
+        self.refresh_from_data_dict(data = new_file_data)
+        return self
 
     def set_ordinal(self, value: int):
         self.ordinal = value
@@ -69,29 +106,69 @@ def set_ordinal(self, value: int):
 class CompoundFile:
     project: Project
     parent_file_data: dict
-    child_files_to_upload: List[CompoundChildFile]
+    child_files: List[CompoundChildFile]
+    instance_list: List[dict]
 
-    def __init__(self, project: Project, name: str, directory_id: int):
+    def __init__(self, project: Project, name: str, directory_id: int, instance_list: List[dict] = [], file_type: str = 'compound'):
         self.project = project
         self.name = name
         self.directory_id = directory_id
-        self.child_files_to_upload = []
+        self.child_files = []
+        self.instance_list = instance_list
+        self.type = file_type
+
+    @staticmethod
+    def from_dict(project: Project, dir_id: int, dict_data: dict):
+        result = CompoundFile(project = project, name = dict_data.get('original_filename'), directory_id = dir_id)
+        result.__refresh_compound_file_from_data_dict(data = dict_data)
+        child_files = result.fetch_child_files()
+        result.child_files = child_files
+        return result
+
+    def fetch_child_files(self, with_instances: bool = False) -> List[CompoundChildFile]:
+        client = self.project
+        endpoint = f"/api/v1/project/{self.project.project_string_id}/file/{self.id}/child-files"
+
+        response = client.session.get(client.host + endpoint, params = {'with_instances': with_instances})
+
+        client.handle_errors(response)
+
+        data = response.json()
+        child_files_data = data['child_files']
+        print('child_files_data', child_files_data)
+        result = []
+        for elm in child_files_data:
+            child_file = CompoundChildFile(root_file = self, child_file_type = elm.get('type'))
+            child_file.refresh_from_data_dict(data = elm)
+            result.append(child_file)
+        return result
+
+    def update_all(self) -> bool:
+        """
+            Syncs parent and child metadata with backend API.
+        :return: True/False
+        """
+        for child in self.child_files:
+            child.update()
+        return True
 
     def __refresh_compound_file_from_data_dict(self, data: dict):
         if not data:
             return
         for key in data:
             setattr(self, key, data[key])
 
-    def remove_compound_file(self, child_file: CompoundChildFile) -> List[CompoundChildFile]:
-        self.child_files_to_upload.remove(child_file)
-        return self.child_files_to_upload
+    def remove_child_file(self, child_file: CompoundChildFile) -> List[CompoundChildFile]:
+        self.child_files.remove(child_file)
+        return self.child_files
 
     def __create_compound_parent_file(self):
         url = f'/api/v1/project/{self.project.project_string_id}/file/new-compound'
         data = {
             'name': self.name,
-            'directory_id': self.directory_id
+            'directory_id': self.directory_id,
+            'instance_list': self.instance_list,
+            'type': self.type
         }
         response = self.project.session.post(url = self.project.host + url,
                                              json = data)
@@ -110,7 +187,8 @@ def __create_child_file(self, child_file: CompoundChildFile):
                 frame_packet_map = child_file.frame_packet_map,
                 assume_new_instances_machine_made = child_file.assume_new_instances_machine_made,
                 convert_names_to_label_files = child_file.convert_names_to_label_files,
-                parent_file_id = self.parent_file_data.get('id')
+                parent_file_id = self.parent_file_data.get('id'),
+                ordinal = child_file.ordinal,
             )
         elif child_file.child_file_type == 'from_url':
             return self.project.file.from_url(
@@ -121,7 +199,8 @@ def __create_child_file(self, child_file: CompoundChildFile):
                 video_split_duration = child_file.video_split_duration,
                 instance_list = child_file.instance_list,
                 frame_packet_map = child_file.frame_packet_map,
-                parent_file_id = self.parent_file_data.get('id')
+                parent_file_id = self.parent_file_data.get('id'),
+                ordinal = child_file.ordinal,
             )
         elif child_file.child_file_type == 'from_blob_path':
             return self.project.file.from_blob_path(
@@ -132,7 +211,8 @@ def __create_child_file(self, child_file: CompoundChildFile):
                 instance_list = child_file.instance_list,
                 file_name = child_file.file_name,
                 frame_packet_map = child_file.frame_packet_map,
-                parent_file_id = self.parent_file_data.get('id')
+                parent_file_id = self.parent_file_data.get('id'),
+                ordinal = child_file.ordinal,
             )
 
     def add_child_from_local(self,
@@ -141,17 +221,21 @@ def add_child_from_local(self,
                              frame_packet_map: dict = None,
                              assume_new_instances_machine_made: bool = True,
                              convert_names_to_label_files: bool = True,
-                             ordinal: int = 0):
+                             ordinal: int = None):
+        if ordinal is None:
+            ordinal = len(self.child_files)
         new_child_file = CompoundChildFile(
+            root_file = self,
             child_file_type = "from_local",
             path = path,
             directory_id = self.directory_id,
             instance_list = instance_list,
             frame_packet_map = frame_packet_map,
             assume_new_instances_machine_made = assume_new_instances_machine_made,
-            convert_names_to_label_files = convert_names_to_label_files
+            convert_names_to_label_files = convert_names_to_label_files,
+            ordinal = ordinal
         )
-        self.child_files_to_upload.append(new_child_file)
+        self.child_files.append(new_child_file)
         return new_child_file
 
     def add_child_file_from_url(self,
@@ -162,8 +246,11 @@ def add_child_file_from_url(self,
                                 video_split_duration: int = None,
                                 instance_list: list = None,
                                 frame_packet_map: dict = None,
-                                ordinal: int = 0):
+                                ordinal: int = None):
+        if ordinal is None:
+            ordinal = len(self.child_files)
         new_child_file = CompoundChildFile(
+            root_file = self,
             child_file_type = "from_url",
             url = url,
             media_type = media_type,
@@ -173,8 +260,9 @@ def add_child_file_from_url(self,
             video_split_duration = video_split_duration,
             instance_list = instance_list,
             frame_packet_map = frame_packet_map,
+            ordinal = ordinal
         )
-        self.child_files_to_upload.append(new_child_file)
+        self.child_files.append(new_child_file)
         return new_child_file
 
     def add_child_from_blob_path(self,
@@ -185,9 +273,12 @@ def add_child_from_blob_path(self,
                                  instance_list: list = None,
                                  file_name: str = None,
                                  frame_packet_map: dict = None,
-                                 ordinal: int = 0
+                                 ordinal: int = None
                                  ):
+        if ordinal is None:
+            ordinal = len(self.child_files)
         new_child_file = CompoundChildFile(
+            root_file = self,
             child_file_type = "from_blob_path",
             blob_path = blob_path,
             bucket_name = bucket_name,
@@ -197,15 +288,16 @@ def add_child_from_blob_path(self,
             instance_list = instance_list,
             file_name = file_name,
             frame_packet_map = frame_packet_map,
+            ordinal = ordinal
         )
-        self.child_files_to_upload.append(new_child_file)
+        self.child_files.append(new_child_file)
         return new_child_file
 
     def upload(self):
-        if len(self.child_files_to_upload) == 0:
+        if len(self.child_files) == 0:
             raise AssertionError('Need to add at least one child file to the compound file before calling upload()')
         parent_file_data: dict = self.__create_compound_parent_file()
 
-        for child_file in self.child_files_to_upload:
+        for child_file in self.child_files:
             self.__create_child_file(child_file)
         return parent_file_data