do check for corrupt images only once

zauberzeug · Feb 2, 2024 · d95c36b · d95c36b
1 parent 6aa1edb
commit d95c36b
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 1 deletion.
diff --git a/.gitignore b/.gitignore
@@ -12,3 +12,4 @@ __pycache__/
 
 .env
 learning_loop_node/tmp
+learning_loop_node/.python-version
diff --git a/learning_loop_node/data_exchanger.py b/learning_loop_node/data_exchanger.py
@@ -60,7 +60,6 @@ async def download_images(self, image_ids: List[str], image_folder: str) -> None
             logging.warning('context was not set yet')
             return
 
-        await self.delete_corrupt_images(image_folder)
         new_image_ids = await asyncio.get_event_loop().run_in_executor(None, DataExchanger.filter_existing_images, image_ids, image_folder)
         paths, ids = create_resource_paths(self.context.organization, self.context.project, new_image_ids)
         await self._download_images(paths, ids, image_folder)

diff --git a/learning_loop_node/trainer/trainer_logic.py b/learning_loop_node/trainer/trainer_logic.py
@@ -424,6 +424,8 @@ async def _do_detections(self) -> None:
             image_ids += new_ids
             logging.info(f'downloading {len(new_ids)} images')
             await self.node.data_exchanger.download_images(new_ids, image_folder)
+        self.detection_progress = 0.42
+        await self.node.data_exchanger.delete_corrupt_images(image_folder)
 
         images = await asyncio.get_event_loop().run_in_executor(None, TrainerLogic.images_for_ids, image_ids, image_folder)
         num_images = len(images)
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,3 +12,4 @@ __pycache__/

		.env
		learning_loop_node/tmp
		learning_loop_node/.python-version