Merge pull request #1 from correlation-one/shadinaif/add-gcs-backend

feat: add GCS backend

Author: shadinaif

.github/workflows/add-depr-ticket-to-depr-board.yml

@@ -2,7 +2,7 @@ name: Python CI
 
 on:
   push:
-    branches: [master]
+    branches: [main]
   pull_request:
     branches:
     - '**'
@@ -15,7 +15,7 @@ jobs:
       matrix:
         os: [ubuntu-20.04]
         python-version: ['3.8']
-        toxenv: [quality, js, django32]
+        toxenv: [quality, django32]
 
     steps:
     - uses: actions/checkout@v2
@@ -40,10 +40,3 @@ jobs:
       env:
         TOXENV: ${{ matrix.toxenv }}
       run: tox
-
-    - name: Run Coverage
-      if: matrix.python-version == '3.8' && matrix.toxenv=='django32'
-      uses: codecov/codecov-action@v1
-      with:
-        flags: unittests
-        fail_ci_if_error: true

.github/workflows/ci.yml

@@ -3,7 +3,7 @@
 
 from django.conf import settings
 
-from . import django_storage, filesystem, s3, swift
+from . import django_storage, gcs, filesystem, s3, swift
 
 
 def get_backend():
@@ -20,5 +20,7 @@ def get_backend():
         return swift.Backend()
     elif backend_setting == "django":
         return django_storage.Backend()
+    elif backend_setting == "gcs":
+        return gcs.GCSBackend()
     else:
         raise ValueError("Invalid ORA2_FILEUPLOAD_BACKEND setting value: %s" % backend_setting)

.github/workflows/commitlint.yml

@@ -0,0 +1,69 @@
+"""GCS Bucket File Upload Backend."""
+
+import functools
+import logging
+from google.cloud import storage
+
+from ..exceptions import FileUploadInternalError
+from .base import BaseBackend
+
+log = logging.getLogger("openassessment.fileupload.api")  # pylint: disable=invalid-name
+
+
+def catch_broad_exception(method):
+    """Decorator to catch broad exceptions, log them, and raise a FileUploadInternalError."""
+    @functools.wraps(method)
+    def wrapper(*args, **kwargs):
+        try:
+            return method(*args, **kwargs)
+        except Exception as ex:  # pylint: disable=broad-except
+            log.exception(
+                f"Internal exception occurred while executing ora2 file-upload backend gcs.{method.__name__}: {str(ex)}"
+            )
+            raise FileUploadInternalError(ex) from ex
+    return wrapper
+
+
+class GCSBackend(BaseBackend):
+    """ GCS Bucket File Upload Backend. """
+
+    @catch_broad_exception
+    def get_upload_url(self, key, content_type):
+        """Get a signed URL for uploading a file to GCS"""
+        bucket_name, key_name = self._retrieve_parameters(key)
+        blob = get_blob_object(bucket_name, key_name)
+        return blob.generate_signed_url(
+            version="v4",
+            expiration=self.UPLOAD_URL_TIMEOUT,
+            method="PUT",
+            content_type=content_type,
+        )
+
+    @catch_broad_exception
+    def get_download_url(self, key):
+        """Get a signed URL for downloading a file from GCS"""
+        bucket_name, key_name = self._retrieve_parameters(key)
+        blob = get_blob_object(bucket_name, key_name)
+        if not blob.exists():
+            return ""
+        return blob.generate_signed_url(
+            version="v4",
+            expiration=self.DOWNLOAD_URL_TIMEOUT,
+            method="GET",
+        )
+
+    @catch_broad_exception
+    def remove_file(self, key):
+        """Remove a file from GCS"""
+        bucket_name, key_name = self._retrieve_parameters(key)
+        blob = get_blob_object(bucket_name, key_name)
+        if blob.exists():
+            blob.delete()
+            return True
+        return False
+
+
+def get_blob_object(bucket_name, key_name):
+    """Get a blob object from GCS"""
+    client = storage.Client()
+    return client.bucket(bucket_name).blob(key_name)

.github/workflows/pypi-publish.yml

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
-#    make upgrade
+#    pip-compile --output-file=requirements/quality.txt requirements/quality.in
 #
 appdirs==1.4.4
     # via
@@ -54,6 +54,10 @@ botocore==1.27.66
     #   boto3
     #   moto
     #   s3transfer
+cachetools==5.3.2
+    # via
+    #   -r requirements/test.txt
+    #   google-auth
 certifi==2022.6.15
     # via
     #   -r requirements/test.txt
@@ -196,6 +200,36 @@ fs-s3fs==0.1.8
     #   -c requirements/constraints.txt
     #   -r requirements/test.txt
     #   django-pyfs
+google-api-core==2.17.0
+    # via
+    #   -r requirements/test.txt
+    #   google-cloud-core
+    #   google-cloud-storage
+google-auth==2.27.0
+    # via
+    #   -r requirements/test.txt
+    #   google-api-core
+    #   google-cloud-core
+    #   google-cloud-storage
+google-cloud-core==2.4.1
+    # via
+    #   -r requirements/test.txt
+    #   google-cloud-storage
+google-cloud-storage==2.14.0
+    # via -r requirements/test.txt
+google-crc32c==1.5.0
+    # via
+    #   -r requirements/test.txt
+    #   google-cloud-storage
+    #   google-resumable-media
+google-resumable-media==2.7.0
+    # via
+    #   -r requirements/test.txt
+    #   google-cloud-storage
+googleapis-common-protos==1.62.0
+    # via
+    #   -r requirements/test.txt
+    #   google-api-core
 html5lib==1.1
     # via -r requirements/test.txt
 idna==2.8
@@ -326,6 +360,11 @@ polib==1.1.1
     # via
     #   -r requirements/test.txt
     #   edx-i18n-tools
+protobuf==4.25.2
+    # via
+    #   -r requirements/test.txt
+    #   google-api-core
+    #   googleapis-common-protos
 psutil==5.9.1
     # via
     #   -r requirements/test.txt
@@ -338,8 +377,13 @@ py==1.11.0
 pyasn1==0.4.8
     # via
     #   -r requirements/test.txt
+    #   pyasn1-modules
     #   python-jose
     #   rsa
+pyasn1-modules==0.3.0
+    # via
+    #   -r requirements/test.txt
+    #   google-auth
 pycodestyle==2.9.1
     # via -r requirements/quality.in
 pycparser==2.21
@@ -431,6 +475,8 @@ requests==2.28.1
     # via
     #   -r requirements/test.txt
     #   docker
+    #   google-api-core
+    #   google-cloud-storage
     #   moto
     #   python-swiftclient
     #   responses
@@ -441,6 +487,7 @@ responses==0.21.0
 rsa==4.9
     # via
     #   -r requirements/test.txt
+    #   google-auth
     #   python-jose
 s3transfer==0.6.0
     # via