PEP8 and utils Fixes

Author: ArneGudermann

src/viur/core/db/__init__.py

@@ -3,12 +3,12 @@
 from .errors import *
 from .query import Query
 # new exports for 3.8
-from .transport import AllocateIDs, Count, Delete, Get, Put, RunInTransaction, allocate_ids, count, delete, get, \
-    is_in_transaction, put, run_in_transaction
+from .transport import (AllocateIDs, Count, Delete, Get, Put, RunInTransaction, allocate_ids, count, delete, get, put,
+                        run_in_transaction)
 from .types import (DATASTORE_BASE_TYPES, Entity, KEY_SPECIAL_PROPERTY, Key, QueryDefinition, SortOrder,
                     currentDbAccessLog)
-from .utils import (GetOrInsert, IsInTransaction, acquireTransactionSuccessMarker, encodeKey, endDataAccessLog,
-                    fixUnindexableProperties, keyHelper, normalizeKey, startDataAccessLog)
+from .utils import (GetOrInsert, IsInTransaction, is_in_transaction, acquireTransactionSuccessMarker, encodeKey,
+                    endDataAccessLog, fix_unindexable_properties, keyHelper, normalizeKey, startDataAccessLog)
 
 __all__ = [
     "KEY_SPECIAL_PROPERTY",
@@ -18,7 +18,7 @@
     "QueryDefinition",
     "Key",
     "Query",
-    "fixUnindexableProperties",
+    "fix_unindexable_properties",
     "normalizeKey",
     "keyHelper",
     "Get",

src/viur/core/db/query.py

@@ -7,7 +7,7 @@
 import typing as t
 
 from .config import conf
-from .transport import Count, Get, runSingleFilter
+from .transport import Count, Get, run_single_filter
 from .types import (
     DATASTORE_BASE_TYPES,
     Entity,
@@ -17,7 +17,7 @@
     TFilters,
     TOrders,
 )
-from .utils import IsInTransaction
+from . import utils
 
 if t.TYPE_CHECKING:
     from viur.core.skeleton import SkeletonInstance, SkelList
@@ -444,39 +444,38 @@ def getKind(self) -> str:
         """
         return self.kind
 
-    def _runSingleFilterQuery(self, query: QueryDefinition, limit: int) -> t.List[Entity]:
+    def _run_single_filter_query(self, query: QueryDefinition, limit: int) -> t.List[Entity]:
         """
         Internal helper function that runs a single query definition on the datastore and returns a list of
         entities found.
         :param query: The querydefinition (filters, orders, distinct etc.) to run against the datastore
         :param limit: How many results should at most be returned
         :return: The first *limit* entities that matches this query
         """
-        return runSingleFilter(query, limit)
+        return run_single_filter(query, limit)
 
-    def _mergeMultiQueryResults(self, inputRes: t.List[t.List[Entity]]) -> t.List[Entity]:
+    def _merge_multi_query_results(self, input_result: t.List[t.List[Entity]]) -> t.List[Entity]:
         """
         Merge the lists of entries into a single list; removing duplicates and restoring sort-order
-        :param inputRes: Nested Lists of Entries returned by each individual query run
+        :param input_result: Nested Lists of Entries returned by each individual query run
         :return: Sorted & deduplicated list of entries
         """
-        seenKeys = set()
+        seen_keys = set()
         res = []
-        for subList in inputRes:
+        for subList in input_result:
             for entry in subList:
                 key = entry.key
-                if key in seenKeys:
+                if key in seen_keys:
                     continue
-                seenKeys.add(key)
+                seen_keys.add(key)
                 res.append(entry)
         # FIXME: What about filters that mix different inequality filters?
         # Currently, we'll now simply ignore any implicit sortorder.
-        return self._resortResult(res, {}, self.queries[0].orders)
+        return self._resort_result(res, {}, self.queries[0].orders)
 
-    def _resortResult(
+    def _resort_result(
         self,
-        entities:
-        t.List[Entity],
+        entities: t.List[Entity],
         filters: t.Dict[str, DATASTORE_BASE_TYPES],
         orders: t.List[t.Tuple[str, SortOrder]],
     ) -> t.List[Entity]:
@@ -578,7 +577,7 @@ def run(self, limit: int = -1) -> t.List[Entity]:
             return []
 
         if self._fulltextQueryString:
-            if IsInTransaction():
+            if utils.is_in_transaction():
                 raise ValueError("Can't run fulltextSearch inside transactions!")  # InvalidStateError FIXME!
             qryStr = self._fulltextQueryString
             self._fulltextQueryString = None  # Reset, so the adapter can still work with this query
@@ -596,17 +595,17 @@ def run(self, limit: int = -1) -> t.List[Entity]:
             res = []
             # We run all queries first (preventing multiple round-trips to the server)
             for singleQuery in self.queries:
-                res.append(self._runSingleFilterQuery(singleQuery, limit if limit != -1 else singleQuery.limit))
+                res.append(self._run_single_filter_query(singleQuery, limit if limit != -1 else singleQuery.limit))
             # Wait for the actual results to arrive and convert the protobuffs to Entries
             res = [self._fixKind(x) for x in res]
             if self._customMultiQueryMerge:
                 # We have a custom merge function, use that
                 res = self._customMultiQueryMerge(self, res, limit if limit != -1 else self.queries[0].limit)
             else:
                 # We must merge (and sort) the results ourself
-                res = self._mergeMultiQueryResults(res)
+                res = self._merge_multi_query_results(res)
         else:  # We have just one single query
-            res = self._fixKind(self._runSingleFilterQuery(self.queries, limit if limit != -1 else self.queries.limit))
+            res = self._fixKind(self._run_single_filter_query(self.queries, limit if limit != -1 else self.queries.limit))
         if res:
             self._lastEntry = res[-1]
         return res
@@ -688,7 +687,7 @@ def iter(self) -> t.Iterator[Entity]:
         elif isinstance(self.queries, list):
             raise ValueError("No iter on Multiqueries")
         while True:
-            qryRes = self._runSingleFilterQuery(self.queries, 20)
+            qryRes = self._run_single_filter_query(self.queries, 20)
             yield from qryRes
             if not self.queries.currentCursor:  # We reached the end of that query
                 break

src/viur/core/db/transport.py

@@ -104,12 +104,9 @@ def Delete(keys: t.Union[Entity, t.List[Entity], Key, t.List[Key]]):
     return delete(keys)
 
 
-def is_in_transaction() -> bool:
-    return __client__.current_transaction is not None
-
-
-@deprecated(version="3.8.0", reason="Use 'db.is_in_transaction' instead")
+@deprecated(version="3.8.0", reason="Use 'db.utils.is_in_transaction' instead")
 def IsInTransaction() -> bool:
+    from .utils import is_in_transaction  # noqa: E402 # import works only here because circular imports
     return is_in_transaction()
 
 
@@ -167,14 +164,15 @@ def Count(kind: str = None, up_to=2 ** 31 - 1, queryDefinition: QueryDefinition
     return count(kind, up_to, queryDefinition)
 
 
-def runSingleFilter(query: QueryDefinition, limit: int) -> t.List[Entity]:
+def run_single_filter(query: QueryDefinition, limit: int) -> t.List[Entity]:
     """
-    Internal helper function that runs a single query definition on the datastore and returns a list of
-    entities found.
-    :param query: The querydefinition (filters, orders, distinct etc.) to run against the datastore
-    :param limit: How many results should at most be returned
-    :return: The first *limit* entities that matches this query
+        Internal helper function that runs a single query definition on the datastore and returns a list of
+        entities found.
+        :param query: The querydefinition (filters, orders, distinct etc.) to run against the datastore
+        :param limit: How many results should at most be returned
+        :return: The first *limit* entities that matches this query
     """
+
     qry = __client__.query(kind=query.kind)
     startCursor = None
     endCursor = None
@@ -217,4 +215,9 @@ def runSingleFilter(query: QueryDefinition, limit: int) -> t.List[Entity]:
     return res
 
 
+@deprecated(version="3.8.0", reason="Use 'run_single_filter' instead")
+def runSingleFilter(query: QueryDefinition, limit: int) -> t.List[Entity]:
+    run_single_filter(query, limit)
+
+
 __all__ = [AllocateIDs, Delete, Get, Put, RunInTransaction, Count]

src/viur/core/db/utils.py

@@ -1,41 +1,44 @@
 import datetime
+from deprecated.sphinx import deprecated
 import typing as t
 
 from .transport import Get, Put, RunInTransaction
 from .types import Entity, Key, currentDbAccessLog, currentTransaction
 
 
-def fixUnindexableProperties(entry: Entity) -> Entity:
+def fix_unindexable_properties(entry: Entity) -> Entity:
     """
         Recursively walk the given Entity and add all properties to the list of unindexed properties if they contain
-        a string longer than 500 bytes (which is maximum size of a string that can be indexed). The datastore would
+        a string longer than 1500 bytes (which is maximum size of a string that can be indexed). The datastore would
         return an error otherwise.
+        https://cloud.google.com/datastore/docs/concepts/limits?hl=en#limits
     :param entry: The entity to fix (inplace)
     :return: The fixed entity
     """
 
-    def hasUnindexableProperty(prop):
+    def has_unindexable_property(prop):
         if isinstance(prop, dict):
-            return any([hasUnindexableProperty(x) for x in prop.values()])
+            return any([has_unindexable_property(x) for x in prop.values()])
         elif isinstance(prop, list):
-            return any([hasUnindexableProperty(x) for x in prop])
+            return any([has_unindexable_property(x) for x in prop])
         elif isinstance(prop, (str, bytes)):
-            return len(prop) >= 500
+            return len(prop) >= 1500
         else:
             return False
 
-    resList = set()
-    for k, v in entry.items():
-        if hasUnindexableProperty(v):
-            if isinstance(v, dict):
-                innerEntry = Entity()
-                innerEntry.update(v)
-                entry[k] = fixUnindexableProperties(innerEntry)
-                if isinstance(v, Entity):
-                    innerEntry.key = v.key
-            else:
-                resList.add(k)
-    entry.exclude_from_indexes = resList
+    unindexable_properties = set()
+    for key, value in entry.items():
+        if not has_unindexable_property(value):
+            continue
+        if isinstance(value, dict):
+            inner_entity = Entity()
+            inner_entity.update(value)
+            entry[key] = fix_unindexable_properties(inner_entity)
+            if isinstance(value, Entity):
+                inner_entity.key = value.key
+        else:
+            unindexable_properties.add(key)
+    entry.exclude_from_indexes = unindexable_properties
     return entry
 
 
@@ -94,10 +97,15 @@ def keyHelper(
     raise NotImplementedError(f"Unsupported key type {type(inKey)}")
 
 
-def IsInTransaction() -> bool:
+def is_in_transaction() -> bool:
     return currentTransaction.get() is not None
 
 
+@deprecated(version="3.8.0", reason="Use 'db.utils.is_in_transaction' instead")
+def IsInTransaction() -> bool:
+    return is_in_transaction()
+
+
 def GetOrInsert(key: Key, **kwargs) -> Entity:
     """
     Either creates a new entity with the given key, or returns the existing one.

src/viur/core/session.py

@@ -121,7 +121,7 @@ def save(self):
 
         dbSession = db.Entity(db.Key(self.kindName, self.cookie_key))
 
-        dbSession["data"] = db.fixUnindexableProperties(self)
+        dbSession["data"] = db.fix_unindexable_properties(self)
         dbSession["static_security_key"] = self.static_security_key
         dbSession["lastseen"] = time.time()
         dbSession["user"] = str(user_key)  # allow filtering for users