Refactor library

- Merge django-eth library here
- Organize imports
- Use postgresql for testing

.travis.yml

@@ -9,6 +9,8 @@ env:
   global:
   - SOURCE_FOLDER=gnosis
   - PIP_USE_MIRRORS=true
+addons:
+  postgresql: "9.6"
 install:
   - pip install -r requirements.txt
   - pip install coveralls

README.rst

@@ -13,6 +13,18 @@ Quick start
 
 Just run ``pip install gnosis-py`` or add it to your **requirements.txt**
 
+Ethereum django utils
+---------------------
+Now django-eth is part of this package, you can find it under `gnosis.eth.django`
+Django ethereum is a set of helpers for working with ethereum using Django and Django Rest framework.
+
+It includes:
+
+- Basic serializers (signature, transaction)
+- Serializer fields (Ethereum address field, hexadecimal field)
+- Model fields (Ethereum address, Ethereum big integer field)
+- Utils for testing
+
 Contributors
 ------------
 - Denís Graña ([email protected])

config/settings/test.py

@@ -1,15 +1,20 @@
 from .base import *
 
 INSTALLED_APPS += (
+    'gnosis.eth.django.tests',
 )
 
 SECRET_KEY = 'testtest'
 DEBUG = True
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': 'mydatabase',
+        'ENGINE': 'django.db.backends.postgresql',
+        'NAME': 'postgres',
+        'USER': 'postgres',
+        # 'PASSWORD': '',
+        'HOST': '127.0.0.1',
+        'PORT': '5432',
     }
 }
 

docker-compose.yml

@@ -0,0 +1,7 @@
+version: '3.5'
+
+services:
+  db:
+    image: postgres:10-alpine
+    ports:
+      - "5432:5432"

gnosis/eth/django/models.py

@@ -0,0 +1,137 @@
+from django import forms
+from django.core import exceptions
+from django.db import DefaultConnectionProxy, models
+from django.utils.translation import gettext_lazy as _
+
+import ethereum.utils
+from hexbytes import HexBytes
+
+from .validators import validate_checksumed_address
+
+connection = DefaultConnectionProxy()
+
+
+class EthereumAddressField(models.CharField):
+    default_validators = [validate_checksumed_address]
+    description = "Ethereum address"
+
+    def __init__(self, *args, **kwargs):
+        kwargs['max_length'] = 42
+        super().__init__(*args, **kwargs)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        del kwargs['max_length']
+        return name, path, args, kwargs
+
+    def from_db_value(self, value, expression, connection):
+        return self.to_python(value)
+
+    def to_python(self, value):
+        value = super().to_python(value)
+        if value:
+            return ethereum.utils.checksum_encode(value)
+        else:
+            return value
+
+    def get_prep_value(self, value):
+        value = super().get_prep_value(value)
+        if value:
+            return ethereum.utils.checksum_encode(value)
+        else:
+            return value
+
+
+class Uint256Field(models.Field):
+    description = _("Ethereum uint256 number")
+    """
+    Field to store ethereum uint256 values. Uses Decimal db type without decimals to store
+    in the database, but retrieve as `int` instead of `Decimal` (https://docs.python.org/3/library/decimal.html)
+    """
+    def __init__(self, *args, **kwargs):
+        self.max_digits, self.decimal_places = 79, 0  # 2 ** 256 is 78 digits
+        super().__init__(*args, **kwargs)
+
+    def get_internal_type(self):
+        return "DecimalField"
+
+    def from_db_value(self, value, expression, connection):
+        return self.to_python(value)
+
+    def get_db_prep_save(self, value, connection):
+        return connection.ops.adapt_decimalfield_value(self.to_python(value),
+                                                       max_digits=self.max_digits,
+                                                       decimal_places=self.decimal_places)
+
+    def get_prep_value(self, value):
+        value = super().get_prep_value(value)
+        return self.to_python(value)
+
+    def to_python(self, value):
+        if value is None:
+            return value
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            raise exceptions.ValidationError(
+                self.error_messages['invalid'],
+                code='invalid',
+                params={'value': value},
+            )
+
+    def formfield(self, **kwargs):
+        defaults = {'form_class': forms.IntegerField}
+        defaults.update(kwargs)
+        return super().formfield(**defaults)
+
+
+class HexField(models.CharField):
+    """
+    Field to store hex values (without 0x). Returns hex with 0x prefix.
+
+    On Database side a CharField is used.
+    """
+    description = "Stores a hex value into an CharField"
+
+    def from_db_value(self, value, expression, connection):
+        return self.to_python(value)
+
+    def to_python(self, value):
+        return value if value is None else HexBytes(value).hex()
+
+    def get_prep_value(self, value):
+        if value is None:
+            return value
+        elif isinstance(value, HexBytes):
+            return value.hex()[2:]  # HexBytes.hex() retrieves hexadecimal with '0x', remove it
+        elif isinstance(value, bytes):
+            return value.hex()  # bytes.hex() retrieves hexadecimal without '0x'
+        else:  # str
+            return HexBytes(value).hex()[2:]
+
+    def formfield(self, **kwargs):
+        # We need max_lenght + 2 on forms because of `0x`
+        defaults = {'max_length': self.max_length + 2}
+        # TODO: Handle multiple backends with different feature flags.
+        if self.null and not connection.features.interprets_empty_strings_as_nulls:
+            defaults['empty_value'] = None
+        defaults.update(kwargs)
+        return super().formfield(**defaults)
+
+    def clean(self, value, model_instance):
+        value = self.to_python(value)
+        self.validate(value, model_instance)
+        # Validation didn't work because of `0x`
+        self.run_validators(value[2:])
+        return value
+
+
+class Sha3HashField(HexField):
+    def __init__(self, *args, **kwargs):
+        kwargs['max_length'] = 64
+        super().__init__(*args, **kwargs)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        del kwargs['max_length']
+        return name, path, args, kwargs

gnosis/eth/django/serializers.py

@@ -0,0 +1,156 @@
+import logging
+
+from django.utils.translation import ugettext_lazy as _
+
+from ethereum.utils import checksum_encode
+from hexbytes import HexBytes
+from rest_framework import serializers
+from rest_framework.exceptions import ValidationError
+
+from ..constants import *
+
+logger = logging.getLogger(__name__)
+
+
+# ================================================ #
+#                Custom Fields
+# ================================================ #
+class EthereumAddressField(serializers.Field):
+    """
+    Ethereum address checksumed
+    https://github.com/ethereum/EIPs/blob/master/EIPS/eip-55.md
+    """
+
+    def __init__(self, allow_zero_address=False, allow_sentinel_address=False, **kwargs):
+        self.allow_zero_address = allow_zero_address
+        self.allow_sentinel_address = allow_sentinel_address
+        super().__init__(**kwargs)
+
+    def to_representation(self, obj):
+        return obj
+
+    def to_internal_value(self, data):
+        # Check if address is valid
+        try:
+            if checksum_encode(data) != data:
+                raise ValueError
+            elif int(data, 16) == 0 and not self.allow_zero_address:
+                raise ValidationError("0x0 address is not allowed")
+            elif int(data, 16) == 1 and not self.allow_sentinel_address:
+                raise ValidationError("0x1 address is not allowed")
+        except ValueError:
+            raise ValidationError("Address %s is not checksumed" % data)
+        except Exception:
+            raise ValidationError("Address %s is not valid" % data)
+
+        return data
+
+
+class HexadecimalField(serializers.Field):
+    """
+    Serializes hexadecimal values starting by `0x`. Empty values should be None or just `0x`.
+    """
+
+    default_error_messages = {
+        'invalid': _('{value} is not an hexadecimal value.'),
+        'blank': _('This field may not be blank.'),
+        'max_length': _('Ensure this field has no more than {max_length} hexadecimal chars (not counting 0x).'),
+        'min_length': _('Ensure this field has at least {min_length} hexadecimal chars (not counting 0x).'),
+    }
+
+    def __init__(self, **kwargs):
+        self.allow_blank = kwargs.pop('allow_blank', False)
+        self.max_length = kwargs.pop('max_length', None)
+        self.min_length = kwargs.pop('min_length', None)
+        super().__init__(**kwargs)
+
+    def to_representation(self, obj):
+        if not obj:
+            return '0x'
+
+        # We can get another types like `memoryview` from django models. `to_internal_value` is not used
+        # when you provide an object instead of a json using `data`. Make sure everything is HexBytes.
+        if hasattr(obj, 'hex'):
+            obj = HexBytes(obj.hex())
+        elif not isinstance(obj, HexBytes):
+            obj = HexBytes(obj)
+        return obj.hex()
+
+    def to_internal_value(self, data):
+        if isinstance(data, (bytes, memoryview)):
+            data = data.hex()
+
+        data = data.strip()  # Trim spaces
+        if data.startswith('0x'):  # Remove 0x prefix
+            data = data[2:]
+
+        if not data:
+            if self.allow_blank:
+                return None
+            else:
+                self.fail('blank')
+
+        data_len = len(data)
+        if self.min_length and data_len < self.min_length:
+            self.fail('min_length', min_length=data_len)
+        elif self.max_length and data_len > self.max_length:
+            self.fail('max_length', max_length=data_len)
+
+        try:
+            return HexBytes(data)
+        except ValueError:
+            self.fail('invalid', value=data)
+
+
+class Sha3HashField(HexadecimalField):
+    def __init__(self, **kwargs):
+        kwargs['max_length'] = 64
+        kwargs['min_length'] = 64
+        super().__init__(**kwargs)
+
+
+# ================================================ #
+#                Base Serializers
+# ================================================ #
+class SignatureSerializer(serializers.Serializer):
+    v = serializers.IntegerField(min_value=SIGNATURE_V_MIN_VALUE,
+                                 max_value=SIGNATURE_V_MAX_VALUE)
+    r = serializers.IntegerField(min_value=SIGNATURE_R_MIN_VALUE,
+                                 max_value=SIGNATURE_R_MAX_VALUE)
+    s = serializers.IntegerField(min_value=SIGNATURE_S_MIN_VALUE,
+                                 max_value=SIGNATURE_S_MAX_VALUE)
+
+
+class TransactionSerializer(serializers.Serializer):
+    from_ = EthereumAddressField()
+    value = serializers.IntegerField(min_value=0)
+    data = HexadecimalField()
+    gas = serializers.IntegerField(min_value=0)
+    gas_price = serializers.IntegerField(min_value=0)
+    nonce = serializers.IntegerField(min_value=0)
+
+    def get_fields(self):
+        result = super().get_fields()
+        # Rename `from_` to `from`
+        from_ = result.pop('from_')
+        result['from'] = from_
+        return result
+
+
+class TransactionResponseSerializer(serializers.Serializer):
+    """
+    Use chars to avoid problems with big ints (i.e. JavaScript)
+    """
+    from_ = EthereumAddressField()
+    value = serializers.IntegerField(min_value=0)
+    data = serializers.CharField()
+    gas = serializers.CharField()
+    gas_price = serializers.CharField()
+    nonce = serializers.IntegerField(min_value=0)
+
+    def get_fields(self):
+        result = super().get_fields()
+        # Rename `from_` to `from`
+        from_ = result.pop('from_')
+        result['from'] = from_
+        return result

gnosis/eth/django/tests/models.py

@@ -0,0 +1,15 @@
+from django.db import models
+
+from ..models import EthereumAddressField, Sha3HashField, Uint256Field
+
+
+class EthereumAddress(models.Model):
+    value = EthereumAddressField(null=True)
+
+
+class Uint256(models.Model):
+    value = Uint256Field(null=True)
+
+
+class Sha3Hash(models.Model):
+    value = Sha3HashField(null=True)