diff --git a/AUTHORS b/AUTHORS index b9a81c63f..7d3000ce5 100644 --- a/AUTHORS +++ b/AUTHORS @@ -255,3 +255,4 @@ that much better: * Filip Kucharczyk (https://github.com/Pacu2) * Eric Timmons (https://github.com/daewok) * Matthew Simpson (https://github.com/mcsimps2) + * Leonardo Domingues (https://github.com/leodmgs) diff --git a/docs/changelog.rst b/docs/changelog.rst index d924a2c18..b23bbd34a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,6 +7,15 @@ Development =========== - (Fill this out as you fix issues and develop your features). - Add Mongo 4.0 to Travis +<<<<<<< HEAD +- BREAKING CHANGE: Removed ``Queryset._ensure_indexes`` and ``Queryset.ensure_indexes`` that were deprecated in 2013. + ``Document.ensure_indexes`` still exists and is the right method to use +- Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264 +- Add support for the `elemMatch` projection operator in .fields() (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267 +- DictField validate failed without default connection (bug introduced in 0.19.0) #2239 +- Remove name parameter in Field constructor e.g `StringField(name="...")`, it was deprecated a while ago in favor of db_field +- Remove method queryset.slave_okay() that was deprecated a while ago and disappeared since pymongo3 +- Fix querying and updating dynamic fields on DynamicEmbeddedDocuments #2251 Changes in 0.19.1 ================= diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index cd1039cbb..379098e5b 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -36,7 +36,6 @@ class BaseField(object): def __init__( self, db_field=None, - name=None, required=False, default=None, unique=False, @@ -51,7 +50,6 @@ def __init__( """ :param db_field: The database field to store this field in (defaults to the name of the field) - :param name: Deprecated - use db_field :param required: If the field is required. Whether it has to have a value or not. Defaults to False. :param default: (optional) The default value for this field if no value @@ -75,11 +73,8 @@ def __init__( existing attributes. Common metadata includes `verbose_name` and `help_text`. """ - self.db_field = (db_field or name) if not primary_key else "_id" + self.db_field = db_field if not primary_key else "_id" - if name: - msg = 'Field\'s "name" attribute deprecated in favour of "db_field"' - warnings.warn(msg, DeprecationWarning) self.required = required or primary_key self.default = default self.unique = bool(unique or unique_with) diff --git a/mongoengine/document.py b/mongoengine/document.py index 23968f17d..3cc0046e9 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -56,7 +56,7 @@ class InvalidCollectionError(Exception): class EmbeddedDocument(six.with_metaclass(DocumentMetaclass, BaseDocument)): - """A :class:`~mongoengine.Document` that isn't stored in its own + r"""A :class:`~mongoengine.Document` that isn't stored in its own collection. :class:`~mongoengine.EmbeddedDocument`\ s should be used as fields on :class:`~mongoengine.Document`\ s through the :class:`~mongoengine.EmbeddedDocumentField` field type. @@ -332,7 +332,7 @@ def save( ): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be - created. + created. Returns the saved object instance. :param force_insert: only try to create a new document, don't allow updates of existing documents. diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 7ec8c0f3c..b92d3c122 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -777,6 +777,9 @@ def lookup_member(self, member_name): if field: return field + if any(doc_type._dynamic for doc_type in doc_and_subclasses): + return DynamicField(db_field=member_name) + def prepare_query_value(self, op, value): if value is not None and not isinstance(value, self.document_type): try: @@ -1088,14 +1091,12 @@ def validate(self, value): msg = "Invalid dictionary key - documents must have only string keys" self.error(msg) - curr_mongo_ver = get_mongodb_version() - - if curr_mongo_ver < MONGODB_36 and key_has_dot_or_dollar(value): - self.error( - 'Invalid dictionary key name - keys may not contain "."' - ' or startswith "$" characters' - ) - elif curr_mongo_ver >= MONGODB_36 and key_starts_with_dollar(value): + # Following condition applies to MongoDB >= 3.6 + # older Mongo has stricter constraints but + # it will be rejected upon insertion anyway + # Having a validation that depends on the MongoDB version + # is not straightforward as the field isn't aware of the connected Mongo + if key_starts_with_dollar(value): self.error( 'Invalid dictionary key name - keys may not startswith "$" characters' ) diff --git a/mongoengine/mongodb_support.py b/mongoengine/mongodb_support.py index 5d437fef8..522f064e6 100644 --- a/mongoengine/mongodb_support.py +++ b/mongoengine/mongodb_support.py @@ -11,7 +11,7 @@ def get_mongodb_version(): - """Return the version of the connected mongoDB (first 2 digits) + """Return the version of the default connected mongoDB (first 2 digits) :return: tuple(int, int) """ diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 50cb37acc..eed304136 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -60,7 +60,6 @@ def __init__(self, document, collection): self._ordering = None self._snapshot = False self._timeout = True - self._slave_okay = False self._read_preference = None self._iter = False self._scalar = [] @@ -694,8 +693,8 @@ def with_id(self, object_id): def in_bulk(self, object_ids): """Retrieve a set of documents by their ids. - :param object_ids: a list or tuple of ``ObjectId``\ s - :rtype: dict of ObjectIds as keys and collection-specific + :param object_ids: a list or tuple of ObjectId's + :rtype: dict of ObjectId's as keys and collection-specific Document subclasses as values. .. versionadded:: 0.3 @@ -775,7 +774,6 @@ def _clone_into(self, new_qs): "_ordering", "_snapshot", "_timeout", - "_slave_okay", "_read_preference", "_iter", "_scalar", @@ -1026,9 +1024,11 @@ def fields(self, _only_called=False, **kwargs): posts = BlogPost.objects(...).fields(comments=0) - To retrieve a subrange of array elements: + To retrieve a subrange or sublist of array elements, + support exist for both the `slice` and `elemMatch` projection operator: posts = BlogPost.objects(...).fields(slice__comments=5) + posts = BlogPost.objects(...).fields(elemMatch__comments="test") :param kwargs: A set of keyword arguments identifying what to include, exclude, or slice. @@ -1037,7 +1037,7 @@ def fields(self, _only_called=False, **kwargs): """ # Check for an operator and transform to mongo-style if there is - operators = ["slice"] + operators = ["slice", "elemMatch"] cleaned_fields = [] for key, value in kwargs.items(): parts = key.split("__") @@ -1140,7 +1140,7 @@ def comment(self, text): def explain(self): """Return an explain plan record for the - :class:`~mongoengine.queryset.QuerySet`\ 's cursor. + :class:`~mongoengine.queryset.QuerySet` cursor. """ return self._cursor.explain() @@ -1170,20 +1170,6 @@ def timeout(self, enabled): queryset._timeout = enabled return queryset - # DEPRECATED. Has no more impact on PyMongo 3+ - def slave_okay(self, enabled): - """Enable or disable the slave_okay when querying. - - :param enabled: whether or not the slave_okay is enabled - - .. deprecated:: Ignored with PyMongo 3+ - """ - msg = "slave_okay is deprecated as it has no impact when using PyMongo 3+." - warnings.warn(msg, DeprecationWarning) - queryset = self.clone() - queryset._slave_okay = enabled - return queryset - def read_preference(self, read_preference): """Change the read_preference when querying. @@ -1958,23 +1944,3 @@ def _chainable_method(self, method_name, val): setattr(queryset, "_" + method_name, val) return queryset - - # Deprecated - def ensure_index(self, **kwargs): - """Deprecated use :func:`Document.ensure_index`""" - msg = ( - "Doc.objects()._ensure_index() is deprecated. " - "Use Doc.ensure_index() instead." - ) - warnings.warn(msg, DeprecationWarning) - self._document.__class__.ensure_index(**kwargs) - return self - - def _ensure_indexes(self): - """Deprecated use :func:`~Document.ensure_indexes`""" - msg = ( - "Doc.objects()._ensure_indexes() is deprecated. " - "Use Doc.ensure_indexes() instead." - ) - warnings.warn(msg, DeprecationWarning) - self._document.__class__.ensure_indexes() diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 0b73e99b1..659a97e25 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -169,9 +169,9 @@ def query(_doc_cls=None, **kwargs): key = ".".join(parts) - if op is None or key not in mongo_query: + if key not in mongo_query: mongo_query[key] = value - elif key in mongo_query: + else: if isinstance(mongo_query[key], dict) and isinstance(value, dict): mongo_query[key].update(value) # $max/minDistance needs to come last - convert to SON diff --git a/setup.py b/setup.py index 5ba84e069..5cba5d9e0 100644 --- a/setup.py +++ b/setup.py @@ -108,6 +108,10 @@ def run_tests(self): "Topic :: Software Development :: Libraries :: Python Modules", ] +PYTHON_VERSION = sys.version_info[0] +PY3 = PYTHON_VERSION == 3 +PY2 = PYTHON_VERSION == 2 + extra_opts = { "packages": find_packages(exclude=["tests", "tests.*"]), "tests_require": [ @@ -116,9 +120,10 @@ def run_tests(self): "coverage<5.0", # recent coverage switched to sqlite format for the .coverage file which isn't handled properly by coveralls "blinker", "Pillow>=2.0.0, <7.0.0", # 7.0.0 dropped Python2 support + "zipp<2.0.0", # (dependency of pytest) dropped python2 support ], } -if sys.version_info[0] == 3: +if PY3: extra_opts["use_2to3"] = True if "test" in sys.argv: extra_opts["packages"] = find_packages() diff --git a/tests/fields/test_complex_datetime_field.py b/tests/fields/test_complex_datetime_field.py index f0a6b96e3..5bd6c56b0 100644 --- a/tests/fields/test_complex_datetime_field.py +++ b/tests/fields/test_complex_datetime_field.py @@ -65,7 +65,7 @@ class LogEntry(Document): for values in itertools.product([2014], mm, dd, hh, ii, ss, microsecond): stored = LogEntry(date=datetime.datetime(*values)).to_mongo()["date"] assert ( - re.match("^\d{4},\d{2},\d{2},\d{2},\d{2},\d{2},\d{6}$", stored) + re.match(r"^\d{4},\d{2},\d{2},\d{2},\d{2},\d{2},\d{6}$", stored) is not None ) @@ -74,7 +74,7 @@ class LogEntry(Document): "date_with_dots" ] assert ( - re.match("^\d{4}.\d{2}.\d{2}.\d{2}.\d{2}.\d{2}.\d{6}$", stored) is not None + re.match(r"^\d{4}.\d{2}.\d{2}.\d{2}.\d{2}.\d{2}.\d{6}$", stored) is not None ) def test_complexdatetime_usage(self): diff --git a/tests/fields/test_dict_field.py b/tests/fields/test_dict_field.py index 44e628f6c..5a886b302 100644 --- a/tests/fields/test_dict_field.py +++ b/tests/fields/test_dict_field.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +from bson import InvalidDocument import pytest from mongoengine import * @@ -19,22 +19,24 @@ class BlogPost(Document): post = BlogPost(info=info).save() assert get_as_pymongo(post) == {"_id": post.id, "info": info} - def test_general_things(self): - """Ensure that dict types work as expected.""" + def test_validate_invalid_type(self): + class BlogPost(Document): + info = DictField() + + BlogPost.drop_collection() + + invalid_infos = ["my post", ["test", "test"], {1: "test"}] + for invalid_info in invalid_infos: + with pytest.raises(ValidationError): + BlogPost(info=invalid_info).validate() + def test_keys_with_dots_or_dollars(self): class BlogPost(Document): info = DictField() BlogPost.drop_collection() post = BlogPost() - post.info = "my post" - with pytest.raises(ValidationError): - post.validate() - - post.info = ["test", "test"] - with pytest.raises(ValidationError): - post.validate() post.info = {"$title": "test"} with pytest.raises(ValidationError): @@ -48,25 +50,34 @@ class BlogPost(Document): with pytest.raises(ValidationError): post.validate() - post.info = {1: "test"} - with pytest.raises(ValidationError): - post.validate() - post.info = {"nested": {"the.title": "test"}} if get_mongodb_version() < MONGODB_36: - with pytest.raises(ValidationError): - post.validate() + # MongoDB < 3.6 rejects dots + # To avoid checking the mongodb version from the DictField class + # we rely on MongoDB to reject the data during the save + post.validate() + with pytest.raises(InvalidDocument): + post.save() else: post.validate() post.info = {"dollar_and_dot": {"te$st.test": "test"}} if get_mongodb_version() < MONGODB_36: - with pytest.raises(ValidationError): - post.validate() + post.validate() + with pytest.raises(InvalidDocument): + post.save() else: post.validate() - post.info = {"title": "test"} + def test_general_things(self): + """Ensure that dict types work as expected.""" + + class BlogPost(Document): + info = DictField() + + BlogPost.drop_collection() + + post = BlogPost(info={"title": "test"}) post.save() post = BlogPost() diff --git a/tests/fields/test_embedded_document_field.py b/tests/fields/test_embedded_document_field.py index eeddac1ee..4fa802e14 100644 --- a/tests/fields/test_embedded_document_field.py +++ b/tests/fields/test_embedded_document_field.py @@ -3,6 +3,7 @@ from mongoengine import ( Document, + DynamicEmbeddedDocument, EmbeddedDocument, EmbeddedDocumentField, GenericEmbeddedDocumentField, @@ -92,6 +93,33 @@ class Person(Document): assert exclude_p.settings.foo2 == p.settings.foo2 assert exclude_p.name == p.name + def test_dynamic_embedded_document_attribute(self): + class DynamicSettings(DynamicEmbeddedDocument): + known_field = StringField() + + class Person(Document): + name = StringField() + settings = EmbeddedDocumentField(DynamicSettings) + + Person.drop_collection() + + p = Person( + settings=DynamicSettings(known_field="abc", dynamic_field1="123"), + name="John", + ).save() + + # Test querying by a dynamic field that is not defined in the schema + assert Person.objects(settings__dynamic_field1="123").first().id == p.id + + p_modified = Person.objects(settings__known_field="abc").modify( + set__settings__dynamic_field1="789" + ) + p.reload() + + # Test if the update occurred successfully + assert p_modified.settings.dynamic_field1 == "123" + assert p.settings.dynamic_field1 == "789" + def test_query_embedded_document_attribute_with_inheritance(self): class BaseSettings(EmbeddedDocument): meta = {"allow_inheritance": True} diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index b30350e67..f6d1a916e 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -4476,6 +4476,74 @@ def test_scalar_cursor_behaviour(self): expected = "[u'A1', u'A2']" assert expected == "%s" % sorted(names) + def test_fields(self): + class Bar(EmbeddedDocument): + v = StringField() + z = StringField() + + class Foo(Document): + x = StringField() + y = IntField() + items = EmbeddedDocumentListField(Bar) + + Foo.drop_collection() + + Foo(x="foo1", y=1).save() + Foo(x="foo2", y=2, items=[]).save() + Foo(x="foo3", y=3, items=[Bar(z="a", v="V")]).save() + Foo( + x="foo4", + y=4, + items=[ + Bar(z="a", v="V"), + Bar(z="b", v="W"), + Bar(z="b", v="X"), + Bar(z="c", v="V"), + ], + ).save() + Foo( + x="foo5", + y=5, + items=[ + Bar(z="b", v="X"), + Bar(z="c", v="V"), + Bar(z="d", v="V"), + Bar(z="e", v="V"), + ], + ).save() + + foos_with_x = list(Foo.objects.order_by("y").fields(x=1)) + + assert all(o.x is not None for o in foos_with_x) + + foos_without_y = list(Foo.objects.order_by("y").fields(y=0)) + + assert all(o.y is None for o in foos_with_x) + + foos_with_sliced_items = list(Foo.objects.order_by("y").fields(slice__items=1)) + + assert foos_with_sliced_items[0].items == [] + assert foos_with_sliced_items[1].items == [] + assert len(foos_with_sliced_items[2].items) == 1 + assert foos_with_sliced_items[2].items[0].z == "a" + assert len(foos_with_sliced_items[3].items) == 1 + assert foos_with_sliced_items[3].items[0].z == "a" + assert len(foos_with_sliced_items[4].items) == 1 + assert foos_with_sliced_items[4].items[0].z == "b" + + foos_with_elem_match_items = list( + Foo.objects.order_by("y").fields(elemMatch__items={"z": "b"}) + ) + + assert foos_with_elem_match_items[0].items == [] + assert foos_with_elem_match_items[1].items == [] + assert foos_with_elem_match_items[2].items == [] + assert len(foos_with_elem_match_items[3].items) == 1 + assert foos_with_elem_match_items[3].items[0].z == "b" + assert foos_with_elem_match_items[3].items[0].v == "W" + assert len(foos_with_elem_match_items[4].items) == 1 + assert foos_with_elem_match_items[4].items[0].z == "b" + def test_elem_match(self): class Foo(EmbeddedDocument): shape = StringField() diff --git a/tests/queryset/test_transform.py b/tests/queryset/test_transform.py index 3898809e6..8d6c2d06b 100644 --- a/tests/queryset/test_transform.py +++ b/tests/queryset/test_transform.py @@ -24,6 +24,12 @@ def test_transform_query(self): } assert transform.query(friend__age__gte=30) == {"friend.age": {"$gte": 30}} assert transform.query(name__exists=True) == {"name": {"$exists": True}} + assert transform.query(name=["Mark"], __raw__={"name": {"$in": "Tom"}}) == { + "$and": [{"name": ["Mark"]}, {"name": {"$in": "Tom"}}] + } + assert transform.query(name__in=["Tom"], __raw__={"name": "Mark"}) == { + "$and": [{"name": {"$in": ["Tom"]}}, {"name": "Mark"}] + } def test_transform_update(self): class LisDoc(Document):