From a4d2f22fd2016e47987eb8c48f541db99eafb067 Mon Sep 17 00:00:00 2001 From: Matt Dennewitz Date: Tue, 23 Mar 2010 00:14:01 -0500 Subject: [PATCH 0001/1279] added 'geo_indexes' to TopLevelDocumentMetaclass; added GeoPointField, a glorified [lat float, lng float] container; added geo lookup operators to QuerySet; added initial geo tests --- mongoengine/base.py | 1 + mongoengine/fields.py | 17 ++++++++++++- mongoengine/queryset.py | 26 ++++++++++++++++---- tests/queryset.py | 54 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 6 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 0a90446..e4b45f2 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -206,6 +206,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): 'max_size': None, 'ordering': [], # default ordering applied at runtime 'indexes': [], # indexes to be ensured at runtime + 'geo_indexes': [], 'id_field': id_field, } diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 9a9f4e0..bb61f1d 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -12,7 +12,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', 'DecimalField', 'URLField', 'GenericReferenceField', - 'BinaryField'] + 'BinaryField', 'GeoPointField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -443,6 +443,7 @@ class GenericReferenceField(BaseField): def prepare_query_value(self, op, value): return self.to_mongo(value)['_ref'] + class BinaryField(BaseField): """A binary data field. """ @@ -462,3 +463,17 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') + + +class GeoPointField(BaseField): + """A list storing a latitude and longitude. + """ + + def validate(self, value): + assert isinstance(value, (list, tuple)) + + if not len(value) == 2: + raise ValidationError('Value must be a two-dimensional point.') + if not isinstance(value[0], (float, int)) and \ + not isinstance(value[1], (float, int)): + raise ValidationError('Both values in point must be float or int.') diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 11dc2bc..f03f97a 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -232,12 +232,17 @@ class QuerySet(object): # Ensure document-defined indexes are created if self._document._meta['indexes']: for key_or_list in self._document._meta['indexes']: - #self.ensure_index(key_or_list) self._collection.ensure_index(key_or_list) # Ensure indexes created by uniqueness constraints for index in self._document._meta['unique_indexes']: self._collection.ensure_index(index, unique=True) + + if self._document._meta['geo_indexes'] and \ + pymongo.version >= "1.5.1": + from pymongo import GEO2D + for index in self._document._meta['geo_indexes']: + self._collection.ensure_index([(index, GEO2D)]) # If _types is being used (for polymorphism), it needs an index if '_types' in self._query: @@ -298,6 +303,7 @@ class QuerySet(object): """ operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', 'all', 'size', 'exists'] + geo_operators = ['within_distance', 'within_box', 'near'] match_operators = ['contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith'] @@ -306,7 +312,7 @@ class QuerySet(object): parts = key.split('__') # Check for an operator and transform to mongo-style if there is op = None - if parts[-1] in operators + match_operators: + if parts[-1] in operators + match_operators + geo_operators: op = parts.pop() if _doc_cls: @@ -320,15 +326,25 @@ class QuerySet(object): singular_ops += match_operators if op in singular_ops: value = field.prepare_query_value(op, value) - elif op in ('in', 'nin', 'all'): + elif op in ('in', 'nin', 'all', 'near'): # 'in', 'nin' and 'all' require a list of values value = [field.prepare_query_value(op, v) for v in value] if field.__class__.__name__ == 'GenericReferenceField': parts.append('_ref') - if op and op not in match_operators: - value = {'$' + op: value} + # if op and op not in match_operators: + if op: + if op in geo_operators: + if op == "within_distance": + value = {'$within': {'$center': value}} + elif op == "near": + value = {'$near': value} + else: + raise NotImplmenetedError, \ + "Geo method has been implemented" + elif op not in match_operators: + value = {'$' + op: value} key = '.'.join(parts) if op is None or key not in mongo_query: diff --git a/tests/queryset.py b/tests/queryset.py index c0bd8fa..004c416 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1070,6 +1070,60 @@ class QuerySetTest(unittest.TestCase): def tearDown(self): self.Person.drop_collection() + def test_near(self): + """Ensure that "near" queries work with and without radii. + """ + class Event(Document): + title = StringField() + location = GeoPointField() + + def __unicode__(self): + return self.title + + meta = {'geo_indexes': ["location"]} + + Event.drop_collection() + + event1 = Event(title="Coltrane Motion @ Double Door", + location=[41.909889, -87.677137]) + event2 = Event(title="Coltrane Motion @ Bottom of the Hill", + location=[37.7749295, -122.4194155]) + event3 = Event(title="Coltrane Motion @ Empty Bottle", + location=[41.900474, -87.686638]) + + event1.save() + event2.save() + event3.save() + + # find all events "near" pitchfork office, chicago. + # note that "near" will show the san francisco event, too, + # although it sorts to last. + events = Event.objects(location__near=[41.9120459, -87.67892]) + self.assertEqual(events.count(), 3) + self.assertEqual(list(events), [event1, event3, event2]) + + # find events within 5 miles of pitchfork office, chicago + point_and_distance = [[41.9120459, -87.67892], 5] + events = Event.objects(location__within_distance=point_and_distance) + self.assertEqual(events.count(), 2) + events = list(events) + self.assertTrue(event2 not in events) + self.assertTrue(event1 in events) + self.assertTrue(event3 in events) + + # find events around san francisco + point_and_distance = [[37.7566023, -122.415579], 10] + events = Event.objects(location__within_distance=point_and_distance) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0], event2) + + # find events within 1 mile of greenpoint, broolyn, nyc, ny + point_and_distance = [[40.7237134, -73.9509714], 1] + events = Event.objects(location__within_distance=point_and_distance) + self.assertEqual(events.count(), 0) + + Event.drop_collection() + class QTest(unittest.TestCase): From 600ca3bcf9d9f87129f8504286f7e2108154b3cd Mon Sep 17 00:00:00 2001 From: Matt Dennewitz Date: Tue, 23 Mar 2010 00:57:26 -0500 Subject: [PATCH 0002/1279] renamed 'test_near' to 'test_geospatial_operators', updated added ordering checks to test --- tests/queryset.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 004c416..c8db29a 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1070,11 +1070,12 @@ class QuerySetTest(unittest.TestCase): def tearDown(self): self.Person.drop_collection() - def test_near(self): - """Ensure that "near" queries work with and without radii. + def test_geospatial_operators(self): + """Ensure that geospatial queries are working. """ class Event(Document): title = StringField() + date = DateTimeField() location = GeoPointField() def __unicode__(self): @@ -1085,10 +1086,13 @@ class QuerySetTest(unittest.TestCase): Event.drop_collection() event1 = Event(title="Coltrane Motion @ Double Door", + date=datetime.now() - timedelta(days=1), location=[41.909889, -87.677137]) event2 = Event(title="Coltrane Motion @ Bottom of the Hill", + date=datetime.now() - timedelta(days=10), location=[37.7749295, -122.4194155]) event3 = Event(title="Coltrane Motion @ Empty Bottle", + date=datetime.now(), location=[41.900474, -87.686638]) event1.save() @@ -1111,6 +1115,12 @@ class QuerySetTest(unittest.TestCase): self.assertTrue(event1 in events) self.assertTrue(event3 in events) + # ensure ordering is respected by "near" + events = Event.objects(location__near=[41.9120459, -87.67892]) + events = events.order_by("-date") + self.assertEqual(events.count(), 3) + self.assertEqual(list(events), [event3, event1, event2]) + # find events around san francisco point_and_distance = [[37.7566023, -122.415579], 10] events = Event.objects(location__within_distance=point_and_distance) @@ -1122,6 +1132,13 @@ class QuerySetTest(unittest.TestCase): events = Event.objects(location__within_distance=point_and_distance) self.assertEqual(events.count(), 0) + # ensure ordering is respected by "within_distance" + point_and_distance = [[41.9120459, -87.67892], 10] + events = Event.objects(location__within_distance=point_and_distance) + events = events.order_by("-date") + self.assertEqual(events.count(), 2) + self.assertEqual(events[0], event3) + Event.drop_collection() From 8f4a579df93f17a301eae23f72a7d3f5f6332ac5 Mon Sep 17 00:00:00 2001 From: Deepak Thukral Date: Sun, 28 Mar 2010 22:22:36 +0200 Subject: [PATCH 0003/1279] DoesNotExist and MultipleObjectsReturned now contributes Document class --- mongoengine/base.py | 29 +++++++++++++++++++++++++++-- mongoengine/queryset.py | 7 ++++--- tests/queryset.py | 7 +++++++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 0a90446..fbd95b3 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1,7 +1,9 @@ -from queryset import QuerySet, QuerySetManager - +import sys import pymongo +from queryset import QuerySet, QuerySetManager +from queryset import DoesNotExist, MultipleObjectsReturned + _document_registry = {} @@ -167,7 +169,22 @@ class DocumentMetaclass(type): for field in new_class._fields.values(): field.owner_document = new_class + module = attrs.pop('__module__') + + new_class.add_to_class('DoesNotExist', subclass_exception('DoesNotExist', + tuple(x.DoesNotExist + for k,x in superclasses.items()) + or (DoesNotExist,), module)) + + new_class.add_to_class('MultipleObjectsReturned', subclass_exception('MultipleObjectsReturned', + tuple(x.MultipleObjectsReturned + for k,x in superclasses.items()) + or (MultipleObjectsReturned,), module)) return new_class + + + def add_to_class(self, name, value): + setattr(self, name, value) class TopLevelDocumentMetaclass(DocumentMetaclass): @@ -417,3 +434,11 @@ class BaseDocument(object): if self.id == other.id: return True return False + +if sys.version_info < (2, 5): + # Prior to Python 2.5, Exception was an old-style class + def subclass_exception(name, parents, unused): + return types.ClassType(name, parents, {}) +else: + def subclass_exception(name, parents, module): + return type(name, parents, {'__module__': module}) \ No newline at end of file diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 11dc2bc..0623719 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -352,9 +352,10 @@ class QuerySet(object): return self[0] elif count > 1: message = u'%d items returned, instead of 1' % count - raise MultipleObjectsReturned(message) + raise self._document.MultipleObjectsReturned(message) else: - raise DoesNotExist('Document not found') + raise self._document.DoesNotExist("%s matching query does not exist." + % self._document._class_name) def get_or_create(self, *q_objs, **query): """Retreive unique object or create, if it doesn't exist. Raises @@ -380,7 +381,7 @@ class QuerySet(object): return self.first() else: message = u'%d items returned, instead of 1' % count - raise MultipleObjectsReturned(message) + raise self._document.MultipleObjectsReturned(message) def first(self): """Retrieve the first object matching the query. diff --git a/tests/queryset.py b/tests/queryset.py index c0bd8fa..668f035 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -147,6 +147,7 @@ class QuerySetTest(unittest.TestCase): """ # Try retrieving when no objects exists self.assertRaises(DoesNotExist, self.Person.objects.get) + self.assertRaises(self.Person.DoesNotExist, self.Person.objects.get) person1 = self.Person(name="User A", age=20) person1.save() @@ -155,6 +156,7 @@ class QuerySetTest(unittest.TestCase): # Retrieve the first person from the database self.assertRaises(MultipleObjectsReturned, self.Person.objects.get) + self.assertRaises(self.Person.MultipleObjectsReturned, self.Person.objects.get) # Use a query to filter the people found to just person2 person = self.Person.objects.get(age=30) @@ -162,6 +164,9 @@ class QuerySetTest(unittest.TestCase): person = self.Person.objects.get(age__lt=30) self.assertEqual(person.name, "User A") + + + def test_get_or_create(self): """Ensure that ``get_or_create`` returns one result or creates a new @@ -175,6 +180,8 @@ class QuerySetTest(unittest.TestCase): # Retrieve the first person from the database self.assertRaises(MultipleObjectsReturned, self.Person.objects.get_or_create) + self.assertRaises(self.Person.MultipleObjectsReturned, + self.Person.objects.get_or_create) # Use a query to filter the people found to just person2 person = self.Person.objects.get_or_create(age=30) From fbcf58c48f84412e5ebe38b20488cdd64a062b56 Mon Sep 17 00:00:00 2001 From: Deepak Thukral Date: Mon, 29 Mar 2010 11:25:17 +0200 Subject: [PATCH 0004/1279] updated documentation --- mongoengine/queryset.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 0623719..6c16207 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -15,7 +15,6 @@ REPR_OUTPUT_SIZE = 20 class DoesNotExist(Exception): pass - class MultipleObjectsReturned(Exception): pass @@ -27,6 +26,8 @@ class InvalidQueryError(Exception): class OperationError(Exception): pass +class InvalidCollectionError(Exception): + pass RE_TYPE = type(re.compile('')) @@ -341,8 +342,9 @@ class QuerySet(object): def get(self, *q_objs, **query): """Retrieve the the matching object raising :class:`~mongoengine.queryset.MultipleObjectsReturned` or - :class:`~mongoengine.queryset.DoesNotExist` exceptions if multiple or - no results are found. + `DocumentName.MultipleObjectsReturned` exception if multiple results and + :class:`~mongoengine.queryset.DoesNotExist` or `DocumentName.DoesNotExist` + if no results are found. .. versionadded:: 0.3 """ @@ -359,10 +361,11 @@ class QuerySet(object): def get_or_create(self, *q_objs, **query): """Retreive unique object or create, if it doesn't exist. Raises - :class:`~mongoengine.queryset.MultipleObjectsReturned` if multiple - results are found. A new document will be created if the document - doesn't exists; a dictionary of default values for the new document - may be provided as a keyword argument called :attr:`defaults`. + :class:`~mongoengine.queryset.MultipleObjectsReturned` or + `DocumentName.MultipleObjectsReturned` if multiple results are found. + A new document will be created if the document doesn't exists; a + dictionary of default values for the new document may be provided as a + keyword argument called :attr:`defaults`. .. versionadded:: 0.3 """ @@ -870,10 +873,6 @@ class QuerySet(object): return repr(data) -class InvalidCollectionError(Exception): - pass - - class QuerySetManager(object): def __init__(self, manager_func=None): From 207fd9fcb74f53757d3b445a88a19d58eee355cf Mon Sep 17 00:00:00 2001 From: Deepak Thukral Date: Mon, 29 Mar 2010 11:27:50 +0200 Subject: [PATCH 0005/1279] keeping import policy in mind --- mongoengine/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index fbd95b3..55323dd 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1,9 +1,9 @@ -import sys -import pymongo - from queryset import QuerySet, QuerySetManager from queryset import DoesNotExist, MultipleObjectsReturned +import sys +import pymongo + _document_registry = {} From 38b2919c0dc88fdc91164e77087da2eb1f424b5e Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 29 Mar 2010 22:02:33 +0200 Subject: [PATCH 0006/1279] added emailfield --- mongoengine/fields.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 9a9f4e0..622145e 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -12,7 +12,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', 'DecimalField', 'URLField', 'GenericReferenceField', - 'BinaryField'] + 'BinaryField', 'EmailField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -62,7 +62,7 @@ class StringField(BaseField): class URLField(StringField): - """A field that validates input as a URL. + """A field that validates input as an URL. .. versionadded:: 0.3 """ @@ -93,6 +93,19 @@ class URLField(StringField): message = 'This URL appears to be a broken link: %s' % e raise ValidationError(message) +class EmailField(StringField): + """A field that validates input as an E-Mail-Address. + """ + + EMAIL_REGEX = re.compile( + r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*" # dot-atom + r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"' # quoted-string + r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$', re.IGNORECASE # domain + ) + + def validate(self, value): + if not EmailField.EMAIL_REGEX.match(value): + raise ValidationError('Invalid Mail-address: %s' % value) class IntField(BaseField): """An integer field. From 2304dac8e3b92714b978e71d336cb2d66f248201 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Tue, 30 Mar 2010 00:04:39 +0200 Subject: [PATCH 0007/1279] added GeoLocationField with auto index-creation for GEO2D --- mongoengine/fields.py | 19 ++++++++++++++++++- mongoengine/queryset.py | 12 ++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 622145e..4c15dc0 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -12,7 +12,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', 'DecimalField', 'URLField', 'GenericReferenceField', - 'BinaryField', 'EmailField'] + 'BinaryField', 'EmailField', 'GeoLocationField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -342,6 +342,23 @@ class DictField(BaseField): def lookup_member(self, member_name): return BaseField(db_field=member_name) +class GeoLocationField(DictField): + """Supports geobased fields""" + + def validate(self, value): + """Make sure that a geo-value is of type (x, y) + """ + if not isinstance(value, tuple) and not isinstance(value, list): + raise ValidationError('GeoLocationField can only hold tuples or lists of (x, y)') + + if len(value) <> 2: + raise ValidationError('GeoLocationField must have exactly two elements (x, y)') + + def to_mongo(self, value): + return {'x': value[0], 'y': value[1]} + + def to_python(self, value): + return value.keys() class ReferenceField(BaseField): """A reference to a document that will be automatically dereferenced on diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 11dc2bc..a3bae3c 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -4,7 +4,6 @@ import pymongo import re import copy - __all__ = ['queryset_manager', 'Q', 'InvalidQueryError', 'InvalidCollectionError'] @@ -228,7 +227,7 @@ class QuerySet(object): """ if not self._accessed_collection: self._accessed_collection = True - + # Ensure document-defined indexes are created if self._document._meta['indexes']: for key_or_list in self._document._meta['indexes']: @@ -242,6 +241,11 @@ class QuerySet(object): # If _types is being used (for polymorphism), it needs an index if '_types' in self._query: self._collection.ensure_index('_types') + + # Ensure all needed field indexes are created + for field_name, field_instance in self._document._fields.iteritems(): + if field_instance.__class__.__name__ == 'GeoLocationField': + self._collection.ensure_index([(field_name, pymongo.GEO2D),]) return self._collection_obj @property @@ -297,7 +301,7 @@ class QuerySet(object): """Transform a query from Django-style format to Mongo format. """ operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', - 'all', 'size', 'exists'] + 'all', 'size', 'exists', 'near'] match_operators = ['contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith'] @@ -308,7 +312,7 @@ class QuerySet(object): op = None if parts[-1] in operators + match_operators: op = parts.pop() - + if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] fields = QuerySet._lookup_field(_doc_cls, parts) From 90200dbe9ce2d6ced0fa5fb0b3b5cd274015a616 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 12 Apr 2010 15:59:20 +0100 Subject: [PATCH 0008/1279] Fixed DecimalField bug --- mongoengine/fields.py | 3 +++ tests/fields.py | 10 ++++++++-- tests/queryset.py | 21 +++++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 9a9f4e0..94a1e24 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -155,6 +155,9 @@ class DecimalField(BaseField): if not isinstance(value, basestring): value = unicode(value) return decimal.Decimal(value) + + def to_mongo(self, value): + return unicode(value) def validate(self, value): if not isinstance(value, decimal.Decimal): diff --git a/tests/fields.py b/tests/fields.py index 986f0e2..24acc02 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -136,12 +136,16 @@ class FieldTest(unittest.TestCase): height = DecimalField(min_value=Decimal('0.1'), max_value=Decimal('3.5')) + Person.drop_collection() + person = Person() person.height = Decimal('1.89') - person.validate() + person.save() + person.reload() + self.assertEqual(person.height, Decimal('1.89')) person.height = '2.0' - person.validate() + person.save() person.height = 0.01 self.assertRaises(ValidationError, person.validate) person.height = Decimal('0.01') @@ -149,6 +153,8 @@ class FieldTest(unittest.TestCase): person.height = Decimal('4.0') self.assertRaises(ValidationError, person.validate) + Person.drop_collection() + def test_boolean_validation(self): """Ensure that invalid values cannot be assigned to boolean fields. """ diff --git a/tests/queryset.py b/tests/queryset.py index 668f035..e1fa6f4 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -623,6 +623,27 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() + def test_update_pull(self): + """Ensure that the 'pull' update operation works correctly. + """ + class Comment(EmbeddedDocument): + content = StringField() + + class BlogPost(Document): + slug = StringField() + comments = ListField(EmbeddedDocumentField(Comment)) + + comment1 = Comment(content="test1") + comment2 = Comment(content="test2") + + post = BlogPost(slug="test", comments=[comment1, comment2]) + post.save() + self.assertTrue(comment2 in post.comments) + + BlogPost.objects(slug="test").update(pull__comments__content="test2") + post.reload() + self.assertTrue(comment2 not in post.comments) + def test_order_by(self): """Ensure that QuerySets may be ordered. """ From a39685d98cb43b7ec30cd1c8aa0873b7186e390b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Peignier?= Date: Sun, 11 Apr 2010 20:14:32 +0200 Subject: [PATCH 0009/1279] make get_or_create returns a tuple with the retrieved or created object and a boolean specifying whether a new object was created --- docs/guide/querying.rst | 4 ++-- mongoengine/queryset.py | 8 +++++--- tests/queryset.py | 15 +++++++++------ 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index eba75d7..bad8b34 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -135,8 +135,8 @@ additional keyword argument, :attr:`defaults` may be provided, which will be used as default values for the new document, in the case that it should need to be created:: - >>> a = User.objects.get_or_create(name='User A', defaults={'age': 30}) - >>> b = User.objects.get_or_create(name='User A', defaults={'age': 40}) + >>> a, created = User.objects.get_or_create(name='User A', defaults={'age': 30}) + >>> b, created = User.objects.get_or_create(name='User A', defaults={'age': 40}) >>> a.name == b.name and a.age == b.age True diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6c16207..43b6eee 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -360,7 +360,9 @@ class QuerySet(object): % self._document._class_name) def get_or_create(self, *q_objs, **query): - """Retreive unique object or create, if it doesn't exist. Raises + """Retrieve unique object or create, if it doesn't exist. Returns a tuple of + ``(object, created)``, where ``object`` is the retrieved or created object + and ``created`` is a boolean specifying whether a new object was created. Raises :class:`~mongoengine.queryset.MultipleObjectsReturned` or `DocumentName.MultipleObjectsReturned` if multiple results are found. A new document will be created if the document doesn't exists; a @@ -379,9 +381,9 @@ class QuerySet(object): query.update(defaults) doc = self._document(**query) doc.save() - return doc + return doc, True elif count == 1: - return self.first() + return self.first(), False else: message = u'%d items returned, instead of 1' % count raise self._document.MultipleObjectsReturned(message) diff --git a/tests/queryset.py b/tests/queryset.py index e1fa6f4..aba3bc7 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -184,15 +184,18 @@ class QuerySetTest(unittest.TestCase): self.Person.objects.get_or_create) # Use a query to filter the people found to just person2 - person = self.Person.objects.get_or_create(age=30) + person, created = self.Person.objects.get_or_create(age=30) self.assertEqual(person.name, "User B") - - person = self.Person.objects.get_or_create(age__lt=30) + self.assertEqual(created, False) + + person, created = self.Person.objects.get_or_create(age__lt=30) self.assertEqual(person.name, "User A") - + self.assertEqual(created, False) + # Try retrieving when no objects exists - new doc should be created - self.Person.objects.get_or_create(age=50, defaults={'name': 'User C'}) - + person, created = self.Person.objects.get_or_create(age=50, defaults={'name': 'User C'}) + self.assertEqual(created, True) + person = self.Person.objects.get(age=50) self.assertEqual(person.name, "User C") From c8e466a160e1eeae20519dbf52dbfc6051c80dc6 Mon Sep 17 00:00:00 2001 From: Josh Ourisman Date: Mon, 12 Apr 2010 12:31:52 -0400 Subject: [PATCH 0010/1279] Moved SortedListField stuff into its own branch --- mongoengine/fields.py | 20 +++++++++++++++++++- tests/fields.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 94a1e24..90d9e5c 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1,6 +1,7 @@ from base import BaseField, ObjectIdField, ValidationError, get_document from document import Document, EmbeddedDocument from connection import _get_db +from operator import itemgetter import re import pymongo @@ -12,7 +13,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', 'DecimalField', 'URLField', 'GenericReferenceField', - 'BinaryField'] + 'BinaryField', 'SortedListField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -310,6 +311,23 @@ class ListField(BaseField): def lookup_member(self, member_name): return self.field.lookup_member(member_name) +class SortedListField(ListField): + """A ListField that sorts the contents of its list before writing to + the database in order to ensure that a sorted list is always + retrieved. + """ + + _ordering = None + + def __init__(self, field, **kwargs): + if 'ordering' in kwargs.keys(): + self._ordering = kwargs.pop('ordering') + super(SortedListField, self).__init__(field, **kwargs) + + def to_mongo(self, value): + if self._ordering is not None: + return sorted([self.field.to_mongo(item) for item in value], key=itemgetter(self._ordering)) + return sorted([self.field.to_mongo(item) for item in value]) class DictField(BaseField): """A dictionary field that wraps a standard Python dictionary. This is diff --git a/tests/fields.py b/tests/fields.py index 24acc02..7e68155 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -218,6 +218,37 @@ class FieldTest(unittest.TestCase): post.comments = 'yay' self.assertRaises(ValidationError, post.validate) + def test_sorted_list_sorting(self): + """Ensure that a sorted list field properly sorts values. + """ + class Comment(EmbeddedDocument): + order = IntField() + content = StringField() + + class BlogPost(Document): + content = StringField() + comments = SortedListField(EmbeddedDocumentField(Comment), ordering='order') + tags = SortedListField(StringField()) + + post = BlogPost(content='Went for a walk today...') + post.save() + + post.tags = ['leisure', 'fun'] + post.save() + post.reload() + self.assertEqual(post.tags, ['fun', 'leisure']) + + comment1 = Comment(content='Good for you', order=1) + comment2 = Comment(content='Yay.', order=0) + comments = [comment1, comment2] + post.comments = comments + post.save() + post.reload() + self.assertEqual(post.comments[0].content, comment2.content) + self.assertEqual(post.comments[1].content, comment1.content) + + BlogPost.drop_collection() + def test_dict_validation(self): """Ensure that dict types work as expected. """ From da3f4c30e29571bc5db4d5c6f8cd8959d36b18bd Mon Sep 17 00:00:00 2001 From: Don Spaulding Date: Wed, 14 Apr 2010 22:40:56 -0500 Subject: [PATCH 0011/1279] Fix doc typos --- docs/guide/defining-documents.rst | 2 +- docs/guide/querying.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index b5655c7..d38e661 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -178,7 +178,7 @@ either a single field name, or a list or tuple of field names:: class User(Document): username = StringField(unique=True) first_name = StringField() - last_name = StringField(unique_with='last_name') + last_name = StringField(unique_with='first_name') Document collections ==================== diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index bad8b34..4447f7d 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -172,7 +172,7 @@ custom manager methods as you like:: @queryset_manager def live_posts(doc_cls, queryset): - return queryset.order_by('-date') + return queryset.filter(published=True) BlogPost(title='test1', published=False).save() BlogPost(title='test2', published=True).save() From e9c92f30ba71dd3b7e5eea5541f76a1fee63c72b Mon Sep 17 00:00:00 2001 From: Don Spaulding Date: Thu, 15 Apr 2010 17:57:23 -0500 Subject: [PATCH 0012/1279] Add description of each of the keyword arguments to BaseField.__init__(), adds description for choices too. --- docs/guide/defining-documents.rst | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index d38e661..8964dab 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -47,6 +47,40 @@ are as follows: * :class:`~mongoengine.ReferenceField` * :class:`~mongoengine.GenericReferenceField` +Field arguments +--------------- +Each field type can be customized by keyword arguments. The following keyword +arguments can be set on all fields: + +:attr:`db_field` (Default: None) + The MongoDB field name. + +:attr:`name` (Default: None) + The mongoengine field name. + +:attr:`required` (Default: False) + If set to True and the field is not set on the document instance, a + :class:`~mongoengine.base.ValidationError` will be raised when the document is + validated. + +:attr:`default` (Default: None) + A value to use when no value is set for this field. + +:attr:`unique` (Default: False) + When True, no documents in the collection will have the same value for this + field. + +:attr:`unique_with` (Default: None) + A field name (or list of field names) that when taken together with this + field, will not have two documents in the collection with the same value. + +:attr:`primary_key` (Default: False) + When True, use this field as a primary key for the collection. + +:attr:`choices` (Default: None) + An iterable of choices to which the value of this field should be limited. + + List fields ----------- MongoDB allows the storage of lists of items. To add a list of items to a From ee0c75a26da5d83a6d8042f0317f8a3e92a39bb3 Mon Sep 17 00:00:00 2001 From: Don Spaulding Date: Thu, 15 Apr 2010 17:59:35 -0500 Subject: [PATCH 0013/1279] Add choices keyword argument to BaseField.__init__() --- mongoengine/base.py | 14 +++++++++++--- tests/fields.py | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 55323dd..78f0650 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -24,7 +24,8 @@ class BaseField(object): _index_with_types = True def __init__(self, db_field=None, name=None, required=False, default=None, - unique=False, unique_with=None, primary_key=False): + unique=False, unique_with=None, primary_key=False, + choices=None): self.db_field = (db_field or name) if not primary_key else '_id' if name: import warnings @@ -36,6 +37,7 @@ class BaseField(object): self.unique = bool(unique or unique_with) self.unique_with = unique_with self.primary_key = primary_key + self.choices = choices def __get__(self, instance, owner): """Descriptor for retrieving a value from a field in a document. Do @@ -79,6 +81,12 @@ class BaseField(object): """ pass + def _validate(self, value): + if self.choices is not None: + if value not in self.choices: + raise ValidationError("Value must be one of %s."%unicode(self.choices)) + self.validate(value) + class ObjectIdField(BaseField): """An field wrapper around MongoDB's ObjectIds. @@ -314,7 +322,7 @@ class BaseDocument(object): for field, value in fields: if value is not None: try: - field.validate(value) + field._validate(value) except (ValueError, AttributeError, AssertionError), e: raise ValidationError('Invalid value for field of type "' + field.__class__.__name__ + '"') @@ -441,4 +449,4 @@ if sys.version_info < (2, 5): return types.ClassType(name, parents, {}) else: def subclass_exception(name, parents, module): - return type(name, parents, {'__module__': module}) \ No newline at end of file + return type(name, parents, {'__module__': module}) diff --git a/tests/fields.py b/tests/fields.py index 7e68155..4050e26 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -588,5 +588,26 @@ class FieldTest(unittest.TestCase): AttachmentRequired.drop_collection() AttachmentSizeLimit.drop_collection() + def test_choices_validation(self): + """Ensure that value is in a container of allowed values. + """ + class Shirt(Document): + size = StringField(max_length=3, choices=('S','M','L','XL','XXL')) + + Shirt.drop_collection() + + shirt = Shirt() + shirt.validate() + + shirt.size = "S" + shirt.validate() + + shirt.size = "XS" + self.assertRaises(ValidationError, shirt.validate) + + Shirt.drop_collection() + + + if __name__ == '__main__': unittest.main() From 48facec5246dcd9c0b3e543eab2de55e25ce4132 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Fri, 16 Apr 2010 16:59:34 +0200 Subject: [PATCH 0014/1279] Fixes tiny documentation error. Adds possibility to add custom validation methods to fields, e. g.: class Customer(Document): country = StringField(validation=lambda value: value in ['DE', 'AT', 'CH']) Replaced some str() with unicode() for i18n reasons. --- docs/guide/defining-documents.rst | 2 +- mongoengine/base.py | 15 ++++++++------- mongoengine/fields.py | 29 +++++++++++++++++++++++++++-- mongoengine/queryset.py | 14 +++++++------- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index d38e661..76791e2 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -22,7 +22,7 @@ objects** as class attributes to the document class:: class Page(Document): title = StringField(max_length=200, required=True) - date_modified = DateTimeField(default=datetime.now) + date_modified = DateTimeField(default=datetime.datetime.now) Fields ====== diff --git a/mongoengine/base.py b/mongoengine/base.py index 55323dd..a83f3e7 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -24,7 +24,7 @@ class BaseField(object): _index_with_types = True def __init__(self, db_field=None, name=None, required=False, default=None, - unique=False, unique_with=None, primary_key=False): + unique=False, unique_with=None, primary_key=False, validation=None): self.db_field = (db_field or name) if not primary_key else '_id' if name: import warnings @@ -36,6 +36,7 @@ class BaseField(object): self.unique = bool(unique or unique_with) self.unique_with = unique_with self.primary_key = primary_key + self.validation = validation def __get__(self, instance, owner): """Descriptor for retrieving a value from a field in a document. Do @@ -77,8 +78,8 @@ class BaseField(object): def validate(self, value): """Perform validation on a value. """ - pass - + if self.validation is not None and not self.validation(value): + raise ValidationError('Value does not match custom validation method.') class ObjectIdField(BaseField): """An field wrapper around MongoDB's ObjectIds. @@ -91,10 +92,10 @@ class ObjectIdField(BaseField): def to_mongo(self, value): if not isinstance(value, pymongo.objectid.ObjectId): try: - return pymongo.objectid.ObjectId(str(value)) + return pymongo.objectid.ObjectId(unicode(value)) except Exception, e: #e.message attribute has been deprecated since Python 2.6 - raise ValidationError(str(e)) + raise ValidationError(unicode(e)) return value def prepare_query_value(self, op, value): @@ -102,7 +103,7 @@ class ObjectIdField(BaseField): def validate(self, value): try: - pymongo.objectid.ObjectId(str(value)) + pymongo.objectid.ObjectId(unicode(value)) except: raise ValidationError('Invalid Object ID') @@ -402,7 +403,7 @@ class BaseDocument(object): # class if unavailable class_name = son.get(u'_cls', cls._class_name) - data = dict((str(key), value) for key, value in son.items()) + data = dict((unicode(key), value) for key, value in son.items()) if '_types' in data: del data['_types'] diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 5eb0042..227af0a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -39,6 +39,8 @@ class StringField(BaseField): if self.regex is not None and self.regex.match(value) is None: message = 'String value did not match validation regex' raise ValidationError(message) + + super(StringField, self).validate(value) def lookup_member(self, member_name): return None @@ -93,6 +95,8 @@ class URLField(StringField): except Exception, e: message = 'This URL appears to be a broken link: %s' % e raise ValidationError(message) + + super(URLField, self).validate(value) class EmailField(StringField): """A field that validates input as an E-Mail-Address. @@ -130,7 +134,8 @@ class IntField(BaseField): if self.max_value is not None and value > self.max_value: raise ValidationError('Integer value is too large') - + + super(IntField, self).validate(value) class FloatField(BaseField): """An floating point number field. @@ -153,6 +158,8 @@ class FloatField(BaseField): if self.max_value is not None and value > self.max_value: raise ValidationError('Float value is too large') + + super(FloatField, self).validate(value) class DecimalField(BaseField): @@ -187,6 +194,8 @@ class DecimalField(BaseField): if self.max_value is not None and value > self.max_value: raise ValidationError('Decimal value is too large') + + super(DecimalField, self).validate(value) class BooleanField(BaseField): @@ -200,6 +209,8 @@ class BooleanField(BaseField): def validate(self, value): assert isinstance(value, bool) + + super(BooleanField, self).validate(value) class DateTimeField(BaseField): @@ -208,6 +219,8 @@ class DateTimeField(BaseField): def validate(self, value): assert isinstance(value, datetime.datetime) + + super(DateTimeField, self).validate(value) class EmbeddedDocumentField(BaseField): @@ -239,6 +252,8 @@ class EmbeddedDocumentField(BaseField): raise ValidationError('Invalid embedded document instance ' 'provided to an EmbeddedDocumentField') self.document.validate(value) + + super(EmbeddedDocumentField, self).validate(value) def lookup_member(self, member_name): return self.document._fields.get(member_name) @@ -315,6 +330,8 @@ class ListField(BaseField): [self.field.validate(item) for item in value] except Exception, err: raise ValidationError('Invalid ListField item (%s)' % str(err)) + + super(ListField, self).validate(value) def prepare_query_value(self, op, value): if op in ('set', 'unset'): @@ -359,6 +376,8 @@ class DictField(BaseField): if any(('.' in k or '$' in k) for k in value): raise ValidationError('Invalid dictionary key name - keys may not ' 'contain "." or "$" characters') + + super(DictField, self).validate(value) def lookup_member(self, member_name): return BaseField(db_field=member_name) @@ -374,6 +393,8 @@ class GeoLocationField(DictField): if len(value) <> 2: raise ValidationError('GeoLocationField must have exactly two elements (x, y)') + + super(GeoLocationField, self).validate(value) def to_mongo(self, value): return {'x': value[0], 'y': value[1]} @@ -443,6 +464,8 @@ class ReferenceField(BaseField): def validate(self, value): assert isinstance(value, (self.document_type, pymongo.dbref.DBRef)) + + super(ReferenceField, self).validate(value) def lookup_member(self, member_name): return self.document_type._fields.get(member_name) @@ -506,10 +529,12 @@ class BinaryField(BaseField): return pymongo.binary.Binary(value) def to_python(self, value): - return str(value) + return unicode(value) def validate(self, value): assert isinstance(value, str) if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') + + super(BinaryField, self).validate(value) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index c57a516..40779ab 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -123,7 +123,7 @@ class Q(object): # Comparing two ObjectIds in Javascript doesn't work.. if isinstance(value, pymongo.objectid.ObjectId): - value = str(value) + value = unicode(value) # Perform the substitution operation_js = op_js % { @@ -497,13 +497,13 @@ class QuerySet(object): map_f_scope = {} if isinstance(map_f, pymongo.code.Code): map_f_scope = map_f.scope - map_f = str(map_f) + map_f = unicode(map_f) map_f = pymongo.code.Code(self._sub_js_fields(map_f), map_f_scope) reduce_f_scope = {} if isinstance(reduce_f, pymongo.code.Code): reduce_f_scope = reduce_f.scope - reduce_f = str(reduce_f) + reduce_f = unicode(reduce_f) reduce_f_code = self._sub_js_fields(reduce_f) reduce_f = pymongo.code.Code(reduce_f_code, reduce_f_scope) @@ -513,7 +513,7 @@ class QuerySet(object): finalize_f_scope = {} if isinstance(finalize_f, pymongo.code.Code): finalize_f_scope = finalize_f.scope - finalize_f = str(finalize_f) + finalize_f = unicode(finalize_f) finalize_f_code = self._sub_js_fields(finalize_f) finalize_f = pymongo.code.Code(finalize_f_code, finalize_f_scope) mr_args['finalize'] = finalize_f @@ -736,7 +736,7 @@ class QuerySet(object): # Older versions of PyMongo don't support 'multi' self._collection.update(self._query, update, safe=safe_update) except pymongo.errors.OperationFailure, e: - raise OperationError('Update failed [%s]' % str(e)) + raise OperationError(u'Update failed [%s]' % unicode(e)) def __iter__(self): return self @@ -752,9 +752,9 @@ class QuerySet(object): field_name = match.group(1).split('.') fields = QuerySet._lookup_field(self._document, field_name) # Substitute the correct name for the field into the javascript - return '["%s"]' % fields[-1].db_field + return u'["%s"]' % fields[-1].db_field - return re.sub('\[\s*~([A-z_][A-z_0-9.]+?)\s*\]', field_sub, code) + return re.sub(u'\[\s*~([A-z_][A-z_0-9.]+?)\s*\]', field_sub, code) def exec_js(self, code, *fields, **options): """Execute a Javascript function on the server. A list of fields may be From f3ca9fa4c5052c997790077b18246d9c7c3468d0 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Fri, 16 Apr 2010 18:00:51 +0200 Subject: [PATCH 0015/1279] Make validation-lists possible. Example: class Doc(Document): country = StringField(validation=['DE', 'AT', 'CH']) --- mongoengine/base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index a83f3e7..41cbed5 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -78,8 +78,11 @@ class BaseField(object): def validate(self, value): """Perform validation on a value. """ - if self.validation is not None and not self.validation(value): - raise ValidationError('Value does not match custom validation method.') + if self.validation is not None: + if isinstance(self.validation, list): + raise ValidationError('Value not in validation list.') + elif callable(self.validation) and not self.validation(value): + raise ValidationError('Value does not match custom validation method.') class ObjectIdField(BaseField): """An field wrapper around MongoDB's ObjectIds. From 170c56bcb93f519bb4ca302d3754e953d77680b1 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Fri, 16 Apr 2010 18:13:11 +0200 Subject: [PATCH 0016/1279] introduced min_length for a StringField --- mongoengine/fields.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 227af0a..e4d95d5 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -22,9 +22,10 @@ class StringField(BaseField): """A unicode string field. """ - def __init__(self, regex=None, max_length=None, **kwargs): + def __init__(self, regex=None, max_length=None, min_length=None, **kwargs): self.regex = re.compile(regex) if regex else None self.max_length = max_length + self.min_length = min_length super(StringField, self).__init__(**kwargs) def to_python(self, value): @@ -35,6 +36,9 @@ class StringField(BaseField): if self.max_length is not None and len(value) > self.max_length: raise ValidationError('String value is too long') + + if self.min_length is not None and len(value) < self.min_length: + raise ValidationError('String value is too short') if self.regex is not None and self.regex.match(value) is None: message = 'String value did not match validation regex' From ef172712da915e4563fbb39f70406d305e96ca13 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Fri, 16 Apr 2010 22:25:45 +0200 Subject: [PATCH 0017/1279] bugfix --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 41cbed5..2494949 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -79,7 +79,7 @@ class BaseField(object): """Perform validation on a value. """ if self.validation is not None: - if isinstance(self.validation, list): + if isinstance(self.validation, list) and value not in self.validation: raise ValidationError('Value not in validation list.') elif callable(self.validation) and not self.validation(value): raise ValidationError('Value does not match custom validation method.') From da57572409b9b7819c886da581ec07859d6805ae Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Sat, 17 Apr 2010 01:23:14 +0200 Subject: [PATCH 0018/1279] Introduced new create_default field argument. If set to true, mongoengine will automagically create an instance of the desired document class (useful if using EmbeddedDocumentField for example): class SubDoc(EmbeddedDocument): url = URLField() class MyDoc(Document): subdoc = EmbeddedDocumentField(SubDoc, create_default=True) With create_default MyDoc().subdoc is automatically instantiated. Hint: default=SubDoc() WON'T work (that's why I've introduced create_default) --- mongoengine/base.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 2494949..c5f0b55 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -24,7 +24,8 @@ class BaseField(object): _index_with_types = True def __init__(self, db_field=None, name=None, required=False, default=None, - unique=False, unique_with=None, primary_key=False, validation=None): + unique=False, unique_with=None, primary_key=False, validation=None, + create_default=False): self.db_field = (db_field or name) if not primary_key else '_id' if name: import warnings @@ -37,6 +38,7 @@ class BaseField(object): self.unique_with = unique_with self.primary_key = primary_key self.validation = validation + self.create_default = create_default def __get__(self, instance, owner): """Descriptor for retrieving a value from a field in a document. Do @@ -53,6 +55,10 @@ class BaseField(object): # Allow callable default values if callable(value): value = value() + + # Check whether we should auto-create a default document + if self.create_default and hasattr(self, 'document'): + value = self.document() return value def __set__(self, instance, value): From e196e229cde21089ea6b4456e0e6926c22661728 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Sat, 17 Apr 2010 01:36:45 +0200 Subject: [PATCH 0019/1279] Accepting a tuple for validation argument. --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index c5f0b55..43a54ee 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -85,7 +85,7 @@ class BaseField(object): """Perform validation on a value. """ if self.validation is not None: - if isinstance(self.validation, list) and value not in self.validation: + if (isinstance(self.validation, list) or isinstance(self.validation, tuple)) and value not in self.validation: raise ValidationError('Value not in validation list.') elif callable(self.validation) and not self.validation(value): raise ValidationError('Value does not match custom validation method.') From 3c7e8be2e73d7b0d809603049faea5eaaffa1ac2 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Sat, 17 Apr 2010 16:59:09 +0200 Subject: [PATCH 0020/1279] Removed create_default since it can be achieved with the `default` argument (like default=MyEmbeddedDocument since default takes callables too). --- mongoengine/base.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 8c814ac..bd3f11a 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -25,7 +25,7 @@ class BaseField(object): def __init__(self, db_field=None, name=None, required=False, default=None, unique=False, unique_with=None, primary_key=False, validation=None, - create_default=False, choices=None): + choices=None): self.db_field = (db_field or name) if not primary_key else '_id' if name: import warnings @@ -38,7 +38,6 @@ class BaseField(object): self.unique_with = unique_with self.primary_key = primary_key self.validation = validation - self.create_default = create_default self.choices = choices def __get__(self, instance, owner): @@ -56,10 +55,6 @@ class BaseField(object): # Allow callable default values if callable(value): value = value() - - # Check whether we should auto-create a default document - if self.create_default and hasattr(self, 'document'): - value = self.document() return value def __set__(self, instance, value): From edfda6ad5bb85e775e5d847a313a09f5224d86e1 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sat, 17 Apr 2010 21:24:06 +0100 Subject: [PATCH 0021/1279] BinaryField returns str not unicode --- mongoengine/fields.py | 5 +++-- mongoengine/queryset.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 5b2f14a..55e5add 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -505,10 +505,11 @@ class BinaryField(BaseField): return pymongo.binary.Binary(value) def to_python(self, value): - return unicode(value) + # Returns str not unicode as this is binary data + return str(value) def validate(self, value): assert isinstance(value, str) if self.max_bytes is not None and len(value) > self.max_bytes: - raise ValidationError('Binary value is too long') \ No newline at end of file + raise ValidationError('Binary value is too long') diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 40779ab..396a745 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -918,7 +918,7 @@ class QuerySetManager(object): opts = {'capped': True, 'size': max_size} if max_documents: opts['max'] = max_documents - self._collection = db.create_collection(collection, opts) + self._collection = db.create_collection(collection, **opts) else: self._collection = db[collection] From 3b4df4615add363febff12c814529633ca0d70d4 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sat, 17 Apr 2010 21:45:11 +0100 Subject: [PATCH 0022/1279] Fixed MRO error that occured on document inheritance --- mongoengine/base.py | 21 +++++++++++---------- tests/document.py | 5 +++++ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index bd3f11a..83faa92 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -187,19 +187,20 @@ class DocumentMetaclass(type): for field in new_class._fields.values(): field.owner_document = new_class - module = attrs.pop('__module__') + module = attrs.get('__module__') - new_class.add_to_class('DoesNotExist', subclass_exception('DoesNotExist', - tuple(x.DoesNotExist - for k,x in superclasses.items()) - or (DoesNotExist,), module)) + base_excs = tuple(base.DoesNotExist for base in bases + if hasattr(base, 'DoesNotExist')) or (DoesNotExist,) + exc = subclass_exception('DoesNotExist', base_excs, module) + new_class.add_to_class('DoesNotExist', exc) - new_class.add_to_class('MultipleObjectsReturned', subclass_exception('MultipleObjectsReturned', - tuple(x.MultipleObjectsReturned - for k,x in superclasses.items()) - or (MultipleObjectsReturned,), module)) - return new_class + base_excs = tuple(base.MultipleObjectsReturned for base in bases + if hasattr(base, 'MultipleObjectsReturned')) + base_excs = base_excs or (MultipleObjectsReturned,) + exc = subclass_exception('MultipleObjectsReturned', base_excs, module) + new_class.add_to_class('MultipleObjectsReturned', exc) + return new_class def add_to_class(self, name, value): setattr(self, name, value) diff --git a/tests/document.py b/tests/document.py index a17aebd..a753fa7 100644 --- a/tests/document.py +++ b/tests/document.py @@ -127,6 +127,11 @@ class DocumentTest(unittest.TestCase): self.assertEqual(Employee._meta['collection'], self.Person._meta['collection']) + # Ensure that MRO error is not raised + class A(Document): pass + class B(A): pass + class C(B): pass + def test_allow_inheritance(self): """Ensure that inheritance may be disabled on simple classes and that _cls and _types will not be used. From eecc6188a7a3056cfacbd757ca2ad3cd5668a1e3 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 19 Apr 2010 11:34:09 +0200 Subject: [PATCH 0023/1279] fixes issue #41 since unicode kwargs is an feature of python 2.6.5 and above. --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 83faa92..2dde93b 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -421,7 +421,7 @@ class BaseDocument(object): # class if unavailable class_name = son.get(u'_cls', cls._class_name) - data = dict((unicode(key), value) for key, value in son.items()) + data = dict((str(key), value) for key, value in son.items()) if '_types' in data: del data['_types'] From 86575cb0351331f084812d518ad3f9bbf3f1f704 Mon Sep 17 00:00:00 2001 From: Matt Dennewitz Date: Mon, 19 Apr 2010 09:39:03 -0500 Subject: [PATCH 0024/1279] can't use unicode strings for __init__ kwargs --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 83faa92..2dde93b 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -421,7 +421,7 @@ class BaseDocument(object): # class if unavailable class_name = son.get(u'_cls', cls._class_name) - data = dict((unicode(key), value) for key, value in son.items()) + data = dict((str(key), value) for key, value in son.items()) if '_types' in data: del data['_types'] From 682326c130e396e617f3a3124cd0bcfadcc532a1 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Fri, 30 Apr 2010 18:04:58 +0200 Subject: [PATCH 0025/1279] documentation bug fixed --- docs/guide/querying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 4447f7d..690569a 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -172,7 +172,7 @@ custom manager methods as you like:: @queryset_manager def live_posts(doc_cls, queryset): - return queryset.filter(published=True) + return queryset(published=True).filter(published=True) BlogPost(title='test1', published=False).save() BlogPost(title='test2', published=True).save() From 9df725165beca247340fcee7f2858fdaa64107a6 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Fri, 14 May 2010 13:35:45 +0200 Subject: [PATCH 0026/1279] =?UTF-8?q?Added=20a=20possibility=20to=20define?= =?UTF-8?q?=20a=20base=20class=20for=20fields=20from=20a=20DictField=20(in?= =?UTF-8?q?stead=20of=20using=20BaseField).=20This=20is=20important=20if?= =?UTF-8?q?=20you=20want=20to=20use=20field-based=20query=20abilities=20li?= =?UTF-8?q?ke=20StringField's=20startswith/endswith/contains.=20Just=20def?= =?UTF-8?q?ine=20`basecls=C2=B4=20when=20defining=20your=20DictField.=20Ex?= =?UTF-8?q?ample:?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit class Test(Document): name = StringField() translations = DictField(basecls=StringField) Without basecls defined: > Test.objects(translations__german__startswith='Deutsch') [] With basecls set to StringField: > Test.objects(translations__german__startswith='Deutsch') [] --- mongoengine/fields.py | 7 ++++++- mongoengine/queryset.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 55e5add..75008d6 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -348,6 +348,11 @@ class DictField(BaseField): .. versionadded:: 0.3 """ + def __init__(self, basecls=None, *args, **kwargs): + self.basecls = basecls or BaseField + assert issubclass(self.basecls, BaseField) + super(DictField, self).__init__(*args, **kwargs) + def validate(self, value): """Make sure that a list of valid fields is being used. """ @@ -360,7 +365,7 @@ class DictField(BaseField): 'contain "." or "$" characters') def lookup_member(self, member_name): - return BaseField(db_field=member_name) + return self.basecls(db_field=member_name) class GeoLocationField(DictField): """Supports geobased fields""" diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 396a745..11b5321 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -313,7 +313,7 @@ class QuerySet(object): op = None if parts[-1] in operators + match_operators: op = parts.pop() - + if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] fields = QuerySet._lookup_field(_doc_cls, parts) From 11c7a15067b1095f68fb045fd878fdcf4e84c1b7 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Fri, 14 May 2010 13:49:13 +0200 Subject: [PATCH 0027/1279] Added test for DictField's basecls. --- tests/queryset.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index aba3bc7..c43b8f1 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1063,6 +1063,29 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() + def test_dict_with_custom_baseclass(self): + """Ensure DictField working with custom base clases. + """ + class Test(Document): + testdict = DictField() + + t = Test(testdict={'f': 'Value'}) + t.save() + + self.assertEqual(len(Test.objects(testdict__f__startswith='Val')), 0) + self.assertEqual(len(Test.objects(testdict__f='Value')), 1) + Test.drop_collection() + + class Test(Document): + testdict = DictField(basecls=StringField) + + t = Test(testdict={'f': 'Value'}) + t.save() + + self.assertEqual(len(Test.objects(testdict__f='Value')), 1) + self.assertEqual(len(Test.objects(testdict__f__startswith='Val')), 1) + Test.drop_collection() + def test_bulk(self): """Ensure bulk querying by object id returns a proper dict. """ From 4972bdb3834e0b5db1c4f20cf4bf0082a813ab76 Mon Sep 17 00:00:00 2001 From: Stephan Jaekel Date: Fri, 14 May 2010 14:02:39 +0200 Subject: [PATCH 0028/1279] ignore empty Q objects when combining Q objects. --- mongoengine/queryset.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 11b5321..3db23ba 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -59,8 +59,10 @@ class Q(object): def _combine(self, other, op): obj = Q() - obj.query = ['('] + copy.deepcopy(self.query) + [op] - obj.query += copy.deepcopy(other.query) + [')'] + if self.query[0]: + obj.query = ['('] + copy.deepcopy(self.query) + [op] + copy.deepcopy(other.query) + [')'] + else: + obj.query = copy.deepcopy(other.query) return obj def __or__(self, other): From 225972e151b8bead6842c6be4205f9e2771b2c3a Mon Sep 17 00:00:00 2001 From: Stephan Jaekel Date: Fri, 14 May 2010 14:03:18 +0200 Subject: [PATCH 0029/1279] Added some handy shortcuts for django users. --- mongoengine/django/shortcuts.py | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 mongoengine/django/shortcuts.py diff --git a/mongoengine/django/shortcuts.py b/mongoengine/django/shortcuts.py new file mode 100644 index 0000000..29bc17a --- /dev/null +++ b/mongoengine/django/shortcuts.py @@ -0,0 +1,45 @@ +from django.http import Http404 +from mongoengine.queryset import QuerySet +from mongoengine.base import BaseDocument + +def _get_queryset(cls): + """Inspired by django.shortcuts.*""" + if isinstance(cls, QuerySet): + return cls + else: + return cls.objects + +def get_document_or_404(cls, *args, **kwargs): + """ + Uses get() to return an document, or raises a Http404 exception if the document + does not exist. + + cls may be a Document or QuerySet object. All other passed + arguments and keyword arguments are used in the get() query. + + Note: Like with get(), an MultipleObjectsReturned will be raised if more than one + object is found. + + Inspired by django.shortcuts.* + """ + queryset = _get_queryset(cls) + try: + return queryset.get(*args, **kwargs) + except queryset._document.DoesNotExist: + raise Http404('No %s matches the given query.' % queryset._document._class_name) + +def get_list_or_404(cls, *args, **kwargs): + """ + Uses filter() to return a list of documents, or raise a Http404 exception if + the list is empty. + + cls may be a Document or QuerySet object. All other passed + arguments and keyword arguments are used in the filter() query. + + Inspired by django.shortcuts.* + """ + queryset = _get_queryset(cls) + obj_list = list(queryset.filter(*args, **kwargs)) + if not obj_list: + raise Http404('No %s matches the given query.' % queryset._document._class_name) + return obj_list From 88da9985327e7b664b844a28039e8ed37e5c1d2a Mon Sep 17 00:00:00 2001 From: Stephan Jaekel Date: Fri, 14 May 2010 14:21:58 +0200 Subject: [PATCH 0030/1279] added test for empty Q objects --- tests/queryset.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index aba3bc7..f184bb0 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1136,5 +1136,18 @@ class QTest(unittest.TestCase): self.assertEqual(q._item_query_as_js(item, test_scope, 0), js) self.assertEqual(scope, test_scope) + def test_empty_q(self): + """Ensure that starting with an empty Q object won't hurt. + """ + q1 = Q() + q2 = Q(age__gte=18) + q3 = Q(name='test') + + query = ['(', {'age__gte': 18}, '||', {'name': 'test'}, ')'] + self.assertEqual((q1 | q2 | q3).query, query) + + query = ['(', {'age__gte': 18}, '&&', {'name': 'test'}, ')'] + self.assertEqual((q1 & q2 & q3).query, query) + if __name__ == '__main__': unittest.main() From f657432be3d33fda30a88d04997a4680a591cee7 Mon Sep 17 00:00:00 2001 From: Stephan Jaekel Date: Fri, 14 May 2010 14:35:27 +0200 Subject: [PATCH 0031/1279] always ignore empty Q objects, if the new Q is empty, the old one will be returned. --- mongoengine/queryset.py | 2 ++ tests/queryset.py | 10 ++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 3db23ba..72a474d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -59,6 +59,8 @@ class Q(object): def _combine(self, other, op): obj = Q() + if not other.query[0]: + return self if self.query[0]: obj.query = ['('] + copy.deepcopy(self.query) + [op] + copy.deepcopy(other.query) + [')'] else: diff --git a/tests/queryset.py b/tests/queryset.py index df2e2e7..9daa73e 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1160,17 +1160,19 @@ class QTest(unittest.TestCase): self.assertEqual(scope, test_scope) def test_empty_q(self): - """Ensure that starting with an empty Q object won't hurt. + """Ensure that empty Q objects won't hurt. """ q1 = Q() q2 = Q(age__gte=18) - q3 = Q(name='test') + q3 = Q() + q4 = Q(name='test') + q5 = Q() query = ['(', {'age__gte': 18}, '||', {'name': 'test'}, ')'] - self.assertEqual((q1 | q2 | q3).query, query) + self.assertEqual((q1 | q2 | q3 | q4 | q5).query, query) query = ['(', {'age__gte': 18}, '&&', {'name': 'test'}, ')'] - self.assertEqual((q1 & q2 & q3).query, query) + self.assertEqual((q1 & q2 & q3 & q4 & q5).query, query) if __name__ == '__main__': unittest.main() From b23353e37639f241675d0416b4fb054d1a6b7b6a Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 24 May 2010 23:03:30 +0100 Subject: [PATCH 0032/1279] Fixed inherited document primary key issue --- mongoengine/base.py | 9 +++++---- tests/document.py | 3 +++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 2dde93b..c8c162b 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -288,13 +288,14 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Check for custom primary key if field.primary_key: - if not new_class._meta['id_field']: + current_pk = new_class._meta['id_field'] + if current_pk and current_pk != field_name: + raise ValueError('Cannot override primary key field') + + if not current_pk: new_class._meta['id_field'] = field_name # Make 'Document.id' an alias to the real primary key field new_class.id = field - #new_class._fields['id'] = field - else: - raise ValueError('Cannot override primary key field') new_class._meta['unique_indexes'] = unique_indexes diff --git a/tests/document.py b/tests/document.py index a753fa7..8bc907c 100644 --- a/tests/document.py +++ b/tests/document.py @@ -344,6 +344,9 @@ class DocumentTest(unittest.TestCase): class EmailUser(User): email = StringField(primary_key=True) self.assertRaises(ValueError, define_invalid_user) + + class EmailUser(User): + email = StringField() user = User(username='test', name='test user') user.save() From a2c78c9063658d566cfe3d12c12b2d47c9672222 Mon Sep 17 00:00:00 2001 From: Flavio Amieiro Date: Wed, 26 May 2010 20:24:57 -0300 Subject: [PATCH 0033/1279] Add 'exact' and 'iexact' match operators for QuerySets --- mongoengine/fields.py | 4 +++- mongoengine/queryset.py | 5 +++-- tests/queryset.py | 24 ++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 75008d6..127f029 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -51,7 +51,7 @@ class StringField(BaseField): if not isinstance(op, basestring): return value - if op.lstrip('i') in ('startswith', 'endswith', 'contains'): + if op.lstrip('i') in ('startswith', 'endswith', 'contains', 'exact'): flags = 0 if op.startswith('i'): flags = re.IGNORECASE @@ -62,6 +62,8 @@ class StringField(BaseField): regex = r'^%s' elif op == 'endswith': regex = r'%s$' + elif op == 'exact': + regex = r'^%s$' value = re.compile(regex % value, flags) return value diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 72a474d..57600b1 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -307,8 +307,9 @@ class QuerySet(object): """ operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', 'all', 'size', 'exists', 'near'] - match_operators = ['contains', 'icontains', 'startswith', - 'istartswith', 'endswith', 'iendswith'] + match_operators = ['contains', 'icontains', 'startswith', + 'istartswith', 'endswith', 'iendswith', + 'exact', 'iexact'] mongo_query = {} for key, value in query.items(): diff --git a/tests/queryset.py b/tests/queryset.py index 9daa73e..e512cd7 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -265,6 +265,30 @@ class QuerySetTest(unittest.TestCase): obj = self.Person.objects(Q(name__iendswith='rossuM')).first() self.assertEqual(obj, person) + # Test exact + obj = self.Person.objects(name__exact='Guido van Rossum').first() + self.assertEqual(obj, person) + obj = self.Person.objects(name__exact='Guido van rossum').first() + self.assertEqual(obj, None) + obj = self.Person.objects(name__exact='Guido van Rossu').first() + self.assertEqual(obj, None) + obj = self.Person.objects(Q(name__exact='Guido van Rossum')).first() + self.assertEqual(obj, person) + obj = self.Person.objects(Q(name__exact='Guido van rossum')).first() + self.assertEqual(obj, None) + obj = self.Person.objects(Q(name__exact='Guido van Rossu')).first() + self.assertEqual(obj, None) + + # Test iexact + obj = self.Person.objects(name__iexact='gUIDO VAN rOSSUM').first() + self.assertEqual(obj, person) + obj = self.Person.objects(name__iexact='gUIDO VAN rOSSU').first() + self.assertEqual(obj, None) + obj = self.Person.objects(Q(name__iexact='gUIDO VAN rOSSUM')).first() + self.assertEqual(obj, person) + obj = self.Person.objects(Q(name__iexact='gUIDO VAN rOSSU')).first() + self.assertEqual(obj, None) + def test_filter_chaining(self): """Ensure filters can be chained together. """ From 467e61bcc159bda179f913d24d52abd20e292f3f Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Fri, 28 May 2010 02:28:42 +0100 Subject: [PATCH 0034/1279] Documentation fix --- docs/guide/querying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 690569a..5ea110c 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -63,7 +63,7 @@ operator name to a key with a double-underscore:: Available operators are as follows: -* ``neq`` -- not equal to +* ``ne`` -- not equal to * ``lt`` -- less than * ``lte`` -- less than or equal to * ``gt`` -- greater than From 6896818bfd13779a457ccafdd8a9cf7bf0c6bc7b Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 30 May 2010 17:40:01 +0100 Subject: [PATCH 0035/1279] Added docs for exact, iexact --- docs/guide/querying.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 5ea110c..113ee43 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -78,6 +78,8 @@ Available operators are as follows: The following operators are available as shortcuts to querying with regular expressions: +* ``exact`` -- string field exactly matches value +* ``iexact`` -- string field exactly matches value (case insensitive) * ``contains`` -- string field contains value * ``icontains`` -- string field contains value (case insensitive) * ``startswith`` -- string field starts with value From 196606438ccd05589ec25221ddf60346285e9101 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 30 May 2010 18:34:06 +0100 Subject: [PATCH 0036/1279] Fixed Q-object list query issue --- mongoengine/queryset.py | 9 +++++++-- tests/queryset.py | 25 +++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 57600b1..069ab11 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -36,7 +36,9 @@ class Q(object): OR = '||' AND = '&&' OPERATORS = { - 'eq': 'this.%(field)s == %(value)s', + 'eq': ('((this.%(field)s instanceof Array) && ' + ' this.%(field)s.indexOf(%(value)s) != -1) ||' + ' this.%(field)s == %(value)s'), 'ne': 'this.%(field)s != %(value)s', 'gt': 'this.%(field)s > %(value)s', 'gte': 'this.%(field)s >= %(value)s', @@ -62,7 +64,8 @@ class Q(object): if not other.query[0]: return self if self.query[0]: - obj.query = ['('] + copy.deepcopy(self.query) + [op] + copy.deepcopy(other.query) + [')'] + obj.query = (['('] + copy.deepcopy(self.query) + [op] + + copy.deepcopy(other.query) + [')']) else: obj.query = copy.deepcopy(other.query) return obj @@ -111,11 +114,13 @@ class Q(object): value, field_js = self._build_op_js(op, key, value, value_name) js_scope[value_name] = value js.append(field_js) + print ' && '.join(js) return ' && '.join(js) def _build_op_js(self, op, key, value, value_name): """Substitute the values in to the correct chunk of Javascript. """ + print op, key, value, value_name if isinstance(value, RE_TYPE): # Regexes are handled specially if op.strip('$') == 'ne': diff --git a/tests/queryset.py b/tests/queryset.py index e512cd7..51f9299 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -503,6 +503,22 @@ class QuerySetTest(unittest.TestCase): obj = self.Person.objects(Q(name__ne=re.compile('^Gui'))).first() self.assertEqual(obj, None) + def test_q_lists(self): + """Ensure that Q objects query ListFields correctly. + """ + class BlogPost(Document): + tags = ListField(StringField()) + + BlogPost.drop_collection() + + BlogPost(tags=['python', 'mongo']).save() + BlogPost(tags=['python']).save() + + self.assertEqual(len(BlogPost.objects(Q(tags='mongo'))), 1) + self.assertEqual(len(BlogPost.objects(Q(tags='python'))), 2) + + BlogPost.drop_collection() + def test_exec_js_query(self): """Ensure that queries are properly formed for use in exec_js. """ @@ -1172,10 +1188,15 @@ class QTest(unittest.TestCase): """ q = Q() examples = [ - ({'name': 'test'}, 'this.name == i0f0', {'i0f0': 'test'}), + + ({'name': 'test'}, ('((this.name instanceof Array) && ' + 'this.name.indexOf(i0f0) != -1) || this.name == i0f0'), + {'i0f0': 'test'}), ({'age': {'$gt': 18}}, 'this.age > i0f0o0', {'i0f0o0': 18}), ({'name': 'test', 'age': {'$gt': 18, '$lte': 65}}, - 'this.age <= i0f0o0 && this.age > i0f0o1 && this.name == i0f1', + ('this.age <= i0f0o0 && this.age > i0f0o1 && ' + '((this.name instanceof Array) && ' + 'this.name.indexOf(i0f1) != -1) || this.name == i0f1'), {'i0f0o0': 65, 'i0f0o1': 18, 'i0f1': 'test'}), ] for item, js, scope in examples: From 0ad343484f932722bdbe5b00b68e2659b8c0ddc4 Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Wed, 2 Jun 2010 20:53:39 +0100 Subject: [PATCH 0037/1279] Added new FileField with GridFS support The API is similar to that of PyMongo and most of the same operations are possible. The FileField can be written too with put(), write() or by using the assignment operator. All three cases are demonstrated in the tests. Metadata can be added to a FileField by assigning keyword arguments when using put() or new_file(). --- docs/apireference.rst | 2 + mongoengine/fields.py | 89 ++++++++++++++++++++++++++++++++++++++++++- tests/fields.py | 56 +++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 1 deletion(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 267b22a..4fff317 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -64,3 +64,5 @@ Fields .. autoclass:: mongoengine.ReferenceField .. autoclass:: mongoengine.GenericReferenceField + +.. autoclass:: mongoengine.FileField diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 127f029..7d9c47f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -7,12 +7,15 @@ import re import pymongo import datetime import decimal +import gridfs +import warnings +import types __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', - 'DecimalField', 'URLField', 'GenericReferenceField', + 'DecimalField', 'URLField', 'GenericReferenceField', 'FileField', 'BinaryField', 'SortedListField', 'EmailField', 'GeoLocationField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -520,3 +523,87 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') + +class GridFSProxy(object): + """Proxy object to handle writing and reading of files to and from GridFS + """ + + def __init__(self): + self.fs = gridfs.GridFS(_get_db()) # Filesystem instance + self.newfile = None # Used for partial writes + self.grid_id = None # Store GridFS id for file + + def __getattr__(self, name): + obj = self.fs.get(self.grid_id) + if name in dir(obj): + return getattr(obj, name) + + def __get__(self, instance, value): + return self + + def new_file(self, **kwargs): + self.newfile = self.fs.new_file(**kwargs) + self.grid_id = self.newfile._id + + def put(self, file, **kwargs): + self.grid_id = self.fs.put(file, **kwargs) + + def write(self, string): + if not self.newfile: + self.new_file() + self.grid_id = self.newfile._id + self.newfile.write(string) + + def writelines(self, lines): + if not self.newfile: + self.new_file() + self.grid_id = self.newfile._id + self.newfile.writelines(lines) + + def read(self): + return self.fs.get(self.grid_id).read() + + def delete(self): + # Delete file from GridFS + self.fs.delete(self.grid_id) + + def close(self): + if self.newfile: + self.newfile.close() + else: + msg = "The close() method is only necessary after calling write()" + warnings.warn(msg) + +class FileField(BaseField): + """A GridFS storage field. + """ + + def __init__(self, **kwargs): + self.gridfs = GridFSProxy() + super(FileField, self).__init__(**kwargs) + + def __get__(self, instance, owner): + if instance is None: + return self + + return self.gridfs + + def __set__(self, instance, value): + if isinstance(value, file) or isinstance(value, str): + # using "FileField() = file/string" notation + self.gridfs.put(value) + else: + instance._data[self.name] = value + + def to_mongo(self, value): + # Store the GridFS file id in MongoDB + return self.gridfs.grid_id + + def to_python(self, value): + # Use stored value (id) to lookup file in GridFS + return self.gridfs.fs.get(value) + + def validate(self, value): + assert isinstance(value, GridFSProxy) + assert isinstance(value.grid_id, pymongo.objectid.ObjectId) + diff --git a/tests/fields.py b/tests/fields.py index 4050e26..e22e6dd 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -3,6 +3,7 @@ import datetime from decimal import Decimal import pymongo +import gridfs from mongoengine import * from mongoengine.connection import _get_db @@ -607,6 +608,61 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() + def test_file_fields(self): + """Ensure that file fields can be written to and their data retrieved + """ + class PutFile(Document): + file = FileField() + + class StreamFile(Document): + file = FileField() + + class SetFile(Document): + file = FileField() + + text = 'Hello, World!' + more_text = 'Foo Bar' + content_type = 'text/plain' + + PutFile.drop_collection() + StreamFile.drop_collection() + SetFile.drop_collection() + + putfile = PutFile() + putfile.file.put(text, content_type=content_type) + putfile.save() + putfile.validate() + result = PutFile.objects.first() + self.assertTrue(putfile == result) + self.assertEquals(result.file.read(), text) + self.assertEquals(result.file.content_type, content_type) + result.file.delete() # Remove file from GridFS + + streamfile = StreamFile() + streamfile.file.new_file(content_type=content_type) + streamfile.file.write(text) + streamfile.file.write(more_text) + streamfile.file.close() + streamfile.save() + streamfile.validate() + result = StreamFile.objects.first() + self.assertTrue(streamfile == result) + self.assertEquals(result.file.read(), text + more_text) + self.assertEquals(result.file.content_type, content_type) + result.file.delete() # Remove file from GridFS + + setfile = SetFile() + setfile.file = text + setfile.save() + setfile.validate() + result = SetFile.objects.first() + self.assertTrue(setfile == result) + self.assertEquals(result.file.read(), text) + result.file.delete() # Remove file from GridFS + + PutFile.drop_collection() + StreamFile.drop_collection() + SetFile.drop_collection() if __name__ == '__main__': From 39b749432a7fe11a899ad3602038217c6301a8dd Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Thu, 3 Jun 2010 08:27:21 +0100 Subject: [PATCH 0038/1279] Tidied code, added replace() method to FileField --- docs/guide/defining-documents.rst | 6 ++++++ mongoengine/base.py | 10 ++++++---- mongoengine/fields.py | 20 +++++++++++++++----- tests/fields.py | 15 +++++++++++++-- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 3c27686..7b8dcd5 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -46,6 +46,12 @@ are as follows: * :class:`~mongoengine.EmbeddedDocumentField` * :class:`~mongoengine.ReferenceField` * :class:`~mongoengine.GenericReferenceField` +* :class:`~mongoengine.BooleanField` +* :class:`~mongoengine.GeoLocationField` +* :class:`~mongoengine.FileField` +* :class:`~mongoengine.EmailField` +* :class:`~mongoengine.SortedListField` +* :class:`~mongoengine.BinaryField` Field arguments --------------- diff --git a/mongoengine/base.py b/mongoengine/base.py index c8c162b..b6d5a63 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -24,8 +24,8 @@ class BaseField(object): _index_with_types = True def __init__(self, db_field=None, name=None, required=False, default=None, - unique=False, unique_with=None, primary_key=False, validation=None, - choices=None): + unique=False, unique_with=None, primary_key=False, + validation=None, choices=None): self.db_field = (db_field or name) if not primary_key else '_id' if name: import warnings @@ -86,13 +86,15 @@ class BaseField(object): # check choices if self.choices is not None: if value not in self.choices: - raise ValidationError("Value must be one of %s."%unicode(self.choices)) + raise ValidationError("Value must be one of %s." + % unicode(self.choices)) # check validation argument if self.validation is not None: if callable(self.validation): if not self.validation(value): - raise ValidationError('Value does not match custom validation method.') + raise ValidationError('Value does not match custom' \ + 'validation method.') else: raise ValueError('validation argument must be a callable.') diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 7d9c47f..a276399 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -530,17 +530,21 @@ class GridFSProxy(object): def __init__(self): self.fs = gridfs.GridFS(_get_db()) # Filesystem instance - self.newfile = None # Used for partial writes + self.newfile = None # Used for partial writes self.grid_id = None # Store GridFS id for file def __getattr__(self, name): - obj = self.fs.get(self.grid_id) + obj = self.get() if name in dir(obj): return getattr(obj, name) def __get__(self, instance, value): return self + def get(self, id=None): + try: return self.fs.get(id or self.grid_id) + except: return None # File has been deleted + def new_file(self, **kwargs): self.newfile = self.fs.new_file(**kwargs) self.grid_id = self.newfile._id @@ -561,11 +565,17 @@ class GridFSProxy(object): self.newfile.writelines(lines) def read(self): - return self.fs.get(self.grid_id).read() + try: return self.get().read() + except: return None def delete(self): - # Delete file from GridFS + # Delete file from GridFS, FileField still remains self.fs.delete(self.grid_id) + self.grid_id = None + + def replace(self, file, **kwargs): + self.delete() + self.put(file, **kwargs) def close(self): if self.newfile: @@ -601,7 +611,7 @@ class FileField(BaseField): def to_python(self, value): # Use stored value (id) to lookup file in GridFS - return self.gridfs.fs.get(value) + return self.gridfs.get() def validate(self, value): assert isinstance(value, GridFSProxy) diff --git a/tests/fields.py b/tests/fields.py index e22e6dd..8dddcb3 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -649,7 +649,10 @@ class FieldTest(unittest.TestCase): self.assertTrue(streamfile == result) self.assertEquals(result.file.read(), text + more_text) self.assertEquals(result.file.content_type, content_type) - result.file.delete() # Remove file from GridFS + result.file.delete() + + # Ensure deleted file returns None + self.assertTrue(result.file.read() == None) setfile = SetFile() setfile.file = text @@ -658,7 +661,15 @@ class FieldTest(unittest.TestCase): result = SetFile.objects.first() self.assertTrue(setfile == result) self.assertEquals(result.file.read(), text) - result.file.delete() # Remove file from GridFS + + # Try replacing file with new one + result.file.replace(more_text) + result.save() + result.validate() + result = SetFile.objects.first() + self.assertTrue(setfile == result) + self.assertEquals(result.file.read(), more_text) + result.file.delete() PutFile.drop_collection() StreamFile.drop_collection() From 86e2797c577d007b1883164867eaca7548703ea9 Mon Sep 17 00:00:00 2001 From: vandersonmota Date: Wed, 9 Jun 2010 22:28:30 -0300 Subject: [PATCH 0039/1279] added a TestCase for tests that uses mongoDB --- mongoengine/django/tests.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 mongoengine/django/tests.py diff --git a/mongoengine/django/tests.py b/mongoengine/django/tests.py new file mode 100644 index 0000000..a8d7c7f --- /dev/null +++ b/mongoengine/django/tests.py @@ -0,0 +1,21 @@ +#coding: utf-8 +from django.test import TestCase +from django.conf import settings + +from mongoengine import connect + +class MongoTestCase(TestCase): + """ + TestCase class that clear the collection between the tests + """ + db_name = 'test_%s' % settings.MONGO_DATABASE_NAME + def __init__(self, methodName='runtest'): + self.db = connect(self.db_name) + super(MongoTestCase, self).__init__(methodName) + + def _post_teardown(self): + super(MongoTestCase, self)._post_teardown() + for collection in self.db.collection_names(): + if collection == 'system.indexes': + continue + self.db.drop_collection(collection) From f5e39c0064e52c03194e09a72c204c409a9ab5b2 Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Tue, 6 Jul 2010 10:25:31 +0200 Subject: [PATCH 0040/1279] Allowed _id to be missing when converting to mongo --- mongoengine/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index c8c162b..fe942b1 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -407,11 +407,15 @@ class BaseDocument(object): value = getattr(self, field_name, None) if value is not None: data[field.db_field] = field.to_mongo(value) + else: + data[field.db_field] = None # Only add _cls and _types if allow_inheritance is not False if not (hasattr(self, '_meta') and self._meta.get('allow_inheritance', True) == False): data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] + if hasattr(self, '_id') and not data['_id']: + del data['_id'] return data @classmethod From 3179c4e4aca20cdaa3daafb7cbc1204d847e2e2f Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Tue, 6 Jul 2010 11:25:49 +0200 Subject: [PATCH 0041/1279] Now only removes _id if none, for real --- mongoengine/base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index fe942b1..253c758 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -414,8 +414,11 @@ class BaseDocument(object): self._meta.get('allow_inheritance', True) == False): data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] - if hasattr(self, '_id') and not data['_id']: - del data['_id'] + try: + if not data['_id']: + del data['_id'] + except KeyError: + pass return data @classmethod From b89d71bfa5ffea3d3aff6c23ab0be4bcfefc930e Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Tue, 6 Jul 2010 14:17:30 +0200 Subject: [PATCH 0042/1279] Do not convert None objects --- mongoengine/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 253c758..8137edd 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -450,7 +450,8 @@ class BaseDocument(object): for field_name, field in cls._fields.items(): if field.db_field in data: - data[field_name] = field.to_python(data[field.db_field]) + value = data[field.db_field] + data[field_name] = value if value is None else field.to_python(value) obj = cls(**data) obj._present_fields = present_fields From 71689fcf234ec3cc9349850de4e4afd7af574650 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Wed, 7 Jul 2010 15:00:46 +0100 Subject: [PATCH 0043/1279] Got within_box working for Geo fields --- docs/changelog.rst | 14 ++++++++++++++ mongoengine/queryset.py | 2 ++ tests/queryset.py | 6 ++++++ 3 files changed, 22 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 479ea21..6f2d6f1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,20 @@ Changelog ========= +Changes in v0.4 +=============== +- Added ``SortedListField`` +- Added ``EmailField`` +- Added ``GeoLocationField`` +- Added ``exact`` and ``iexact`` match operators to ``QuerySet`` +- Added ``get_document_or_404`` and ``get_list_or_404`` Django shortcuts +- Fixed bug in Q-objects +- Fixed document inheritance primary key issue +- Base class can now be defined for ``DictField`` +- Fixed MRO error that occured on document inheritance +- Introduced ``min_length`` for ``StringField`` +- Other minor fixes + Changes in v0.3 =============== - Added MapReduce support diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 5a837c4..4840b53 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -351,6 +351,8 @@ class QuerySet(object): value = {'$within': {'$center': value}} elif op == "near": value = {'$near': value} + elif op == 'within_box': + value = {'$within': {'$box': value}} else: raise NotImplementedError("Geo method '%s' has not " "been implemented" % op) diff --git a/tests/queryset.py b/tests/queryset.py index a7719b8..a84c8c5 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1232,6 +1232,12 @@ class QuerySetTest(unittest.TestCase): events = events.order_by("-date") self.assertEqual(events.count(), 2) self.assertEqual(events[0], event3) + + # check that within_box works + box = [(35.0, -125.0), (40.0, -100.0)] + events = Event.objects(location__within_box=box) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0].id, event2.id) Event.drop_collection() From c2163ecee5674e4379d6db7ad6b46254e59e2ee4 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Wed, 7 Jul 2010 15:12:14 +0100 Subject: [PATCH 0044/1279] Added test for Geo indexes --- docs/changelog.rst | 2 +- tests/fields.py | 18 ++++++++++++++++++ tests/queryset.py | 2 -- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6f2d6f1..8dd5b00 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,7 @@ Changes in v0.4 =============== - Added ``SortedListField`` - Added ``EmailField`` -- Added ``GeoLocationField`` +- Added ``GeoPointField`` - Added ``exact`` and ``iexact`` match operators to ``QuerySet`` - Added ``get_document_or_404`` and ``get_list_or_404`` Django shortcuts - Fixed bug in Q-objects diff --git a/tests/fields.py b/tests/fields.py index 4050e26..80ce3b6 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -607,6 +607,24 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() + def test_geo_indexes(self): + """Ensure that indexes are created automatically for GeoPointFields. + """ + class Event(Document): + title = StringField() + location = GeoPointField() + + Event.drop_collection() + event = Event(title="Coltrane Motion @ Double Door", + location=[41.909889, -87.677137]) + event.save() + + info = Event.objects._collection.index_information() + self.assertTrue(u'location_2d' in info) + self.assertTrue(info[u'location_2d'] == [(u'location', u'2d')]) + + Event.drop_collection() + if __name__ == '__main__': diff --git a/tests/queryset.py b/tests/queryset.py index a84c8c5..4187d55 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1175,8 +1175,6 @@ class QuerySetTest(unittest.TestCase): def __unicode__(self): return self.title - meta = {'geo_indexes': ["location"]} - Event.drop_collection() event1 = Event(title="Coltrane Motion @ Double Door", From acbc741037c00736ce1dbdcd39bd77ddf6f663b3 Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Thu, 15 Jul 2010 18:20:29 +0200 Subject: [PATCH 0045/1279] Made list store empty list by default --- mongoengine/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 127f029..6bfcff9 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -261,6 +261,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field + kwargs.setdefault("default", []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): From 0512dd4c25e0f8395c317725fa149a64025faf3e Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Thu, 3 Jun 2010 03:53:39 +0800 Subject: [PATCH 0046/1279] Added new FileField with GridFS support The API is similar to that of PyMongo and most of the same operations are possible. The FileField can be written too with put(), write() or by using the assignment operator. All three cases are demonstrated in the tests. Metadata can be added to a FileField by assigning keyword arguments when using put() or new_file(). --- docs/apireference.rst | 2 + mongoengine/fields.py | 89 ++++++++++++++++++++++++++++++++++++++++++- tests/fields.py | 56 +++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 1 deletion(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 267b22a..4fff317 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -64,3 +64,5 @@ Fields .. autoclass:: mongoengine.ReferenceField .. autoclass:: mongoengine.GenericReferenceField + +.. autoclass:: mongoengine.FileField diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 127f029..7d9c47f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -7,12 +7,15 @@ import re import pymongo import datetime import decimal +import gridfs +import warnings +import types __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', - 'DecimalField', 'URLField', 'GenericReferenceField', + 'DecimalField', 'URLField', 'GenericReferenceField', 'FileField', 'BinaryField', 'SortedListField', 'EmailField', 'GeoLocationField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -520,3 +523,87 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') + +class GridFSProxy(object): + """Proxy object to handle writing and reading of files to and from GridFS + """ + + def __init__(self): + self.fs = gridfs.GridFS(_get_db()) # Filesystem instance + self.newfile = None # Used for partial writes + self.grid_id = None # Store GridFS id for file + + def __getattr__(self, name): + obj = self.fs.get(self.grid_id) + if name in dir(obj): + return getattr(obj, name) + + def __get__(self, instance, value): + return self + + def new_file(self, **kwargs): + self.newfile = self.fs.new_file(**kwargs) + self.grid_id = self.newfile._id + + def put(self, file, **kwargs): + self.grid_id = self.fs.put(file, **kwargs) + + def write(self, string): + if not self.newfile: + self.new_file() + self.grid_id = self.newfile._id + self.newfile.write(string) + + def writelines(self, lines): + if not self.newfile: + self.new_file() + self.grid_id = self.newfile._id + self.newfile.writelines(lines) + + def read(self): + return self.fs.get(self.grid_id).read() + + def delete(self): + # Delete file from GridFS + self.fs.delete(self.grid_id) + + def close(self): + if self.newfile: + self.newfile.close() + else: + msg = "The close() method is only necessary after calling write()" + warnings.warn(msg) + +class FileField(BaseField): + """A GridFS storage field. + """ + + def __init__(self, **kwargs): + self.gridfs = GridFSProxy() + super(FileField, self).__init__(**kwargs) + + def __get__(self, instance, owner): + if instance is None: + return self + + return self.gridfs + + def __set__(self, instance, value): + if isinstance(value, file) or isinstance(value, str): + # using "FileField() = file/string" notation + self.gridfs.put(value) + else: + instance._data[self.name] = value + + def to_mongo(self, value): + # Store the GridFS file id in MongoDB + return self.gridfs.grid_id + + def to_python(self, value): + # Use stored value (id) to lookup file in GridFS + return self.gridfs.fs.get(value) + + def validate(self, value): + assert isinstance(value, GridFSProxy) + assert isinstance(value.grid_id, pymongo.objectid.ObjectId) + diff --git a/tests/fields.py b/tests/fields.py index 4050e26..e22e6dd 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -3,6 +3,7 @@ import datetime from decimal import Decimal import pymongo +import gridfs from mongoengine import * from mongoengine.connection import _get_db @@ -607,6 +608,61 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() + def test_file_fields(self): + """Ensure that file fields can be written to and their data retrieved + """ + class PutFile(Document): + file = FileField() + + class StreamFile(Document): + file = FileField() + + class SetFile(Document): + file = FileField() + + text = 'Hello, World!' + more_text = 'Foo Bar' + content_type = 'text/plain' + + PutFile.drop_collection() + StreamFile.drop_collection() + SetFile.drop_collection() + + putfile = PutFile() + putfile.file.put(text, content_type=content_type) + putfile.save() + putfile.validate() + result = PutFile.objects.first() + self.assertTrue(putfile == result) + self.assertEquals(result.file.read(), text) + self.assertEquals(result.file.content_type, content_type) + result.file.delete() # Remove file from GridFS + + streamfile = StreamFile() + streamfile.file.new_file(content_type=content_type) + streamfile.file.write(text) + streamfile.file.write(more_text) + streamfile.file.close() + streamfile.save() + streamfile.validate() + result = StreamFile.objects.first() + self.assertTrue(streamfile == result) + self.assertEquals(result.file.read(), text + more_text) + self.assertEquals(result.file.content_type, content_type) + result.file.delete() # Remove file from GridFS + + setfile = SetFile() + setfile.file = text + setfile.save() + setfile.validate() + result = SetFile.objects.first() + self.assertTrue(setfile == result) + self.assertEquals(result.file.read(), text) + result.file.delete() # Remove file from GridFS + + PutFile.drop_collection() + StreamFile.drop_collection() + SetFile.drop_collection() if __name__ == '__main__': From 6bfd6c322b166d55eb151de33e9eaee7ca214b8a Mon Sep 17 00:00:00 2001 From: martin Date: Fri, 18 Jun 2010 10:41:23 +0800 Subject: [PATCH 0047/1279] Fixed bug with GeoLocationField --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 7d9c47f..5893049 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -388,7 +388,7 @@ class GeoLocationField(DictField): return {'x': value[0], 'y': value[1]} def to_python(self, value): - return value.keys() + return (value['x'], value['y']) class ReferenceField(BaseField): """A reference to a document that will be automatically dereferenced on From 47bfeec115a297c1a859543e842797751ab9c14a Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Thu, 3 Jun 2010 15:27:21 +0800 Subject: [PATCH 0048/1279] Tidied code, added replace() method to FileField --- docs/guide/defining-documents.rst | 6 ++++++ mongoengine/base.py | 10 ++++++---- mongoengine/fields.py | 20 +++++++++++++++----- tests/fields.py | 15 +++++++++++++-- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 3c27686..7b8dcd5 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -46,6 +46,12 @@ are as follows: * :class:`~mongoengine.EmbeddedDocumentField` * :class:`~mongoengine.ReferenceField` * :class:`~mongoengine.GenericReferenceField` +* :class:`~mongoengine.BooleanField` +* :class:`~mongoengine.GeoLocationField` +* :class:`~mongoengine.FileField` +* :class:`~mongoengine.EmailField` +* :class:`~mongoengine.SortedListField` +* :class:`~mongoengine.BinaryField` Field arguments --------------- diff --git a/mongoengine/base.py b/mongoengine/base.py index c8c162b..b6d5a63 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -24,8 +24,8 @@ class BaseField(object): _index_with_types = True def __init__(self, db_field=None, name=None, required=False, default=None, - unique=False, unique_with=None, primary_key=False, validation=None, - choices=None): + unique=False, unique_with=None, primary_key=False, + validation=None, choices=None): self.db_field = (db_field or name) if not primary_key else '_id' if name: import warnings @@ -86,13 +86,15 @@ class BaseField(object): # check choices if self.choices is not None: if value not in self.choices: - raise ValidationError("Value must be one of %s."%unicode(self.choices)) + raise ValidationError("Value must be one of %s." + % unicode(self.choices)) # check validation argument if self.validation is not None: if callable(self.validation): if not self.validation(value): - raise ValidationError('Value does not match custom validation method.') + raise ValidationError('Value does not match custom' \ + 'validation method.') else: raise ValueError('validation argument must be a callable.') diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 5893049..ebefbb7 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -530,17 +530,21 @@ class GridFSProxy(object): def __init__(self): self.fs = gridfs.GridFS(_get_db()) # Filesystem instance - self.newfile = None # Used for partial writes + self.newfile = None # Used for partial writes self.grid_id = None # Store GridFS id for file def __getattr__(self, name): - obj = self.fs.get(self.grid_id) + obj = self.get() if name in dir(obj): return getattr(obj, name) def __get__(self, instance, value): return self + def get(self, id=None): + try: return self.fs.get(id or self.grid_id) + except: return None # File has been deleted + def new_file(self, **kwargs): self.newfile = self.fs.new_file(**kwargs) self.grid_id = self.newfile._id @@ -561,11 +565,17 @@ class GridFSProxy(object): self.newfile.writelines(lines) def read(self): - return self.fs.get(self.grid_id).read() + try: return self.get().read() + except: return None def delete(self): - # Delete file from GridFS + # Delete file from GridFS, FileField still remains self.fs.delete(self.grid_id) + self.grid_id = None + + def replace(self, file, **kwargs): + self.delete() + self.put(file, **kwargs) def close(self): if self.newfile: @@ -601,7 +611,7 @@ class FileField(BaseField): def to_python(self, value): # Use stored value (id) to lookup file in GridFS - return self.gridfs.fs.get(value) + return self.gridfs.get() def validate(self, value): assert isinstance(value, GridFSProxy) diff --git a/tests/fields.py b/tests/fields.py index e22e6dd..8dddcb3 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -649,7 +649,10 @@ class FieldTest(unittest.TestCase): self.assertTrue(streamfile == result) self.assertEquals(result.file.read(), text + more_text) self.assertEquals(result.file.content_type, content_type) - result.file.delete() # Remove file from GridFS + result.file.delete() + + # Ensure deleted file returns None + self.assertTrue(result.file.read() == None) setfile = SetFile() setfile.file = text @@ -658,7 +661,15 @@ class FieldTest(unittest.TestCase): result = SetFile.objects.first() self.assertTrue(setfile == result) self.assertEquals(result.file.read(), text) - result.file.delete() # Remove file from GridFS + + # Try replacing file with new one + result.file.replace(more_text) + result.save() + result.validate() + result = SetFile.objects.first() + self.assertTrue(setfile == result) + self.assertEquals(result.file.read(), more_text) + result.file.delete() PutFile.drop_collection() StreamFile.drop_collection() From 9596a25bb92212806e98fd77b6ced1b79c2ed1bf Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 19 Jul 2010 00:56:16 +0200 Subject: [PATCH 0049/1279] Fixed documentation bug. --- docs/guide/querying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 113ee43..1fd2ed5 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -174,7 +174,7 @@ custom manager methods as you like:: @queryset_manager def live_posts(doc_cls, queryset): - return queryset(published=True).filter(published=True) + return queryset.filter(published=True) BlogPost(title='test1', published=False).save() BlogPost(title='test2', published=True).save() From f9057e1a288dea4b99ac1936a75696071476870d Mon Sep 17 00:00:00 2001 From: martin Date: Thu, 24 Jun 2010 00:56:51 +0800 Subject: [PATCH 0050/1279] Fixed bug in FileField, proxy was not getting the grid_id set --- mongoengine/fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index ebefbb7..d2b41ab 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -542,6 +542,7 @@ class GridFSProxy(object): return self def get(self, id=None): + if id: self.grid_id = id try: return self.fs.get(id or self.grid_id) except: return None # File has been deleted @@ -611,7 +612,7 @@ class FileField(BaseField): def to_python(self, value): # Use stored value (id) to lookup file in GridFS - return self.gridfs.get() + return self.gridfs.get(id=value) def validate(self, value): assert isinstance(value, GridFSProxy) From ec519f20fa8e1c769ffb6ae766dd00bb36f9eabc Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 19 Jul 2010 01:32:28 +0200 Subject: [PATCH 0051/1279] Makes the tests compatible to pymongo 1.7+. Not backwards compatible! --- tests/document.py | 14 ++++++++------ tests/fields.py | 2 +- tests/queryset.py | 5 +++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/document.py b/tests/document.py index 8bc907c..1160b35 100644 --- a/tests/document.py +++ b/tests/document.py @@ -264,11 +264,12 @@ class DocumentTest(unittest.TestCase): # Indexes are lazy so use list() to perform query list(BlogPost.objects) info = BlogPost.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] - in info.values()) - self.assertTrue([('_types', 1), ('addDate', -1)] in info.values()) + in info) + self.assertTrue([('_types', 1), ('addDate', -1)] in info) # tags is a list field so it shouldn't have _types in the index - self.assertTrue([('tags', 1)] in info.values()) + self.assertTrue([('tags', 1)] in info) class ExtendedBlogPost(BlogPost): title = StringField() @@ -278,10 +279,11 @@ class DocumentTest(unittest.TestCase): list(ExtendedBlogPost.objects) info = ExtendedBlogPost.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] - in info.values()) - self.assertTrue([('_types', 1), ('addDate', -1)] in info.values()) - self.assertTrue([('_types', 1), ('title', 1)] in info.values()) + in info) + self.assertTrue([('_types', 1), ('addDate', -1)] in info) + self.assertTrue([('_types', 1), ('title', 1)] in info) BlogPost.drop_collection() diff --git a/tests/fields.py b/tests/fields.py index d95f4d3..136437b 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -689,7 +689,7 @@ class FieldTest(unittest.TestCase): info = Event.objects._collection.index_information() self.assertTrue(u'location_2d' in info) - self.assertTrue(info[u'location_2d'] == [(u'location', u'2d')]) + self.assertTrue(info[u'location_2d']['key'] == [(u'location', u'2d')]) Event.drop_collection() diff --git a/tests/queryset.py b/tests/queryset.py index 4187d55..0424d32 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1087,8 +1087,9 @@ class QuerySetTest(unittest.TestCase): # Indexes are lazy so use list() to perform query list(BlogPost.objects) info = BlogPost.objects._collection.index_information() - self.assertTrue([('_types', 1)] in info.values()) - self.assertTrue([('_types', 1), ('date', -1)] in info.values()) + info = [value['key'] for key, value in info.iteritems()] + self.assertTrue([('_types', 1)] in info) + self.assertTrue([('_types', 1), ('date', -1)] in info) BlogPost.drop_collection() From 6093e88eebf932d04741364f53de77878198c1f1 Mon Sep 17 00:00:00 2001 From: Daniel Hasselrot Date: Fri, 16 Jul 2010 00:20:29 +0800 Subject: [PATCH 0052/1279] Made list store empty list by default --- mongoengine/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 24c5b56..8e8a97c 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -263,6 +263,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field + kwargs.setdefault("default", []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): From 03c0fd9ada5eb1c90a9213fe4579e64d9a6a111c Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 19 Jul 2010 19:01:53 +0200 Subject: [PATCH 0053/1279] Make default value of DictField an empty dict instead of None. --- mongoengine/fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 8e8a97c..759697a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -263,7 +263,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field - kwargs.setdefault("default", []) + kwargs.setdefault('default', []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): @@ -356,6 +356,7 @@ class DictField(BaseField): def __init__(self, basecls=None, *args, **kwargs): self.basecls = basecls or BaseField assert issubclass(self.basecls, BaseField) + kwargs.setdefault('default', {}) super(DictField, self).__init__(*args, **kwargs) def validate(self, value): From aa00feb6a55eba7f720969828242436103817e64 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Tue, 20 Jul 2010 22:46:00 +0200 Subject: [PATCH 0054/1279] FileField's values are now optional. When no value is applied, no File object is created and referenced. --- mongoengine/base.py | 4 ++-- mongoengine/fields.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 806d83b..086c787 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -339,8 +339,8 @@ class BaseDocument(object): try: field._validate(value) except (ValueError, AttributeError, AssertionError), e: - raise ValidationError('Invalid value for field of type "' + - field.__class__.__name__ + '"') + raise ValidationError('Invalid value for field of type "%s": %s' + % (field.__class__.__name__, value)) elif field.required: raise ValidationError('Field "%s" is required' % field.name) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 759697a..f84f751 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -591,15 +591,20 @@ class FileField(BaseField): def to_mongo(self, value): # Store the GridFS file id in MongoDB - return self.gridfs.grid_id + if self.gridfs.grid_id is not None: + return self.gridfs.grid_id + return None def to_python(self, value): # Use stored value (id) to lookup file in GridFS - return self.gridfs.get(id=value) + if self.gridfs.grid_id is not None: + return self.gridfs.get(id=value) + return None def validate(self, value): - assert isinstance(value, GridFSProxy) - assert isinstance(value.grid_id, pymongo.objectid.ObjectId) + if value.grid_id is not None: + assert isinstance(value, GridFSProxy) + assert isinstance(value.grid_id, pymongo.objectid.ObjectId) class GeoPointField(BaseField): """A list storing a latitude and longitude. From be651caa688c31b434d641c270e1bf073b3c576a Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 25 Jul 2010 15:02:37 +0100 Subject: [PATCH 0055/1279] Removed a couple of sneaky print statements --- mongoengine/queryset.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 4840b53..f81adb5 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -114,13 +114,11 @@ class Q(object): value, field_js = self._build_op_js(op, key, value, value_name) js_scope[value_name] = value js.append(field_js) - print ' && '.join(js) return ' && '.join(js) def _build_op_js(self, op, key, value, value_name): """Substitute the values in to the correct chunk of Javascript. """ - print op, key, value, value_name if isinstance(value, RE_TYPE): # Regexes are handled specially if op.strip('$') == 'ne': From 9f98025b8c13d9e845f5de68e29e825f5517b245 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 25 Jul 2010 15:29:02 +0100 Subject: [PATCH 0056/1279] Added QuerySet.distinct. Closes #44. --- mongoengine/queryset.py | 9 +++++++++ tests/queryset.py | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index f81adb5..1e42cd1 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -604,6 +604,15 @@ class QuerySet(object): # Integer index provided elif isinstance(key, int): return self._document._from_son(self._cursor[key]) + + def distinct(self, field): + """Return a list of distinct values for a given field. + + :param field: the field to select distinct values from + + .. versionadded:: 0.4 + """ + return self._collection.distinct(field) def only(self, *fields): """Load only a subset of this document's fields. :: diff --git a/tests/queryset.py b/tests/queryset.py index 0424d32..3691d89 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -970,6 +970,16 @@ class QuerySetTest(unittest.TestCase): self.Person(name='ageless person').save() self.assertEqual(int(self.Person.objects.sum('age')), sum(ages)) + def test_distinct(self): + """Ensure that the QuerySet.distinct method works. + """ + self.Person(name='Mr Orange', age=20).save() + self.Person(name='Mr White', age=20).save() + self.Person(name='Mr Orange', age=30).save() + self.assertEqual(self.Person.objects.distinct('name'), + ['Mr Orange', 'Mr White']) + self.assertEqual(self.Person.objects.distinct('age'), [20, 30]) + def test_custom_manager(self): """Ensure that custom QuerySetManager instances work as expected. """ From 13316e5380f9886b0d8c65458096311b8382233a Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 17:35:09 +0200 Subject: [PATCH 0057/1279] Introduced new Document.objects.create, like django has. It creates a new object, saves it and returns the new object instance. --- mongoengine/queryset.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 1e42cd1..e176c54 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1,5 +1,6 @@ from connection import _get_db +import pprint import pymongo import re import copy @@ -414,6 +415,14 @@ class QuerySet(object): message = u'%d items returned, instead of 1' % count raise self._document.MultipleObjectsReturned(message) + def create(self, **kwargs): + """Create new object. Returns the saved object instance. + .. versionadded:: 0.4 + """ + doc = self._document(**kwargs) + doc.save() + return doc + def first(self): """Retrieve the first object matching the query. """ @@ -667,7 +676,6 @@ class QuerySet(object): plan = self._cursor.explain() if format: - import pprint plan = pprint.pformat(plan) return plan From 327452622e3081c81b05916c61454a9fd97d992d Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:22:26 +0200 Subject: [PATCH 0058/1279] Handle DBRefs correctly within Q objects. Closes #55 --- mongoengine/queryset.py | 11 +++++++++++ tests/queryset.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index e176c54..00a7f7a 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -133,6 +133,17 @@ class Q(object): if isinstance(value, pymongo.objectid.ObjectId): value = unicode(value) + # Handle DBRef + if isinstance(value, pymongo.dbref.DBRef): + # this.created_user.$id == "4c4c56f8cc1831418c000000" + op_js = '(this.%(field)s.$id == "%(id)s" &&'\ + ' this.%(field)s.$ref == "%(ref)s")' % { + 'field': key, + 'id': unicode(value.id), + 'ref': unicode(value.collection) + } + value = None + # Perform the substitution operation_js = op_js % { 'field': key, diff --git a/tests/queryset.py b/tests/queryset.py index 3691d89..8cbd9a4 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1304,6 +1304,20 @@ class QTest(unittest.TestCase): query = ['(', {'age__gte': 18}, '&&', {'name': 'test'}, ')'] self.assertEqual((q1 & q2 & q3 & q4 & q5).query, query) + + def test_q_with_dbref(self): + """Ensure Q objects handle DBRefs correctly""" + class User(Document): + pass + + class Post(Document): + created_user = ReferenceField(User) + + user = User.objects.create() + Post.objects.create(created_user=user) + + self.assertEqual(Post.objects.filter(created_user=user).count(), 1) + self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) if __name__ == '__main__': unittest.main() From 51065e7a4dda96943dadfcbdfcca6111dfa1f9f5 Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:33:33 +0200 Subject: [PATCH 0059/1279] Closes #46 by instantiating a new default instance for every field by request. --- mongoengine/base.py | 5 ++++- mongoengine/fields.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 086c787..10ff121 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -52,7 +52,10 @@ class BaseField(object): # Get value from document instance if available, if not use default value = instance._data.get(self.name) if value is None: - value = self.default + if callable(self.default): # fixes #46 + value = self.default() + else: + value = self.default # Allow callable default values if callable(value): value = value() diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f84f751..670e3cd 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -263,7 +263,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field - kwargs.setdefault('default', []) + kwargs.setdefault('default', lambda: []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): @@ -356,7 +356,7 @@ class DictField(BaseField): def __init__(self, basecls=None, *args, **kwargs): self.basecls = basecls or BaseField assert issubclass(self.basecls, BaseField) - kwargs.setdefault('default', {}) + kwargs.setdefault('default', lambda: {}) super(DictField, self).__init__(*args, **kwargs) def validate(self, value): From 9d82911f6338747411788291175739f436d95709 Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:38:24 +0200 Subject: [PATCH 0060/1279] Added tests for #46. --- tests/fields.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/fields.py b/tests/fields.py index 136437b..ef776d4 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -693,5 +693,18 @@ class FieldTest(unittest.TestCase): Event.drop_collection() + def test_ensure_unique_default_instances(self): + """Ensure that every document has it's own unique default instance.""" + class D(Document): + data = DictField() + data2 = DictField(default=lambda: {}) + + d1 = D() + d1.data['foo'] = 'bar' + d1.data2['foo'] = 'bar' + d2 = D() + self.assertEqual(d2.data, {}) + self.assertEqual(d2.data2, {}) + if __name__ == '__main__': unittest.main() From 386c48b116191a754e3cec550ccdbd5246efdc97 Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:43:11 +0200 Subject: [PATCH 0061/1279] Typo. --- tests/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fields.py b/tests/fields.py index ef776d4..6e42ea4 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -694,7 +694,7 @@ class FieldTest(unittest.TestCase): Event.drop_collection() def test_ensure_unique_default_instances(self): - """Ensure that every document has it's own unique default instance.""" + """Ensure that every field has it's own unique default instance.""" class D(Document): data = DictField() data2 = DictField(default=lambda: {}) From 9411b38508ac261f166dac1f8f8adc7f01084334 Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 18:45:49 +0200 Subject: [PATCH 0062/1279] Removed unnecessary comment. --- mongoengine/queryset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 00a7f7a..be29ae8 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -135,7 +135,6 @@ class Q(object): # Handle DBRef if isinstance(value, pymongo.dbref.DBRef): - # this.created_user.$id == "4c4c56f8cc1831418c000000" op_js = '(this.%(field)s.$id == "%(id)s" &&'\ ' this.%(field)s.$ref == "%(ref)s")' % { 'field': key, From 2f991ac6f18172008ed71f1d97ab6ee71ea360ba Mon Sep 17 00:00:00 2001 From: flosch Date: Sun, 25 Jul 2010 19:02:15 +0200 Subject: [PATCH 0063/1279] Added all() method to get all document instances from a document. Extended the FileField's tests with testing on empty filefield. --- mongoengine/queryset.py | 4 ++++ tests/fields.py | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index be29ae8..ffbe525 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -238,6 +238,10 @@ class QuerySet(object): """An alias of :meth:`~mongoengine.queryset.QuerySet.__call__` """ return self.__call__(*q_objs, **query) + + def all(self): + """Returns all documents.""" + return self.__call__() @property def _collection(self): diff --git a/tests/fields.py b/tests/fields.py index 6e42ea4..1e53d23 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -674,7 +674,12 @@ class FieldTest(unittest.TestCase): PutFile.drop_collection() StreamFile.drop_collection() SetFile.drop_collection() - + + # Make sure FileField is optional and not required + class DemoFile(Document): + file = FileField() + d = DemoFile.objects.create() + def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """ From 7ab2e21c106fc89875b1ae4ae79ecd7e0988ded3 Mon Sep 17 00:00:00 2001 From: flosch Date: Mon, 26 Jul 2010 16:42:10 +0200 Subject: [PATCH 0064/1279] Handle unsafe expressions when using startswith/endswith/contains with unsafe expressions. Closes #58 --- mongoengine/fields.py | 3 +++ tests/queryset.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 670e3cd..bd81d3a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -66,6 +66,9 @@ class StringField(BaseField): regex = r'%s$' elif op == 'exact': regex = r'^%s$' + + # escape unsafe characters which could lead to a re.error + value = re.escape(value) value = re.compile(regex % value, flags) return value diff --git a/tests/queryset.py b/tests/queryset.py index 8cbd9a4..1efd034 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -288,6 +288,13 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(obj, person) obj = self.Person.objects(Q(name__iexact='gUIDO VAN rOSSU')).first() self.assertEqual(obj, None) + + # Test unsafe expressions + person = self.Person(name='Guido van Rossum [.\'Geek\']') + person.save() + + obj = self.Person.objects(Q(name__icontains='[.\'Geek')).first() + self.assertEqual(obj, person) def test_filter_chaining(self): """Ensure filters can be chained together. From 6791f205af3d15a0f4ccaf357692ba868b6ebfff Mon Sep 17 00:00:00 2001 From: flosch Date: Mon, 26 Jul 2010 16:50:09 +0200 Subject: [PATCH 0065/1279] Style fix. --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index bd81d3a..30a11f2 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -66,7 +66,7 @@ class StringField(BaseField): regex = r'%s$' elif op == 'exact': regex = r'^%s$' - + # escape unsafe characters which could lead to a re.error value = re.escape(value) value = re.compile(regex % value, flags) From 21d267cb112e9030a0f17271fecd052dbe35b50a Mon Sep 17 00:00:00 2001 From: flosch Date: Mon, 26 Jul 2010 17:28:59 +0200 Subject: [PATCH 0066/1279] Now order_by() works like queries for referencing deeper fields (replacing . with __). old: order_by('mydoc.myattr') / new: order_by('mydoc__myattr'). Closes #45 --- mongoengine/queryset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index ffbe525..0b9218a 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -670,11 +670,13 @@ class QuerySet(object): """ key_list = [] for key in keys: + if not key: continue direction = pymongo.ASCENDING if key[0] == '-': direction = pymongo.DESCENDING if key[0] in ('-', '+'): key = key[1:] + key = key.replace('__', '.') key_list.append((key, direction)) self._ordering = key_list From 1147ac43506283a37554d09242894e9088f2133b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Peignier?= Date: Sun, 23 May 2010 00:19:50 +0800 Subject: [PATCH 0067/1279] ignore virtualenv directory --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 42dcc6e..57cf1b7 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ docs/.build docs/_build build/ dist/ -mongoengine.egg-info/ \ No newline at end of file +mongoengine.egg-info/ +env/ \ No newline at end of file From b96e27a7e4675c0564cff05bcba66f7662b6099d Mon Sep 17 00:00:00 2001 From: Theo Julienne Date: Fri, 30 Jul 2010 22:09:00 +1000 Subject: [PATCH 0068/1279] Allow documents to override the 'objects' QuerySetManager --- mongoengine/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index c8c162b..ed8214a 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -252,7 +252,10 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Set up collection manager, needs the class to have fields so use # DocumentMetaclass before instantiating CollectionManager object new_class = super_new(cls, name, bases, attrs) - new_class.objects = QuerySetManager() + + # Provide a default queryset unless one has been manually provided + if not 'objects' in dir(new_class): + new_class.objects = QuerySetManager() user_indexes = [QuerySet._build_index_spec(new_class, spec) for spec in meta['indexes']] + base_indexes From 198ccc028a9d24e7879db583f8a263a061255742 Mon Sep 17 00:00:00 2001 From: Greg Turner Date: Fri, 6 Aug 2010 20:29:09 +1000 Subject: [PATCH 0069/1279] made list queries work with regexes (e.g. istartswith) --- mongoengine/fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 127f029..73bc756 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -319,8 +319,8 @@ class ListField(BaseField): def prepare_query_value(self, op, value): if op in ('set', 'unset'): - return [self.field.to_mongo(v) for v in value] - return self.field.to_mongo(value) + return [self.field.prepare_query_value(op, v) for v in value] + return self.field.prepare_query_value(op, value) def lookup_member(self, member_name): return self.field.lookup_member(member_name) From 809fe44b43decc6f271d3eedd1f637db4149f24c Mon Sep 17 00:00:00 2001 From: Greg Turner Date: Thu, 12 Aug 2010 15:14:20 +1000 Subject: [PATCH 0070/1279] Added a __raw__ parameter for passing query dictionaries directly to pymongo --- mongoengine/queryset.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 069ab11..b246021 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -318,6 +318,11 @@ class QuerySet(object): mongo_query = {} for key, value in query.items(): + + if key == "__raw__": + mongo_query.update(value) + return mongo_query + parts = key.split('__') # Check for an operator and transform to mongo-style if there is op = None From 6373e20696533630979327ed0183ea2cbbc6cc48 Mon Sep 17 00:00:00 2001 From: Greg Turner Date: Fri, 13 Aug 2010 22:28:26 +1000 Subject: [PATCH 0071/1279] Better error reporting on a validation error for a list. --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 73bc756..866dc7e 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -315,7 +315,7 @@ class ListField(BaseField): try: [self.field.validate(item) for item in value] except Exception, err: - raise ValidationError('Invalid ListField item (%s)' % str(err)) + raise ValidationError('Invalid ListField item (%s)' % str(item)) def prepare_query_value(self, op, value): if op in ('set', 'unset'): From d274576b4739d77803cc4adfda5e069be865d909 Mon Sep 17 00:00:00 2001 From: Greg Turner Date: Fri, 13 Aug 2010 22:30:36 +1000 Subject: [PATCH 0072/1279] Fixed premature return for query gen --- mongoengine/queryset.py | 57 ++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b246021..3f9b7c5 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -321,40 +321,39 @@ class QuerySet(object): if key == "__raw__": mongo_query.update(value) - return mongo_query + else: + parts = key.split('__') + # Check for an operator and transform to mongo-style if there is + op = None + if parts[-1] in operators + match_operators: + op = parts.pop() - parts = key.split('__') - # Check for an operator and transform to mongo-style if there is - op = None - if parts[-1] in operators + match_operators: - op = parts.pop() + if _doc_cls: + # Switch field names to proper names [set in Field(name='foo')] + fields = QuerySet._lookup_field(_doc_cls, parts) + parts = [field.db_field for field in fields] - if _doc_cls: - # Switch field names to proper names [set in Field(name='foo')] - fields = QuerySet._lookup_field(_doc_cls, parts) - parts = [field.db_field for field in fields] + # Convert value to proper value + field = fields[-1] + singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte'] + singular_ops += match_operators + if op in singular_ops: + value = field.prepare_query_value(op, value) + elif op in ('in', 'nin', 'all'): + # 'in', 'nin' and 'all' require a list of values + value = [field.prepare_query_value(op, v) for v in value] - # Convert value to proper value - field = fields[-1] - singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte'] - singular_ops += match_operators - if op in singular_ops: - value = field.prepare_query_value(op, value) - elif op in ('in', 'nin', 'all'): - # 'in', 'nin' and 'all' require a list of values - value = [field.prepare_query_value(op, v) for v in value] + if field.__class__.__name__ == 'GenericReferenceField': + parts.append('_ref') - if field.__class__.__name__ == 'GenericReferenceField': - parts.append('_ref') + if op and op not in match_operators: + value = {'$' + op: value} - if op and op not in match_operators: - value = {'$' + op: value} - - key = '.'.join(parts) - if op is None or key not in mongo_query: - mongo_query[key] = value - elif key in mongo_query and isinstance(mongo_query[key], dict): - mongo_query[key].update(value) + key = '.'.join(parts) + if op is None or key not in mongo_query: + mongo_query[key] = value + elif key in mongo_query and isinstance(mongo_query[key], dict): + mongo_query[key].update(value) return mongo_query From 17addbefe2e7befe45edcc764add45c397d4202b Mon Sep 17 00:00:00 2001 From: sp Date: Wed, 18 Aug 2010 16:50:52 -0400 Subject: [PATCH 0073/1279] made it more like Django's user model --- mongoengine/django/auth.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index d4b0ff0..da0005c 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -32,6 +32,9 @@ class User(Document): last_login = DateTimeField(default=datetime.datetime.now) date_joined = DateTimeField(default=datetime.datetime.now) + def __unicode__(self): + return self.username + def get_full_name(self): """Returns the users first and last names, separated by a space. """ From 7de9adc6b1677b5e9c2b7b2b6d5b670effdd5ead Mon Sep 17 00:00:00 2001 From: Richard Henry Date: Sat, 28 Aug 2010 09:16:02 +0100 Subject: [PATCH 0074/1279] Adding support for pop operations to QuerySet.update and QuerySet.update_one --- docs/guide/querying.rst | 1 + mongoengine/queryset.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 113ee43..7de6c5c 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -399,6 +399,7 @@ that you may use with these methods: * ``unset`` -- delete a particular value (since MongoDB v1.3+) * ``inc`` -- increment a value by a given amount * ``dec`` -- decrement a value by a given amount +* ``pop`` -- remove the last item from a list * ``push`` -- append a value to a list * ``push_all`` -- append several values to a list * ``pull`` -- remove a value from a list diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 069ab11..182dfd5 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -661,8 +661,8 @@ class QuerySet(object): def _transform_update(cls, _doc_cls=None, **update): """Transform an update spec from Django-style format to Mongo format. """ - operators = ['set', 'unset', 'inc', 'dec', 'push', 'push_all', 'pull', - 'pull_all'] + operators = ['set', 'unset', 'inc', 'dec', 'pop', 'push', 'push_all', + 'pull', 'pull_all'] mongo_update = {} for key, value in update.items(): @@ -688,7 +688,7 @@ class QuerySet(object): # Convert value to proper value field = fields[-1] - if op in (None, 'set', 'unset', 'push', 'pull'): + if op in (None, 'set', 'unset', 'pop', 'push', 'pull'): value = field.prepare_query_value(op, value) elif op in ('pushAll', 'pullAll'): value = [field.prepare_query_value(op, v) for v in value] From d99c5973c3cb6da18670c6592ff396d18bb0e946 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 12:52:24 +0100 Subject: [PATCH 0075/1279] Fixed Q object DBRef test bug --- tests/queryset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 8cbd9a4..b1f1657 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1307,6 +1307,8 @@ class QTest(unittest.TestCase): def test_q_with_dbref(self): """Ensure Q objects handle DBRefs correctly""" + connect(db='mongoenginetest') + class User(Document): pass From 3b62cf80cde0b83715662ce9bb01053dc13a7060 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 13:00:34 +0100 Subject: [PATCH 0076/1279] Fixed {Dict,List}Field default issue. Closes #46. --- mongoengine/fields.py | 6 +++--- tests/fields.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f84f751..d8d310d 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -263,7 +263,7 @@ class ListField(BaseField): raise ValidationError('Argument to ListField constructor must be ' 'a valid field') self.field = field - kwargs.setdefault('default', []) + kwargs.setdefault('default', lambda: []) super(ListField, self).__init__(**kwargs) def __get__(self, instance, owner): @@ -356,7 +356,7 @@ class DictField(BaseField): def __init__(self, basecls=None, *args, **kwargs): self.basecls = basecls or BaseField assert issubclass(self.basecls, BaseField) - kwargs.setdefault('default', {}) + kwargs.setdefault('default', lambda: {}) super(DictField, self).__init__(*args, **kwargs) def validate(self, value): @@ -623,4 +623,4 @@ class GeoPointField(BaseField): raise ValidationError('Value must be a two-dimensional point.') if (not isinstance(value[0], (float, int)) and not isinstance(value[1], (float, int))): - raise ValidationError('Both values in point must be float or int.') \ No newline at end of file + raise ValidationError('Both values in point must be float or int.') diff --git a/tests/fields.py b/tests/fields.py index 136437b..ef776d4 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -693,5 +693,18 @@ class FieldTest(unittest.TestCase): Event.drop_collection() + def test_ensure_unique_default_instances(self): + """Ensure that every document has it's own unique default instance.""" + class D(Document): + data = DictField() + data2 = DictField(default=lambda: {}) + + d1 = D() + d1.data['foo'] = 'bar' + d1.data2['foo'] = 'bar' + d2 = D() + self.assertEqual(d2.data, {}) + self.assertEqual(d2.data2, {}) + if __name__ == '__main__': unittest.main() From 4fb6fcabefa04f9ab4b410ce1cf3b6293e0ec52c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 13:54:20 +0100 Subject: [PATCH 0077/1279] Added test for overriding objects --- tests/queryset.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/queryset.py b/tests/queryset.py index b339d61..0c7ba05 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -992,10 +992,15 @@ class QuerySetTest(unittest.TestCase): """ class BlogPost(Document): tags = ListField(StringField()) + deleted = BooleanField(default=False) + + @queryset_manager + def objects(doc_cls, queryset): + return queryset(deleted=False) @queryset_manager def music_posts(doc_cls, queryset): - return queryset(tags='music') + return queryset(tags='music', deleted=False) BlogPost.drop_collection() @@ -1005,6 +1010,8 @@ class QuerySetTest(unittest.TestCase): post2.save() post3 = BlogPost(tags=['film', 'actors']) post3.save() + post4 = BlogPost(tags=['film', 'actors'], deleted=True) + post4.save() self.assertEqual([p.id for p in BlogPost.objects], [post1.id, post2.id, post3.id]) From 95efa39b5229081deef179ae4b08299a50b18ccb Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 30 Aug 2010 14:56:18 +0200 Subject: [PATCH 0078/1279] Added *.egg to .gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 57cf1b7..51a9ca1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.pyc .*.swp +*.egg docs/.build docs/_build build/ From e0911a5fe00d77bb74b0b8b2d8e5d6d963afc687 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 30 Aug 2010 14:58:58 +0200 Subject: [PATCH 0079/1279] Replaced slow exception handling with has_key. --- mongoengine/base.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 4523a26..e0e71d3 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -415,11 +415,8 @@ class BaseDocument(object): self._meta.get('allow_inheritance', True) == False): data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] - try: - if not data['_id']: - del data['_id'] - except KeyError: - pass + if data.has_key('_id') and not data['_id']: + del data['_id'] return data @classmethod From 1ed9a36d0a15eef4038568ccc974d14bfb425c42 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 14:02:02 +0100 Subject: [PATCH 0080/1279] Added test for pop operation --- tests/queryset.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 0c7ba05..2f86953 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -671,6 +671,11 @@ class QuerySetTest(unittest.TestCase): post.reload() self.assertTrue('db' in post.tags and 'nosql' in post.tags) + tags = post.tags[:-1] + BlogPost.objects.update(pop__tags=1) + post.reload() + self.assertEqual(post.tags, tags) + BlogPost.drop_collection() def test_update_pull(self): From 5b230b90b984b20d09322d17dd6b43b93db050f7 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 15:34:29 +0100 Subject: [PATCH 0081/1279] Doc fix (all operator) --- docs/guide/querying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 8656308..bef19bc 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -71,7 +71,7 @@ Available operators are as follows: * ``in`` -- value is in list (a list of values should be provided) * ``nin`` -- value is not in list (a list of values should be provided) * ``mod`` -- ``value % x == y``, where ``x`` and ``y`` are two provided values -* ``all`` -- every item in array is in list of values provided +* ``all`` -- every item in list of values provided is in array * ``size`` -- the size of the array is * ``exists`` -- value for field exists From 185e7a6a7e27884ac4896bf5c6a6881db8d419a0 Mon Sep 17 00:00:00 2001 From: Florian Schlachter Date: Mon, 30 Aug 2010 18:38:41 +0200 Subject: [PATCH 0082/1279] Better way of checking if new_class has an 'objects' attribute. --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 3172e9a..9505d41 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -257,7 +257,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): new_class = super_new(cls, name, bases, attrs) # Provide a default queryset unless one has been manually provided - if not 'objects' in dir(new_class): + if not hasattr(new_class, 'objects'): new_class.objects = QuerySetManager() user_indexes = [QuerySet._build_index_spec(new_class, spec) From dcc8d22cec6d59392c8470a9325d0d8fedc6d4fa Mon Sep 17 00:00:00 2001 From: Mircea Pasoi Date: Tue, 24 Aug 2010 16:22:56 -0700 Subject: [PATCH 0083/1279] Proper unique index name when using db_field. --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 9505d41..92a450d 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -269,7 +269,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Generate a list of indexes needed by uniqueness constraints if field.unique: field.required = True - unique_fields = [field_name] + unique_fields = [field.db_field] # Add any unique_with fields to the back of the index spec if field.unique_with: From 266f33adc4730859c272d9a8233769b0d9e54205 Mon Sep 17 00:00:00 2001 From: Mircea Pasoi Date: Tue, 24 Aug 2010 19:16:54 -0700 Subject: [PATCH 0084/1279] Bug fix for gridfs FileField. --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index dc5dbe0..49e9d05 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -600,7 +600,7 @@ class FileField(BaseField): def to_python(self, value): # Use stored value (id) to lookup file in GridFS - if self.gridfs.grid_id is not None: + if value is not None: return self.gridfs.get(id=value) return None From 3e30d71263356082ef5d391a2ea614d74790bb32 Mon Sep 17 00:00:00 2001 From: Mircea Pasoi Date: Fri, 27 Aug 2010 20:25:07 -0700 Subject: [PATCH 0085/1279] Support for background and drop_dups indexing options. --- mongoengine/queryset.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index dd7fcef..7969c19 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -172,7 +172,8 @@ class QuerySet(object): self._limit = None self._skip = None - def ensure_index(self, key_or_list): + def ensure_index(self, key_or_list, drop_dups=False, background=False, + **kwargs): """Ensure that the given indexes are in place. :param key_or_list: a single index key or a list of index keys (to @@ -180,7 +181,8 @@ class QuerySet(object): or a **-** to determine the index ordering """ index_list = QuerySet._build_index_spec(self._document, key_or_list) - self._collection.ensure_index(index_list) + self._collection.ensure_index(index_list, drop_dups=drop_dups, + background=background) return self @classmethod @@ -250,25 +252,33 @@ class QuerySet(object): """ if not self._accessed_collection: self._accessed_collection = True + + background = self._document._meta.get('index_background', False) + drop_dups = self._document._meta.get('index_drop_dups', False) + index_opts = self._document._meta.get('index_options', {}) # Ensure document-defined indexes are created if self._document._meta['indexes']: for key_or_list in self._document._meta['indexes']: - self._collection.ensure_index(key_or_list) + self._collection.ensure_index(key_or_list, + background=background, **index_opts) # Ensure indexes created by uniqueness constraints for index in self._document._meta['unique_indexes']: - self._collection.ensure_index(index, unique=True) + self._collection.ensure_index(index, unique=True, + background=background, drop_dups=drop_dups, **index_opts) # If _types is being used (for polymorphism), it needs an index if '_types' in self._query: - self._collection.ensure_index('_types') + self._collection.ensure_index('_types', + background=background, **index_opts) # Ensure all needed field indexes are created for field in self._document._fields.values(): if field.__class__._geo_index: index_spec = [(field.db_field, pymongo.GEO2D)] - self._collection.ensure_index(index_spec) + self._collection.ensure_index(index_spec, + background=background, **index_opts) return self._collection_obj From f1aec68f239d77edff296e05764c419c4187b2d0 Mon Sep 17 00:00:00 2001 From: Mircea Pasoi Date: Fri, 27 Aug 2010 20:46:44 -0700 Subject: [PATCH 0086/1279] Inherit index options. --- mongoengine/base.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index 92a450d..836817d 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -230,12 +230,18 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): id_field = None base_indexes = [] + base_meta = {} # Subclassed documents inherit collection from superclass for base in bases: if hasattr(base, '_meta') and 'collection' in base._meta: collection = base._meta['collection'] + # Propagate index options. + for key in ('index_background', 'index_drop_dups', 'index_opts'): + if key in base._meta: + base_meta[key] = base._meta[key] + id_field = id_field or base._meta.get('id_field') base_indexes += base._meta.get('indexes', []) @@ -246,7 +252,11 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): 'ordering': [], # default ordering applied at runtime 'indexes': [], # indexes to be ensured at runtime 'id_field': id_field, + 'index_background': False, + 'index_drop_dups': False, + 'index_opts': {}, } + meta.update(base_meta) # Apply document-defined meta options meta.update(attrs.get('meta', {})) From 17642c8a8cc2e37741a1d10be04d704770679193 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 19:48:17 +0100 Subject: [PATCH 0087/1279] Fixed QuerySet.average issue that ignored 0 --- mongoengine/queryset.py | 2 +- tests/queryset.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 7969c19..662fa8c 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -892,7 +892,7 @@ class QuerySet(object): var total = 0.0; var num = 0; db[collection].find(query).forEach(function(doc) { - if (doc[averageField]) { + if (doc[averageField] !== undefined) { total += doc[averageField]; num += 1; } diff --git a/tests/queryset.py b/tests/queryset.py index 2f86953..0c6c3ca 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -960,11 +960,14 @@ class QuerySetTest(unittest.TestCase): def test_average(self): """Ensure that field can be averaged correctly. """ + self.Person(name='person', age=0).save() + self.assertEqual(int(self.Person.objects.average('age')), 0) + ages = [23, 54, 12, 94, 27] for i, age in enumerate(ages): self.Person(name='test%s' % i, age=age).save() - avg = float(sum(ages)) / len(ages) + avg = float(sum(ages)) / (len(ages) + 1) # take into account the 0 self.assertAlmostEqual(int(self.Person.objects.average('age')), avg) self.Person(name='ageless person').save() @@ -1340,5 +1343,6 @@ class QTest(unittest.TestCase): self.assertEqual(Post.objects.filter(created_user=user).count(), 1) self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) + if __name__ == '__main__': unittest.main() From 69012e8ad11a7e4312450ff71fc80445e007270f Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 19:59:49 +0100 Subject: [PATCH 0088/1279] Fixed incorrect $pull test --- tests/queryset.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 0c6c3ca..e391224 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -681,23 +681,17 @@ class QuerySetTest(unittest.TestCase): def test_update_pull(self): """Ensure that the 'pull' update operation works correctly. """ - class Comment(EmbeddedDocument): - content = StringField() - class BlogPost(Document): slug = StringField() - comments = ListField(EmbeddedDocumentField(Comment)) + tags = ListField(StringField()) - comment1 = Comment(content="test1") - comment2 = Comment(content="test2") - - post = BlogPost(slug="test", comments=[comment1, comment2]) + post = BlogPost(slug="test", tags=['code', 'mongodb', 'code']) post.save() - self.assertTrue(comment2 in post.comments) - BlogPost.objects(slug="test").update(pull__comments__content="test2") + BlogPost.objects(slug="test").update(pull__tags="code") post.reload() - self.assertTrue(comment2 not in post.comments) + self.assertTrue('code' not in post.tags) + self.assertEqual(len(post.tags), 1) def test_order_by(self): """Ensure that QuerySets may be ordered. From 32e66b29f44f3015be099851201241caee92054f Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 30 Aug 2010 22:12:05 +0100 Subject: [PATCH 0089/1279] Fixed FileField problem caused by shared objects --- mongoengine/fields.py | 50 ++++++++++++++++++++++++++++++------------- tests/fields.py | 22 +++++++++++++++++++ 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 49e9d05..ffcfb53 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -510,14 +510,15 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') + class GridFSProxy(object): """Proxy object to handle writing and reading of files to and from GridFS """ - def __init__(self): + def __init__(self, grid_id=None): self.fs = gridfs.GridFS(_get_db()) # Filesystem instance self.newfile = None # Used for partial writes - self.grid_id = None # Store GridFS id for file + self.grid_id = grid_id # Store GridFS id for file def __getattr__(self, name): obj = self.get() @@ -528,9 +529,12 @@ class GridFSProxy(object): return self def get(self, id=None): - if id: self.grid_id = id - try: return self.fs.get(id or self.grid_id) - except: return None # File has been deleted + if id: + self.grid_id = id + try: + return self.fs.get(id or self.grid_id) + except: + return None # File has been deleted def new_file(self, **kwargs): self.newfile = self.fs.new_file(**kwargs) @@ -552,8 +556,10 @@ class GridFSProxy(object): self.newfile.writelines(lines) def read(self): - try: return self.get().read() - except: return None + try: + return self.get().read() + except: + return None def delete(self): # Delete file from GridFS, FileField still remains @@ -571,38 +577,52 @@ class GridFSProxy(object): msg = "The close() method is only necessary after calling write()" warnings.warn(msg) + class FileField(BaseField): """A GridFS storage field. """ def __init__(self, **kwargs): - self.gridfs = GridFSProxy() super(FileField, self).__init__(**kwargs) def __get__(self, instance, owner): if instance is None: return self - return self.gridfs + # Check if a file already exists for this model + grid_file = instance._data.get(self.name) + if grid_file: + return grid_file + return GridFSProxy() def __set__(self, instance, value): if isinstance(value, file) or isinstance(value, str): # using "FileField() = file/string" notation - self.gridfs.put(value) + grid_file = instance._data.get(self.name) + # If a file already exists, delete it + if grid_file: + try: + grid_file.delete() + except: + pass + # Create a new file with the new data + grid_file.put(value) + else: + # Create a new proxy object as we don't already have one + instance._data[self.name] = GridFSProxy() + instance._data[self.name].put(value) else: instance._data[self.name] = value def to_mongo(self, value): # Store the GridFS file id in MongoDB - if self.gridfs.grid_id is not None: - return self.gridfs.grid_id + if isinstance(value, GridFSProxy) and value.grid_id is not None: + return value.grid_id return None def to_python(self, value): - # Use stored value (id) to lookup file in GridFS if value is not None: - return self.gridfs.get(id=value) - return None + return GridFSProxy(value) def validate(self, value): if value.grid_id is not None: diff --git a/tests/fields.py b/tests/fields.py index 1e53d23..8c72719 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -680,6 +680,28 @@ class FieldTest(unittest.TestCase): file = FileField() d = DemoFile.objects.create() + def test_file_uniqueness(self): + """Ensure that each instance of a FileField is unique + """ + class TestFile(Document): + name = StringField() + file = FileField() + + # First instance + testfile = TestFile() + testfile.name = "Hello, World!" + testfile.file.put('Hello, World!') + testfile.save() + + # Second instance + testfiledupe = TestFile() + data = testfiledupe.file.read() # Should be None + + self.assertTrue(testfile.name != testfiledupe.name) + self.assertTrue(testfile.file.read() != data) + + TestFile.drop_collection() + def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """ From 1849f75ad04a8a50c73862a6d0794fe65016343c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Tue, 31 Aug 2010 00:23:59 +0100 Subject: [PATCH 0090/1279] Made GridFSProxy a bit stricter / safer --- mongoengine/fields.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index ffcfb53..418f57c 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -511,6 +511,10 @@ class BinaryField(BaseField): raise ValidationError('Binary value is too long') +class GridFSError(Exception): + pass + + class GridFSProxy(object): """Proxy object to handle writing and reading of files to and from GridFS """ @@ -541,12 +545,18 @@ class GridFSProxy(object): self.grid_id = self.newfile._id def put(self, file, **kwargs): + if self.grid_id: + raise GridFSError('This document alreay has a file. Either delete ' + 'it or call replace to overwrite it') self.grid_id = self.fs.put(file, **kwargs) def write(self, string): - if not self.newfile: + if self.grid_id: + if not self.newfile: + raise GridFSError('This document alreay has a file. Either ' + 'delete it or call replace to overwrite it') + else: self.new_file() - self.grid_id = self.newfile._id self.newfile.write(string) def writelines(self, lines): From 2af5f3c56ebc7feff6a0a1cd95a77e12b425efed Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Tue, 31 Aug 2010 00:24:30 +0100 Subject: [PATCH 0091/1279] Added support for querying by array position. Closes #36. --- mongoengine/queryset.py | 6 +++++- tests/queryset.py | 43 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 662fa8c..b8ca125 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -344,6 +344,8 @@ class QuerySet(object): mongo_query = {} for key, value in query.items(): parts = key.split('__') + indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()] + parts = [part for part in parts if not part.isdigit()] # Check for an operator and transform to mongo-style if there is op = None if parts[-1] in operators + match_operators + geo_operators: @@ -381,7 +383,9 @@ class QuerySet(object): "been implemented" % op) elif op not in match_operators: value = {'$' + op: value} - + + for i, part in indices: + parts.insert(i, part) key = '.'.join(parts) if op is None or key not in mongo_query: mongo_query[key] = value diff --git a/tests/queryset.py b/tests/queryset.py index e391224..3d714be 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -165,8 +165,49 @@ class QuerySetTest(unittest.TestCase): person = self.Person.objects.get(age__lt=30) self.assertEqual(person.name, "User A") + def test_find_array_position(self): + """Ensure that query by array position works. + """ + class Comment(EmbeddedDocument): + name = StringField() + + class Post(EmbeddedDocument): + comments = ListField(EmbeddedDocumentField(Comment)) + + class Blog(Document): + tags = ListField(StringField()) + posts = ListField(EmbeddedDocumentField(Post)) + + Blog.drop_collection() - + Blog.objects.create(tags=['a', 'b']) + self.assertEqual(len(Blog.objects(tags__0='a')), 1) + self.assertEqual(len(Blog.objects(tags__0='b')), 0) + self.assertEqual(len(Blog.objects(tags__1='a')), 0) + self.assertEqual(len(Blog.objects(tags__1='b')), 1) + + Blog.drop_collection() + + comment1 = Comment(name='testa') + comment2 = Comment(name='testb') + post1 = Post(comments=[comment1, comment2]) + post2 = Post(comments=[comment2, comment2]) + blog1 = Blog.objects.create(posts=[post1, post2]) + blog2 = Blog.objects.create(posts=[post2, post1]) + + blog = Blog.objects(posts__0__comments__0__name='testa').get() + self.assertEqual(blog, blog1) + + query = Blog.objects(posts__1__comments__1__name='testb') + self.assertEqual(len(query), 2) + + query = Blog.objects(posts__1__comments__1__name='testa') + self.assertEqual(len(query), 0) + + query = Blog.objects(posts__0__comments__1__name='testa') + self.assertEqual(len(query), 0) + + Blog.drop_collection() def test_get_or_create(self): """Ensure that ``get_or_create`` returns one result or creates a new From 449f5a00dccce95b3efebd59438ee981ddfb7435 Mon Sep 17 00:00:00 2001 From: Nicolas Perriault Date: Sat, 11 Sep 2010 17:45:57 +0200 Subject: [PATCH 0092/1279] added a 'validate' option to Document.save() +docs +tests --- docs/guide/defining-documents.rst | 14 ++++++++++++++ mongoengine/document.py | 6 ++++-- tests/document.py | 10 ++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 3c27686..dff3ed6 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -214,6 +214,20 @@ either a single field name, or a list or tuple of field names:: first_name = StringField() last_name = StringField(unique_with='first_name') +Skipping Document validation on save +------------------------------------ +You can also skip the whole document validation process by setting +``validate=False`` when caling the :meth:`~mongoengine.document.Document.save` +method:: + + class Recipient(Document): + name = StringField() + email = EmailField() + + recipient = Recipient(name='admin', email='root@localhost') + recipient.save() # will raise a ValidationError while + recipient.save(validate=False) # won't + Document collections ==================== Document classes that inherit **directly** from :class:`~mongoengine.Document` diff --git a/mongoengine/document.py b/mongoengine/document.py index e5dec14..af2a5e2 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -56,7 +56,7 @@ class Document(BaseDocument): __metaclass__ = TopLevelDocumentMetaclass - def save(self, safe=True, force_insert=False): + def save(self, safe=True, force_insert=False, validate=True): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be created. @@ -67,8 +67,10 @@ class Document(BaseDocument): :param safe: check if the operation succeeded before returning :param force_insert: only try to create a new document, don't allow updates of existing documents + :param validate: validates the document; set to ``False`` for skiping """ - self.validate() + if validate: + self.validate() doc = self.to_mongo() try: collection = self.__class__.objects._collection diff --git a/tests/document.py b/tests/document.py index 8bc907c..80cf3f0 100644 --- a/tests/document.py +++ b/tests/document.py @@ -446,6 +446,16 @@ class DocumentTest(unittest.TestCase): self.assertEqual(person_obj['name'], 'Test User') self.assertEqual(person_obj['age'], 30) self.assertEqual(person_obj['_id'], person.id) + # Test skipping validation on save + class Recipient(Document): + email = EmailField(required=True) + + recipient = Recipient(email='root@localhost') + self.assertRaises(ValidationError, recipient.save) + try: + recipient.save(validate=False) + except ValidationError: + fail() def test_delete(self): """Ensure that document may be deleted using the delete method. From f11ee1f9cf60979fbb5c6768219225202d510951 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Wed, 15 Sep 2010 09:47:13 +0100 Subject: [PATCH 0093/1279] Added support for using custom QuerySet classes --- mongoengine/base.py | 1 + mongoengine/queryset.py | 3 ++- tests/queryset.py | 20 ++++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 836817d..0cbd707 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -255,6 +255,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): 'index_background': False, 'index_drop_dups': False, 'index_opts': {}, + 'queryset_class': QuerySet, } meta.update(base_meta) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b8ca125..8b48609 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -992,7 +992,8 @@ class QuerySetManager(object): self._collection = db[collection] # owner is the document that contains the QuerySetManager - queryset = QuerySet(owner, self._collection) + queryset_class = owner._meta['queryset_class'] or QuerySet + queryset = queryset_class(owner, self._collection) if self._manager_func: if self._manager_func.func_code.co_argcount == 1: queryset = self._manager_func(queryset) diff --git a/tests/queryset.py b/tests/queryset.py index 3d714be..4491be8 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1378,6 +1378,26 @@ class QTest(unittest.TestCase): self.assertEqual(Post.objects.filter(created_user=user).count(), 1) self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) + def test_custom_querysets(self): + """Ensure that custom QuerySet classes may be used. + """ + class CustomQuerySet(QuerySet): + def not_empty(self): + return len(self) > 0 + + class Post(Document): + meta = {'queryset_class': CustomQuerySet} + + Post.drop_collection() + + self.assertTrue(isinstance(Post.objects, CustomQuerySet)) + self.assertFalse(Post.objects.not_empty()) + + Post().save() + self.assertTrue(Post.objects.not_empty()) + + Post.drop_collection() + if __name__ == '__main__': unittest.main() From b7e84031e310561469b99d62c99dd354e4fa6fe5 Mon Sep 17 00:00:00 2001 From: Greg Turner Date: Thu, 16 Sep 2010 14:37:18 +1000 Subject: [PATCH 0094/1279] Escape strings for regex query. --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 866dc7e..7904589 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -64,7 +64,7 @@ class StringField(BaseField): regex = r'%s$' elif op == 'exact': regex = r'^%s$' - value = re.compile(regex % value, flags) + value = re.compile(regex % re.escape(value), flags) return value From 20dd7562e0e7307f3a53c77d33521cb2f826cf5c Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Thu, 16 Sep 2010 17:19:58 -0400 Subject: [PATCH 0095/1279] Adding multiprocessing support to mongoengine by using the identity of the process to define the connection. Each 'thread' gets its own pymongo connection. --- mongoengine/connection.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index ec3bf78..94cc6ea 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,5 +1,5 @@ from pymongo import Connection - +import multiprocessing __all__ = ['ConnectionError', 'connect'] @@ -8,12 +8,12 @@ _connection_settings = { 'host': 'localhost', 'port': 27017, } -_connection = None +_connection = {} _db_name = None _db_username = None _db_password = None -_db = None +_db = {} class ConnectionError(Exception): @@ -22,32 +22,39 @@ class ConnectionError(Exception): def _get_connection(): global _connection + identity = get_identity() # Connect to the database if not already connected - if _connection is None: + if _connection.get(identity) is None: try: - _connection = Connection(**_connection_settings) + _connection[identity] = Connection(**_connection_settings) except: raise ConnectionError('Cannot connect to the database') - return _connection + return _connection[identity] def _get_db(): global _db, _connection + identity = get_identity() # Connect if not already connected - if _connection is None: - _connection = _get_connection() + if _connection.get(identity) is None: + _connection[identity] = _get_connection() - if _db is None: + if _db.get(identity) is None: # _db_name will be None if the user hasn't called connect() if _db_name is None: raise ConnectionError('Not connected to the database') # Get DB from current connection and authenticate if necessary - _db = _connection[_db_name] + _db[identity] = _connection[identity][_db_name] if _db_username and _db_password: - _db.authenticate(_db_username, _db_password) + _db[identity].authenticate(_db_username, _db_password) - return _db + return _db[identity] +def get_identity(): + identity = multiprocessing.current_process()._identity + identity = 0 if not identity else identity[0] + return identity + def connect(db, username=None, password=None, **kwargs): """Connect to the database specified by the 'db' argument. Connection settings may be provided here as well if the database is not running on From 73092dcb33ed64e64aedc3d32fa84780c6e534bb Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 19 Sep 2010 17:17:37 +0100 Subject: [PATCH 0096/1279] QuerySet update method returns num affected docs --- mongoengine/queryset.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b8ca125..c199d64 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -766,7 +766,8 @@ class QuerySet(object): return mongo_update def update(self, safe_update=True, upsert=False, **update): - """Perform an atomic update on the fields matched by the query. + """Perform an atomic update on the fields matched by the query. When + ``safe_update`` is used, the number of affected documents is returned. :param safe: check if the operation succeeded before returning :param update: Django-style update keyword arguments @@ -778,8 +779,10 @@ class QuerySet(object): update = QuerySet._transform_update(self._document, **update) try: - self._collection.update(self._query, update, safe=safe_update, - upsert=upsert, multi=True) + ret = self._collection.update(self._query, update, multi=True, + upsert=upsert, safe=safe_update) + if ret is not None and 'n' in ret: + return ret['n'] except pymongo.errors.OperationFailure, err: if unicode(err) == u'multi not coded yet': message = u'update() method requires MongoDB 1.1.3+' @@ -787,7 +790,8 @@ class QuerySet(object): raise OperationError(u'Update failed (%s)' % unicode(err)) def update_one(self, safe_update=True, upsert=False, **update): - """Perform an atomic update on first field matched by the query. + """Perform an atomic update on first field matched by the query. When + ``safe_update`` is used, the number of affected documents is returned. :param safe: check if the operation succeeded before returning :param update: Django-style update keyword arguments @@ -799,11 +803,14 @@ class QuerySet(object): # Explicitly provide 'multi=False' to newer versions of PyMongo # as the default may change to 'True' if pymongo.version >= '1.1.1': - self._collection.update(self._query, update, safe=safe_update, - upsert=upsert, multi=False) + ret = self._collection.update(self._query, update, multi=False, + upsert=upsert, safe=safe_update) else: # Older versions of PyMongo don't support 'multi' - self._collection.update(self._query, update, safe=safe_update) + ret = self._collection.update(self._query, update, + safe=safe_update) + if ret is not None and 'n' in ret: + return ret['n'] except pymongo.errors.OperationFailure, e: raise OperationError(u'Update failed [%s]' % unicode(e)) From bb2487914987972c4d7ff6960de1012d53a7f566 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 19 Sep 2010 20:00:53 +0100 Subject: [PATCH 0097/1279] Moved custom queryset test to correct place --- tests/queryset.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 4491be8..59a8216 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1307,6 +1307,26 @@ class QuerySetTest(unittest.TestCase): Event.drop_collection() + def test_custom_querysets(self): + """Ensure that custom QuerySet classes may be used. + """ + class CustomQuerySet(QuerySet): + def not_empty(self): + return len(self) > 0 + + class Post(Document): + meta = {'queryset_class': CustomQuerySet} + + Post.drop_collection() + + self.assertTrue(isinstance(Post.objects, CustomQuerySet)) + self.assertFalse(Post.objects.not_empty()) + + Post().save() + self.assertTrue(Post.objects.not_empty()) + + Post.drop_collection() + class QTest(unittest.TestCase): @@ -1378,26 +1398,6 @@ class QTest(unittest.TestCase): self.assertEqual(Post.objects.filter(created_user=user).count(), 1) self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) - def test_custom_querysets(self): - """Ensure that custom QuerySet classes may be used. - """ - class CustomQuerySet(QuerySet): - def not_empty(self): - return len(self) > 0 - - class Post(Document): - meta = {'queryset_class': CustomQuerySet} - - Post.drop_collection() - - self.assertTrue(isinstance(Post.objects, CustomQuerySet)) - self.assertFalse(Post.objects.not_empty()) - - Post().save() - self.assertTrue(Post.objects.not_empty()) - - Post.drop_collection() - if __name__ == '__main__': unittest.main() From 98bc0a7c10f23c4c7582c222ec0d1432ebe5b567 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sat, 25 Sep 2010 22:47:09 +0100 Subject: [PATCH 0098/1279] Raise AttributeError when necessary on GridFSProxy --- mongoengine/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 418f57c..87d52fd 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -528,6 +528,7 @@ class GridFSProxy(object): obj = self.get() if name in dir(obj): return getattr(obj, name) + raise AttributeError def __get__(self, instance, value): return self From b5eb3ea1cded1bb4c93e13298e86e3d85241c081 Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Wed, 29 Sep 2010 23:36:58 +0100 Subject: [PATCH 0099/1279] Added a Django storage backend. - New GridFSStorage storage backend - New FileDocument document for storing files in GridFS - Whitespace cleaned up in various files --- docs/changelog.rst | 2 + docs/django.rst | 41 ++++++++++++- mongoengine/base.py | 28 ++++----- mongoengine/django/storage.py | 112 ++++++++++++++++++++++++++++++++++ mongoengine/document.py | 12 ++-- mongoengine/fields.py | 78 ++++++----------------- mongoengine/queryset.py | 20 +++--- tests/connnection.py | 44 +++++++++++++ tests/fields.py | 2 +- 9 files changed, 248 insertions(+), 91 deletions(-) create mode 100644 mongoengine/django/storage.py create mode 100644 tests/connnection.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 8dd5b00..29f49cf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,8 @@ Changelog Changes in v0.4 =============== +- Added ``GridFSStorage`` Django storage backend +- Added ``FileField`` for GridFS support - Added ``SortedListField`` - Added ``EmailField`` - Added ``GeoPointField`` diff --git a/docs/django.rst b/docs/django.rst index 92a8a52..2cce3f0 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -19,7 +19,7 @@ MongoDB but still use many of the Django authentication infrastucture (such as the :func:`login_required` decorator and the :func:`authenticate` function). To enable the MongoEngine auth backend, add the following to you **settings.py** file:: - + AUTHENTICATION_BACKENDS = ( 'mongoengine.django.auth.MongoEngineBackend', ) @@ -44,3 +44,42 @@ into you settings module:: SESSION_ENGINE = 'mongoengine.django.sessions' .. versionadded:: 0.2.1 + +Storage +======= +With MongoEngine's support for GridFS via the FileField, it is useful to have a +Django file storage backend that wraps this. The new storage module is called +GridFSStorage. Using it is very similar to using the default FileSystemStorage.:: + + fs = mongoengine.django.GridFSStorage() + + filename = fs.save('hello.txt', 'Hello, World!') + +All of the `Django Storage API methods +`_ have been +implemented except ``path()``. If the filename provided already exists, an +underscore and a number (before # the file extension, if one exists) will be +appended to the filename until the generated filename doesn't exist. The +``save()`` method will return the new filename.:: + + > fs.exists('hello.txt') + True + > fs.open('hello.txt').read() + 'Hello, World!' + > fs.size('hello.txt') + 13 + > fs.url('hello.txt') + 'http://your_media_url/hello.txt' + > fs.open('hello.txt').name + 'hello.txt' + > fs.listdir() + ([], [u'hello.txt']) + +All files will be saved and retrieved in GridFS via the ``FileDocument`` document, +allowing easy access to the files without the GridFSStorage backend.:: + + > from mongoengine.django.storage import FileDocument + > FileDocument.objects() + [] + +.. versionadded:: 0.4 diff --git a/mongoengine/base.py b/mongoengine/base.py index 836817d..91a12ab 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -23,7 +23,7 @@ class BaseField(object): # Fields may have _types inserted into indexes by default _index_with_types = True _geo_index = False - + def __init__(self, db_field=None, name=None, required=False, default=None, unique=False, unique_with=None, primary_key=False, validation=None, choices=None): @@ -89,7 +89,7 @@ class BaseField(object): if value not in self.choices: raise ValidationError("Value must be one of %s." % unicode(self.choices)) - + # check validation argument if self.validation is not None: if callable(self.validation): @@ -98,13 +98,13 @@ class BaseField(object): 'validation method.') else: raise ValueError('validation argument must be a callable.') - + self.validate(value) class ObjectIdField(BaseField): """An field wrapper around MongoDB's ObjectIds. """ - + def to_python(self, value): return value # return unicode(value) @@ -150,7 +150,7 @@ class DocumentMetaclass(type): # Get superclasses from superclass superclasses[base._class_name] = base superclasses.update(base._superclasses) - + if hasattr(base, '_meta'): # Ensure that the Document class may be subclassed - # inheritance may be disabled to remove dependency on @@ -191,20 +191,20 @@ class DocumentMetaclass(type): field.owner_document = new_class module = attrs.get('__module__') - + base_excs = tuple(base.DoesNotExist for base in bases if hasattr(base, 'DoesNotExist')) or (DoesNotExist,) exc = subclass_exception('DoesNotExist', base_excs, module) new_class.add_to_class('DoesNotExist', exc) - + base_excs = tuple(base.MultipleObjectsReturned for base in bases if hasattr(base, 'MultipleObjectsReturned')) base_excs = base_excs or (MultipleObjectsReturned,) exc = subclass_exception('MultipleObjectsReturned', base_excs, module) new_class.add_to_class('MultipleObjectsReturned', exc) - + return new_class - + def add_to_class(self, name, value): setattr(self, name, value) @@ -227,7 +227,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): return super_new(cls, name, bases, attrs) collection = name.lower() - + id_field = None base_indexes = [] base_meta = {} @@ -265,7 +265,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Set up collection manager, needs the class to have fields so use # DocumentMetaclass before instantiating CollectionManager object new_class = super_new(cls, name, bases, attrs) - + # Provide a default queryset unless one has been manually provided if not hasattr(new_class, 'objects'): new_class.objects = QuerySetManager() @@ -273,7 +273,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): user_indexes = [QuerySet._build_index_spec(new_class, spec) for spec in meta['indexes']] + base_indexes new_class._meta['indexes'] = user_indexes - + unique_indexes = [] for field_name, field in new_class._fields.items(): # Generate a list of indexes needed by uniqueness constraints @@ -431,7 +431,7 @@ class BaseDocument(object): if data.has_key('_id') and not data['_id']: del data['_id'] return data - + @classmethod def _from_son(cls, son): """Create an instance of a Document (subclass) from a PyMongo SON. @@ -468,7 +468,7 @@ class BaseDocument(object): obj = cls(**data) obj._present_fields = present_fields return obj - + def __eq__(self, other): if isinstance(other, self.__class__) and hasattr(other, 'id'): if self.id == other.id: diff --git a/mongoengine/django/storage.py b/mongoengine/django/storage.py new file mode 100644 index 0000000..341455c --- /dev/null +++ b/mongoengine/django/storage.py @@ -0,0 +1,112 @@ +import os +import itertools +import urlparse + +from mongoengine import * +from django.conf import settings +from django.core.files.storage import Storage +from django.core.exceptions import ImproperlyConfigured + + +class FileDocument(Document): + """A document used to store a single file in GridFS. + """ + file = FileField() + + +class GridFSStorage(Storage): + """A custom storage backend to store files in GridFS + """ + + def __init__(self, base_url=None): + + if base_url is None: + base_url = settings.MEDIA_URL + self.base_url = base_url + self.document = FileDocument + self.field = 'file' + + def delete(self, name): + """Deletes the specified file from the storage system. + """ + if self.exists(name): + doc = self.document.objects.first() + field = getattr(doc, self.field) + self._get_doc_with_name(name).delete() # Delete the FileField + field.delete() # Delete the FileDocument + + def exists(self, name): + """Returns True if a file referened by the given name already exists in the + storage system, or False if the name is available for a new file. + """ + doc = self._get_doc_with_name(name) + if doc: + field = getattr(doc, self.field) + return bool(field.name) + else: + return False + + def listdir(self, path=None): + """Lists the contents of the specified path, returning a 2-tuple of lists; + the first item being directories, the second item being files. + """ + def name(doc): + return getattr(doc, self.field).name + docs = self.document.objects + return [], [name(d) for d in docs if name(d)] + + def size(self, name): + """Returns the total size, in bytes, of the file specified by name. + """ + doc = self._get_doc_with_name(name) + if doc: + return getattr(doc, self.field).length + else: + raise ValueError("No such file or directory: '%s'" % name) + + def url(self, name): + """Returns an absolute URL where the file's contents can be accessed + directly by a web browser. + """ + if self.base_url is None: + raise ValueError("This file is not accessible via a URL.") + return urlparse.urljoin(self.base_url, name).replace('\\', '/') + + def _get_doc_with_name(self, name): + """Find the documents in the store with the given name + """ + docs = self.document.objects + doc = [d for d in docs if getattr(d, self.field).name == name] + if doc: + return doc[0] + else: + return None + + def _open(self, name, mode='rb'): + doc = self._get_doc_with_name(name) + if doc: + return getattr(doc, self.field) + else: + raise ValueError("No file found with the name '%s'." % name) + + def get_available_name(self, name): + """Returns a filename that's free on the target storage system, and + available for new content to be written to. + """ + file_root, file_ext = os.path.splitext(name) + # If the filename already exists, add an underscore and a number (before + # the file extension, if one exists) to the filename until the generated + # filename doesn't exist. + count = itertools.count(1) + while self.exists(name): + # file_ext includes the dot. + name = os.path.join("%s_%s%s" % (file_root, count.next(), file_ext)) + + return name + + def _save(self, name, content): + doc = self.document() + getattr(doc, self.field).put(content, filename=name) + doc.save() + + return name diff --git a/mongoengine/document.py b/mongoengine/document.py index e5dec14..368b580 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -15,7 +15,7 @@ class EmbeddedDocument(BaseDocument): fields on :class:`~mongoengine.Document`\ s through the :class:`~mongoengine.EmbeddedDocumentField` field type. """ - + __metaclass__ = DocumentMetaclass @@ -119,23 +119,23 @@ class Document(BaseDocument): class MapReduceDocument(object): """A document returned from a map/reduce query. - + :param collection: An instance of :class:`~pymongo.Collection` :param key: Document/result key, often an instance of :class:`~pymongo.objectid.ObjectId`. If supplied as an ``ObjectId`` found in the given ``collection``, the object can be accessed via the ``object`` property. :param value: The result(s) for this key. - + .. versionadded:: 0.3 """ - + def __init__(self, document, collection, key, value): self._document = document self._collection = collection self.key = key self.value = value - + @property def object(self): """Lazy-load the object referenced by ``self.key``. ``self.key`` @@ -143,7 +143,7 @@ class MapReduceDocument(object): """ id_field = self._document()._meta['id_field'] id_field_type = type(id_field) - + if not isinstance(self.key, id_field_type): try: self.key = id_field_type(self.key) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 76bc4fb..1b689f2 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -16,11 +16,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', 'DecimalField', 'URLField', 'GenericReferenceField', 'FileField', -<<<<<<< HEAD - 'BinaryField', 'SortedListField', 'EmailField', 'GeoLocationField'] -======= 'BinaryField', 'SortedListField', 'EmailField', 'GeoPointField'] ->>>>>>> 32e66b29f44f3015be099851201241caee92054f RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -42,7 +38,7 @@ class StringField(BaseField): if self.max_length is not None and len(value) > self.max_length: raise ValidationError('String value is too long') - + if self.min_length is not None and len(value) < self.min_length: raise ValidationError('String value is too short') @@ -350,7 +346,8 @@ class SortedListField(ListField): def to_mongo(self, value): if self._ordering is not None: - return sorted([self.field.to_mongo(item) for item in value], key=itemgetter(self._ordering)) + return sorted([self.field.to_mongo(item) for item in value], + key=itemgetter(self._ordering)) return sorted([self.field.to_mongo(item) for item in value]) class DictField(BaseField): @@ -514,25 +511,17 @@ class BinaryField(BaseField): if self.max_bytes is not None and len(value) > self.max_bytes: raise ValidationError('Binary value is too long') -<<<<<<< HEAD -======= ->>>>>>> 32e66b29f44f3015be099851201241caee92054f class GridFSProxy(object): """Proxy object to handle writing and reading of files to and from GridFS + + .. versionadded:: 0.4 """ -<<<<<<< HEAD - def __init__(self): - self.fs = gridfs.GridFS(_get_db()) # Filesystem instance - self.newfile = None # Used for partial writes - self.grid_id = None # Store GridFS id for file -======= def __init__(self, grid_id=None): self.fs = gridfs.GridFS(_get_db()) # Filesystem instance self.newfile = None # Used for partial writes self.grid_id = grid_id # Store GridFS id for file ->>>>>>> 32e66b29f44f3015be099851201241caee92054f def __getattr__(self, name): obj = self.get() @@ -543,17 +532,13 @@ class GridFSProxy(object): return self def get(self, id=None): -<<<<<<< HEAD - try: return self.fs.get(id or self.grid_id) - except: return None # File has been deleted -======= if id: self.grid_id = id try: return self.fs.get(id or self.grid_id) except: - return None # File has been deleted ->>>>>>> 32e66b29f44f3015be099851201241caee92054f + # File has been deleted + return None def new_file(self, **kwargs): self.newfile = self.fs.new_file(**kwargs) @@ -575,20 +560,19 @@ class GridFSProxy(object): self.newfile.writelines(lines) def read(self): -<<<<<<< HEAD - try: return self.get().read() - except: return None -======= try: return self.get().read() except: return None ->>>>>>> 32e66b29f44f3015be099851201241caee92054f def delete(self): # Delete file from GridFS, FileField still remains self.fs.delete(self.grid_id) - self.grid_id = None + + #self.grid_id = None + # Doesn't make a difference because will be put back in when + # reinstantiated We should delete all the metadata stored with the + # file too def replace(self, file, **kwargs): self.delete() @@ -601,41 +585,30 @@ class GridFSProxy(object): msg = "The close() method is only necessary after calling write()" warnings.warn(msg) -<<<<<<< HEAD -======= ->>>>>>> 32e66b29f44f3015be099851201241caee92054f class FileField(BaseField): """A GridFS storage field. + + .. versionadded:: 0.4 """ def __init__(self, **kwargs): -<<<<<<< HEAD - self.gridfs = GridFSProxy() -======= ->>>>>>> 32e66b29f44f3015be099851201241caee92054f super(FileField, self).__init__(**kwargs) def __get__(self, instance, owner): if instance is None: return self -<<<<<<< HEAD - return self.gridfs -======= # Check if a file already exists for this model grid_file = instance._data.get(self.name) - if grid_file: - return grid_file + self.grid_file = grid_file + if self.grid_file: + return self.grid_file return GridFSProxy() ->>>>>>> 32e66b29f44f3015be099851201241caee92054f def __set__(self, instance, value): if isinstance(value, file) or isinstance(value, str): # using "FileField() = file/string" notation -<<<<<<< HEAD - self.gridfs.put(value) -======= grid_file = instance._data.get(self.name) # If a file already exists, delete it if grid_file: @@ -649,24 +622,11 @@ class FileField(BaseField): # Create a new proxy object as we don't already have one instance._data[self.name] = GridFSProxy() instance._data[self.name].put(value) ->>>>>>> 32e66b29f44f3015be099851201241caee92054f else: instance._data[self.name] = value def to_mongo(self, value): # Store the GridFS file id in MongoDB -<<<<<<< HEAD - return self.gridfs.grid_id - - def to_python(self, value): - # Use stored value (id) to lookup file in GridFS - return self.gridfs.get() - - def validate(self, value): - assert isinstance(value, GridFSProxy) - assert isinstance(value.grid_id, pymongo.objectid.ObjectId) - -======= if isinstance(value, GridFSProxy) and value.grid_id is not None: return value.grid_id return None @@ -680,6 +640,7 @@ class FileField(BaseField): assert isinstance(value, GridFSProxy) assert isinstance(value.grid_id, pymongo.objectid.ObjectId) + class GeoPointField(BaseField): """A list storing a latitude and longitude. """ @@ -692,10 +653,9 @@ class GeoPointField(BaseField): if not isinstance(value, (list, tuple)): raise ValidationError('GeoPointField can only accept tuples or ' 'lists of (x, y)') - + if not len(value) == 2: raise ValidationError('Value must be a two-dimensional point.') if (not isinstance(value[0], (float, int)) and not isinstance(value[1], (float, int))): raise ValidationError('Both values in point must be float or int.') ->>>>>>> 32e66b29f44f3015be099851201241caee92054f diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 662fa8c..2fb8a9d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -163,7 +163,7 @@ class QuerySet(object): self._where_clause = None self._loaded_fields = [] self._ordering = [] - + # If inheritance is allowed, only return instances and instances of # subclasses of the class being used if document._meta.get('allow_inheritance'): @@ -240,7 +240,7 @@ class QuerySet(object): """An alias of :meth:`~mongoengine.queryset.QuerySet.__call__` """ return self.__call__(*q_objs, **query) - + def all(self): """Returns all documents.""" return self.__call__() @@ -256,7 +256,7 @@ class QuerySet(object): background = self._document._meta.get('index_background', False) drop_dups = self._document._meta.get('index_drop_dups', False) index_opts = self._document._meta.get('index_options', {}) - + # Ensure document-defined indexes are created if self._document._meta['indexes']: for key_or_list in self._document._meta['indexes']: @@ -267,12 +267,12 @@ class QuerySet(object): for index in self._document._meta['unique_indexes']: self._collection.ensure_index(index, unique=True, background=background, drop_dups=drop_dups, **index_opts) - + # If _types is being used (for polymorphism), it needs an index if '_types' in self._query: self._collection.ensure_index('_types', background=background, **index_opts) - + # Ensure all needed field indexes are created for field in self._document._fields.values(): if field.__class__._geo_index: @@ -471,7 +471,7 @@ class QuerySet(object): def in_bulk(self, object_ids): """Retrieve a set of documents by their ids. - + :param object_ids: a list or tuple of ``ObjectId``\ s :rtype: dict of ObjectIds as keys and collection-specific Document subclasses as values. @@ -483,7 +483,7 @@ class QuerySet(object): docs = self._collection.find({'_id': {'$in': object_ids}}) for doc in docs: doc_map[doc['_id']] = self._document._from_son(doc) - + return doc_map def next(self): @@ -637,7 +637,7 @@ class QuerySet(object): # Integer index provided elif isinstance(key, int): return self._document._from_son(self._cursor[key]) - + def distinct(self, field): """Return a list of distinct values for a given field. @@ -649,9 +649,9 @@ class QuerySet(object): def only(self, *fields): """Load only a subset of this document's fields. :: - + post = BlogPost.objects(...).only("title") - + :param fields: fields to include .. versionadded:: 0.3 diff --git a/tests/connnection.py b/tests/connnection.py new file mode 100644 index 0000000..1903a5f --- /dev/null +++ b/tests/connnection.py @@ -0,0 +1,44 @@ +import unittest +import datetime +import pymongo + +import mongoengine.connection +from mongoengine import * +from mongoengine.connection import _get_db, _get_connection + + +class ConnectionTest(unittest.TestCase): + + def tearDown(self): + mongoengine.connection._connection_settings = {} + mongoengine.connection._connections = {} + mongoengine.connection._dbs = {} + + def test_connect(self): + """Ensure that the connect() method works properly. + """ + connect('mongoenginetest') + + conn = _get_connection() + self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + + db = _get_db() + self.assertTrue(isinstance(db, pymongo.database.Database)) + self.assertEqual(db.name, 'mongoenginetest') + + def test_register_connection(self): + """Ensure that connections with different aliases may be registered. + """ + register_connection('testdb', 'mongoenginetest2') + + self.assertRaises(ConnectionError, _get_connection) + conn = _get_connection('testdb') + self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + + db = _get_db('testdb') + self.assertTrue(isinstance(db, pymongo.database.Database)) + self.assertEqual(db.name, 'mongoenginetest2') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/fields.py b/tests/fields.py index 536a9f1..f5f38fc 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -693,7 +693,7 @@ class FieldTest(unittest.TestCase): testfile.name = "Hello, World!" testfile.file.put('Hello, World!') testfile.save() - + # Second instance testfiledupe = TestFile() data = testfiledupe.file.read() # Should be None From 2c8f00410301d3a184e03cf012a0737533478be4 Mon Sep 17 00:00:00 2001 From: sib Date: Thu, 30 Sep 2010 02:53:44 -0300 Subject: [PATCH 0100/1279] added update operator for addToSet --- mongoengine/queryset.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 069ab11..4d0f113 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -662,7 +662,7 @@ class QuerySet(object): """Transform an update spec from Django-style format to Mongo format. """ operators = ['set', 'unset', 'inc', 'dec', 'push', 'push_all', 'pull', - 'pull_all'] + 'pull_all', 'add_to_set'] mongo_update = {} for key, value in update.items(): @@ -680,7 +680,9 @@ class QuerySet(object): op = 'inc' if value > 0: value = -value - + elif op == 'add_to_set': + op = op.replace('_to_set', 'ToSet') + if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] fields = QuerySet._lookup_field(_doc_cls, parts) @@ -688,7 +690,7 @@ class QuerySet(object): # Convert value to proper value field = fields[-1] - if op in (None, 'set', 'unset', 'push', 'pull'): + if op in (None, 'set', 'unset', 'push', 'pull', 'addToSet'): value = field.prepare_query_value(op, value) elif op in ('pushAll', 'pullAll'): value = [field.prepare_query_value(op, v) for v in value] From 72c7a010ff6d782d850c245deee1857f87b9a1f4 Mon Sep 17 00:00:00 2001 From: sib Date: Thu, 30 Sep 2010 03:05:15 -0300 Subject: [PATCH 0101/1279] added unit test for addToSet --- tests/queryset.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 51f9299..10825d0 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -664,6 +664,11 @@ class QuerySetTest(unittest.TestCase): post.reload() self.assertTrue('db' in post.tags and 'nosql' in post.tags) + BlogPost.objects.update_one(add_to_set__tags='unique') + BlogPost.objects.update_one(add_to_set__tags='unique') + post.reload() + self.assertEqual(post.tags.count('unique'), 1) + BlogPost.drop_collection() def test_update_pull(self): From 159923fae237f1c292b1eae901ea2fea6b653763 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 3 Oct 2010 01:48:42 +0100 Subject: [PATCH 0102/1279] Made lists of recursive reference fields possible --- mongoengine/fields.py | 23 ++++++++++++++++++++++- tests/fields.py | 12 +++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 87d52fd..65b397d 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -20,6 +20,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', RECURSIVE_REFERENCE_CONSTANT = 'self' + class StringField(BaseField): """A unicode string field. """ @@ -105,6 +106,7 @@ class URLField(StringField): message = 'This URL appears to be a broken link: %s' % e raise ValidationError(message) + class EmailField(StringField): """A field that validates input as an E-Mail-Address. """ @@ -119,6 +121,7 @@ class EmailField(StringField): if not EmailField.EMAIL_REGEX.match(value): raise ValidationError('Invalid Mail-address: %s' % value) + class IntField(BaseField): """An integer field. """ @@ -142,6 +145,7 @@ class IntField(BaseField): if self.max_value is not None and value > self.max_value: raise ValidationError('Integer value is too large') + class FloatField(BaseField): """An floating point number field. """ @@ -197,6 +201,7 @@ class DecimalField(BaseField): if self.max_value is not None and value > self.max_value: raise ValidationError('Decimal value is too large') + class BooleanField(BaseField): """A boolean field type. @@ -209,6 +214,7 @@ class BooleanField(BaseField): def validate(self, value): assert isinstance(value, bool) + class DateTimeField(BaseField): """A datetime field. """ @@ -216,6 +222,7 @@ class DateTimeField(BaseField): def validate(self, value): assert isinstance(value, datetime.datetime) + class EmbeddedDocumentField(BaseField): """An embedded document field. Only valid values are subclasses of :class:`~mongoengine.EmbeddedDocument`. @@ -331,6 +338,16 @@ class ListField(BaseField): def lookup_member(self, member_name): return self.field.lookup_member(member_name) + def _set_owner_document(self, owner_document): + self.field.owner_document = owner_document + self._owner_document = owner_document + + def _get_owner_document(self, owner_document): + self._owner_document = owner_document + + owner_document = property(_get_owner_document, _set_owner_document) + + class SortedListField(ListField): """A ListField that sorts the contents of its list before writing to the database in order to ensure that a sorted list is always @@ -346,9 +363,11 @@ class SortedListField(ListField): def to_mongo(self, value): if self._ordering is not None: - return sorted([self.field.to_mongo(item) for item in value], key=itemgetter(self._ordering)) + return sorted([self.field.to_mongo(item) for item in value], + key=itemgetter(self._ordering)) return sorted([self.field.to_mongo(item) for item in value]) + class DictField(BaseField): """A dictionary field that wraps a standard Python dictionary. This is similar to an embedded document, but the structure is not defined. @@ -442,6 +461,7 @@ class ReferenceField(BaseField): def lookup_member(self, member_name): return self.document_type._fields.get(member_name) + class GenericReferenceField(BaseField): """A reference to *any* :class:`~mongoengine.document.Document` subclass that will be automatically dereferenced on access (lazily). @@ -640,6 +660,7 @@ class FileField(BaseField): assert isinstance(value, GridFSProxy) assert isinstance(value.grid_id, pymongo.objectid.ObjectId) + class GeoPointField(BaseField): """A list storing a latitude and longitude. """ diff --git a/tests/fields.py b/tests/fields.py index 8c72719..622016c 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -394,14 +394,24 @@ class FieldTest(unittest.TestCase): class Employee(Document): name = StringField() boss = ReferenceField('self') + friends = ListField(ReferenceField('self')) bill = Employee(name='Bill Lumbergh') bill.save() - peter = Employee(name='Peter Gibbons', boss=bill) + + michael = Employee(name='Michael Bolton') + michael.save() + + samir = Employee(name='Samir Nagheenanajar') + samir.save() + + friends = [michael, samir] + peter = Employee(name='Peter Gibbons', boss=bill, friends=friends) peter.save() peter = Employee.objects.with_id(peter.id) self.assertEqual(peter.boss, bill) + self.assertEqual(peter.friends, friends) def test_undefined_reference(self): """Ensure that ReferenceFields may reference undefined Documents. From 4012722a8d15bb3adc6d8581f092882b474b2ced Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 3 Oct 2010 15:01:45 +0100 Subject: [PATCH 0103/1279] QuerySet.item_frequencies works with non-list fields --- mongoengine/queryset.py | 28 ++++++++++++++++++++-------- tests/queryset.py | 9 ++++++++- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 73a4529..016430d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -916,20 +916,27 @@ class QuerySet(object): """ return self.exec_js(average_func, field) - def item_frequencies(self, list_field, normalize=False): - """Returns a dictionary of all items present in a list field across + def item_frequencies(self, field, normalize=False): + """Returns a dictionary of all items present in a field across the whole queried set of documents, and their corresponding frequency. This is useful for generating tag clouds, or searching documents. - :param list_field: the list field to use + If the field is a :class:`~mongoengine.ListField`, the items within + each list will be counted individually. + + :param field: the field to use :param normalize: normalize the results so they add to 1.0 """ freq_func = """ - function(listField) { + function(field) { if (options.normalize) { var total = 0.0; db[collection].find(query).forEach(function(doc) { - total += doc[listField].length; + if (doc[field].constructor == Array) { + total += doc[field].length; + } else { + total++; + } }); } @@ -939,14 +946,19 @@ class QuerySet(object): inc /= total; } db[collection].find(query).forEach(function(doc) { - doc[listField].forEach(function(item) { + if (doc[field].constructor == Array) { + doc[field].forEach(function(item) { + frequencies[item] = inc + (frequencies[item] || 0); + }); + } else { + var item = doc[field]; frequencies[item] = inc + (frequencies[item] || 0); - }); + } }); return frequencies; } """ - return self.exec_js(freq_func, list_field, normalize=normalize) + return self.exec_js(freq_func, field, normalize=normalize) def __repr__(self): limit = REPR_OUTPUT_SIZE + 1 diff --git a/tests/queryset.py b/tests/queryset.py index ab28ff3..62cf495 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -973,7 +973,7 @@ class QuerySetTest(unittest.TestCase): BlogPost(hits=1, tags=['music', 'film', 'actors']).save() BlogPost(hits=2, tags=['music']).save() - BlogPost(hits=3, tags=['music', 'actors']).save() + BlogPost(hits=2, tags=['music', 'actors']).save() f = BlogPost.objects.item_frequencies('tags') f = dict((key, int(val)) for key, val in f.items()) @@ -995,6 +995,13 @@ class QuerySetTest(unittest.TestCase): self.assertAlmostEqual(f['actors'], 2.0/6.0) self.assertAlmostEqual(f['film'], 1.0/6.0) + # Check item_frequencies works for non-list fields + f = BlogPost.objects.item_frequencies('hits') + f = dict((key, int(val)) for key, val in f.items()) + self.assertEqual(set(['1', '2']), set(f.keys())) + self.assertEqual(f['1'], 1) + self.assertEqual(f['2'], 2) + BlogPost.drop_collection() def test_average(self): From 556eed0151ea845bef1cb761f84cc07cadf4329c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 3 Oct 2010 15:22:47 +0100 Subject: [PATCH 0104/1279] QuerySet.distinct respects query. Closes #64. --- mongoengine/queryset.py | 2 +- tests/queryset.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 016430d..48936e6 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -649,7 +649,7 @@ class QuerySet(object): .. versionadded:: 0.4 """ - return self._collection.distinct(field) + return self._cursor.distinct(field) def only(self, *fields): """Load only a subset of this document's fields. :: diff --git a/tests/queryset.py b/tests/queryset.py index 62cf495..2271c36 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1038,9 +1038,13 @@ class QuerySetTest(unittest.TestCase): self.Person(name='Mr Orange', age=20).save() self.Person(name='Mr White', age=20).save() self.Person(name='Mr Orange', age=30).save() - self.assertEqual(self.Person.objects.distinct('name'), - ['Mr Orange', 'Mr White']) - self.assertEqual(self.Person.objects.distinct('age'), [20, 30]) + self.Person(name='Mr Pink', age=30).save() + self.assertEqual(set(self.Person.objects.distinct('name')), + set(['Mr Orange', 'Mr White', 'Mr Pink'])) + self.assertEqual(set(self.Person.objects.distinct('age')), + set([20, 30])) + self.assertEqual(set(self.Person.objects(age=30).distinct('name')), + set(['Mr Orange', 'Mr Pink'])) def test_custom_manager(self): """Ensure that custom QuerySetManager instances work as expected. From 9c9903664a5fe39d6f40728bc877027ec658cb81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Peignier?= Date: Sun, 3 Oct 2010 18:50:35 +0200 Subject: [PATCH 0105/1279] add support for pk property in documents and filters --- docs/guide/document-instances.rst | 7 +++++++ mongoengine/base.py | 25 +++++++++++++++++++------ mongoengine/queryset.py | 3 +++ tests/document.py | 26 ++++++++++++++++++++++++++ tests/queryset.py | 24 +++++++++++++++++++++++- 5 files changed, 78 insertions(+), 7 deletions(-) diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index b5a1f02..7b5d165 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -59,6 +59,13 @@ you may still use :attr:`id` to access the primary key if you want:: >>> bob.id == bob.email == 'bob@example.com' True +You can also access the document's "primary key" using the :attr:`pk` field; in +is an alias to :attr:`id`:: + + >>> page = Page(title="Another Test Page") + >>> page.save() + >>> page.id == page.pk + .. note:: If you define your own primary key field, the field implicitly becomes required, so a :class:`ValidationError` will be thrown if you don't provide diff --git a/mongoengine/base.py b/mongoengine/base.py index 0cbd707..addcd6b 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -330,14 +330,17 @@ class BaseDocument(object): def __init__(self, **values): self._data = {} + # Assign default values to instance + for attr_name in self._fields.keys(): + # Use default value if present + value = getattr(self, attr_name, None) + setattr(self, attr_name, value) # Assign initial values to instance - for attr_name, attr_value in self._fields.items(): - if attr_name in values: + for attr_name in values.keys(): + try: setattr(self, attr_name, values.pop(attr_name)) - else: - # Use default value if present - value = getattr(self, attr_name, None) - setattr(self, attr_name, value) + except AttributeError: + pass def validate(self): """Ensure that all fields' values are valid and that required fields @@ -373,6 +376,16 @@ class BaseDocument(object): all_subclasses.update(subclass._get_subclasses()) return all_subclasses + @apply + def pk(): + """Primary key alias + """ + def fget(self): + return getattr(self, self._meta['id_field']) + def fset(self, value): + return setattr(self, self._meta['id_field'], value) + return property(fget, fset) + def __iter__(self): return iter(self._fields) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 48936e6..69a110f 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -312,6 +312,9 @@ class QuerySet(object): for field_name in parts: if field is None: # Look up first field from the document + if field_name == 'pk': + # Deal with "primary key" alias + field_name = document._meta['id_field'] field = document._fields[field_name] else: # Look up subfield on the previous field diff --git a/tests/document.py b/tests/document.py index c2e0d6f..81f492c 100644 --- a/tests/document.py +++ b/tests/document.py @@ -355,12 +355,26 @@ class DocumentTest(unittest.TestCase): user_obj = User.objects.first() self.assertEqual(user_obj.id, 'test') + self.assertEqual(user_obj.pk, 'test') user_son = User.objects._collection.find_one() self.assertEqual(user_son['_id'], 'test') self.assertTrue('username' not in user_son['_id']) User.drop_collection() + + user = User(pk='mongo', name='mongo user') + user.save() + + user_obj = User.objects.first() + self.assertEqual(user_obj.id, 'mongo') + self.assertEqual(user_obj.pk, 'mongo') + + user_son = User.objects._collection.find_one() + self.assertEqual(user_son['_id'], 'mongo') + self.assertTrue('username' not in user_son['_id']) + + User.drop_collection() def test_creation(self): """Ensure that document may be created using keyword arguments. @@ -479,6 +493,18 @@ class DocumentTest(unittest.TestCase): collection = self.db[self.Person._meta['collection']] person_obj = collection.find_one({'name': 'Test User'}) self.assertEqual(str(person_obj['_id']), '497ce96f395f2f052a494fd4') + + def test_save_custom_pk(self): + """Ensure that a document may be saved with a custom _id using pk alias. + """ + # Create person object and save it to the database + person = self.Person(name='Test User', age=30, + pk='497ce96f395f2f052a494fd4') + person.save() + # Ensure that the object is in the database with the correct _id + collection = self.db[self.Person._meta['collection']] + person_obj = collection.find_one({'name': 'Test User'}) + self.assertEqual(str(person_obj['_id']), '497ce96f395f2f052a494fd4') def test_save_list(self): """Ensure that a list field may be properly saved. diff --git a/tests/queryset.py b/tests/queryset.py index 2271c36..92d3e4c 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1094,7 +1094,8 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() data = {'title': 'Post 1', 'comments': [Comment(content='test')]} - BlogPost(**data).save() + post = BlogPost(**data) + post.save() self.assertTrue('postTitle' in BlogPost.objects(title=data['title'])._query) @@ -1102,12 +1103,33 @@ class QuerySetTest(unittest.TestCase): BlogPost.objects(title=data['title'])._query) self.assertEqual(len(BlogPost.objects(title=data['title'])), 1) + self.assertTrue('_id' in BlogPost.objects(pk=post.id)._query) + self.assertEqual(len(BlogPost.objects(pk=post.id)), 1) + self.assertTrue('postComments.commentContent' in BlogPost.objects(comments__content='test')._query) self.assertEqual(len(BlogPost.objects(comments__content='test')), 1) BlogPost.drop_collection() + def test_query_pk_field_name(self): + """Ensure that the correct "primary key" field name is used when querying + """ + class BlogPost(Document): + title = StringField(primary_key=True, db_field='postTitle') + + BlogPost.drop_collection() + + data = { 'title':'Post 1' } + post = BlogPost(**data) + post.save() + + self.assertTrue('_id' in BlogPost.objects(pk=data['title'])._query) + self.assertTrue('_id' in BlogPost.objects(title=data['title'])._query) + self.assertEqual(len(BlogPost.objects(pk=data['title'])), 1) + + BlogPost.drop_collection() + def test_query_value_conversion(self): """Ensure that query values are properly converted when necessary. """ From 62388cb740deb5be6845225f2294bb94abac30dc Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 3 Oct 2010 21:08:28 +0100 Subject: [PATCH 0106/1279] Started work on new Q-object implementation --- mongoengine/queryset.py | 112 ++++++++++++++++++++++++++++++++++++++++ tests/queryset.py | 25 +++++++-- 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 48936e6..ad3c2de 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -15,6 +15,7 @@ REPR_OUTPUT_SIZE = 20 class DoesNotExist(Exception): pass + class MultipleObjectsReturned(Exception): pass @@ -26,12 +27,123 @@ class InvalidQueryError(Exception): class OperationError(Exception): pass + class InvalidCollectionError(Exception): pass + RE_TYPE = type(re.compile('')) +class QNodeVisitor(object): + + def visit_combination(self, combination): + return combination + + def visit_query(self, query): + return query + + +class SimplificationVisitor(QNodeVisitor): + + def visit_combination(self, combination): + if combination.operation != combination.AND: + return combination + + if any(not isinstance(node, NewQ) for node in combination.children): + return combination + + query_ops = set() + query = {} + for node in combination.children: + ops = set(node.query.keys()) + intersection = ops.intersection(query_ops) + if intersection: + msg = 'Duplicate query contitions: ' + raise InvalidQueryError(msg + ', '.join(intersection)) + + query_ops.update(ops) + query.update(copy.deepcopy(node.query)) + return NewQ(**query) + + +class QueryCompilerVisitor(QNodeVisitor): + + def __init__(self, document): + self.document = document + + def visit_combination(self, combination): + if combination.operation == combination.OR: + return combination + return combination + + def visit_query(self, query): + return QuerySet._transform_query(self.document, **query.query) + + +class QNode(object): + + AND = 0 + OR = 1 + + def to_query(self, document): + query = self.accept(SimplificationVisitor()) + query = query.accept(QueryCompilerVisitor(document)) + return query + + def accept(self, visitor): + raise NotImplementedError + + def _combine(self, other, operation): + if other.empty: + return self + + if self.empty: + return other + + return QCombination(operation, [self, other]) + + @property + def empty(self): + return False + + def __or__(self, other): + return self._combine(other, self.OR) + + def __and__(self, other): + return self._combine(other, self.AND) + + +class QCombination(QNode): + + def __init__(self, operation, children): + self.operation = operation + self.children = children + + def accept(self, visitor): + for i in range(len(self.children)): + self.children[i] = self.children[i].accept(visitor) + + return visitor.visit_combination(self) + + @property + def empty(self): + return not bool(self.query) + + +class NewQ(QNode): + + def __init__(self, **query): + self.query = query + + def accept(self, visitor): + return visitor.visit_query(self) + + @property + def empty(self): + return not bool(self.query) + + class Q(object): OR = '||' diff --git a/tests/queryset.py b/tests/queryset.py index 2271c36..6095251 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -6,7 +6,7 @@ import pymongo from datetime import datetime, timedelta from mongoengine.queryset import (QuerySet, MultipleObjectsReturned, - DoesNotExist) + DoesNotExist, NewQ) from mongoengine import * @@ -53,9 +53,6 @@ class QuerySetTest(unittest.TestCase): person2 = self.Person(name="User B", age=30) person2.save() - q1 = Q(name='test') - q2 = Q(age__gte=18) - # Find all people in the collection people = self.Person.objects self.assertEqual(len(people), 2) @@ -1415,5 +1412,25 @@ class QTest(unittest.TestCase): self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) +class NewQTest(unittest.TestCase): + + def test_and_combination(self): + class TestDoc(Document): + x = IntField() + + # Check than an error is raised when conflicting queries are anded + def invalid_combination(): + query = NewQ(x__lt=7) & NewQ(x__lt=3) + query.to_query(TestDoc) + self.assertRaises(InvalidQueryError, invalid_combination) + + # Check normal cases work without an error + query = NewQ(x__lt=7) & NewQ(x__gt=3) + + q1 = NewQ(x__lt=7) + q2 = NewQ(x__gt=3) + query = (q1 & q2).to_query(TestDoc) + self.assertEqual(query, {'x': {'$lt': 7, '$gt': 3}}) + if __name__ == '__main__': unittest.main() From a3c46fec0778dd6dba5aa9d693eb9cecac530f0c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 3 Oct 2010 21:26:26 +0100 Subject: [PATCH 0107/1279] Compilation of combinations - simple $or now works --- mongoengine/queryset.py | 32 +++++++++++++++++++------------- tests/queryset.py | 14 ++++++++++++++ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index ad3c2de..b3fe29f 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -43,6 +43,20 @@ class QNodeVisitor(object): def visit_query(self, query): return query + def _query_conjunction(self, queries): + query_ops = set() + combined_query = {} + for query in queries: + ops = set(query.keys()) + intersection = ops.intersection(query_ops) + if intersection: + msg = 'Duplicate query contitions: ' + raise InvalidQueryError(msg + ', '.join(intersection)) + + query_ops.update(ops) + combined_query.update(copy.deepcopy(query)) + return combined_query + class SimplificationVisitor(QNodeVisitor): @@ -53,18 +67,8 @@ class SimplificationVisitor(QNodeVisitor): if any(not isinstance(node, NewQ) for node in combination.children): return combination - query_ops = set() - query = {} - for node in combination.children: - ops = set(node.query.keys()) - intersection = ops.intersection(query_ops) - if intersection: - msg = 'Duplicate query contitions: ' - raise InvalidQueryError(msg + ', '.join(intersection)) - - query_ops.update(ops) - query.update(copy.deepcopy(node.query)) - return NewQ(**query) + queries = [node.query for node in combination.children] + return NewQ(**self._query_conjunction(queries)) class QueryCompilerVisitor(QNodeVisitor): @@ -74,7 +78,9 @@ class QueryCompilerVisitor(QNodeVisitor): def visit_combination(self, combination): if combination.operation == combination.OR: - return combination + return {'$or': combination.children} + elif combination.operation == combination.AND: + return self._query_conjunction(combination.children) return combination def visit_query(self, query): diff --git a/tests/queryset.py b/tests/queryset.py index 6095251..6d3114e 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1432,5 +1432,19 @@ class NewQTest(unittest.TestCase): query = (q1 & q2).to_query(TestDoc) self.assertEqual(query, {'x': {'$lt': 7, '$gt': 3}}) + def test_or_combination(self): + class TestDoc(Document): + x = IntField() + + q1 = NewQ(x__lt=3) + q2 = NewQ(x__gt=7) + query = (q1 | q2).to_query(TestDoc) + self.assertEqual(query, { + '$or': [ + {'x': {'$lt': 3}}, + {'x': {'$gt': 7}}, + ] + }) + if __name__ == '__main__': unittest.main() From db2f64c290c5469c0e82952cb3c9ef0c5457b0f8 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 3 Oct 2010 23:01:44 +0100 Subject: [PATCH 0108/1279] Made query-tree code a bit clearer --- mongoengine/queryset.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b3fe29f..21bd44d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -36,18 +36,28 @@ RE_TYPE = type(re.compile('')) class QNodeVisitor(object): + """Base visitor class for visiting Q-object nodes in a query tree. + """ def visit_combination(self, combination): + """Called by QCombination objects. + """ return combination def visit_query(self, query): + """Called by (New)Q objects. + """ return query def _query_conjunction(self, queries): + """Merges two query dicts - effectively &ing them together. + """ query_ops = set() combined_query = {} for query in queries: ops = set(query.keys()) + # Make sure that the same operation isn't applied more than once + # to a single field intersection = ops.intersection(query_ops) if intersection: msg = 'Duplicate query contitions: ' @@ -59,11 +69,15 @@ class QNodeVisitor(object): class SimplificationVisitor(QNodeVisitor): + """Simplifies query trees by combinging unnecessary 'and' connection nodes + into a single Q-object. + """ def visit_combination(self, combination): if combination.operation != combination.AND: return combination + # The simplification only applies to 'simple' queries if any(not isinstance(node, NewQ) for node in combination.children): return combination @@ -72,6 +86,9 @@ class SimplificationVisitor(QNodeVisitor): class QueryCompilerVisitor(QNodeVisitor): + """Compiles the nodes in a query tree to a PyMongo-compatible query + dictionary. + """ def __init__(self, document): self.document = document @@ -88,6 +105,8 @@ class QueryCompilerVisitor(QNodeVisitor): class QNode(object): + """Base class for nodes in query trees. + """ AND = 0 OR = 1 @@ -101,6 +120,8 @@ class QNode(object): raise NotImplementedError def _combine(self, other, operation): + """Combine this node with another node into a QCombination object. + """ if other.empty: return self @@ -121,6 +142,9 @@ class QNode(object): class QCombination(QNode): + """Represents the combination of several conditions by a given logical + operator. + """ def __init__(self, operation, children): self.operation = operation @@ -138,6 +162,9 @@ class QCombination(QNode): class NewQ(QNode): + """A simple query object, used in a query tree to build up more complex + query structures. + """ def __init__(self, **query): self.query = query From c0f7c4ca2ddabb33a5d7dcfd87dfe9f8059844bc Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 3 Oct 2010 23:22:36 +0100 Subject: [PATCH 0109/1279] Fixed error in empty property on QCombination --- mongoengine/queryset.py | 2 +- tests/queryset.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 21bd44d..2c822c7 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -158,7 +158,7 @@ class QCombination(QNode): @property def empty(self): - return not bool(self.query) + return not bool(self.children) class NewQ(QNode): diff --git a/tests/queryset.py b/tests/queryset.py index 6d3114e..6a33764 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1417,6 +1417,7 @@ class NewQTest(unittest.TestCase): def test_and_combination(self): class TestDoc(Document): x = IntField() + y = StringField() # Check than an error is raised when conflicting queries are anded def invalid_combination(): @@ -1432,6 +1433,15 @@ class NewQTest(unittest.TestCase): query = (q1 & q2).to_query(TestDoc) self.assertEqual(query, {'x': {'$lt': 7, '$gt': 3}}) + # More complex nested example + query = NewQ(x__lt=100) & NewQ(y__ne='NotMyString') + query &= NewQ(y__in=['a', 'b', 'c']) & NewQ(x__gt=-100) + mongo_query = { + 'x': {'$lt': 100, '$gt': -100}, + 'y': {'$ne': 'NotMyString', '$in': ['a', 'b', 'c']}, + } + self.assertEqual(query.to_query(TestDoc), mongo_query) + def test_or_combination(self): class TestDoc(Document): x = IntField() From 8e651542015ba7de526477d17a5cf661aca949e2 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 4 Oct 2010 00:06:42 +0100 Subject: [PATCH 0110/1279] Added a tree transformer, got complex ANDs working --- mongoengine/queryset.py | 39 +++++++++++++++++++++++++++++++++++++++ tests/queryset.py | 14 ++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 2c822c7..9650fe1 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -85,6 +85,44 @@ class SimplificationVisitor(QNodeVisitor): return NewQ(**self._query_conjunction(queries)) +class QueryTreeTransformerVisitor(QNodeVisitor): + """Transforms the query tree in to a form that may be used with MongoDB. + """ + + def visit_combination(self, combination): + if combination.operation == combination.AND: + # MongoDB doesn't allow us to have too many $or operations in our + # queries, so the aim is to move the ORs up the tree to one + # 'master' $or. Firstly, we must find all the necessary parts (part + # of an AND combination or just standard Q object), and store them + # separately from the OR parts. + or_parts = [] + and_parts = [] + for node in combination.children: + if isinstance(node, QCombination): + if node.operation == node.OR: + # Any of the children in an $or component may cause + # the query to succeed + or_parts += node.children + elif node.operation == node.AND: + and_parts.append(node) + elif isinstance(node, NewQ): + and_parts.append(node) + + # Now we combine the parts into a usable query. AND together all of + # the necessary parts. Then for each $or part, create a new query + # that ANDs the necessary part with the $or part. + clauses = [] + for or_part in or_parts: + q_object = reduce(lambda a, b: a & b, and_parts, NewQ()) + clauses.append(q_object & or_part) + + # Finally, $or the generated clauses in to one query. Each of the + # clauses is sufficient for the query to succeed. + return reduce(lambda a, b: a | b, clauses, NewQ()) + return combination + + class QueryCompilerVisitor(QNodeVisitor): """Compiles the nodes in a query tree to a PyMongo-compatible query dictionary. @@ -113,6 +151,7 @@ class QNode(object): def to_query(self, document): query = self.accept(SimplificationVisitor()) + query = query.accept(QueryTreeTransformerVisitor()) query = query.accept(QueryCompilerVisitor(document)) return query diff --git a/tests/queryset.py b/tests/queryset.py index 6a33764..f83a8d4 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1456,5 +1456,19 @@ class NewQTest(unittest.TestCase): ] }) + def test_and_or_combination(self): + class TestDoc(Document): + x = IntField() + + query = NewQ(x__gt=0) | NewQ(x__exists=False) + query &= NewQ(x__lt=100) | NewQ(x__in=[100, 200, 3000]) + print query.to_query(TestDoc) +# self.assertEqual(query.to_query(TestDoc, { +# '$or': [ +# {'x': {'$lt': 3}}, +# {'x': {'$gt': 7}}, +# ] +# }) + if __name__ == '__main__': unittest.main() From 3fcc0e97895d1f2d2a1bea150f15d1f8aa35b7e9 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 4 Oct 2010 02:10:37 +0100 Subject: [PATCH 0111/1279] Combining OR nodes works, fixed other Q-object bugs --- mongoengine/queryset.py | 97 +++++++++++++++++++++++++++++++---------- tests/queryset.py | 62 ++++++++++++++++++++++---- 2 files changed, 126 insertions(+), 33 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 9650fe1..06b67ab 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -4,6 +4,7 @@ import pprint import pymongo import re import copy +import itertools __all__ = ['queryset_manager', 'Q', 'InvalidQueryError', 'InvalidCollectionError'] @@ -49,8 +50,22 @@ class QNodeVisitor(object): """ return query + +class SimplificationVisitor(QNodeVisitor): + """Simplifies query trees by combinging unnecessary 'and' connection nodes + into a single Q-object. + """ + + def visit_combination(self, combination): + if combination.operation == combination.AND: + # The simplification only applies to 'simple' queries + if all(isinstance(node, NewQ) for node in combination.children): + queries = [node.query for node in combination.children] + return NewQ(**self._query_conjunction(queries)) + return combination + def _query_conjunction(self, queries): - """Merges two query dicts - effectively &ing them together. + """Merges query dicts - effectively &ing them together. """ query_ops = set() combined_query = {} @@ -68,23 +83,6 @@ class QNodeVisitor(object): return combined_query -class SimplificationVisitor(QNodeVisitor): - """Simplifies query trees by combinging unnecessary 'and' connection nodes - into a single Q-object. - """ - - def visit_combination(self, combination): - if combination.operation != combination.AND: - return combination - - # The simplification only applies to 'simple' queries - if any(not isinstance(node, NewQ) for node in combination.children): - return combination - - queries = [node.query for node in combination.children] - return NewQ(**self._query_conjunction(queries)) - - class QueryTreeTransformerVisitor(QNodeVisitor): """Transforms the query tree in to a form that may be used with MongoDB. """ @@ -96,14 +94,14 @@ class QueryTreeTransformerVisitor(QNodeVisitor): # 'master' $or. Firstly, we must find all the necessary parts (part # of an AND combination or just standard Q object), and store them # separately from the OR parts. - or_parts = [] + or_groups = [] and_parts = [] for node in combination.children: if isinstance(node, QCombination): if node.operation == node.OR: # Any of the children in an $or component may cause # the query to succeed - or_parts += node.children + or_groups.append(node.children) elif node.operation == node.AND: and_parts.append(node) elif isinstance(node, NewQ): @@ -113,13 +111,27 @@ class QueryTreeTransformerVisitor(QNodeVisitor): # the necessary parts. Then for each $or part, create a new query # that ANDs the necessary part with the $or part. clauses = [] - for or_part in or_parts: + for or_group in itertools.product(*or_groups): q_object = reduce(lambda a, b: a & b, and_parts, NewQ()) - clauses.append(q_object & or_part) + q_object = reduce(lambda a, b: a & b, or_group, q_object) + clauses.append(q_object) # Finally, $or the generated clauses in to one query. Each of the # clauses is sufficient for the query to succeed. return reduce(lambda a, b: a | b, clauses, NewQ()) + + if combination.operation == combination.OR: + children = [] + # Crush any nested ORs in to this combination as MongoDB doesn't + # support nested $or operations + for node in combination.children: + if (isinstance(node, QCombination) and + node.operation == combination.OR): + children += node.children + else: + children.append(node) + combination.children = children + return combination @@ -135,12 +147,42 @@ class QueryCompilerVisitor(QNodeVisitor): if combination.operation == combination.OR: return {'$or': combination.children} elif combination.operation == combination.AND: - return self._query_conjunction(combination.children) + return self._mongo_query_conjunction(combination.children) return combination def visit_query(self, query): return QuerySet._transform_query(self.document, **query.query) + def _mongo_query_conjunction(self, queries): + """Merges Mongo query dicts - effectively &ing them together. + """ + combined_query = {} + for query in queries: + for field, ops in query.items(): + if field not in combined_query: + combined_query[field] = ops + else: + # The field is already present in the query the only way + # we can merge is if both the existing value and the new + # value are operation dicts, reject anything else + if (not isinstance(combined_query[field], dict) or + not isinstance(ops, dict)): + message = 'Conflicting values for ' + field + raise InvalidQueryError(message) + + current_ops = set(combined_query[field].keys()) + new_ops = set(ops.keys()) + # Make sure that the same operation isn't applied more than + # once to a single field + intersection = current_ops.intersection(new_ops) + if intersection: + msg = 'Duplicate query contitions: ' + raise InvalidQueryError(msg + ', '.join(intersection)) + + # Right! We've got two non-overlapping dicts of operations! + combined_query[field].update(copy.deepcopy(ops)) + return combined_query + class QNode(object): """Base class for nodes in query trees. @@ -187,7 +229,14 @@ class QCombination(QNode): def __init__(self, operation, children): self.operation = operation - self.children = children + self.children = [] + for node in children: + # If the child is a combination of the same type, we can merge its + # children directly into this combinations children + if isinstance(node, QCombination) and node.operation == operation: + self.children += node.children + else: + self.children.append(node) def accept(self, visitor): for i in range(len(self.children)): diff --git a/tests/queryset.py b/tests/queryset.py index f83a8d4..0f8c9a9 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1415,6 +1415,8 @@ class QTest(unittest.TestCase): class NewQTest(unittest.TestCase): def test_and_combination(self): + """Ensure that Q-objects correctly AND together. + """ class TestDoc(Document): x = IntField() y = StringField() @@ -1443,6 +1445,8 @@ class NewQTest(unittest.TestCase): self.assertEqual(query.to_query(TestDoc), mongo_query) def test_or_combination(self): + """Ensure that Q-objects correctly OR together. + """ class TestDoc(Document): x = IntField() @@ -1457,18 +1461,58 @@ class NewQTest(unittest.TestCase): }) def test_and_or_combination(self): + """Ensure that Q-objects handle ANDing ORed components. + """ class TestDoc(Document): x = IntField() + y = BooleanField() + + query = (NewQ(x__gt=0) | NewQ(x__exists=False)) + query &= NewQ(x__lt=100) + self.assertEqual(query.to_query(TestDoc), { + '$or': [ + {'x': {'$lt': 100, '$gt': 0}}, + {'x': {'$lt': 100, '$exists': False}}, + ] + }) + + q1 = (NewQ(x__gt=0) | NewQ(x__exists=False)) + q2 = (NewQ(x__lt=100) | NewQ(y=True)) + query = (q1 & q2).to_query(TestDoc) + + self.assertEqual(['$or'], query.keys()) + conditions = [ + {'x': {'$lt': 100, '$gt': 0}}, + {'x': {'$lt': 100, '$exists': False}}, + {'x': {'$gt': 0}, 'y': True}, + {'x': {'$exists': False}, 'y': True}, + ] + self.assertEqual(len(conditions), len(query['$or'])) + for condition in conditions: + self.assertTrue(condition in query['$or']) + + def test_or_and_or_combination(self): + """Ensure that Q-objects handle ORing ANDed ORed components. :) + """ + class TestDoc(Document): + x = IntField() + y = BooleanField() + + q1 = (NewQ(x__gt=0) & (NewQ(y=True) | NewQ(y__exists=False))) + q2 = (NewQ(x__lt=100) & (NewQ(y=False) | NewQ(y__exists=False))) + query = (q1 | q2).to_query(TestDoc) + + self.assertEqual(['$or'], query.keys()) + conditions = [ + {'x': {'$gt': 0}, 'y': True}, + {'x': {'$gt': 0}, 'y': {'$exists': False}}, + {'x': {'$lt': 100}, 'y':False}, + {'x': {'$lt': 100}, 'y': {'$exists': False}}, + ] + self.assertEqual(len(conditions), len(query['$or'])) + for condition in conditions: + self.assertTrue(condition in query['$or']) - query = NewQ(x__gt=0) | NewQ(x__exists=False) - query &= NewQ(x__lt=100) | NewQ(x__in=[100, 200, 3000]) - print query.to_query(TestDoc) -# self.assertEqual(query.to_query(TestDoc, { -# '$or': [ -# {'x': {'$lt': 3}}, -# {'x': {'$gt': 7}}, -# ] -# }) if __name__ == '__main__': unittest.main() From 76cb851c40c00b11700689b72b022541de3b422a Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 4 Oct 2010 11:41:07 +0100 Subject: [PATCH 0112/1279] Replaced old Q-object with new, revamped Q-object --- mongoengine/queryset.py | 154 +++++++--------------------------------- tests/queryset.py | 86 ++++++++-------------- 2 files changed, 52 insertions(+), 188 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 06b67ab..2eabfb0 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -59,9 +59,9 @@ class SimplificationVisitor(QNodeVisitor): def visit_combination(self, combination): if combination.operation == combination.AND: # The simplification only applies to 'simple' queries - if all(isinstance(node, NewQ) for node in combination.children): + if all(isinstance(node, Q) for node in combination.children): queries = [node.query for node in combination.children] - return NewQ(**self._query_conjunction(queries)) + return Q(**self._query_conjunction(queries)) return combination def _query_conjunction(self, queries): @@ -104,7 +104,7 @@ class QueryTreeTransformerVisitor(QNodeVisitor): or_groups.append(node.children) elif node.operation == node.AND: and_parts.append(node) - elif isinstance(node, NewQ): + elif isinstance(node, Q): and_parts.append(node) # Now we combine the parts into a usable query. AND together all of @@ -112,13 +112,13 @@ class QueryTreeTransformerVisitor(QNodeVisitor): # that ANDs the necessary part with the $or part. clauses = [] for or_group in itertools.product(*or_groups): - q_object = reduce(lambda a, b: a & b, and_parts, NewQ()) + q_object = reduce(lambda a, b: a & b, and_parts, Q()) q_object = reduce(lambda a, b: a & b, or_group, q_object) clauses.append(q_object) # Finally, $or the generated clauses in to one query. Each of the # clauses is sufficient for the query to succeed. - return reduce(lambda a, b: a | b, clauses, NewQ()) + return reduce(lambda a, b: a | b, clauses, Q()) if combination.operation == combination.OR: children = [] @@ -249,7 +249,7 @@ class QCombination(QNode): return not bool(self.children) -class NewQ(QNode): +class Q(QNode): """A simple query object, used in a query tree to build up more complex query structures. """ @@ -265,124 +265,6 @@ class NewQ(QNode): return not bool(self.query) -class Q(object): - - OR = '||' - AND = '&&' - OPERATORS = { - 'eq': ('((this.%(field)s instanceof Array) && ' - ' this.%(field)s.indexOf(%(value)s) != -1) ||' - ' this.%(field)s == %(value)s'), - 'ne': 'this.%(field)s != %(value)s', - 'gt': 'this.%(field)s > %(value)s', - 'gte': 'this.%(field)s >= %(value)s', - 'lt': 'this.%(field)s < %(value)s', - 'lte': 'this.%(field)s <= %(value)s', - 'lte': 'this.%(field)s <= %(value)s', - 'in': '%(value)s.indexOf(this.%(field)s) != -1', - 'nin': '%(value)s.indexOf(this.%(field)s) == -1', - 'mod': '%(field)s %% %(value)s', - 'all': ('%(value)s.every(function(a){' - 'return this.%(field)s.indexOf(a) != -1 })'), - 'size': 'this.%(field)s.length == %(value)s', - 'exists': 'this.%(field)s != null', - 'regex_eq': '%(value)s.test(this.%(field)s)', - 'regex_ne': '!%(value)s.test(this.%(field)s)', - } - - def __init__(self, **query): - self.query = [query] - - def _combine(self, other, op): - obj = Q() - if not other.query[0]: - return self - if self.query[0]: - obj.query = (['('] + copy.deepcopy(self.query) + [op] + - copy.deepcopy(other.query) + [')']) - else: - obj.query = copy.deepcopy(other.query) - return obj - - def __or__(self, other): - return self._combine(other, self.OR) - - def __and__(self, other): - return self._combine(other, self.AND) - - def as_js(self, document): - js = [] - js_scope = {} - for i, item in enumerate(self.query): - if isinstance(item, dict): - item_query = QuerySet._transform_query(document, **item) - # item_query will values will either be a value or a dict - js.append(self._item_query_as_js(item_query, js_scope, i)) - else: - js.append(item) - return pymongo.code.Code(' '.join(js), js_scope) - - def _item_query_as_js(self, item_query, js_scope, item_num): - # item_query will be in one of the following forms - # {'age': 25, 'name': 'Test'} - # {'age': {'$lt': 25}, 'name': {'$in': ['Test', 'Example']} - # {'age': {'$lt': 25, '$gt': 18}} - js = [] - for i, (key, value) in enumerate(item_query.items()): - op = 'eq' - # Construct a variable name for the value in the JS - value_name = 'i%sf%s' % (item_num, i) - if isinstance(value, dict): - # Multiple operators for this field - for j, (op, value) in enumerate(value.items()): - # Create a custom variable name for this operator - op_value_name = '%so%s' % (value_name, j) - # Construct the JS that uses this op - value, operation_js = self._build_op_js(op, key, value, - op_value_name) - # Update the js scope with the value for this op - js_scope[op_value_name] = value - js.append(operation_js) - else: - # Construct the JS for this field - value, field_js = self._build_op_js(op, key, value, value_name) - js_scope[value_name] = value - js.append(field_js) - return ' && '.join(js) - - def _build_op_js(self, op, key, value, value_name): - """Substitute the values in to the correct chunk of Javascript. - """ - if isinstance(value, RE_TYPE): - # Regexes are handled specially - if op.strip('$') == 'ne': - op_js = Q.OPERATORS['regex_ne'] - else: - op_js = Q.OPERATORS['regex_eq'] - else: - op_js = Q.OPERATORS[op.strip('$')] - - # Comparing two ObjectIds in Javascript doesn't work.. - if isinstance(value, pymongo.objectid.ObjectId): - value = unicode(value) - - # Handle DBRef - if isinstance(value, pymongo.dbref.DBRef): - op_js = '(this.%(field)s.$id == "%(id)s" &&'\ - ' this.%(field)s.$ref == "%(ref)s")' % { - 'field': key, - 'id': unicode(value.id), - 'ref': unicode(value.collection) - } - value = None - - # Perform the substitution - operation_js = op_js % { - 'field': key, - 'value': value_name - } - return value, operation_js - class QuerySet(object): """A set of results returned from a query. Wraps a MongoDB cursor, providing :class:`~mongoengine.Document` objects as the results. @@ -392,7 +274,9 @@ class QuerySet(object): self._document = document self._collection_obj = collection self._accessed_collection = False - self._query = {} + self._mongo_query = None + self._query_obj = Q() + self._initial_query = {} self._where_clause = None self._loaded_fields = [] self._ordering = [] @@ -400,11 +284,18 @@ class QuerySet(object): # If inheritance is allowed, only return instances and instances of # subclasses of the class being used if document._meta.get('allow_inheritance'): - self._query = {'_types': self._document._class_name} + self._initial_query = {'_types': self._document._class_name} self._cursor_obj = None self._limit = None self._skip = None + @property + def _query(self): + if self._mongo_query is None: + self._mongo_query = self._query_obj.to_query(self._document) + self._mongo_query.update(self._initial_query) + return self._mongo_query + def ensure_index(self, key_or_list, drop_dups=False, background=False, **kwargs): """Ensure that the given indexes are in place. @@ -463,10 +354,13 @@ class QuerySet(object): objects, only the last one will be used :param query: Django-style query keyword arguments """ + #if q_obj: + #self._where_clause = q_obj.as_js(self._document) + query = Q(**query) if q_obj: - self._where_clause = q_obj.as_js(self._document) - query = QuerySet._transform_query(_doc_cls=self._document, **query) - self._query.update(query) + query &= q_obj + self._query_obj &= query + self._mongo_query = None return self def filter(self, *q_objs, **query): @@ -616,7 +510,7 @@ class QuerySet(object): "been implemented" % op) elif op not in match_operators: value = {'$' + op: value} - + for i, part in indices: parts.insert(i, part) key = '.'.join(parts) diff --git a/tests/queryset.py b/tests/queryset.py index 0f8c9a9..c941ecd 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -6,7 +6,7 @@ import pymongo from datetime import datetime, timedelta from mongoengine.queryset import (QuerySet, MultipleObjectsReturned, - DoesNotExist, NewQ) + DoesNotExist) from mongoengine import * @@ -153,7 +153,8 @@ class QuerySetTest(unittest.TestCase): # Retrieve the first person from the database self.assertRaises(MultipleObjectsReturned, self.Person.objects.get) - self.assertRaises(self.Person.MultipleObjectsReturned, self.Person.objects.get) + self.assertRaises(self.Person.MultipleObjectsReturned, + self.Person.objects.get) # Use a query to filter the people found to just person2 person = self.Person.objects.get(age=30) @@ -231,7 +232,8 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(created, False) # Try retrieving when no objects exists - new doc should be created - person, created = self.Person.objects.get_or_create(age=50, defaults={'name': 'User C'}) + kwargs = dict(age=50, defaults={'name': 'User C'}) + person, created = self.Person.objects.get_or_create(**kwargs) self.assertEqual(created, True) person = self.Person.objects.get(age=50) @@ -545,6 +547,7 @@ class QuerySetTest(unittest.TestCase): obj = self.Person.objects(Q(name__ne=re.compile('^bob'))).first() self.assertEqual(obj, person) + obj = self.Person.objects(Q(name__ne=re.compile('^Gui'))).first() self.assertEqual(obj, None) @@ -1343,43 +1346,6 @@ class QuerySetTest(unittest.TestCase): class QTest(unittest.TestCase): - def test_or_and(self): - """Ensure that Q objects may be combined correctly. - """ - q1 = Q(name='test') - q2 = Q(age__gte=18) - - query = ['(', {'name': 'test'}, '||', {'age__gte': 18}, ')'] - self.assertEqual((q1 | q2).query, query) - - query = ['(', {'name': 'test'}, '&&', {'age__gte': 18}, ')'] - self.assertEqual((q1 & q2).query, query) - - query = ['(', '(', {'name': 'test'}, '&&', {'age__gte': 18}, ')', '||', - {'name': 'example'}, ')'] - self.assertEqual((q1 & q2 | Q(name='example')).query, query) - - def test_item_query_as_js(self): - """Ensure that the _item_query_as_js utilitiy method works properly. - """ - q = Q() - examples = [ - - ({'name': 'test'}, ('((this.name instanceof Array) && ' - 'this.name.indexOf(i0f0) != -1) || this.name == i0f0'), - {'i0f0': 'test'}), - ({'age': {'$gt': 18}}, 'this.age > i0f0o0', {'i0f0o0': 18}), - ({'name': 'test', 'age': {'$gt': 18, '$lte': 65}}, - ('this.age <= i0f0o0 && this.age > i0f0o1 && ' - '((this.name instanceof Array) && ' - 'this.name.indexOf(i0f1) != -1) || this.name == i0f1'), - {'i0f0o0': 65, 'i0f0o1': 18, 'i0f1': 'test'}), - ] - for item, js, scope in examples: - test_scope = {} - self.assertEqual(q._item_query_as_js(item, test_scope, 0), js) - self.assertEqual(scope, test_scope) - def test_empty_q(self): """Ensure that empty Q objects won't hurt. """ @@ -1389,11 +1355,15 @@ class QTest(unittest.TestCase): q4 = Q(name='test') q5 = Q() - query = ['(', {'age__gte': 18}, '||', {'name': 'test'}, ')'] - self.assertEqual((q1 | q2 | q3 | q4 | q5).query, query) + class Person(Document): + name = StringField() + age = IntField() - query = ['(', {'age__gte': 18}, '&&', {'name': 'test'}, ')'] - self.assertEqual((q1 & q2 & q3 & q4 & q5).query, query) + query = {'$or': [{'age': {'$gte': 18}}, {'name': 'test'}]} + self.assertEqual((q1 | q2 | q3 | q4 | q5).to_query(Person), query) + + query = {'age': {'$gte': 18}, 'name': 'test'} + self.assertEqual((q1 & q2 & q3 & q4 & q5).to_query(Person), query) def test_q_with_dbref(self): """Ensure Q objects handle DBRefs correctly""" @@ -1423,21 +1393,21 @@ class NewQTest(unittest.TestCase): # Check than an error is raised when conflicting queries are anded def invalid_combination(): - query = NewQ(x__lt=7) & NewQ(x__lt=3) + query = Q(x__lt=7) & Q(x__lt=3) query.to_query(TestDoc) self.assertRaises(InvalidQueryError, invalid_combination) # Check normal cases work without an error - query = NewQ(x__lt=7) & NewQ(x__gt=3) + query = Q(x__lt=7) & Q(x__gt=3) - q1 = NewQ(x__lt=7) - q2 = NewQ(x__gt=3) + q1 = Q(x__lt=7) + q2 = Q(x__gt=3) query = (q1 & q2).to_query(TestDoc) self.assertEqual(query, {'x': {'$lt': 7, '$gt': 3}}) # More complex nested example - query = NewQ(x__lt=100) & NewQ(y__ne='NotMyString') - query &= NewQ(y__in=['a', 'b', 'c']) & NewQ(x__gt=-100) + query = Q(x__lt=100) & Q(y__ne='NotMyString') + query &= Q(y__in=['a', 'b', 'c']) & Q(x__gt=-100) mongo_query = { 'x': {'$lt': 100, '$gt': -100}, 'y': {'$ne': 'NotMyString', '$in': ['a', 'b', 'c']}, @@ -1450,8 +1420,8 @@ class NewQTest(unittest.TestCase): class TestDoc(Document): x = IntField() - q1 = NewQ(x__lt=3) - q2 = NewQ(x__gt=7) + q1 = Q(x__lt=3) + q2 = Q(x__gt=7) query = (q1 | q2).to_query(TestDoc) self.assertEqual(query, { '$or': [ @@ -1467,8 +1437,8 @@ class NewQTest(unittest.TestCase): x = IntField() y = BooleanField() - query = (NewQ(x__gt=0) | NewQ(x__exists=False)) - query &= NewQ(x__lt=100) + query = (Q(x__gt=0) | Q(x__exists=False)) + query &= Q(x__lt=100) self.assertEqual(query.to_query(TestDoc), { '$or': [ {'x': {'$lt': 100, '$gt': 0}}, @@ -1476,8 +1446,8 @@ class NewQTest(unittest.TestCase): ] }) - q1 = (NewQ(x__gt=0) | NewQ(x__exists=False)) - q2 = (NewQ(x__lt=100) | NewQ(y=True)) + q1 = (Q(x__gt=0) | Q(x__exists=False)) + q2 = (Q(x__lt=100) | Q(y=True)) query = (q1 & q2).to_query(TestDoc) self.assertEqual(['$or'], query.keys()) @@ -1498,8 +1468,8 @@ class NewQTest(unittest.TestCase): x = IntField() y = BooleanField() - q1 = (NewQ(x__gt=0) & (NewQ(y=True) | NewQ(y__exists=False))) - q2 = (NewQ(x__lt=100) & (NewQ(y=False) | NewQ(y__exists=False))) + q1 = (Q(x__gt=0) & (Q(y=True) | Q(y__exists=False))) + q2 = (Q(x__lt=100) & (Q(y=False) | Q(y__exists=False))) query = (q1 | q2).to_query(TestDoc) self.assertEqual(['$or'], query.keys()) From b4c54b1b6257a2b5a262eb0c794aa1371f7aa1fb Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 4 Oct 2010 11:41:49 +0100 Subject: [PATCH 0113/1279] Added support for the $not operator --- mongoengine/queryset.py | 130 +++++++++++++++++++++++++++++++++++++++- tests/queryset.py | 16 ++++- 2 files changed, 142 insertions(+), 4 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 2eabfb0..e275bb3 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -265,6 +265,124 @@ class Q(QNode): return not bool(self.query) +class OldQ(object): + + OR = '||' + AND = '&&' + OPERATORS = { + 'eq': ('((this.%(field)s instanceof Array) && ' + ' this.%(field)s.indexOf(%(value)s) != -1) ||' + ' this.%(field)s == %(value)s'), + 'ne': 'this.%(field)s != %(value)s', + 'gt': 'this.%(field)s > %(value)s', + 'gte': 'this.%(field)s >= %(value)s', + 'lt': 'this.%(field)s < %(value)s', + 'lte': 'this.%(field)s <= %(value)s', + 'lte': 'this.%(field)s <= %(value)s', + 'in': '%(value)s.indexOf(this.%(field)s) != -1', + 'nin': '%(value)s.indexOf(this.%(field)s) == -1', + 'mod': '%(field)s %% %(value)s', + 'all': ('%(value)s.every(function(a){' + 'return this.%(field)s.indexOf(a) != -1 })'), + 'size': 'this.%(field)s.length == %(value)s', + 'exists': 'this.%(field)s != null', + 'regex_eq': '%(value)s.test(this.%(field)s)', + 'regex_ne': '!%(value)s.test(this.%(field)s)', + } + + def __init__(self, **query): + self.query = [query] + + def _combine(self, other, op): + obj = Q() + if not other.query[0]: + return self + if self.query[0]: + obj.query = (['('] + copy.deepcopy(self.query) + [op] + + copy.deepcopy(other.query) + [')']) + else: + obj.query = copy.deepcopy(other.query) + return obj + + def __or__(self, other): + return self._combine(other, self.OR) + + def __and__(self, other): + return self._combine(other, self.AND) + + def as_js(self, document): + js = [] + js_scope = {} + for i, item in enumerate(self.query): + if isinstance(item, dict): + item_query = QuerySet._transform_query(document, **item) + # item_query will values will either be a value or a dict + js.append(self._item_query_as_js(item_query, js_scope, i)) + else: + js.append(item) + return pymongo.code.Code(' '.join(js), js_scope) + + def _item_query_as_js(self, item_query, js_scope, item_num): + # item_query will be in one of the following forms + # {'age': 25, 'name': 'Test'} + # {'age': {'$lt': 25}, 'name': {'$in': ['Test', 'Example']} + # {'age': {'$lt': 25, '$gt': 18}} + js = [] + for i, (key, value) in enumerate(item_query.items()): + op = 'eq' + # Construct a variable name for the value in the JS + value_name = 'i%sf%s' % (item_num, i) + if isinstance(value, dict): + # Multiple operators for this field + for j, (op, value) in enumerate(value.items()): + # Create a custom variable name for this operator + op_value_name = '%so%s' % (value_name, j) + # Construct the JS that uses this op + value, operation_js = self._build_op_js(op, key, value, + op_value_name) + # Update the js scope with the value for this op + js_scope[op_value_name] = value + js.append(operation_js) + else: + # Construct the JS for this field + value, field_js = self._build_op_js(op, key, value, value_name) + js_scope[value_name] = value + js.append(field_js) + return ' && '.join(js) + + def _build_op_js(self, op, key, value, value_name): + """Substitute the values in to the correct chunk of Javascript. + """ + if isinstance(value, RE_TYPE): + # Regexes are handled specially + if op.strip('$') == 'ne': + op_js = Q.OPERATORS['regex_ne'] + else: + op_js = Q.OPERATORS['regex_eq'] + else: + op_js = Q.OPERATORS[op.strip('$')] + + # Comparing two ObjectIds in Javascript doesn't work.. + if isinstance(value, pymongo.objectid.ObjectId): + value = unicode(value) + + # Handle DBRef + if isinstance(value, pymongo.dbref.DBRef): + op_js = '(this.%(field)s.$id == "%(id)s" &&'\ + ' this.%(field)s.$ref == "%(ref)s")' % { + 'field': key, + 'id': unicode(value.id), + 'ref': unicode(value.collection) + } + value = None + + # Perform the substitution + operation_js = op_js % { + 'field': key, + 'value': value_name + } + return value, operation_js + class QuerySet(object): """A set of results returned from a query. Wraps a MongoDB cursor, providing :class:`~mongoengine.Document` objects as the results. @@ -462,7 +580,7 @@ class QuerySet(object): """Transform a query from Django-style format to Mongo format. """ operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', - 'all', 'size', 'exists'] + 'all', 'size', 'exists', 'not'] geo_operators = ['within_distance', 'within_box', 'near'] match_operators = ['contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith', @@ -478,6 +596,11 @@ class QuerySet(object): if parts[-1] in operators + match_operators + geo_operators: op = parts.pop() + negate = False + if parts[-1] == 'not': + parts.pop() + negate = True + if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] fields = QuerySet._lookup_field(_doc_cls, parts) @@ -485,7 +608,7 @@ class QuerySet(object): # Convert value to proper value field = fields[-1] - singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte'] + singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte', 'not'] singular_ops += match_operators if op in singular_ops: value = field.prepare_query_value(op, value) @@ -511,6 +634,9 @@ class QuerySet(object): elif op not in match_operators: value = {'$' + op: value} + if negate: + value = {'$not': value} + for i, part in indices: parts.insert(i, part) key = '.'.join(parts) diff --git a/tests/queryset.py b/tests/queryset.py index c941ecd..32d1902 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -336,6 +336,18 @@ class QuerySetTest(unittest.TestCase): obj = self.Person.objects(Q(name__icontains='[.\'Geek')).first() self.assertEqual(obj, person) + def test_not(self): + """Ensure that the __not operator works as expected. + """ + alice = self.Person(name='Alice', age=25) + alice.save() + + obj = self.Person.objects(name__iexact='alice').first() + self.assertEqual(obj, alice) + + obj = self.Person.objects(name__not__iexact='alice').first() + self.assertEqual(obj, None) + def test_filter_chaining(self): """Ensure filters can be chained together. """ @@ -545,10 +557,10 @@ class QuerySetTest(unittest.TestCase): obj = self.Person.objects(Q(name=re.compile('^gui', re.I))).first() self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__ne=re.compile('^bob'))).first() + obj = self.Person.objects(Q(name__not=re.compile('^bob'))).first() self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__ne=re.compile('^Gui'))).first() + obj = self.Person.objects(Q(name__not=re.compile('^Gui'))).first() self.assertEqual(obj, None) def test_q_lists(self): From 4742328b908e57b3d87e2c0752ed8d4b51763e55 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 4 Oct 2010 12:10:29 +0100 Subject: [PATCH 0114/1279] Delete stale cursor when query is filtered. Closes #62. --- mongoengine/queryset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index e275bb3..6d4ad55 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -479,6 +479,7 @@ class QuerySet(object): query &= q_obj self._query_obj &= query self._mongo_query = None + self._cursor_obj = None return self def filter(self, *q_objs, **query): From 3acfd907202afa4f2712f6e7fce9c34c2e67c8a0 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 4 Oct 2010 14:58:00 +0100 Subject: [PATCH 0115/1279] Added some imports for PyMongo 1.9 compatibility. --- mongoengine/base.py | 1 + mongoengine/fields.py | 3 +++ mongoengine/queryset.py | 3 +++ 3 files changed, 7 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index 0cbd707..1f7ba1f 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -3,6 +3,7 @@ from queryset import DoesNotExist, MultipleObjectsReturned import sys import pymongo +import pymongo.objectid _document_registry = {} diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 65b397d..8fcb1d6 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -5,6 +5,9 @@ from operator import itemgetter import re import pymongo +import pymongo.dbref +import pymongo.son +import pymongo.binary import datetime import decimal import gridfs diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 48936e6..9941785 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -2,6 +2,9 @@ from connection import _get_db import pprint import pymongo +import pymongo.code +import pymongo.dbref +import pymongo.objectid import re import copy From 92471445ec5aede3d6ffb14e6b63026156c6d623 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Tue, 5 Oct 2010 00:46:13 +0100 Subject: [PATCH 0116/1279] Fix changing databases Conflicts: mongoengine/connection.py mongoengine/queryset.py --- mongoengine/connection.py | 22 ++++++++++++---------- mongoengine/queryset.py | 18 ++++++++++-------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 94cc6ea..814fde1 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -4,11 +4,12 @@ import multiprocessing __all__ = ['ConnectionError', 'connect'] -_connection_settings = { +_connection_defaults = { 'host': 'localhost', 'port': 27017, } _connection = {} +_connection_settings = _connection_defaults.copy() _db_name = None _db_username = None @@ -20,25 +21,25 @@ class ConnectionError(Exception): pass -def _get_connection(): +def _get_connection(reconnect=False): global _connection identity = get_identity() # Connect to the database if not already connected - if _connection.get(identity) is None: + if _connection.get(identity) is None or reconnect: try: _connection[identity] = Connection(**_connection_settings) except: raise ConnectionError('Cannot connect to the database') return _connection[identity] -def _get_db(): +def _get_db(reconnect=False): global _db, _connection identity = get_identity() # Connect if not already connected - if _connection.get(identity) is None: - _connection[identity] = _get_connection() + if _connection.get(identity) is None or reconnect: + _connection[identity] = _get_connection(reconnect=reconnect) - if _db.get(identity) is None: + if _db.get(identity) is None or reconnect: # _db_name will be None if the user hasn't called connect() if _db_name is None: raise ConnectionError('Not connected to the database') @@ -61,9 +62,10 @@ def connect(db, username=None, password=None, **kwargs): the default port on localhost. If authentication is needed, provide username and password arguments as well. """ - global _connection_settings, _db_name, _db_username, _db_password - _connection_settings.update(kwargs) + global _connection_settings, _db_name, _db_username, _db_password, _db + _connection_settings = dict(_connection_defaults, **kwargs) _db_name = db _db_username = username _db_password = password - return _get_db() \ No newline at end of file + return _get_db(reconnect=True) + diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 9941785..fae2aab 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -977,7 +977,7 @@ class QuerySetManager(object): def __init__(self, manager_func=None): self._manager_func = manager_func - self._collection = None + self._collections = {} def __get__(self, instance, owner): """Descriptor for instantiating a new QuerySet object when @@ -987,8 +987,8 @@ class QuerySetManager(object): # Document class being used rather than a document object return self - if self._collection is None: - db = _get_db() + db = _get_db() + if db not in self._collections: collection = owner._meta['collection'] # Create collection as a capped collection if specified @@ -998,10 +998,10 @@ class QuerySetManager(object): max_documents = owner._meta['max_documents'] if collection in db.collection_names(): - self._collection = db[collection] + self._collections[db] = db[collection] # The collection already exists, check if its capped # options match the specified capped options - options = self._collection.options() + options = self._collections[db].options() if options.get('max') != max_documents or \ options.get('size') != max_size: msg = ('Cannot create collection "%s" as a capped ' @@ -1012,13 +1012,15 @@ class QuerySetManager(object): opts = {'capped': True, 'size': max_size} if max_documents: opts['max'] = max_documents - self._collection = db.create_collection(collection, **opts) + self._collections[db] = db.create_collection( + collection, **opts + ) else: - self._collection = db[collection] + self._collections[db] = db[collection] # owner is the document that contains the QuerySetManager queryset_class = owner._meta['queryset_class'] or QuerySet - queryset = queryset_class(owner, self._collection) + queryset = queryset_class(owner, self._collections[db]) if self._manager_func: if self._manager_func.func_code.co_argcount == 1: queryset = self._manager_func(queryset) From 833fa3d94dcb35fd10131894f1d45f57000e0fbe Mon Sep 17 00:00:00 2001 From: Jaime Date: Wed, 6 Oct 2010 19:56:12 +0100 Subject: [PATCH 0117/1279] Added note about the use of default parameters --- docs/guide/defining-documents.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 3c27686..a8e9924 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -66,6 +66,25 @@ arguments can be set on all fields: :attr:`default` (Default: None) A value to use when no value is set for this field. + The definion of default parameters follow `the general rules on Python + `__, + which means that some care should be taken when dealing with default mutable objects + (like in :class:`~mongoengine.ListField` or :class:`~mongoengine.DictField`):: + + class ExampleFirst(Document): + # Default an empty list + values = ListField(IntField(), default=list) + + class ExampleSecond(Document): + # Default a set of values + values = ListField(IntField(), default=lambda: [1,2,3]) + + class ExampleDangerous(Document): + # This can make an .append call to add values to the default (and all the following objects), + # instead to just an object + values = ListField(IntField(), default=[1,2,3]) + + :attr:`unique` (Default: False) When True, no documents in the collection will have the same value for this field. From 34fa5cd241b4bbc59877c8891b6214bfd314b88c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 4 Oct 2010 14:58:00 +0100 Subject: [PATCH 0118/1279] Added some imports for PyMongo 1.9 compatibility. --- mongoengine/base.py | 1 + mongoengine/fields.py | 3 +++ mongoengine/queryset.py | 3 +++ 3 files changed, 7 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index c8c162b..22347a2 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -3,6 +3,7 @@ from queryset import DoesNotExist, MultipleObjectsReturned import sys import pymongo +import pymongo.objectid _document_registry = {} diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 127f029..7883f78 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -5,6 +5,9 @@ from operator import itemgetter import re import pymongo +import pymongo.dbref +import pymongo.son +import pymongo.binary import datetime import decimal diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 069ab11..faf0cf4 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1,6 +1,9 @@ from connection import _get_db import pymongo +import pymongo.code +import pymongo.dbref +import pymongo.objectid import re import copy From f6661419816157bec89db3f97b4f8d4a95c67897 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 13:23:11 +0100 Subject: [PATCH 0119/1279] Added test for list of referencefields --- tests/fields.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/fields.py b/tests/fields.py index 622016c..e30f843 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -189,6 +189,9 @@ class FieldTest(unittest.TestCase): def test_list_validation(self): """Ensure that a list field only accepts lists with valid elements. """ + class User(Document): + pass + class Comment(EmbeddedDocument): content = StringField() @@ -196,6 +199,7 @@ class FieldTest(unittest.TestCase): content = StringField() comments = ListField(EmbeddedDocumentField(Comment)) tags = ListField(StringField()) + authors = ListField(ReferenceField(User)) post = BlogPost(content='Went for a walk today...') post.validate() @@ -210,15 +214,21 @@ class FieldTest(unittest.TestCase): post.tags = ('fun', 'leisure') post.validate() - comments = [Comment(content='Good for you'), Comment(content='Yay.')] - post.comments = comments - post.validate() - post.comments = ['a'] self.assertRaises(ValidationError, post.validate) post.comments = 'yay' self.assertRaises(ValidationError, post.validate) + comments = [Comment(content='Good for you'), Comment(content='Yay.')] + post.comments = comments + post.validate() + + post.authors = [Comment()] + self.assertRaises(ValidationError, post.validate) + + post.authors = [User()] + post.validate() + def test_sorted_list_sorting(self): """Ensure that a sorted list field properly sorts values. """ From 3591593ac73dbd0a2275e5335b0ceaace232048d Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 13:55:48 +0100 Subject: [PATCH 0120/1279] Fixed GenericReferenceField query issue --- mongoengine/fields.py | 2 +- mongoengine/queryset.py | 3 --- tests/fields.py | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f65ca15..9fc50d6 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -509,7 +509,7 @@ class GenericReferenceField(BaseField): return {'_cls': document.__class__.__name__, '_ref': ref} def prepare_query_value(self, op, value): - return self.to_mongo(value)['_ref'] + return self.to_mongo(value) class BinaryField(BaseField): diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a3d4c54..06524dc 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -624,9 +624,6 @@ class QuerySet(object): # 'in', 'nin' and 'all' require a list of values value = [field.prepare_query_value(op, v) for v in value] - if field.__class__.__name__ == 'GenericReferenceField': - parts.append('_ref') - # if op and op not in match_operators: if op: if op in geo_operators: diff --git a/tests/fields.py b/tests/fields.py index 1df484b..e30f843 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -545,7 +545,6 @@ class FieldTest(unittest.TestCase): user.save() user = User.objects(bookmarks__all=[post_1, link_1]).first() - print User.objects(bookmarks__all=[post_1, link_1]).explain() self.assertEqual(user.bookmarks[0], post_1) self.assertEqual(user.bookmarks[1], link_1) From 26723992e30d1db03c40250c7635374d0cc502d8 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 14:14:05 +0100 Subject: [PATCH 0121/1279] Combined Q-object tests --- tests/queryset.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 32d1902..38e1cb3 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1393,9 +1393,6 @@ class QTest(unittest.TestCase): self.assertEqual(Post.objects.filter(created_user=user).count(), 1) self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) - -class NewQTest(unittest.TestCase): - def test_and_combination(self): """Ensure that Q-objects correctly AND together. """ From 012352cf24e55558c9638ff29ad36010f98e34a6 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 14:21:55 +0100 Subject: [PATCH 0122/1279] Added snapshot and timeout methods to QuerySet --- mongoengine/queryset.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 06524dc..ead659d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -401,6 +401,8 @@ class QuerySet(object): self._where_clause = None self._loaded_fields = [] self._ordering = [] + self._snapshot = False + self._timeout = True # If inheritance is allowed, only return instances and instances of # subclasses of the class being used @@ -534,9 +536,12 @@ class QuerySet(object): @property def _cursor(self): if self._cursor_obj is None: - cursor_args = {} + cursor_args = { + 'snapshot': self._snapshot, + 'timeout': self._timeout, + } if self._loaded_fields: - cursor_args = {'fields': self._loaded_fields} + cursor_args['fields'] = self._loaded_fields self._cursor_obj = self._collection.find(self._query, **cursor_args) # Apply where clauses to cursor @@ -967,6 +972,20 @@ class QuerySet(object): plan = pprint.pformat(plan) return plan + def snapshot(self, enabled): + """Enable or disable snapshot mode when querying. + + :param enabled: whether or not snapshot mode is enabled + """ + self._snapshot = enabled + + def timeout(self, enabled): + """Enable or disable the default mongod timeout when querying. + + :param enabled: whether or not the timeout is used + """ + self._timeout = enabled + def delete(self, safe=False): """Delete the documents matched by the query. From 36993029ad4aa5b3a929e68b626eec083078738b Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 14:22:45 +0100 Subject: [PATCH 0123/1279] Removed old Q-object implementation --- mongoengine/queryset.py | 118 ---------------------------------------- 1 file changed, 118 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index ead659d..f6296dd 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -268,124 +268,6 @@ class Q(QNode): return not bool(self.query) -class OldQ(object): - - OR = '||' - AND = '&&' - OPERATORS = { - 'eq': ('((this.%(field)s instanceof Array) && ' - ' this.%(field)s.indexOf(%(value)s) != -1) ||' - ' this.%(field)s == %(value)s'), - 'ne': 'this.%(field)s != %(value)s', - 'gt': 'this.%(field)s > %(value)s', - 'gte': 'this.%(field)s >= %(value)s', - 'lt': 'this.%(field)s < %(value)s', - 'lte': 'this.%(field)s <= %(value)s', - 'lte': 'this.%(field)s <= %(value)s', - 'in': '%(value)s.indexOf(this.%(field)s) != -1', - 'nin': '%(value)s.indexOf(this.%(field)s) == -1', - 'mod': '%(field)s %% %(value)s', - 'all': ('%(value)s.every(function(a){' - 'return this.%(field)s.indexOf(a) != -1 })'), - 'size': 'this.%(field)s.length == %(value)s', - 'exists': 'this.%(field)s != null', - 'regex_eq': '%(value)s.test(this.%(field)s)', - 'regex_ne': '!%(value)s.test(this.%(field)s)', - } - - def __init__(self, **query): - self.query = [query] - - def _combine(self, other, op): - obj = Q() - if not other.query[0]: - return self - if self.query[0]: - obj.query = (['('] + copy.deepcopy(self.query) + [op] + - copy.deepcopy(other.query) + [')']) - else: - obj.query = copy.deepcopy(other.query) - return obj - - def __or__(self, other): - return self._combine(other, self.OR) - - def __and__(self, other): - return self._combine(other, self.AND) - - def as_js(self, document): - js = [] - js_scope = {} - for i, item in enumerate(self.query): - if isinstance(item, dict): - item_query = QuerySet._transform_query(document, **item) - # item_query will values will either be a value or a dict - js.append(self._item_query_as_js(item_query, js_scope, i)) - else: - js.append(item) - return pymongo.code.Code(' '.join(js), js_scope) - - def _item_query_as_js(self, item_query, js_scope, item_num): - # item_query will be in one of the following forms - # {'age': 25, 'name': 'Test'} - # {'age': {'$lt': 25}, 'name': {'$in': ['Test', 'Example']} - # {'age': {'$lt': 25, '$gt': 18}} - js = [] - for i, (key, value) in enumerate(item_query.items()): - op = 'eq' - # Construct a variable name for the value in the JS - value_name = 'i%sf%s' % (item_num, i) - if isinstance(value, dict): - # Multiple operators for this field - for j, (op, value) in enumerate(value.items()): - # Create a custom variable name for this operator - op_value_name = '%so%s' % (value_name, j) - # Construct the JS that uses this op - value, operation_js = self._build_op_js(op, key, value, - op_value_name) - # Update the js scope with the value for this op - js_scope[op_value_name] = value - js.append(operation_js) - else: - # Construct the JS for this field - value, field_js = self._build_op_js(op, key, value, value_name) - js_scope[value_name] = value - js.append(field_js) - return ' && '.join(js) - - def _build_op_js(self, op, key, value, value_name): - """Substitute the values in to the correct chunk of Javascript. - """ - if isinstance(value, RE_TYPE): - # Regexes are handled specially - if op.strip('$') == 'ne': - op_js = Q.OPERATORS['regex_ne'] - else: - op_js = Q.OPERATORS['regex_eq'] - else: - op_js = Q.OPERATORS[op.strip('$')] - - # Comparing two ObjectIds in Javascript doesn't work.. - if isinstance(value, pymongo.objectid.ObjectId): - value = unicode(value) - - # Handle DBRef - if isinstance(value, pymongo.dbref.DBRef): - op_js = '(this.%(field)s.$id == "%(id)s" &&'\ - ' this.%(field)s.$ref == "%(ref)s")' % { - 'field': key, - 'id': unicode(value.id), - 'ref': unicode(value.collection) - } - value = None - - # Perform the substitution - operation_js = op_js % { - 'field': key, - 'value': value_name - } - return value, operation_js - class QuerySet(object): """A set of results returned from a query. Wraps a MongoDB cursor, providing :class:`~mongoengine.Document` objects as the results. From 6817f3b7ba1b6f25554937179cd112a636140e01 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 15:40:49 +0100 Subject: [PATCH 0124/1279] Updated docs for v0.4 --- docs/apireference.rst | 2 ++ docs/changelog.rst | 11 ++++++- docs/guide/defining-documents.rst | 6 +++- docs/guide/querying.rst | 51 +++++++++++++++++++++++++++++++ docs/index.rst | 2 +- mongoengine/fields.py | 6 ++++ mongoengine/queryset.py | 1 + 7 files changed, 76 insertions(+), 3 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 4fff317..34d4536 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -66,3 +66,5 @@ Fields .. autoclass:: mongoengine.GenericReferenceField .. autoclass:: mongoengine.FileField + +.. autoclass:: mongoengine.GeoPointField diff --git a/docs/changelog.rst b/docs/changelog.rst index 8dd5b00..64bdf90 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,16 +4,25 @@ Changelog Changes in v0.4 =============== +- New Q-object implementation, which is no longer based on Javascript - Added ``SortedListField`` - Added ``EmailField`` - Added ``GeoPointField`` - Added ``exact`` and ``iexact`` match operators to ``QuerySet`` - Added ``get_document_or_404`` and ``get_list_or_404`` Django shortcuts -- Fixed bug in Q-objects +- Added new query operators for Geo queries +- Added ``not`` query operator +- Added new update operators: ``pop`` and ``add_to_set`` +- Added ``__raw__`` query parameter - Fixed document inheritance primary key issue +- Added support for querying by array element position - Base class can now be defined for ``DictField`` - Fixed MRO error that occured on document inheritance +- Added ``QuerySet.distinct``, ``QuerySet.create``, ``QuerySet.snapshot``, + ``QuerySet.timeout`` and ``QuerySet.all`` +- Subsequent calls to ``connect()`` now work - Introduced ``min_length`` for ``StringField`` +- Fixed multi-process connection issue - Other minor fixes Changes in v0.3 diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index dc136f2..106d4ec 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -47,11 +47,11 @@ are as follows: * :class:`~mongoengine.ReferenceField` * :class:`~mongoengine.GenericReferenceField` * :class:`~mongoengine.BooleanField` -* :class:`~mongoengine.GeoLocationField` * :class:`~mongoengine.FileField` * :class:`~mongoengine.EmailField` * :class:`~mongoengine.SortedListField` * :class:`~mongoengine.BinaryField` +* :class:`~mongoengine.GeoPointField` Field arguments --------------- @@ -298,6 +298,10 @@ or a **-** sign. Note that direction only matters on multi-field indexes. :: meta = { 'indexes': ['title', ('title', '-rating')] } + +.. note:: + Geospatial indexes will be automatically created for all + :class:`~mongoengine.GeoPointField`\ s Ordering ======== diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index bef19bc..58bf9f6 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -53,6 +53,16 @@ lists that contain that item will be matched:: # 'tags' list Page.objects(tags='coding') +Raw queries +----------- +It is possible to provide a raw PyMongo query as a query parameter, which will +be integrated directly into the query. This is done using the ``__raw__`` +keyword argument:: + + Page.objects(__raw__={'tags': 'coding'}) + +.. versionadded:: 0.4 + Query operators =============== Operators other than equality may also be used in queries; just attach the @@ -68,6 +78,8 @@ Available operators are as follows: * ``lte`` -- less than or equal to * ``gt`` -- greater than * ``gte`` -- greater than or equal to +* ``not`` -- negate a standard check, may be used before other operators (e.g. + ``Q(age__not__mod=5)``) * ``in`` -- value is in list (a list of values should be provided) * ``nin`` -- value is not in list (a list of values should be provided) * ``mod`` -- ``value % x == y``, where ``x`` and ``y`` are two provided values @@ -89,6 +101,27 @@ expressions: .. versionadded:: 0.3 +There are a few special operators for performing geographical queries, that +may used with :class:`~mongoengine.GeoPointField`\ s: + +* ``within_distance`` -- provide a list containing a point and a maximum + distance (e.g. [(41.342, -87.653), 5]) +* ``within_box`` -- filter documents to those within a given bounding box (e.g. + [(35.0, -125.0), (40.0, -100.0)]) +* ``near`` -- order the documents by how close they are to a given point + +.. versionadded:: 0.4 + +Querying by position +==================== +It is possible to query by position in a list by using a numerical value as a +query operator. So if you wanted to find all pages whose first tag was ``db``, +you could use the following query:: + + BlogPost.objects(tags__0='db') + +.. versionadded:: 0.4 + Limiting and skipping results ============================= Just as with traditional ORMs, you may limit the number of results returned, or @@ -181,6 +214,22 @@ custom manager methods as you like:: assert len(BlogPost.objects) == 2 assert len(BlogPost.live_posts) == 1 +Custom QuerySets +================ +Should you want to add custom methods for interacting with or filtering +documents, extending the :class:`~mongoengine.queryset.QuerySet` class may be +the way to go. To use a custom :class:`~mongoengine.queryset.QuerySet` class on +a document, set ``queryset_class`` to the custom class in a +:class:`~mongoengine.Document`\ s ``meta`` dictionary:: + + class AwesomerQuerySet(QuerySet): + pass + + class Page(Document): + meta = {'queryset_class': AwesomerQuerySet} + +.. versionadded:: 0.4 + Aggregation =========== MongoDB provides some aggregation methods out of the box, but there are not as @@ -402,8 +451,10 @@ that you may use with these methods: * ``pop`` -- remove the last item from a list * ``push`` -- append a value to a list * ``push_all`` -- append several values to a list +* ``pop`` -- remove the first or last element of a list * ``pull`` -- remove a value from a list * ``pull_all`` -- remove several values from a list +* ``add_to_set`` -- add value to a list only if its not in the list already The syntax for atomic updates is similar to the querying syntax, but the modifier comes before the field, not after it:: diff --git a/docs/index.rst b/docs/index.rst index a28b344..ccb7fbe 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,7 +7,7 @@ MongoDB. To install it, simply run .. code-block:: console - # easy_install -U mongoengine + # pip install -U mongoengine The source is available on `GitHub `_. diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 9fc50d6..62d2ef2 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -112,6 +112,8 @@ class URLField(StringField): class EmailField(StringField): """A field that validates input as an E-Mail-Address. + + .. versionadded:: 0.4 """ EMAIL_REGEX = re.compile( @@ -355,6 +357,8 @@ class SortedListField(ListField): """A ListField that sorts the contents of its list before writing to the database in order to ensure that a sorted list is always retrieved. + + .. versionadded:: 0.4 """ _ordering = None @@ -666,6 +670,8 @@ class FileField(BaseField): class GeoPointField(BaseField): """A list storing a latitude and longitude. + + .. versionadded:: 0.4 """ _geo_index = True diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index f6296dd..86f8b1d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -590,6 +590,7 @@ class QuerySet(object): def create(self, **kwargs): """Create new object. Returns the saved object instance. + .. versionadded:: 0.4 """ doc = self._document(**kwargs) From 007f116bfaec40205626075ef9050ab35b1753df Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 15:42:31 +0100 Subject: [PATCH 0125/1279] Increment version to 0.4 --- docs/changelog.rst | 1 + mongoengine/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 64bdf90..277d8e9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -14,6 +14,7 @@ Changes in v0.4 - Added ``not`` query operator - Added new update operators: ``pop`` and ``add_to_set`` - Added ``__raw__`` query parameter +- Added support for custom querysets - Fixed document inheritance primary key issue - Added support for querying by array element position - Base class can now be defined for ``DictField`` diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index e01d31a..6d18ffe 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ __all__ = (document.__all__ + fields.__all__ + connection.__all__ + __author__ = 'Harry Marr' -VERSION = (0, 3, 0) +VERSION = (0, 4, 0) def get_version(): version = '%s.%s' % (VERSION[0], VERSION[1]) From dcec61e9b2738f9fc3bd0f09a2cb1bc631438c66 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 16:36:22 +0100 Subject: [PATCH 0126/1279] Raise AttributeError when necessary on QuerySet[] --- mongoengine/queryset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 86f8b1d..a8d739d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -787,6 +787,7 @@ class QuerySet(object): # Integer index provided elif isinstance(key, int): return self._document._from_son(self._cursor[key]) + raise AttributeError def distinct(self, field): """Return a list of distinct values for a given field. From e93c4c87d8953db1c4da24415c0da3c463523ab9 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 17 Oct 2010 17:41:20 +0100 Subject: [PATCH 0127/1279] Fixed inheritance collection issue --- mongoengine/queryset.py | 15 +++++++-------- tests/document.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a8d739d..7422bb8 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1146,9 +1146,8 @@ class QuerySetManager(object): return self db = _get_db() - if db not in self._collections: - collection = owner._meta['collection'] - + collection = owner._meta['collection'] + if (db, collection) not in self._collections: # Create collection as a capped collection if specified if owner._meta['max_size'] or owner._meta['max_documents']: # Get max document limit and max byte size from meta @@ -1156,10 +1155,10 @@ class QuerySetManager(object): max_documents = owner._meta['max_documents'] if collection in db.collection_names(): - self._collections[db] = db[collection] + self._collections[(db, collection)] = db[collection] # The collection already exists, check if its capped # options match the specified capped options - options = self._collections[db].options() + options = self._collections[(db, collection)].options() if options.get('max') != max_documents or \ options.get('size') != max_size: msg = ('Cannot create collection "%s" as a capped ' @@ -1170,15 +1169,15 @@ class QuerySetManager(object): opts = {'capped': True, 'size': max_size} if max_documents: opts['max'] = max_documents - self._collections[db] = db.create_collection( + self._collections[(db, collection)] = db.create_collection( collection, **opts ) else: - self._collections[db] = db[collection] + self._collections[(db, collection)] = db[collection] # owner is the document that contains the QuerySetManager queryset_class = owner._meta['queryset_class'] or QuerySet - queryset = queryset_class(owner, self._collections[db]) + queryset = queryset_class(owner, self._collections[(db, collection)]) if self._manager_func: if self._manager_func.func_code.co_argcount == 1: queryset = self._manager_func(queryset) diff --git a/tests/document.py b/tests/document.py index c2e0d6f..aa90181 100644 --- a/tests/document.py +++ b/tests/document.py @@ -200,6 +200,37 @@ class DocumentTest(unittest.TestCase): Person.drop_collection() self.assertFalse(collection in self.db.collection_names()) + def test_inherited_collections(self): + """Ensure that subclassed documents don't override parents' collections. + """ + class Drink(Document): + name = StringField() + + class AlcoholicDrink(Drink): + meta = {'collection': 'booze'} + + class Drinker(Document): + drink = GenericReferenceField() + + Drink.drop_collection() + AlcoholicDrink.drop_collection() + Drinker.drop_collection() + + red_bull = Drink(name='Red Bull') + red_bull.save() + + programmer = Drinker(drink=red_bull) + programmer.save() + + beer = AlcoholicDrink(name='Beer') + beer.save() + + real_person = Drinker(drink=beer) + real_person.save() + + self.assertEqual(Drinker.objects[0].drink.name, red_bull.name) + self.assertEqual(Drinker.objects[1].drink.name, beer.name) + def test_capped_collection(self): """Ensure that capped collections work properly. """ From dc7181a3fdef5e5a1724683677fd5bc21298db8f Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Sun, 17 Oct 2010 23:43:58 +0100 Subject: [PATCH 0128/1279] Begun GridFS documentation --- docs/guide/gridfs.rst | 26 ++++++++++++++++++++++++++ docs/guide/index.rst | 1 + mongoengine/django/auth.py | 7 +++---- mongoengine/fields.py | 1 + 4 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 docs/guide/gridfs.rst diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst new file mode 100644 index 0000000..7c6e8ac --- /dev/null +++ b/docs/guide/gridfs.rst @@ -0,0 +1,26 @@ +====== +GridFS +====== +GridFS support comes in the form of the :class:`~mongoengine.FileField` field +object. This field acts as a file-like object and provides a couple of +different ways of inserting and retrieving data. Metadata such as content-type +can also be stored alongside the stored files. In the following example, an +document is created to store details about animals, including a photo: + + class Animal(Document): + genus = StringField() + family = StringField() + photo = FileField() + + marmot = Animal('Marmota', 'Sciuridae') + + marmot_photo = open('marmot.jpg') # Retrieve a photo from disk + marmot.photo = marmot_photo # Store the photo in the document + + marmot.save() + +So adding file data to a document is as easy as adding data to any other + +.. versionadded:: 0.4 + + diff --git a/docs/guide/index.rst b/docs/guide/index.rst index 7fdfe93..aac7246 100644 --- a/docs/guide/index.rst +++ b/docs/guide/index.rst @@ -10,3 +10,4 @@ User Guide defining-documents document-instances querying + gridfs diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index da0005c..595852e 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -75,10 +75,9 @@ class User(Document): email address. """ now = datetime.datetime.now() - + # Normalize the address by lowercasing the domain part of the email # address. - # Not sure why we'r allowing null email when its not allowed in django if email is not None: try: email_name, domain_part = email.strip().split('@', 1) @@ -86,12 +85,12 @@ class User(Document): pass else: email = '@'.join([email_name, domain_part.lower()]) - + user = User(username=username, email=email, date_joined=now) user.set_password(password) user.save() return user - + def get_and_delete_messages(self): return [] diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 86da2e3..23a88de 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -20,6 +20,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', RECURSIVE_REFERENCE_CONSTANT = 'self' + class StringField(BaseField): """A unicode string field. """ From 0902b957641e09861f0864e09501b174624577f0 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 18 Oct 2010 00:27:40 +0100 Subject: [PATCH 0129/1279] Added support for recursive embedded documents --- mongoengine/base.py | 7 +++---- mongoengine/fields.py | 33 +++++++++++++++++++++------------ tests/fields.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 16 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 1f7ba1f..2253e4a 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -204,6 +204,9 @@ class DocumentMetaclass(type): exc = subclass_exception('MultipleObjectsReturned', base_excs, module) new_class.add_to_class('MultipleObjectsReturned', exc) + global _document_registry + _document_registry[name] = new_class + return new_class def add_to_class(self, name, value): @@ -216,8 +219,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): """ def __new__(cls, name, bases, attrs): - global _document_registry - super_new = super(TopLevelDocumentMetaclass, cls).__new__ # Classes defined in this package are abstract and should not have # their own metadata with DB collection, etc. @@ -322,8 +323,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): new_class._fields['id'] = ObjectIdField(db_field='_id') new_class.id = new_class._fields['id'] - _document_registry[name] = new_class - return new_class diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 62d2ef2..63107f2 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -233,33 +233,43 @@ class EmbeddedDocumentField(BaseField): :class:`~mongoengine.EmbeddedDocument`. """ - def __init__(self, document, **kwargs): - if not issubclass(document, EmbeddedDocument): - raise ValidationError('Invalid embedded document class provided ' - 'to an EmbeddedDocumentField') - self.document = document + def __init__(self, document_type, **kwargs): + if not isinstance(document_type, basestring): + if not issubclass(document_type, EmbeddedDocument): + raise ValidationError('Invalid embedded document class ' + 'provided to an EmbeddedDocumentField') + self.document_type_obj = document_type super(EmbeddedDocumentField, self).__init__(**kwargs) + @property + def document_type(self): + if isinstance(self.document_type_obj, basestring): + if self.document_type_obj == RECURSIVE_REFERENCE_CONSTANT: + self.document_type_obj = self.owner_document + else: + self.document_type_obj = get_document(self.document_type_obj) + return self.document_type_obj + def to_python(self, value): - if not isinstance(value, self.document): - return self.document._from_son(value) + if not isinstance(value, self.document_type): + return self.document_type._from_son(value) return value def to_mongo(self, value): - return self.document.to_mongo(value) + return self.document_type.to_mongo(value) def validate(self, value): """Make sure that the document instance is an instance of the EmbeddedDocument subclass provided when the document was defined. """ # Using isinstance also works for subclasses of self.document - if not isinstance(value, self.document): + if not isinstance(value, self.document_type): raise ValidationError('Invalid embedded document instance ' 'provided to an EmbeddedDocumentField') - self.document.validate(value) + self.document_type.validate(value) def lookup_member(self, member_name): - return self.document._fields.get(member_name) + return self.document_type._fields.get(member_name) def prepare_query_value(self, op, value): return self.to_mongo(value) @@ -413,7 +423,6 @@ class ReferenceField(BaseField): raise ValidationError('Argument to ReferenceField constructor ' 'must be a document class or a string') self.document_type_obj = document_type - self.document_obj = None super(ReferenceField, self).__init__(**kwargs) @property diff --git a/tests/fields.py b/tests/fields.py index e30f843..208b464 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -423,6 +423,36 @@ class FieldTest(unittest.TestCase): self.assertEqual(peter.boss, bill) self.assertEqual(peter.friends, friends) + def test_recursive_embedding(self): + """Ensure that EmbeddedDocumentFields can contain their own documents. + """ + class Tree(Document): + name = StringField() + children = ListField(EmbeddedDocumentField('TreeNode')) + + class TreeNode(EmbeddedDocument): + name = StringField() + children = ListField(EmbeddedDocumentField('self')) + + tree = Tree(name="Tree") + + first_child = TreeNode(name="Child 1") + tree.children.append(first_child) + + second_child = TreeNode(name="Child 2") + first_child.children.append(second_child) + + third_child = TreeNode(name="Child 3") + first_child.children.append(third_child) + + tree.save() + + tree_obj = Tree.objects.first() + self.assertEqual(len(tree.children), 1) + self.assertEqual(tree.children[0].name, first_child.name) + self.assertEqual(tree.children[0].children[0].name, second_child.name) + self.assertEqual(tree.children[0].children[1].name, third_child.name) + def test_undefined_reference(self): """Ensure that ReferenceFields may reference undefined Documents. """ From 67736c849dd65d6dd96cef6e7bcc41cdb8503c22 Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Mon, 18 Oct 2010 00:55:44 +0100 Subject: [PATCH 0130/1279] Finished GridFS Documentation * Also made GridFS replace test pass --- docs/django.rst | 32 ++++++++++--------- docs/guide/gridfs.rst | 71 +++++++++++++++++++++++++++++++++++++---- docs/guide/querying.rst | 8 ++--- mongoengine/fields.py | 5 +-- 4 files changed, 88 insertions(+), 28 deletions(-) diff --git a/docs/django.rst b/docs/django.rst index 2cce3f0..8a49057 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -47,9 +47,10 @@ into you settings module:: Storage ======= -With MongoEngine's support for GridFS via the FileField, it is useful to have a -Django file storage backend that wraps this. The new storage module is called -GridFSStorage. Using it is very similar to using the default FileSystemStorage.:: +With MongoEngine's support for GridFS via the :class:`~mongoengine.FileField`, +it is useful to have a Django file storage backend that wraps this. The new +storage module is called :class:`~mongoengine.django.GridFSStorage`. Using it +is very similar to using the default FileSystemStorage.:: fs = mongoengine.django.GridFSStorage() @@ -57,29 +58,30 @@ GridFSStorage. Using it is very similar to using the default FileSystemStorage.: All of the `Django Storage API methods `_ have been -implemented except ``path()``. If the filename provided already exists, an +implemented except :func:`path`. If the filename provided already exists, an underscore and a number (before # the file extension, if one exists) will be appended to the filename until the generated filename doesn't exist. The -``save()`` method will return the new filename.:: +:func:`save` method will return the new filename.:: - > fs.exists('hello.txt') + >>> fs.exists('hello.txt') True - > fs.open('hello.txt').read() + >>> fs.open('hello.txt').read() 'Hello, World!' - > fs.size('hello.txt') + >>> fs.size('hello.txt') 13 - > fs.url('hello.txt') + >>> fs.url('hello.txt') 'http://your_media_url/hello.txt' - > fs.open('hello.txt').name + >>> fs.open('hello.txt').name 'hello.txt' - > fs.listdir() + >>> fs.listdir() ([], [u'hello.txt']) -All files will be saved and retrieved in GridFS via the ``FileDocument`` document, -allowing easy access to the files without the GridFSStorage backend.:: +All files will be saved and retrieved in GridFS via the :class::`FileDocument` +document, allowing easy access to the files without the GridFSStorage +backend.:: - > from mongoengine.django.storage import FileDocument - > FileDocument.objects() + >>> from mongoengine.django.storage import FileDocument + >>> FileDocument.objects() [] .. versionadded:: 0.4 diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst index 7c6e8ac..503610c 100644 --- a/docs/guide/gridfs.rst +++ b/docs/guide/gridfs.rst @@ -1,11 +1,17 @@ ====== GridFS ====== + +.. versionadded:: 0.4 + +Writing +------- + GridFS support comes in the form of the :class:`~mongoengine.FileField` field object. This field acts as a file-like object and provides a couple of -different ways of inserting and retrieving data. Metadata such as content-type -can also be stored alongside the stored files. In the following example, an -document is created to store details about animals, including a photo: +different ways of inserting and retrieving data. Arbitrary metadata such as +content type can also be stored alongside the files. In the following example, +a document is created to store details about animals, including a photo:: class Animal(Document): genus = StringField() @@ -14,13 +20,64 @@ document is created to store details about animals, including a photo: marmot = Animal('Marmota', 'Sciuridae') - marmot_photo = open('marmot.jpg') # Retrieve a photo from disk - marmot.photo = marmot_photo # Store the photo in the document + marmot_photo = open('marmot.jpg', 'r') # Retrieve a photo from disk + marmot.photo = marmot_photo # Store the photo in the document + marmot.photo.content_type = 'image/jpeg' # Store metadata marmot.save() -So adding file data to a document is as easy as adding data to any other +Another way of writing to a :class:`~mongoengine.FileField` is to use the +:func:`put` method. This allows for metadata to be stored in the same call as +the file:: -.. versionadded:: 0.4 + marmot.photo.put(marmot_photo, content_type='image/jpeg') + + marmot.save() + +Retrieval +--------- + +So using the :class:`~mongoengine.FileField` is just like using any other +field. The file can also be retrieved just as easily:: + + marmot = Animal.objects('Marmota').first() + photo = marmot.photo.read() + content_type = marmot.photo.content_type + +Streaming +--------- + +Streaming data into a :class:`~mongoengine.FileField` is achieved in a +slightly different manner. First, a new file must be created by calling the +:func:`new_file` method. Data can then be written using :func:`write`:: + + marmot.photo.new_file() + marmot.photo.write('some_image_data') + marmot.photo.write('some_more_image_data') + marmot.photo.close() + + marmot.photo.save() + +Deletion +-------- + +Deleting stored files is achieved with the :func:`delete` method:: + + marmot.photo.delete() + +.. note:: + The FileField in a Document actually only stores the ID of a file in a + separate GridFS collection. This means that deleting a document + with a defined FileField does not actually delete the file. You must be + careful to delete any files in a Document as above before deleting the + Document itself. +Replacing files +--------------- + +Files can be replaced with the :func:`replace` method. This works just like +the :func:`put` method so even metadata can (and should) be replaced:: + + another_marmot = open('another_marmot.png', 'r') + marmot.photo.replace(another_marmot, content_type='image/png') diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 58bf9f6..832fed5 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -34,7 +34,7 @@ arguments. The keys in the keyword arguments correspond to fields on the Fields on embedded documents may also be referred to using field lookup syntax by using a double-underscore in place of the dot in object attribute access syntax:: - + # This will return a QuerySet that will only iterate over pages that have # been written by a user whose 'country' field is set to 'uk' uk_pages = Page.objects(author__country='uk') @@ -67,7 +67,7 @@ Query operators =============== Operators other than equality may also be used in queries; just attach the operator name to a key with a double-underscore:: - + # Only find users whose age is 18 or less young_users = Users.objects(age__lte=18) @@ -144,7 +144,7 @@ You may also index the query to retrieve a single result. If an item at that index does not exists, an :class:`IndexError` will be raised. A shortcut for retrieving the first result and returning :attr:`None` if no result exists is provided (:meth:`~mongoengine.queryset.QuerySet.first`):: - + >>> # Make sure there are no users >>> User.drop_collection() >>> User.objects[0] @@ -458,7 +458,7 @@ that you may use with these methods: The syntax for atomic updates is similar to the querying syntax, but the modifier comes before the field, not after it:: - + >>> post = BlogPost(title='Test', page_views=0, tags=['database']) >>> post.save() >>> BlogPost.objects(id=post.id).update_one(inc__page_views=1) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 30a508d..ef9540f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -586,14 +586,14 @@ class GridFSProxy(object): def put(self, file, **kwargs): if self.grid_id: - raise GridFSError('This document alreay has a file. Either delete ' + raise GridFSError('This document already has a file. Either delete ' 'it or call replace to overwrite it') self.grid_id = self.fs.put(file, **kwargs) def write(self, string): if self.grid_id: if not self.newfile: - raise GridFSError('This document alreay has a file. Either ' + raise GridFSError('This document already has a file. Either ' 'delete it or call replace to overwrite it') else: self.new_file() @@ -622,6 +622,7 @@ class GridFSProxy(object): def replace(self, file, **kwargs): self.delete() + self.grid_id = None self.put(file, **kwargs) def close(self): From 5580b003b5aca4dc272c1e046f0100657faf1a25 Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Mon, 18 Oct 2010 01:30:32 +0100 Subject: [PATCH 0131/1279] Added self to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 93ecfa8..93fe819 100644 --- a/AUTHORS +++ b/AUTHORS @@ -2,3 +2,4 @@ Harry Marr Matt Dennewitz Deepak Thukral Florian Schlachter +Steve Challis From d6cb5b9abe64309b4e57dd36888bac4c0f34e6d8 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 18 Oct 2010 10:21:23 +0100 Subject: [PATCH 0132/1279] Removed invalid connection tests --- tests/connnection.py | 44 -------------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 tests/connnection.py diff --git a/tests/connnection.py b/tests/connnection.py deleted file mode 100644 index 1903a5f..0000000 --- a/tests/connnection.py +++ /dev/null @@ -1,44 +0,0 @@ -import unittest -import datetime -import pymongo - -import mongoengine.connection -from mongoengine import * -from mongoengine.connection import _get_db, _get_connection - - -class ConnectionTest(unittest.TestCase): - - def tearDown(self): - mongoengine.connection._connection_settings = {} - mongoengine.connection._connections = {} - mongoengine.connection._dbs = {} - - def test_connect(self): - """Ensure that the connect() method works properly. - """ - connect('mongoenginetest') - - conn = _get_connection() - self.assertTrue(isinstance(conn, pymongo.connection.Connection)) - - db = _get_db() - self.assertTrue(isinstance(db, pymongo.database.Database)) - self.assertEqual(db.name, 'mongoenginetest') - - def test_register_connection(self): - """Ensure that connections with different aliases may be registered. - """ - register_connection('testdb', 'mongoenginetest2') - - self.assertRaises(ConnectionError, _get_connection) - conn = _get_connection('testdb') - self.assertTrue(isinstance(conn, pymongo.connection.Connection)) - - db = _get_db('testdb') - self.assertTrue(isinstance(db, pymongo.database.Database)) - self.assertEqual(db.name, 'mongoenginetest2') - - -if __name__ == '__main__': - unittest.main() From d7c42861fbcf32754d53559fbc0863ad78c8cabd Mon Sep 17 00:00:00 2001 From: Steve Challis Date: Mon, 18 Oct 2010 10:25:06 +0100 Subject: [PATCH 0133/1279] Minor GridFS corrections --- docs/guide/gridfs.rst | 4 ++-- mongoengine/fields.py | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst index 503610c..0cd0653 100644 --- a/docs/guide/gridfs.rst +++ b/docs/guide/gridfs.rst @@ -21,7 +21,7 @@ a document is created to store details about animals, including a photo:: marmot = Animal('Marmota', 'Sciuridae') marmot_photo = open('marmot.jpg', 'r') # Retrieve a photo from disk - marmot.photo = marmot_photo # Store the photo in the document + marmot.photo = marmot_photo # Store photo in the document marmot.photo.content_type = 'image/jpeg' # Store metadata marmot.save() @@ -40,7 +40,7 @@ Retrieval So using the :class:`~mongoengine.FileField` is just like using any other field. The file can also be retrieved just as easily:: - marmot = Animal.objects('Marmota').first() + marmot = Animal.objects(genus='Marmota').first() photo = marmot.photo.read() content_type = marmot.photo.content_type diff --git a/mongoengine/fields.py b/mongoengine/fields.py index ef9540f..e95fd65 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -614,15 +614,10 @@ class GridFSProxy(object): def delete(self): # Delete file from GridFS, FileField still remains self.fs.delete(self.grid_id) - - #self.grid_id = None - # Doesn't make a difference because will be put back in when - # reinstantiated We should delete all the metadata stored with the - # file too + self.grid_id = None def replace(self, file, **kwargs): self.delete() - self.grid_id = None self.put(file, **kwargs) def close(self): From 3b88a4f7288dd83a0bbbec2393127fc7abc24b03 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Tue, 19 Oct 2010 12:28:34 +0100 Subject: [PATCH 0134/1279] Fixed a couple of errors in the docs --- docs/apireference.rst | 4 ++++ docs/guide/querying.rst | 6 ------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 34d4536..a3d287a 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -41,6 +41,8 @@ Fields .. autoclass:: mongoengine.URLField +.. autoclass:: mongoengine.EmailField + .. autoclass:: mongoengine.IntField .. autoclass:: mongoengine.FloatField @@ -57,6 +59,8 @@ Fields .. autoclass:: mongoengine.ListField +.. autoclass:: mongoengine.SortedListField + .. autoclass:: mongoengine.BinaryField .. autoclass:: mongoengine.ObjectIdField diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 832fed5..1caed2d 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -325,12 +325,6 @@ calling it with keyword arguments:: # Get top posts Post.objects((Q(featured=True) & Q(hits__gte=1000)) | Q(hits__gte=5000)) -.. warning:: - Only use these advanced queries if absolutely necessary as they will execute - significantly slower than regular queries. This is because they are not - natively supported by MongoDB -- they are compiled to Javascript and sent - to the server for execution. - Server-side javascript execution ================================ Javascript functions may be written and sent to the server for execution. The From 2560145551099f15626b1a209ea6de8974817de4 Mon Sep 17 00:00:00 2001 From: Rached Ben Mustapha Date: Tue, 19 Oct 2010 22:23:08 +0000 Subject: [PATCH 0135/1279] add a failing test for the pagination bug --- tests/queryset.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 6ca4174..37fe750 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1377,6 +1377,23 @@ class QuerySetTest(unittest.TestCase): Post.drop_collection() + def test_call_after_limits_set(self): + """Ensure that re-filtering after slicing works + """ + class Post(Document): + title = StringField() + + Post.drop_collection() + + post1 = Post(title="Post 1") + post1.save() + post2 = Post(title="Post 2") + post2.save() + + posts = Post.objects.all()[0:1] + self.assertEqual(len(list(posts())), 1) + + Post.drop_collection() class QTest(unittest.TestCase): From 18baa2dd7a4e909169b694cb6ec36214c5a51506 Mon Sep 17 00:00:00 2001 From: Rached Ben Mustapha Date: Tue, 19 Oct 2010 22:40:36 +0000 Subject: [PATCH 0136/1279] fix calling a queryset after skip and limit have been set --- mongoengine/queryset.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 519dda0..f484961 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -434,6 +434,12 @@ class QuerySet(object): if self._document._meta['ordering']: self.order_by(*self._document._meta['ordering']) + if self._limit is not None: + self._cursor_obj.limit(self._limit) + + if self._skip is not None: + self._cursor_obj.skip(self._skip) + return self._cursor_obj @classmethod From f0c5dd1bce63f7ce3c7cdbcba33a4bc367674e3c Mon Sep 17 00:00:00 2001 From: Viktor Kerkez Date: Sat, 23 Oct 2010 22:33:03 +0200 Subject: [PATCH 0137/1279] Small fix for Python 2.5 --- .gitignore | 4 +++- mongoengine/base.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 51a9ca1..d292729 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,6 @@ docs/_build build/ dist/ mongoengine.egg-info/ -env/ \ No newline at end of file +env/ +.project +.pydevproject \ No newline at end of file diff --git a/mongoengine/base.py b/mongoengine/base.py index 6b74cb0..ac15df6 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -491,6 +491,7 @@ class BaseDocument(object): if sys.version_info < (2, 5): # Prior to Python 2.5, Exception was an old-style class + import types def subclass_exception(name, parents, unused): return types.ClassType(name, parents, {}) else: From ef15733efea51bf61f2d01a58b0c87d7560f1bb5 Mon Sep 17 00:00:00 2001 From: Viktor Kerkez Date: Sat, 23 Oct 2010 22:35:37 +0200 Subject: [PATCH 0138/1279] Added creation_counter to BaseField in order to provied form modules with a way to sort fields i order user specified them (same technique is used in Django) --- mongoengine/base.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index ac15df6..c647208 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -25,6 +25,12 @@ class BaseField(object): _index_with_types = True _geo_index = False + # These track each time a Field instance is created. Used to retain order. + # The auto_creation_counter is used for fields that MongoEngine implicitly + # creates, creation_counter is used for all user-specified fields. + creation_counter = 0 + auto_creation_counter = -1 + def __init__(self, db_field=None, name=None, required=False, default=None, unique=False, unique_with=None, primary_key=False, validation=None, choices=None): @@ -41,6 +47,13 @@ class BaseField(object): self.primary_key = primary_key self.validation = validation self.choices = choices + # Adjust the appropriate creation counter, and save our local copy. + if self.db_field == '_id': + self.creation_counter = BaseField.auto_creation_counter + BaseField.auto_creation_counter -= 1 + else: + self.creation_counter = BaseField.creation_counter + BaseField.creation_counter += 1 def __get__(self, instance, owner): """Descriptor for retrieving a value from a field in a document. Do From a3830be4c9481ff2be0043e413dfdca838f90a4e Mon Sep 17 00:00:00 2001 From: Ales Zoulek Date: Thu, 28 Oct 2010 01:03:57 +0200 Subject: [PATCH 0139/1279] QuerySet.only() supports subfields + tests --- mongoengine/queryset.py | 6 +----- tests/queryset.py | 45 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 519dda0..0b65168 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -812,11 +812,7 @@ class QuerySet(object): """ self._loaded_fields = [] for field in fields: - if '.' in field: - raise InvalidQueryError('Subfields cannot be used as ' - 'arguments to QuerySet.only') - # Translate field name - field = QuerySet._lookup_field(self._document, field)[-1].db_field + field = ".".join(f.db_field for f in QuerySet._lookup_field(self._document, field.split('.'))) self._loaded_fields.append(field) # _cls is needed for polymorphism diff --git a/tests/queryset.py b/tests/queryset.py index 6ca4174..84e450b 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -452,6 +452,51 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(obj.salary, employee.salary) self.assertEqual(obj.name, None) + def test_only_with_subfields(self): + class User(EmbeddedDocument): + name = StringField() + email = StringField() + + class Comment(EmbeddedDocument): + title = StringField() + text = StringField() + + class BlogPost(Document): + content = StringField() + author = EmbeddedDocumentField(User) + comments = ListField(EmbeddedDocumentField(Comment)) + + BlogPost.drop_collection() + + post = BlogPost(content='Had a good coffee today...') + post.author = User(name='Test User') + post.comments = [Comment(title='I aggree', text='Great post!'), Comment(title='Coffee', text='I hate coffee')] + post.save() + + obj = BlogPost.objects.only('author.name',).get() + self.assertEqual(obj.content, None) + self.assertEqual(obj.author.email, None) + self.assertEqual(obj.author.name, 'Test User') + self.assertEqual(obj.comments, []) + + obj = BlogPost.objects.only('content', 'comments.title',).get() + self.assertEqual(obj.content, 'Had a good coffee today...') + self.assertEqual(obj.author, None) + self.assertEqual(obj.comments[0].title, 'I aggree') + self.assertEqual(obj.comments[1].title, 'Coffee') + self.assertEqual(obj.comments[0].text, None) + self.assertEqual(obj.comments[1].text, None) + + obj = BlogPost.objects.only('comments',).get() + self.assertEqual(obj.content, None) + self.assertEqual(obj.author, None) + self.assertEqual(obj.comments[0].title, 'I aggree') + self.assertEqual(obj.comments[1].title, 'Coffee') + self.assertEqual(obj.comments[0].text, 'Great post!') + self.assertEqual(obj.comments[1].text, 'I hate coffee') + + BlogPost.drop_collection() + def test_find_embedded(self): """Ensure that an embedded document is properly returned from a query. """ From 6b880aa8b358d6a80c35525adced2da1d801830c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Mon, 1 Nov 2010 00:43:30 +0000 Subject: [PATCH 0140/1279] Fixed order-then-filter issue --- mongoengine/queryset.py | 4 +++- tests/queryset.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index f484961..5737cb3 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -431,7 +431,9 @@ class QuerySet(object): self._cursor_obj.where(self._where_clause) # apply default ordering - if self._document._meta['ordering']: + if self._ordering: + self._cursor_obj.sort(self._ordering) + elif self._document._meta['ordering']: self.order_by(*self._document._meta['ordering']) if self._limit is not None: diff --git a/tests/queryset.py b/tests/queryset.py index 37fe750..7059659 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1395,6 +1395,24 @@ class QuerySetTest(unittest.TestCase): Post.drop_collection() + def test_order_then_filter(self): + """Ensure that ordering still works after filtering. + """ + class Number(Document): + n = IntField() + + Number.drop_collection() + + n2 = Number.objects.create(n=2) + n1 = Number.objects.create(n=1) + + self.assertEqual(list(Number.objects), [n2, n1]) + self.assertEqual(list(Number.objects.order_by('n')), [n1, n2]) + self.assertEqual(list(Number.objects.order_by('n').filter()), [n1, n2]) + + Number.drop_collection() + + class QTest(unittest.TestCase): def test_empty_q(self): From e1282028a576b0a8c2bd0f9e9699f854e98db23d Mon Sep 17 00:00:00 2001 From: Viktor Kerkez Date: Mon, 1 Nov 2010 14:54:55 +0100 Subject: [PATCH 0141/1279] Added django style choices --- mongoengine/base.py | 10 +++++----- mongoengine/fields.py | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index c647208..589042e 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -100,9 +100,9 @@ class BaseField(object): def _validate(self, value): # check choices if self.choices is not None: - if value not in self.choices: - raise ValidationError("Value must be one of %s." - % unicode(self.choices)) + option_keys = [option_key for option_key, option_value in self.choices] + if value not in option_keys: + raise ValidationError("Value must be one of %s." % unicode(option_keys)) # check validation argument if self.validation is not None: @@ -254,8 +254,8 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Propagate index options. for key in ('index_background', 'index_drop_dups', 'index_opts'): - if key in base._meta: - base_meta[key] = base._meta[key] + if key in base._meta: + base_meta[key] = base._meta[key] id_field = id_field or base._meta.get('id_field') base_indexes += base._meta.get('indexes', []) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e95fd65..e95312f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -12,7 +12,6 @@ import datetime import decimal import gridfs import warnings -import types __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', From bda4776a18522704469d16c0c58b855f0cb434a9 Mon Sep 17 00:00:00 2001 From: Ales Zoulek Date: Wed, 3 Nov 2010 16:37:41 +0100 Subject: [PATCH 0142/1279] added Queryset.exclude() + tests --- mongoengine/queryset.py | 68 +++++++++++++++++--- tests/queryset.py | 133 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 191 insertions(+), 10 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index e46380b..8ce5ec7 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -268,6 +268,48 @@ class Q(QNode): return not bool(self.query) +class QueryFieldList(object): + """Object that handles combinations of .only() and .exclude() calls""" + ONLY = True + EXCLUDE = False + + def __init__(self, fields=[], direction=ONLY, always_include=[]): + self.direction = direction + self.fields = set(fields) + self.always_include = set(always_include) + + def as_dict(self): + return dict((field, self.direction) for field in self.fields) + + def __add__(self, f): + if not self.fields: + self.fields = f.fields + self.direction = f.direction + elif self.direction is self.ONLY and f.direction is self.ONLY: + self.fields = self.fields.intersection(f.fields) + elif self.direction is self.EXCLUDE and f.direction is self.EXCLUDE: + self.fields = self.fields.union(f.fields) + elif self.direction is self.ONLY and f.direction is self.EXCLUDE: + self.fields -= f.fields + elif self.direction is self.EXCLUDE and f.direction is self.ONLY: + self.direction = self.ONLY + self.fields = f.fields - self.fields + + if self.always_include: + if self.direction is self.ONLY and self.fields: + self.fields = self.fields.union(self.always_include) + else: + self.fields -= self.always_include + return self + + def reset(self): + self.fields = set([]) + self.direction = self.ONLY + + def __nonzero__(self): + return bool(self.fields) + + class QuerySet(object): """A set of results returned from a query. Wraps a MongoDB cursor, providing :class:`~mongoengine.Document` objects as the results. @@ -281,7 +323,7 @@ class QuerySet(object): self._query_obj = Q() self._initial_query = {} self._where_clause = None - self._loaded_fields = [] + self._loaded_fields = QueryFieldList() self._ordering = [] self._snapshot = False self._timeout = True @@ -290,6 +332,7 @@ class QuerySet(object): # subclasses of the class being used if document._meta.get('allow_inheritance'): self._initial_query = {'_types': self._document._class_name} + self._loaded_fields = QueryFieldList(always_include=['_cls']) self._cursor_obj = None self._limit = None self._skip = None @@ -423,7 +466,7 @@ class QuerySet(object): 'timeout': self._timeout, } if self._loaded_fields: - cursor_args['fields'] = self._loaded_fields + cursor_args['fields'] = self._loaded_fields.as_dict() self._cursor_obj = self._collection.find(self._query, **cursor_args) # Apply where clauses to cursor @@ -818,15 +861,22 @@ class QuerySet(object): .. versionadded:: 0.3 """ - self._loaded_fields = [] + fields = self._fields_to_dbfields(fields) + self._loaded_fields += QueryFieldList(fields, direction=QueryFieldList.ONLY) + return self + + + def exclude(self, *fields): + fields = self._fields_to_dbfields(fields) + self._loaded_fields += QueryFieldList(fields, direction=QueryFieldList.EXCLUDE) + return self + + def _fields_to_dbfields(self, fields): + ret = [] for field in fields: field = ".".join(f.db_field for f in QuerySet._lookup_field(self._document, field.split('.'))) - self._loaded_fields.append(field) - - # _cls is needed for polymorphism - if self._document._meta.get('allow_inheritance'): - self._loaded_fields += ['_cls'] - return self + ret.append(field) + return ret def order_by(self, *keys): """Order the :class:`~mongoengine.queryset.QuerySet` by the keys. The diff --git a/tests/queryset.py b/tests/queryset.py index 8b25524..4e1302e 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -6,7 +6,7 @@ import pymongo from datetime import datetime, timedelta from mongoengine.queryset import (QuerySet, MultipleObjectsReturned, - DoesNotExist) + DoesNotExist, QueryFieldList) from mongoengine import * @@ -497,6 +497,81 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() + def test_exclude(self): + class User(EmbeddedDocument): + name = StringField() + email = StringField() + + class Comment(EmbeddedDocument): + title = StringField() + text = StringField() + + class BlogPost(Document): + content = StringField() + author = EmbeddedDocumentField(User) + comments = ListField(EmbeddedDocumentField(Comment)) + + BlogPost.drop_collection() + + post = BlogPost(content='Had a good coffee today...') + post.author = User(name='Test User') + post.comments = [Comment(title='I aggree', text='Great post!'), Comment(title='Coffee', text='I hate coffee')] + post.save() + + obj = BlogPost.objects.exclude('author', 'comments.text').get() + self.assertEqual(obj.author, None) + self.assertEqual(obj.content, 'Had a good coffee today...') + self.assertEqual(obj.comments[0].title, 'I aggree') + self.assertEqual(obj.comments[0].text, None) + + BlogPost.drop_collection() + + def test_exclude_only_combining(self): + class Attachment(EmbeddedDocument): + name = StringField() + content = StringField() + + class Email(Document): + sender = StringField() + to = StringField() + subject = StringField() + body = StringField() + content_type = StringField() + attachments = ListField(EmbeddedDocumentField(Attachment)) + + Email.drop_collection() + email = Email(sender='me', to='you', subject='From Russia with Love', body='Hello!', content_type='text/plain') + email.attachments = [ + Attachment(name='file1.doc', content='ABC'), + Attachment(name='file2.doc', content='XYZ'), + ] + email.save() + + obj = Email.objects.exclude('content_type').exclude('body').get() + self.assertEqual(obj.sender, 'me') + self.assertEqual(obj.to, 'you') + self.assertEqual(obj.subject, 'From Russia with Love') + self.assertEqual(obj.body, None) + self.assertEqual(obj.content_type, None) + + obj = Email.objects.only('sender', 'to').exclude('body', 'sender').get() + self.assertEqual(obj.sender, None) + self.assertEqual(obj.to, 'you') + self.assertEqual(obj.subject, None) + self.assertEqual(obj.body, None) + self.assertEqual(obj.content_type, None) + + obj = Email.objects.exclude('attachments.content').exclude('body').only('to', 'attachments.name').get() + self.assertEqual(obj.attachments[0].name, 'file1.doc') + self.assertEqual(obj.attachments[0].content, None) + self.assertEqual(obj.sender, None) + self.assertEqual(obj.to, 'you') + self.assertEqual(obj.subject, None) + self.assertEqual(obj.body, None) + self.assertEqual(obj.content_type, None) + + Email.drop_collection() + def test_find_embedded(self): """Ensure that an embedded document is properly returned from a query. """ @@ -1594,6 +1669,62 @@ class QTest(unittest.TestCase): for condition in conditions: self.assertTrue(condition in query['$or']) +class QueryFieldListTest(unittest.TestCase): + def test_empty(self): + q = QueryFieldList() + self.assertFalse(q) + + q = QueryFieldList(always_include=['_cls']) + self.assertFalse(q) + + def test_include_include(self): + q = QueryFieldList() + q += QueryFieldList(fields=['a', 'b'], direction=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'a': True, 'b': True}) + q += QueryFieldList(fields=['b', 'c'], direction=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'b': True}) + + def test_include_exclude(self): + q = QueryFieldList() + q += QueryFieldList(fields=['a', 'b'], direction=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'a': True, 'b': True}) + q += QueryFieldList(fields=['b', 'c'], direction=QueryFieldList.EXCLUDE) + self.assertEqual(q.as_dict(), {'a': True}) + + def test_exclude_exclude(self): + q = QueryFieldList() + q += QueryFieldList(fields=['a', 'b'], direction=QueryFieldList.EXCLUDE) + self.assertEqual(q.as_dict(), {'a': False, 'b': False}) + q += QueryFieldList(fields=['b', 'c'], direction=QueryFieldList.EXCLUDE) + self.assertEqual(q.as_dict(), {'a': False, 'b': False, 'c': False}) + + def test_exclude_include(self): + q = QueryFieldList() + q += QueryFieldList(fields=['a', 'b'], direction=QueryFieldList.EXCLUDE) + self.assertEqual(q.as_dict(), {'a': False, 'b': False}) + q += QueryFieldList(fields=['b', 'c'], direction=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'c': True}) + + def test_always_include(self): + q = QueryFieldList(always_include=['x', 'y']) + q += QueryFieldList(fields=['a', 'b', 'x'], direction=QueryFieldList.EXCLUDE) + q += QueryFieldList(fields=['b', 'c'], direction=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'c': True}) + + + def test_reset(self): + q = QueryFieldList(always_include=['x', 'y']) + q += QueryFieldList(fields=['a', 'b', 'x'], direction=QueryFieldList.EXCLUDE) + q += QueryFieldList(fields=['b', 'c'], direction=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'c': True}) + q.reset() + self.assertFalse(q) + q += QueryFieldList(fields=['b', 'c'], direction=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'b': True, 'c': True}) + + + + if __name__ == '__main__': unittest.main() From 89646439e7afbffe7209d218f454ee0fa03a2f14 Mon Sep 17 00:00:00 2001 From: Deepak Thukral Date: Wed, 10 Nov 2010 21:02:59 +0100 Subject: [PATCH 0143/1279] fixed typo in error message --- mongoengine/queryset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index e46380b..5520100 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -78,7 +78,7 @@ class SimplificationVisitor(QNodeVisitor): # to a single field intersection = ops.intersection(query_ops) if intersection: - msg = 'Duplicate query contitions: ' + msg = 'Duplicate query conditions: ' raise InvalidQueryError(msg + ', '.join(intersection)) query_ops.update(ops) @@ -179,7 +179,7 @@ class QueryCompilerVisitor(QNodeVisitor): # once to a single field intersection = current_ops.intersection(new_ops) if intersection: - msg = 'Duplicate query contitions: ' + msg = 'Duplicate query conditions: ' raise InvalidQueryError(msg + ', '.join(intersection)) # Right! We've got two non-overlapping dicts of operations! From 66baa4eb6185ca27b2ccc92d68450ecaa17112fd Mon Sep 17 00:00:00 2001 From: Ales Zoulek Date: Wed, 10 Nov 2010 22:01:27 +0100 Subject: [PATCH 0144/1279] QS.all_fields - resets previous .only() and .exlude() --- mongoengine/queryset.py | 15 +++++++++++++++ tests/queryset.py | 23 +++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 8ce5ec7..e075537 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -867,11 +867,26 @@ class QuerySet(object): def exclude(self, *fields): + """Opposite to .only(), exclude some document's fields. :: + + post = BlogPost.objects(...).exclude("comments") + + :param fields: fields to exclude + """ fields = self._fields_to_dbfields(fields) self._loaded_fields += QueryFieldList(fields, direction=QueryFieldList.EXCLUDE) return self + def all_fields(self): + """Include all fields. Reset all previously calls of .only() and .exclude(). :: + + post = BlogPost.objects(...).exclude("comments").only("title").all_fields() + """ + self._loaded_fields = QueryFieldList(always_include=self._loaded_fields.always_include) + return self + def _fields_to_dbfields(self, fields): + """Translate fields paths to its db equivalents""" ret = [] for field in fields: field = ".".join(f.db_field for f in QuerySet._lookup_field(self._document, field.split('.'))) diff --git a/tests/queryset.py b/tests/queryset.py index 4e1302e..88e7737 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -572,6 +572,29 @@ class QuerySetTest(unittest.TestCase): Email.drop_collection() + def test_all_fields(self): + + class Email(Document): + sender = StringField() + to = StringField() + subject = StringField() + body = StringField() + content_type = StringField() + + Email.drop_collection() + + email = Email(sender='me', to='you', subject='From Russia with Love', body='Hello!', content_type='text/plain') + email.save() + + obj = Email.objects.exclude('content_type', 'body').only('to', 'body').all_fields().get() + self.assertEqual(obj.sender, 'me') + self.assertEqual(obj.to, 'you') + self.assertEqual(obj.subject, 'From Russia with Love') + self.assertEqual(obj.body, 'Hello!') + self.assertEqual(obj.content_type, 'text/plain') + + Email.drop_collection() + def test_find_embedded(self): """Ensure that an embedded document is properly returned from a query. """ From 9c8411b251ab0557063fc60d50c32c888f55c613 Mon Sep 17 00:00:00 2001 From: Viktor Kerkez Date: Thu, 11 Nov 2010 18:19:00 +0100 Subject: [PATCH 0145/1279] Choice field test updated --- .gitignore | 1 + tests/fields.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d292729..d67429a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,6 @@ build/ dist/ mongoengine.egg-info/ env/ +.settings .project .pydevproject \ No newline at end of file diff --git a/tests/fields.py b/tests/fields.py index 5602cde..034ec61 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -644,7 +644,8 @@ class FieldTest(unittest.TestCase): """Ensure that value is in a container of allowed values. """ class Shirt(Document): - size = StringField(max_length=3, choices=('S','M','L','XL','XXL')) + size = StringField(max_length=3, choices=(('S', 'Small'), ('M', 'Medium'), ('L', 'Large'), + ('XL', 'Extra Large'), ('XXL', 'Extra Extra Large'))) Shirt.drop_collection() From b12c34334ccfeef41b948eef6bedae22c7254bdf Mon Sep 17 00:00:00 2001 From: Deepak Thukral Date: Thu, 18 Nov 2010 20:44:51 +0100 Subject: [PATCH 0146/1279] added test case for issue 103 --- tests/queryset.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 8b25524..e130ece 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -539,33 +539,47 @@ class QuerySetTest(unittest.TestCase): """Ensure that Q objects may be used to query for documents. """ class BlogPost(Document): + title = StringField() publish_date = DateTimeField() published = BooleanField() BlogPost.drop_collection() - post1 = BlogPost(publish_date=datetime(2010, 1, 8), published=False) + post1 = BlogPost(title='Test 1', publish_date=datetime(2010, 1, 8), published=False) post1.save() - post2 = BlogPost(publish_date=datetime(2010, 1, 15), published=True) + post2 = BlogPost(title='Test 2', publish_date=datetime(2010, 1, 15), published=True) post2.save() - post3 = BlogPost(published=True) + post3 = BlogPost(title='Test 3', published=True) post3.save() - post4 = BlogPost(publish_date=datetime(2010, 1, 8)) + post4 = BlogPost(title='Test 4', publish_date=datetime(2010, 1, 8)) post4.save() - post5 = BlogPost(publish_date=datetime(2010, 1, 15)) + post5 = BlogPost(title='Test 1', publish_date=datetime(2010, 1, 15)) post5.save() - post6 = BlogPost(published=False) + post6 = BlogPost(title='Test 1', published=False) post6.save() # Check ObjectId lookup works obj = BlogPost.objects(id=post1.id).first() self.assertEqual(obj, post1) + # Check Q object combination with one does not exist + q = BlogPost.objects(Q(title='Test 5') | Q(published=True)) + posts = [post.id for post in q] + + published_posts = (post2, post3) + self.assertTrue(all(obj.id in posts for obj in published_posts)) + + q = BlogPost.objects(Q(title='Test 1') | Q(published=True)) + posts = [post.id for post in q] + published_posts = (post1, post2, post3, post5, post6) + self.assertTrue(all(obj.id in posts for obj in published_posts)) + + # Check Q object combination date = datetime(2010, 1, 10) q = BlogPost.objects(Q(publish_date__lte=date) | Q(published=True)) From ca56785cbcf3747428b219f1ce0029c7e80bfd29 Mon Sep 17 00:00:00 2001 From: sshwsfc Date: Thu, 18 Nov 2010 21:33:05 -0800 Subject: [PATCH 0147/1279] add some prepare_query_value method for fields --- mongoengine/fields.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e95fd65..f658b10 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -150,6 +150,8 @@ class IntField(BaseField): if self.max_value is not None and value > self.max_value: raise ValidationError('Integer value is too large') + def prepare_query_value(self, op, value): + return int(value) class FloatField(BaseField): """An floating point number field. @@ -173,6 +175,9 @@ class FloatField(BaseField): if self.max_value is not None and value > self.max_value: raise ValidationError('Float value is too large') + def prepare_query_value(self, op, value): + return float(value) + class DecimalField(BaseField): """A fixed-point decimal number field. @@ -227,6 +232,40 @@ class DateTimeField(BaseField): def validate(self, value): assert isinstance(value, datetime.datetime) + def prepare_query_value(self, op, value): + if value is None: + return value + if isinstance(value, datetime.datetime): + return value + if isinstance(value, datetime.date): + return datetime.datetime(value.year, value.month, value.day) + + # Attempt to parse a datetime: + #value = smart_str(value) + # split usecs, because they are not recognized by strptime. + if '.' in value: + try: + value, usecs = value.split('.') + usecs = int(usecs) + except ValueError: + return None + else: + usecs = 0 + kwargs = {'microsecond': usecs} + try: # Seconds are optional, so try converting seconds first. + return datetime.datetime(*time.strptime(value, '%Y-%m-%d %H:%M:%S')[:6], + **kwargs) + + except ValueError: + try: # Try without seconds. + return datetime.datetime(*time.strptime(value, '%Y-%m-%d %H:%M')[:5], + **kwargs) + except ValueError: # Try without hour/minutes/seconds. + try: + return datetime.datetime(*time.strptime(value, '%Y-%m-%d')[:3], + **kwargs) + except ValueError: + return None class EmbeddedDocumentField(BaseField): """An embedded document field. Only valid values are subclasses of From ca8c3981c4f1fcf35a83e29395647366ffff6283 Mon Sep 17 00:00:00 2001 From: sshwsfc Date: Thu, 18 Nov 2010 22:35:11 -0800 Subject: [PATCH 0148/1279] --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f658b10..7942e5e 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -8,7 +8,7 @@ import pymongo import pymongo.dbref import pymongo.son import pymongo.binary -import datetime +import datetime, time import decimal import gridfs import warnings From cec8b67b08ae9d4776b3a19b0e8979b2237a328c Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 5 Dec 2010 20:47:24 +0000 Subject: [PATCH 0149/1279] Added test for unsetting fields --- tests/queryset.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 8b25524..de3f426 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -778,6 +778,11 @@ class QuerySetTest(unittest.TestCase): BlogPost.objects.update_one(add_to_set__tags='unique') post.reload() self.assertEqual(post.tags.count('unique'), 1) + + self.assertNotEqual(post.hits, None) + BlogPost.objects.update_one(unset__hits=1) + post.reload() + self.assertEqual(post.hits, None) BlogPost.drop_collection() From 3a0523dd796ed2b7e41ac7569df1a32cc1686488 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sun, 5 Dec 2010 21:43:04 +0000 Subject: [PATCH 0150/1279] Fixed issue with unset operation --- mongoengine/queryset.py | 3 +-- tests/queryset.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index e46380b..49c8f69 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -917,8 +917,7 @@ class QuerySet(object): # Convert value to proper value field = fields[-1] - if op in (None, 'set', 'unset', 'pop', 'push', 'pull', - 'addToSet'): + if op in (None, 'set', 'push', 'pull', 'addToSet'): value = field.prepare_query_value(op, value) elif op in ('pushAll', 'pullAll'): value = [field.prepare_query_value(op, v) for v in value] diff --git a/tests/queryset.py b/tests/queryset.py index de3f426..374fdb5 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1462,6 +1462,27 @@ class QuerySetTest(unittest.TestCase): Number.drop_collection() + def test_unset_reference(self): + class Comment(Document): + text = StringField() + + class Post(Document): + comment = ReferenceField(Comment) + + Comment.drop_collection() + Post.drop_collection() + + comment = Comment.objects.create(text='test') + post = Post.objects.create(comment=comment) + + self.assertEqual(post.comment, comment) + Post.objects.update(unset__comment=1) + post.reload() + self.assertEqual(post.comment, None) + + Comment.drop_collection() + Post.drop_collection() + class QTest(unittest.TestCase): From 62cc8d2ab3a00cc2af4c8cce2f8942bd2b13a3f1 Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Sun, 5 Dec 2010 00:13:55 -0800 Subject: [PATCH 0151/1279] Fix: redefinition of "datetime" from line 6. --- tests/queryset.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 374fdb5..6362555 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -351,8 +351,6 @@ class QuerySetTest(unittest.TestCase): def test_filter_chaining(self): """Ensure filters can be chained together. """ - from datetime import datetime - class BlogPost(Document): title = StringField() is_published = BooleanField() From 67fcdca6d4f1b4fd791ad3485eff0a1b76b26e9b Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Sun, 5 Dec 2010 14:30:19 +0100 Subject: [PATCH 0152/1279] Fix: PyFlakes pointed out this missing import. --- mongoengine/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index 6b74cb0..3dd2cb0 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -492,6 +492,7 @@ class BaseDocument(object): if sys.version_info < (2, 5): # Prior to Python 2.5, Exception was an old-style class def subclass_exception(name, parents, unused): + import types return types.ClassType(name, parents, {}) else: def subclass_exception(name, parents, module): From 4f3eacd72cc807344bc06e69306b5174994be4eb Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Sun, 5 Dec 2010 14:30:50 +0100 Subject: [PATCH 0153/1279] Fix: whitespace. This broke my Vim auto-folds. --- tests/fields.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/fields.py b/tests/fields.py index 5602cde..d36a080 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -523,29 +523,29 @@ class FieldTest(unittest.TestCase): Link.drop_collection() Post.drop_collection() Bookmark.drop_collection() - + link_1 = Link(title="Pitchfork") link_1.save() - + post_1 = Post(title="Behind the Scenes of the Pavement Reunion") post_1.save() - + bm = Bookmark(bookmark_object=post_1) bm.save() - + bm = Bookmark.objects(bookmark_object=post_1).first() - + self.assertEqual(bm.bookmark_object, post_1) self.assertTrue(isinstance(bm.bookmark_object, Post)) - + bm.bookmark_object = link_1 bm.save() - + bm = Bookmark.objects(bookmark_object=link_1).first() - + self.assertEqual(bm.bookmark_object, link_1) self.assertTrue(isinstance(bm.bookmark_object, Link)) - + Link.drop_collection() Post.drop_collection() Bookmark.drop_collection() @@ -555,23 +555,23 @@ class FieldTest(unittest.TestCase): """ class Link(Document): title = StringField() - + class Post(Document): title = StringField() - + class User(Document): bookmarks = ListField(GenericReferenceField()) - + Link.drop_collection() Post.drop_collection() User.drop_collection() - + link_1 = Link(title="Pitchfork") link_1.save() - + post_1 = Post(title="Behind the Scenes of the Pavement Reunion") post_1.save() - + user = User(bookmarks=[post_1, link_1]) user.save() From 86233bcdf539874c9cddef6f883abd84f68329a3 Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Sun, 5 Dec 2010 08:08:55 -0800 Subject: [PATCH 0154/1279] Added initial implementation of cascading document deletion. The current implementation is still very basic and needs some polish. The essence of it is that each Document gets a new meta attribute called "delete_rules" that is a dictionary containing (documentclass, fieldname) as key and the actual delete rule as a value. (Possible values are DO_NOTHING, NULLIFY, CASCADE and DENY. Of those, only CASCADE is currently implented.) --- mongoengine/base.py | 3 +++ mongoengine/document.py | 27 ++++++++++++++++++++++++++- mongoengine/fields.py | 3 ++- tests/document.py | 25 +++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 3dd2cb0..29de82f 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -190,6 +190,8 @@ class DocumentMetaclass(type): new_class = super_new(cls, name, bases, attrs) for field in new_class._fields.values(): field.owner_document = new_class + if hasattr(field, 'delete_rule') and field.delete_rule: + field.document_type._meta['delete_rules'][(new_class, field.name)] = field.delete_rule module = attrs.get('__module__') @@ -258,6 +260,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): 'index_drop_dups': False, 'index_opts': {}, 'queryset_class': QuerySet, + 'delete_rules': {}, } meta.update(base_meta) diff --git a/mongoengine/document.py b/mongoengine/document.py index fef737d..0686716 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -6,9 +6,16 @@ from connection import _get_db import pymongo -__all__ = ['Document', 'EmbeddedDocument', 'ValidationError', 'OperationError'] +__all__ = ['Document', 'EmbeddedDocument', 'ValidationError', 'OperationError', + 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY'] +# Delete rules +DO_NOTHING = 0 +NULLIFY = 1 +CASCADE = 2 +DENY = 3 + class EmbeddedDocument(BaseDocument): """A :class:`~mongoengine.Document` that isn't stored in its own collection. :class:`~mongoengine.EmbeddedDocument`\ s should be used as @@ -92,6 +99,13 @@ class Document(BaseDocument): :param safe: check if the operation succeeded before returning """ + for rule_entry in self._meta['delete_rules']: + document_cls, field_name = rule_entry + rule = self._meta['delete_rules'][rule_entry] + + if rule == CASCADE: + document_cls.objects(**{field_name: self.id}).delete(safe=safe) + id_field = self._meta['id_field'] object_id = self._fields[id_field].to_mongo(self[id_field]) try: @@ -100,6 +114,17 @@ class Document(BaseDocument): message = u'Could not delete document (%s)' % err.message raise OperationError(message) + @classmethod + def register_delete_rule(cls, document_cls, field_name, rule): + """This method registers the delete rules to apply when removing this + object. This could go into the Document class. + """ + if rule == DO_NOTHING: + return + + cls._meta['delete_rules'][(document_cls, field_name)] = rule + + def reload(self): """Reloads all attributes from the database. diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e95fd65..01ec1f7 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -417,12 +417,13 @@ class ReferenceField(BaseField): access (lazily). """ - def __init__(self, document_type, **kwargs): + def __init__(self, document_type, delete_rule=None, **kwargs): if not isinstance(document_type, basestring): if not issubclass(document_type, (Document, basestring)): raise ValidationError('Argument to ReferenceField constructor ' 'must be a document class or a string') self.document_type_obj = document_type + self.delete_rule = delete_rule super(ReferenceField, self).__init__(**kwargs) @property diff --git a/tests/document.py b/tests/document.py index c056763..d5807c9 100644 --- a/tests/document.py +++ b/tests/document.py @@ -624,6 +624,31 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + + def test_cascade_delete(self): + """Ensure that a referenced document is also deleted upon deletion. + """ + + class BlogPost(Document): + meta = {'collection': 'blogpost_1'} + content = StringField() + author = ReferenceField(self.Person, delete_rule=CASCADE) + + self.Person.drop_collection() + BlogPost.drop_collection() + + author = self.Person(name='Test User') + author.save() + + post = BlogPost(content = 'Watched some TV') + post.author = author + post.save() + + # Delete the Person, which should lead to deletion of the BlogPost, too + author.delete() + self.assertEqual(len(BlogPost.objects), 0) + + def tearDown(self): self.Person.drop_collection() From bba3aeb4fa06091561e601bf9d5dd72690416ddb Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Sun, 5 Dec 2010 11:10:11 -0800 Subject: [PATCH 0155/1279] Actually *use* the register_delete_rule classmethod, since it's there. --- mongoengine/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 29de82f..9f8c1e7 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -191,7 +191,8 @@ class DocumentMetaclass(type): for field in new_class._fields.values(): field.owner_document = new_class if hasattr(field, 'delete_rule') and field.delete_rule: - field.document_type._meta['delete_rules'][(new_class, field.name)] = field.delete_rule + field.document_type.register_delete_rule(new_class, field.name, + field.delete_rule) module = attrs.get('__module__') From dd21ce9eac4156936f17e7106c1b048fe6069015 Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Sun, 5 Dec 2010 13:40:39 -0800 Subject: [PATCH 0156/1279] Initial implementation of the NULLIFY rule. --- mongoengine/document.py | 13 +++++++++++++ tests/document.py | 11 ++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 0686716..3b812ab 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -105,6 +105,19 @@ class Document(BaseDocument): if rule == CASCADE: document_cls.objects(**{field_name: self.id}).delete(safe=safe) + elif rule == NULLIFY: + # TODO: For now, this makes the nullify test pass, but it would + # be nicer to use any of these two atomic versions: + # + # document_cls.objects(**{field_name: self.id}).update(**{'unset__%s' % field_name: 1}) + # or + # document_cls.objects(**{field_name: self.id}).update(**{'set__%s' % field_name: None}) + # + # However, I'm getting ValidationError: 1/None is not a valid ObjectId + # Anybody got a clue? + for doc in document_cls.objects(**{field_name: self.id}): + doc.reviewer = None + doc.save() id_field = self._meta['id_field'] object_id = self._fields[id_field].to_mongo(self[id_field]) diff --git a/tests/document.py b/tests/document.py index d5807c9..7f92320 100644 --- a/tests/document.py +++ b/tests/document.py @@ -625,7 +625,7 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() - def test_cascade_delete(self): + def test_delete_rule_cascade_and_nullify(self): """Ensure that a referenced document is also deleted upon deletion. """ @@ -633,6 +633,7 @@ class DocumentTest(unittest.TestCase): meta = {'collection': 'blogpost_1'} content = StringField() author = ReferenceField(self.Person, delete_rule=CASCADE) + reviewer = ReferenceField(self.Person, delete_rule=NULLIFY) self.Person.drop_collection() BlogPost.drop_collection() @@ -640,10 +641,18 @@ class DocumentTest(unittest.TestCase): author = self.Person(name='Test User') author.save() + reviewer = self.Person(name='Re Viewer') + reviewer.save() + post = BlogPost(content = 'Watched some TV') post.author = author + post.reviewer = reviewer post.save() + reviewer.delete() + self.assertEqual(len(BlogPost.objects), 1) # No effect on the BlogPost + self.assertEqual(BlogPost.objects.get().reviewer, None) + # Delete the Person, which should lead to deletion of the BlogPost, too author.delete() self.assertEqual(len(BlogPost.objects), 0) From ad1aa5bd3e4f66ba18ae98b04af16a2e8aa60291 Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Sun, 5 Dec 2010 13:47:32 -0800 Subject: [PATCH 0157/1279] Add tests that need to be satisfied. --- tests/document.py | 12 ++++++++++++ tests/queryset.py | 15 +++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/tests/document.py b/tests/document.py index 7f92320..dc63e32 100644 --- a/tests/document.py +++ b/tests/document.py @@ -657,6 +657,18 @@ class DocumentTest(unittest.TestCase): author.delete() self.assertEqual(len(BlogPost.objects), 0) + def test_delete_rule_cascade_recurs(self): + """Ensure that a recursive chain of documents is also deleted upon + cascaded deletion. + """ + self.fail() + + def test_delete_rule_deny(self): + """Ensure that a document cannot be referenced if there are still + documents referring to it. + """ + self.fail() + def tearDown(self): self.Person.drop_collection() diff --git a/tests/queryset.py b/tests/queryset.py index 6362555..32bbc4b 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -734,6 +734,21 @@ class QuerySetTest(unittest.TestCase): self.Person.objects.delete() self.assertEqual(len(self.Person.objects), 0) + def test_delete_rule_cascade(self): + """Ensure cascading deletion of referring documents from the database. + """ + self.fail() + + def test_delete_rule_nullify(self): + """Ensure nullification of references to deleted documents. + """ + self.fail() + + def test_delete_rule_deny(self): + """Ensure deletion gets denied on documents that still have references to them. + """ + self.fail() + def test_update(self): """Ensure that atomic updates work properly. """ From d21434dfd648332f903b0ebe99d10197f740ce03 Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Sun, 5 Dec 2010 22:40:01 -0800 Subject: [PATCH 0158/1279] Make the nullification an atomic operation. This shortcut works now, since hmarr fixed the unset bug in dev. --- mongoengine/document.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 3b812ab..39442f6 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -106,18 +106,7 @@ class Document(BaseDocument): if rule == CASCADE: document_cls.objects(**{field_name: self.id}).delete(safe=safe) elif rule == NULLIFY: - # TODO: For now, this makes the nullify test pass, but it would - # be nicer to use any of these two atomic versions: - # - # document_cls.objects(**{field_name: self.id}).update(**{'unset__%s' % field_name: 1}) - # or - # document_cls.objects(**{field_name: self.id}).update(**{'set__%s' % field_name: None}) - # - # However, I'm getting ValidationError: 1/None is not a valid ObjectId - # Anybody got a clue? - for doc in document_cls.objects(**{field_name: self.id}): - doc.reviewer = None - doc.save() + document_cls.objects(**{field_name: self.id}).update(**{'unset__%s' % field_name: 1}) id_field = self._meta['id_field'] object_id = self._fields[id_field].to_mongo(self[id_field]) From f3da5bc092df9e8ae78a3b81f3bb3af2506d55f5 Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Sun, 5 Dec 2010 23:03:40 -0800 Subject: [PATCH 0159/1279] Fix: potential NameError bug in test case. --- tests/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/document.py b/tests/document.py index dc63e32..e5ff3b1 100644 --- a/tests/document.py +++ b/tests/document.py @@ -502,7 +502,7 @@ class DocumentTest(unittest.TestCase): try: recipient.save(validate=False) except ValidationError: - fail() + self.fail() def test_delete(self): """Ensure that document may be deleted using the delete method. From b06d7948543870cc4ca0bb41a4450e18d76053ec Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Sun, 5 Dec 2010 23:43:19 -0800 Subject: [PATCH 0160/1279] Implementation of DENY rules. --- mongoengine/document.py | 14 +++++++++++++- tests/document.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 39442f6..38831b2 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -99,6 +99,17 @@ class Document(BaseDocument): :param safe: check if the operation succeeded before returning """ + # Check for DENY rules before actually deleting/nullifying any other + # references + for rule_entry in self._meta['delete_rules']: + document_cls, field_name = rule_entry + rule = self._meta['delete_rules'][rule_entry] + if rule == DENY and document_cls.objects(**{field_name: self.id}).count() > 0: + msg = u'Could not delete document (at least %s.%s refers to it)' % \ + (document_cls.__name__, field_name) + logging.error(msg) + raise OperationError(msg) + for rule_entry in self._meta['delete_rules']: document_cls, field_name = rule_entry rule = self._meta['delete_rules'][rule_entry] @@ -106,7 +117,8 @@ class Document(BaseDocument): if rule == CASCADE: document_cls.objects(**{field_name: self.id}).delete(safe=safe) elif rule == NULLIFY: - document_cls.objects(**{field_name: self.id}).update(**{'unset__%s' % field_name: 1}) + document_cls.objects(**{field_name: + self.id}).update(**{'unset__%s' % field_name: 1}) id_field = self._meta['id_field'] object_id = self._fields[id_field].to_mongo(self[id_field]) diff --git a/tests/document.py b/tests/document.py index e5ff3b1..9965799 100644 --- a/tests/document.py +++ b/tests/document.py @@ -667,7 +667,36 @@ class DocumentTest(unittest.TestCase): """Ensure that a document cannot be referenced if there are still documents referring to it. """ - self.fail() + + class BlogPost(Document): + content = StringField() + author = ReferenceField(self.Person, delete_rule=DENY) + + self.Person.drop_collection() + BlogPost.drop_collection() + + author = self.Person(name='Test User') + author.save() + + post = BlogPost(content = 'Watched some TV') + post.author = author + post.save() + + # Delete the Person should be denied + self.assertRaises(OperationError, author.delete) # Should raise denied error + self.assertEqual(len(BlogPost.objects), 1) # No objects may have been deleted + self.assertEqual(len(self.Person.objects), 1) + + # Other users, that don't have BlogPosts must be removable, like normal + author = self.Person(name='Another User') + author.save() + + self.assertEqual(len(self.Person.objects), 2) + author.delete() + self.assertEqual(len(self.Person.objects), 1) + + self.Person.drop_collection() + BlogPost.drop_collection() def tearDown(self): From 20eb920cb487457c016e1524348fcd57eace6d50 Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Mon, 6 Dec 2010 00:06:03 -0800 Subject: [PATCH 0161/1279] Change test docstring. --- tests/document.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/document.py b/tests/document.py index 9965799..11af8b2 100644 --- a/tests/document.py +++ b/tests/document.py @@ -658,8 +658,8 @@ class DocumentTest(unittest.TestCase): self.assertEqual(len(BlogPost.objects), 0) def test_delete_rule_cascade_recurs(self): - """Ensure that a recursive chain of documents is also deleted upon - cascaded deletion. + """Ensure that a chain of documents is also deleted upon cascaded + deletion. """ self.fail() From 3c98a4bff56be07e495f83d2df52e29131005b5b Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Mon, 6 Dec 2010 00:07:30 -0800 Subject: [PATCH 0162/1279] Remove accidentally left behind debugging message. --- mongoengine/document.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 38831b2..d89d687 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -107,7 +107,6 @@ class Document(BaseDocument): if rule == DENY and document_cls.objects(**{field_name: self.id}).count() > 0: msg = u'Could not delete document (at least %s.%s refers to it)' % \ (document_cls.__name__, field_name) - logging.error(msg) raise OperationError(msg) for rule_entry in self._meta['delete_rules']: From a68cb2026671e9e7bd58938147f04d9c910b126a Mon Sep 17 00:00:00 2001 From: Igor Ivanov Date: Thu, 9 Dec 2010 08:38:47 -0800 Subject: [PATCH 0163/1279] Allow 0 or "" to be used as valid _id value. --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 6b74cb0..77d0c0d 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -442,7 +442,7 @@ class BaseDocument(object): self._meta.get('allow_inheritance', True) == False): data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] - if data.has_key('_id') and not data['_id']: + if data.has_key('_id') and data['_id'] is None: del data['_id'] return data From 07dae64d660235e681427494bd5a2d2dfc0f05dd Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Mon, 13 Dec 2010 12:36:24 -0800 Subject: [PATCH 0164/1279] More the deletion code over to the QuerySet object. The Document object doens't have any delete_rule specific code anymore, and leverages the QuerySet's ability to deny/cascade/nullify its relations. --- mongoengine/document.py | 20 -------------------- mongoengine/queryset.py | 22 ++++++++++++++++++++++ tests/document.py | 30 +++++++++++++++++++++++++++++- tests/queryset.py | 15 ++++++++++++++- 4 files changed, 65 insertions(+), 22 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index d89d687..d1a031a 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -99,26 +99,6 @@ class Document(BaseDocument): :param safe: check if the operation succeeded before returning """ - # Check for DENY rules before actually deleting/nullifying any other - # references - for rule_entry in self._meta['delete_rules']: - document_cls, field_name = rule_entry - rule = self._meta['delete_rules'][rule_entry] - if rule == DENY and document_cls.objects(**{field_name: self.id}).count() > 0: - msg = u'Could not delete document (at least %s.%s refers to it)' % \ - (document_cls.__name__, field_name) - raise OperationError(msg) - - for rule_entry in self._meta['delete_rules']: - document_cls, field_name = rule_entry - rule = self._meta['delete_rules'][rule_entry] - - if rule == CASCADE: - document_cls.objects(**{field_name: self.id}).delete(safe=safe) - elif rule == NULLIFY: - document_cls.objects(**{field_name: - self.id}).update(**{'unset__%s' % field_name: 1}) - id_field = self._meta['id_field'] object_id = self._fields[id_field].to_mongo(self[id_field]) try: diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 49c8f69..82efd4f 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -882,6 +882,28 @@ class QuerySet(object): :param safe: check if the operation succeeded before returning """ + from document import CASCADE, DENY, NULLIFY + + doc = self._document + + # Check for DENY rules before actually deleting/nullifying any other + # references + for rule_entry in doc._meta['delete_rules']: + document_cls, field_name = rule_entry + rule = doc._meta['delete_rules'][rule_entry] + if rule == DENY and document_cls.objects(**{field_name + '__in': self}).count() > 0: + msg = u'Could not delete document (at least %s.%s refers to it)' % \ + (document_cls.__name__, field_name) + raise OperationError(msg) + + for rule_entry in doc._meta['delete_rules']: + document_cls, field_name = rule_entry + rule = doc._meta['delete_rules'][rule_entry] + if rule == CASCADE: + document_cls.objects(**{field_name + '__in': self}).delete(safe=safe) + elif rule == NULLIFY: + document_cls.objects(**{field_name + '__in': self}).update(**{'unset__%s' % field_name: 1}) + self._collection.remove(self._query, safe=safe) @classmethod diff --git a/tests/document.py b/tests/document.py index 11af8b2..221d22b 100644 --- a/tests/document.py +++ b/tests/document.py @@ -661,7 +661,35 @@ class DocumentTest(unittest.TestCase): """Ensure that a chain of documents is also deleted upon cascaded deletion. """ - self.fail() + + class BlogPost(Document): + content = StringField() + author = ReferenceField(self.Person, delete_rule=CASCADE) + + class Comment(Document): + text = StringField() + post = ReferenceField(BlogPost, delete_rule=CASCADE) + + + author = self.Person(name='Test User') + author.save() + + post = BlogPost(content = 'Watched some TV') + post.author = author + post.save() + + comment = Comment(text = 'Kudos.') + comment.post = post + comment.save() + + # Delete the Person, which should lead to deletion of the BlogPost, and, + # recursively to the Comment, too + author.delete() + self.assertEqual(len(Comment.objects), 0) + + self.Person.drop_collection() + BlogPost.drop_collection() + Comment.drop_collection() def test_delete_rule_deny(self): """Ensure that a document cannot be referenced if there are still diff --git a/tests/queryset.py b/tests/queryset.py index 32bbc4b..fecbaec 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -737,7 +737,20 @@ class QuerySetTest(unittest.TestCase): def test_delete_rule_cascade(self): """Ensure cascading deletion of referring documents from the database. """ - self.fail() + class BlogPost(Document): + content = StringField() + author = ReferenceField(self.Person, delete_rule=CASCADE) + BlogPost.drop_collection() + + me = self.Person(name='Test User') + me.save() + + post = BlogPost(content='Watching TV', author=me) + post.save() + + self.assertEqual(1, BlogPost.objects.count()) + self.Person.objects.delete() + self.assertEqual(0, BlogPost.objects.count()) def test_delete_rule_nullify(self): """Ensure nullification of references to deleted documents. From 5b118f64ec0b32cca5909d4fa4809227e4794034 Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Mon, 13 Dec 2010 12:54:26 -0800 Subject: [PATCH 0165/1279] Add tests for nullification and denial on the queryset. --- tests/queryset.py | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index fecbaec..132549d 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -755,12 +755,46 @@ class QuerySetTest(unittest.TestCase): def test_delete_rule_nullify(self): """Ensure nullification of references to deleted documents. """ - self.fail() + class Category(Document): + name = StringField() + + class BlogPost(Document): + content = StringField() + category = ReferenceField(Category, delete_rule=NULLIFY) + + BlogPost.drop_collection() + Category.drop_collection() + + lameness = Category(name='Lameness') + lameness.save() + + post = BlogPost(content='Watching TV', category=lameness) + post.save() + + self.assertEqual(1, BlogPost.objects.count()) + self.assertEqual('Lameness', BlogPost.objects.first().category.name) + Category.objects.delete() + self.assertEqual(1, BlogPost.objects.count()) + self.assertEqual(None, BlogPost.objects.first().category) def test_delete_rule_deny(self): - """Ensure deletion gets denied on documents that still have references to them. + """Ensure deletion gets denied on documents that still have references + to them. """ - self.fail() + class BlogPost(Document): + content = StringField() + author = ReferenceField(self.Person, delete_rule=DENY) + + BlogPost.drop_collection() + self.Person.drop_collection() + + me = self.Person(name='Test User') + me.save() + + post = BlogPost(content='Watching TV', author=me) + post.save() + + self.assertRaises(OperationError, self.Person.objects.delete) def test_update(self): """Ensure that atomic updates work properly. From 4d5164c5804882978ec607b695c13cfcbaf4b7be Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Mon, 13 Dec 2010 13:24:20 -0800 Subject: [PATCH 0166/1279] Use multiple objects in the test. This is to ensure only the intended subset is deleted and not all objects. --- tests/queryset.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 132549d..d6ec46b 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -744,13 +744,16 @@ class QuerySetTest(unittest.TestCase): me = self.Person(name='Test User') me.save() + someoneelse = self.Person(name='Some-one Else') + someoneelse.save() - post = BlogPost(content='Watching TV', author=me) - post.save() + BlogPost(content='Watching TV', author=me).save() + BlogPost(content='Chilling out', author=me).save() + BlogPost(content='Pro Testing', author=someoneelse).save() + self.assertEqual(3, BlogPost.objects.count()) + self.Person.objects(name='Test User').delete() self.assertEqual(1, BlogPost.objects.count()) - self.Person.objects.delete() - self.assertEqual(0, BlogPost.objects.count()) def test_delete_rule_nullify(self): """Ensure nullification of references to deleted documents. From 3b55deb472638cb98a94fa59f7163709660393ed Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Mon, 13 Dec 2010 13:25:49 -0800 Subject: [PATCH 0167/1279] Remove unused meta data. --- tests/document.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/document.py b/tests/document.py index 221d22b..e768b54 100644 --- a/tests/document.py +++ b/tests/document.py @@ -630,7 +630,6 @@ class DocumentTest(unittest.TestCase): """ class BlogPost(Document): - meta = {'collection': 'blogpost_1'} content = StringField() author = ReferenceField(self.Person, delete_rule=CASCADE) reviewer = ReferenceField(self.Person, delete_rule=NULLIFY) From f30fd71c5ee6af832cdb9e01f9fdb915fef421ea Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Mon, 13 Dec 2010 13:42:01 -0800 Subject: [PATCH 0168/1279] Refactor: put the delete rule constants into the queryset module, too. --- mongoengine/document.py | 13 +++---------- mongoengine/queryset.py | 11 ++++++++--- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index d1a031a..504e14e 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,21 +1,14 @@ from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, ValidationError) -from queryset import OperationError +from queryset import OperationError, DO_NOTHING from connection import _get_db import pymongo -__all__ = ['Document', 'EmbeddedDocument', 'ValidationError', 'OperationError', - 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY'] +__all__ = ['Document', 'EmbeddedDocument', 'ValidationError', 'OperationError'] -# Delete rules -DO_NOTHING = 0 -NULLIFY = 1 -CASCADE = 2 -DENY = 3 - class EmbeddedDocument(BaseDocument): """A :class:`~mongoengine.Document` that isn't stored in its own collection. :class:`~mongoengine.EmbeddedDocument`\ s should be used as @@ -110,7 +103,7 @@ class Document(BaseDocument): @classmethod def register_delete_rule(cls, document_cls, field_name, rule): """This method registers the delete rules to apply when removing this - object. This could go into the Document class. + object. """ if rule == DO_NOTHING: return diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 82efd4f..c400a61 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -10,11 +10,18 @@ import copy import itertools __all__ = ['queryset_manager', 'Q', 'InvalidQueryError', - 'InvalidCollectionError'] + 'InvalidCollectionError', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY'] + # The maximum number of items to display in a QuerySet.__repr__ REPR_OUTPUT_SIZE = 20 +# Delete rules +DO_NOTHING = 0 +NULLIFY = 1 +CASCADE = 2 +DENY = 3 + class DoesNotExist(Exception): pass @@ -882,8 +889,6 @@ class QuerySet(object): :param safe: check if the operation succeeded before returning """ - from document import CASCADE, DENY, NULLIFY - doc = self._document # Check for DENY rules before actually deleting/nullifying any other From 620f4a222ea8c3d0177a741e234223909f433555 Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Tue, 14 Dec 2010 02:01:25 -0800 Subject: [PATCH 0169/1279] Don't check for DO_NOTHING in the delete rule registration method. It is already checked before it is invoked. This saves the ugly import of DO_NOTHING inside document.py. --- mongoengine/base.py | 3 ++- mongoengine/document.py | 5 +---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 9f8c1e7..42db460 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1,5 +1,6 @@ from queryset import QuerySet, QuerySetManager from queryset import DoesNotExist, MultipleObjectsReturned +from queryset import DO_NOTHING import sys import pymongo @@ -190,7 +191,7 @@ class DocumentMetaclass(type): new_class = super_new(cls, name, bases, attrs) for field in new_class._fields.values(): field.owner_document = new_class - if hasattr(field, 'delete_rule') and field.delete_rule: + if hasattr(field, 'delete_rule') and field.delete_rule > DO_NOTHING: field.document_type.register_delete_rule(new_class, field.name, field.delete_rule) diff --git a/mongoengine/document.py b/mongoengine/document.py index 504e14e..e64092e 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,6 +1,6 @@ from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, ValidationError) -from queryset import OperationError, DO_NOTHING +from queryset import OperationError from connection import _get_db import pymongo @@ -105,9 +105,6 @@ class Document(BaseDocument): """This method registers the delete rules to apply when removing this object. """ - if rule == DO_NOTHING: - return - cls._meta['delete_rules'][(document_cls, field_name)] = rule From 16e1f72e657895e6491f4111e621c510784f596a Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Tue, 14 Dec 2010 03:39:14 -0800 Subject: [PATCH 0170/1279] Avoid confusing semantics when comparing delete rules. --- mongoengine/base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 42db460..405f642 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -191,9 +191,10 @@ class DocumentMetaclass(type): new_class = super_new(cls, name, bases, attrs) for field in new_class._fields.values(): field.owner_document = new_class - if hasattr(field, 'delete_rule') and field.delete_rule > DO_NOTHING: + delete_rule = getattr(field, 'delete_rule', DO_NOTHING) + if delete_rule != DO_NOTHING: field.document_type.register_delete_rule(new_class, field.name, - field.delete_rule) + delete_rule) module = attrs.get('__module__') From ffc8b21f67c1e43617f5cd33f71192974a12ed99 Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Tue, 14 Dec 2010 03:50:49 -0800 Subject: [PATCH 0171/1279] Some tests broke over the default None value. --- mongoengine/fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 01ec1f7..235694a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1,4 +1,5 @@ from base import BaseField, ObjectIdField, ValidationError, get_document +from queryset import DO_NOTHING from document import Document, EmbeddedDocument from connection import _get_db from operator import itemgetter @@ -417,7 +418,7 @@ class ReferenceField(BaseField): access (lazily). """ - def __init__(self, document_type, delete_rule=None, **kwargs): + def __init__(self, document_type, delete_rule=DO_NOTHING, **kwargs): if not isinstance(document_type, basestring): if not issubclass(document_type, (Document, basestring)): raise ValidationError('Argument to ReferenceField constructor ' From e05e6b89f38562c6063154e9e9cb87fca40dde39 Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Thu, 16 Dec 2010 11:53:12 +0100 Subject: [PATCH 0172/1279] Add safe_update parameter to updates. --- mongoengine/queryset.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index c400a61..e12b308 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -907,7 +907,9 @@ class QuerySet(object): if rule == CASCADE: document_cls.objects(**{field_name + '__in': self}).delete(safe=safe) elif rule == NULLIFY: - document_cls.objects(**{field_name + '__in': self}).update(**{'unset__%s' % field_name: 1}) + document_cls.objects(**{field_name + '__in': self}).update( + safe_update=safe, + **{'unset__%s' % field_name: 1}) self._collection.remove(self._query, safe=safe) From 52f5deb456eea6e9e06236f084c9b13364c0e33a Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Mon, 20 Dec 2010 05:23:27 -0800 Subject: [PATCH 0173/1279] Add documentation for the delete_rule argument. --- docs/guide/defining-documents.rst | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 106d4ec..a2c598c 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -193,6 +193,37 @@ as the constructor's argument:: class ProfilePage(Document): content = StringField() +Dealing with deletion of referred documents +''''''''''''''''''''''''''''''''''''''''''' +By default, MongoDB doesn't check the integrity of your data, so deleting +documents that other documents still hold references to will lead to consistency +issues. Mongoengine's :class:`ReferenceField` adds some functionality to +safeguard against these kinds of database integrity problems, providing each +reference with a delete rule specification. A delete rule is specified by +supplying the :attr:`delete_rule` attribute on the :class:`ReferenceField` +definition, like this:: + + class Employee(Document): + ... + profile_page = ReferenceField('ProfilePage', delete_rule=mongoengine.NULLIFY) + +Its value can take any of the following constants: + +:const:`mongoengine.DO_NOTHING` + This is the default and won't do anything. Deletes are fast, but may + cause database inconsistency or dangling references. +:const:`mongoengine.DENY` + Deletion is denied if there still exist references to the object being + deleted. +:const:`mongoengine.NULLIFY` + Any object's fields still referring to the object being deleted are + removed (using MongoDB's "unset" operation), effectively nullifying the + relationship. +:const:`mongoengine.CASCADE` + Any object containing fields that are refererring to the object being + deleted are deleted first. + + Generic reference fields '''''''''''''''''''''''' A second kind of reference field also exists, From 07ef58c1a7e757c211bf6036768839d0471dc976 Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Mon, 20 Dec 2010 05:50:42 -0800 Subject: [PATCH 0174/1279] Rename delete_rule -> reverse_delete_rule. --- docs/guide/defining-documents.rst | 4 ++-- mongoengine/base.py | 2 +- mongoengine/fields.py | 4 ++-- tests/document.py | 16 ++++++++-------- tests/queryset.py | 12 ++++++------ 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index a2c598c..2b64ca3 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -200,8 +200,8 @@ documents that other documents still hold references to will lead to consistency issues. Mongoengine's :class:`ReferenceField` adds some functionality to safeguard against these kinds of database integrity problems, providing each reference with a delete rule specification. A delete rule is specified by -supplying the :attr:`delete_rule` attribute on the :class:`ReferenceField` -definition, like this:: +supplying the :attr:`reverse_delete_rule` attributes on the +:class:`ReferenceField` definition, like this:: class Employee(Document): ... diff --git a/mongoengine/base.py b/mongoengine/base.py index 405f642..a59cdba 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -191,7 +191,7 @@ class DocumentMetaclass(type): new_class = super_new(cls, name, bases, attrs) for field in new_class._fields.values(): field.owner_document = new_class - delete_rule = getattr(field, 'delete_rule', DO_NOTHING) + delete_rule = getattr(field, 'reverse_delete_rule', DO_NOTHING) if delete_rule != DO_NOTHING: field.document_type.register_delete_rule(new_class, field.name, delete_rule) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 235694a..5fdde1e 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -418,13 +418,13 @@ class ReferenceField(BaseField): access (lazily). """ - def __init__(self, document_type, delete_rule=DO_NOTHING, **kwargs): + def __init__(self, document_type, reverse_delete_rule=DO_NOTHING, **kwargs): if not isinstance(document_type, basestring): if not issubclass(document_type, (Document, basestring)): raise ValidationError('Argument to ReferenceField constructor ' 'must be a document class or a string') self.document_type_obj = document_type - self.delete_rule = delete_rule + self.reverse_delete_rule = reverse_delete_rule super(ReferenceField, self).__init__(**kwargs) @property diff --git a/tests/document.py b/tests/document.py index e768b54..67c21a4 100644 --- a/tests/document.py +++ b/tests/document.py @@ -625,14 +625,14 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() - def test_delete_rule_cascade_and_nullify(self): + def test_reverse_delete_rule_cascade_and_nullify(self): """Ensure that a referenced document is also deleted upon deletion. """ class BlogPost(Document): content = StringField() - author = ReferenceField(self.Person, delete_rule=CASCADE) - reviewer = ReferenceField(self.Person, delete_rule=NULLIFY) + author = ReferenceField(self.Person, reverse_delete_rule=CASCADE) + reviewer = ReferenceField(self.Person, reverse_delete_rule=NULLIFY) self.Person.drop_collection() BlogPost.drop_collection() @@ -656,18 +656,18 @@ class DocumentTest(unittest.TestCase): author.delete() self.assertEqual(len(BlogPost.objects), 0) - def test_delete_rule_cascade_recurs(self): + def test_reverse_delete_rule_cascade_recurs(self): """Ensure that a chain of documents is also deleted upon cascaded deletion. """ class BlogPost(Document): content = StringField() - author = ReferenceField(self.Person, delete_rule=CASCADE) + author = ReferenceField(self.Person, reverse_delete_rule=CASCADE) class Comment(Document): text = StringField() - post = ReferenceField(BlogPost, delete_rule=CASCADE) + post = ReferenceField(BlogPost, reverse_delete_rule=CASCADE) author = self.Person(name='Test User') @@ -690,14 +690,14 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() Comment.drop_collection() - def test_delete_rule_deny(self): + def test_reverse_delete_rule_deny(self): """Ensure that a document cannot be referenced if there are still documents referring to it. """ class BlogPost(Document): content = StringField() - author = ReferenceField(self.Person, delete_rule=DENY) + author = ReferenceField(self.Person, reverse_delete_rule=DENY) self.Person.drop_collection() BlogPost.drop_collection() diff --git a/tests/queryset.py b/tests/queryset.py index d6ec46b..f95974e 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -734,12 +734,12 @@ class QuerySetTest(unittest.TestCase): self.Person.objects.delete() self.assertEqual(len(self.Person.objects), 0) - def test_delete_rule_cascade(self): + def test_reverse_delete_rule_cascade(self): """Ensure cascading deletion of referring documents from the database. """ class BlogPost(Document): content = StringField() - author = ReferenceField(self.Person, delete_rule=CASCADE) + author = ReferenceField(self.Person, reverse_delete_rule=CASCADE) BlogPost.drop_collection() me = self.Person(name='Test User') @@ -755,7 +755,7 @@ class QuerySetTest(unittest.TestCase): self.Person.objects(name='Test User').delete() self.assertEqual(1, BlogPost.objects.count()) - def test_delete_rule_nullify(self): + def test_reverse_delete_rule_nullify(self): """Ensure nullification of references to deleted documents. """ class Category(Document): @@ -763,7 +763,7 @@ class QuerySetTest(unittest.TestCase): class BlogPost(Document): content = StringField() - category = ReferenceField(Category, delete_rule=NULLIFY) + category = ReferenceField(Category, reverse_delete_rule=NULLIFY) BlogPost.drop_collection() Category.drop_collection() @@ -780,13 +780,13 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(1, BlogPost.objects.count()) self.assertEqual(None, BlogPost.objects.first().category) - def test_delete_rule_deny(self): + def test_reverse_delete_rule_deny(self): """Ensure deletion gets denied on documents that still have references to them. """ class BlogPost(Document): content = StringField() - author = ReferenceField(self.Person, delete_rule=DENY) + author = ReferenceField(self.Person, reverse_delete_rule=DENY) BlogPost.drop_collection() self.Person.drop_collection() From 0f68df3b4a9c7c770b25ca72f0e912e54c205b5c Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Mon, 20 Dec 2010 05:52:21 -0800 Subject: [PATCH 0175/1279] Fix line width. --- docs/guide/defining-documents.rst | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 2b64ca3..80d2cd3 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -205,23 +205,22 @@ supplying the :attr:`reverse_delete_rule` attributes on the class Employee(Document): ... - profile_page = ReferenceField('ProfilePage', delete_rule=mongoengine.NULLIFY) + profile_page = ReferenceField('ProfilePage', reverse_delete_rule=mongoengine.NULLIFY) Its value can take any of the following constants: :const:`mongoengine.DO_NOTHING` - This is the default and won't do anything. Deletes are fast, but may - cause database inconsistency or dangling references. + This is the default and won't do anything. Deletes are fast, but may cause + database inconsistency or dangling references. :const:`mongoengine.DENY` Deletion is denied if there still exist references to the object being deleted. :const:`mongoengine.NULLIFY` - Any object's fields still referring to the object being deleted are - removed (using MongoDB's "unset" operation), effectively nullifying the - relationship. + Any object's fields still referring to the object being deleted are removed + (using MongoDB's "unset" operation), effectively nullifying the relationship. :const:`mongoengine.CASCADE` - Any object containing fields that are refererring to the object being - deleted are deleted first. + Any object containing fields that are refererring to the object being deleted + are deleted first. Generic reference fields From 03a757bc6efa52d85cf26a9d0d5f8086c0234571 Mon Sep 17 00:00:00 2001 From: Vincent Driessen Date: Tue, 21 Dec 2010 01:19:27 -0800 Subject: [PATCH 0176/1279] Add a safety note on using the new delete rules. --- docs/guide/defining-documents.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 80d2cd3..de0e727 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -193,6 +193,7 @@ as the constructor's argument:: class ProfilePage(Document): content = StringField() + Dealing with deletion of referred documents ''''''''''''''''''''''''''''''''''''''''''' By default, MongoDB doesn't check the integrity of your data, so deleting @@ -207,6 +208,11 @@ supplying the :attr:`reverse_delete_rule` attributes on the ... profile_page = ReferenceField('ProfilePage', reverse_delete_rule=mongoengine.NULLIFY) +The declaration in this example means that when an :class:`Employee` object is +removed, the :class:`ProfilePage` that belongs to that employee is removed as +well. If a whole batch of employees is removed, all profile pages that are +linked are removed as well. + Its value can take any of the following constants: :const:`mongoengine.DO_NOTHING` @@ -223,6 +229,23 @@ Its value can take any of the following constants: are deleted first. +.. warning:: + A safety note on setting up these delete rules! Since the delete rules are + not recorded on the database level by MongoDB itself, but instead at runtime, + in-memory, by the MongoEngine module, it is of the upmost importance + that the module that declares the relationship is loaded **BEFORE** the + delete is invoked. + + If, for example, the :class:`Employee` object lives in the + :mod:`payroll` app, and the :class:`ProfilePage` in the :mod:`people` + app, it is extremely important that the :mod:`people` app is loaded + before any employee is removed, because otherwise, MongoEngine could + never know this relationship exists. + + In Django, be sure to put all apps that have such delete rule declarations in + their :file:`models.py` in the :const:`INSTALLED_APPS` tuple. + + Generic reference fields '''''''''''''''''''''''' A second kind of reference field also exists, From 0acb2d904db38edc0820c3553ba1e0abf3a5b750 Mon Sep 17 00:00:00 2001 From: Serge Matveenko Date: Tue, 21 Dec 2010 18:11:33 +0300 Subject: [PATCH 0177/1279] Add hidden (.*) files to .gitignore but not the .gitignore itself. --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 51a9ca1..9c61296 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.* +!.gitignore *.pyc .*.swp *.egg @@ -6,4 +8,4 @@ docs/_build build/ dist/ mongoengine.egg-info/ -env/ \ No newline at end of file +env/ From 846f5a868f345431fd2cae4cf5dab483cab604fc Mon Sep 17 00:00:00 2001 From: Serge Matveenko Date: Tue, 21 Dec 2010 18:16:00 +0300 Subject: [PATCH 0178/1279] Fix Issue#116: Use cls instead of User in create_user. --- mongoengine/django/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 595852e..41d307c 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -86,7 +86,7 @@ class User(Document): else: email = '@'.join([email_name, domain_part.lower()]) - user = User(username=username, email=email, date_joined=now) + user = cls(username=username, email=email, date_joined=now) user.set_password(password) user.save() return user From 34b923b7ac93262521542877df9f96c7cf822d6a Mon Sep 17 00:00:00 2001 From: Serge Matveenko Date: Tue, 21 Dec 2010 18:29:51 +0300 Subject: [PATCH 0179/1279] Fix Issue#115: Possibility to bypass class_name check in queryset. --- mongoengine/queryset.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 519dda0..1826157 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -285,6 +285,7 @@ class QuerySet(object): self._ordering = [] self._snapshot = False self._timeout = True + self._class_check = True # If inheritance is allowed, only return instances and instances of # subclasses of the class being used @@ -298,7 +299,8 @@ class QuerySet(object): def _query(self): if self._mongo_query is None: self._mongo_query = self._query_obj.to_query(self._document) - self._mongo_query.update(self._initial_query) + if self._class_check: + self._mongo_query.update(self._initial_query) return self._mongo_query def ensure_index(self, key_or_list, drop_dups=False, background=False, @@ -349,7 +351,7 @@ class QuerySet(object): return index_list - def __call__(self, q_obj=None, **query): + def __call__(self, q_obj=None, class_check=True, **query): """Filter the selected documents by calling the :class:`~mongoengine.queryset.QuerySet` with a query. @@ -357,6 +359,8 @@ class QuerySet(object): the query; the :class:`~mongoengine.queryset.QuerySet` is filtered multiple times with different :class:`~mongoengine.queryset.Q` objects, only the last one will be used + :param class_check: If set to False bypass class name check when + querying collection :param query: Django-style query keyword arguments """ #if q_obj: @@ -367,6 +371,7 @@ class QuerySet(object): self._query_obj &= query self._mongo_query = None self._cursor_obj = None + self._class_check = class_check return self def filter(self, *q_objs, **query): From ba9813e5a37f0b32292ab599d3030ea6aaa45141 Mon Sep 17 00:00:00 2001 From: Nick Vlku Date: Sun, 9 Jan 2011 22:30:18 -0500 Subject: [PATCH 0180/1279] Fixed Issue 122: item_frequencies doesn't work if tag is also the name of a native js function Did this by checking if the item is a native function, if it is I set it to an initial numeric value. Future occurrences of the tag count correctly. --- mongoengine/queryset.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 11d4706..17ebc2e 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1209,11 +1209,19 @@ class QuerySet(object): db[collection].find(query).forEach(function(doc) { if (doc[field].constructor == Array) { doc[field].forEach(function(item) { - frequencies[item] = inc + (frequencies[item] || 0); + var preValue = 0; + if (!isNaN(frequencies[item])) { + preValue = frequencies[item]; + } + frequencies[item] = inc + preValue; }); } else { var item = doc[field]; - frequencies[item] = inc + (frequencies[item] || 0); + var preValue = 0; + if (!isNaN(frequencies[item])) { + preValue = frequencies[item]; + } + frequencies[item] = inc + preValue; } }); return frequencies; From 53d66b72673550efa9316acd097c4dc8275a8346 Mon Sep 17 00:00:00 2001 From: Michael Henson Date: Thu, 27 Jan 2011 23:51:10 -0500 Subject: [PATCH 0181/1279] Added QuerySet.clone() to support copying querysets --- mongoengine/queryset.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 519dda0..5d48e6e 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -293,6 +293,20 @@ class QuerySet(object): self._cursor_obj = None self._limit = None self._skip = None + + def clone(self): + """Creates a copy of the current :class:`~mongoengine.queryset.QuerySet`""" + c = self.__class__(self._document, self._collection_obj) + + copy_props = ('_initial_query', '_query_obj', '_where_clause', + '_loaded_fields', '_ordering', '_snapshot', + '_timeout', '_limit', '_skip') + + for prop in copy_props: + val = getattr(self, prop) + setattr(c, prop, copy.deepcopy(val)) + + return c @property def _query(self): From 6f7d7537f2fe5fbadffbe174fcbef7395922ae01 Mon Sep 17 00:00:00 2001 From: Nick Vlku Date: Sun, 6 Mar 2011 18:59:29 -0500 Subject: [PATCH 0182/1279] Added a test to verify that if a native JS function is put in as a tag, item_frequencies no longer fails (added the word 'watch' as a tag) --- tests/queryset.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index d0cdf10..d503cf3 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1208,13 +1208,13 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() - BlogPost(hits=1, tags=['music', 'film', 'actors']).save() + BlogPost(hits=1, tags=['music', 'film', 'actors', 'watch']).save() BlogPost(hits=2, tags=['music']).save() BlogPost(hits=2, tags=['music', 'actors']).save() f = BlogPost.objects.item_frequencies('tags') f = dict((key, int(val)) for key, val in f.items()) - self.assertEqual(set(['music', 'film', 'actors']), set(f.keys())) + self.assertEqual(set(['music', 'film', 'actors', 'watch']), set(f.keys())) self.assertEqual(f['music'], 3) self.assertEqual(f['actors'], 2) self.assertEqual(f['film'], 1) @@ -1228,9 +1228,9 @@ class QuerySetTest(unittest.TestCase): # Check that normalization works f = BlogPost.objects.item_frequencies('tags', normalize=True) - self.assertAlmostEqual(f['music'], 3.0/6.0) - self.assertAlmostEqual(f['actors'], 2.0/6.0) - self.assertAlmostEqual(f['film'], 1.0/6.0) + self.assertAlmostEqual(f['music'], 3.0/7.0) + self.assertAlmostEqual(f['actors'], 2.0/7.0) + self.assertAlmostEqual(f['film'], 1.0/7.0) # Check item_frequencies works for non-list fields f = BlogPost.objects.item_frequencies('hits') From 2c7469c62acaef77272383428b1ddcdee574b149 Mon Sep 17 00:00:00 2001 From: Stuart Rackham Date: Sun, 3 Apr 2011 15:21:00 +1200 Subject: [PATCH 0183/1279] Additional file-like behavior for FileField (optional size arg for read method; fixed seek and tell methods for reading files). --- mongoengine/fields.py | 12 +++++++++--- tests/fields.py | 6 ++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e95fd65..7639c7b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -561,6 +561,7 @@ class GridFSProxy(object): self.fs = gridfs.GridFS(_get_db()) # Filesystem instance self.newfile = None # Used for partial writes self.grid_id = grid_id # Store GridFS id for file + self.gridout = None def __getattr__(self, name): obj = self.get() @@ -574,8 +575,12 @@ class GridFSProxy(object): def get(self, id=None): if id: self.grid_id = id + if self.grid_id is None: + return None try: - return self.fs.get(id or self.grid_id) + if self.gridout is None: + self.gridout = self.fs.get(self.grid_id) + return self.gridout except: # File has been deleted return None @@ -605,9 +610,9 @@ class GridFSProxy(object): self.grid_id = self.newfile._id self.newfile.writelines(lines) - def read(self): + def read(self, size=-1): try: - return self.get().read() + return self.get().read(size) except: return None @@ -615,6 +620,7 @@ class GridFSProxy(object): # Delete file from GridFS, FileField still remains self.fs.delete(self.grid_id) self.grid_id = None + self.grid_out = None def replace(self, file, **kwargs): self.delete() diff --git a/tests/fields.py b/tests/fields.py index 5602cde..c76935d 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -700,6 +700,12 @@ class FieldTest(unittest.TestCase): self.assertTrue(streamfile == result) self.assertEquals(result.file.read(), text + more_text) self.assertEquals(result.file.content_type, content_type) + result.file.seek(0) + self.assertEquals(result.file.tell(), 0) + self.assertEquals(result.file.read(len(text)), text) + self.assertEquals(result.file.tell(), len(text)) + self.assertEquals(result.file.read(len(more_text)), more_text) + self.assertEquals(result.file.tell(), len(text + more_text)) result.file.delete() # Ensure deleted file returns None From bd84d08b959f60d0adc2d39e1e51d1be4f5f42fb Mon Sep 17 00:00:00 2001 From: Stuart Rackham Date: Mon, 4 Apr 2011 13:44:36 +1200 Subject: [PATCH 0184/1279] Fixed misspelt variable name. --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 7639c7b..186826a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -620,7 +620,7 @@ class GridFSProxy(object): # Delete file from GridFS, FileField still remains self.fs.delete(self.grid_id) self.grid_id = None - self.grid_out = None + self.gridout = None def replace(self, file, **kwargs): self.delete() From 829df581f03f9a3245e36a7d4f5686e6d5647b91 Mon Sep 17 00:00:00 2001 From: Stuart Rackham Date: Mon, 4 Apr 2011 15:19:57 +1200 Subject: [PATCH 0185/1279] Drop gridfs close warning --- mongoengine/fields.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e95fd65..f39d6e6 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -623,9 +623,6 @@ class GridFSProxy(object): def close(self): if self.newfile: self.newfile.close() - else: - msg = "The close() method is only necessary after calling write()" - warnings.warn(msg) class FileField(BaseField): From 96dbeea171cc7d081ccd269705752722bad146bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BD=D1=85=D0=B1=D0=B0=D1=8F=D1=80=20=D0=9B=D1=85?= =?UTF-8?q?=D0=B0=D0=B3=D0=B2=D0=B0=D0=B4=D0=BE=D1=80=D0=B6?= Date: Tue, 12 Apr 2011 20:23:16 +0800 Subject: [PATCH 0186/1279] Added __hash__, __ne__ with test. --- mongoengine/base.py | 11 +++++++++++ tests/document.py | 48 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index 6b74cb0..6340e31 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -489,6 +489,17 @@ class BaseDocument(object): return True return False + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + """ For list, dic key """ + if self.pk is None: + # For new object + return super(BaseDocument,self).__hash__() + else: + return hash(self.pk) + if sys.version_info < (2, 5): # Prior to Python 2.5, Exception was an old-style class def subclass_exception(name, parents, unused): diff --git a/tests/document.py b/tests/document.py index c056763..280b671 100644 --- a/tests/document.py +++ b/tests/document.py @@ -627,6 +627,54 @@ class DocumentTest(unittest.TestCase): def tearDown(self): self.Person.drop_collection() + def test_document_hash(self): + """Test document in list, dict, set + """ + class User(Document): + pass + + class BlogPost(Document): + pass + + # Clear old datas + User.drop_collection() + BlogPost.drop_collection() + + u1 = User.objects.create() + u2 = User.objects.create() + u3 = User.objects.create() + u4 = User() # New object + + b1 = BlogPost.objects.create() + b2 = BlogPost.objects.create() + + # in List + all_user_list = list(User.objects.all()) + + self.assertTrue(u1 in all_user_list) + self.assertTrue(u2 in all_user_list) + self.assertTrue(u3 in all_user_list) + self.assertFalse(u4 in all_user_list) # New object + self.assertFalse(b1 in all_user_list) # Other object + self.assertFalse(b2 in all_user_list) # Other object + + # in Dict + all_user_dic = {} + for u in User.objects.all(): + all_user_dic[u] = "OK" + + self.assertEqual(all_user_dic.get(u1, False), "OK" ) + self.assertEqual(all_user_dic.get(u2, False), "OK" ) + self.assertEqual(all_user_dic.get(u3, False), "OK" ) + self.assertEqual(all_user_dic.get(u4, False), False ) # New object + self.assertEqual(all_user_dic.get(b1, False), False ) # Other object + self.assertEqual(all_user_dic.get(b2, False), False ) # Other object + + # in Set + all_user_set = set(User.objects.all()) + + self.assertTrue(u1 in all_user_set ) + if __name__ == '__main__': unittest.main() From 76cbb668437267e41ed840b1177e6af8cda4f2c4 Mon Sep 17 00:00:00 2001 From: Alistair Roche Date: Thu, 28 Apr 2011 14:31:19 +0100 Subject: [PATCH 0187/1279] Fixed error with _lookup_field It was failing when given multiple fields --- mongoengine/queryset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 9f24dea..e227362 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -522,6 +522,7 @@ class QuerySet(object): raise InvalidQueryError('Cannot resolve field "%s"' % field_name) fields.append(field) + field = None return fields @classmethod From c2fef4e791916117e9df1b7678aaebef4b894ac5 Mon Sep 17 00:00:00 2001 From: Matt Chisholm Date: Tue, 3 May 2011 23:50:20 +0200 Subject: [PATCH 0188/1279] error in documentation; need to use keyword arg to create Comment object --- docs/guide/defining-documents.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 106d4ec..4c9de93 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -134,8 +134,8 @@ document class as the first argument:: class Page(Document): comments = ListField(EmbeddedDocumentField(Comment)) - comment1 = Comment('Good work!') - comment2 = Comment('Nice article!') + comment1 = Comment(content='Good work!') + comment2 = Comment(content='Nice article!') page = Page(comments=[comment1, comment2]) Dictionary Fields From 9c1ad5f631011bf8c5178d1e4ceb292def43c2e6 Mon Sep 17 00:00:00 2001 From: Gregg Lind Date: Wed, 4 May 2011 18:01:06 -0700 Subject: [PATCH 0189/1279] Tiny spelling correction / clarification. --- mongoengine/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index fef737d..f15e683 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -67,7 +67,7 @@ class Document(BaseDocument): :param safe: check if the operation succeeded before returning :param force_insert: only try to create a new document, don't allow updates of existing documents - :param validate: validates the document; set to ``False`` for skiping + :param validate: validates the document; set to ``False`` to skip. """ if validate: self.validate() From f0277736e2144130b209e0a11667e50c098ec1bc Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 9 May 2011 10:22:37 +0100 Subject: [PATCH 0190/1279] Updated queryset to handle latest version of pymongo map_reduce now requires an output. Reverted previous _lookup_field change, until a test case is produced for the incorrect behaviour. --- mongoengine/queryset.py | 14 ++++++-------- tests/queryset.py | 9 ++++++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index e227362..f5d5c5f 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -522,7 +522,6 @@ class QuerySet(object): raise InvalidQueryError('Cannot resolve field "%s"' % field_name) fields.append(field) - field = None return fields @classmethod @@ -731,7 +730,7 @@ class QuerySet(object): def __len__(self): return self.count() - def map_reduce(self, map_f, reduce_f, finalize_f=None, limit=None, + def map_reduce(self, map_f, reduce_f, output, finalize_f=None, limit=None, scope=None, keep_temp=False): """Perform a map/reduce query using the current query spec and ordering. While ``map_reduce`` respects ``QuerySet`` chaining, @@ -745,26 +744,26 @@ class QuerySet(object): :param map_f: map function, as :class:`~pymongo.code.Code` or string :param reduce_f: reduce function, as :class:`~pymongo.code.Code` or string + :param output: output collection name :param finalize_f: finalize function, an optional function that performs any post-reduction processing. :param scope: values to insert into map/reduce global scope. Optional. :param limit: number of objects from current query to provide to map/reduce method - :param keep_temp: keep temporary table (boolean, default ``True``) Returns an iterator yielding :class:`~mongoengine.document.MapReduceDocument`. - .. note:: Map/Reduce requires server version **>= 1.1.1**. The PyMongo + .. note:: Map/Reduce changed in server version **>= 1.7.4**. The PyMongo :meth:`~pymongo.collection.Collection.map_reduce` helper requires - PyMongo version **>= 1.2**. + PyMongo version **>= 1.11**. .. versionadded:: 0.3 """ from document import MapReduceDocument if not hasattr(self._collection, "map_reduce"): - raise NotImplementedError("Requires MongoDB >= 1.1.1") + raise NotImplementedError("Requires MongoDB >= 1.7.1") map_f_scope = {} if isinstance(map_f, pymongo.code.Code): @@ -795,8 +794,7 @@ class QuerySet(object): if limit: mr_args['limit'] = limit - - results = self._collection.map_reduce(map_f, reduce_f, **mr_args) + results = self._collection.map_reduce(map_f, reduce_f, output, **mr_args) results = results.find() if self._ordering: diff --git a/tests/queryset.py b/tests/queryset.py index d0cdf10..746e8c2 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1027,7 +1027,7 @@ class QuerySetTest(unittest.TestCase): """ # run a map/reduce operation spanning all posts - results = BlogPost.objects.map_reduce(map_f, reduce_f) + results = BlogPost.objects.map_reduce(map_f, reduce_f, "myresults") results = list(results) self.assertEqual(len(results), 4) @@ -1076,7 +1076,7 @@ class QuerySetTest(unittest.TestCase): } """ - results = BlogPost.objects.map_reduce(map_f, reduce_f) + results = BlogPost.objects.map_reduce(map_f, reduce_f, "myresults") results = list(results) self.assertEqual(results[0].object, post1) @@ -1187,6 +1187,7 @@ class QuerySetTest(unittest.TestCase): results = Link.objects.order_by("-value") results = results.map_reduce(map_f, reduce_f, + "myresults", finalize_f=finalize_f, scope=scope) results = list(results) @@ -1451,7 +1452,9 @@ class QuerySetTest(unittest.TestCase): """ class Test(Document): testdict = DictField() - + + Test.drop_collection() + t = Test(testdict={'f': 'Value'}) t.save() From 3b7a8ce449ef6ca4f4c09775a2a5cdbed248a2aa Mon Sep 17 00:00:00 2001 From: Matt Chisholm Date: Mon, 9 May 2011 22:29:27 +0200 Subject: [PATCH 0191/1279] change comments to reflect that the geospatial queries use degrees, not miles --- tests/queryset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 6ca4174..3592cc6 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1316,7 +1316,7 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(events.count(), 3) self.assertEqual(list(events), [event1, event3, event2]) - # find events within 5 miles of pitchfork office, chicago + # find events within 5 degrees of pitchfork office, chicago point_and_distance = [[41.9120459, -87.67892], 5] events = Event.objects(location__within_distance=point_and_distance) self.assertEqual(events.count(), 2) @@ -1331,13 +1331,13 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(events.count(), 3) self.assertEqual(list(events), [event3, event1, event2]) - # find events around san francisco + # find events within 10 degrees of san francisco point_and_distance = [[37.7566023, -122.415579], 10] events = Event.objects(location__within_distance=point_and_distance) self.assertEqual(events.count(), 1) self.assertEqual(events[0], event2) - # find events within 1 mile of greenpoint, broolyn, nyc, ny + # find events within 1 degree of greenpoint, broolyn, nyc, ny point_and_distance = [[40.7237134, -73.9509714], 1] events = Event.objects(location__within_distance=point_and_distance) self.assertEqual(events.count(), 0) From 608f08c267aff857eba5bc28ef9fc70fd11abe71 Mon Sep 17 00:00:00 2001 From: Matt Chisholm Date: Tue, 10 May 2011 12:28:56 +0200 Subject: [PATCH 0192/1279] Implement spherical geospatial query operators & unit tests fixes https://github.com/hmarr/mongoengine/issues/163 --- mongoengine/queryset.py | 6 ++++- tests/queryset.py | 52 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 519dda0..008d240 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -475,7 +475,7 @@ class QuerySet(object): """ operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', 'all', 'size', 'exists', 'not'] - geo_operators = ['within_distance', 'within_box', 'near'] + geo_operators = ['within_distance', 'within_spherical_distance', 'within_box', 'near', 'near_sphere'] match_operators = ['contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith', 'exact', 'iexact'] @@ -519,8 +519,12 @@ class QuerySet(object): if op in geo_operators: if op == "within_distance": value = {'$within': {'$center': value}} + elif op == "within_spherical_distance": + value = {'$within': {'$centerSphere': value}} elif op == "near": value = {'$near': value} + elif op == "near_sphere": + value = {'$nearSphere': value} elif op == 'within_box': value = {'$within': {'$box': value}} else: diff --git a/tests/queryset.py b/tests/queryset.py index 3592cc6..03256d5 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1357,6 +1357,58 @@ class QuerySetTest(unittest.TestCase): Event.drop_collection() + def test_spherical_geospatial_operators(self): + """Ensure that spherical geospatial queries are working + """ + class Point(Document): + location = GeoPointField() + + Point.drop_collection() + + # These points are one degree apart, which (according to Google Maps) + # is about 110 km apart at this place on the Earth. + north_point = Point(location=[-122, 38]) # Near Concord, CA + south_point = Point(location=[-122, 37]) # Near Santa Cruz, CA + north_point.save() + south_point.save() + + earth_radius = 6378.009; # in km (needs to be a float for dividing by) + + # Finds both points because they are within 60 km of the reference + # point equidistant between them. + points = Point.objects(location__near_sphere=[-122, 37.5]) + self.assertEqual(points.count(), 2) + + # Finds both points, but orders the north point first because it's + # closer to the reference point to the north. + points = Point.objects(location__near_sphere=[-122, 38.5]) + self.assertEqual(points.count(), 2) + self.assertEqual(points[0].id, north_point.id) + self.assertEqual(points[1].id, south_point.id) + + # Finds both points, but orders the south point first because it's + # closer to the reference point to the south. + points = Point.objects(location__near_sphere=[-122, 36.5]) + self.assertEqual(points.count(), 2) + self.assertEqual(points[0].id, south_point.id) + self.assertEqual(points[1].id, north_point.id) + + # Same behavior for _within_spherical_distance + points = Point.objects( + location__within_spherical_distance=[[-122, 37.5], 60/earth_radius] + ); + self.assertEqual(points.count(), 2) + + # Finds only one point because only the first point is within 60km of + # the reference point to the south. + points = Point.objects( + location__within_spherical_distance=[[-122, 36.5], 60/earth_radius] + ); + self.assertEqual(points.count(), 1) + self.assertEqual(points[0].id, south_point.id) + + Point.drop_collection() + def test_custom_querysets(self): """Ensure that custom QuerySet classes may be used. """ From 6cf0cf9e7d8076773ebfebb60d4e56faa9cd49f8 Mon Sep 17 00:00:00 2001 From: Matt Chisholm Date: Tue, 10 May 2011 12:31:31 +0200 Subject: [PATCH 0193/1279] slight reordering of test code for clarity --- tests/queryset.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 03256d5..6d4f161 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1379,6 +1379,12 @@ class QuerySetTest(unittest.TestCase): points = Point.objects(location__near_sphere=[-122, 37.5]) self.assertEqual(points.count(), 2) + # Same behavior for _within_spherical_distance + points = Point.objects( + location__within_spherical_distance=[[-122, 37.5], 60/earth_radius] + ); + self.assertEqual(points.count(), 2) + # Finds both points, but orders the north point first because it's # closer to the reference point to the north. points = Point.objects(location__near_sphere=[-122, 38.5]) @@ -1393,12 +1399,6 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(points[0].id, south_point.id) self.assertEqual(points[1].id, north_point.id) - # Same behavior for _within_spherical_distance - points = Point.objects( - location__within_spherical_distance=[[-122, 37.5], 60/earth_radius] - ); - self.assertEqual(points.count(), 2) - # Finds only one point because only the first point is within 60km of # the reference point to the south. points = Point.objects( From 31521ccff5d984d3321d1ec3791521cead3ae37f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 18 May 2011 10:30:07 +0100 Subject: [PATCH 0194/1279] Added queryset clone support and tests, thanks to hensom Fixes #130 --- mongoengine/queryset.py | 32 +++++------ tests/queryset.py | 120 ++++++++++++++++++++++++++++------------ 2 files changed, 101 insertions(+), 51 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index bafb7c1..e0c6213 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -344,19 +344,19 @@ class QuerySet(object): self._cursor_obj = None self._limit = None self._skip = None - + def clone(self): """Creates a copy of the current :class:`~mongoengine.queryset.QuerySet`""" c = self.__class__(self._document, self._collection_obj) - + copy_props = ('_initial_query', '_query_obj', '_where_clause', '_loaded_fields', '_ordering', '_snapshot', '_timeout', '_limit', '_skip') - + for prop in copy_props: val = getattr(self, prop) setattr(c, prop, copy.deepcopy(val)) - + return c @property @@ -493,7 +493,7 @@ class QuerySet(object): } if self._loaded_fields: cursor_args['fields'] = self._loaded_fields.as_dict() - self._cursor_obj = self._collection.find(self._query, + self._cursor_obj = self._collection.find(self._query, **cursor_args) # Apply where clauses to cursor if self._where_clause: @@ -553,8 +553,8 @@ class QuerySet(object): operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', 'all', 'size', 'exists', 'not'] geo_operators = ['within_distance', 'within_spherical_distance', 'within_box', 'near', 'near_sphere'] - match_operators = ['contains', 'icontains', 'startswith', - 'istartswith', 'endswith', 'iendswith', + match_operators = ['contains', 'icontains', 'startswith', + 'istartswith', 'endswith', 'iendswith', 'exact', 'iexact'] mongo_query = {} @@ -644,8 +644,8 @@ class QuerySet(object): % self._document._class_name) def get_or_create(self, *q_objs, **query): - """Retrieve unique object or create, if it doesn't exist. Returns a tuple of - ``(object, created)``, where ``object`` is the retrieved or created object + """Retrieve unique object or create, if it doesn't exist. Returns a tuple of + ``(object, created)``, where ``object`` is the retrieved or created object and ``created`` is a boolean specifying whether a new object was created. Raises :class:`~mongoengine.queryset.MultipleObjectsReturned` or `DocumentName.MultipleObjectsReturned` if multiple results are found. @@ -857,7 +857,7 @@ class QuerySet(object): self._skip, self._limit = key.start, key.stop except IndexError, err: # PyMongo raises an error if key.start == key.stop, catch it, - # bin it, kill it. + # bin it, kill it. start = key.start or 0 if start >= 0 and key.stop >= 0 and key.step is None: if start == key.stop: @@ -1052,7 +1052,7 @@ class QuerySet(object): return mongo_update def update(self, safe_update=True, upsert=False, **update): - """Perform an atomic update on the fields matched by the query. When + """Perform an atomic update on the fields matched by the query. When ``safe_update`` is used, the number of affected documents is returned. :param safe: check if the operation succeeded before returning @@ -1076,7 +1076,7 @@ class QuerySet(object): raise OperationError(u'Update failed (%s)' % unicode(err)) def update_one(self, safe_update=True, upsert=False, **update): - """Perform an atomic update on first field matched by the query. When + """Perform an atomic update on first field matched by the query. When ``safe_update`` is used, the number of affected documents is returned. :param safe: check if the operation succeeded before returning @@ -1104,8 +1104,8 @@ class QuerySet(object): return self def _sub_js_fields(self, code): - """When fields are specified with [~fieldname] syntax, where - *fieldname* is the Python name of a field, *fieldname* will be + """When fields are specified with [~fieldname] syntax, where + *fieldname* is the Python name of a field, *fieldname* will be substituted for the MongoDB name of the field (specified using the :attr:`name` keyword argument in a field's constructor). """ @@ -1128,9 +1128,9 @@ class QuerySet(object): options specified as keyword arguments. As fields in MongoEngine may use different names in the database (set - using the :attr:`db_field` keyword argument to a :class:`Field` + using the :attr:`db_field` keyword argument to a :class:`Field` constructor), a mechanism exists for replacing MongoEngine field names - with the database field names in Javascript code. When accessing a + with the database field names in Javascript code. When accessing a field, use square-bracket notation, and prefix the MongoEngine field name with a tilde (~). diff --git a/tests/queryset.py b/tests/queryset.py index d4d7fb3..2543178 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -162,7 +162,7 @@ class QuerySetTest(unittest.TestCase): person = self.Person.objects.get(age__lt=30) self.assertEqual(person.name, "User A") - + def test_find_array_position(self): """Ensure that query by array position works. """ @@ -177,7 +177,7 @@ class QuerySetTest(unittest.TestCase): posts = ListField(EmbeddedDocumentField(Post)) Blog.drop_collection() - + Blog.objects.create(tags=['a', 'b']) self.assertEqual(len(Blog.objects(tags__0='a')), 1) self.assertEqual(len(Blog.objects(tags__0='b')), 0) @@ -226,16 +226,16 @@ class QuerySetTest(unittest.TestCase): person, created = self.Person.objects.get_or_create(age=30) self.assertEqual(person.name, "User B") self.assertEqual(created, False) - + person, created = self.Person.objects.get_or_create(age__lt=30) self.assertEqual(person.name, "User A") self.assertEqual(created, False) - + # Try retrieving when no objects exists - new doc should be created kwargs = dict(age=50, defaults={'name': 'User C'}) person, created = self.Person.objects.get_or_create(**kwargs) self.assertEqual(created, True) - + person = self.Person.objects.get(age=50) self.assertEqual(person.name, "User C") @@ -328,7 +328,7 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(obj, person) obj = self.Person.objects(Q(name__iexact='gUIDO VAN rOSSU')).first() self.assertEqual(obj, None) - + # Test unsafe expressions person = self.Person(name='Guido van Rossum [.\'Geek\']') person.save() @@ -674,7 +674,7 @@ class QuerySetTest(unittest.TestCase): posts = [post.id for post in q] published_posts = (post1, post2, post3, post5, post6) self.assertTrue(all(obj.id in posts for obj in published_posts)) - + # Check Q object combination date = datetime(2010, 1, 10) @@ -714,7 +714,7 @@ class QuerySetTest(unittest.TestCase): obj = self.Person.objects(Q(name__not=re.compile('^bob'))).first() self.assertEqual(obj, person) - + obj = self.Person.objects(Q(name__not=re.compile('^Gui'))).first() self.assertEqual(obj, None) @@ -786,7 +786,7 @@ class QuerySetTest(unittest.TestCase): class BlogPost(Document): name = StringField(db_field='doc-name') - comments = ListField(EmbeddedDocumentField(Comment), + comments = ListField(EmbeddedDocumentField(Comment), db_field='cmnts') BlogPost.drop_collection() @@ -958,7 +958,7 @@ class QuerySetTest(unittest.TestCase): BlogPost.objects.update_one(unset__hits=1) post.reload() self.assertEqual(post.hits, None) - + BlogPost.drop_collection() def test_update_pull(self): @@ -1038,7 +1038,7 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(film.value, 3) BlogPost.drop_collection() - + def test_map_reduce_with_custom_object_ids(self): """Ensure that QuerySet.map_reduce works properly with custom primary keys. @@ -1047,24 +1047,24 @@ class QuerySetTest(unittest.TestCase): class BlogPost(Document): title = StringField(primary_key=True) tags = ListField(StringField()) - + post1 = BlogPost(title="Post #1", tags=["mongodb", "mongoengine"]) post2 = BlogPost(title="Post #2", tags=["django", "mongodb"]) post3 = BlogPost(title="Post #3", tags=["hitchcock films"]) - + post1.save() post2.save() post3.save() - + self.assertEqual(BlogPost._fields['title'].db_field, '_id') self.assertEqual(BlogPost._meta['id_field'], 'title') - + map_f = """ function() { emit(this._id, 1); } """ - + # reduce to a list of tag ids and counts reduce_f = """ function(key, values) { @@ -1075,10 +1075,10 @@ class QuerySetTest(unittest.TestCase): return total; } """ - + results = BlogPost.objects.map_reduce(map_f, reduce_f, "myresults") results = list(results) - + self.assertEqual(results[0].object, post1) self.assertEqual(results[1].object, post2) self.assertEqual(results[2].object, post3) @@ -1168,7 +1168,7 @@ class QuerySetTest(unittest.TestCase): finalize_f = """ function(key, value) { - // f(sec_since_epoch,y,z) = + // f(sec_since_epoch,y,z) = // log10(z) + ((y*sec_since_epoch) / 45000) z_10 = Math.log(value.z) / Math.log(10); weight = z_10 + ((value.y * value.t_s) / 45000); @@ -1452,9 +1452,9 @@ class QuerySetTest(unittest.TestCase): """ class Test(Document): testdict = DictField() - + Test.drop_collection() - + t = Test(testdict={'f': 'Value'}) t.save() @@ -1517,12 +1517,12 @@ class QuerySetTest(unittest.TestCase): title = StringField() date = DateTimeField() location = GeoPointField() - + def __unicode__(self): return self.title - + Event.drop_collection() - + event1 = Event(title="Coltrane Motion @ Double Door", date=datetime.now() - timedelta(days=1), location=[41.909889, -87.677137]) @@ -1532,7 +1532,7 @@ class QuerySetTest(unittest.TestCase): event3 = Event(title="Coltrane Motion @ Empty Bottle", date=datetime.now(), location=[41.900474, -87.686638]) - + event1.save() event2.save() event3.save() @@ -1552,24 +1552,24 @@ class QuerySetTest(unittest.TestCase): self.assertTrue(event2 not in events) self.assertTrue(event1 in events) self.assertTrue(event3 in events) - + # ensure ordering is respected by "near" events = Event.objects(location__near=[41.9120459, -87.67892]) events = events.order_by("-date") self.assertEqual(events.count(), 3) self.assertEqual(list(events), [event3, event1, event2]) - + # find events within 10 degrees of san francisco point_and_distance = [[37.7566023, -122.415579], 10] events = Event.objects(location__within_distance=point_and_distance) self.assertEqual(events.count(), 1) self.assertEqual(events[0], event2) - + # find events within 1 degree of greenpoint, broolyn, nyc, ny point_and_distance = [[40.7237134, -73.9509714], 1] events = Event.objects(location__within_distance=point_and_distance) self.assertEqual(events.count(), 0) - + # ensure ordering is respected by "within_distance" point_and_distance = [[41.9120459, -87.67892], 10] events = Event.objects(location__within_distance=point_and_distance) @@ -1582,7 +1582,7 @@ class QuerySetTest(unittest.TestCase): events = Event.objects(location__within_box=box) self.assertEqual(events.count(), 1) self.assertEqual(events[0].id, event2.id) - + Event.drop_collection() def test_spherical_geospatial_operators(self): @@ -1692,6 +1692,35 @@ class QuerySetTest(unittest.TestCase): Number.drop_collection() + def test_clone(self): + """Ensure that cloning clones complex querysets + """ + class Number(Document): + n = IntField() + + Number.drop_collection() + + for i in xrange(1, 101): + t = Number(n=i) + t.save() + + test = Number.objects + test2 = test.clone() + self.assertFalse(test == test2) + self.assertEqual(test.count(), test2.count()) + + test = test.filter(n__gt=11) + test2 = test.clone() + self.assertFalse(test == test2) + self.assertEqual(test.count(), test2.count()) + + test = test.limit(10) + test2 = test.clone() + self.assertFalse(test == test2) + self.assertEqual(test.count(), test2.count()) + + Number.drop_collection() + def test_unset_reference(self): class Comment(Document): text = StringField() @@ -1734,7 +1763,7 @@ class QTest(unittest.TestCase): query = {'age': {'$gte': 18}, 'name': 'test'} self.assertEqual((q1 & q2 & q3 & q4 & q5).to_query(Person), query) - + def test_q_with_dbref(self): """Ensure Q objects handle DBRefs correctly""" connect(db='mongoenginetest') @@ -1776,7 +1805,7 @@ class QTest(unittest.TestCase): query = Q(x__lt=100) & Q(y__ne='NotMyString') query &= Q(y__in=['a', 'b', 'c']) & Q(x__gt=-100) mongo_query = { - 'x': {'$lt': 100, '$gt': -100}, + 'x': {'$lt': 100, '$gt': -100}, 'y': {'$ne': 'NotMyString', '$in': ['a', 'b', 'c']}, } self.assertEqual(query.to_query(TestDoc), mongo_query) @@ -1850,6 +1879,30 @@ class QTest(unittest.TestCase): for condition in conditions: self.assertTrue(condition in query['$or']) + + def test_q_clone(self): + + class TestDoc(Document): + x = IntField() + + TestDoc.drop_collection() + for i in xrange(1, 101): + t = TestDoc(x=i) + t.save() + + # Check normal cases work without an error + test = TestDoc.objects(Q(x__lt=7) & Q(x__gt=3)) + + self.assertEqual(test.count(), 3) + + test2 = test.clone() + self.assertEqual(test2.count(), 3) + self.assertFalse(test2 == test) + + test2.filter(x=6) + self.assertEqual(test2.count(), 1) + self.assertEqual(test.count(), 3) + class QueryFieldListTest(unittest.TestCase): def test_empty(self): q = QueryFieldList() @@ -1904,8 +1957,5 @@ class QueryFieldListTest(unittest.TestCase): self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'b': True, 'c': True}) - - - if __name__ == '__main__': unittest.main() From 1a049ee49d3e5fc758f9c0d5268282599f9268dc Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 18 May 2011 11:06:14 +0100 Subject: [PATCH 0195/1279] Added regression test case for mongoengine/issues/155 --- tests/queryset.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 2543178..6a87a1e 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -105,6 +105,10 @@ class QuerySetTest(unittest.TestCase): people = list(self.Person.objects[1:1]) self.assertEqual(len(people), 0) + # Test slice out of range + people = list(self.Person.objects[80000:80001]) + self.assertEqual(len(people), 0) + def test_find_one(self): """Ensure that a query using find_one returns a valid result. """ From 1781c4638b0bc7b510c6c3cea27305fcb933941e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 18 May 2011 11:41:23 +0100 Subject: [PATCH 0196/1279] Changed how the connection identity key is made Uses the current thread identity as well as the process idenity to form the key. Fixes #151 --- mongoengine/connection.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 814fde1..fc6c768 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,5 +1,6 @@ from pymongo import Connection import multiprocessing +import threading __all__ = ['ConnectionError', 'connect'] @@ -22,6 +23,8 @@ class ConnectionError(Exception): def _get_connection(reconnect=False): + """Handles the connection to the database + """ global _connection identity = get_identity() # Connect to the database if not already connected @@ -33,6 +36,9 @@ def _get_connection(reconnect=False): return _connection[identity] def _get_db(reconnect=False): + """Handles database connections and authentication based on the current + identity + """ global _db, _connection identity = get_identity() # Connect if not already connected @@ -52,12 +58,17 @@ def _get_db(reconnect=False): return _db[identity] def get_identity(): + """Creates an identity key based on the current process and thread + identity. + """ identity = multiprocessing.current_process()._identity identity = 0 if not identity else identity[0] + + identity = (identity, threading.current_thread().ident) return identity - + def connect(db, username=None, password=None, **kwargs): - """Connect to the database specified by the 'db' argument. Connection + """Connect to the database specified by the 'db' argument. Connection settings may be provided here as well if the database is not running on the default port on localhost. If authentication is needed, provide username and password arguments as well. From 7ba40062d3a8b661feba22db94711a472c14a172 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 18 May 2011 12:18:33 +0100 Subject: [PATCH 0197/1279] Fixes ordering with custom db field names Closes #125 --- mongoengine/queryset.py | 4 ++++ tests/queryset.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index e0c6213..3b37c3d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -939,6 +939,10 @@ class QuerySet(object): if key[0] in ('-', '+'): key = key[1:] key = key.replace('__', '.') + try: + key = QuerySet._translate_field_name(self._document, key) + except: + pass key_list.append((key, direction)) self._ordering = key_list diff --git a/tests/queryset.py b/tests/queryset.py index 6a87a1e..48ce627 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1746,6 +1746,20 @@ class QuerySetTest(unittest.TestCase): Comment.drop_collection() Post.drop_collection() + def test_order_works_with_custom_db_field_names(self): + class Number(Document): + n = IntField(db_field='number') + + Number.drop_collection() + + n2 = Number.objects.create(n=2) + n1 = Number.objects.create(n=1) + + self.assertEqual(list(Number.objects), [n2,n1]) + self.assertEqual(list(Number.objects.order_by('n')), [n1,n2]) + + Number.drop_collection() + class QTest(unittest.TestCase): From 5cbc76ea81c78a379d8be038437fdb49edcb7e13 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 18 May 2011 12:26:51 +0100 Subject: [PATCH 0198/1279] Pep8 --- tests/document.py | 68 +++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/tests/document.py b/tests/document.py index 0da7b93..45f1c3c 100644 --- a/tests/document.py +++ b/tests/document.py @@ -7,7 +7,7 @@ from mongoengine.connection import _get_db class DocumentTest(unittest.TestCase): - + def setUp(self): connect(db='mongoenginetest') self.db = _get_db() @@ -38,7 +38,7 @@ class DocumentTest(unittest.TestCase): name = name_field age = age_field non_field = True - + self.assertEqual(Person._fields['name'], name_field) self.assertEqual(Person._fields['age'], age_field) self.assertFalse('non_field' in Person._fields) @@ -60,7 +60,7 @@ class DocumentTest(unittest.TestCase): mammal_superclasses = {'Animal': Animal} self.assertEqual(Mammal._superclasses, mammal_superclasses) - + dog_superclasses = { 'Animal': Animal, 'Animal.Mammal': Mammal, @@ -68,7 +68,7 @@ class DocumentTest(unittest.TestCase): self.assertEqual(Dog._superclasses, dog_superclasses) def test_get_subclasses(self): - """Ensure that the correct list of subclasses is retrieved by the + """Ensure that the correct list of subclasses is retrieved by the _get_subclasses method. """ class Animal(Document): pass @@ -78,15 +78,15 @@ class DocumentTest(unittest.TestCase): class Dog(Mammal): pass mammal_subclasses = { - 'Animal.Mammal.Dog': Dog, + 'Animal.Mammal.Dog': Dog, 'Animal.Mammal.Human': Human } self.assertEqual(Mammal._get_subclasses(), mammal_subclasses) - + animal_subclasses = { 'Animal.Fish': Fish, 'Animal.Mammal': Mammal, - 'Animal.Mammal.Dog': Dog, + 'Animal.Mammal.Dog': Dog, 'Animal.Mammal.Human': Human } self.assertEqual(Animal._get_subclasses(), animal_subclasses) @@ -124,7 +124,7 @@ class DocumentTest(unittest.TestCase): self.assertTrue('name' in Employee._fields) self.assertTrue('salary' in Employee._fields) - self.assertEqual(Employee._meta['collection'], + self.assertEqual(Employee._meta['collection'], self.Person._meta['collection']) # Ensure that MRO error is not raised @@ -146,7 +146,7 @@ class DocumentTest(unittest.TestCase): class Dog(Animal): pass self.assertRaises(ValueError, create_dog_class) - + # Check that _cls etc aren't present on simple documents dog = Animal(name='dog') dog.save() @@ -161,7 +161,7 @@ class DocumentTest(unittest.TestCase): class Employee(self.Person): meta = {'allow_inheritance': False} self.assertRaises(ValueError, create_employee_class) - + # Test the same for embedded documents class Comment(EmbeddedDocument): content = StringField() @@ -186,7 +186,7 @@ class DocumentTest(unittest.TestCase): class Person(Document): name = StringField() meta = {'collection': collection} - + user = Person(name="Test User") user.save() self.assertTrue(collection in self.db.collection_names()) @@ -280,7 +280,7 @@ class DocumentTest(unittest.TestCase): tags = ListField(StringField()) meta = { 'indexes': [ - '-date', + '-date', 'tags', ('category', '-date') ], @@ -296,12 +296,12 @@ class DocumentTest(unittest.TestCase): list(BlogPost.objects) info = BlogPost.objects._collection.index_information() info = [value['key'] for key, value in info.iteritems()] - self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] + self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] in info) self.assertTrue([('_types', 1), ('addDate', -1)] in info) # tags is a list field so it shouldn't have _types in the index self.assertTrue([('tags', 1)] in info) - + class ExtendedBlogPost(BlogPost): title = StringField() meta = {'indexes': ['title']} @@ -311,7 +311,7 @@ class DocumentTest(unittest.TestCase): list(ExtendedBlogPost.objects) info = ExtendedBlogPost.objects._collection.index_information() info = [value['key'] for key, value in info.iteritems()] - self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] + self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] in info) self.assertTrue([('_types', 1), ('addDate', -1)] in info) self.assertTrue([('_types', 1), ('title', 1)] in info) @@ -380,7 +380,7 @@ class DocumentTest(unittest.TestCase): class EmailUser(User): email = StringField() - + user = User(username='test', name='test user') user.save() @@ -391,20 +391,20 @@ class DocumentTest(unittest.TestCase): user_son = User.objects._collection.find_one() self.assertEqual(user_son['_id'], 'test') self.assertTrue('username' not in user_son['_id']) - + User.drop_collection() - + user = User(pk='mongo', name='mongo user') user.save() - + user_obj = User.objects.first() self.assertEqual(user_obj.id, 'mongo') self.assertEqual(user_obj.pk, 'mongo') - + user_son = User.objects._collection.find_one() self.assertEqual(user_son['_id'], 'mongo') self.assertTrue('username' not in user_son['_id']) - + User.drop_collection() def test_creation(self): @@ -457,18 +457,18 @@ class DocumentTest(unittest.TestCase): """ class Comment(EmbeddedDocument): content = StringField() - + self.assertTrue('content' in Comment._fields) self.assertFalse('id' in Comment._fields) self.assertFalse('collection' in Comment._meta) - + def test_embedded_document_validation(self): """Ensure that embedded documents may be validated. """ class Comment(EmbeddedDocument): date = DateTimeField() content = StringField(required=True) - + comment = Comment() self.assertRaises(ValidationError, comment.validate) @@ -496,7 +496,7 @@ class DocumentTest(unittest.TestCase): # Test skipping validation on save class Recipient(Document): email = EmailField(required=True) - + recipient = Recipient(email='root@localhost') self.assertRaises(ValidationError, recipient.save) try: @@ -517,19 +517,19 @@ class DocumentTest(unittest.TestCase): """Ensure that a document may be saved with a custom _id. """ # Create person object and save it to the database - person = self.Person(name='Test User', age=30, + person = self.Person(name='Test User', age=30, id='497ce96f395f2f052a494fd4') person.save() # Ensure that the object is in the database with the correct _id collection = self.db[self.Person._meta['collection']] person_obj = collection.find_one({'name': 'Test User'}) self.assertEqual(str(person_obj['_id']), '497ce96f395f2f052a494fd4') - + def test_save_custom_pk(self): """Ensure that a document may be saved with a custom _id using pk alias. """ # Create person object and save it to the database - person = self.Person(name='Test User', age=30, + person = self.Person(name='Test User', age=30, pk='497ce96f395f2f052a494fd4') person.save() # Ensure that the object is in the database with the correct _id @@ -565,7 +565,7 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() def test_save_embedded_document(self): - """Ensure that a document with an embedded document field may be + """Ensure that a document with an embedded document field may be saved in the database. """ class EmployeeDetails(EmbeddedDocument): @@ -591,7 +591,7 @@ class DocumentTest(unittest.TestCase): def test_save_reference(self): """Ensure that a document reference field may be saved in the database. """ - + class BlogPost(Document): meta = {'collection': 'blogpost_1'} content = StringField() @@ -610,7 +610,7 @@ class DocumentTest(unittest.TestCase): post_obj = BlogPost.objects.first() # Test laziness - self.assertTrue(isinstance(post_obj._data['author'], + self.assertTrue(isinstance(post_obj._data['author'], pymongo.dbref.DBRef)) self.assertTrue(isinstance(post_obj.author, self.Person)) self.assertEqual(post_obj.author.name, 'Test User') @@ -737,7 +737,7 @@ class DocumentTest(unittest.TestCase): class BlogPost(Document): pass - + # Clear old datas User.drop_collection() BlogPost.drop_collection() @@ -774,9 +774,9 @@ class DocumentTest(unittest.TestCase): # in Set all_user_set = set(User.objects.all()) - + self.assertTrue(u1 in all_user_set ) - + if __name__ == '__main__': unittest.main() From 7526272f84f2867f1619e1e947292fea3ec57b1c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 18 May 2011 12:27:33 +0100 Subject: [PATCH 0199/1279] Added test example of updating an embedded field Closes #139 --- tests/document.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/document.py b/tests/document.py index 45f1c3c..6f9d9ec 100644 --- a/tests/document.py +++ b/tests/document.py @@ -588,6 +588,34 @@ class DocumentTest(unittest.TestCase): # Ensure that the 'details' embedded object saved correctly self.assertEqual(employee_obj['details']['position'], 'Developer') + def test_updating_an_embedded_document(self): + """Ensure that a document with an embedded document field may be + saved in the database. + """ + class EmployeeDetails(EmbeddedDocument): + position = StringField() + + class Employee(self.Person): + salary = IntField() + details = EmbeddedDocumentField(EmployeeDetails) + + # Create employee object and save it to the database + employee = Employee(name='Test Employee', age=50, salary=20000) + employee.details = EmployeeDetails(position='Developer') + employee.save() + + # Test updating an embedded document + promoted_employee = Employee.objects.get(name='Test Employee') + promoted_employee.details.position = 'Senior Developer' + promoted_employee.save() + + collection = self.db[self.Person._meta['collection']] + employee_obj = collection.find_one({'name': 'Test Employee'}) + self.assertEqual(employee_obj['name'], 'Test Employee') + self.assertEqual(employee_obj['age'], 50) + # Ensure that the 'details' embedded object saved correctly + self.assertEqual(employee_obj['details']['position'], 'Senior Developer') + def test_save_reference(self): """Ensure that a document reference field may be saved in the database. """ From 5d5a84dbcf1fcd2bcd603dd417c3c2c75e1d6252 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 18 May 2011 16:24:35 +0100 Subject: [PATCH 0200/1279] Spacing issue cleaned up --- mongoengine/queryset.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 3b37c3d..58ea61c 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -346,18 +346,18 @@ class QuerySet(object): self._skip = None def clone(self): - """Creates a copy of the current :class:`~mongoengine.queryset.QuerySet`""" - c = self.__class__(self._document, self._collection_obj) + """Creates a copy of the current :class:`~mongoengine.queryset.QuerySet`""" + c = self.__class__(self._document, self._collection_obj) - copy_props = ('_initial_query', '_query_obj', '_where_clause', + copy_props = ('_initial_query', '_query_obj', '_where_clause', '_loaded_fields', '_ordering', '_snapshot', '_timeout', '_limit', '_skip') - for prop in copy_props: - val = getattr(self, prop) - setattr(c, prop, copy.deepcopy(val)) + for prop in copy_props: + val = getattr(self, prop) + setattr(c, prop, copy.deepcopy(val)) - return c + return c @property def _query(self): From 371dbf009fe6f0637a758d8cdb07d3be4e311f51 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 18 May 2011 16:39:19 +0100 Subject: [PATCH 0201/1279] Updated QuerySet to allow more granular fields control. Added a fields method and tests showing the retrival of subranges of List Fields. Refs #167 --- mongoengine/queryset.py | 58 +++++++++++++++++++++++++------------ tests/queryset.py | 64 ++++++++++++++++++++++++++++++++--------- 2 files changed, 90 insertions(+), 32 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 58ea61c..54d7643 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -8,6 +8,7 @@ import pymongo.objectid import re import copy import itertools +import operator __all__ = ['queryset_manager', 'Q', 'InvalidQueryError', 'InvalidCollectionError', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY'] @@ -280,30 +281,30 @@ class QueryFieldList(object): ONLY = True EXCLUDE = False - def __init__(self, fields=[], direction=ONLY, always_include=[]): - self.direction = direction + def __init__(self, fields=[], value=ONLY, always_include=[]): + self.value = value self.fields = set(fields) self.always_include = set(always_include) def as_dict(self): - return dict((field, self.direction) for field in self.fields) + return dict((field, self.value) for field in self.fields) def __add__(self, f): if not self.fields: self.fields = f.fields - self.direction = f.direction - elif self.direction is self.ONLY and f.direction is self.ONLY: + self.value = f.value + elif self.value is self.ONLY and f.value is self.ONLY: self.fields = self.fields.intersection(f.fields) - elif self.direction is self.EXCLUDE and f.direction is self.EXCLUDE: + elif self.value is self.EXCLUDE and f.value is self.EXCLUDE: self.fields = self.fields.union(f.fields) - elif self.direction is self.ONLY and f.direction is self.EXCLUDE: + elif self.value is self.ONLY and f.value is self.EXCLUDE: self.fields -= f.fields - elif self.direction is self.EXCLUDE and f.direction is self.ONLY: - self.direction = self.ONLY + elif self.value is self.EXCLUDE and f.value is self.ONLY: + self.value = self.ONLY self.fields = f.fields - self.fields if self.always_include: - if self.direction is self.ONLY and self.fields: + if self.value is self.ONLY and self.fields: self.fields = self.fields.union(self.always_include) else: self.fields -= self.always_include @@ -311,7 +312,7 @@ class QueryFieldList(object): def reset(self): self.fields = set([]) - self.direction = self.ONLY + self.value = self.ONLY def __nonzero__(self): return bool(self.fields) @@ -890,10 +891,8 @@ class QuerySet(object): .. versionadded:: 0.3 """ - fields = self._fields_to_dbfields(fields) - self._loaded_fields += QueryFieldList(fields, direction=QueryFieldList.ONLY) - return self - + fields = dict([(f, QueryFieldList.ONLY) for f in fields]) + return self.fields(**fields) def exclude(self, *fields): """Opposite to .only(), exclude some document's fields. :: @@ -902,8 +901,31 @@ class QuerySet(object): :param fields: fields to exclude """ - fields = self._fields_to_dbfields(fields) - self._loaded_fields += QueryFieldList(fields, direction=QueryFieldList.EXCLUDE) + fields = dict([(f, QueryFieldList.EXCLUDE) for f in fields]) + return self.fields(**fields) + + def fields(self, **kwargs): + """Manipulate how you load this document's fields. Used by `.only()` + and `.exclude()` to manipulate which fields to retrieve. Fields also + allows for a greater level of control for example: + + Retrieving a Subrange of Array Elements + --------------------------------------- + + You can use the $slice operator to retrieve a subrange of elements in + an array :: + + post = BlogPost.objects(...).fields(comments={"$slice": 5}) // first 5 comments + + :param kwargs: A dictionary identifying what to include + + .. versionadded:: 0.5 + """ + fields = sorted(kwargs.iteritems(), key=operator.itemgetter(1)) + for value, group in itertools.groupby(fields, lambda x: x[1]): + fields = [field for field, value in group] + fields = self._fields_to_dbfields(fields) + self._loaded_fields += QueryFieldList(fields, value=value) return self def all_fields(self): @@ -1277,7 +1299,7 @@ class QuerySetManager(object): # Create collection as a capped collection if specified if owner._meta['max_size'] or owner._meta['max_documents']: # Get max document limit and max byte size from meta - max_size = owner._meta['max_size'] or 10000000 # 10MB default + max_size = owner._meta['max_size'] or 10000000 # 10MB default max_documents = owner._meta['max_documents'] if collection in db.collection_names(): diff --git a/tests/queryset.py b/tests/queryset.py index 48ce627..1961d7c 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -597,6 +597,38 @@ class QuerySetTest(unittest.TestCase): Email.drop_collection() + def test_custom_fields(self): + """Ensure that query slicing an array works. + """ + + class Numbers(Document): + n = ListField(IntField()) + + Numbers.drop_collection() + + numbers = Numbers(n=[0,1,2,3,4,5,-5,-4,-3,-2,-1]) + numbers.save() + + # first three + numbers = Numbers.objects.fields(n={"$slice": 3}).get() + self.assertEquals(numbers.n, [0, 1, 2]) + + # last three + numbers = Numbers.objects.fields(n={"$slice": -3}).get() + self.assertEquals(numbers.n, [-3, -2, -1]) + + # skip 2, limit 3 + numbers = Numbers.objects.fields(n={"$slice": [2, 3]}).get() + self.assertEquals(numbers.n, [2, 3, 4]) + + # skip to fifth from last, limit 4 + numbers = Numbers.objects.fields(n={"$slice": [-5, 4]}).get() + self.assertEquals(numbers.n, [-5, -4, -3, -2]) + + # skip to fifth from last, limit 10 + numbers = Numbers.objects.fields(n={"$slice": [-5, 10]}).get() + self.assertEquals(numbers.n, [-5, -4, -3, -2, -1]) + def test_find_embedded(self): """Ensure that an embedded document is properly returned from a query. """ @@ -1931,49 +1963,53 @@ class QueryFieldListTest(unittest.TestCase): def test_include_include(self): q = QueryFieldList() - q += QueryFieldList(fields=['a', 'b'], direction=QueryFieldList.ONLY) + q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.ONLY) self.assertEqual(q.as_dict(), {'a': True, 'b': True}) - q += QueryFieldList(fields=['b', 'c'], direction=QueryFieldList.ONLY) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) self.assertEqual(q.as_dict(), {'b': True}) def test_include_exclude(self): q = QueryFieldList() - q += QueryFieldList(fields=['a', 'b'], direction=QueryFieldList.ONLY) + q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.ONLY) self.assertEqual(q.as_dict(), {'a': True, 'b': True}) - q += QueryFieldList(fields=['b', 'c'], direction=QueryFieldList.EXCLUDE) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.EXCLUDE) self.assertEqual(q.as_dict(), {'a': True}) def test_exclude_exclude(self): q = QueryFieldList() - q += QueryFieldList(fields=['a', 'b'], direction=QueryFieldList.EXCLUDE) + q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.EXCLUDE) self.assertEqual(q.as_dict(), {'a': False, 'b': False}) - q += QueryFieldList(fields=['b', 'c'], direction=QueryFieldList.EXCLUDE) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.EXCLUDE) self.assertEqual(q.as_dict(), {'a': False, 'b': False, 'c': False}) def test_exclude_include(self): q = QueryFieldList() - q += QueryFieldList(fields=['a', 'b'], direction=QueryFieldList.EXCLUDE) + q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.EXCLUDE) self.assertEqual(q.as_dict(), {'a': False, 'b': False}) - q += QueryFieldList(fields=['b', 'c'], direction=QueryFieldList.ONLY) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) self.assertEqual(q.as_dict(), {'c': True}) def test_always_include(self): q = QueryFieldList(always_include=['x', 'y']) - q += QueryFieldList(fields=['a', 'b', 'x'], direction=QueryFieldList.EXCLUDE) - q += QueryFieldList(fields=['b', 'c'], direction=QueryFieldList.ONLY) + q += QueryFieldList(fields=['a', 'b', 'x'], value=QueryFieldList.EXCLUDE) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'c': True}) - def test_reset(self): q = QueryFieldList(always_include=['x', 'y']) - q += QueryFieldList(fields=['a', 'b', 'x'], direction=QueryFieldList.EXCLUDE) - q += QueryFieldList(fields=['b', 'c'], direction=QueryFieldList.ONLY) + q += QueryFieldList(fields=['a', 'b', 'x'], value=QueryFieldList.EXCLUDE) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'c': True}) q.reset() self.assertFalse(q) - q += QueryFieldList(fields=['b', 'c'], direction=QueryFieldList.ONLY) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'b': True, 'c': True}) + def test_using_a_slice(self): + q = QueryFieldList() + q += QueryFieldList(fields=['a'], value={"$slice": 5}) + self.assertEqual(q.as_dict(), {'a': {"$slice": 5}}) + if __name__ == '__main__': unittest.main() From fc2aff342bed0bdc764af7d4dc96850161477149 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 18 May 2011 17:37:41 +0100 Subject: [PATCH 0202/1279] Unique indexes are created before user declared indexes This ensures that indexes are created with the unique flag, if a user declares the index, that would automatically be declared by the `unique_indexes` logic. Thanks to btubbs for the test case. Fixes #129 --- mongoengine/queryset.py | 10 +++++----- tests/document.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 58ea61c..6da11fa 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -459,17 +459,17 @@ class QuerySet(object): drop_dups = self._document._meta.get('index_drop_dups', False) index_opts = self._document._meta.get('index_options', {}) + # Ensure indexes created by uniqueness constraints + for index in self._document._meta['unique_indexes']: + self._collection.ensure_index(index, unique=True, + background=background, drop_dups=drop_dups, **index_opts) + # Ensure document-defined indexes are created if self._document._meta['indexes']: for key_or_list in self._document._meta['indexes']: self._collection.ensure_index(key_or_list, background=background, **index_opts) - # Ensure indexes created by uniqueness constraints - for index in self._document._meta['unique_indexes']: - self._collection.ensure_index(index, unique=True, - background=background, drop_dups=drop_dups, **index_opts) - # If _types is being used (for polymorphism), it needs an index if '_types' in self._query: self._collection.ensure_index('_types', diff --git a/tests/document.py b/tests/document.py index 6f9d9ec..66efdf9 100644 --- a/tests/document.py +++ b/tests/document.py @@ -357,6 +357,29 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + def test_unique_and_indexes(self): + """Ensure that 'unique' constraints aren't overridden by + meta.indexes. + """ + class Customer(Document): + cust_id = IntField(unique=True, required=True) + meta = { + 'indexes': ['cust_id'], + 'allow_inheritance': False, + } + + Customer.drop_collection() + cust = Customer(cust_id=1) + cust.save() + + cust_dupe = Customer(cust_id=1) + try: + cust_dupe.save() + raise AssertionError, "We saved a dupe!" + except OperationError: + pass + Customer.drop_collection() + def test_custom_id_field(self): """Ensure that documents may be created with custom primary keys. """ From 95c2643f63558ccc2a707fa425f670b597f4e2a2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 18 May 2011 20:31:28 +0100 Subject: [PATCH 0203/1279] Added test showing primary=True behaviour. If you set a field as primary, then unexpected behaviour can occur. You won't create a duplicate but you will update an existing document. Closes #138 --- tests/document.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/document.py b/tests/document.py index 66efdf9..dee0b71 100644 --- a/tests/document.py +++ b/tests/document.py @@ -380,6 +380,28 @@ class DocumentTest(unittest.TestCase): pass Customer.drop_collection() + def test_unique_and_primary(self): + """If you set a field as primary, then unexpected behaviour can occur. + You won't create a duplicate but you will update an existing document. + """ + + class User(Document): + name = StringField(primary_key=True, unique=True) + password = StringField() + + User.drop_collection() + + user = User(name='huangz', password='secret') + user.save() + + user = User(name='huangz', password='secret2') + user.save() + + self.assertEqual(User.objects.count(), 1) + self.assertEqual(User.objects.get().password, 'secret2') + + User.drop_collection() + def test_custom_id_field(self): """Ensure that documents may be created with custom primary keys. """ From fb61c9a765d0605a27f720d7a24d43aa580e109b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 19 May 2011 09:55:34 +0100 Subject: [PATCH 0204/1279] Regression test for mysterious uniqueness constraint when inserting into mongoengine Closes #143 Thanks to tfausak for the test case. --- tests/document.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/document.py b/tests/document.py index dee0b71..f6d8ea4 100644 --- a/tests/document.py +++ b/tests/document.py @@ -798,6 +798,25 @@ class DocumentTest(unittest.TestCase): self.Person.drop_collection() BlogPost.drop_collection() + def subclasses_and_unique_keys_works(self): + + class A(Document): + pass + + class B(A): + foo = BooleanField(unique=True) + + A.drop_collection() + B.drop_collection() + + A().save() + A().save() + B(foo=True).save() + + self.assertEquals(A.objects.count(), 2) + self.assertEquals(B.objects.count(), 1) + A.drop_collection() + B.drop_collection() def tearDown(self): self.Person.drop_collection() From da8a057edecb5e3246e7ad14bdd5eb7f08363ed1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 19 May 2011 12:41:38 +0100 Subject: [PATCH 0205/1279] Added test showing documents can be pickled Refs #135 --- tests/document.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/document.py b/tests/document.py index f6d8ea4..84d0068 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1,11 +1,23 @@ import unittest from datetime import datetime import pymongo +import pickle from mongoengine import * +from mongoengine.base import BaseField from mongoengine.connection import _get_db +class PickleEmbedded(EmbeddedDocument): + date = DateTimeField(default=datetime.now) + +class PickleTest(Document): + number = IntField() + string = StringField() + embedded = EmbeddedDocumentField(PickleEmbedded) + lists = ListField(StringField()) + + class DocumentTest(unittest.TestCase): def setUp(self): @@ -869,6 +881,23 @@ class DocumentTest(unittest.TestCase): self.assertTrue(u1 in all_user_set ) + def test_picklable(self): + + pickle_doc = PickleTest(number=1, string="OH HAI", lists=['1', '2']) + pickle_doc.embedded = PickleEmbedded() + pickle_doc.save() + + pickled_doc = pickle.dumps(pickle_doc) + resurrected = pickle.loads(pickled_doc) + + self.assertEquals(resurrected, pickle_doc) + + resurrected.string = "Working" + resurrected.save() + + pickle_doc.reload() + self.assertEquals(resurrected, pickle_doc) + if __name__ == '__main__': unittest.main() From b3251818cc3b55e7dab1dc7e45ab2f8be82f7269 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 19 May 2011 13:04:14 +0100 Subject: [PATCH 0206/1279] Added regression test for custom queryset ordering Closes #126 --- tests/queryset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 48ce627..51224ea 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1294,6 +1294,7 @@ class QuerySetTest(unittest.TestCase): class BlogPost(Document): tags = ListField(StringField()) deleted = BooleanField(default=False) + date = DateTimeField(default=datetime.now) @queryset_manager def objects(doc_cls, queryset): @@ -1301,7 +1302,7 @@ class QuerySetTest(unittest.TestCase): @queryset_manager def music_posts(doc_cls, queryset): - return queryset(tags='music', deleted=False) + return queryset(tags='music', deleted=False).order_by('-date') BlogPost.drop_collection() @@ -1317,7 +1318,7 @@ class QuerySetTest(unittest.TestCase): self.assertEqual([p.id for p in BlogPost.objects], [post1.id, post2.id, post3.id]) self.assertEqual([p.id for p in BlogPost.music_posts], - [post1.id, post2.id]) + [post2.id, post1.id]) BlogPost.drop_collection() From 40b69baa2991f750bdf879058ddb2d1e6a9a0c05 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 19 May 2011 16:49:00 +0100 Subject: [PATCH 0207/1279] Implementing Write Concern Added write_options dict to save, update, update_one and get_or_create. Thanks to justquick for the initial ticket and code. Refs #132 --- mongoengine/document.py | 38 ++++++++++++++++++++++++-------------- mongoengine/queryset.py | 32 +++++++++++++++++++++++--------- tests/document.py | 20 ++++++++++++++++++++ 3 files changed, 67 insertions(+), 23 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 196662c..771b922 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -40,44 +40,54 @@ class Document(BaseDocument): presence of `_cls` and `_types`, set :attr:`allow_inheritance` to ``False`` in the :attr:`meta` dictionary. - A :class:`~mongoengine.Document` may use a **Capped Collection** by + A :class:`~mongoengine.Document` may use a **Capped Collection** by specifying :attr:`max_documents` and :attr:`max_size` in the :attr:`meta` dictionary. :attr:`max_documents` is the maximum number of documents that - is allowed to be stored in the collection, and :attr:`max_size` is the - maximum size of the collection in bytes. If :attr:`max_size` is not - specified and :attr:`max_documents` is, :attr:`max_size` defaults to + is allowed to be stored in the collection, and :attr:`max_size` is the + maximum size of the collection in bytes. If :attr:`max_size` is not + specified and :attr:`max_documents` is, :attr:`max_size` defaults to 10000000 bytes (10MB). Indexes may be created by specifying :attr:`indexes` in the :attr:`meta` - dictionary. The value should be a list of field names or tuples of field + dictionary. The value should be a list of field names or tuples of field names. Index direction may be specified by prefixing the field names with a **+** or **-** sign. """ __metaclass__ = TopLevelDocumentMetaclass - def save(self, safe=True, force_insert=False, validate=True): + def save(self, safe=True, force_insert=False, validate=True, write_options=None): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be created. - If ``safe=True`` and the operation is unsuccessful, an + If ``safe=True`` and the operation is unsuccessful, an :class:`~mongoengine.OperationError` will be raised. :param safe: check if the operation succeeded before returning - :param force_insert: only try to create a new document, don't allow + :param force_insert: only try to create a new document, don't allow updates of existing documents :param validate: validates the document; set to ``False`` to skip. + :param write_options: Extra keyword arguments are passed down to + :meth:`~pymongo.collection.Collection.save` OR + :meth:`~pymongo.collection.Collection.insert` + which will be used as options for the resultant ``getLastError`` command. + For example, ``save(..., w=2, fsync=True)`` will wait until at least two servers + have recorded the write and will force an fsync on each server being written to. """ if validate: self.validate() + + if not write_options: + write_options = {} + doc = self.to_mongo() try: collection = self.__class__.objects._collection if force_insert: - object_id = collection.insert(doc, safe=safe) + object_id = collection.insert(doc, safe=safe, **write_options) else: - object_id = collection.save(doc, safe=safe) + object_id = collection.save(doc, safe=safe, **write_options) except pymongo.errors.OperationFailure, err: message = 'Could not save document (%s)' if u'duplicate key' in unicode(err): @@ -131,9 +141,9 @@ class MapReduceDocument(object): """A document returned from a map/reduce query. :param collection: An instance of :class:`~pymongo.Collection` - :param key: Document/result key, often an instance of - :class:`~pymongo.objectid.ObjectId`. If supplied as - an ``ObjectId`` found in the given ``collection``, + :param key: Document/result key, often an instance of + :class:`~pymongo.objectid.ObjectId`. If supplied as + an ``ObjectId`` found in the given ``collection``, the object can be accessed via the ``object`` property. :param value: The result(s) for this key. @@ -148,7 +158,7 @@ class MapReduceDocument(object): @property def object(self): - """Lazy-load the object referenced by ``self.key``. ``self.key`` + """Lazy-load the object referenced by ``self.key``. ``self.key`` should be the ``primary_key``. """ id_field = self._document()._meta['id_field'] diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6da11fa..683aac5 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -643,7 +643,7 @@ class QuerySet(object): raise self._document.DoesNotExist("%s matching query does not exist." % self._document._class_name) - def get_or_create(self, *q_objs, **query): + def get_or_create(self, write_options=None, *q_objs, **query): """Retrieve unique object or create, if it doesn't exist. Returns a tuple of ``(object, created)``, where ``object`` is the retrieved or created object and ``created`` is a boolean specifying whether a new object was created. Raises @@ -653,6 +653,10 @@ class QuerySet(object): dictionary of default values for the new document may be provided as a keyword argument called :attr:`defaults`. + :param write_options: optional extra keyword arguments used if we + have to create a new document. + Passes any write_options onto :meth:`~mongoengine.document.Document.save` + .. versionadded:: 0.3 """ defaults = query.get('defaults', {}) @@ -664,7 +668,7 @@ class QuerySet(object): if count == 0: query.update(defaults) doc = self._document(**query) - doc.save() + doc.save(write_options=write_options) return doc, True elif count == 1: return self.first(), False @@ -1055,22 +1059,27 @@ class QuerySet(object): return mongo_update - def update(self, safe_update=True, upsert=False, **update): + def update(self, safe_update=True, upsert=False, write_options=None, **update): """Perform an atomic update on the fields matched by the query. When ``safe_update`` is used, the number of affected documents is returned. - :param safe: check if the operation succeeded before returning - :param update: Django-style update keyword arguments + :param safe_update: check if the operation succeeded before returning + :param upsert: Any existing document with that "_id" is overwritten. + :param write_options: extra keyword arguments for :meth:`~pymongo.collection.Collection.update` .. versionadded:: 0.2 """ if pymongo.version < '1.1.1': raise OperationError('update() method requires PyMongo 1.1.1+') + if not write_options: + write_options = {} + update = QuerySet._transform_update(self._document, **update) try: ret = self._collection.update(self._query, update, multi=True, - upsert=upsert, safe=safe_update) + upsert=upsert, safe=safe_update, + **write_options) if ret is not None and 'n' in ret: return ret['n'] except pymongo.errors.OperationFailure, err: @@ -1079,22 +1088,27 @@ class QuerySet(object): raise OperationError(message) raise OperationError(u'Update failed (%s)' % unicode(err)) - def update_one(self, safe_update=True, upsert=False, **update): + def update_one(self, safe_update=True, upsert=False, write_options=None, **update): """Perform an atomic update on first field matched by the query. When ``safe_update`` is used, the number of affected documents is returned. - :param safe: check if the operation succeeded before returning + :param safe_update: check if the operation succeeded before returning + :param upsert: Any existing document with that "_id" is overwritten. + :param write_options: extra keyword arguments for :meth:`~pymongo.collection.Collection.update` :param update: Django-style update keyword arguments .. versionadded:: 0.2 """ + if not write_options: + write_options = {} update = QuerySet._transform_update(self._document, **update) try: # Explicitly provide 'multi=False' to newer versions of PyMongo # as the default may change to 'True' if pymongo.version >= '1.1.1': ret = self._collection.update(self._query, update, multi=False, - upsert=upsert, safe=safe_update) + upsert=upsert, safe=safe_update, + **write_options) else: # Older versions of PyMongo don't support 'multi' ret = self._collection.update(self._query, update, diff --git a/tests/document.py b/tests/document.py index 84d0068..cef6e8c 100644 --- a/tests/document.py +++ b/tests/document.py @@ -898,6 +898,26 @@ class DocumentTest(unittest.TestCase): pickle_doc.reload() self.assertEquals(resurrected, pickle_doc) + def test_write_options(self): + """Test that passing write_options works""" + + self.Person.drop_collection() + + write_options = {"fsync": True} + + author, created = self.Person.objects.get_or_create( + name='Test User', write_options=write_options) + author.save(write_options=write_options) + + self.Person.objects.update(set__name='Ross', write_options=write_options) + + author = self.Person.objects.first() + self.assertEquals(author.name, 'Ross') + + self.Person.objects.update_one(set__name='Test User', write_options=write_options) + author = self.Person.objects.first() + self.assertEquals(author.name, 'Test User') + if __name__ == '__main__': unittest.main() From 08d1689268c846118a4f6d07772ce6f6b29649a6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 20 May 2011 09:47:41 +0100 Subject: [PATCH 0208/1279] Updated to handle the converntional api style for slicing a field Added testcase to demonstrate embedded slicing as well. Refs #167 --- mongoengine/queryset.py | 19 +++++++++++--- tests/queryset.py | 55 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 54d7643..8469e71 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -548,7 +548,7 @@ class QuerySet(object): return '.'.join(parts) @classmethod - def _transform_query(cls, _doc_cls=None, **query): + def _transform_query(cls, _doc_cls=None, _field_operation=False, **query): """Transform a query from Django-style format to Mongo format. """ operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', @@ -915,13 +915,26 @@ class QuerySet(object): You can use the $slice operator to retrieve a subrange of elements in an array :: - post = BlogPost.objects(...).fields(comments={"$slice": 5}) // first 5 comments + post = BlogPost.objects(...).fields(slice__comments=5) // first 5 comments :param kwargs: A dictionary identifying what to include .. versionadded:: 0.5 """ - fields = sorted(kwargs.iteritems(), key=operator.itemgetter(1)) + + # Check for an operator and transform to mongo-style if there is + operators = ["slice"] + cleaned_fields = [] + for key, value in kwargs.items(): + parts = key.split('__') + op = None + if parts[0] in operators: + op = parts.pop(0) + value = {'$' + op: value} + key = '.'.join(parts) + cleaned_fields.append((key, value)) + + fields = sorted(cleaned_fields, key=operator.itemgetter(1)) for value, group in itertools.groupby(fields, lambda x: x[1]): fields = [field for field, value in group] fields = self._fields_to_dbfields(fields) diff --git a/tests/queryset.py b/tests/queryset.py index 1961d7c..e29a6d9 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -597,10 +597,9 @@ class QuerySetTest(unittest.TestCase): Email.drop_collection() - def test_custom_fields(self): + def test_slicing_fields(self): """Ensure that query slicing an array works. """ - class Numbers(Document): n = ListField(IntField()) @@ -610,25 +609,69 @@ class QuerySetTest(unittest.TestCase): numbers.save() # first three - numbers = Numbers.objects.fields(n={"$slice": 3}).get() + numbers = Numbers.objects.fields(slice__n=3).get() self.assertEquals(numbers.n, [0, 1, 2]) # last three - numbers = Numbers.objects.fields(n={"$slice": -3}).get() + numbers = Numbers.objects.fields(slice__n=-3).get() self.assertEquals(numbers.n, [-3, -2, -1]) # skip 2, limit 3 - numbers = Numbers.objects.fields(n={"$slice": [2, 3]}).get() + numbers = Numbers.objects.fields(slice__n=[2, 3]).get() self.assertEquals(numbers.n, [2, 3, 4]) # skip to fifth from last, limit 4 - numbers = Numbers.objects.fields(n={"$slice": [-5, 4]}).get() + numbers = Numbers.objects.fields(slice__n=[-5, 4]).get() self.assertEquals(numbers.n, [-5, -4, -3, -2]) # skip to fifth from last, limit 10 + numbers = Numbers.objects.fields(slice__n=[-5, 10]).get() + self.assertEquals(numbers.n, [-5, -4, -3, -2, -1]) + + # skip to fifth from last, limit 10 dict method numbers = Numbers.objects.fields(n={"$slice": [-5, 10]}).get() self.assertEquals(numbers.n, [-5, -4, -3, -2, -1]) + def test_slicing_nested_fields(self): + """Ensure that query slicing an embedded array works. + """ + + class EmbeddedNumber(EmbeddedDocument): + n = ListField(IntField()) + + class Numbers(Document): + embedded = EmbeddedDocumentField(EmbeddedNumber) + + Numbers.drop_collection() + + numbers = Numbers() + numbers.embedded = EmbeddedNumber(n=[0,1,2,3,4,5,-5,-4,-3,-2,-1]) + numbers.save() + + # first three + numbers = Numbers.objects.fields(slice__embedded__n=3).get() + self.assertEquals(numbers.embedded.n, [0, 1, 2]) + + # last three + numbers = Numbers.objects.fields(slice__embedded__n=-3).get() + self.assertEquals(numbers.embedded.n, [-3, -2, -1]) + + # skip 2, limit 3 + numbers = Numbers.objects.fields(slice__embedded__n=[2, 3]).get() + self.assertEquals(numbers.embedded.n, [2, 3, 4]) + + # skip to fifth from last, limit 4 + numbers = Numbers.objects.fields(slice__embedded__n=[-5, 4]).get() + self.assertEquals(numbers.embedded.n, [-5, -4, -3, -2]) + + # skip to fifth from last, limit 10 + numbers = Numbers.objects.fields(slice__embedded__n=[-5, 10]).get() + self.assertEquals(numbers.embedded.n, [-5, -4, -3, -2, -1]) + + # skip to fifth from last, limit 10 dict method + numbers = Numbers.objects.fields(embedded__n={"$slice": [-5, 10]}).get() + self.assertEquals(numbers.embedded.n, [-5, -4, -3, -2, -1]) + def test_find_embedded(self): """Ensure that an embedded document is properly returned from a query. """ From 9260ff9e83365a13bd75334c60d3eb33d2fdf5ef Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 20 May 2011 10:22:22 +0100 Subject: [PATCH 0209/1279] Updated docs and added a NotRegistered exception For handling GenericReferences that reference documents that haven't been imported. Closes #170 --- mongoengine/base.py | 38 ++++++++++++++--------- mongoengine/fields.py | 7 +++-- tests/fields.py | 70 ++++++++++++++++++++++++++++++++----------- 3 files changed, 81 insertions(+), 34 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 9d0b823..ede9083 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -7,22 +7,32 @@ import pymongo import pymongo.objectid -_document_registry = {} - -def get_document(name): - return _document_registry[name] +class NotRegistered(Exception): + pass class ValidationError(Exception): pass +_document_registry = {} + +def get_document(name): + if name not in _document_registry: + raise NotRegistered(""" + `%s` has not been registered in the document registry. + Importing the document class automatically registers it, has it + been imported? + """.strip() % name) + return _document_registry[name] + + class BaseField(object): """A base class for fields in a MongoDB document. Instances of this class may be added to subclasses of `Document` to define a document's schema. """ - # Fields may have _types inserted into indexes by default + # Fields may have _types inserted into indexes by default _index_with_types = True _geo_index = False @@ -32,7 +42,7 @@ class BaseField(object): creation_counter = 0 auto_creation_counter = -1 - def __init__(self, db_field=None, name=None, required=False, default=None, + def __init__(self, db_field=None, name=None, required=False, default=None, unique=False, unique_with=None, primary_key=False, validation=None, choices=None): self.db_field = (db_field or name) if not primary_key else '_id' @@ -57,7 +67,7 @@ class BaseField(object): BaseField.creation_counter += 1 def __get__(self, instance, owner): - """Descriptor for retrieving a value from a field in a document. Do + """Descriptor for retrieving a value from a field in a document. Do any necessary conversion between Python and MongoDB types. """ if instance is None: @@ -167,8 +177,8 @@ class DocumentMetaclass(type): superclasses.update(base._superclasses) if hasattr(base, '_meta'): - # Ensure that the Document class may be subclassed - - # inheritance may be disabled to remove dependency on + # Ensure that the Document class may be subclassed - + # inheritance may be disabled to remove dependency on # additional fields _cls and _types if base._meta.get('allow_inheritance', True) == False: raise ValueError('Document %s may not be subclassed' % @@ -211,12 +221,12 @@ class DocumentMetaclass(type): module = attrs.get('__module__') - base_excs = tuple(base.DoesNotExist for base in bases + base_excs = tuple(base.DoesNotExist for base in bases if hasattr(base, 'DoesNotExist')) or (DoesNotExist,) exc = subclass_exception('DoesNotExist', base_excs, module) new_class.add_to_class('DoesNotExist', exc) - base_excs = tuple(base.MultipleObjectsReturned for base in bases + base_excs = tuple(base.MultipleObjectsReturned for base in bases if hasattr(base, 'MultipleObjectsReturned')) base_excs = base_excs or (MultipleObjectsReturned,) exc = subclass_exception('MultipleObjectsReturned', base_excs, module) @@ -238,9 +248,9 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): def __new__(cls, name, bases, attrs): super_new = super(TopLevelDocumentMetaclass, cls).__new__ - # Classes defined in this package are abstract and should not have + # Classes defined in this package are abstract and should not have # their own metadata with DB collection, etc. - # __metaclass__ is only set on the class with the __metaclass__ + # __metaclass__ is only set on the class with the __metaclass__ # attribute (i.e. it is not set on subclasses). This differentiates # 'real' documents from the 'Document' class if attrs.get('__metaclass__') == TopLevelDocumentMetaclass: @@ -366,7 +376,7 @@ class BaseDocument(object): are present. """ # Get a list of tuples of field names and their current values - fields = [(field, getattr(self, name)) + fields = [(field, getattr(self, name)) for name, field in self._fields.items()] # Ensure that each field is matched to a valid value diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 527eb15..0cc8219 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -339,7 +339,7 @@ class ListField(BaseField): if isinstance(self.field, ReferenceField): referenced_type = self.field.document_type - # Get value from document instance if available + # Get value from document instance if available value_list = instance._data.get(self.name) if value_list: deref_list = [] @@ -522,6 +522,9 @@ class GenericReferenceField(BaseField): """A reference to *any* :class:`~mongoengine.document.Document` subclass that will be automatically dereferenced on access (lazily). + note: Any documents used as a generic reference must be registered in the + document registry. Importing the model will automatically register it. + .. versionadded:: 0.3 """ @@ -648,7 +651,7 @@ class GridFSProxy(object): if not self.newfile: self.new_file() self.grid_id = self.newfile._id - self.newfile.writelines(lines) + self.newfile.writelines(lines) def read(self, size=-1): try: diff --git a/tests/fields.py b/tests/fields.py index c867187..38409b6 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -7,6 +7,7 @@ import gridfs from mongoengine import * from mongoengine.connection import _get_db +from mongoengine.base import _document_registry, NotRegistered class FieldTest(unittest.TestCase): @@ -45,7 +46,7 @@ class FieldTest(unittest.TestCase): """ class Person(Document): name = StringField() - + person = Person(name='Test User') self.assertEqual(person.id, None) @@ -95,7 +96,7 @@ class FieldTest(unittest.TestCase): link.url = 'http://www.google.com:8080' link.validate() - + def test_int_validation(self): """Ensure that invalid values cannot be assigned to int fields. """ @@ -129,12 +130,12 @@ class FieldTest(unittest.TestCase): self.assertRaises(ValidationError, person.validate) person.height = 4.0 self.assertRaises(ValidationError, person.validate) - + def test_decimal_validation(self): """Ensure that invalid values cannot be assigned to decimal fields. """ class Person(Document): - height = DecimalField(min_value=Decimal('0.1'), + height = DecimalField(min_value=Decimal('0.1'), max_value=Decimal('3.5')) Person.drop_collection() @@ -249,7 +250,7 @@ class FieldTest(unittest.TestCase): post.save() post.reload() self.assertEqual(post.tags, ['fun', 'leisure']) - + comment1 = Comment(content='Good for you', order=1) comment2 = Comment(content='Yay.', order=0) comments = [comment1, comment2] @@ -315,7 +316,7 @@ class FieldTest(unittest.TestCase): person.validate() def test_embedded_document_inheritance(self): - """Ensure that subclasses of embedded documents may be provided to + """Ensure that subclasses of embedded documents may be provided to EmbeddedDocumentFields of the superclass' type. """ class User(EmbeddedDocument): @@ -327,7 +328,7 @@ class FieldTest(unittest.TestCase): class BlogPost(Document): content = StringField() author = EmbeddedDocumentField(User) - + post = BlogPost(content='What I did today...') post.author = User(name='Test User') post.author = PowerUser(name='Test User', power=47) @@ -370,7 +371,7 @@ class FieldTest(unittest.TestCase): User.drop_collection() BlogPost.drop_collection() - + def test_list_item_dereference(self): """Ensure that DBRef items in ListFields are dereferenced. """ @@ -434,7 +435,7 @@ class FieldTest(unittest.TestCase): class TreeNode(EmbeddedDocument): name = StringField() children = ListField(EmbeddedDocumentField('self')) - + tree = Tree(name="Tree") first_child = TreeNode(name="Child 1") @@ -442,7 +443,7 @@ class FieldTest(unittest.TestCase): second_child = TreeNode(name="Child 2") first_child.children.append(second_child) - + third_child = TreeNode(name="Child 3") first_child.children.append(third_child) @@ -506,20 +507,20 @@ class FieldTest(unittest.TestCase): Member.drop_collection() BlogPost.drop_collection() - + def test_generic_reference(self): """Ensure that a GenericReferenceField properly dereferences items. """ class Link(Document): title = StringField() meta = {'allow_inheritance': False} - + class Post(Document): title = StringField() - + class Bookmark(Document): bookmark_object = GenericReferenceField() - + Link.drop_collection() Post.drop_collection() Bookmark.drop_collection() @@ -574,16 +575,49 @@ class FieldTest(unittest.TestCase): user = User(bookmarks=[post_1, link_1]) user.save() - + user = User.objects(bookmarks__all=[post_1, link_1]).first() - + self.assertEqual(user.bookmarks[0], post_1) self.assertEqual(user.bookmarks[1], link_1) - + Link.drop_collection() Post.drop_collection() User.drop_collection() + def test_generic_reference_document_not_registered(self): + """Ensure dereferencing out of the document registry throws a + `NotRegistered` error. + """ + class Link(Document): + title = StringField() + + class User(Document): + bookmarks = ListField(GenericReferenceField()) + + Link.drop_collection() + User.drop_collection() + + link_1 = Link(title="Pitchfork") + link_1.save() + + user = User(bookmarks=[link_1]) + user.save() + + # Mimic User and Link definitions being in a different file + # and the Link model not being imported in the User file. + del(_document_registry["Link"]) + + user = User.objects.first() + try: + user.bookmarks + raise AssertionError, "Link was removed from the registry" + except NotRegistered: + pass + + Link.drop_collection() + User.drop_collection() + def test_binary_fields(self): """Ensure that binary fields can be stored and retrieved. """ @@ -727,7 +761,7 @@ class FieldTest(unittest.TestCase): result = SetFile.objects.first() self.assertTrue(setfile == result) self.assertEquals(result.file.read(), more_text) - result.file.delete() + result.file.delete() PutFile.drop_collection() StreamFile.drop_collection() From 5f53cda3ab3a320c1a303a44b9c61d350900a91c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 20 May 2011 10:55:01 +0100 Subject: [PATCH 0210/1279] Added regression test for #94 --- tests/queryset.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 51224ea..82dae6c 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1761,6 +1761,25 @@ class QuerySetTest(unittest.TestCase): Number.drop_collection() + def test_order_works_with_primary(self): + """Ensure that order_by and primary work. + """ + class Number(Document): + n = IntField(primary_key=True) + + Number.drop_collection() + + Number(n=1).save() + Number(n=2).save() + Number(n=3).save() + + numbers = [n.n for n in Number.objects.order_by('-n')] + self.assertEquals([3, 2, 1], numbers) + + numbers = [n.n for n in Number.objects.order_by('+n')] + self.assertEquals([1, 2, 3], numbers) + Number.drop_collection() + class QTest(unittest.TestCase): From 07e71d9ce9ac42ca7f9cb3d33550d619e7f99bdd Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 20 May 2011 14:18:16 +0100 Subject: [PATCH 0211/1279] Regression test for collection names an primary ordering Closes #91 --- tests/document.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/document.py b/tests/document.py index cef6e8c..77c8269 100644 --- a/tests/document.py +++ b/tests/document.py @@ -212,6 +212,22 @@ class DocumentTest(unittest.TestCase): Person.drop_collection() self.assertFalse(collection in self.db.collection_names()) + def test_collection_name_and_primary(self): + """Ensure that a collection with a specified name may be used. + """ + + class Person(Document): + name = StringField(primary_key=True) + meta = {'collection': 'app'} + + user = Person(name="Test User") + user.save() + + user_obj = Person.objects[0] + self.assertEqual(user_obj.name, "Test User") + + Person.drop_collection() + def test_inherited_collections(self): """Ensure that subclassed documents don't override parents' collections. """ From 1b72ea9cc161995c9a2c34c9929620f9f0f7a79e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 20 May 2011 16:09:03 +0100 Subject: [PATCH 0212/1279] Fixed detection of unique=True in embedded documents. Added some more test cases - thanks to @heyman for the initial test case. Closes #172 Refs #171 --- mongoengine/base.py | 50 +++++++++++++++++++++++-------------- tests/document.py | 61 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 19 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index ede9083..9a6b5f1 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -306,6 +306,30 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): for spec in meta['indexes']] + base_indexes new_class._meta['indexes'] = user_indexes + unique_indexes = cls._unique_with_indexes(new_class) + new_class._meta['unique_indexes'] = unique_indexes + + for field_name, field in new_class._fields.items(): + # Check for custom primary key + if field.primary_key: + current_pk = new_class._meta['id_field'] + if current_pk and current_pk != field_name: + raise ValueError('Cannot override primary key field') + + if not current_pk: + new_class._meta['id_field'] = field_name + # Make 'Document.id' an alias to the real primary key field + new_class.id = field + + if not new_class._meta['id_field']: + new_class._meta['id_field'] = 'id' + new_class._fields['id'] = ObjectIdField(db_field='_id') + new_class.id = new_class._fields['id'] + + return new_class + + @classmethod + def _unique_with_indexes(cls, new_class, namespace=""): unique_indexes = [] for field_name, field in new_class._fields.items(): # Generate a list of indexes needed by uniqueness constraints @@ -331,28 +355,16 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): unique_fields += unique_with # Add the new index to the list - index = [(f, pymongo.ASCENDING) for f in unique_fields] + index = [("%s%s" % (namespace, f), pymongo.ASCENDING) for f in unique_fields] unique_indexes.append(index) - # Check for custom primary key - if field.primary_key: - current_pk = new_class._meta['id_field'] - if current_pk and current_pk != field_name: - raise ValueError('Cannot override primary key field') + # Grab any embedded document field unique indexes + if field.__class__.__name__ == "EmbeddedDocumentField": + field_namespace = "%s." % field_name + unique_indexes += cls._unique_with_indexes(field.document_type, + field_namespace) - if not current_pk: - new_class._meta['id_field'] = field_name - # Make 'Document.id' an alias to the real primary key field - new_class.id = field - - new_class._meta['unique_indexes'] = unique_indexes - - if not new_class._meta['id_field']: - new_class._meta['id_field'] = 'id' - new_class._fields['id'] = ObjectIdField(db_field='_id') - new_class.id = new_class._fields['id'] - - return new_class + return unique_indexes class BaseDocument(object): diff --git a/tests/document.py b/tests/document.py index 77c8269..8f47ec3 100644 --- a/tests/document.py +++ b/tests/document.py @@ -362,6 +362,10 @@ class DocumentTest(unittest.TestCase): post2 = BlogPost(title='test2', slug='test') self.assertRaises(OperationError, post2.save) + + def test_unique_with(self): + """Ensure that unique_with constraints are applied to fields. + """ class Date(EmbeddedDocument): year = IntField(db_field='yr') @@ -385,6 +389,63 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + def test_unique_embedded_document(self): + """Ensure that uniqueness constraints are applied to fields on embedded documents. + """ + class SubDocument(EmbeddedDocument): + year = IntField(db_field='yr') + slug = StringField(unique=True) + + class BlogPost(Document): + title = StringField() + sub = EmbeddedDocumentField(SubDocument) + + BlogPost.drop_collection() + + post1 = BlogPost(title='test1', sub=SubDocument(year=2009, slug="test")) + post1.save() + + # sub.slug is different so won't raise exception + post2 = BlogPost(title='test2', sub=SubDocument(year=2010, slug='another-slug')) + post2.save() + + # Now there will be two docs with the same sub.slug + post3 = BlogPost(title='test3', sub=SubDocument(year=2010, slug='test')) + self.assertRaises(OperationError, post3.save) + + BlogPost.drop_collection() + + def test_unique_with_embedded_document_and_embedded_unique(self): + """Ensure that uniqueness constraints are applied to fields on + embedded documents. And work with unique_with as well. + """ + class SubDocument(EmbeddedDocument): + year = IntField(db_field='yr') + slug = StringField(unique=True) + + class BlogPost(Document): + title = StringField(unique_with='sub.year') + sub = EmbeddedDocumentField(SubDocument) + + BlogPost.drop_collection() + + post1 = BlogPost(title='test1', sub=SubDocument(year=2009, slug="test")) + post1.save() + + # sub.slug is different so won't raise exception + post2 = BlogPost(title='test2', sub=SubDocument(year=2010, slug='another-slug')) + post2.save() + + # Now there will be two docs with the same sub.slug + post3 = BlogPost(title='test3', sub=SubDocument(year=2010, slug='test')) + self.assertRaises(OperationError, post3.save) + + # Now there will be two docs with the same title and year + post3 = BlogPost(title='test1', sub=SubDocument(year=2009, slug='test-1')) + self.assertRaises(OperationError, post3.save) + + BlogPost.drop_collection() + def test_unique_and_indexes(self): """Ensure that 'unique' constraints aren't overridden by meta.indexes. From 36034ee15fad413392c009e8f0d367669d355cb5 Mon Sep 17 00:00:00 2001 From: Alistair Roche Date: Mon, 23 May 2011 18:27:01 +0100 Subject: [PATCH 0213/1279] 'set__comments__0__body="asdf"' syntax works --- mongoengine/base.py | 21 ++++++++++----------- mongoengine/fields.py | 4 ++-- mongoengine/queryset.py | 15 +++++++++++---- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 9d0b823..495e241 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -22,7 +22,7 @@ class BaseField(object): may be added to subclasses of `Document` to define a document's schema. """ - # Fields may have _types inserted into indexes by default + # Fields may have _types inserted into indexes by default _index_with_types = True _geo_index = False @@ -32,7 +32,7 @@ class BaseField(object): creation_counter = 0 auto_creation_counter = -1 - def __init__(self, db_field=None, name=None, required=False, default=None, + def __init__(self, db_field=None, name=None, required=False, default=None, unique=False, unique_with=None, primary_key=False, validation=None, choices=None): self.db_field = (db_field or name) if not primary_key else '_id' @@ -57,7 +57,7 @@ class BaseField(object): BaseField.creation_counter += 1 def __get__(self, instance, owner): - """Descriptor for retrieving a value from a field in a document. Do + """Descriptor for retrieving a value from a field in a document. Do any necessary conversion between Python and MongoDB types. """ if instance is None: @@ -167,8 +167,8 @@ class DocumentMetaclass(type): superclasses.update(base._superclasses) if hasattr(base, '_meta'): - # Ensure that the Document class may be subclassed - - # inheritance may be disabled to remove dependency on + # Ensure that the Document class may be subclassed - + # inheritance may be disabled to remove dependency on # additional fields _cls and _types if base._meta.get('allow_inheritance', True) == False: raise ValueError('Document %s may not be subclassed' % @@ -190,7 +190,6 @@ class DocumentMetaclass(type): attrs['_class_name'] = '.'.join(reversed(class_name)) attrs['_superclasses'] = superclasses - # Add the document's fields to the _fields attribute for attr_name, attr_value in attrs.items(): if hasattr(attr_value, "__class__") and \ @@ -211,12 +210,12 @@ class DocumentMetaclass(type): module = attrs.get('__module__') - base_excs = tuple(base.DoesNotExist for base in bases + base_excs = tuple(base.DoesNotExist for base in bases if hasattr(base, 'DoesNotExist')) or (DoesNotExist,) exc = subclass_exception('DoesNotExist', base_excs, module) new_class.add_to_class('DoesNotExist', exc) - base_excs = tuple(base.MultipleObjectsReturned for base in bases + base_excs = tuple(base.MultipleObjectsReturned for base in bases if hasattr(base, 'MultipleObjectsReturned')) base_excs = base_excs or (MultipleObjectsReturned,) exc = subclass_exception('MultipleObjectsReturned', base_excs, module) @@ -238,9 +237,9 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): def __new__(cls, name, bases, attrs): super_new = super(TopLevelDocumentMetaclass, cls).__new__ - # Classes defined in this package are abstract and should not have + # Classes defined in this package are abstract and should not have # their own metadata with DB collection, etc. - # __metaclass__ is only set on the class with the __metaclass__ + # __metaclass__ is only set on the class with the __metaclass__ # attribute (i.e. it is not set on subclasses). This differentiates # 'real' documents from the 'Document' class if attrs.get('__metaclass__') == TopLevelDocumentMetaclass: @@ -366,7 +365,7 @@ class BaseDocument(object): are present. """ # Get a list of tuples of field names and their current values - fields = [(field, getattr(self, name)) + fields = [(field, getattr(self, name)) for name, field in self._fields.items()] # Ensure that each field is matched to a valid value diff --git a/mongoengine/fields.py b/mongoengine/fields.py index c06fdd4..9fcfcc2 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -339,7 +339,7 @@ class ListField(BaseField): if isinstance(self.field, ReferenceField): referenced_type = self.field.document_type - # Get value from document instance if available + # Get value from document instance if available value_list = instance._data.get(self.name) if value_list: deref_list = [] @@ -643,7 +643,7 @@ class GridFSProxy(object): if not self.newfile: self.new_file() self.grid_id = self.newfile._id - self.newfile.writelines(lines) + self.newfile.writelines(lines) def read(self): try: diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6da11fa..f58328a 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -523,6 +523,10 @@ class QuerySet(object): fields = [] field = None for field_name in parts: + if field_name.isdigit(): + fields.append(field_name) + field = field.field + continue if field is None: # Look up first field from the document if field_name == 'pk': @@ -620,7 +624,6 @@ class QuerySet(object): mongo_query[key] = value elif key in mongo_query and isinstance(mongo_query[key], dict): mongo_query[key].update(value) - return mongo_query def get(self, *q_objs, **query): @@ -1010,7 +1013,6 @@ class QuerySet(object): """ operators = ['set', 'unset', 'inc', 'dec', 'pop', 'push', 'push_all', 'pull', 'pull_all', 'add_to_set'] - mongo_update = {} for key, value in update.items(): parts = key.split('__') @@ -1033,10 +1035,15 @@ class QuerySet(object): if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] fields = QuerySet._lookup_field(_doc_cls, parts) - parts = [field.db_field for field in fields] - + parts = [] + for field in fields: + if isinstance(field, str): + parts.append(field) + else: + parts.append(field.db_field) # Convert value to proper value field = fields[-1] + if op in (None, 'set', 'push', 'pull', 'addToSet'): value = field.prepare_query_value(op, value) elif op in ('pushAll', 'pullAll'): From 1126c85903431fc789ba7afa9220d16720a5b40a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 24 May 2011 11:26:46 +0100 Subject: [PATCH 0214/1279] Added Custom Objects Managers Managers can now be directly declared in a Document eg:: ```python class CustomQuerySetManager(QuerySetManager): @staticmethod def get_queryset(doc_cls, queryset): return queryset(is_published=True) class Post(Document): is_published = BooleanField(default=False) published = CustomQuerySetManager() ``` Refactored the name of the `_manager_func` to `get_queryset` to mark it as part the public API. If declaring a Manager with a get_queryset method, it should be a staticmethod, that accepts the document_class and the queryset. Note - you can still use decorators in fact in the example below, we effectively do the same thing as the first example and is much less verbose. ```python class Post(Document): is_published = BooleanField(default=False) @queryset_manager def published(doc_cls, queryset): return queryset(is_published=True) ``` Thanks to @theojulienne for the initial impetus and code sample #108 --- mongoengine/base.py | 6 +++-- mongoengine/queryset.py | 17 +++++++------- tests/queryset.py | 52 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 63 insertions(+), 12 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 9a6b5f1..77c2d7d 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -299,8 +299,10 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): new_class = super_new(cls, name, bases, attrs) # Provide a default queryset unless one has been manually provided - if not hasattr(new_class, 'objects'): - new_class.objects = QuerySetManager() + manager = attrs.get('objects', QuerySetManager()) + if hasattr(manager, 'queryset_class'): + meta['queryset_class'] = manager.queryset_class + new_class.objects = manager user_indexes = [QuerySet._build_index_spec(new_class, spec) for spec in meta['indexes']] + base_indexes diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 239e146..3583d0f 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -428,8 +428,6 @@ class QuerySet(object): querying collection :param query: Django-style query keyword arguments """ - #if q_obj: - #self._where_clause = q_obj.as_js(self._document) query = Q(**query) if q_obj: query &= q_obj @@ -1308,8 +1306,11 @@ class QuerySet(object): class QuerySetManager(object): - def __init__(self, manager_func=None): - self._manager_func = manager_func + get_queryset = None + + def __init__(self, queryset_func=None): + if queryset_func: + self.get_queryset = queryset_func self._collections = {} def __get__(self, instance, owner): @@ -1353,11 +1354,11 @@ class QuerySetManager(object): # owner is the document that contains the QuerySetManager queryset_class = owner._meta['queryset_class'] or QuerySet queryset = queryset_class(owner, self._collections[(db, collection)]) - if self._manager_func: - if self._manager_func.func_code.co_argcount == 1: - queryset = self._manager_func(queryset) + if self.get_queryset: + if self.get_queryset.func_code.co_argcount == 1: + queryset = self.get_queryset(queryset) else: - queryset = self._manager_func(owner, queryset) + queryset = self.get_queryset(owner, queryset) return queryset diff --git a/tests/queryset.py b/tests/queryset.py index 5a2c46c..777f9e3 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -5,8 +5,9 @@ import unittest import pymongo from datetime import datetime, timedelta -from mongoengine.queryset import (QuerySet, MultipleObjectsReturned, - DoesNotExist, QueryFieldList) +from mongoengine.queryset import (QuerySet, QuerySetManager, + MultipleObjectsReturned, DoesNotExist, + QueryFieldList) from mongoengine import * @@ -1737,6 +1738,53 @@ class QuerySetTest(unittest.TestCase): Post.drop_collection() + def test_custom_querysets_set_manager_directly(self): + """Ensure that custom QuerySet classes may be used. + """ + + class CustomQuerySet(QuerySet): + def not_empty(self): + return len(self) > 0 + + class CustomQuerySetManager(QuerySetManager): + queryset_class = CustomQuerySet + + class Post(Document): + objects = CustomQuerySetManager() + + Post.drop_collection() + + self.assertTrue(isinstance(Post.objects, CustomQuerySet)) + self.assertFalse(Post.objects.not_empty()) + + Post().save() + self.assertTrue(Post.objects.not_empty()) + + Post.drop_collection() + + def test_custom_querysets_managers_directly(self): + """Ensure that custom QuerySet classes may be used. + """ + + class CustomQuerySetManager(QuerySetManager): + + @staticmethod + def get_queryset(doc_cls, queryset): + return queryset(is_published=True) + + class Post(Document): + is_published = BooleanField(default=False) + published = CustomQuerySetManager() + + Post.drop_collection() + + Post().save() + Post(is_published=True).save() + self.assertEquals(Post.objects.count(), 2) + self.assertEquals(Post.published.count(), 1) + + Post.drop_collection() + def test_call_after_limits_set(self): """Ensure that re-filtering after slicing works """ From 118c0deb7a7bd48170eb49624f79f737419c5342 Mon Sep 17 00:00:00 2001 From: Alistair Roche Date: Tue, 24 May 2011 11:31:44 +0100 Subject: [PATCH 0215/1279] Fixed list-indexing syntax; created tests. --- mongoengine/queryset.py | 16 +++++++++++++- tests/queryset.py | 49 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 239e146..e6c9335 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -524,6 +524,15 @@ class QuerySet(object): fields = [] field = None for field_name in parts: + # Handle ListField indexing: + if field_name.isdigit(): + try: + field = field.field + except AttributeError, err: + raise InvalidQueryError( + "Can't use index on unsubscriptable field (%s)" % err) + fields.append(field_name) + continue if field is None: # Look up first field from the document if field_name == 'pk': @@ -1072,7 +1081,12 @@ class QuerySet(object): if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] fields = QuerySet._lookup_field(_doc_cls, parts) - parts = [field.db_field for field in fields] + parts = [] + for field in fields: + if isinstance(field, str): + parts.append(field) + else: + parts.append(field.db_field) # Convert value to proper value field = fields[-1] diff --git a/tests/queryset.py b/tests/queryset.py index 5a2c46c..b0693f6 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -211,6 +211,55 @@ class QuerySetTest(unittest.TestCase): Blog.drop_collection() + def test_update_array_position(self): + """Ensure that updating by array position works. + + Check update() and update_one() can take syntax like: + set__posts__1__comments__1__name="testc" + Check that it only works for ListFields. + """ + class Comment(EmbeddedDocument): + name = StringField() + + class Post(EmbeddedDocument): + comments = ListField(EmbeddedDocumentField(Comment)) + + class Blog(Document): + tags = ListField(StringField()) + posts = ListField(EmbeddedDocumentField(Post)) + + Blog.drop_collection() + + comment1 = Comment(name='testa') + comment2 = Comment(name='testb') + post1 = Post(comments=[comment1, comment2]) + post2 = Post(comments=[comment2, comment2]) + blog1 = Blog.objects.create(posts=[post1, post2]) + blog2 = Blog.objects.create(posts=[post2, post1]) + + # Update all of the first comments of second posts of all blogs + blog = Blog.objects().update(set__posts__1__comments__0__name="testc") + testc_blogs = Blog.objects(posts__1__comments__0__name="testc") + self.assertEqual(len(testc_blogs), 2) + + Blog.drop_collection() + + blog1 = Blog.objects.create(posts=[post1, post2]) + blog2 = Blog.objects.create(posts=[post2, post1]) + + # Update only the first blog returned by the query + blog = Blog.objects().update_one( + set__posts__1__comments__1__name="testc") + testc_blogs = Blog.objects(posts__1__comments__1__name="testc") + self.assertEqual(len(testc_blogs), 1) + + # Check that using this indexing syntax on a non-list fails + def non_list_indexing(): + Blog.objects().update(set__posts__1__comments__0__name__1="asdf") + self.assertRaises(InvalidQueryError, non_list_indexing) + + Blog.drop_collection() + def test_get_or_create(self): """Ensure that ``get_or_create`` returns one result or creates a new document. From 305fd4b232e480f4aaa85999d87e6a645e4e5c43 Mon Sep 17 00:00:00 2001 From: Alistair Roche Date: Tue, 24 May 2011 11:44:43 +0100 Subject: [PATCH 0216/1279] Fixed whitespace --- mongoengine/base.py | 1 + mongoengine/queryset.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index 5188c31..77c2d7d 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -200,6 +200,7 @@ class DocumentMetaclass(type): attrs['_class_name'] = '.'.join(reversed(class_name)) attrs['_superclasses'] = superclasses + # Add the document's fields to the _fields attribute for attr_name, attr_value in attrs.items(): if hasattr(attr_value, "__class__") and \ diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index d7d349a..087ee48 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -628,6 +628,7 @@ class QuerySet(object): mongo_query[key] = value elif key in mongo_query and isinstance(mongo_query[key], dict): mongo_query[key].update(value) + return mongo_query def get(self, *q_objs, **query): @@ -1055,6 +1056,7 @@ class QuerySet(object): """ operators = ['set', 'unset', 'inc', 'dec', 'pop', 'push', 'push_all', 'pull', 'pull_all', 'add_to_set'] + mongo_update = {} for key, value in update.items(): parts = key.split('__') From 088c40f9f2d296f65fd493fd5d98baf02f9125eb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 24 May 2011 12:30:12 +0100 Subject: [PATCH 0217/1279] Added Abstract Base Classes Thanks to @theojulienne for the code :) #108 --- mongoengine/base.py | 12 +++++++++++- tests/document.py | 34 +++++++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 77c2d7d..ffceb79 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -253,7 +253,16 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # __metaclass__ is only set on the class with the __metaclass__ # attribute (i.e. it is not set on subclasses). This differentiates # 'real' documents from the 'Document' class - if attrs.get('__metaclass__') == TopLevelDocumentMetaclass: + # + # Also assume a class is abstract if it has abstract set to True in + # its meta dictionary. This allows custom Document superclasses. + if (attrs.get('__metaclass__') == TopLevelDocumentMetaclass or + ('meta' in attrs and attrs['meta'].get('abstract', False))): + # Make sure no base class was non-abstract + non_abstract_bases = [b for b in bases + if hasattr(b,'_meta') and not b._meta.get('abstract', False)] + if non_abstract_bases: + raise ValueError("Abstract document cannot have non-abstract base") return super_new(cls, name, bases, attrs) collection = name.lower() @@ -276,6 +285,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): base_indexes += base._meta.get('indexes', []) meta = { + 'abstract': False, 'collection': collection, 'max_documents': None, 'max_size': None, diff --git a/tests/document.py b/tests/document.py index 8f47ec3..fe67312 100644 --- a/tests/document.py +++ b/tests/document.py @@ -29,6 +29,9 @@ class DocumentTest(unittest.TestCase): age = IntField() self.Person = Person + def tearDown(self): + self.Person.drop_collection() + def test_drop_collection(self): """Ensure that the collection may be dropped from the database. """ @@ -188,6 +191,34 @@ class DocumentTest(unittest.TestCase): self.assertFalse('_cls' in comment.to_mongo()) self.assertFalse('_types' in comment.to_mongo()) + def test_abstract_documents(self): + """Ensure that a document superclass can be marked as abstract + thereby not using it as the name for the collection.""" + + class Animal(Document): + name = StringField() + meta = {'abstract': True} + + class Fish(Animal): pass + class Guppy(Fish): pass + + class Mammal(Animal): + meta = {'abstract': True} + class Human(Mammal): pass + + self.assertFalse('collection' in Animal._meta) + self.assertFalse('collection' in Mammal._meta) + + self.assertEqual(Fish._meta['collection'], 'fish') + self.assertEqual(Guppy._meta['collection'], 'fish') + self.assertEqual(Human._meta['collection'], 'human') + + def create_bad_abstract(): + class EvilHuman(Human): + evil = BooleanField(default=True) + meta = {'abstract': True} + self.assertRaises(ValueError, create_bad_abstract) + def test_collection_name(self): """Ensure that a collection with a specified name may be used. """ @@ -907,9 +938,6 @@ class DocumentTest(unittest.TestCase): A.drop_collection() B.drop_collection() - def tearDown(self): - self.Person.drop_collection() - def test_document_hash(self): """Test document in list, dict, set """ From 32bab13a8acc0e091094468be0a6d890719bfec5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 24 May 2011 12:50:48 +0100 Subject: [PATCH 0218/1279] Added MapField, similar to DictField Similar to DictField except the value of each entry is always of a certain (declared) field type. Thanks again to @theojulienne for the code #108 --- mongoengine/fields.py | 93 ++++++++++++++++++++++++++++++++++++++++++- tests/fields.py | 61 ++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 0cc8219..d1f9b66 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -17,7 +17,7 @@ import warnings __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', - 'ObjectIdField', 'ReferenceField', 'ValidationError', + 'ObjectIdField', 'ReferenceField', 'ValidationError', 'MapField', 'DecimalField', 'URLField', 'GenericReferenceField', 'FileField', 'BinaryField', 'SortedListField', 'EmailField', 'GeoPointField'] @@ -451,6 +451,97 @@ class DictField(BaseField): def lookup_member(self, member_name): return self.basecls(db_field=member_name) + +class MapField(BaseField): + """A field that maps a name to a specified field type. Similar to + a DictField, except the 'value' of each item must match the specified + field type. + + .. versionadded:: 0.5 + """ + + def __init__(self, field=None, *args, **kwargs): + if not isinstance(field, BaseField): + raise ValidationError('Argument to MapField constructor must be ' + 'a valid field') + self.field = field + kwargs.setdefault('default', lambda: {}) + super(MapField, self).__init__(*args, **kwargs) + + def validate(self, value): + """Make sure that a list of valid fields is being used. + """ + if not isinstance(value, dict): + raise ValidationError('Only dictionaries may be used in a ' + 'DictField') + + if any(('.' in k or '$' in k) for k in value): + raise ValidationError('Invalid dictionary key name - keys may not ' + 'contain "." or "$" characters') + + try: + [self.field.validate(item) for item in value.values()] + except Exception, err: + raise ValidationError('Invalid MapField item (%s)' % str(item)) + + def __get__(self, instance, owner): + """Descriptor to automatically dereference references. + """ + if instance is None: + # Document class being used rather than a document object + return self + + if isinstance(self.field, ReferenceField): + referenced_type = self.field.document_type + # Get value from document instance if available + value_dict = instance._data.get(self.name) + if value_dict: + deref_dict = [] + for key,value in value_dict.iteritems(): + # Dereference DBRefs + if isinstance(value, (pymongo.dbref.DBRef)): + value = _get_db().dereference(value) + deref_dict[key] = referenced_type._from_son(value) + else: + deref_dict[key] = value + instance._data[self.name] = deref_dict + + if isinstance(self.field, GenericReferenceField): + value_dict = instance._data.get(self.name) + if value_dict: + deref_dict = [] + for key,value in value_dict.iteritems(): + # Dereference DBRefs + if isinstance(value, (dict, pymongo.son.SON)): + deref_dict[key] = self.field.dereference(value) + else: + deref_dict[key] = value + instance._data[self.name] = deref_dict + + return super(MapField, self).__get__(instance, owner) + + def to_python(self, value): + return dict( [(key,self.field.to_python(item)) for key,item in value.iteritems()] ) + + def to_mongo(self, value): + return dict( [(key,self.field.to_mongo(item)) for key,item in value.iteritems()] ) + + def prepare_query_value(self, op, value): + return self.field.prepare_query_value(op, value) + + def lookup_member(self, member_name): + return self.field.lookup_member(member_name) + + def _set_owner_document(self, owner_document): + self.field.owner_document = owner_document + self._owner_document = owner_document + + def _get_owner_document(self, owner_document): + self._owner_document = owner_document + + owner_document = property(_get_owner_document, _set_owner_document) + + class ReferenceField(BaseField): """A reference to a document that will be automatically dereferenced on access (lazily). diff --git a/tests/fields.py b/tests/fields.py index 38409b6..62bd3a1 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -825,5 +825,66 @@ class FieldTest(unittest.TestCase): self.assertEqual(d2.data, {}) self.assertEqual(d2.data2, {}) + def test_mapfield(self): + """Ensure that the MapField handles the declared type.""" + + class Simple(Document): + mapping = MapField(IntField()) + + Simple.drop_collection() + + e = Simple() + e.mapping['someint'] = 1 + e.save() + + def create_invalid_mapping(): + e.mapping['somestring'] = "abc" + e.save() + + self.assertRaises(ValidationError, create_invalid_mapping) + + def create_invalid_class(): + class NoDeclaredType(Document): + mapping = MapField() + + self.assertRaises(ValidationError, create_invalid_class) + + Simple.drop_collection() + + def test_complex_mapfield(self): + """Ensure that the MapField can handle complex declared types.""" + + class SettingBase(EmbeddedDocument): + pass + + class StringSetting(SettingBase): + value = StringField() + + class IntegerSetting(SettingBase): + value = IntField() + + class Extensible(Document): + mapping = MapField(EmbeddedDocumentField(SettingBase)) + + Extensible.drop_collection() + + e = Extensible() + e.mapping['somestring'] = StringSetting(value='foo') + e.mapping['someint'] = IntegerSetting(value=42) + e.save() + + e2 = Extensible.objects.get(id=e.id) + self.assertTrue(isinstance(e2.mapping['somestring'], StringSetting)) + self.assertTrue(isinstance(e2.mapping['someint'], IntegerSetting)) + + def create_invalid_mapping(): + e.mapping['someint'] = 123 + e.save() + + self.assertRaises(ValidationError, create_invalid_mapping) + + Extensible.drop_collection() + + if __name__ == '__main__': unittest.main() From 7ecf84395a698818440b70104dc2f4d5984a4386 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 24 May 2011 14:07:58 +0100 Subject: [PATCH 0219/1279] Improved DictFields Allow searching multiple levels deep in DictFields Allow DictField entries containing strings to use matching operators Thanks again to @theojulien for the initial code #108 --- mongoengine/fields.py | 12 +++++++++++- tests/fields.py | 23 +++++++++++++++++++++-- tests/queryset.py | 2 +- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index d1f9b66..11366dd 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -449,7 +449,17 @@ class DictField(BaseField): 'contain "." or "$" characters') def lookup_member(self, member_name): - return self.basecls(db_field=member_name) + return DictField(basecls=self.basecls, db_field=member_name) + + def prepare_query_value(self, op, value): + match_operators = ['contains', 'icontains', 'startswith', + 'istartswith', 'endswith', 'iendswith', + 'exact', 'iexact'] + + if op in match_operators and isinstance(value, basestring): + return StringField().prepare_query_value(op, value) + + return super(DictField,self).prepare_query_value(op, value) class MapField(BaseField): diff --git a/tests/fields.py b/tests/fields.py index 62bd3a1..00b1c88 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -262,12 +262,14 @@ class FieldTest(unittest.TestCase): BlogPost.drop_collection() - def test_dict_validation(self): + def test_dict_field(self): """Ensure that dict types work as expected. """ class BlogPost(Document): info = DictField() + BlogPost.drop_collection() + post = BlogPost() post.info = 'my post' self.assertRaises(ValidationError, post.validate) @@ -282,7 +284,24 @@ class FieldTest(unittest.TestCase): self.assertRaises(ValidationError, post.validate) post.info = {'title': 'test'} - post.validate() + post.save() + + post = BlogPost() + post.info = {'details': {'test': 'test'}} + post.save() + + post = BlogPost() + post.info = {'details': {'test': 3}} + post.save() + + self.assertEquals(BlogPost.objects.count(), 3) + self.assertEquals(BlogPost.objects.filter(info__title__exact='test').count(), 1) + self.assertEquals(BlogPost.objects.filter(info__details__test__exact='test').count(), 1) + + # Confirm handles non strings or non existing keys + self.assertEquals(BlogPost.objects.filter(info__details__test__exact=5).count(), 0) + self.assertEquals(BlogPost.objects.filter(info__made_up__test__exact='test').count(), 0) + BlogPost.drop_collection() def test_embedded_document_validation(self): """Ensure that invalid embedded documents cannot be assigned to diff --git a/tests/queryset.py b/tests/queryset.py index 777f9e3..6c0b686 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1539,7 +1539,7 @@ class QuerySetTest(unittest.TestCase): t = Test(testdict={'f': 'Value'}) t.save() - self.assertEqual(len(Test.objects(testdict__f__startswith='Val')), 0) + self.assertEqual(len(Test.objects(testdict__f__startswith='Val')), 1) self.assertEqual(len(Test.objects(testdict__f='Value')), 1) Test.drop_collection() From c3a88404356c40702ca0204193ede1b1bd21e0ad Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 24 May 2011 20:27:19 +0100 Subject: [PATCH 0220/1279] Blinker signals added --- mongoengine/base.py | 6 ++ mongoengine/document.py | 10 ++++ mongoengine/signals.py | 41 +++++++++++++ setup.py | 6 +- tests/signals.py | 130 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 190 insertions(+), 3 deletions(-) create mode 100644 mongoengine/signals.py create mode 100644 tests/signals.py diff --git a/mongoengine/base.py b/mongoengine/base.py index ffceb79..101bb73 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -2,6 +2,8 @@ from queryset import QuerySet, QuerySetManager from queryset import DoesNotExist, MultipleObjectsReturned from queryset import DO_NOTHING +from mongoengine import signals + import sys import pymongo import pymongo.objectid @@ -382,6 +384,8 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): class BaseDocument(object): def __init__(self, **values): + signals.pre_init.send(self, values=values) + self._data = {} # Assign default values to instance for attr_name in self._fields.keys(): @@ -395,6 +399,8 @@ class BaseDocument(object): except AttributeError: pass + signals.post_init.send(self) + def validate(self): """Ensure that all fields' values are valid and that required fields are present. diff --git a/mongoengine/document.py b/mongoengine/document.py index 771b922..b563f42 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,3 +1,4 @@ +from mongoengine import signals from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, ValidationError) from queryset import OperationError @@ -75,6 +76,8 @@ class Document(BaseDocument): For example, ``save(..., w=2, fsync=True)`` will wait until at least two servers have recorded the write and will force an fsync on each server being written to. """ + signals.pre_save.send(self) + if validate: self.validate() @@ -82,6 +85,7 @@ class Document(BaseDocument): write_options = {} doc = self.to_mongo() + created = '_id' not in doc try: collection = self.__class__.objects._collection if force_insert: @@ -96,12 +100,16 @@ class Document(BaseDocument): id_field = self._meta['id_field'] self[id_field] = self._fields[id_field].to_python(object_id) + signals.post_save.send(self, created=created) + def delete(self, safe=False): """Delete the :class:`~mongoengine.Document` from the database. This will only take effect if the document has been previously saved. :param safe: check if the operation succeeded before returning """ + signals.pre_delete.send(self) + id_field = self._meta['id_field'] object_id = self._fields[id_field].to_mongo(self[id_field]) try: @@ -110,6 +118,8 @@ class Document(BaseDocument): message = u'Could not delete document (%s)' % err.message raise OperationError(message) + signals.post_delete.send(self) + @classmethod def register_delete_rule(cls, document_cls, field_name, rule): """This method registers the delete rules to apply when removing this diff --git a/mongoengine/signals.py b/mongoengine/signals.py new file mode 100644 index 0000000..4caa553 --- /dev/null +++ b/mongoengine/signals.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +signals_available = False +try: + from blinker import Namespace + signals_available = True +except ImportError: + class Namespace(object): + def signal(self, name, doc=None): + return _FakeSignal(name, doc) + + class _FakeSignal(object): + """If blinker is unavailable, create a fake class with the same + interface that allows sending of signals but will fail with an + error on anything else. Instead of doing anything on send, it + will just ignore the arguments and do nothing instead. + """ + + def __init__(self, name, doc=None): + self.name = name + self.__doc__ = doc + + def _fail(self, *args, **kwargs): + raise RuntimeError('signalling support is unavailable ' + 'because the blinker library is ' + 'not installed.') + send = lambda *a, **kw: None + connect = disconnect = has_receivers_for = receivers_for = \ + temporarily_connected_to = _fail + del _fail + +# the namespace for code signals. If you are not mongoengine code, do +# not put signals in here. Create your own namespace instead. +_signals = Namespace() + +pre_init = _signals.signal('pre_init') +post_init = _signals.signal('post_init') +pre_save = _signals.signal('pre_save') +post_save = _signals.signal('post_save') +pre_delete = _signals.signal('pre_delete') +post_delete = _signals.signal('post_delete') diff --git a/setup.py b/setup.py index e0585b7..01a201d 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ def get_version(version_tuple): version = '%s.%s' % (version, version_tuple[2]) return version -# Dirty hack to get version number from monogengine/__init__.py - we can't +# Dirty hack to get version number from monogengine/__init__.py - we can't # import it as it depends on PyMongo and PyMongo isn't installed until this # file is read init = os.path.join(os.path.dirname(__file__), 'mongoengine', '__init__.py') @@ -45,6 +45,6 @@ setup(name='mongoengine', long_description=LONG_DESCRIPTION, platforms=['any'], classifiers=CLASSIFIERS, - install_requires=['pymongo'], - test_suite='tests', + install_requires=['pymongo', 'blinker'], + test_suite='tests.signals', ) diff --git a/tests/signals.py b/tests/signals.py new file mode 100644 index 0000000..fff2d39 --- /dev/null +++ b/tests/signals.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +import unittest + +from mongoengine import * +from mongoengine import signals + +signal_output = [] + + +class SignalTests(unittest.TestCase): + """ + Testing signals before/after saving and deleting. + """ + + def get_signal_output(self, fn, *args, **kwargs): + # Flush any existing signal output + global signal_output + signal_output = [] + fn(*args, **kwargs) + return signal_output + + def setUp(self): + connect(db='mongoenginetest') + class Author(Document): + name = StringField() + + def __unicode__(self): + return self.name + + @classmethod + def pre_init(cls, instance, **kwargs): + signal_output.append('pre_init signal, %s' % cls.__name__) + signal_output.append(str(kwargs['values'])) + + @classmethod + def post_init(cls, instance, **kwargs): + signal_output.append('post_init signal, %s' % instance) + + @classmethod + def pre_save(cls, instance, **kwargs): + signal_output.append('pre_save signal, %s' % instance) + + @classmethod + def post_save(cls, instance, **kwargs): + signal_output.append('post_save signal, %s' % instance) + if 'created' in kwargs: + if kwargs['created']: + signal_output.append('Is created') + else: + signal_output.append('Is updated') + + @classmethod + def pre_delete(cls, instance, **kwargs): + signal_output.append('pre_delete signal, %s' % instance) + + @classmethod + def post_delete(cls, instance, **kwargs): + signal_output.append('post_delete signal, %s' % instance) + + self.Author = Author + + # Save up the number of connected signals so that we can check at the end + # that all the signals we register get properly unregistered + self.pre_signals = ( + len(signals.pre_init.receivers), + len(signals.post_init.receivers), + len(signals.pre_save.receivers), + len(signals.post_save.receivers), + len(signals.pre_delete.receivers), + len(signals.post_delete.receivers) + ) + + signals.pre_init.connect(Author.pre_init) + signals.post_init.connect(Author.post_init) + signals.pre_save.connect(Author.pre_save) + signals.post_save.connect(Author.post_save) + signals.pre_delete.connect(Author.pre_delete) + signals.post_delete.connect(Author.post_delete) + + def tearDown(self): + signals.pre_init.disconnect(self.Author.pre_init) + signals.post_init.disconnect(self.Author.post_init) + signals.post_delete.disconnect(self.Author.post_delete) + signals.pre_delete.disconnect(self.Author.pre_delete) + signals.post_save.disconnect(self.Author.post_save) + signals.pre_save.disconnect(self.Author.pre_save) + + # Check that all our signals got disconnected properly. + post_signals = ( + len(signals.pre_init.receivers), + len(signals.post_init.receivers), + len(signals.pre_save.receivers), + len(signals.post_save.receivers), + len(signals.pre_delete.receivers), + len(signals.post_delete.receivers) + ) + + self.assertEqual(self.pre_signals, post_signals) + + def test_model_signals(self): + """ Model saves should throw some signals. """ + + def create_author(): + a1 = self.Author(name='Bill Shakespeare') + + self.assertEqual(self.get_signal_output(create_author), [ + "pre_init signal, Author", + "{'name': 'Bill Shakespeare'}", + "post_init signal, Bill Shakespeare", + ]) + + a1 = self.Author(name='Bill Shakespeare') + self.assertEqual(self.get_signal_output(a1.save), [ + "pre_save signal, Bill Shakespeare", + "post_save signal, Bill Shakespeare", + "Is created" + ]) + + a1.reload() + a1.name='William Shakespeare' + self.assertEqual(self.get_signal_output(a1.save), [ + "pre_save signal, William Shakespeare", + "post_save signal, William Shakespeare", + "Is updated" + ]) + + self.assertEqual(self.get_signal_output(a1.delete), [ + 'pre_delete signal, William Shakespeare', + 'post_delete signal, William Shakespeare', + ]) \ No newline at end of file From 0708d1bedc53d933750e2b871a1c16626627b1f7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 25 May 2011 09:34:50 +0100 Subject: [PATCH 0221/1279] Run all tests... --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 01a201d..d3be64b 100644 --- a/setup.py +++ b/setup.py @@ -46,5 +46,5 @@ setup(name='mongoengine', platforms=['any'], classifiers=CLASSIFIERS, install_requires=['pymongo', 'blinker'], - test_suite='tests.signals', + test_suite='tests', ) From 3861103585beba17b886d3da044fe49017b638cb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 25 May 2011 09:36:25 +0100 Subject: [PATCH 0222/1279] Updated connection exception to provide more info on the cause. Fixes #178 --- mongoengine/connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index fc6c768..7b5cd21 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -31,8 +31,8 @@ def _get_connection(reconnect=False): if _connection.get(identity) is None or reconnect: try: _connection[identity] = Connection(**_connection_settings) - except: - raise ConnectionError('Cannot connect to the database') + except Exception, e: + raise ConnectionError("Cannot connect to the database:\n%s" % e) return _connection[identity] def _get_db(reconnect=False): From 60c8254f58986f43a25f9190bb97fb07b0742b64 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 25 May 2011 11:10:42 +0100 Subject: [PATCH 0223/1279] Tweaks to item_frequencies Updated to use a ternary statement and added tests Refs #124 #122 Thanks to @nickvlku for the code. --- mongoengine/queryset.py | 12 ++---------- tests/queryset.py | 13 ++++++++----- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 54d4845..f5020ab 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1298,19 +1298,11 @@ class QuerySet(object): db[collection].find(query).forEach(function(doc) { if (doc[field].constructor == Array) { doc[field].forEach(function(item) { - var preValue = 0; - if (!isNaN(frequencies[item])) { - preValue = frequencies[item]; - } - frequencies[item] = inc + preValue; + frequencies[item] = inc + (isNaN(frequencies[item]) ? 0: frequencies[item]); }); } else { var item = doc[field]; - var preValue = 0; - if (!isNaN(frequencies[item])) { - preValue = frequencies[item]; - } - frequencies[item] = inc + preValue; + frequencies[item] = inc + (isNaN(frequencies[item]) ? 0: frequencies[item]); } }); return frequencies; diff --git a/tests/queryset.py b/tests/queryset.py index 5cf0895..9b71140 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1339,7 +1339,7 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() BlogPost(hits=1, tags=['music', 'film', 'actors', 'watch']).save() - BlogPost(hits=2, tags=['music']).save() + BlogPost(hits=2, tags=['music', 'watch']).save() BlogPost(hits=2, tags=['music', 'actors']).save() f = BlogPost.objects.item_frequencies('tags') @@ -1347,20 +1347,23 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(set(['music', 'film', 'actors', 'watch']), set(f.keys())) self.assertEqual(f['music'], 3) self.assertEqual(f['actors'], 2) + self.assertEqual(f['watch'], 2) self.assertEqual(f['film'], 1) # Ensure query is taken into account f = BlogPost.objects(hits__gt=1).item_frequencies('tags') f = dict((key, int(val)) for key, val in f.items()) - self.assertEqual(set(['music', 'actors']), set(f.keys())) + self.assertEqual(set(['music', 'actors', 'watch']), set(f.keys())) self.assertEqual(f['music'], 2) self.assertEqual(f['actors'], 1) + self.assertEqual(f['watch'], 1) # Check that normalization works f = BlogPost.objects.item_frequencies('tags', normalize=True) - self.assertAlmostEqual(f['music'], 3.0/7.0) - self.assertAlmostEqual(f['actors'], 2.0/7.0) - self.assertAlmostEqual(f['film'], 1.0/7.0) + self.assertAlmostEqual(f['music'], 3.0/8.0) + self.assertAlmostEqual(f['actors'], 2.0/8.0) + self.assertAlmostEqual(f['watch'], 2.0/8.0) + self.assertAlmostEqual(f['film'], 1.0/8.0) # Check item_frequencies works for non-list fields f = BlogPost.objects.item_frequencies('hits') From b1cdd1eb268680aa0425aead0f1080560f080e5c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 25 May 2011 12:01:41 +0100 Subject: [PATCH 0224/1279] Updated docs regarding ReferenceFields Closes #149 --- docs/tutorial.rst | 15 +++++++++++++++ mongoengine/fields.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 5db2c4d..63f8fe9 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -152,6 +152,21 @@ We can then store a list of comment documents in our post document:: tags = ListField(StringField(max_length=30)) comments = ListField(EmbeddedDocumentField(Comment)) +Handling deletions of references +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The :class:`~mongoengine.ReferenceField` object takes a keyword +`reverse_delete_rule` for handling deletion rules if the reference is deleted. +To delete all the posts if a user is deleted set the rule:: + + class Post(Document): + title = StringField(max_length=120, required=True) + author = ReferenceField(User, reverse_delete_rule=CASCADE) + tags = ListField(StringField(max_length=30)) + comments = ListField(EmbeddedDocumentField(Comment)) + +See :class:`~mongoengine.ReferenceField` for more information. + Adding data to our Tumblelog ============================ Now that we've defined how our documents will be structured, let's start adding diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 11366dd..b12c507 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -555,9 +555,24 @@ class MapField(BaseField): class ReferenceField(BaseField): """A reference to a document that will be automatically dereferenced on access (lazily). + + Use the `reverse_delete_rule` to handle what should happen if the document + the field is referencing is deleted. + + The options are: + + * DO_NOTHING - don't do anything (default). + * NULLIFY - Updates the reference to null. + * CASCADE - Deletes the documents associated with the reference. + * DENY - Prevent the deletion of the reference object. """ def __init__(self, document_type, reverse_delete_rule=DO_NOTHING, **kwargs): + """Initialises the Reference Field. + + :param reverse_delete_rule: Determines what to do when the referring + object is deleted + """ if not isinstance(document_type, basestring): if not issubclass(document_type, (Document, basestring)): raise ValidationError('Argument to ReferenceField constructor ' From fac3f038a84fb768894f0885f85f3cedaf7ac1dc Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 25 May 2011 12:20:56 +0100 Subject: [PATCH 0225/1279] Added regression test for issue with unset and pop Closes #118 --- tests/queryset.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 9b71140..a45aaf3 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1105,6 +1105,32 @@ class QuerySetTest(unittest.TestCase): self.assertTrue('code' not in post.tags) self.assertEqual(len(post.tags), 1) + def test_update_one_pop_generic_reference(self): + + class BlogTag(Document): + name = StringField(required=True) + + class BlogPost(Document): + slug = StringField() + tags = ListField(ReferenceField(BlogTag), required=True) + + tag_1 = BlogTag(name='code') + tag_1.save() + tag_2 = BlogTag(name='mongodb') + tag_2.save() + + post = BlogPost(slug="test", tags=[tag_1]) + post.save() + + post = BlogPost(slug="test-2", tags=[tag_1, tag_2]) + post.save() + self.assertEqual(len(post.tags), 2) + + BlogPost.objects(slug="test-2").update_one(pop__tags=-1) + + post.reload() + self.assertEqual(len(post.tags), 1) + def test_order_by(self): """Ensure that QuerySets may be ordered. """ From eb892241ee3bd1f98d2d8c0fe420841ddb21037c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 25 May 2011 13:31:01 +0100 Subject: [PATCH 0226/1279] Added regression test for editting embedded documents Closes #35 --- tests/queryset.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index a45aaf3..296377f 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1114,6 +1114,9 @@ class QuerySetTest(unittest.TestCase): slug = StringField() tags = ListField(ReferenceField(BlogTag), required=True) + BlogPost.drop_collection() + BlogTag.drop_collection() + tag_1 = BlogTag(name='code') tag_1.save() tag_2 = BlogTag(name='mongodb') @@ -1131,6 +1134,40 @@ class QuerySetTest(unittest.TestCase): post.reload() self.assertEqual(len(post.tags), 1) + BlogPost.drop_collection() + BlogTag.drop_collection() + + def test_editting_embedded_objects(self): + + class BlogTag(EmbeddedDocument): + name = StringField(required=True) + + class BlogPost(Document): + slug = StringField() + tags = ListField(EmbeddedDocumentField(BlogTag), required=True) + + BlogPost.drop_collection() + + tag_1 = BlogTag(name='code') + tag_2 = BlogTag(name='mongodb') + + post = BlogPost(slug="test", tags=[tag_1]) + post.save() + + post = BlogPost(slug="test-2", tags=[tag_1, tag_2]) + post.save() + self.assertEqual(len(post.tags), 2) + + BlogPost.objects(slug="test-2").update_one(set__tags__0__name="python") + post.reload() + self.assertEquals(post.tags[0].name, 'python') + + BlogPost.objects(slug="test-2").update_one(pop__tags=-1) + post.reload() + self.assertEqual(len(post.tags), 1) + + BlogPost.drop_collection() + def test_order_by(self): """Ensure that QuerySets may be ordered. """ From 5ab13518dbecc826965232a34744bd6ce1cba31e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 25 May 2011 13:50:52 +0100 Subject: [PATCH 0227/1279] Added test confirming order_by reference field doesnt work --- tests/queryset.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 296377f..a2d78d7 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1187,6 +1187,29 @@ class QuerySetTest(unittest.TestCase): ages = [p.age for p in self.Person.objects.order_by('-name')] self.assertEqual(ages, [30, 40, 20]) + def test_confirm_order_by_reference_wont_work(self): + """Ordering by reference is not possible. Use map / reduce.. or + denormalise""" + + class Author(Document): + author = ReferenceField(self.Person) + + Author.drop_collection() + + person_a = self.Person(name="User A", age=20) + person_a.save() + person_b = self.Person(name="User B", age=40) + person_b.save() + person_c = self.Person(name="User C", age=30) + person_c.save() + + Author(author=person_a).save() + Author(author=person_b).save() + Author(author=person_c).save() + + names = [a.author.name for a in Author.objects.order_by('-author__age')] + self.assertEqual(names, ['User A', 'User B', 'User C']) + def test_map_reduce(self): """Ensure map/reduce is both mapping and reducing. """ From bf6f03a4129ead4975c16822f7516fbae3696f85 Mon Sep 17 00:00:00 2001 From: Alistair Roche Date: Wed, 25 May 2011 17:25:39 +0100 Subject: [PATCH 0228/1279] Improved MapFields setting --- mongoengine/fields.py | 6 +++++- tests/queryset.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index b12c507..b2aab5a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -537,7 +537,11 @@ class MapField(BaseField): return dict( [(key,self.field.to_mongo(item)) for key,item in value.iteritems()] ) def prepare_query_value(self, op, value): - return self.field.prepare_query_value(op, value) + if op not in ('set', 'unset'): + return self.field.prepare_query_value(op, value) + for key in value: + value[key] = self.field.prepare_query_value(op, value[key]) + return value def lookup_member(self, member_name): return self.field.lookup_member(member_name) diff --git a/tests/queryset.py b/tests/queryset.py index a2d78d7..82cd870 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -261,6 +261,44 @@ class QuerySetTest(unittest.TestCase): Blog.drop_collection() + def test_mapfield_update(self): + """Ensure that the MapField can be updated.""" + class Member(EmbeddedDocument): + gender = StringField() + age = IntField() + + class Club(Document): + members = MapField(EmbeddedDocumentField(Member)) + + Club.drop_collection() + + club = Club() + club.members['John'] = Member(gender="M", age=13) + club.save() + + Club.objects().update( + set__members={"John": Member(gender="F", age=14)}) + + club = Club.objects().first() + self.assertEqual(club.members['John'].gender, "F") + self.assertEqual(club.members['John'].age, 14) + + def test_dictfield_update(self): + """Ensure that the MapField can be updated.""" + class Club(Document): + members = DictField() + + club = Club() + club.members['John'] = dict(gender="M", age=13) + club.save() + + Club.objects().update( + set__members={"John": dict(gender="F", age=14)}) + + club = Club.objects().first() + self.assertEqual(club.members['John']['gender'], "F") + self.assertEqual(club.members['John']['age'], 14) + def test_get_or_create(self): """Ensure that ``get_or_create`` returns one result or creates a new document. From 97a13103441b61ebde77f1ca510b0bf556feafd1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 May 2011 11:11:00 +0100 Subject: [PATCH 0229/1279] Tweakin test --- tests/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queryset.py b/tests/queryset.py index 82cd870..d5611fd 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -284,7 +284,7 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(club.members['John'].age, 14) def test_dictfield_update(self): - """Ensure that the MapField can be updated.""" + """Ensure that the DictField can be updated.""" class Club(Document): members = DictField() From 9dd3504765fe209030213164fe6e3f37a7c4bbb2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 May 2011 11:56:56 +0100 Subject: [PATCH 0230/1279] Updated changelog --- docs/changelog.rst | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index d7c6fe8..29d03bc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,42 @@ Changelog ========= +Changes in dev +============== + +- Updated connection exception so it provides more info on the cause. +- Added searching multiple levels deep in ``DictField`` +- Added ``DictField`` entries containing strings to use matching operators +- Added ``MapField``, similar to ``DictField`` +- Added Abstract Base Classes +- Added Custom Objects Managers +- Added sliced subfields updating +- Added ``NotRegistered`` exception if dereferencing ``Document`` not in the registry +- Added a write concern for ``save``, ``update``, ``update_one`` and ``get_or_create`` +- Added slicing / subarray fetching controls +- Fixed various unique index and other index issues +- Fixed threaded connection issues +- Added spherical geospatial query operators +- Updated queryset to handle latest version of pymongo + map_reduce now requires an output. +- Added ``Document`` __hash__, __ne__ for pickling +- Added ``FileField`` optional size arg for read method +- Fixed ``FileField`` seek and tell methods for reading files +- Added ``QuerySet.clone`` to support copying querysets +- Fixed item_frequencies when using name thats the same as a native js function +- Added reverse delete rules +- Fixed issue with unset operation +- Fixed Q-object bug +- Added ``QuerySet.all_fields`` resets previous .only() and .exlude() +- Added ``QuerySet.exclude`` +- Added django style choices +- Fixed order and filter issue +- Added ``QuerySet.only`` subfield support +- Added creation_counter to ``BaseField`` allowing fields to be sorted in the + way the user has specified them +- Fixed various errors +- Added many tests + Changes in v0.4 =============== - Added ``GridFSStorage`` Django storage backend From c903af032fd65614585e9b9a377abf14f046fdfc Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 May 2011 15:44:43 +0100 Subject: [PATCH 0231/1279] Added inline_map_reduce functionality Also added map_reduce method for calculating item_frequencies Closes #183 --- docs/changelog.rst | 2 ++ mongoengine/queryset.py | 63 ++++++++++++++++++++++++++++++++++++--- setup.py | 2 +- tests/queryset.py | 65 ++++++++++++++++++++++++++--------------- 4 files changed, 104 insertions(+), 28 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 29d03bc..686b326 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,8 @@ Changelog Changes in dev ============== +- Added optional map_reduce method item_frequencies +- Added inline_map_reduce option to map_reduce - Updated connection exception so it provides more info on the cause. - Added searching multiple levels deep in ``DictField`` - Added ``DictField`` entries containing strings to use matching operators diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index f5020ab..17a1b0d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -774,7 +774,8 @@ class QuerySet(object): :param map_f: map function, as :class:`~pymongo.code.Code` or string :param reduce_f: reduce function, as :class:`~pymongo.code.Code` or string - :param output: output collection name + :param output: output collection name, if set to 'inline' will try to + use :class:`~pymongo.collection.Collection.inline_map_reduce` :param finalize_f: finalize function, an optional function that performs any post-reduction processing. :param scope: values to insert into map/reduce global scope. Optional. @@ -824,8 +825,17 @@ class QuerySet(object): if limit: mr_args['limit'] = limit - results = self._collection.map_reduce(map_f, reduce_f, output, **mr_args) - results = results.find() + + if output == 'inline' or (not keep_temp and not self._ordering): + map_reduce_function = 'inline_map_reduce' + else: + map_reduce_function = 'map_reduce' + mr_args['out'] = output + + results = getattr(self._collection, map_reduce_function)(map_f, reduce_f, **mr_args) + + if map_reduce_function == 'map_reduce': + results = results.find() if self._ordering: results = results.sort(self._ordering) @@ -1266,7 +1276,7 @@ class QuerySet(object): """ return self.exec_js(average_func, field) - def item_frequencies(self, field, normalize=False): + def item_frequencies(self, field, normalize=False, map_reduce=False): """Returns a dictionary of all items present in a field across the whole queried set of documents, and their corresponding frequency. This is useful for generating tag clouds, or searching documents. @@ -1276,7 +1286,52 @@ class QuerySet(object): :param field: the field to use :param normalize: normalize the results so they add to 1.0 + :param map_reduce: Use map_reduce over exec_js """ + if map_reduce: + return self._item_frequencies_map_reduce(field, normalize=normalize) + return self._item_frequencies_exec_js(field, normalize=normalize) + + def _item_frequencies_map_reduce(self, field, normalize=False): + map_func = """ + function() { + if (this[~%(field)s].constructor == Array) { + this[~%(field)s].forEach(function(item) { + emit(item, 1); + }); + } else { + emit(this[~%(field)s], 1); + } + } + """ % dict(field=field) + reduce_func = """ + function(key, values) { + var total = 0; + var valuesSize = values.length; + for (var i=0; i < valuesSize; i++) { + total += parseInt(values[i], 10); + } + return total; + } + """ + values = self.map_reduce(map_func, reduce_func, 'inline', keep_temp=False) + frequencies = {} + for f in values: + key = f.key + if isinstance(key, float): + if int(key) == key: + key = int(key) + key = str(key) + frequencies[key] = f.value + + if normalize: + count = sum(frequencies.values()) + frequencies = dict([(k, v/count) for k,v in frequencies.items()]) + + return frequencies + + def _item_frequencies_exec_js(self, field, normalize=False): + """Uses exec_js to execute""" freq_func = """ function(field) { if (options.normalize) { diff --git a/setup.py b/setup.py index e0585b7..0c19d8d 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ def get_version(version_tuple): version = '%s.%s' % (version, version_tuple[2]) return version -# Dirty hack to get version number from monogengine/__init__.py - we can't +# Dirty hack to get version number from monogengine/__init__.py - we can't # import it as it depends on PyMongo and PyMongo isn't installed until this # file is read init = os.path.join(os.path.dirname(__file__), 'mongoengine', '__init__.py') diff --git a/tests/queryset.py b/tests/queryset.py index d5611fd..1f03fbd 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1466,35 +1466,54 @@ class QuerySetTest(unittest.TestCase): BlogPost(hits=2, tags=['music', 'watch']).save() BlogPost(hits=2, tags=['music', 'actors']).save() - f = BlogPost.objects.item_frequencies('tags') - f = dict((key, int(val)) for key, val in f.items()) - self.assertEqual(set(['music', 'film', 'actors', 'watch']), set(f.keys())) - self.assertEqual(f['music'], 3) - self.assertEqual(f['actors'], 2) - self.assertEqual(f['watch'], 2) - self.assertEqual(f['film'], 1) + def test_assertions(f): + f = dict((key, int(val)) for key, val in f.items()) + self.assertEqual(set(['music', 'film', 'actors', 'watch']), set(f.keys())) + self.assertEqual(f['music'], 3) + self.assertEqual(f['actors'], 2) + self.assertEqual(f['watch'], 2) + self.assertEqual(f['film'], 1) + + exec_js = BlogPost.objects.item_frequencies('tags') + map_reduce = BlogPost.objects.item_frequencies('tags', map_reduce=True) + test_assertions(exec_js) + test_assertions(map_reduce) # Ensure query is taken into account - f = BlogPost.objects(hits__gt=1).item_frequencies('tags') - f = dict((key, int(val)) for key, val in f.items()) - self.assertEqual(set(['music', 'actors', 'watch']), set(f.keys())) - self.assertEqual(f['music'], 2) - self.assertEqual(f['actors'], 1) - self.assertEqual(f['watch'], 1) + def test_assertions(f): + f = dict((key, int(val)) for key, val in f.items()) + self.assertEqual(set(['music', 'actors', 'watch']), set(f.keys())) + self.assertEqual(f['music'], 2) + self.assertEqual(f['actors'], 1) + self.assertEqual(f['watch'], 1) + + exec_js = BlogPost.objects(hits__gt=1).item_frequencies('tags') + map_reduce = BlogPost.objects(hits__gt=1).item_frequencies('tags', map_reduce=True) + test_assertions(exec_js) + test_assertions(map_reduce) # Check that normalization works - f = BlogPost.objects.item_frequencies('tags', normalize=True) - self.assertAlmostEqual(f['music'], 3.0/8.0) - self.assertAlmostEqual(f['actors'], 2.0/8.0) - self.assertAlmostEqual(f['watch'], 2.0/8.0) - self.assertAlmostEqual(f['film'], 1.0/8.0) + def test_assertions(f): + self.assertAlmostEqual(f['music'], 3.0/8.0) + self.assertAlmostEqual(f['actors'], 2.0/8.0) + self.assertAlmostEqual(f['watch'], 2.0/8.0) + self.assertAlmostEqual(f['film'], 1.0/8.0) + + exec_js = BlogPost.objects.item_frequencies('tags', normalize=True) + map_reduce = BlogPost.objects.item_frequencies('tags', normalize=True, map_reduce=True) + test_assertions(exec_js) + test_assertions(map_reduce) # Check item_frequencies works for non-list fields - f = BlogPost.objects.item_frequencies('hits') - f = dict((key, int(val)) for key, val in f.items()) - self.assertEqual(set(['1', '2']), set(f.keys())) - self.assertEqual(f['1'], 1) - self.assertEqual(f['2'], 2) + def test_assertions(f): + self.assertEqual(set(['1', '2']), set(f.keys())) + self.assertEqual(f['1'], 1) + self.assertEqual(f['2'], 2) + + exec_js = BlogPost.objects.item_frequencies('hits') + map_reduce = BlogPost.objects.item_frequencies('hits', map_reduce=True) + test_assertions(exec_js) + test_assertions(map_reduce) BlogPost.drop_collection() From 6f5bd7b0b90eb33760896ba907634013d404b4c8 Mon Sep 17 00:00:00 2001 From: Colin Howe Date: Thu, 26 May 2011 18:54:52 +0100 Subject: [PATCH 0232/1279] Test needs a connection... --- tests/queryset.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 1f03fbd..081ffb3 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -2101,6 +2101,9 @@ class QuerySetTest(unittest.TestCase): class QTest(unittest.TestCase): + def setUp(self): + connect(db='mongoenginetest') + def test_empty_q(self): """Ensure that empty Q objects won't hurt. """ From 1fa47206aa817dc4556e703de9121d69fb8b064c Mon Sep 17 00:00:00 2001 From: Colin Howe Date: Thu, 26 May 2011 19:39:41 +0100 Subject: [PATCH 0233/1279] Support for sparse indexes and omitting types from indexes --- mongoengine/queryset.py | 31 ++++++++++++++++++++++--------- tests/document.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 17a1b0d..68afefc 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -382,15 +382,17 @@ class QuerySet(object): return self @classmethod - def _build_index_spec(cls, doc_cls, key_or_list): + def _build_index_spec(cls, doc_cls, spec): """Build a PyMongo index spec from a MongoEngine index spec. """ - if isinstance(key_or_list, basestring): - key_or_list = [key_or_list] + if isinstance(spec, basestring): + spec = {'fields': [spec]} + if isinstance(spec, (list, tuple)): + spec = {'fields': spec} index_list = [] use_types = doc_cls._meta.get('allow_inheritance', True) - for key in key_or_list: + for key in spec['fields']: # Get direction from + or - direction = pymongo.ASCENDING if key.startswith("-"): @@ -411,10 +413,18 @@ class QuerySet(object): use_types = False # If _types is being used, prepend it to every specified index - if doc_cls._meta.get('allow_inheritance') and use_types: + if (spec.get('types', True) and doc_cls._meta.get('allow_inheritance') + and use_types): index_list.insert(0, ('_types', 1)) - return index_list + spec['fields'] = index_list + + if spec.get('sparse', False) and len(spec['fields']) > 1: + raise ValueError( + 'Sparse indexes can only have one field in them. ' + 'See https://jira.mongodb.org/browse/SERVER-2193') + + return spec def __call__(self, q_obj=None, class_check=True, **query): """Filter the selected documents by calling the @@ -465,9 +475,12 @@ class QuerySet(object): # Ensure document-defined indexes are created if self._document._meta['indexes']: - for key_or_list in self._document._meta['indexes']: - self._collection.ensure_index(key_or_list, - background=background, **index_opts) + for spec in self._document._meta['indexes']: + opts = index_opts.copy() + opts['unique'] = spec.get('unique', False) + opts['sparse'] = spec.get('sparse', False) + self._collection.ensure_index(spec['fields'], + background=background, **opts) # If _types is being used (for polymorphism), it needs an index if '_types' in self._query: diff --git a/tests/document.py b/tests/document.py index fe67312..a812046 100644 --- a/tests/document.py +++ b/tests/document.py @@ -377,6 +377,40 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + + def test_dictionary_indexes(self): + """Ensure that indexes are used when meta[indexes] contains dictionaries + instead of lists. + """ + class BlogPost(Document): + date = DateTimeField(db_field='addDate', default=datetime.now) + category = StringField() + tags = ListField(StringField()) + meta = { + 'indexes': [ + { 'fields': ['-date'], 'unique': True, + 'sparse': True, 'types': False }, + ], + } + + BlogPost.drop_collection() + + info = BlogPost.objects._collection.index_information() + # _id, '-date' + self.assertEqual(len(info), 3) + + # Indexes are lazy so use list() to perform query + list(BlogPost.objects) + info = BlogPost.objects._collection.index_information() + info = [(value['key'], + value.get('unique', False), + value.get('sparse', False)) + for key, value in info.iteritems()] + self.assertTrue(([('addDate', -1)], True, True) in info) + + BlogPost.drop_collection() + + def test_unique(self): """Ensure that uniqueness constraints are applied to fields. """ From 5d778648e697651ca681d5347051e6071cfe8487 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 May 2011 11:33:40 +0100 Subject: [PATCH 0234/1279] Inital tests for dereferencing improvements --- mongoengine/base.py | 1 + mongoengine/fields.py | 215 +++++++++++++++++++++++-------- mongoengine/tests.py | 58 +++++++++ tests/dereference.py | 288 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 512 insertions(+), 50 deletions(-) create mode 100644 mongoengine/tests.py create mode 100644 tests/dereference.py diff --git a/mongoengine/base.py b/mongoengine/base.py index ffceb79..4e3154f 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -126,6 +126,7 @@ class BaseField(object): self.validate(value) + class ObjectIdField(BaseField): """An field wrapper around MongoDB's ObjectIds. """ diff --git a/mongoengine/fields.py b/mongoengine/fields.py index b2aab5a..c21829c 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -337,33 +337,54 @@ class ListField(BaseField): # Document class being used rather than a document object return self - if isinstance(self.field, ReferenceField): - referenced_type = self.field.document_type - # Get value from document instance if available - value_list = instance._data.get(self.name) - if value_list: - deref_list = [] - for value in value_list: - # Dereference DBRefs - if isinstance(value, (pymongo.dbref.DBRef)): - value = _get_db().dereference(value) - deref_list.append(referenced_type._from_son(value)) - else: - deref_list.append(value) - instance._data[self.name] = deref_list + # Get value from document instance if available + value_list = instance._data.get(self.name) + if isinstance(self.field, ReferenceField) and value_list: + db = _get_db() + value_list = [(k,v) for k,v in enumerate(value_list)] + deref_list = [] + collections = {} - if isinstance(self.field, GenericReferenceField): - value_list = instance._data.get(self.name) - if value_list: - deref_list = [] - for value in value_list: - # Dereference DBRefs - if isinstance(value, (dict, pymongo.son.SON)): - deref_list.append(self.field.dereference(value)) - else: - deref_list.append(value) - instance._data[self.name] = deref_list + for k, v in value_list: + deref_list.append(v) + # Save any DBRefs + if isinstance(v, (pymongo.dbref.DBRef)): + collections.setdefault(v.collection, []).append((k, v)) + # For each collection get the references + for collection, dbrefs in collections.items(): + id_map = dict([(v.id, k) for k, v in dbrefs]) + references = db[collection].find({'_id': {'$in': id_map.keys()}}) + for ref in references: + key = id_map[ref['_id']] + deref_list[key] = get_document(ref['_cls'])._from_son(ref) + instance._data[self.name] = deref_list + + # Get value from document instance if available + if isinstance(self.field, GenericReferenceField) and value_list: + + db = _get_db() + value_list = [(k,v) for k,v in enumerate(value_list)] + deref_list = [] + classes = {} + + for k, v in value_list: + deref_list.append(v) + # Save any DBRefs + if isinstance(v, (dict, pymongo.son.SON)): + classes.setdefault(v['_cls'], []).append((k, v)) + + # For each collection get the references + for doc_cls, dbrefs in classes.items(): + id_map = dict([(v['_ref'].id, k) for k, v in dbrefs]) + doc_cls = get_document(doc_cls) + collection = doc_cls._meta['collection'] + references = db[collection].find({'_id': {'$in': id_map.keys()}}) + + for ref in references: + key = id_map[ref['_id']] + deref_list[key] = doc_cls._from_son(ref) + instance._data[self.name] = deref_list return super(ListField, self).__get__(instance, owner) def to_python(self, value): @@ -501,32 +522,53 @@ class MapField(BaseField): # Document class being used rather than a document object return self - if isinstance(self.field, ReferenceField): - referenced_type = self.field.document_type - # Get value from document instance if available - value_dict = instance._data.get(self.name) - if value_dict: - deref_dict = [] - for key,value in value_dict.iteritems(): - # Dereference DBRefs - if isinstance(value, (pymongo.dbref.DBRef)): - value = _get_db().dereference(value) - deref_dict[key] = referenced_type._from_son(value) - else: - deref_dict[key] = value - instance._data[self.name] = deref_dict + # Get value from document instance if available + value_list = instance._data.get(self.name) + if isinstance(self.field, ReferenceField) and value_list: + db = _get_db() + deref_dict = {} + collections = {} - if isinstance(self.field, GenericReferenceField): - value_dict = instance._data.get(self.name) - if value_dict: - deref_dict = [] - for key,value in value_dict.iteritems(): - # Dereference DBRefs - if isinstance(value, (dict, pymongo.son.SON)): - deref_dict[key] = self.field.dereference(value) - else: - deref_dict[key] = value - instance._data[self.name] = deref_dict + for k, v in value_list.items(): + deref_dict[k] = v + # Save any DBRefs + if isinstance(v, (pymongo.dbref.DBRef)): + collections.setdefault(v.collection, []).append((k, v)) + + # For each collection get the references + for collection, dbrefs in collections.items(): + id_map = dict([(v.id, k) for k, v in dbrefs]) + references = db[collection].find({'_id': {'$in': id_map.keys()}}) + for ref in references: + key = id_map[ref['_id']] + deref_dict[key] = get_document(ref['_cls'])._from_son(ref) + instance._data[self.name] = deref_dict + + # Get value from document instance if available + if isinstance(self.field, GenericReferenceField) and value_list: + + db = _get_db() + value_list = [(k,v) for k,v in value_list.items()] + deref_dict = {} + classes = {} + + for k, v in value_list: + deref_dict[k] = v + # Save any DBRefs + if isinstance(v, (dict, pymongo.son.SON)): + classes.setdefault(v['_cls'], []).append((k, v)) + + # For each collection get the references + for doc_cls, dbrefs in classes.items(): + id_map = dict([(v['_ref'].id, k) for k, v in dbrefs]) + doc_cls = get_document(doc_cls) + collection = doc_cls._meta['collection'] + references = db[collection].find({'_id': {'$in': id_map.keys()}}) + + for ref in references: + key = id_map[ref['_id']] + deref_dict[key] = doc_cls._from_son(ref) + instance._data[self.name] = deref_dict return super(MapField, self).__get__(instance, owner) @@ -869,3 +911,76 @@ class GeoPointField(BaseField): if (not isinstance(value[0], (float, int)) and not isinstance(value[1], (float, int))): raise ValidationError('Both values in point must be float or int.') + + + +class DereferenceMixin(object): + """ WORK IN PROGRESS""" + + def __get__(self, instance, owner): + """Descriptor to automatically dereference references. + """ + if instance is None: + # Document class being used rather than a document object + return self + + # Get value from document instance if available + value_list = instance._data.get(self.name) + if not value_list: + return super(MapField, self).__get__(instance, owner) + + is_dict = True + if not hasattr(value_list, 'items'): + is_dict = False + value_list = dict([(k,v) for k,v in enumerate(value_list)]) + + if isinstance(self.field, ReferenceField) and value_list: + db = _get_db() + dbref = {} + if not is_dict: + dbref = [] + collections = {} + + for k, v in value_list.items(): + dbref[k] = v + # Save any DBRefs + if isinstance(v, (pymongo.dbref.DBRef)): + collections.setdefault(v.collection, []).append((k, v)) + + # For each collection get the references + for collection, dbrefs in collections.items(): + id_map = dict([(v.id, k) for k, v in dbrefs]) + references = db[collection].find({'_id': {'$in': id_map.keys()}}) + for ref in references: + key = id_map[ref['_id']] + dbref[key] = get_document(ref['_cls'])._from_son(ref) + + instance._data[self.name] = dbref + + # Get value from document instance if available + if isinstance(self.field, GenericReferenceField) and value_list: + + db = _get_db() + value_list = [(k,v) for k,v in value_list.items()] + dbref = {} + classes = {} + + for k, v in value_list: + dbref[k] = v + # Save any DBRefs + if isinstance(v, (dict, pymongo.son.SON)): + classes.setdefault(v['_cls'], []).append((k, v)) + + # For each collection get the references + for doc_cls, dbrefs in classes.items(): + id_map = dict([(v['_ref'].id, k) for k, v in dbrefs]) + doc_cls = get_document(doc_cls) + collection = doc_cls._meta['collection'] + references = db[collection].find({'_id': {'$in': id_map.keys()}}) + + for ref in references: + key = id_map[ref['_id']] + dbref[key] = doc_cls._from_son(ref) + instance._data[self.name] = dbref + + return super(DereferenceField, self).__get__(instance, owner) \ No newline at end of file diff --git a/mongoengine/tests.py b/mongoengine/tests.py new file mode 100644 index 0000000..4932bc2 --- /dev/null +++ b/mongoengine/tests.py @@ -0,0 +1,58 @@ +from mongoengine.connection import _get_db + +class query_counter(object): + """ Query_counter contextmanager to get the number of queries. """ + + def __init__(self): + """ Construct the query_counter. """ + self.counter = 0 + self.db = _get_db() + + def __enter__(self): + """ On every with block we need to drop the profile collection. """ + self.db.set_profiling_level(0) + self.db.system.profile.drop() + self.db.set_profiling_level(2) + return self + + def __exit__(self, t, value, traceback): + """ Reset the profiling level. """ + self.db.set_profiling_level(0) + + def __eq__(self, value): + """ == Compare querycounter. """ + return value == self._get_count() + + def __ne__(self, value): + """ != Compare querycounter. """ + return not self.__eq__(value) + + def __lt__(self, value): + """ < Compare querycounter. """ + return self._get_count() < value + + def __le__(self, value): + """ <= Compare querycounter. """ + return self._get_count() <= value + + def __gt__(self, value): + """ > Compare querycounter. """ + return self._get_count() > value + + def __ge__(self, value): + """ >= Compare querycounter. """ + return self._get_count() >= value + + def __int__(self): + """ int representation. """ + return self._get_count() + + def __repr__(self): + """ repr query_counter as the number of queries. """ + return u"%s" % self._get_count() + + def _get_count(self): + """ Get the number of queries. """ + count = self.db.system.profile.find().count() - self.counter + self.counter += 1 + return count diff --git a/tests/dereference.py b/tests/dereference.py new file mode 100644 index 0000000..2764ee7 --- /dev/null +++ b/tests/dereference.py @@ -0,0 +1,288 @@ +import unittest + +from mongoengine import * +from mongoengine.connection import _get_db +from mongoengine.tests import query_counter + + +class FieldTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = _get_db() + + def ztest_list_item_dereference(self): + """Ensure that DBRef items in ListFields are dereferenced. + """ + class User(Document): + name = StringField() + + class Group(Document): + members = ListField(ReferenceField(User)) + + User.drop_collection() + Group.drop_collection() + + for i in xrange(1, 51): + user = User(name='user %s' % i) + user.save() + + group = Group(members=User.objects) + group.save() + + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first() + self.assertEqual(q, 1) + + [m for m in group_obj.members] + self.assertEqual(q, 2) + + User.drop_collection() + Group.drop_collection() + + def ztest_recursive_reference(self): + """Ensure that ReferenceFields can reference their own documents. + """ + class Employee(Document): + name = StringField() + boss = ReferenceField('self') + friends = ListField(ReferenceField('self')) + + bill = Employee(name='Bill Lumbergh') + bill.save() + + michael = Employee(name='Michael Bolton') + michael.save() + + samir = Employee(name='Samir Nagheenanajar') + samir.save() + + friends = [michael, samir] + peter = Employee(name='Peter Gibbons', boss=bill, friends=friends) + peter.save() + + with query_counter() as q: + self.assertEqual(q, 0) + + peter = Employee.objects.with_id(peter.id) + self.assertEqual(q, 1) + + peter.boss + self.assertEqual(q, 2) + + peter.friends + self.assertEqual(q, 3) + + def ztest_generic_reference(self): + + class UserA(Document): + name = StringField() + + class UserB(Document): + name = StringField() + + class UserC(Document): + name = StringField() + + class Group(Document): + members = ListField(GenericReferenceField()) + + UserA.drop_collection() + UserB.drop_collection() + UserC.drop_collection() + Group.drop_collection() + + members = [] + for i in xrange(1, 51): + a = UserA(name='User A %s' % i) + a.save() + + b = UserB(name='User B %s' % i) + b.save() + + c = UserC(name='User C %s' % i) + c.save() + + members += [a, b, c] + + group = Group(members=members) + group.save() + + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first() + self.assertEqual(q, 1) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + UserA.drop_collection() + UserB.drop_collection() + UserC.drop_collection() + Group.drop_collection() + + def test_map_field_reference(self): + + class User(Document): + name = StringField() + + class Group(Document): + members = MapField(ReferenceField(User)) + + User.drop_collection() + Group.drop_collection() + + members = [] + for i in xrange(1, 51): + user = User(name='user %s' % i) + user.save() + members.append(user) + + group = Group(members=dict([(str(u.id), u) for u in members])) + group.save() + + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first() + self.assertEqual(q, 1) + + [m for m in group_obj.members] + self.assertEqual(q, 2) + + User.drop_collection() + Group.drop_collection() + + def ztest_generic_reference_dict_field(self): + + class UserA(Document): + name = StringField() + + class UserB(Document): + name = StringField() + + class UserC(Document): + name = StringField() + + class Group(Document): + members = DictField() + + UserA.drop_collection() + UserB.drop_collection() + UserC.drop_collection() + Group.drop_collection() + + members = [] + for i in xrange(1, 51): + a = UserA(name='User A %s' % i) + a.save() + + b = UserB(name='User B %s' % i) + b.save() + + c = UserC(name='User C %s' % i) + c.save() + + members += [a, b, c] + + group = Group(members=dict([(str(u.id), u) for u in members])) + group.save() + + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first() + self.assertEqual(q, 1) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + group.members = {} + group.save() + + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first() + self.assertEqual(q, 1) + + [m for m in group_obj.members] + self.assertEqual(q, 1) + + UserA.drop_collection() + UserB.drop_collection() + UserC.drop_collection() + Group.drop_collection() + + def test_generic_reference_map_field(self): + + class UserA(Document): + name = StringField() + + class UserB(Document): + name = StringField() + + class UserC(Document): + name = StringField() + + class Group(Document): + members = MapField(GenericReferenceField()) + + UserA.drop_collection() + UserB.drop_collection() + UserC.drop_collection() + Group.drop_collection() + + members = [] + for i in xrange(1, 51): + a = UserA(name='User A %s' % i) + a.save() + + b = UserB(name='User B %s' % i) + b.save() + + c = UserC(name='User C %s' % i) + c.save() + + members += [a, b, c] + + group = Group(members=dict([(str(u.id), u) for u in members])) + group.save() + + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first() + self.assertEqual(q, 1) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + group.members = {} + group.save() + + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first() + self.assertEqual(q, 1) + + [m for m in group_obj.members] + self.assertEqual(q, 1) + + UserA.drop_collection() + UserB.drop_collection() + UserC.drop_collection() + Group.drop_collection() \ No newline at end of file From 40df08c74c44546fd04f23f1cba4da0f5f162d0e Mon Sep 17 00:00:00 2001 From: Colin Howe Date: Sun, 29 May 2011 13:33:00 +0100 Subject: [PATCH 0235/1279] Fix QuerySet.ensure_index for new index specs --- mongoengine/queryset.py | 10 +++++++--- tests/queryset.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 68afefc..2de15ed 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -376,9 +376,13 @@ class QuerySet(object): construct a multi-field index); keys may be prefixed with a **+** or a **-** to determine the index ordering """ - index_list = QuerySet._build_index_spec(self._document, key_or_list) - self._collection.ensure_index(index_list, drop_dups=drop_dups, - background=background) + index_spec = QuerySet._build_index_spec(self._document, key_or_list) + self._collection.ensure_index( + index_spec['fields'], + drop_dups=drop_dups, + background=background, + sparse=index_spec.get('sparse', False), + unique=index_spec.get('unique', False)) return self @classmethod diff --git a/tests/queryset.py b/tests/queryset.py index 081ffb3..8d04690 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -2099,6 +2099,22 @@ class QuerySetTest(unittest.TestCase): Number.drop_collection() + def test_ensure_index(self): + """Ensure that manual creation of indexes works. + """ + class Comment(Document): + message = StringField() + + Comment.objects.ensure_index('message') + + info = Comment.objects._collection.index_information() + info = [(value['key'], + value.get('unique', False), + value.get('sparse', False)) + for key, value in info.iteritems()] + self.assertTrue(([('_types', 1), ('message', 1)], False, False) in info) + + class QTest(unittest.TestCase): def setUp(self): From 9a2cf206b22f7e9697b5e2d7ea47d37230f68206 Mon Sep 17 00:00:00 2001 From: Colin Howe Date: Sun, 29 May 2011 13:38:54 +0100 Subject: [PATCH 0236/1279] Documentation for new-style indices --- docs/guide/defining-documents.rst | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index e333674..a524520 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -341,9 +341,10 @@ Indexes You can specify indexes on collections to make querying faster. This is done by creating a list of index specifications called :attr:`indexes` in the :attr:`~mongoengine.Document.meta` dictionary, where an index specification may -either be a single field name, or a tuple containing multiple field names. A -direction may be specified on fields by prefixing the field name with a **+** -or a **-** sign. Note that direction only matters on multi-field indexes. :: +either be a single field name, a tuple containing multiple field names, or a +dictionary containing a full index definition. A direction may be specified on +fields by prefixing the field name with a **+** or a **-** sign. Note that +direction only matters on multi-field indexes. :: class Page(Document): title = StringField() @@ -352,6 +353,21 @@ or a **-** sign. Note that direction only matters on multi-field indexes. :: 'indexes': ['title', ('title', '-rating')] } +If a dictionary is passed then the following options are available: + +:attr:`fields` (Default: None) + The fields to index. Specified in the same format as described above. + +:attr:`types` (Default: True) + Whether the index should have the :attr:`_types` field added automatically + to the start of the index. + +:attr:`sparse` (Default: False) + Whether the index should be sparse. + +:attr:`unique` (Default: False) + Whether the index should be sparse. + .. note:: Geospatial indexes will be automatically created for all :class:`~mongoengine.GeoPointField`\ s From ec7effa0ef8c3a71d1f8dd0695639f60763b9858 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 6 Jun 2011 11:04:06 +0100 Subject: [PATCH 0237/1279] Added DereferenceBaseField class Handles the lazy dereferencing of all items in a list / dict. Improves query efficiency by an order of magnitude. --- mongoengine/base.py | 82 ++++++++++++++++ mongoengine/fields.py | 223 ++++-------------------------------------- tests/dereference.py | 6 +- 3 files changed, 104 insertions(+), 207 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 4e3154f..ce61547 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -5,6 +5,7 @@ from queryset import DO_NOTHING import sys import pymongo import pymongo.objectid +from operator import itemgetter class NotRegistered(Exception): @@ -127,6 +128,87 @@ class BaseField(object): self.validate(value) +class DereferenceBaseField(BaseField): + """Handles the lazy dereferencing of a queryset. Will dereference all + items in a list / dict rather than one at a time. + """ + + def __get__(self, instance, owner): + """Descriptor to automatically dereference references. + """ + from fields import ReferenceField, GenericReferenceField + from connection import _get_db + + if instance is None: + # Document class being used rather than a document object + return self + + # Get value from document instance if available + value_list = instance._data.get(self.name) + if not value_list: + return super(DereferenceBaseField, self).__get__(instance, owner) + + is_list = False + if not hasattr(value_list, 'items'): + is_list = True + value_list = dict([(k,v) for k,v in enumerate(value_list)]) + + if isinstance(self.field, ReferenceField) and value_list: + db = _get_db() + dbref = {} + collections = {} + + for k, v in value_list.items(): + dbref[k] = v + # Save any DBRefs + if isinstance(v, (pymongo.dbref.DBRef)): + collections.setdefault(v.collection, []).append((k, v)) + + # For each collection get the references + for collection, dbrefs in collections.items(): + id_map = dict([(v.id, k) for k, v in dbrefs]) + references = db[collection].find({'_id': {'$in': id_map.keys()}}) + for ref in references: + key = id_map[ref['_id']] + dbref[key] = get_document(ref['_cls'])._from_son(ref) + + if is_list: + dbref = [v for k,v in sorted(dbref.items(), key=itemgetter(0))] + instance._data[self.name] = dbref + + # Get value from document instance if available + if isinstance(self.field, GenericReferenceField) and value_list: + db = _get_db() + value_list = [(k,v) for k,v in value_list.items()] + dbref = {} + classes = {} + + for k, v in value_list: + dbref[k] = v + # Save any DBRefs + if isinstance(v, (dict, pymongo.son.SON)): + classes.setdefault(v['_cls'], []).append((k, v)) + + # For each collection get the references + for doc_cls, dbrefs in classes.items(): + id_map = dict([(v['_ref'].id, k) for k, v in dbrefs]) + doc_cls = get_document(doc_cls) + collection = doc_cls._meta['collection'] + references = db[collection].find({'_id': {'$in': id_map.keys()}}) + + for ref in references: + key = id_map[ref['_id']] + dbref[key] = doc_cls._from_son(ref) + + if is_list: + dbref = [v for k,v in sorted(dbref.items(), key=itemgetter(0))] + + instance._data[self.name] = dbref + + return super(DereferenceBaseField, self).__get__(instance, owner) + + + class ObjectIdField(BaseField): """An field wrapper around MongoDB's ObjectIds. """ diff --git a/mongoengine/fields.py b/mongoengine/fields.py index c21829c..dc03fc0 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1,4 +1,5 @@ -from base import BaseField, ObjectIdField, ValidationError, get_document +from base import (BaseField, DereferenceBaseField, ObjectIdField, + ValidationError, get_document) from queryset import DO_NOTHING from document import Document, EmbeddedDocument from connection import _get_db @@ -12,7 +13,6 @@ import pymongo.binary import datetime, time import decimal import gridfs -import warnings __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', @@ -118,8 +118,8 @@ class EmailField(StringField): EMAIL_REGEX = re.compile( r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*" # dot-atom - r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"' # quoted-string - r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$', re.IGNORECASE # domain + r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"' # quoted-string + r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$', re.IGNORECASE # domain ) def validate(self, value): @@ -153,6 +153,7 @@ class IntField(BaseField): def prepare_query_value(self, op, value): return int(value) + class FloatField(BaseField): """An floating point number field. """ @@ -178,6 +179,7 @@ class FloatField(BaseField): def prepare_query_value(self, op, value): return float(value) + class DecimalField(BaseField): """A fixed-point decimal number field. @@ -252,21 +254,21 @@ class DateTimeField(BaseField): else: usecs = 0 kwargs = {'microsecond': usecs} - try: # Seconds are optional, so try converting seconds first. + try: # Seconds are optional, so try converting seconds first. return datetime.datetime(*time.strptime(value, '%Y-%m-%d %H:%M:%S')[:6], **kwargs) - except ValueError: - try: # Try without seconds. + try: # Try without seconds. return datetime.datetime(*time.strptime(value, '%Y-%m-%d %H:%M')[:5], **kwargs) - except ValueError: # Try without hour/minutes/seconds. + except ValueError: # Try without hour/minutes/seconds. try: return datetime.datetime(*time.strptime(value, '%Y-%m-%d')[:3], **kwargs) except ValueError: return None + class EmbeddedDocumentField(BaseField): """An embedded document field. Only valid values are subclasses of :class:`~mongoengine.EmbeddedDocument`. @@ -314,7 +316,7 @@ class EmbeddedDocumentField(BaseField): return self.to_mongo(value) -class ListField(BaseField): +class ListField(DereferenceBaseField): """A list field that wraps a standard field, allowing multiple instances of the field to be used as a list in the database. """ @@ -330,63 +332,6 @@ class ListField(BaseField): kwargs.setdefault('default', lambda: []) super(ListField, self).__init__(**kwargs) - def __get__(self, instance, owner): - """Descriptor to automatically dereference references. - """ - if instance is None: - # Document class being used rather than a document object - return self - - # Get value from document instance if available - value_list = instance._data.get(self.name) - if isinstance(self.field, ReferenceField) and value_list: - db = _get_db() - value_list = [(k,v) for k,v in enumerate(value_list)] - deref_list = [] - collections = {} - - for k, v in value_list: - deref_list.append(v) - # Save any DBRefs - if isinstance(v, (pymongo.dbref.DBRef)): - collections.setdefault(v.collection, []).append((k, v)) - - # For each collection get the references - for collection, dbrefs in collections.items(): - id_map = dict([(v.id, k) for k, v in dbrefs]) - references = db[collection].find({'_id': {'$in': id_map.keys()}}) - for ref in references: - key = id_map[ref['_id']] - deref_list[key] = get_document(ref['_cls'])._from_son(ref) - instance._data[self.name] = deref_list - - # Get value from document instance if available - if isinstance(self.field, GenericReferenceField) and value_list: - - db = _get_db() - value_list = [(k,v) for k,v in enumerate(value_list)] - deref_list = [] - classes = {} - - for k, v in value_list: - deref_list.append(v) - # Save any DBRefs - if isinstance(v, (dict, pymongo.son.SON)): - classes.setdefault(v['_cls'], []).append((k, v)) - - # For each collection get the references - for doc_cls, dbrefs in classes.items(): - id_map = dict([(v['_ref'].id, k) for k, v in dbrefs]) - doc_cls = get_document(doc_cls) - collection = doc_cls._meta['collection'] - references = db[collection].find({'_id': {'$in': id_map.keys()}}) - - for ref in references: - key = id_map[ref['_id']] - deref_list[key] = doc_cls._from_son(ref) - instance._data[self.name] = deref_list - return super(ListField, self).__get__(instance, owner) - def to_python(self, value): return [self.field.to_python(item) for item in value] @@ -480,10 +425,10 @@ class DictField(BaseField): if op in match_operators and isinstance(value, basestring): return StringField().prepare_query_value(op, value) - return super(DictField,self).prepare_query_value(op, value) + return super(DictField, self).prepare_query_value(op, value) -class MapField(BaseField): +class MapField(DereferenceBaseField): """A field that maps a name to a specified field type. Similar to a DictField, except the 'value' of each item must match the specified field type. @@ -515,68 +460,11 @@ class MapField(BaseField): except Exception, err: raise ValidationError('Invalid MapField item (%s)' % str(item)) - def __get__(self, instance, owner): - """Descriptor to automatically dereference references. - """ - if instance is None: - # Document class being used rather than a document object - return self - - # Get value from document instance if available - value_list = instance._data.get(self.name) - if isinstance(self.field, ReferenceField) and value_list: - db = _get_db() - deref_dict = {} - collections = {} - - for k, v in value_list.items(): - deref_dict[k] = v - # Save any DBRefs - if isinstance(v, (pymongo.dbref.DBRef)): - collections.setdefault(v.collection, []).append((k, v)) - - # For each collection get the references - for collection, dbrefs in collections.items(): - id_map = dict([(v.id, k) for k, v in dbrefs]) - references = db[collection].find({'_id': {'$in': id_map.keys()}}) - for ref in references: - key = id_map[ref['_id']] - deref_dict[key] = get_document(ref['_cls'])._from_son(ref) - instance._data[self.name] = deref_dict - - # Get value from document instance if available - if isinstance(self.field, GenericReferenceField) and value_list: - - db = _get_db() - value_list = [(k,v) for k,v in value_list.items()] - deref_dict = {} - classes = {} - - for k, v in value_list: - deref_dict[k] = v - # Save any DBRefs - if isinstance(v, (dict, pymongo.son.SON)): - classes.setdefault(v['_cls'], []).append((k, v)) - - # For each collection get the references - for doc_cls, dbrefs in classes.items(): - id_map = dict([(v['_ref'].id, k) for k, v in dbrefs]) - doc_cls = get_document(doc_cls) - collection = doc_cls._meta['collection'] - references = db[collection].find({'_id': {'$in': id_map.keys()}}) - - for ref in references: - key = id_map[ref['_id']] - deref_dict[key] = doc_cls._from_son(ref) - instance._data[self.name] = deref_dict - - return super(MapField, self).__get__(instance, owner) - def to_python(self, value): - return dict( [(key,self.field.to_python(item)) for key,item in value.iteritems()] ) + return dict([(key, self.field.to_python(item)) for key, item in value.iteritems()]) def to_mongo(self, value): - return dict( [(key,self.field.to_mongo(item)) for key,item in value.iteritems()] ) + return dict([(key, self.field.to_mongo(item)) for key, item in value.iteritems()]) def prepare_query_value(self, op, value): if op not in ('set', 'unset'): @@ -794,11 +682,11 @@ class GridFSProxy(object): self.newfile = self.fs.new_file(**kwargs) self.grid_id = self.newfile._id - def put(self, file, **kwargs): + def put(self, file_obj, **kwargs): if self.grid_id: raise GridFSError('This document already has a file. Either delete ' 'it or call replace to overwrite it') - self.grid_id = self.fs.put(file, **kwargs) + self.grid_id = self.fs.put(file_obj, **kwargs) def write(self, string): if self.grid_id: @@ -827,9 +715,9 @@ class GridFSProxy(object): self.grid_id = None self.gridout = None - def replace(self, file, **kwargs): + def replace(self, file_obj, **kwargs): self.delete() - self.put(file, **kwargs) + self.put(file_obj, **kwargs) def close(self): if self.newfile: @@ -911,76 +799,3 @@ class GeoPointField(BaseField): if (not isinstance(value[0], (float, int)) and not isinstance(value[1], (float, int))): raise ValidationError('Both values in point must be float or int.') - - - -class DereferenceMixin(object): - """ WORK IN PROGRESS""" - - def __get__(self, instance, owner): - """Descriptor to automatically dereference references. - """ - if instance is None: - # Document class being used rather than a document object - return self - - # Get value from document instance if available - value_list = instance._data.get(self.name) - if not value_list: - return super(MapField, self).__get__(instance, owner) - - is_dict = True - if not hasattr(value_list, 'items'): - is_dict = False - value_list = dict([(k,v) for k,v in enumerate(value_list)]) - - if isinstance(self.field, ReferenceField) and value_list: - db = _get_db() - dbref = {} - if not is_dict: - dbref = [] - collections = {} - - for k, v in value_list.items(): - dbref[k] = v - # Save any DBRefs - if isinstance(v, (pymongo.dbref.DBRef)): - collections.setdefault(v.collection, []).append((k, v)) - - # For each collection get the references - for collection, dbrefs in collections.items(): - id_map = dict([(v.id, k) for k, v in dbrefs]) - references = db[collection].find({'_id': {'$in': id_map.keys()}}) - for ref in references: - key = id_map[ref['_id']] - dbref[key] = get_document(ref['_cls'])._from_son(ref) - - instance._data[self.name] = dbref - - # Get value from document instance if available - if isinstance(self.field, GenericReferenceField) and value_list: - - db = _get_db() - value_list = [(k,v) for k,v in value_list.items()] - dbref = {} - classes = {} - - for k, v in value_list: - dbref[k] = v - # Save any DBRefs - if isinstance(v, (dict, pymongo.son.SON)): - classes.setdefault(v['_cls'], []).append((k, v)) - - # For each collection get the references - for doc_cls, dbrefs in classes.items(): - id_map = dict([(v['_ref'].id, k) for k, v in dbrefs]) - doc_cls = get_document(doc_cls) - collection = doc_cls._meta['collection'] - references = db[collection].find({'_id': {'$in': id_map.keys()}}) - - for ref in references: - key = id_map[ref['_id']] - dbref[key] = doc_cls._from_son(ref) - instance._data[self.name] = dbref - - return super(DereferenceField, self).__get__(instance, owner) \ No newline at end of file diff --git a/tests/dereference.py b/tests/dereference.py index 2764ee7..b6cee89 100644 --- a/tests/dereference.py +++ b/tests/dereference.py @@ -11,7 +11,7 @@ class FieldTest(unittest.TestCase): connect(db='mongoenginetest') self.db = _get_db() - def ztest_list_item_dereference(self): + def test_list_item_dereference(self): """Ensure that DBRef items in ListFields are dereferenced. """ class User(Document): @@ -42,7 +42,7 @@ class FieldTest(unittest.TestCase): User.drop_collection() Group.drop_collection() - def ztest_recursive_reference(self): + def test_recursive_reference(self): """Ensure that ReferenceFields can reference their own documents. """ class Employee(Document): @@ -75,7 +75,7 @@ class FieldTest(unittest.TestCase): peter.friends self.assertEqual(q, 3) - def ztest_generic_reference(self): + def test_generic_reference(self): class UserA(Document): name = StringField() From 7312db5c252bf3c395357cba3b7254cdccd1c6c0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 6 Jun 2011 11:07:27 +0100 Subject: [PATCH 0238/1279] Updated docs / authors. Thanks @jorgebastida for the awesome query_counter test context manager. --- AUTHORS | 1 + docs/changelog.rst | 2 ++ mongoengine/tests.py | 1 + 3 files changed, 4 insertions(+) diff --git a/AUTHORS b/AUTHORS index 93fe819..aecdcaa 100644 --- a/AUTHORS +++ b/AUTHORS @@ -3,3 +3,4 @@ Matt Dennewitz Deepak Thukral Florian Schlachter Steve Challis +Ross Lawley diff --git a/docs/changelog.rst b/docs/changelog.rst index 686b326..58da0d9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,8 @@ Changelog Changes in dev ============== +- Added query_counter context manager for tests +- Added DereferenceBaseField - for improved performance in field dereferencing - Added optional map_reduce method item_frequencies - Added inline_map_reduce option to map_reduce - Updated connection exception so it provides more info on the cause. diff --git a/mongoengine/tests.py b/mongoengine/tests.py index 4932bc2..9584bc7 100644 --- a/mongoengine/tests.py +++ b/mongoengine/tests.py @@ -1,5 +1,6 @@ from mongoengine.connection import _get_db + class query_counter(object): """ Query_counter contextmanager to get the number of queries. """ From 0e4507811611b80be6529f2376c5e3e9b4d5bdef Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 6 Jun 2011 11:34:43 +0100 Subject: [PATCH 0239/1279] Added Blinker signal support --- docs/changelog.rst | 1 + docs/guide/index.rst | 1 + mongoengine/__init__.py | 4 +++- mongoengine/signals.py | 3 +++ 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 58da0d9..659bdb4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added blinker signal support - Added query_counter context manager for tests - Added DereferenceBaseField - for improved performance in field dereferencing - Added optional map_reduce method item_frequencies diff --git a/docs/guide/index.rst b/docs/guide/index.rst index aac7246..d56e747 100644 --- a/docs/guide/index.rst +++ b/docs/guide/index.rst @@ -11,3 +11,4 @@ User Guide document-instances querying gridfs + signals diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 6d18ffe..de635f9 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -6,9 +6,11 @@ import connection from connection import * import queryset from queryset import * +import signals +from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + - queryset.__all__) + queryset.__all__ + signals.__all__) __author__ = 'Harry Marr' diff --git a/mongoengine/signals.py b/mongoengine/signals.py index 4caa553..0a69753 100644 --- a/mongoengine/signals.py +++ b/mongoengine/signals.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- +__all__ = ['pre_init', 'post_init', 'pre_save', 'post_save', + 'pre_delete', 'post_delete'] + signals_available = False try: from blinker import Namespace From 74b5043ef9441938c6668af6eb510adccc8e531a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 6 Jun 2011 11:39:58 +0100 Subject: [PATCH 0240/1279] Added signals documentation --- docs/guide/signals.rst | 49 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/guide/signals.rst diff --git a/docs/guide/signals.rst b/docs/guide/signals.rst new file mode 100644 index 0000000..d80a421 --- /dev/null +++ b/docs/guide/signals.rst @@ -0,0 +1,49 @@ +.. _signals: + +Signals +======= + +.. versionadded:: 0.5 + +Signal support is provided by the excellent `blinker`_ library and +will gracefully fall back if it is not available. + + +The following document signals exist in MongoEngine and are pretty self explaintary: + + * `mongoengine.signals.pre_init` + * `mongoengine.signals.post_init` + * `mongoengine.signals.pre_save` + * `mongoengine.signals.post_save` + * `mongoengine.signals.pre_delete` + * `mongoengine.signals.post_delete` + +Example usage:: + + from mongoengine import * + from mongoengine import signals + + class Author(Document): + name = StringField() + + def __unicode__(self): + return self.name + + @classmethod + def pre_save(cls, instance, **kwargs): + logging.debug("Pre Save: %s" % instance.name) + + @classmethod + def post_save(cls, instance, **kwargs): + logging.debug("Post Save: %s" % instance.name) + if 'created' in kwargs: + if kwargs['created']: + logging.debug("Created") + else: + logging.debug("Updated") + + signals.pre_save.connect(Author.pre_save) + signals.post_save.connect(Author.post_save) + + +.. _blinker: http://pypi.python.org/pypi/blinker \ No newline at end of file From 56f00a64d77655bee2d00ebd783d07655a6900ff Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 6 Jun 2011 12:37:06 +0100 Subject: [PATCH 0241/1279] Added bulk insert method. Updated changelog and added tests / query_counter tests --- docs/changelog.rst | 1 + mongoengine/queryset.py | 42 ++++++++++++++++++++- tests/queryset.py | 83 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 659bdb4..29ecdf7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added insert method for bulk inserts - Added blinker signal support - Added query_counter context manager for tests - Added DereferenceBaseField - for improved performance in field dereferencing diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 2de15ed..0e87db7 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -378,7 +378,7 @@ class QuerySet(object): """ index_spec = QuerySet._build_index_spec(self._document, key_or_list) self._collection.ensure_index( - index_spec['fields'], + index_spec['fields'], drop_dups=drop_dups, background=background, sparse=index_spec.get('sparse', False), @@ -719,6 +719,46 @@ class QuerySet(object): result = None return result + def insert(self, doc_or_docs, load_bulk=True): + """bulk insert documents + + :param docs_or_doc: a document or list of documents to be inserted + :param load_bulk (optional): If True returns the list of document instances + + By default returns document instances, set ``load_bulk`` to False to + return just ``ObjectIds`` + + .. versionadded:: 0.5 + """ + from document import Document + + docs = doc_or_docs + return_one = False + if isinstance(docs, Document) or issubclass(docs.__class__, Document): + return_one = True + docs = [docs] + + raw = [] + for doc in docs: + if not isinstance(doc, self._document): + msg = "Some documents inserted aren't instances of %s" % str(self._document) + raise OperationError(msg) + if doc.pk: + msg = "Some documents have ObjectIds use doc.update() instead" + raise OperationError(msg) + raw.append(doc.to_mongo()) + + ids = self._collection.insert(raw) + + if not load_bulk: + return return_one and ids[0] or ids + + documents = self.in_bulk(ids) + results = [] + for obj_id in ids: + results.append(documents.get(obj_id)) + return return_one and results[0] or results + def with_id(self, object_id): """Retrieve the object matching the id provided. diff --git a/tests/queryset.py b/tests/queryset.py index 8d04690..0b64e3e 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -9,6 +9,7 @@ from mongoengine.queryset import (QuerySet, QuerySetManager, MultipleObjectsReturned, DoesNotExist, QueryFieldList) from mongoengine import * +from mongoengine.tests import query_counter class QuerySetTest(unittest.TestCase): @@ -331,6 +332,88 @@ class QuerySetTest(unittest.TestCase): person = self.Person.objects.get(age=50) self.assertEqual(person.name, "User C") + def test_bulk_insert(self): + """Ensure that query by array position works. + """ + + class Comment(EmbeddedDocument): + name = StringField() + + class Post(EmbeddedDocument): + comments = ListField(EmbeddedDocumentField(Comment)) + + class Blog(Document): + title = StringField() + tags = ListField(StringField()) + posts = ListField(EmbeddedDocumentField(Post)) + + Blog.drop_collection() + + with query_counter() as q: + self.assertEqual(q, 0) + + comment1 = Comment(name='testa') + comment2 = Comment(name='testb') + post1 = Post(comments=[comment1, comment2]) + post2 = Post(comments=[comment2, comment2]) + + blogs = [] + for i in xrange(1, 100): + blogs.append(Blog(title="post %s" % i, posts=[post1, post2])) + + Blog.objects.insert(blogs, load_bulk=False) + self.assertEqual(q, 2) # 1 for the inital connection and 1 for the insert + + Blog.objects.insert(blogs) + self.assertEqual(q, 4) # 1 for insert, and 1 for in bulk + + Blog.drop_collection() + + comment1 = Comment(name='testa') + comment2 = Comment(name='testb') + post1 = Post(comments=[comment1, comment2]) + post2 = Post(comments=[comment2, comment2]) + blog1 = Blog(title="code", posts=[post1, post2]) + blog2 = Blog(title="mongodb", posts=[post2, post1]) + blog1, blog2 = Blog.objects.insert([blog1, blog2]) + self.assertEqual(blog1.title, "code") + self.assertEqual(blog2.title, "mongodb") + + self.assertEqual(Blog.objects.count(), 2) + + # test handles people trying to upsert + def throw_operation_error(): + blogs = Blog.objects + Blog.objects.insert(blogs) + + self.assertRaises(OperationError, throw_operation_error) + + # test handles other classes being inserted + def throw_operation_error_wrong_doc(): + class Author(Document): + pass + Blog.objects.insert(Author()) + + self.assertRaises(OperationError, throw_operation_error_wrong_doc) + + def throw_operation_error_not_a_document(): + Blog.objects.insert("HELLO WORLD") + + self.assertRaises(OperationError, throw_operation_error_not_a_document) + + Blog.drop_collection() + + blog1 = Blog(title="code", posts=[post1, post2]) + blog1 = Blog.objects.insert(blog1) + self.assertEqual(blog1.title, "code") + self.assertEqual(Blog.objects.count(), 1) + + Blog.drop_collection() + blog1 = Blog(title="code", posts=[post1, post2]) + obj_id = Blog.objects.insert(blog1, load_bulk=False) + self.assertEquals(obj_id.__class__.__name__, 'ObjectId') + + def test_repeated_iteration(self): """Ensure that QuerySet rewinds itself one iteration finishes. """ From 55e20bda12ea6ee7a39d6d5ebdf124bfb5cc4689 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 6 Jun 2011 14:35:46 +0100 Subject: [PATCH 0242/1279] Added slave_okay syntax to querysets. * slave_okay (optional): if True, allows this query to be run against a replica secondary. --- mongoengine/queryset.py | 7 ++++++- tests/queryset.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 0e87db7..7b4fef3 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -336,6 +336,7 @@ class QuerySet(object): self._snapshot = False self._timeout = True self._class_check = True + self._slave_okay = False # If inheritance is allowed, only return instances and instances of # subclasses of the class being used @@ -430,7 +431,7 @@ class QuerySet(object): return spec - def __call__(self, q_obj=None, class_check=True, **query): + def __call__(self, q_obj=None, class_check=True, slave_okay=False, **query): """Filter the selected documents by calling the :class:`~mongoengine.queryset.QuerySet` with a query. @@ -440,6 +441,8 @@ class QuerySet(object): objects, only the last one will be used :param class_check: If set to False bypass class name check when querying collection + :param slave_okay: if True, allows this query to be run against a + replica secondary. :param query: Django-style query keyword arguments """ query = Q(**query) @@ -449,6 +452,7 @@ class QuerySet(object): self._mongo_query = None self._cursor_obj = None self._class_check = class_check + self._slave_okay = slave_okay return self def filter(self, *q_objs, **query): @@ -506,6 +510,7 @@ class QuerySet(object): cursor_args = { 'snapshot': self._snapshot, 'timeout': self._timeout, + 'slave_okay': self._slave_okay } if self._loaded_fields: cursor_args['fields'] = self._loaded_fields.as_dict() diff --git a/tests/queryset.py b/tests/queryset.py index 0b64e3e..28d4486 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -413,6 +413,19 @@ class QuerySetTest(unittest.TestCase): obj_id = Blog.objects.insert(blog1, load_bulk=False) self.assertEquals(obj_id.__class__.__name__, 'ObjectId') + def test_slave_okay(self): + """Ensures that a query can take slave_okay syntax + """ + person1 = self.Person(name="User A", age=20) + person1.save() + person2 = self.Person(name="User B", age=30) + person2.save() + + # Retrieve the first person from the database + person = self.Person.objects(slave_okay=True).first() + self.assertTrue(isinstance(person, self.Person)) + self.assertEqual(person.name, "User A") + self.assertEqual(person.age, 20) def test_repeated_iteration(self): """Ensure that QuerySet rewinds itself one iteration finishes. From 711db45c022cae092069432d42e9267411f80008 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 6 Jun 2011 14:36:44 +0100 Subject: [PATCH 0243/1279] Changelist updated --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 29ecdf7..ed877eb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added slave_okay kwarg to queryset - Added insert method for bulk inserts - Added blinker signal support - Added query_counter context manager for tests From d63bf0abde50b50ef091426aa0cef9b1646ed308 Mon Sep 17 00:00:00 2001 From: kuno Date: Tue, 7 Jun 2011 20:19:29 +0800 Subject: [PATCH 0244/1279] fixed import path typo in django documents --- docs/django.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/django.rst b/docs/django.rst index 8a49057..bbfbb56 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -52,7 +52,7 @@ it is useful to have a Django file storage backend that wraps this. The new storage module is called :class:`~mongoengine.django.GridFSStorage`. Using it is very similar to using the default FileSystemStorage.:: - fs = mongoengine.django.GridFSStorage() + fs = mongoengine.django.storage.GridFSStorage() filename = fs.save('hello.txt', 'Hello, World!') From c059ad47f24394c3bb3f4b4f24a6f9e91280c4ab Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 7 Jun 2011 15:14:41 +0100 Subject: [PATCH 0245/1279] Updated django docs refs #186 --- docs/django.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/django.rst b/docs/django.rst index bbfbb56..4478b94 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -49,10 +49,11 @@ Storage ======= With MongoEngine's support for GridFS via the :class:`~mongoengine.FileField`, it is useful to have a Django file storage backend that wraps this. The new -storage module is called :class:`~mongoengine.django.GridFSStorage`. Using it -is very similar to using the default FileSystemStorage.:: - - fs = mongoengine.django.storage.GridFSStorage() +storage module is called :class:`~mongoengine.django.storage.GridFSStorage`. +Using it is very similar to using the default FileSystemStorage.:: + + from mongoengine.django.storage import GridFSStorage + fs = GridFSStorage() filename = fs.save('hello.txt', 'Hello, World!') From cfcd77b193da1eb03ef5632f88cd2189f58b2974 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 8 Jun 2011 10:33:56 +0100 Subject: [PATCH 0246/1279] Added tests displaying datetime behaviour. Updated datetimefield documentation --- mongoengine/fields.py | 4 +++ tests/fields.py | 60 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index dc03fc0..1995d34 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -229,6 +229,10 @@ class BooleanField(BaseField): class DateTimeField(BaseField): """A datetime field. + + Note: Microseconds are rounded to the nearest millisecond. + Pre UTC microsecond support is effecively broken see + `tests.field.test_datetime` for more information. """ def validate(self, value): diff --git a/tests/fields.py b/tests/fields.py index 00b1c88..320e33d 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -187,6 +187,66 @@ class FieldTest(unittest.TestCase): log.time = '1pm' self.assertRaises(ValidationError, log.validate) + def test_datetime(self): + """Tests showing pymongo datetime fields handling of microseconds. + Microseconds are rounded to the nearest millisecond and pre UTC + handling is wonky. + + See: http://api.mongodb.org/python/current/api/bson/son.html#dt + """ + class LogEntry(Document): + date = DateTimeField() + + LogEntry.drop_collection() + + # Post UTC - microseconds are rounded (down) nearest millisecond and dropped + d1 = datetime.datetime(1970, 01, 01, 00, 00, 01, 999) + d2 = datetime.datetime(1970, 01, 01, 00, 00, 01) + log = LogEntry() + log.date = d1 + log.save() + log.reload() + self.assertNotEquals(log.date, d1) + self.assertEquals(log.date, d2) + + # Post UTC - microseconds are rounded (down) nearest millisecond + d1 = datetime.datetime(1970, 01, 01, 00, 00, 01, 9999) + d2 = datetime.datetime(1970, 01, 01, 00, 00, 01, 9000) + log.date = d1 + log.save() + log.reload() + self.assertNotEquals(log.date, d1) + self.assertEquals(log.date, d2) + + # Pre UTC dates microseconds below 1000 are dropped + d1 = datetime.datetime(1969, 12, 31, 23, 59, 59, 999) + d2 = datetime.datetime(1969, 12, 31, 23, 59, 59) + log.date = d1 + log.save() + log.reload() + self.assertNotEquals(log.date, d1) + self.assertEquals(log.date, d2) + + # Pre UTC microseconds above 1000 is wonky. + # log.date has an invalid microsecond value so I can't construct + # a date to compare. + # + # However, the timedelta is predicable with pre UTC timestamps + # It always adds 16 seconds and [777216-776217] microseconds + for i in xrange(1001, 3113, 33): + d1 = datetime.datetime(1969, 12, 31, 23, 59, 59, i) + log.date = d1 + log.save() + log.reload() + self.assertNotEquals(log.date, d1) + + delta = log.date - d1 + self.assertEquals(delta.seconds, 16) + microseconds = 777216 - (i % 1000) + self.assertEquals(delta.microseconds, microseconds) + + LogEntry.drop_collection() + def test_list_validation(self): """Ensure that a list field only accepts lists with valid elements. """ From d15f5ccbf43e31557c43eb238028537e9a59c089 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 8 Jun 2011 10:41:08 +0100 Subject: [PATCH 0247/1279] Added _slave_okay to clone --- mongoengine/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 7b4fef3..a1e1245 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -353,7 +353,7 @@ class QuerySet(object): copy_props = ('_initial_query', '_query_obj', '_where_clause', '_loaded_fields', '_ordering', '_snapshot', - '_timeout', '_limit', '_skip') + '_timeout', '_limit', '_skip', '_slave_okay') for prop in copy_props: val = getattr(self, prop) From 3c88faa889e01071c6953992307112f20140f2f7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 8 Jun 2011 12:06:26 +0100 Subject: [PATCH 0248/1279] Updated slave_okay syntax Now inline with .timeout() and .snapshot(). Made them chainable - so its easier to use and added tests for cursor_args --- mongoengine/queryset.py | 37 ++++++++++++++++++++++++++----------- tests/queryset.py | 26 +++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a1e1245..f542cc8 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -452,7 +452,6 @@ class QuerySet(object): self._mongo_query = None self._cursor_obj = None self._class_check = class_check - self._slave_okay = slave_okay return self def filter(self, *q_objs, **query): @@ -504,18 +503,23 @@ class QuerySet(object): return self._collection_obj + @property + def _cursor_args(self): + cursor_args = { + 'snapshot': self._snapshot, + 'timeout': self._timeout, + 'slave_okay': self._slave_okay + } + if self._loaded_fields: + cursor_args['fields'] = self._loaded_fields.as_dict() + return cursor_args + @property def _cursor(self): if self._cursor_obj is None: - cursor_args = { - 'snapshot': self._snapshot, - 'timeout': self._timeout, - 'slave_okay': self._slave_okay - } - if self._loaded_fields: - cursor_args['fields'] = self._loaded_fields.as_dict() + self._cursor_obj = self._collection.find(self._query, - **cursor_args) + **self._cursor_args) # Apply where clauses to cursor if self._where_clause: self._cursor_obj.where(self._where_clause) @@ -772,7 +776,7 @@ class QuerySet(object): id_field = self._document._meta['id_field'] object_id = self._document._fields[id_field].to_mongo(object_id) - result = self._collection.find_one({'_id': object_id}) + result = self._collection.find_one({'_id': object_id}, **self._cursor_args) if result is not None: result = self._document._from_son(result) return result @@ -788,7 +792,8 @@ class QuerySet(object): """ doc_map = {} - docs = self._collection.find({'_id': {'$in': object_ids}}) + docs = self._collection.find({'_id': {'$in': object_ids}}, + **self._cursor_args) for doc in docs: doc_map[doc['_id']] = self._document._from_son(doc) @@ -1085,6 +1090,7 @@ class QuerySet(object): :param enabled: whether or not snapshot mode is enabled """ self._snapshot = enabled + return self def timeout(self, enabled): """Enable or disable the default mongod timeout when querying. @@ -1092,6 +1098,15 @@ class QuerySet(object): :param enabled: whether or not the timeout is used """ self._timeout = enabled + return self + + def slave_okay(self, enabled): + """Enable or disable the slave_okay when querying. + + :param enabled: whether or not the slave_okay is enabled + """ + self._slave_okay = enabled + return self def delete(self, safe=False): """Delete the documents matched by the query. diff --git a/tests/queryset.py b/tests/queryset.py index 28d4486..1947254 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -422,11 +422,35 @@ class QuerySetTest(unittest.TestCase): person2.save() # Retrieve the first person from the database - person = self.Person.objects(slave_okay=True).first() + person = self.Person.objects.slave_okay(True).first() self.assertTrue(isinstance(person, self.Person)) self.assertEqual(person.name, "User A") self.assertEqual(person.age, 20) + def test_cursor_args(self): + """Ensures the cursor args can be set as expected + """ + p = self.Person.objects + # Check default + self.assertEqual(p._cursor_args, + {'snapshot': False, 'slave_okay': False, 'timeout': True}) + + p.snapshot(False).slave_okay(False).timeout(False) + self.assertEqual(p._cursor_args, + {'snapshot': False, 'slave_okay': False, 'timeout': False}) + + p.snapshot(True).slave_okay(False).timeout(False) + self.assertEqual(p._cursor_args, + {'snapshot': True, 'slave_okay': False, 'timeout': False}) + + p.snapshot(True).slave_okay(True).timeout(False) + self.assertEqual(p._cursor_args, + {'snapshot': True, 'slave_okay': True, 'timeout': False}) + + p.snapshot(True).slave_okay(True).timeout(True) + self.assertEqual(p._cursor_args, + {'snapshot': True, 'slave_okay': True, 'timeout': True}) + def test_repeated_iteration(self): """Ensure that QuerySet rewinds itself one iteration finishes. """ From 7c62fdc0b82f13bae0796b0d749ecb87002240a7 Mon Sep 17 00:00:00 2001 From: Colin Howe Date: Wed, 8 Jun 2011 12:20:58 +0100 Subject: [PATCH 0249/1279] Allow for types to never be auto-prepended to indices --- mongoengine/queryset.py | 9 ++++++--- tests/queryset.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 17a1b0d..303afb6 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -410,8 +410,10 @@ class QuerySet(object): if use_types and not all(f._index_with_types for f in fields): use_types = False - # If _types is being used, prepend it to every specified index - if doc_cls._meta.get('allow_inheritance') and use_types: + # If _types is being used, create an index for it + index_types = doc_cls._meta.get('index_types', True) + allow_inheritance = doc_cls._meta.get('allow_inheritance') + if index_types and allow_inheritance and use_types: index_list.insert(0, ('_types', 1)) return index_list @@ -457,6 +459,7 @@ class QuerySet(object): background = self._document._meta.get('index_background', False) drop_dups = self._document._meta.get('index_drop_dups', False) index_opts = self._document._meta.get('index_options', {}) + index_types = self._document._meta.get('index_types', True) # Ensure indexes created by uniqueness constraints for index in self._document._meta['unique_indexes']: @@ -470,7 +473,7 @@ class QuerySet(object): background=background, **index_opts) # If _types is being used (for polymorphism), it needs an index - if '_types' in self._query: + if index_types and '_types' in self._query: self._collection.ensure_index('_types', background=background, **index_opts) diff --git a/tests/queryset.py b/tests/queryset.py index 1f03fbd..1e5e7a5 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1710,6 +1710,22 @@ class QuerySetTest(unittest.TestCase): self.assertTrue([('_types', 1)] in info) self.assertTrue([('_types', 1), ('date', -1)] in info) + def test_dont_index_types(self): + """Ensure that index_types will, when disabled, prevent _types + being added to all indices. + """ + class BlogPost(Document): + date = DateTimeField() + meta = {'index_types': False, + 'indexes': ['-date']} + + # Indexes are lazy so use list() to perform query + list(BlogPost.objects) + info = BlogPost.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] + self.assertTrue([('_types', 1)] not in info) + self.assertTrue([('date', -1)] in info) + BlogPost.drop_collection() class BlogPost(Document): From aa32d4301479a7cd45071ca3e5607ebe319f225e Mon Sep 17 00:00:00 2001 From: Colin Howe Date: Wed, 8 Jun 2011 12:36:32 +0100 Subject: [PATCH 0250/1279] Pydoc update --- mongoengine/document.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mongoengine/document.py b/mongoengine/document.py index b563f42..cae8343 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -53,6 +53,11 @@ class Document(BaseDocument): dictionary. The value should be a list of field names or tuples of field names. Index direction may be specified by prefixing the field names with a **+** or **-** sign. + + By default, _types will be added to the start of every index (that + doesn't contain a list) if allow_inheritence is True. This can be + disabled by either setting types to False on the specific index or + by setting index_types to False on the meta dictionary for the document. """ __metaclass__ = TopLevelDocumentMetaclass From 6dc2672dbab4d0914e838c4df867daa911a33dcf Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 8 Jun 2011 13:03:42 +0100 Subject: [PATCH 0251/1279] Updated changelog --- docs/changelog.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ed877eb..0a2a273 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,7 +5,8 @@ Changelog Changes in dev ============== -- Added slave_okay kwarg to queryset +- Added queryset.slave_okay(enabled) method +- Updated queryset.timeout(enabled) and queryset.snapshot(enabled) to be chainable - Added insert method for bulk inserts - Added blinker signal support - Added query_counter context manager for tests From d32dd9ff62c0984af5062a4b52f974bb009b22a3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 8 Jun 2011 13:07:08 +0100 Subject: [PATCH 0252/1279] Added _get_FIELD_display() for handy choice field display lookups closes #188 --- docs/changelog.rst | 1 + mongoengine/base.py | 12 +++++++++++- tests/fields.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0a2a273..c76b115 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added get_FIELD_display() method for easy choice field displaying. - Added queryset.slave_okay(enabled) method - Updated queryset.timeout(enabled) and queryset.snapshot(enabled) to be chainable - Added insert method for bulk inserts diff --git a/mongoengine/base.py b/mongoengine/base.py index 76bb1ab..3875fea 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -8,6 +8,7 @@ import sys import pymongo import pymongo.objectid from operator import itemgetter +from functools import partial class NotRegistered(Exception): @@ -61,6 +62,7 @@ class BaseField(object): self.primary_key = primary_key self.validation = validation self.choices = choices + # Adjust the appropriate creation counter, and save our local copy. if self.db_field == '_id': self.creation_counter = BaseField.auto_creation_counter @@ -471,7 +473,10 @@ class BaseDocument(object): self._data = {} # Assign default values to instance - for attr_name in self._fields.keys(): + for attr_name, field in self._fields.items(): + if field.choices: # dynamically adds a way to get the display value for a field with choices + setattr(self, 'get_%s_display' % attr_name, partial(self._get_FIELD_display, field=field)) + # Use default value if present value = getattr(self, attr_name, None) setattr(self, attr_name, value) @@ -484,6 +489,11 @@ class BaseDocument(object): signals.post_init.send(self) + def _get_FIELD_display(self, field): + """Returns the display value for a choice field""" + value = getattr(self, field.name) + return dict(field.choices).get(value, value) + def validate(self): """Ensure that all fields' values are valid and that required fields are present. diff --git a/tests/fields.py b/tests/fields.py index 320e33d..d897004 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -773,6 +773,35 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() + def test_choices_get_field_display(self): + """Test dynamic helper for returning the display value of a choices field. + """ + class Shirt(Document): + size = StringField(max_length=3, choices=(('S', 'Small'), ('M', 'Medium'), ('L', 'Large'), + ('XL', 'Extra Large'), ('XXL', 'Extra Extra Large'))) + style = StringField(max_length=3, choices=(('S', 'Small'), ('B', 'Baggy'), ('W', 'wide')), default='S') + + Shirt.drop_collection() + + shirt = Shirt() + + self.assertEqual(shirt.get_size_display(), None) + self.assertEqual(shirt.get_style_display(), 'Small') + + shirt.size = "XXL" + shirt.style = "B" + self.assertEqual(shirt.get_size_display(), 'Extra Extra Large') + self.assertEqual(shirt.get_style_display(), 'Baggy') + + # Set as Z - an invalid choice + shirt.size = "Z" + shirt.style = "Z" + self.assertEqual(shirt.get_size_display(), 'Z') + self.assertEqual(shirt.get_style_display(), 'Z') + self.assertRaises(ValidationError, shirt.validate) + + Shirt.drop_collection() + def test_file_fields(self): """Ensure that file fields can be written to and their data retrieved """ From 602d7dad0020937364f7076a1930d46209d6009d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 8 Jun 2011 17:10:26 +0100 Subject: [PATCH 0253/1279] Improvements to Abstract Base Classes Added test example highlighting what to do to migrate a class from complex (allows inheritance) to simple. --- mongoengine/base.py | 13 +++++-- tests/document.py | 90 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 97 insertions(+), 6 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 3875fea..8a0a1f2 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -263,7 +263,7 @@ class DocumentMetaclass(type): superclasses[base._class_name] = base superclasses.update(base._superclasses) - if hasattr(base, '_meta'): + if hasattr(base, '_meta') and not base._meta.get('abstract'): # Ensure that the Document class may be subclassed - # inheritance may be disabled to remove dependency on # additional fields _cls and _types @@ -280,7 +280,7 @@ class DocumentMetaclass(type): # Only simple classes - direct subclasses of Document - may set # allow_inheritance to False - if not simple_class and not meta['allow_inheritance']: + if not simple_class and not meta['allow_inheritance'] and not meta['abstract']: raise ValueError('Only direct subclasses of Document may set ' '"allow_inheritance" to False') attrs['_meta'] = meta @@ -360,8 +360,9 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Subclassed documents inherit collection from superclass for base in bases: - if hasattr(base, '_meta') and 'collection' in base._meta: - collection = base._meta['collection'] + if hasattr(base, '_meta'): + if 'collection' in base._meta: + collection = base._meta['collection'] # Propagate index options. for key in ('index_background', 'index_drop_dups', 'index_opts'): @@ -370,6 +371,9 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): id_field = id_field or base._meta.get('id_field') base_indexes += base._meta.get('indexes', []) + # Propagate 'allow_inheritance' + if 'allow_inheritance' in base._meta: + base_meta['allow_inheritance'] = base._meta['allow_inheritance'] meta = { 'abstract': False, @@ -384,6 +388,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): 'index_opts': {}, 'queryset_class': QuerySet, 'delete_rules': {}, + 'allow_inheritance': True } meta.update(base_meta) diff --git a/tests/document.py b/tests/document.py index a812046..1454146 100644 --- a/tests/document.py +++ b/tests/document.py @@ -151,12 +151,12 @@ class DocumentTest(unittest.TestCase): """Ensure that inheritance may be disabled on simple classes and that _cls and _types will not be used. """ + class Animal(Document): - meta = {'allow_inheritance': False} name = StringField() + meta = {'allow_inheritance': False} Animal.drop_collection() - def create_dog_class(): class Dog(Animal): pass @@ -191,6 +191,92 @@ class DocumentTest(unittest.TestCase): self.assertFalse('_cls' in comment.to_mongo()) self.assertFalse('_types' in comment.to_mongo()) + def test_allow_inheritance_abstract_document(self): + """Ensure that abstract documents can set inheritance rules and that + _cls and _types will not be used. + """ + class FinalDocument(Document): + meta = {'abstract': True, + 'allow_inheritance': False} + + class Animal(FinalDocument): + name = StringField() + + Animal.drop_collection() + def create_dog_class(): + class Dog(Animal): + pass + self.assertRaises(ValueError, create_dog_class) + + # Check that _cls etc aren't present on simple documents + dog = Animal(name='dog') + dog.save() + collection = self.db[Animal._meta['collection']] + obj = collection.find_one() + self.assertFalse('_cls' in obj) + self.assertFalse('_types' in obj) + + Animal.drop_collection() + + def test_how_to_turn_off_inheritance(self): + """Demonstrates migrating from allow_inheritance = True to False. + """ + class Animal(Document): + name = StringField() + meta = { + 'indexes': ['name'] + } + + Animal.drop_collection() + + dog = Animal(name='dog') + dog.save() + + collection = self.db[Animal._meta['collection']] + obj = collection.find_one() + self.assertTrue('_cls' in obj) + self.assertTrue('_types' in obj) + + info = collection.index_information() + info = [value['key'] for key, value in info.iteritems()] + self.assertEquals([[(u'_id', 1)], [(u'_types', 1)], [(u'_types', 1), (u'name', 1)]], info) + + # Turn off inheritance + class Animal(Document): + name = StringField() + meta = { + 'allow_inheritance': False, + 'indexes': ['name'] + } + collection.update({}, {"$unset": {"_types": 1, "_cls": 1}}, False, True) + + # Confirm extra data is removed + obj = collection.find_one() + self.assertFalse('_cls' in obj) + self.assertFalse('_types' in obj) + + info = collection.index_information() + info = [value['key'] for key, value in info.iteritems()] + self.assertEquals([[(u'_id', 1)], [(u'_types', 1)], [(u'_types', 1), (u'name', 1)]], info) + + info = collection.index_information() + indexes_to_drop = [key for key, value in info.iteritems() if '_types' in dict(value['key'])] + for index in indexes_to_drop: + collection.drop_index(index) + + info = collection.index_information() + info = [value['key'] for key, value in info.iteritems()] + self.assertEquals([[(u'_id', 1)]], info) + + # Recreate indexes + dog = Animal.objects.first() + dog.save() + info = collection.index_information() + info = [value['key'] for key, value in info.iteritems()] + self.assertEquals([[(u'_id', 1)], [(u'name', 1),]], info) + + Animal.drop_collection() + def test_abstract_documents(self): """Ensure that a document superclass can be marked as abstract thereby not using it as the name for the collection.""" From 4b9bacf7316275d2c0c1efa7b5850b98374679cc Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 6 Jun 2011 17:21:54 +0100 Subject: [PATCH 0254/1279] Added ComplexBaseField * Handles the efficient lazy dereferencing of DBrefs. * Handles complex nested values in ListFields and DictFields * Allows for both strictly declared ListFields and DictFields where the embedded value must be of a field type or no restrictions where the values can be a mix of field types / values. * Handles DBrefences of documents where allow_inheritance = False. --- mongoengine/base.py | 206 +++++++++++++++++++++------- mongoengine/fields.py | 102 +++----------- mongoengine/queryset.py | 47 +++++-- tests/dereference.py | 112 +++++++++++++++- tests/fields.py | 287 +++++++++++++++++++++++++++++++--------- 5 files changed, 555 insertions(+), 199 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 8a0a1f2..a22795c 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -132,15 +132,19 @@ class BaseField(object): self.validate(value) -class DereferenceBaseField(BaseField): - """Handles the lazy dereferencing of a queryset. Will dereference all +class ComplexBaseField(BaseField): + """Handles complex fields, such as lists / dictionaries. + + Allows for nesting of embedded documents inside complex types. + Handles the lazy dereferencing of a queryset by lazily dereferencing all items in a list / dict rather than one at a time. """ + field = None + def __get__(self, instance, owner): """Descriptor to automatically dereference references. """ - from fields import ReferenceField, GenericReferenceField from connection import _get_db if instance is None: @@ -149,68 +153,175 @@ class DereferenceBaseField(BaseField): # Get value from document instance if available value_list = instance._data.get(self.name) - if not value_list: - return super(DereferenceBaseField, self).__get__(instance, owner) + if not value_list or isinstance(value_list, basestring): + return super(ComplexBaseField, self).__get__(instance, owner) is_list = False if not hasattr(value_list, 'items'): is_list = True value_list = dict([(k,v) for k,v in enumerate(value_list)]) - if isinstance(self.field, ReferenceField) and value_list: - db = _get_db() - dbref = {} - collections = {} + for k,v in value_list.items(): + if isinstance(v, dict) and '_cls' in v and '_ref' not in v: + value_list[k] = get_document(v['_cls'].split('.')[-1])._from_son(v) - for k, v in value_list.items(): - dbref[k] = v - # Save any DBRefs + # Handle all dereferencing + db = _get_db() + dbref = {} + collections = {} + for k, v in value_list.items(): + dbref[k] = v + # Save any DBRefs + if isinstance(v, (pymongo.dbref.DBRef)): + # direct reference (DBRef) + collections.setdefault(v.collection, []).append((k, v)) + elif isinstance(v, (dict, pymongo.son.SON)) and '_ref' in v: + # generic reference + collection = get_document(v['_cls'])._meta['collection'] + collections.setdefault(collection, []).append((k, v)) + + # For each collection get the references + for collection, dbrefs in collections.items(): + id_map = {} + for k, v in dbrefs: if isinstance(v, (pymongo.dbref.DBRef)): - collections.setdefault(v.collection, []).append((k, v)) + # direct reference (DBRef), has no _cls information + id_map[v.id] = (k, None) + elif isinstance(v, (dict, pymongo.son.SON)) and '_ref' in v: + # generic reference - includes _cls information + id_map[v['_ref'].id] = (k, get_document(v['_cls'])) - # For each collection get the references - for collection, dbrefs in collections.items(): - id_map = dict([(v.id, k) for k, v in dbrefs]) - references = db[collection].find({'_id': {'$in': id_map.keys()}}) - for ref in references: - key = id_map[ref['_id']] - dbref[key] = get_document(ref['_cls'])._from_son(ref) + references = db[collection].find({'_id': {'$in': id_map.keys()}}) + for ref in references: + key, doc_cls = id_map[ref['_id']] + if not doc_cls: # If no doc_cls get it from the referenced doc + doc_cls = get_document(ref['_cls']) + dbref[key] = doc_cls._from_son(ref) - if is_list: - dbref = [v for k,v in sorted(dbref.items(), key=itemgetter(0))] - instance._data[self.name] = dbref + if is_list: + dbref = [v for k,v in sorted(dbref.items(), key=itemgetter(0))] + instance._data[self.name] = dbref + return super(ComplexBaseField, self).__get__(instance, owner) - # Get value from document instance if available - if isinstance(self.field, GenericReferenceField) and value_list: - db = _get_db() - value_list = [(k,v) for k,v in value_list.items()] - dbref = {} - classes = {} + def to_python(self, value): + """Convert a MongoDB-compatible type to a Python type. + """ + from mongoengine import Document - for k, v in value_list: - dbref[k] = v - # Save any DBRefs - if isinstance(v, (dict, pymongo.son.SON)): - classes.setdefault(v['_cls'], []).append((k, v)) + if isinstance(value, basestring): + return value - # For each collection get the references - for doc_cls, dbrefs in classes.items(): - id_map = dict([(v['_ref'].id, k) for k, v in dbrefs]) - doc_cls = get_document(doc_cls) - collection = doc_cls._meta['collection'] - references = db[collection].find({'_id': {'$in': id_map.keys()}}) + if hasattr(value, 'to_python'): + return value.to_python() - for ref in references: - key = id_map[ref['_id']] - dbref[key] = doc_cls._from_son(ref) + is_list = False + if not hasattr(value, 'items'): + try: + is_list = True + value = dict([(k,v) for k,v in enumerate(value)]) + except TypeError: # Not iterable return the value + return value - if is_list: - dbref = [v for k,v in sorted(dbref.items(), key=itemgetter(0))] + if self.field: + value_dict = dict([(key, self.field.to_python(item)) for key, item in value.items()]) + else: + value_dict = {} + for k,v in value.items(): + if isinstance(v, Document): + # We need the id from the saved object to create the DBRef + if v.pk is None: + raise ValidationError('You can only reference documents once ' + 'they have been saved to the database') + collection = v._meta['collection'] + value_dict[k] = pymongo.dbref.DBRef(collection, v.pk) + elif hasattr(v, 'to_python'): + value_dict[k] = v.to_python() + else: + value_dict[k] = self.to_python(v) - instance._data[self.name] = dbref + if is_list: # Convert back to a list + return [v for k,v in sorted(value_dict.items(), key=itemgetter(0))] + return value_dict - return super(DereferenceBaseField, self).__get__(instance, owner) + def to_mongo(self, value): + """Convert a Python type to a MongoDB-compatible type. + """ + from mongoengine import Document + if isinstance(value, basestring): + return value + + if hasattr(value, 'to_mongo'): + return value.to_mongo() + + is_list = False + if not hasattr(value, 'items'): + try: + is_list = True + value = dict([(k,v) for k,v in enumerate(value)]) + except TypeError: # Not iterable return the value + return value + + if self.field: + value_dict = dict([(key, self.field.to_mongo(item)) for key, item in value.items()]) + else: + value_dict = {} + for k,v in value.items(): + if isinstance(v, Document): + # We need the id from the saved object to create the DBRef + if v.pk is None: + raise ValidationError('You can only reference documents once ' + 'they have been saved to the database') + + # If its a document that is not inheritable it won't have + # _types / _cls data so make it a generic reference allows + # us to dereference + meta = getattr(v, 'meta', getattr(v, '_meta', {})) + if meta and not meta['allow_inheritance'] and not self.field: + from fields import GenericReferenceField + value_dict[k] = GenericReferenceField().to_mongo(v) + else: + collection = v._meta['collection'] + value_dict[k] = pymongo.dbref.DBRef(collection, v.pk) + elif hasattr(v, 'to_mongo'): + value_dict[k] = v.to_mongo() + else: + value_dict[k] = self.to_mongo(v) + + if is_list: # Convert back to a list + return [v for k,v in sorted(value_dict.items(), key=itemgetter(0))] + return value_dict + + def validate(self, value): + """If field provided ensure the value is valid. + """ + if self.field: + try: + if hasattr(value, 'iteritems'): + [self.field.validate(v) for k,v in value.iteritems()] + else: + [self.field.validate(v) for v in value] + except Exception, err: + raise ValidationError('Invalid %s item (%s)' % ( + self.field.__class__.__name__, str(v))) + + def prepare_query_value(self, op, value): + return self.to_mongo(value) + + def lookup_member(self, member_name): + if self.field: + return self.field.lookup_member(member_name) + return None + + def _set_owner_document(self, owner_document): + if self.field: + self.field.owner_document = owner_document + self._owner_document = owner_document + + def _get_owner_document(self, owner_document): + self._owner_document = owner_document + + owner_document = property(_get_owner_document, _set_owner_document) class ObjectIdField(BaseField): @@ -219,7 +330,6 @@ class ObjectIdField(BaseField): def to_python(self, value): return value - # return unicode(value) def to_mongo(self, value): if not isinstance(value, pymongo.objectid.ObjectId): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 1995d34..f9b2580 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1,4 +1,4 @@ -from base import (BaseField, DereferenceBaseField, ObjectIdField, +from base import (BaseField, ComplexBaseField, ObjectIdField, ValidationError, get_document) from queryset import DO_NOTHING from document import Document, EmbeddedDocument @@ -301,6 +301,8 @@ class EmbeddedDocumentField(BaseField): return value def to_mongo(self, value): + if isinstance(value, basestring): + return value return self.document_type.to_mongo(value) def validate(self, value): @@ -320,7 +322,7 @@ class EmbeddedDocumentField(BaseField): return self.to_mongo(value) -class ListField(DereferenceBaseField): +class ListField(ComplexBaseField): """A list field that wraps a standard field, allowing multiple instances of the field to be used as a list in the database. """ @@ -328,48 +330,25 @@ class ListField(DereferenceBaseField): # ListFields cannot be indexed with _types - MongoDB doesn't support this _index_with_types = False - def __init__(self, field, **kwargs): - if not isinstance(field, BaseField): - raise ValidationError('Argument to ListField constructor must be ' - 'a valid field') + def __init__(self, field=None, **kwargs): self.field = field kwargs.setdefault('default', lambda: []) super(ListField, self).__init__(**kwargs) - def to_python(self, value): - return [self.field.to_python(item) for item in value] - - def to_mongo(self, value): - return [self.field.to_mongo(item) for item in value] - def validate(self, value): """Make sure that a list of valid fields is being used. """ if not isinstance(value, (list, tuple)): raise ValidationError('Only lists and tuples may be used in a ' 'list field') - - try: - [self.field.validate(item) for item in value] - except Exception, err: - raise ValidationError('Invalid ListField item (%s)' % str(item)) + super(ListField, self).validate(value) def prepare_query_value(self, op, value): - if op in ('set', 'unset'): - return [self.field.prepare_query_value(op, v) for v in value] - return self.field.prepare_query_value(op, value) - - def lookup_member(self, member_name): - return self.field.lookup_member(member_name) - - def _set_owner_document(self, owner_document): - self.field.owner_document = owner_document - self._owner_document = owner_document - - def _get_owner_document(self, owner_document): - self._owner_document = owner_document - - owner_document = property(_get_owner_document, _set_owner_document) + if self.field: + if op in ('set', 'unset') and not isinstance(value, basestring): + return [self.field.prepare_query_value(op, v) for v in value] + return self.field.prepare_query_value(op, value) + return super(ListField, self).prepare_query_value(op, value) class SortedListField(ListField): @@ -388,20 +367,21 @@ class SortedListField(ListField): super(SortedListField, self).__init__(field, **kwargs) def to_mongo(self, value): + value = super(SortedListField, self).to_mongo(value) if self._ordering is not None: - return sorted([self.field.to_mongo(item) for item in value], - key=itemgetter(self._ordering)) - return sorted([self.field.to_mongo(item) for item in value]) + return sorted(value, key=itemgetter(self._ordering)) + return sorted(value) -class DictField(BaseField): +class DictField(ComplexBaseField): """A dictionary field that wraps a standard Python dictionary. This is similar to an embedded document, but the structure is not defined. .. versionadded:: 0.3 """ - def __init__(self, basecls=None, *args, **kwargs): + def __init__(self, basecls=None, field=None, *args, **kwargs): + self.field = field self.basecls = basecls or BaseField assert issubclass(self.basecls, BaseField) kwargs.setdefault('default', lambda: {}) @@ -417,6 +397,7 @@ class DictField(BaseField): if any(('.' in k or '$' in k) for k in value): raise ValidationError('Invalid dictionary key name - keys may not ' 'contain "." or "$" characters') + super(DictField, self).validate(value) def lookup_member(self, member_name): return DictField(basecls=self.basecls, db_field=member_name) @@ -432,7 +413,7 @@ class DictField(BaseField): return super(DictField, self).prepare_query_value(op, value) -class MapField(DereferenceBaseField): +class MapField(DictField): """A field that maps a name to a specified field type. Similar to a DictField, except the 'value' of each item must match the specified field type. @@ -444,50 +425,7 @@ class MapField(DereferenceBaseField): if not isinstance(field, BaseField): raise ValidationError('Argument to MapField constructor must be ' 'a valid field') - self.field = field - kwargs.setdefault('default', lambda: {}) - super(MapField, self).__init__(*args, **kwargs) - - def validate(self, value): - """Make sure that a list of valid fields is being used. - """ - if not isinstance(value, dict): - raise ValidationError('Only dictionaries may be used in a ' - 'DictField') - - if any(('.' in k or '$' in k) for k in value): - raise ValidationError('Invalid dictionary key name - keys may not ' - 'contain "." or "$" characters') - - try: - [self.field.validate(item) for item in value.values()] - except Exception, err: - raise ValidationError('Invalid MapField item (%s)' % str(item)) - - def to_python(self, value): - return dict([(key, self.field.to_python(item)) for key, item in value.iteritems()]) - - def to_mongo(self, value): - return dict([(key, self.field.to_mongo(item)) for key, item in value.iteritems()]) - - def prepare_query_value(self, op, value): - if op not in ('set', 'unset'): - return self.field.prepare_query_value(op, value) - for key in value: - value[key] = self.field.prepare_query_value(op, value[key]) - return value - - def lookup_member(self, member_name): - return self.field.lookup_member(member_name) - - def _set_owner_document(self, owner_document): - self.field.owner_document = owner_document - self._owner_document = owner_document - - def _get_owner_document(self, owner_document): - self._owner_document = owner_document - - owner_document = property(_get_owner_document, _set_owner_document) + super(MapField, self).__init__(field=field, *args, **kwargs) class ReferenceField(BaseField): diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 1dfe55a..666567e 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -549,11 +549,12 @@ class QuerySet(object): parts = [parts] fields = [] field = None + for field_name in parts: # Handle ListField indexing: if field_name.isdigit(): try: - field = field.field + new_field = field.field except AttributeError, err: raise InvalidQueryError( "Can't use index on unsubscriptable field (%s)" % err) @@ -567,11 +568,17 @@ class QuerySet(object): field = document._fields[field_name] else: # Look up subfield on the previous field - field = field.lookup_member(field_name) - if field is None: + new_field = field.lookup_member(field_name) + from base import ComplexBaseField + if not new_field and isinstance(field, ComplexBaseField): + fields.append(field_name) + continue + elif not new_field: raise InvalidQueryError('Cannot resolve field "%s"' - % field_name) + % field_name) + field = new_field # update field to the new field type fields.append(field) + return fields @classmethod @@ -615,14 +622,33 @@ class QuerySet(object): if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] fields = QuerySet._lookup_field(_doc_cls, parts) - parts = [field.db_field for field in fields] + parts = [] + + cleaned_fields = [] + append_field = True + for field in fields: + if isinstance(field, str): + parts.append(field) + append_field = False + else: + parts.append(field.db_field) + if append_field: + cleaned_fields.append(field) # Convert value to proper value - field = fields[-1] + field = cleaned_fields[-1] + singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte', 'not'] singular_ops += match_operators if op in singular_ops: - value = field.prepare_query_value(op, value) + if isinstance(field, basestring): + if op in match_operators and isinstance(value, basestring): + from mongoengine import StringField + value = StringField().prepare_query_value(op, value) + else: + value = field + else: + value = field.prepare_query_value(op, value) elif op in ('in', 'nin', 'all', 'near'): # 'in', 'nin' and 'all' require a list of values value = [field.prepare_query_value(op, v) for v in value] @@ -1170,14 +1196,19 @@ class QuerySet(object): fields = QuerySet._lookup_field(_doc_cls, parts) parts = [] + cleaned_fields = [] + append_field = True for field in fields: if isinstance(field, str): parts.append(field) + append_field = False else: parts.append(field.db_field) + if append_field: + cleaned_fields.append(field) # Convert value to proper value - field = fields[-1] + field = cleaned_fields[-1] if op in (None, 'set', 'push', 'pull', 'addToSet'): value = field.prepare_query_value(op, value) diff --git a/tests/dereference.py b/tests/dereference.py index b6cee89..6879272 100644 --- a/tests/dereference.py +++ b/tests/dereference.py @@ -122,6 +122,64 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] self.assertEqual(q, 4) + for m in group_obj.members: + self.assertTrue('User' in m.__class__.__name__) + + UserA.drop_collection() + UserB.drop_collection() + UserC.drop_collection() + Group.drop_collection() + + def test_list_field_complex(self): + + class UserA(Document): + name = StringField() + + class UserB(Document): + name = StringField() + + class UserC(Document): + name = StringField() + + class Group(Document): + members = ListField() + + UserA.drop_collection() + UserB.drop_collection() + UserC.drop_collection() + Group.drop_collection() + + members = [] + for i in xrange(1, 51): + a = UserA(name='User A %s' % i) + a.save() + + b = UserB(name='User B %s' % i) + b.save() + + c = UserC(name='User C %s' % i) + c.save() + + members += [a, b, c] + + group = Group(members=members) + group.save() + + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first() + self.assertEqual(q, 1) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + for m in group_obj.members: + self.assertTrue('User' in m.__class__.__name__) + UserA.drop_collection() UserB.drop_collection() UserC.drop_collection() @@ -156,10 +214,13 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] self.assertEqual(q, 2) + for k, m in group_obj.members.iteritems(): + self.assertTrue(isinstance(m, User)) + User.drop_collection() Group.drop_collection() - def ztest_generic_reference_dict_field(self): + def test_dict_field(self): class UserA(Document): name = StringField() @@ -206,6 +267,9 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] self.assertEqual(q, 4) + for k, m in group_obj.members.iteritems(): + self.assertTrue('User' in m.__class__.__name__) + group.members = {} group.save() @@ -218,11 +282,54 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] self.assertEqual(q, 1) + for k, m in group_obj.members.iteritems(): + self.assertTrue('User' in m.__class__.__name__) + UserA.drop_collection() UserB.drop_collection() UserC.drop_collection() Group.drop_collection() + def test_dict_field_no_field_inheritance(self): + + class UserA(Document): + name = StringField() + meta = {'allow_inheritance': False} + + class Group(Document): + members = DictField() + + UserA.drop_collection() + Group.drop_collection() + + members = [] + for i in xrange(1, 51): + a = UserA(name='User A %s' % i) + a.save() + + members += [a] + + group = Group(members=dict([(str(u.id), u) for u in members])) + group.save() + + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first() + self.assertEqual(q, 1) + + [m for m in group_obj.members] + self.assertEqual(q, 2) + + [m for m in group_obj.members] + self.assertEqual(q, 2) + + for k, m in group_obj.members.iteritems(): + self.assertTrue(isinstance(m, UserA)) + + UserA.drop_collection() + Group.drop_collection() + def test_generic_reference_map_field(self): class UserA(Document): @@ -270,6 +377,9 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] self.assertEqual(q, 4) + for k, m in group_obj.members.iteritems(): + self.assertTrue('User' in m.__class__.__name__) + group.members = {} group.save() diff --git a/tests/fields.py b/tests/fields.py index d897004..4d51ed5 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -322,6 +322,108 @@ class FieldTest(unittest.TestCase): BlogPost.drop_collection() + def test_list_field(self): + """Ensure that list types work as expected. + """ + class BlogPost(Document): + info = ListField() + + BlogPost.drop_collection() + + post = BlogPost() + post.info = 'my post' + self.assertRaises(ValidationError, post.validate) + + post.info = {'title': 'test'} + self.assertRaises(ValidationError, post.validate) + + post.info = ['test'] + post.save() + + post = BlogPost() + post.info = [{'test': 'test'}] + post.save() + + post = BlogPost() + post.info = [{'test': 3}] + post.save() + + + self.assertEquals(BlogPost.objects.count(), 3) + self.assertEquals(BlogPost.objects.filter(info__exact='test').count(), 1) + self.assertEquals(BlogPost.objects.filter(info__0__test='test').count(), 1) + + # Confirm handles non strings or non existing keys + self.assertEquals(BlogPost.objects.filter(info__0__test__exact='5').count(), 0) + self.assertEquals(BlogPost.objects.filter(info__100__test__exact='test').count(), 0) + BlogPost.drop_collection() + + def test_list_field_Strict(self): + """Ensure that list field handles validation if provided a strict field type.""" + + class Simple(Document): + mapping = ListField(field=IntField()) + + Simple.drop_collection() + + e = Simple() + e.mapping = [1] + e.save() + + def create_invalid_mapping(): + e.mapping = ["abc"] + e.save() + + self.assertRaises(ValidationError, create_invalid_mapping) + + Simple.drop_collection() + + def test_list_field_complex(self): + """Ensure that the list fields can handle the complex types.""" + + class SettingBase(EmbeddedDocument): + pass + + class StringSetting(SettingBase): + value = StringField() + + class IntegerSetting(SettingBase): + value = IntField() + + class Simple(Document): + mapping = ListField() + + Simple.drop_collection() + e = Simple() + e.mapping.append(StringSetting(value='foo')) + e.mapping.append(IntegerSetting(value=42)) + e.mapping.append({'number': 1, 'string': 'Hi!', 'float': 1.001, + 'complex': IntegerSetting(value=42), 'list': + [IntegerSetting(value=42), StringSetting(value='foo')]}) + e.save() + + e2 = Simple.objects.get(id=e.id) + self.assertTrue(isinstance(e2.mapping[0], StringSetting)) + self.assertTrue(isinstance(e2.mapping[1], IntegerSetting)) + + # Test querying + self.assertEquals(Simple.objects.filter(mapping__1__value=42).count(), 1) + self.assertEquals(Simple.objects.filter(mapping__2__number=1).count(), 1) + self.assertEquals(Simple.objects.filter(mapping__2__complex__value=42).count(), 1) + self.assertEquals(Simple.objects.filter(mapping__2__list__0__value=42).count(), 1) + self.assertEquals(Simple.objects.filter(mapping__2__list__1__value='foo').count(), 1) + + # Confirm can update + Simple.objects().update(set__mapping__1=IntegerSetting(value=10)) + self.assertEquals(Simple.objects.filter(mapping__1__value=10).count(), 1) + + Simple.objects().update( + set__mapping__2__list__1=StringSetting(value='Boo')) + self.assertEquals(Simple.objects.filter(mapping__2__list__1__value='foo').count(), 0) + self.assertEquals(Simple.objects.filter(mapping__2__list__1__value='Boo').count(), 1) + + Simple.drop_collection() + def test_dict_field(self): """Ensure that dict types work as expected. """ @@ -363,6 +465,131 @@ class FieldTest(unittest.TestCase): self.assertEquals(BlogPost.objects.filter(info__made_up__test__exact='test').count(), 0) BlogPost.drop_collection() + def test_dictfield_Strict(self): + """Ensure that dict field handles validation if provided a strict field type.""" + + class Simple(Document): + mapping = DictField(field=IntField()) + + Simple.drop_collection() + + e = Simple() + e.mapping['someint'] = 1 + e.save() + + def create_invalid_mapping(): + e.mapping['somestring'] = "abc" + e.save() + + self.assertRaises(ValidationError, create_invalid_mapping) + + Simple.drop_collection() + + def test_dictfield_complex(self): + """Ensure that the dict field can handle the complex types.""" + + class SettingBase(EmbeddedDocument): + pass + + class StringSetting(SettingBase): + value = StringField() + + class IntegerSetting(SettingBase): + value = IntField() + + class Simple(Document): + mapping = DictField() + + Simple.drop_collection() + e = Simple() + e.mapping['somestring'] = StringSetting(value='foo') + e.mapping['someint'] = IntegerSetting(value=42) + e.mapping['nested_dict'] = {'number': 1, 'string': 'Hi!', 'float': 1.001, + 'complex': IntegerSetting(value=42), 'list': + [IntegerSetting(value=42), StringSetting(value='foo')]} + e.save() + + e2 = Simple.objects.get(id=e.id) + self.assertTrue(isinstance(e2.mapping['somestring'], StringSetting)) + self.assertTrue(isinstance(e2.mapping['someint'], IntegerSetting)) + + # Test querying + self.assertEquals(Simple.objects.filter(mapping__someint__value=42).count(), 1) + self.assertEquals(Simple.objects.filter(mapping__nested_dict__number=1).count(), 1) + self.assertEquals(Simple.objects.filter(mapping__nested_dict__complex__value=42).count(), 1) + self.assertEquals(Simple.objects.filter(mapping__nested_dict__list__0__value=42).count(), 1) + self.assertEquals(Simple.objects.filter(mapping__nested_dict__list__1__value='foo').count(), 1) + + # Confirm can update + Simple.objects().update( + set__mapping={"someint": IntegerSetting(value=10)}) + Simple.objects().update( + set__mapping__nested_dict__list__1=StringSetting(value='Boo')) + self.assertEquals(Simple.objects.filter(mapping__nested_dict__list__1__value='foo').count(), 0) + self.assertEquals(Simple.objects.filter(mapping__nested_dict__list__1__value='Boo').count(), 1) + + Simple.drop_collection() + + def test_mapfield(self): + """Ensure that the MapField handles the declared type.""" + + class Simple(Document): + mapping = MapField(IntField()) + + Simple.drop_collection() + + e = Simple() + e.mapping['someint'] = 1 + e.save() + + def create_invalid_mapping(): + e.mapping['somestring'] = "abc" + e.save() + + self.assertRaises(ValidationError, create_invalid_mapping) + + def create_invalid_class(): + class NoDeclaredType(Document): + mapping = MapField() + + self.assertRaises(ValidationError, create_invalid_class) + + Simple.drop_collection() + + def test_complex_mapfield(self): + """Ensure that the MapField can handle complex declared types.""" + + class SettingBase(EmbeddedDocument): + pass + + class StringSetting(SettingBase): + value = StringField() + + class IntegerSetting(SettingBase): + value = IntField() + + class Extensible(Document): + mapping = MapField(EmbeddedDocumentField(SettingBase)) + + Extensible.drop_collection() + + e = Extensible() + e.mapping['somestring'] = StringSetting(value='foo') + e.mapping['someint'] = IntegerSetting(value=42) + e.save() + + e2 = Extensible.objects.get(id=e.id) + self.assertTrue(isinstance(e2.mapping['somestring'], StringSetting)) + self.assertTrue(isinstance(e2.mapping['someint'], IntegerSetting)) + + def create_invalid_mapping(): + e.mapping['someint'] = 123 + e.save() + + self.assertRaises(ValidationError, create_invalid_mapping) + + Extensible.drop_collection() + def test_embedded_document_validation(self): """Ensure that invalid embedded documents cannot be assigned to embedded document fields. @@ -933,66 +1160,6 @@ class FieldTest(unittest.TestCase): self.assertEqual(d2.data, {}) self.assertEqual(d2.data2, {}) - def test_mapfield(self): - """Ensure that the MapField handles the declared type.""" - - class Simple(Document): - mapping = MapField(IntField()) - - Simple.drop_collection() - - e = Simple() - e.mapping['someint'] = 1 - e.save() - - def create_invalid_mapping(): - e.mapping['somestring'] = "abc" - e.save() - - self.assertRaises(ValidationError, create_invalid_mapping) - - def create_invalid_class(): - class NoDeclaredType(Document): - mapping = MapField() - - self.assertRaises(ValidationError, create_invalid_class) - - Simple.drop_collection() - - def test_complex_mapfield(self): - """Ensure that the MapField can handle complex declared types.""" - - class SettingBase(EmbeddedDocument): - pass - - class StringSetting(SettingBase): - value = StringField() - - class IntegerSetting(SettingBase): - value = IntField() - - class Extensible(Document): - mapping = MapField(EmbeddedDocumentField(SettingBase)) - - Extensible.drop_collection() - - e = Extensible() - e.mapping['somestring'] = StringSetting(value='foo') - e.mapping['someint'] = IntegerSetting(value=42) - e.save() - - e2 = Extensible.objects.get(id=e.id) - self.assertTrue(isinstance(e2.mapping['somestring'], StringSetting)) - self.assertTrue(isinstance(e2.mapping['someint'], IntegerSetting)) - - def create_invalid_mapping(): - e.mapping['someint'] = 123 - e.save() - - self.assertRaises(ValidationError, create_invalid_mapping) - - Extensible.drop_collection() - if __name__ == '__main__': unittest.main() From b9255f73c381c820d13ec30fba499a3fe6868a3e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 9 Jun 2011 11:28:57 +0100 Subject: [PATCH 0255/1279] Updated docs --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c76b115..f4be4ca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,13 +5,13 @@ Changelog Changes in dev ============== +- Added ComplexBaseField - for improved flexibility and performance. - Added get_FIELD_display() method for easy choice field displaying. - Added queryset.slave_okay(enabled) method - Updated queryset.timeout(enabled) and queryset.snapshot(enabled) to be chainable - Added insert method for bulk inserts - Added blinker signal support - Added query_counter context manager for tests -- Added DereferenceBaseField - for improved performance in field dereferencing - Added optional map_reduce method item_frequencies - Added inline_map_reduce option to map_reduce - Updated connection exception so it provides more info on the cause. From a66417e9d098b03b5dfaf04ab23fa8d185dd38e2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 9 Jun 2011 11:31:47 +0100 Subject: [PATCH 0256/1279] pep8 update --- tests/fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fields.py b/tests/fields.py index 4d51ed5..1b19998 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -358,7 +358,7 @@ class FieldTest(unittest.TestCase): self.assertEquals(BlogPost.objects.filter(info__100__test__exact='test').count(), 0) BlogPost.drop_collection() - def test_list_field_Strict(self): + def test_list_field_strict(self): """Ensure that list field handles validation if provided a strict field type.""" class Simple(Document): @@ -465,7 +465,7 @@ class FieldTest(unittest.TestCase): self.assertEquals(BlogPost.objects.filter(info__made_up__test__exact='test').count(), 0) BlogPost.drop_collection() - def test_dictfield_Strict(self): + def test_dictfield_strict(self): """Ensure that dict field handles validation if provided a strict field type.""" class Simple(Document): From 199b4eb860a93c581c1ddfc915f7094fc28de678 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 9 Jun 2011 12:08:37 +0100 Subject: [PATCH 0257/1279] Added django_tests and regression test for order_by Refs #190 --- setup.py | 2 +- tests/django_tests.py | 44 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 tests/django_tests.py diff --git a/setup.py b/setup.py index d3be64b..1f65ae5 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,6 @@ setup(name='mongoengine', long_description=LONG_DESCRIPTION, platforms=['any'], classifiers=CLASSIFIERS, - install_requires=['pymongo', 'blinker'], + install_requires=['pymongo', 'blinker', 'django>=1.3'], test_suite='tests', ) diff --git a/tests/django_tests.py b/tests/django_tests.py new file mode 100644 index 0000000..e5e2602 --- /dev/null +++ b/tests/django_tests.py @@ -0,0 +1,44 @@ + +# -*- coding: utf-8 -*- + +import unittest + +from mongoengine import * + + +class QuerySetTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + + class Person(Document): + name = StringField() + age = IntField() + self.Person = Person + + def test_order_by_in_django_template(self): + """Ensure that QuerySets are properly ordered in Django template. + """ + self.Person.drop_collection() + + self.Person(name="A", age=20).save() + self.Person(name="D", age=10).save() + self.Person(name="B", age=40).save() + self.Person(name="C", age=30).save() + + from django.conf import settings + settings.configure() + from django.template import Context, Template + + t = Template("{% for o in ol %}{{ o.name }}-{{ o.age }}:{% endfor %}") + + d = {"ol": self.Person.objects.order_by('-name')} + self.assertEqual(t.render(Context(d)), u'D-10:C-30:B-40:A-20:') + d = {"ol": self.Person.objects.order_by('+name')} + self.assertEqual(t.render(Context(d)), u'A-20:B-40:C-30:D-10:') + d = {"ol": self.Person.objects.order_by('-age')} + self.assertEqual(t.render(Context(d)), u'B-40:C-30:A-20:D-10:') + d = {"ol": self.Person.objects.order_by('+age')} + self.assertEqual(t.render(Context(d)), u'D-10:A-20:C-30:B-40:') + + self.Person.drop_collection() \ No newline at end of file From 417bb1b35d21c4bf02cb0acfd95f5b1ff6c49d70 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 9 Jun 2011 12:15:36 +0100 Subject: [PATCH 0258/1279] Added regression test for #185 --- tests/django_tests.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/django_tests.py b/tests/django_tests.py index e5e2602..6be1ea2 100644 --- a/tests/django_tests.py +++ b/tests/django_tests.py @@ -5,6 +5,9 @@ import unittest from mongoengine import * +from django.template import Context, Template +from django.conf import settings +settings.configure() class QuerySetTest(unittest.TestCase): @@ -26,10 +29,6 @@ class QuerySetTest(unittest.TestCase): self.Person(name="B", age=40).save() self.Person(name="C", age=30).save() - from django.conf import settings - settings.configure() - from django.template import Context, Template - t = Template("{% for o in ol %}{{ o.name }}-{{ o.age }}:{% endfor %}") d = {"ol": self.Person.objects.order_by('-name')} @@ -41,4 +40,18 @@ class QuerySetTest(unittest.TestCase): d = {"ol": self.Person.objects.order_by('+age')} self.assertEqual(t.render(Context(d)), u'D-10:A-20:C-30:B-40:') - self.Person.drop_collection() \ No newline at end of file + self.Person.drop_collection() + + def test_q_object_filter_in_template(self): + + self.Person.drop_collection() + + self.Person(name="A", age=20).save() + self.Person(name="D", age=10).save() + self.Person(name="B", age=40).save() + self.Person(name="C", age=30).save() + + t = Template("{% for o in ol %}{{ o.name }}-{{ o.age }}:{% endfor %}") + + d = {"ol": self.Person.objects.filter(Q(age=10) | Q(name="C"))} + self.assertEqual(t.render(Context(d)), u'D-10:C-30:') \ No newline at end of file From b2848b85194dee2429d35036c96a6c800cef42bf Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 9 Jun 2011 14:20:21 +0100 Subject: [PATCH 0259/1279] Added ComplexDateTimeField Thanks to @pelletier for the code. Refs #187 --- docs/apireference.rst | 2 + mongoengine/fields.py | 97 +++++++++++++++++++++++++++++++++++++++- tests/fields.py | 101 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 198 insertions(+), 2 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index a3d287a..2442803 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -53,6 +53,8 @@ Fields .. autoclass:: mongoengine.DateTimeField +.. autoclass:: mongoengine.ComplexDateTimeField + .. autoclass:: mongoengine.EmbeddedDocumentField .. autoclass:: mongoengine.DictField diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f9b2580..5d5304a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -18,8 +18,9 @@ import gridfs __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', 'MapField', - 'DecimalField', 'URLField', 'GenericReferenceField', 'FileField', - 'BinaryField', 'SortedListField', 'EmailField', 'GeoPointField'] + 'DecimalField', 'ComplexDateTimeField', 'URLField', + 'GenericReferenceField', 'FileField', 'BinaryField', + 'SortedListField', 'EmailField', 'GeoPointField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -273,6 +274,98 @@ class DateTimeField(BaseField): return None +class ComplexDateTimeField(StringField): + """ + ComplexDateTimeField handles microseconds exactly instead of rounding + like DateTimeField does. + + Derives from a StringField so you can do `gte` and `lte` filtering by + using lexicographical comparison when filtering / sorting strings. + + The stored string has the following format: + + YYYY,MM,DD,HH,MM,SS,NNNNNN + + Where NNNNNN is the number of microseconds of the represented `datetime`. + The `,` as the separator can be easily modified by passing the `separator` + keyword when initializing the field. + """ + + def __init__(self, separator=',', **kwargs): + self.names = ['year', 'month', 'day', 'hour', 'minute', 'second', + 'microsecond'] + self.separtor = separator + super(ComplexDateTimeField, self).__init__(**kwargs) + + def _leading_zero(self, number): + """ + Converts the given number to a string. + + If it has only one digit, a leading zero so as it has always at least + two digits. + """ + if int(number) < 10: + return "0%s" % number + else: + return str(number) + + def _convert_from_datetime(self, val): + """ + Convert a `datetime` object to a string representation (which will be + stored in MongoDB). This is the reverse function of + `_convert_from_string`. + + >>> a = datetime(2011, 6, 8, 20, 26, 24, 192284) + >>> RealDateTimeField()._convert_from_datetime(a) + '2011,06,08,20,26,24,192284' + """ + data = [] + for name in self.names: + data.append(self._leading_zero(getattr(val, name))) + return ','.join(data) + + def _convert_from_string(self, data): + """ + Convert a string representation to a `datetime` object (the object you + will manipulate). This is the reverse function of + `_convert_from_datetime`. + + >>> a = '2011,06,08,20,26,24,192284' + >>> ComplexDateTimeField()._convert_from_string(a) + datetime.datetime(2011, 6, 8, 20, 26, 24, 192284) + """ + data = data.split(',') + data = map(int, data) + values = {} + for i in range(7): + values[self.names[i]] = data[i] + return datetime.datetime(**values) + + def __get__(self, instance, owner): + data = super(ComplexDateTimeField, self).__get__(instance, owner) + if data == None: + return datetime.datetime.now() + return self._convert_from_string(data) + + def __set__(self, obj, val): + data = self._convert_from_datetime(val) + return super(ComplexDateTimeField, self).__set__(obj, data) + + def validate(self, value): + if not isinstance(value, datetime.datetime): + raise ValidationError('Only datetime objects may used in a \ + ComplexDateTimeField') + + def to_python(self, value): + return self._convert_from_string(value) + + def to_mongo(self, value): + return self._convert_from_datetime(value) + + def prepare_query_value(self, op, value): + return self._convert_from_datetime(value) + + class EmbeddedDocumentField(BaseField): """An embedded document field. Only valid values are subclasses of :class:`~mongoengine.EmbeddedDocument`. diff --git a/tests/fields.py b/tests/fields.py index 1b19998..531167c 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -247,6 +247,107 @@ class FieldTest(unittest.TestCase): LogEntry.drop_collection() + def test_complexdatetime_storage(self): + """Tests for complex datetime fields - which can handle microseconds + without rounding. + """ + class LogEntry(Document): + date = ComplexDateTimeField() + + LogEntry.drop_collection() + + # Post UTC - microseconds are rounded (down) nearest millisecond and dropped - with default datetimefields + d1 = datetime.datetime(1970, 01, 01, 00, 00, 01, 999) + log = LogEntry() + log.date = d1 + log.save() + log.reload() + self.assertEquals(log.date, d1) + + # Post UTC - microseconds are rounded (down) nearest millisecond - with default datetimefields + d1 = datetime.datetime(1970, 01, 01, 00, 00, 01, 9999) + log.date = d1 + log.save() + log.reload() + self.assertEquals(log.date, d1) + + # Pre UTC dates microseconds below 1000 are dropped - with default datetimefields + d1 = datetime.datetime(1969, 12, 31, 23, 59, 59, 999) + log.date = d1 + log.save() + log.reload() + self.assertEquals(log.date, d1) + + # Pre UTC microseconds above 1000 is wonky - with default datetimefields + # log.date has an invalid microsecond value so I can't construct + # a date to compare. + for i in xrange(1001, 3113, 33): + d1 = datetime.datetime(1969, 12, 31, 23, 59, 59, i) + log.date = d1 + log.save() + log.reload() + self.assertEquals(log.date, d1) + log1 = LogEntry.objects.get(date=d1) + self.assertEqual(log, log1) + + LogEntry.drop_collection() + + def test_complexdatetime_usage(self): + """Tests for complex datetime fields - which can handle microseconds + without rounding. + """ + class LogEntry(Document): + date = ComplexDateTimeField() + + LogEntry.drop_collection() + + d1 = datetime.datetime(1970, 01, 01, 00, 00, 01, 999) + log = LogEntry() + log.date = d1 + log.save() + + log1 = LogEntry.objects.get(date=d1) + self.assertEquals(log, log1) + + LogEntry.drop_collection() + + # create 60 log entries + for i in xrange(1950, 2010): + d = datetime.datetime(i, 01, 01, 00, 00, 01, 999) + LogEntry(date=d).save() + + self.assertEqual(LogEntry.objects.count(), 60) + + # Test ordering + logs = LogEntry.objects.order_by("date") + count = logs.count() + i = 0 + while i == count-1: + self.assertTrue(logs[i].date <= logs[i+1].date) + i +=1 + + logs = LogEntry.objects.order_by("-date") + count = logs.count() + i = 0 + while i == count-1: + self.assertTrue(logs[i].date >= logs[i+1].date) + i +=1 + + # Test searching + logs = LogEntry.objects.filter(date__gte=datetime.datetime(1980,1,1)) + self.assertEqual(logs.count(), 30) + + logs = LogEntry.objects.filter(date__lte=datetime.datetime(1980,1,1)) + self.assertEqual(logs.count(), 30) + + logs = LogEntry.objects.filter( + date__lte=datetime.datetime(2011,1,1), + date__gte=datetime.datetime(2000,1,1), + ) + self.assertEqual(logs.count(), 10) + + LogEntry.drop_collection() + def test_list_validation(self): """Ensure that a list field only accepts lists with valid elements. """ From fb09fde2097bd557a9173c749f45a8688cf62050 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 9 Jun 2011 14:26:52 +0100 Subject: [PATCH 0260/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index f4be4ca..0bbb5b8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added ComplexDateTimeField - Handles datetimes correctly with microseconds - Added ComplexBaseField - for improved flexibility and performance. - Added get_FIELD_display() method for easy choice field displaying. - Added queryset.slave_okay(enabled) method From fd7f882011ce548efd7ae5fcb0f59fd38d38e98b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 9 Jun 2011 16:09:06 +0100 Subject: [PATCH 0261/1279] Save no longer tramples over documents now sets or unsets explicit fields. Fixes #146, refs #18 Thanks @zhangcheng for the initial code --- docs/changelog.rst | 5 ++- mongoengine/base.py | 9 +++-- mongoengine/document.py | 10 +++++ setup.py | 2 +- tests/document.py | 84 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 104 insertions(+), 6 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0bbb5b8..ecd7ef5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,9 +5,10 @@ Changelog Changes in dev ============== +- Fixed saving so sets updated values rather than overwrites - Added ComplexDateTimeField - Handles datetimes correctly with microseconds -- Added ComplexBaseField - for improved flexibility and performance. -- Added get_FIELD_display() method for easy choice field displaying. +- Added ComplexBaseField - for improved flexibility and performance +- Added get_FIELD_display() method for easy choice field displaying - Added queryset.slave_okay(enabled) method - Updated queryset.timeout(enabled) and queryset.snapshot(enabled) to be chainable - Added insert method for bulk inserts diff --git a/mongoengine/base.py b/mongoengine/base.py index a22795c..aed17bc 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -92,6 +92,9 @@ class BaseField(object): """Descriptor for assigning a value to a field in a document. """ instance._data[self.name] = value + # If the field set is in the _present_fields list add it so we can track + if hasattr(instance, '_present_fields') and self.name not in instance._present_fields: + instance._present_fields.append(self.name) def to_python(self, value): """Convert a MongoDB-compatible type to a Python type. @@ -592,13 +595,14 @@ class BaseDocument(object): if field.choices: # dynamically adds a way to get the display value for a field with choices setattr(self, 'get_%s_display' % attr_name, partial(self._get_FIELD_display, field=field)) - # Use default value if present value = getattr(self, attr_name, None) setattr(self, attr_name, value) + # Assign initial values to instance for attr_name in values.keys(): try: - setattr(self, attr_name, values.pop(attr_name)) + value = values.pop(attr_name) + setattr(self, attr_name, value) except AttributeError: pass @@ -739,7 +743,6 @@ class BaseDocument(object): cls = subclasses[class_name] present_fields = data.keys() - for field_name, field in cls._fields.items(): if field.db_field in data: value = data[field.db_field] diff --git a/mongoengine/document.py b/mongoengine/document.py index cae8343..e25bea0 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -95,6 +95,16 @@ class Document(BaseDocument): collection = self.__class__.objects._collection if force_insert: object_id = collection.insert(doc, safe=safe, **write_options) + elif '_id' in doc: + # Perform a set rather than a save - this will only save set fields + object_id = doc.pop('_id') + collection.update({'_id': object_id}, {"$set": doc}, upsert=True, safe=safe, **write_options) + + # Find and unset any fields explicitly set to None + if hasattr(self, '_present_fields'): + removals = dict([(k, 1) for k in self._present_fields if k not in doc and k != '_id']) + if removals: + collection.update({'_id': object_id}, {"$unset": removals}, upsert=True, safe=safe, **write_options) else: object_id = collection.save(doc, safe=safe, **write_options) except pymongo.errors.OperationFailure, err: diff --git a/setup.py b/setup.py index 1f65ae5..37ec437 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,6 @@ setup(name='mongoengine', long_description=LONG_DESCRIPTION, platforms=['any'], classifiers=CLASSIFIERS, - install_requires=['pymongo', 'blinker', 'django>=1.3'], + install_requires=['pymongo', 'blinker', 'django==1.3'], test_suite='tests', ) diff --git a/tests/document.py b/tests/document.py index 1454146..f0af8f2 100644 --- a/tests/document.py +++ b/tests/document.py @@ -789,6 +789,90 @@ class DocumentTest(unittest.TestCase): except ValidationError: self.fail() + def test_update(self): + """Ensure that an existing document is updated instead of be overwritten. + """ + # Create person object and save it to the database + person = self.Person(name='Test User', age=30) + person.save() + + # Create same person object, with same id, without age + same_person = self.Person(name='Test') + same_person.id = person.id + same_person.save() + + # Confirm only one object + self.assertEquals(self.Person.objects.count(), 1) + + # reload + person.reload() + same_person.reload() + + # Confirm the same + self.assertEqual(person, same_person) + self.assertEqual(person.name, same_person.name) + self.assertEqual(person.age, same_person.age) + + # Confirm the saved values + self.assertEqual(person.name, 'Test') + self.assertEqual(person.age, 30) + + # Test only / exclude only updates included fields + person = self.Person.objects.only('name').get() + person.name = 'User' + person.save() + + person.reload() + self.assertEqual(person.name, 'User') + self.assertEqual(person.age, 30) + + # test exclude only updates set fields + person = self.Person.objects.exclude('name').get() + person.age = 21 + person.save() + + person.reload() + self.assertEqual(person.name, 'User') + self.assertEqual(person.age, 21) + + # Test only / exclude can set non excluded / included fields + person = self.Person.objects.only('name').get() + person.name = 'Test' + person.age = 30 + person.save() + + person.reload() + self.assertEqual(person.name, 'Test') + self.assertEqual(person.age, 30) + + # test exclude only updates set fields + person = self.Person.objects.exclude('name').get() + person.name = 'User' + person.age = 21 + person.save() + + person.reload() + self.assertEqual(person.name, 'User') + self.assertEqual(person.age, 21) + + # Confirm does remove unrequired fields + person = self.Person.objects.exclude('name').get() + person.age = None + person.save() + + person.reload() + self.assertEqual(person.name, 'User') + self.assertEqual(person.age, None) + + person = self.Person.objects.get() + person.name = None + person.age = None + person.save() + + person.reload() + self.assertEqual(person.name, None) + self.assertEqual(person.age, None) + def test_delete(self): """Ensure that document may be deleted using the delete method. """ From 82fbe7128f78d103f10b814d503717fe85b4cd0e Mon Sep 17 00:00:00 2001 From: Colin Howe Date: Fri, 10 Jun 2011 17:31:42 +0100 Subject: [PATCH 0262/1279] Improve validation warnings --- mongoengine/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index aed17bc..43a2196 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -627,8 +627,8 @@ class BaseDocument(object): try: field._validate(value) except (ValueError, AttributeError, AssertionError), e: - raise ValidationError('Invalid value for field of type "%s": %s' - % (field.__class__.__name__, value)) + raise ValidationError('Invalid value for field named "%s" of type "%s": %s' + % (field.name, field.__class__.__name__, value)) elif field.required: raise ValidationError('Field "%s" is required' % field.name) From 7b293783191f24302a2b8dd0b579ae8b839aae79 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 13 Jun 2011 12:40:12 +0100 Subject: [PATCH 0263/1279] Fixes issue converting to mongo --- mongoengine/fields.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 5d5304a..967ce83 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -394,7 +394,7 @@ class EmbeddedDocumentField(BaseField): return value def to_mongo(self, value): - if isinstance(value, basestring): + if not isinstance(value, self.document_type): return value return self.document_type.to_mongo(value) @@ -438,7 +438,8 @@ class ListField(ComplexBaseField): def prepare_query_value(self, op, value): if self.field: - if op in ('set', 'unset') and not isinstance(value, basestring): + if op in ('set', 'unset') and (not isinstance(value, basestring) + and hasattr(value, '__iter__')): return [self.field.prepare_query_value(op, v) for v in value] return self.field.prepare_query_value(op, value) return super(ListField, self).prepare_query_value(op, value) From ea35fb1c54d3f2deb76a31a21314d4b4cc29d177 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 13 Jun 2011 12:49:09 +0100 Subject: [PATCH 0264/1279] More robust _present_fields additions --- mongoengine/base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index aed17bc..592a678 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -91,9 +91,10 @@ class BaseField(object): def __set__(self, instance, value): """Descriptor for assigning a value to a field in a document. """ - instance._data[self.name] = value + key = self.name + instance._data[key] = value # If the field set is in the _present_fields list add it so we can track - if hasattr(instance, '_present_fields') and self.name not in instance._present_fields: + if hasattr(instance, '_present_fields') and key and key not in instance._present_fields: instance._present_fields.append(self.name) def to_python(self, value): From 0ed79a839d78ad69dcd3c60f51acfe5e83348f44 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 10 Jun 2011 17:22:05 +0100 Subject: [PATCH 0265/1279] Added delta tracking to documents. All saves on exisiting items do set / unset operations only on changed fields. * Note lists and dicts generally do set operations for things like pop() del[key] As there is no easy map to unset and explicitly matches the new list / dict fixes #18 --- docs/changelog.rst | 1 + docs/guide/document-instances.rst | 19 ++- mongoengine/base.py | 209 ++++++++++++++++++++++--- mongoengine/document.py | 69 +++++--- mongoengine/fields.py | 31 +++- tests/dereference.py | 4 +- tests/django_tests.py | 1 - tests/document.py | 251 +++++++++++++++++++++++++++++- tests/fields.py | 32 +++- 9 files changed, 552 insertions(+), 65 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ecd7ef5..54efb4f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added delta tracking now only sets / unsets explicitly changed fields - Fixed saving so sets updated values rather than overwrites - Added ComplexDateTimeField - Handles datetimes correctly with microseconds - Added ComplexBaseField - for improved flexibility and performance diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index 7b5d165..aeed7cd 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -18,10 +18,21 @@ attribute syntax:: Saving and deleting documents ============================= -To save the document to the database, call the -:meth:`~mongoengine.Document.save` method. If the document does not exist in -the database, it will be created. If it does already exist, it will be -updated. +MongoEngine tracks changes to documents to provide efficient saving. To save +the document to the database, call the :meth:`~mongoengine.Document.save` method. +If the document does not exist in the database, it will be created. If it does +already exist, then any changes will be updated atomically. For example:: + + >>> page = Page(title="Test Page") + >>> page.save() # Performs an insert + >>> page.title = "My Page" + >>> page.save() # Performs an atomic set on the title field. + +.. note:: + Changes to documents are tracked and on the whole perform `set` operations. + + * ``list_field.pop(0)`` - *sets* the resulting list + * ``del(list_field)`` - *unsets* whole list To delete a document, call the :meth:`~mongoengine.Document.delete` method. Note that this will only work if the document exists in the database and has a diff --git a/mongoengine/base.py b/mongoengine/base.py index 592a678..292184e 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -4,6 +4,7 @@ from queryset import DO_NOTHING from mongoengine import signals +import weakref import sys import pymongo import pymongo.objectid @@ -86,16 +87,19 @@ class BaseField(object): # Allow callable default values if callable(value): value = value() + + # Convert lists / values so we can watch for any changes on them + if isinstance(value, (list, tuple)) and not isinstance(value, BaseList): + value = BaseList(value, instance=instance, name=self.name) + elif isinstance(value, dict) and not isinstance(value, BaseDict): + value = BaseDict(value, instance=instance, name=self.name) return value def __set__(self, instance, value): """Descriptor for assigning a value to a field in a document. """ - key = self.name - instance._data[key] = value - # If the field set is in the _present_fields list add it so we can track - if hasattr(instance, '_present_fields') and key and key not in instance._present_fields: - instance._present_fields.append(self.name) + instance._data[self.name] = value + instance._mark_as_changed(self.name) def to_python(self, value): """Convert a MongoDB-compatible type to a Python type. @@ -173,21 +177,27 @@ class ComplexBaseField(BaseField): db = _get_db() dbref = {} collections = {} - for k, v in value_list.items(): - dbref[k] = v + for k,v in value_list.items(): + # Save any DBRefs if isinstance(v, (pymongo.dbref.DBRef)): # direct reference (DBRef) - collections.setdefault(v.collection, []).append((k, v)) - elif isinstance(v, (dict, pymongo.son.SON)) and '_ref' in v: - # generic reference - collection = get_document(v['_cls'])._meta['collection'] - collections.setdefault(collection, []).append((k, v)) + collections.setdefault(v.collection, []).append((k,v)) + elif isinstance(v, (dict, pymongo.son.SON)): + if '_ref' in v: + # generic reference + collection = get_document(v['_cls'])._meta['collection'] + collections.setdefault(collection, []).append((k,v)) + else: + # Use BaseDict so can watch any changes + dbref[k] = BaseDict(v, instance=instance, name=self.name) + else: + dbref[k] = v # For each collection get the references for collection, dbrefs in collections.items(): id_map = {} - for k, v in dbrefs: + for k,v in dbrefs: if isinstance(v, (pymongo.dbref.DBRef)): # direct reference (DBRef), has no _cls information id_map[v.id] = (k, None) @@ -203,7 +213,9 @@ class ComplexBaseField(BaseField): dbref[key] = doc_cls._from_son(ref) if is_list: - dbref = [v for k,v in sorted(dbref.items(), key=itemgetter(0))] + dbref = BaseList([v for k,v in sorted(dbref.items(), key=itemgetter(0))], instance=instance, name=self.name) + else: + dbref = BaseDict(dbref, instance=instance, name=self.name) instance._data[self.name] = dbref return super(ComplexBaseField, self).__get__(instance, owner) @@ -304,7 +316,7 @@ class ComplexBaseField(BaseField): if hasattr(value, 'iteritems'): [self.field.validate(v) for k,v in value.iteritems()] else: - [self.field.validate(v) for v in value] + [self.field.validate(v) for v in value] except Exception, err: raise ValidationError('Invalid %s item (%s)' % ( self.field.__class__.__name__, str(v))) @@ -714,7 +726,7 @@ class BaseDocument(object): self._meta.get('allow_inheritance', True) == False): data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] - if data.has_key('_id') and data['_id'] is None: + if '_id' in data and data['_id'] is None: del data['_id'] return data @@ -751,9 +763,71 @@ class BaseDocument(object): else field.to_python(value)) obj = cls(**data) - obj._present_fields = present_fields + obj._changed_fields = [] return obj + def _mark_as_changed(self, key): + """Marks a key as explicitly changed by the user + """ + if not key: + return + if hasattr(self, '_changed_fields') and key not in self._changed_fields: + self._changed_fields.append(key) + + def _get_changed_fields(self, key=''): + """Returns a list of all fields that have explicitly been changed. + """ + from mongoengine import EmbeddedDocument + _changed_fields = [] + _changed_fields += getattr(self, '_changed_fields', []) + + for field_name in self._fields: + key = '%s.' % field_name + field = getattr(self, field_name, None) + if isinstance(field, EmbeddedDocument): # Grab all embedded fields that have been changed + _changed_fields += ["%s%s" % (key, k) for k in field._get_changed_fields(key) if k] + elif isinstance(field, (list, tuple)): # Loop list fields as they contain documents + for index, value in enumerate(field): + if not hasattr(value, '_get_changed_fields'): + continue + list_key = "%s%s." % (key, index) + _changed_fields += ["%s%s" % (list_key, k) for k in value._get_changed_fields(list_key) if k] + return _changed_fields + + def _delta(self): + """Returns the delta (set, unset) of the changes for a document. + Gets any values that have been explicitly changed. + """ + # Handles cases where not loaded from_son but has _id + doc = self.to_mongo() + set_fields = self._get_changed_fields() + set_data = {} + unset_data = {} + if hasattr(self, '_changed_fields'): + set_data = {} + # Fetch each set item from its path + for path in set_fields: + parts = path.split('.') + d = doc + for p in parts: + if hasattr(d, '__getattr__'): + d = getattr(p, d) + elif p.isdigit(): + d = d[int(p)] + else: + d = d.get(p) + set_data[path] = d + else: + set_data = doc + if '_id' in set_data: + del(set_data['_id']) + + for k,v in set_data.items(): + if not v: + del(set_data[k]) + unset_data[k] = 1 + return set_data, unset_data + def __eq__(self, other): if isinstance(other, self.__class__) and hasattr(other, 'id'): if self.id == other.id: @@ -764,13 +838,112 @@ class BaseDocument(object): return not self.__eq__(other) def __hash__(self): - """ For list, dic key """ + """ For list, dict key """ if self.pk is None: # For new object return super(BaseDocument,self).__hash__() else: return hash(self.pk) + +class BaseList(list): + """A special list so we can watch any changes + """ + + def __init__(self, list_items, instance, name): + self.instance = weakref.proxy(instance) + self.name = name + super(BaseList, self).__init__(list_items) + + def __setitem__(self, *args, **kwargs): + if hasattr(self, 'instance') and hasattr(self, 'name'): + self.instance._mark_as_changed(self.name) + super(BaseDict, self).__setitem__(*args, **kwargs) + + def __delitem__(self, *args, **kwargs): + self.instance._mark_as_changed(self.name) + super(BaseList, self).__delitem__(*args, **kwargs) + + def __delete__(self, *args, **kwargs): + if hasattr(self, 'instance') and hasattr(self, 'name'): + import ipdb; ipdb.set_trace() + self.instance._mark_as_changed(self.name) + delattr(self, 'instance') + delattr(self, 'name') + super(BaseDict, self).__delete__(*args, **kwargs) + + def append(self, *args, **kwargs): + self.instance._mark_as_changed(self.name) + return super(BaseList, self).append(*args, **kwargs) + + def extend(self, *args, **kwargs): + self.instance._mark_as_changed(self.name) + return super(BaseList, self).extend(*args, **kwargs) + + def insert(self, *args, **kwargs): + self.instance._mark_as_changed(self.name) + return super(BaseList, self).insert(*args, **kwargs) + + def pop(self, *args, **kwargs): + self.instance._mark_as_changed(self.name) + return super(BaseList, self).pop(*args, **kwargs) + + def remove(self, *args, **kwargs): + self.instance._mark_as_changed(self.name) + return super(BaseList, self).remove(*args, **kwargs) + + def reverse(self, *args, **kwargs): + self.instance._mark_as_changed(self.name) + return super(BaseList, self).reverse(*args, **kwargs) + + def sort(self, *args, **kwargs): + self.instance._mark_as_changed(self.name) + return super(BaseList, self).sort(*args, **kwargs) + + +class BaseDict(dict): + """A special dict so we can watch any changes + """ + + def __init__(self, dict_items, instance, name): + self.instance = weakref.proxy(instance) + self.name = name + super(BaseDict, self).__init__(dict_items) + + def __setitem__(self, *args, **kwargs): + if hasattr(self, 'instance') and hasattr(self, 'name'): + self.instance._mark_as_changed(self.name) + super(BaseDict, self).__setitem__(*args, **kwargs) + + def __setattr__(self, *args, **kwargs): + if hasattr(self, 'instance') and hasattr(self, 'name'): + self.instance._mark_as_changed(self.name) + super(BaseDict, self).__setattr__(*args, **kwargs) + + def __delete__(self, *args, **kwargs): + self.instance._mark_as_changed(self.name) + super(BaseDict, self).__delete__(*args, **kwargs) + + def __delitem__(self, *args, **kwargs): + self.instance._mark_as_changed(self.name) + super(BaseDict, self).__delitem__(*args, **kwargs) + + def __delattr__(self, *args, **kwargs): + self.instance._mark_as_changed(self.name) + super(BaseDict, self).__delattr__(*args, **kwargs) + + def clear(self, *args, **kwargs): + self.instance._mark_as_changed(self.name) + super(BaseDict, self).clear(*args, **kwargs) + + def pop(self, *args, **kwargs): + self.instance._mark_as_changed(self.name) + super(BaseDict, self).clear(*args, **kwargs) + + def popitem(self, *args, **kwargs): + self.instance._mark_as_changed(self.name) + super(BaseDict, self).clear(*args, **kwargs) + if sys.version_info < (2, 5): # Prior to Python 2.5, Exception was an old-style class import types diff --git a/mongoengine/document.py b/mongoengine/document.py index e25bea0..2f40eec 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,12 +1,11 @@ from mongoengine import signals from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, - ValidationError) + ValidationError, BaseDict, BaseList) from queryset import OperationError from connection import _get_db import pymongo - __all__ = ['Document', 'EmbeddedDocument', 'ValidationError', 'OperationError'] @@ -19,6 +18,18 @@ class EmbeddedDocument(BaseDocument): __metaclass__ = DocumentMetaclass + def __delattr__(self, *args, **kwargs): + """Handle deletions of fields""" + field_name = args[0] + if field_name in self._fields: + default = self._fields[field_name].default + if callable(default): + default = default() + setattr(self, field_name, default) + else: + super(EmbeddedDocument, self).__delattr__(*args, **kwargs) + + class Document(BaseDocument): """The base class used for defining the structure and properties of @@ -59,7 +70,6 @@ class Document(BaseDocument): disabled by either setting types to False on the specific index or by setting index_types to False on the meta dictionary for the document. """ - __metaclass__ = TopLevelDocumentMetaclass def save(self, safe=True, force_insert=False, validate=True, write_options=None): @@ -95,18 +105,15 @@ class Document(BaseDocument): collection = self.__class__.objects._collection if force_insert: object_id = collection.insert(doc, safe=safe, **write_options) - elif '_id' in doc: - # Perform a set rather than a save - this will only save set fields - object_id = doc.pop('_id') - collection.update({'_id': object_id}, {"$set": doc}, upsert=True, safe=safe, **write_options) - - # Find and unset any fields explicitly set to None - if hasattr(self, '_present_fields'): - removals = dict([(k, 1) for k in self._present_fields if k not in doc and k != '_id']) - if removals: - collection.update({'_id': object_id}, {"$unset": removals}, upsert=True, safe=safe, **write_options) - else: + if created: object_id = collection.save(doc, safe=safe, **write_options) + else: + object_id = doc['_id'] + updates, removals = self._delta() + if updates: + collection.update({'_id': object_id}, {"$set": updates}, upsert=True, safe=safe, **write_options) + if removals: + collection.update({'_id': object_id}, {"$unset": removals}, upsert=True, safe=safe, **write_options) except pymongo.errors.OperationFailure, err: message = 'Could not save document (%s)' if u'duplicate key' in unicode(err): @@ -114,7 +121,7 @@ class Document(BaseDocument): raise OperationError(message % unicode(err)) id_field = self._meta['id_field'] self[id_field] = self._fields[id_field].to_python(object_id) - + self._changed_fields = [] signals.post_save.send(self, created=created) def delete(self, safe=False): @@ -135,14 +142,6 @@ class Document(BaseDocument): signals.post_delete.send(self) - @classmethod - def register_delete_rule(cls, document_cls, field_name, rule): - """This method registers the delete rules to apply when removing this - object. - """ - cls._meta['delete_rules'][(document_cls, field_name)] = rule - - def reload(self): """Reloads all attributes from the database. @@ -151,7 +150,29 @@ class Document(BaseDocument): id_field = self._meta['id_field'] obj = self.__class__.objects(**{id_field: self[id_field]}).first() for field in self._fields: - setattr(self, field, obj[field]) + setattr(self, field, self._reload(field, obj[field])) + self._changed_fields = [] + + def _reload(self, key, value): + """Used by :meth:`~mongoengine.Document.reload` to ensure the + correct instance is linked to self. + """ + if isinstance(value, BaseDict): + value = [(k, self._reload(k,v)) for k,v in value.items()] + value = BaseDict(value, instance=self, name=key) + elif isinstance(value, BaseList): + value = [self._reload(key, v) for v in value] + value = BaseList(value, instance=self, name=key) + elif isinstance(value, EmbeddedDocument): + value._changed_fields = [] + return value + + @classmethod + def register_delete_rule(cls, document_cls, field_name, rule): + """This method registers the delete rules to apply when removing this + object. + """ + cls._meta['delete_rules'][(document_cls, field_name)] = rule @classmethod def drop_collection(cls): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 967ce83..eeb4c2c 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -347,9 +347,9 @@ class ComplexDateTimeField(StringField): return datetime.datetime.now() return self._convert_from_string(data) - def __set__(self, obj, val): - data = self._convert_from_datetime(val) - return super(ComplexDateTimeField, self).__set__(obj, data) + def __set__(self, instance, value): + value = self._convert_from_datetime(value) + return super(ComplexDateTimeField, self).__set__(instance, value) def validate(self, value): if not isinstance(value, datetime.datetime): @@ -686,11 +686,13 @@ class GridFSProxy(object): .. versionadded:: 0.4 """ - def __init__(self, grid_id=None): + def __init__(self, grid_id=None, key=None, instance=None): self.fs = gridfs.GridFS(_get_db()) # Filesystem instance self.newfile = None # Used for partial writes self.grid_id = grid_id # Store GridFS id for file self.gridout = None + self.key = key + self.instance = instance def __getattr__(self, name): obj = self.get() @@ -723,6 +725,7 @@ class GridFSProxy(object): raise GridFSError('This document already has a file. Either delete ' 'it or call replace to overwrite it') self.grid_id = self.fs.put(file_obj, **kwargs) + self._mark_as_changed() def write(self, string): if self.grid_id: @@ -750,6 +753,12 @@ class GridFSProxy(object): self.fs.delete(self.grid_id) self.grid_id = None self.gridout = None + self._mark_as_changed() + + def _mark_as_changed(self): + """Inform the instance that `self.key` has been changed""" + if self.instance: + self.instance._mark_as_changed(self.key) def replace(self, file_obj, **kwargs): self.delete() @@ -777,10 +786,14 @@ class FileField(BaseField): grid_file = instance._data.get(self.name) self.grid_file = grid_file if self.grid_file: + if not self.grid_file.key: + self.grid_file.key = self.name + self.grid_file.instance = instance return self.grid_file - return GridFSProxy() + return GridFSProxy(key=self.name, instance=instance) def __set__(self, instance, value): + key = self.name if isinstance(value, file) or isinstance(value, str): # using "FileField() = file/string" notation grid_file = instance._data.get(self.name) @@ -794,10 +807,12 @@ class FileField(BaseField): grid_file.put(value) else: # Create a new proxy object as we don't already have one - instance._data[self.name] = GridFSProxy() - instance._data[self.name].put(value) + instance._data[key] = GridFSProxy(key=key, instance=instance) + instance._data[key].put(value) else: - instance._data[self.name] = value + instance._data[key] = value + + instance._mark_as_changed(key) def to_mongo(self, value): # Store the GridFS file id in MongoDB diff --git a/tests/dereference.py b/tests/dereference.py index 6879272..4040d5b 100644 --- a/tests/dereference.py +++ b/tests/dereference.py @@ -281,9 +281,7 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] self.assertEqual(q, 1) - - for k, m in group_obj.members.iteritems(): - self.assertTrue('User' in m.__class__.__name__) + self.assertEqual(group_obj.members, {}) UserA.drop_collection() UserB.drop_collection() diff --git a/tests/django_tests.py b/tests/django_tests.py index 6be1ea2..ee8084c 100644 --- a/tests/django_tests.py +++ b/tests/django_tests.py @@ -1,4 +1,3 @@ - # -*- coding: utf-8 -*- import unittest diff --git a/tests/document.py b/tests/document.py index f0af8f2..4c89080 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2,6 +2,7 @@ import unittest from datetime import datetime import pymongo import pickle +import weakref from mongoengine import * from mongoengine.base import BaseField @@ -11,6 +12,7 @@ from mongoengine.connection import _get_db class PickleEmbedded(EmbeddedDocument): date = DateTimeField(default=datetime.now) + class PickleTest(Document): number = IntField() string = StringField() @@ -717,6 +719,47 @@ class DocumentTest(unittest.TestCase): self.assertEqual(person.name, "Mr Test User") self.assertEqual(person.age, 21) + def test_reload_referencing(self): + """Ensures reloading updates weakrefs correctly + """ + class Embedded(EmbeddedDocument): + dict_field = DictField() + list_field = ListField() + + class Doc(Document): + dict_field = DictField() + list_field = ListField() + embedded_field = EmbeddedDocumentField(Embedded) + + Doc.drop_collection + doc = Doc() + doc.dict_field = {'hello': 'world'} + doc.list_field = ['1', 2, {'hello': 'world'}] + + embedded_1 = Embedded() + embedded_1.dict_field = {'hello': 'world'} + embedded_1.list_field = ['1', 2, {'hello': 'world'}] + doc.embedded_field = embedded_1 + doc.save() + + doc.reload() + doc.list_field.append(1) + doc.dict_field['woot'] = "woot" + doc.embedded_field.list_field.append(1) + doc.embedded_field.dict_field['woot'] = "woot" + + self.assertEquals(doc._get_changed_fields(), [ + 'list_field', 'dict_field', 'embedded_field.list_field', + 'embedded_field.dict_field']) + doc.save() + + doc.reload() + self.assertEquals(doc._get_changed_fields(), []) + self.assertEquals(len(doc.list_field), 4) + self.assertEquals(len(doc.dict_field), 2) + self.assertEquals(len(doc.embedded_field.list_field), 4) + self.assertEquals(len(doc.embedded_field.dict_field), 2) + def test_dictionary_access(self): """Ensure that dictionary-style field access works properly. """ @@ -873,6 +916,197 @@ class DocumentTest(unittest.TestCase): self.assertEqual(person.name, None) self.assertEqual(person.age, None) + def test_delta(self): + + class Doc(Document): + string_field = StringField() + int_field = IntField() + dict_field = DictField() + list_field = ListField() + + Doc.drop_collection + doc = Doc() + doc.save() + + doc = Doc.objects.first() + self.assertEquals(doc._get_changed_fields(), []) + self.assertEquals(doc._delta(), ({}, {})) + + doc.string_field = 'hello' + self.assertEquals(doc._delta(), ({'string_field': 'hello'}, {})) + + doc._changed_fields = [] + doc.int_field = 1 + self.assertEquals(doc._delta(), ({'int_field': 1}, {})) + + doc._changed_fields = [] + dict_value = {'hello': 'world', 'ping': 'pong'} + doc.dict_field = dict_value + self.assertEquals(doc._delta(), ({'dict_field': dict_value}, {})) + + doc._changed_fields = [] + list_value = ['1', 2, {'hello': 'world'}] + doc.list_field = list_value + self.assertEquals(doc._delta(), ({'list_field': list_value}, {})) + + # Test unsetting + doc._changed_fields = [] + doc._unset_fields = [] + doc.dict_field = {} + self.assertEquals(doc._delta(), ({}, {'dict_field': 1})) + + doc._changed_fields = [] + doc._unset_fields = {} + doc.list_field = [] + self.assertEquals(doc._delta(), ({}, {'list_field': 1})) + + def test_delta_recursive(self): + + class Embedded(EmbeddedDocument): + string_field = StringField() + int_field = IntField() + dict_field = DictField() + list_field = ListField() + + class Doc(Document): + string_field = StringField() + int_field = IntField() + dict_field = DictField() + list_field = ListField() + embedded_field = EmbeddedDocumentField(Embedded) + + Doc.drop_collection + doc = Doc() + doc.save() + + doc = Doc.objects.first() + self.assertEquals(doc._get_changed_fields(), []) + self.assertEquals(doc._delta(), ({}, {})) + + embedded_1 = Embedded() + embedded_1.string_field = 'hello' + embedded_1.int_field = 1 + embedded_1.dict_field = {'hello': 'world'} + embedded_1.list_field = ['1', 2, {'hello': 'world'}] + doc.embedded_field = embedded_1 + + embedded_delta = { + '_types': ['Embedded'], + '_cls': 'Embedded', + 'string_field': 'hello', + 'int_field': 1, + 'dict_field': {'hello': 'world'}, + 'list_field': ['1', 2, {'hello': 'world'}] + } + self.assertEquals(doc.embedded_field._delta(), (embedded_delta, {})) + self.assertEquals(doc._delta(), ({'embedded_field': embedded_delta}, {})) + + doc.save() + doc.reload() + + doc.embedded_field.dict_field = {} + self.assertEquals(doc.embedded_field._delta(), ({}, {'dict_field': 1})) + self.assertEquals(doc._delta(), ({}, {'embedded_field.dict_field': 1})) + doc.save() + doc.reload() + self.assertEquals(doc.embedded_field.dict_field, {}) + + doc.embedded_field.list_field = [] + self.assertEquals(doc.embedded_field._delta(), ({}, {'list_field': 1})) + self.assertEquals(doc._delta(), ({}, {'embedded_field.list_field': 1})) + doc.save() + doc.reload() + self.assertEquals(doc.embedded_field.list_field, []) + + embedded_2 = Embedded() + embedded_2.string_field = 'hello' + embedded_2.int_field = 1 + embedded_2.dict_field = {'hello': 'world'} + embedded_2.list_field = ['1', 2, {'hello': 'world'}] + + doc.embedded_field.list_field = ['1', 2, embedded_2] + self.assertEquals(doc.embedded_field._delta(), ({ + 'list_field': ['1', 2, { + '_cls': 'Embedded', + '_types': ['Embedded'], + 'string_field': 'hello', + 'dict_field': {'hello': 'world'}, + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], + }] + }, {})) + + self.assertEquals(doc._delta(), ({ + 'embedded_field.list_field': ['1', 2, { + '_cls': 'Embedded', + '_types': ['Embedded'], + 'string_field': 'hello', + 'dict_field': {'hello': 'world'}, + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], + }] + }, {})) + doc.save() + doc.reload() + + self.assertEquals(doc.embedded_field.list_field[0], '1') + self.assertEquals(doc.embedded_field.list_field[1], 2) + for k in doc.embedded_field.list_field[2]._fields: + self.assertEquals(doc.embedded_field.list_field[2][k], embedded_2[k]) + + doc.embedded_field.list_field[2].string_field = 'world' + self.assertEquals(doc.embedded_field._delta(), ({'list_field.2.string_field': 'world'}, {})) + self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.string_field': 'world'}, {})) + doc.save() + doc.reload() + self.assertEquals(doc.embedded_field.list_field[2].string_field, 'world') + + # Test list native methods + doc.embedded_field.list_field[2].list_field.pop(0) + self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}]}, {})) + doc.save() + doc.reload() + + doc.embedded_field.list_field[2].list_field.append(1) + self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}, 1]}, {})) + doc.save() + doc.reload() + self.assertEquals(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) + + doc.embedded_field.list_field[2].list_field.sort() + doc.save() + doc.reload() + self.assertEquals(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) + + del(doc.embedded_field.list_field[2].list_field[2]['hello']) + self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) + doc.save() + doc.reload() + + del(doc.embedded_field.list_field[2].list_field) + self.assertEquals(doc._delta(), ({}, {'embedded_field.list_field.2.list_field': 1})) + + def test_save_only_changed_fields(self): + """Ensure save only sets / unsets changed fields + """ + + # Create person object and save it to the database + person = self.Person(name='Test User', age=30) + person.save() + person.reload() + + same_person = self.Person.objects.get() + + person.age = 21 + same_person.name = 'User' + + person.save() + same_person.save() + + person = self.Person.objects.get() + self.assertEquals(person.name, 'User') + self.assertEquals(person.age, 21) + def test_delete(self): """Ensure that document may be deleted using the delete method. """ @@ -978,12 +1212,19 @@ class DocumentTest(unittest.TestCase): promoted_employee.details.position = 'Senior Developer' promoted_employee.save() - collection = self.db[self.Person._meta['collection']] - employee_obj = collection.find_one({'name': 'Test Employee'}) - self.assertEqual(employee_obj['name'], 'Test Employee') - self.assertEqual(employee_obj['age'], 50) + promoted_employee.reload() + self.assertEqual(promoted_employee.name, 'Test Employee') + self.assertEqual(promoted_employee.age, 50) # Ensure that the 'details' embedded object saved correctly - self.assertEqual(employee_obj['details']['position'], 'Senior Developer') + self.assertEqual(promoted_employee.details.position, 'Senior Developer') + + # Test removal + promoted_employee.details = None + promoted_employee.save() + + promoted_employee.reload() + self.assertEqual(promoted_employee.details, None) + def test_save_reference(self): """Ensure that a document reference field may be saved in the database. diff --git a/tests/fields.py b/tests/fields.py index 531167c..79cd519 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -843,6 +843,7 @@ class FieldTest(unittest.TestCase): name = StringField() children = ListField(EmbeddedDocumentField('self')) + Tree.drop_collection tree = Tree(name="Tree") first_child = TreeNode(name="Child 1") @@ -853,15 +854,42 @@ class FieldTest(unittest.TestCase): third_child = TreeNode(name="Child 3") first_child.children.append(third_child) - tree.save() - tree_obj = Tree.objects.first() self.assertEqual(len(tree.children), 1) self.assertEqual(tree.children[0].name, first_child.name) self.assertEqual(tree.children[0].children[0].name, second_child.name) self.assertEqual(tree.children[0].children[1].name, third_child.name) + # Test updating + tree.children[0].name = 'I am Child 1' + tree.children[0].children[0].name = 'I am Child 2' + tree.children[0].children[1].name = 'I am Child 3' + tree.save() + + self.assertEqual(tree.children[0].name, 'I am Child 1') + self.assertEqual(tree.children[0].children[0].name, 'I am Child 2') + self.assertEqual(tree.children[0].children[1].name, 'I am Child 3') + + # Test removal + self.assertEqual(len(tree.children[0].children), 2) + del(tree.children[0].children[1]) + + tree.save() + self.assertEqual(len(tree.children[0].children), 1) + + tree.children[0].children.pop(0) + tree.save() + self.assertEqual(len(tree.children[0].children), 0) + self.assertEqual(tree.children[0].children, []) + + tree.children[0].children.insert(0, third_child) + tree.children[0].children.insert(0, second_child) + tree.save() + self.assertEqual(len(tree.children[0].children), 2) + self.assertEqual(tree.children[0].children[0].name, second_child.name) + self.assertEqual(tree.children[0].children[1].name, third_child.name) + def test_undefined_reference(self): """Ensure that ReferenceFields may reference undefined Documents. """ From 4c2b83d9cae5323aa4b8f90132b51107c9a58db8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 14 Jun 2011 15:00:26 +0100 Subject: [PATCH 0266/1279] Remove errant __delete__ method --- mongoengine/base.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 41794f9..8a0ded5 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -864,14 +864,6 @@ class BaseList(list): self.instance._mark_as_changed(self.name) super(BaseList, self).__delitem__(*args, **kwargs) - def __delete__(self, *args, **kwargs): - if hasattr(self, 'instance') and hasattr(self, 'name'): - import ipdb; ipdb.set_trace() - self.instance._mark_as_changed(self.name) - delattr(self, 'instance') - delattr(self, 'name') - super(BaseDict, self).__delete__(*args, **kwargs) - def append(self, *args, **kwargs): self.instance._mark_as_changed(self.name) return super(BaseList, self).append(*args, **kwargs) From 576db9ca88bef73aedb107ce7edf2e502acaf7f5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 14 Jun 2011 15:09:03 +0100 Subject: [PATCH 0267/1279] Fixes DateTimeField handling of date objects. Fixes #191 --- mongoengine/fields.py | 10 +++++++--- tests/fields.py | 12 ++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index eeb4c2c..ca18255 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -232,12 +232,16 @@ class DateTimeField(BaseField): """A datetime field. Note: Microseconds are rounded to the nearest millisecond. - Pre UTC microsecond support is effecively broken see - `tests.field.test_datetime` for more information. + Pre UTC microsecond support is effecively broken. + Use :class:`~mongoengine.fields.ComplexDateTimeField` if you + need accurate microsecond support. """ def validate(self, value): - assert isinstance(value, datetime.datetime) + assert isinstance(value, (datetime.datetime, datetime.date)) + + def to_mongo(self, value): + return self.prepare_query_value(None, value) def prepare_query_value(self, op, value): if value is None: diff --git a/tests/fields.py b/tests/fields.py index 79cd519..773ba93 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -182,6 +182,9 @@ class FieldTest(unittest.TestCase): log.time = datetime.datetime.now() log.validate() + log.time = datetime.date.today() + log.validate() + log.time = -1 self.assertRaises(ValidationError, log.validate) log.time = '1pm' @@ -199,6 +202,15 @@ class FieldTest(unittest.TestCase): LogEntry.drop_collection() + # Test can save dates + log = LogEntry() + log.date = datetime.date.today() + log.save() + log.reload() + self.assertEquals(log.date.date(), datetime.date.today()) + + LogEntry.drop_collection() + # Post UTC - microseconds are rounded (down) nearest millisecond and dropped d1 = datetime.datetime(1970, 01, 01, 00, 00, 01, 999) d2 = datetime.datetime(1970, 01, 01, 00, 00, 01) From cb1dfdfac6f7a27ca2bd9bc99fd39e88d47219dd Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 14 Jun 2011 16:56:04 +0100 Subject: [PATCH 0268/1279] Fixes to signals The sender is the class of the document not the instance - easier to hook into --- docs/guide/signals.rst | 12 +++--- mongoengine/base.py | 4 +- mongoengine/document.py | 8 ++-- tests/signals.py | 87 ++++++++++++++++++++++++++++++++--------- 4 files changed, 81 insertions(+), 30 deletions(-) diff --git a/docs/guide/signals.rst b/docs/guide/signals.rst index d80a421..3c3159f 100644 --- a/docs/guide/signals.rst +++ b/docs/guide/signals.rst @@ -30,20 +30,20 @@ Example usage:: return self.name @classmethod - def pre_save(cls, instance, **kwargs): - logging.debug("Pre Save: %s" % instance.name) + def pre_save(cls, sender, document, **kwargs): + logging.debug("Pre Save: %s" % document.name) @classmethod - def post_save(cls, instance, **kwargs): - logging.debug("Post Save: %s" % instance.name) + def post_save(cls, sender, document, **kwargs): + logging.debug("Post Save: %s" % document.name) if 'created' in kwargs: if kwargs['created']: logging.debug("Created") else: logging.debug("Updated") - signals.pre_save.connect(Author.pre_save) - signals.post_save.connect(Author.post_save) + signals.pre_save.connect(Author.pre_save, sender=Author) + signals.post_save.connect(Author.post_save, sender=Author) .. _blinker: http://pypi.python.org/pypi/blinker \ No newline at end of file diff --git a/mongoengine/base.py b/mongoengine/base.py index 8a0ded5..c5b704e 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -600,7 +600,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): class BaseDocument(object): def __init__(self, **values): - signals.pre_init.send(self, values=values) + signals.pre_init.send(self.__class__, document=self, values=values) self._data = {} # Assign default values to instance @@ -619,7 +619,7 @@ class BaseDocument(object): except AttributeError: pass - signals.post_init.send(self) + signals.post_init.send(self.__class__, document=self) def _get_FIELD_display(self, field): """Returns the display value for a choice field""" diff --git a/mongoengine/document.py b/mongoengine/document.py index 2f40eec..69b19e2 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -91,7 +91,7 @@ class Document(BaseDocument): For example, ``save(..., w=2, fsync=True)`` will wait until at least two servers have recorded the write and will force an fsync on each server being written to. """ - signals.pre_save.send(self) + signals.pre_save.send(self.__class__, document=self) if validate: self.validate() @@ -122,7 +122,7 @@ class Document(BaseDocument): id_field = self._meta['id_field'] self[id_field] = self._fields[id_field].to_python(object_id) self._changed_fields = [] - signals.post_save.send(self, created=created) + signals.post_save.send(self.__class__, document=self, created=created) def delete(self, safe=False): """Delete the :class:`~mongoengine.Document` from the database. This @@ -130,7 +130,7 @@ class Document(BaseDocument): :param safe: check if the operation succeeded before returning """ - signals.pre_delete.send(self) + signals.pre_delete.send(self.__class__, document=self) id_field = self._meta['id_field'] object_id = self._fields[id_field].to_mongo(self[id_field]) @@ -140,7 +140,7 @@ class Document(BaseDocument): message = u'Could not delete document (%s)' % err.message raise OperationError(message) - signals.post_delete.send(self) + signals.post_delete.send(self.__class__, document=self) def reload(self): """Reloads all attributes from the database. diff --git a/tests/signals.py b/tests/signals.py index fff2d39..9c41337 100644 --- a/tests/signals.py +++ b/tests/signals.py @@ -28,21 +28,21 @@ class SignalTests(unittest.TestCase): return self.name @classmethod - def pre_init(cls, instance, **kwargs): + def pre_init(cls, sender, document, *args, **kwargs): signal_output.append('pre_init signal, %s' % cls.__name__) signal_output.append(str(kwargs['values'])) @classmethod - def post_init(cls, instance, **kwargs): - signal_output.append('post_init signal, %s' % instance) + def post_init(cls, sender, document, **kwargs): + signal_output.append('post_init signal, %s' % document) @classmethod - def pre_save(cls, instance, **kwargs): - signal_output.append('pre_save signal, %s' % instance) + def pre_save(cls, sender, document, **kwargs): + signal_output.append('pre_save signal, %s' % document) @classmethod - def post_save(cls, instance, **kwargs): - signal_output.append('post_save signal, %s' % instance) + def post_save(cls, sender, document, **kwargs): + signal_output.append('post_save signal, %s' % document) if 'created' in kwargs: if kwargs['created']: signal_output.append('Is created') @@ -50,15 +50,52 @@ class SignalTests(unittest.TestCase): signal_output.append('Is updated') @classmethod - def pre_delete(cls, instance, **kwargs): - signal_output.append('pre_delete signal, %s' % instance) + def pre_delete(cls, sender, document, **kwargs): + signal_output.append('pre_delete signal, %s' % document) @classmethod - def post_delete(cls, instance, **kwargs): - signal_output.append('post_delete signal, %s' % instance) - + def post_delete(cls, sender, document, **kwargs): + signal_output.append('post_delete signal, %s' % document) self.Author = Author + + class Another(Document): + name = StringField() + + def __unicode__(self): + return self.name + + @classmethod + def pre_init(cls, sender, document, **kwargs): + signal_output.append('pre_init Another signal, %s' % cls.__name__) + signal_output.append(str(kwargs['values'])) + + @classmethod + def post_init(cls, sender, document, **kwargs): + signal_output.append('post_init Another signal, %s' % document) + + @classmethod + def pre_save(cls, sender, document, **kwargs): + signal_output.append('pre_save Another signal, %s' % document) + + @classmethod + def post_save(cls, sender, document, **kwargs): + signal_output.append('post_save Another signal, %s' % document) + if 'created' in kwargs: + if kwargs['created']: + signal_output.append('Is created') + else: + signal_output.append('Is updated') + + @classmethod + def pre_delete(cls, sender, document, **kwargs): + signal_output.append('pre_delete Another signal, %s' % document) + + @classmethod + def post_delete(cls, sender, document, **kwargs): + signal_output.append('post_delete Another signal, %s' % document) + + self.Another = Another # Save up the number of connected signals so that we can check at the end # that all the signals we register get properly unregistered self.pre_signals = ( @@ -70,12 +107,19 @@ class SignalTests(unittest.TestCase): len(signals.post_delete.receivers) ) - signals.pre_init.connect(Author.pre_init) - signals.post_init.connect(Author.post_init) - signals.pre_save.connect(Author.pre_save) - signals.post_save.connect(Author.post_save) - signals.pre_delete.connect(Author.pre_delete) - signals.post_delete.connect(Author.post_delete) + signals.pre_init.connect(Author.pre_init, sender=Author) + signals.post_init.connect(Author.post_init, sender=Author) + signals.pre_save.connect(Author.pre_save, sender=Author) + signals.post_save.connect(Author.post_save, sender=Author) + signals.pre_delete.connect(Author.pre_delete, sender=Author) + signals.post_delete.connect(Author.post_delete, sender=Author) + + signals.pre_init.connect(Another.pre_init, sender=Another) + signals.post_init.connect(Another.post_init, sender=Another) + signals.pre_save.connect(Another.pre_save, sender=Another) + signals.post_save.connect(Another.post_save, sender=Another) + signals.pre_delete.connect(Another.pre_delete, sender=Another) + signals.post_delete.connect(Another.post_delete, sender=Another) def tearDown(self): signals.pre_init.disconnect(self.Author.pre_init) @@ -85,6 +129,13 @@ class SignalTests(unittest.TestCase): signals.post_save.disconnect(self.Author.post_save) signals.pre_save.disconnect(self.Author.pre_save) + signals.pre_init.disconnect(self.Another.pre_init) + signals.post_init.disconnect(self.Another.post_init) + signals.post_delete.disconnect(self.Another.post_delete) + signals.pre_delete.disconnect(self.Another.pre_delete) + signals.post_save.disconnect(self.Another.post_save) + signals.pre_save.disconnect(self.Another.pre_save) + # Check that all our signals got disconnected properly. post_signals = ( len(signals.pre_init.receivers), From 0338ac17b1be78050e82e1222a84c805b870bf9c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 15 Jun 2011 08:55:31 +0100 Subject: [PATCH 0269/1279] Fixes multiple assignment issue preventing saves Thanks to @wpjunior for the ticket and testcase Also fixed bug in BaseList fixes #195 --- mongoengine/base.py | 6 +++--- tests/document.py | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index c5b704e..1ca1680 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -784,9 +784,9 @@ class BaseDocument(object): for field_name in self._fields: key = '%s.' % field_name field = getattr(self, field_name, None) - if isinstance(field, EmbeddedDocument): # Grab all embedded fields that have been changed + if isinstance(field, EmbeddedDocument) and field_name not in _changed_fields: # Grab all embedded fields that have been changed _changed_fields += ["%s%s" % (key, k) for k in field._get_changed_fields(key) if k] - elif isinstance(field, (list, tuple)): # Loop list fields as they contain documents + elif isinstance(field, (list, tuple)) and field_name not in _changed_fields: # Loop list fields as they contain documents for index, value in enumerate(field): if not hasattr(value, '_get_changed_fields'): continue @@ -858,7 +858,7 @@ class BaseList(list): def __setitem__(self, *args, **kwargs): if hasattr(self, 'instance') and hasattr(self, 'name'): self.instance._mark_as_changed(self.name) - super(BaseDict, self).__setitem__(*args, **kwargs) + super(BaseList, self).__setitem__(*args, **kwargs) def __delitem__(self, *args, **kwargs): self.instance._mark_as_changed(self.name) diff --git a/tests/document.py b/tests/document.py index 4c89080..4f90ba2 100644 --- a/tests/document.py +++ b/tests/document.py @@ -933,31 +933,35 @@ class DocumentTest(unittest.TestCase): self.assertEquals(doc._delta(), ({}, {})) doc.string_field = 'hello' + self.assertEquals(doc._get_changed_fields(), ['string_field']) self.assertEquals(doc._delta(), ({'string_field': 'hello'}, {})) doc._changed_fields = [] doc.int_field = 1 + self.assertEquals(doc._get_changed_fields(), ['int_field']) self.assertEquals(doc._delta(), ({'int_field': 1}, {})) doc._changed_fields = [] dict_value = {'hello': 'world', 'ping': 'pong'} doc.dict_field = dict_value + self.assertEquals(doc._get_changed_fields(), ['dict_field']) self.assertEquals(doc._delta(), ({'dict_field': dict_value}, {})) doc._changed_fields = [] list_value = ['1', 2, {'hello': 'world'}] doc.list_field = list_value + self.assertEquals(doc._get_changed_fields(), ['list_field']) self.assertEquals(doc._delta(), ({'list_field': list_value}, {})) # Test unsetting doc._changed_fields = [] - doc._unset_fields = [] doc.dict_field = {} + self.assertEquals(doc._get_changed_fields(), ['dict_field']) self.assertEquals(doc._delta(), ({}, {'dict_field': 1})) doc._changed_fields = [] - doc._unset_fields = {} doc.list_field = [] + self.assertEquals(doc._get_changed_fields(), ['list_field']) self.assertEquals(doc._delta(), ({}, {'list_field': 1})) def test_delta_recursive(self): @@ -990,6 +994,8 @@ class DocumentTest(unittest.TestCase): embedded_1.list_field = ['1', 2, {'hello': 'world'}] doc.embedded_field = embedded_1 + self.assertEquals(doc._get_changed_fields(), ['embedded_field']) + embedded_delta = { '_types': ['Embedded'], '_cls': 'Embedded', @@ -1005,6 +1011,7 @@ class DocumentTest(unittest.TestCase): doc.reload() doc.embedded_field.dict_field = {} + self.assertEquals(doc._get_changed_fields(), ['embedded_field.dict_field']) self.assertEquals(doc.embedded_field._delta(), ({}, {'dict_field': 1})) self.assertEquals(doc._delta(), ({}, {'embedded_field.dict_field': 1})) doc.save() @@ -1012,6 +1019,7 @@ class DocumentTest(unittest.TestCase): self.assertEquals(doc.embedded_field.dict_field, {}) doc.embedded_field.list_field = [] + self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field']) self.assertEquals(doc.embedded_field._delta(), ({}, {'list_field': 1})) self.assertEquals(doc._delta(), ({}, {'embedded_field.list_field': 1})) doc.save() @@ -1025,6 +1033,7 @@ class DocumentTest(unittest.TestCase): embedded_2.list_field = ['1', 2, {'hello': 'world'}] doc.embedded_field.list_field = ['1', 2, embedded_2] + self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field']) self.assertEquals(doc.embedded_field._delta(), ({ 'list_field': ['1', 2, { '_cls': 'Embedded', @@ -1055,12 +1064,38 @@ class DocumentTest(unittest.TestCase): self.assertEquals(doc.embedded_field.list_field[2][k], embedded_2[k]) doc.embedded_field.list_field[2].string_field = 'world' + self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field.2.string_field']) self.assertEquals(doc.embedded_field._delta(), ({'list_field.2.string_field': 'world'}, {})) self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.string_field': 'world'}, {})) doc.save() doc.reload() self.assertEquals(doc.embedded_field.list_field[2].string_field, 'world') + # Test multiple assignments + doc.embedded_field.list_field[2].string_field = 'hello world' + doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] + self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field']) + self.assertEquals(doc.embedded_field._delta(), ({ + 'list_field': ['1', 2, { + '_types': ['Embedded'], + '_cls': 'Embedded', + 'string_field': 'hello world', + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], + 'dict_field': {'hello': 'world'}}]}, {})) + self.assertEquals(doc._delta(), ({ + 'embedded_field.list_field': ['1', 2, { + '_types': ['Embedded'], + '_cls': 'Embedded', + 'string_field': 'hello world', + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], + 'dict_field': {'hello': 'world'}} + ]}, {})) + doc.save() + doc.reload() + self.assertEquals(doc.embedded_field.list_field[2].string_field, 'hello world') + # Test list native methods doc.embedded_field.list_field[2].list_field.pop(0) self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}]}, {})) From 94cad89e321b92239171fd0a2f11095fa2f01b09 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 15 Jun 2011 11:22:27 +0100 Subject: [PATCH 0270/1279] Fixes to item_frequencies - now handles path lookups fixes #194 --- .gitignore | 1 + mongoengine/queryset.py | 39 ++++++++++++++++++------- tests/queryset.py | 63 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 8951a0c..315674f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ env/ .settings .project .pydevproject +tests/bugfix.py diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 666567e..4ffa532 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1303,7 +1303,16 @@ class QuerySet(object): # Substitute the correct name for the field into the javascript return u'["%s"]' % fields[-1].db_field - return re.sub(u'\[\s*~([A-z_][A-z_0-9.]+?)\s*\]', field_sub, code) + def field_path_sub(match): + # Extract just the field name, and look up the field objects + field_name = match.group(1).split('.') + fields = QuerySet._lookup_field(self._document, field_name) + # Substitute the correct name for the field into the javascript + return ".".join([f.db_field for f in fields]) + + code = re.sub(u'\[\s*~([A-z_][A-z_0-9.]+?)\s*\]', field_sub, code) + code = re.sub(u'\{\{\s*~([A-z_][A-z_0-9.]+?)\s*\}\}', field_path_sub, code) + return code def exec_js(self, code, *fields, **options): """Execute a Javascript function on the server. A list of fields may be @@ -1405,12 +1414,15 @@ class QuerySet(object): def _item_frequencies_map_reduce(self, field, normalize=False): map_func = """ function() { - if (this[~%(field)s].constructor == Array) { - this[~%(field)s].forEach(function(item) { + path = '{{~%(field)s}}'.split('.'); + field = this; + for (p in path) { field = field[path[p]]; } + if (field.constructor == Array) { + field.forEach(function(item) { emit(item, 1); }); } else { - emit(this[~%(field)s], 1); + emit(field, 1); } } """ % dict(field=field) @@ -1443,12 +1455,16 @@ class QuerySet(object): def _item_frequencies_exec_js(self, field, normalize=False): """Uses exec_js to execute""" freq_func = """ - function(field) { + function(path) { + path = path.split('.'); + if (options.normalize) { var total = 0.0; db[collection].find(query).forEach(function(doc) { - if (doc[field].constructor == Array) { - total += doc[field].length; + field = doc; + for (p in path) { field = field[path[p]]; } + if (field.constructor == Array) { + total += field.length; } else { total++; } @@ -1461,18 +1477,21 @@ class QuerySet(object): inc /= total; } db[collection].find(query).forEach(function(doc) { - if (doc[field].constructor == Array) { - doc[field].forEach(function(item) { + field = doc; + for (p in path) { field = field[path[p]]; } + if (field.constructor == Array) { + field.forEach(function(item) { frequencies[item] = inc + (isNaN(frequencies[item]) ? 0: frequencies[item]); }); } else { - var item = doc[field]; + var item = field; frequencies[item] = inc + (isNaN(frequencies[item]) ? 0: frequencies[item]); } }); return frequencies; } """ + return self.exec_js(freq_func, field, normalize=normalize) def __repr__(self): diff --git a/tests/queryset.py b/tests/queryset.py index 37140f4..cc219fb 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1116,6 +1116,11 @@ class QuerySetTest(unittest.TestCase): ] self.assertEqual(results, expected_results) + # Test template style + code = "{{~comments.content}}" + sub_code = BlogPost.objects._sub_js_fields(code) + self.assertEquals("cmnts.body", sub_code) + BlogPost.drop_collection() def test_delete(self): @@ -1637,6 +1642,64 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() + def test_item_frequencies_on_embedded(self): + """Ensure that item frequencies are properly generated from lists. + """ + + class Phone(EmbeddedDocument): + number = StringField() + + class Person(Document): + name = StringField() + phone = EmbeddedDocumentField(Phone) + + Person.drop_collection() + + doc = Person(name="Guido") + doc.phone = Phone(number='62-3331-1656') + doc.save() + + doc = Person(name="Marr") + doc.phone = Phone(number='62-3331-1656') + doc.save() + + doc = Person(name="WP Junior") + doc.phone = Phone(number='62-3332-1656') + doc.save() + + + def test_assertions(f): + f = dict((key, int(val)) for key, val in f.items()) + self.assertEqual(set(['62-3331-1656', '62-3332-1656']), set(f.keys())) + self.assertEqual(f['62-3331-1656'], 2) + self.assertEqual(f['62-3332-1656'], 1) + + exec_js = Person.objects.item_frequencies('phone.number') + map_reduce = Person.objects.item_frequencies('phone.number', map_reduce=True) + test_assertions(exec_js) + test_assertions(map_reduce) + + # Ensure query is taken into account + def test_assertions(f): + f = dict((key, int(val)) for key, val in f.items()) + self.assertEqual(set(['62-3331-1656']), set(f.keys())) + self.assertEqual(f['62-3331-1656'], 2) + + exec_js = Person.objects(phone__number='62-3331-1656').item_frequencies('phone.number') + map_reduce = Person.objects(phone__number='62-3331-1656').item_frequencies('phone.number', map_reduce=True) + test_assertions(exec_js) + test_assertions(map_reduce) + + # Check that normalization works + def test_assertions(f): + self.assertEqual(f['62-3331-1656'], 2.0/3.0) + self.assertEqual(f['62-3332-1656'], 1.0/3.0) + + exec_js = Person.objects.item_frequencies('phone.number', normalize=True) + map_reduce = Person.objects.item_frequencies('phone.number', normalize=True, map_reduce=True) + test_assertions(exec_js) + test_assertions(map_reduce) + def test_average(self): """Ensure that field can be averaged correctly. """ From ffb3e8b7b9a9f1387566ad41c4a515686a06d975 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 15 Jun 2011 11:28:41 +0100 Subject: [PATCH 0271/1279] Added help_text and verbose_name to fields closes #192 --- mongoengine/base.py | 4 +++- tests/fields.py | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 1ca1680..d50cf95 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -49,7 +49,7 @@ class BaseField(object): def __init__(self, db_field=None, name=None, required=False, default=None, unique=False, unique_with=None, primary_key=False, - validation=None, choices=None): + validation=None, choices=None, verbose_name=None, help_text=None): self.db_field = (db_field or name) if not primary_key else '_id' if name: import warnings @@ -63,6 +63,8 @@ class BaseField(object): self.primary_key = primary_key self.validation = validation self.choices = choices + self.verbose_name = verbose_name + self.help_text = help_text # Adjust the appropriate creation counter, and save our local copy. if self.db_field == '_id': diff --git a/tests/fields.py b/tests/fields.py index 773ba93..c13f9e3 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -21,12 +21,15 @@ class FieldTest(unittest.TestCase): """ class Person(Document): name = StringField() - age = IntField(default=30) - userid = StringField(default=lambda: 'test') + age = IntField(default=30, help_text="Your real age") + userid = StringField(default=lambda: 'test', verbose_name="User Identity") person = Person(name='Test Person') self.assertEqual(person._data['age'], 30) self.assertEqual(person._data['userid'], 'test') + self.assertEqual(person._fields['name'].help_text, None) + self.assertEqual(person._fields['age'].help_text, "Your real age") + self.assertEqual(person._fields['userid'].verbose_name, "User Identity") def test_required_values(self): """Ensure that required field constraints are enforced. From 5411cc55731bc6ecf43075b08acbf00eccafb83e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 15 Jun 2011 11:30:10 +0100 Subject: [PATCH 0272/1279] Updated changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 54efb4f..11218e2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,8 @@ Changelog Changes in dev ============== +- Added help_text and verbose_name to fields to help with some form libs +- Updated item_frequencies to handle embedded document lookups - Added delta tracking now only sets / unsets explicitly changed fields - Fixed saving so sets updated values rather than overwrites - Added ComplexDateTimeField - Handles datetimes correctly with microseconds From 967e72723b692114aa8357387b6b6292d1aab868 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 15 Jun 2011 14:55:38 +0100 Subject: [PATCH 0273/1279] Added note to item_frequencies method. Current implementation is relatively simple, for complex schemas the user will have to write their own map reduce. --- mongoengine/queryset.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 4ffa532..76d4d1c 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1400,6 +1400,12 @@ class QuerySet(object): the whole queried set of documents, and their corresponding frequency. This is useful for generating tag clouds, or searching documents. + .. note:: + Can only do direct simple mappings and cannot map across + :class:`~mongoengine.ReferenceField` or + :class:`~mongoengine.GenericReferenceField` for more complex + counting a manual map reduce call would is required. + If the field is a :class:`~mongoengine.ListField`, the items within each list will be counted individually. From 658b85d3277c6c7478ca426ed64f544f59f811e9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 15 Jun 2011 16:51:49 +0100 Subject: [PATCH 0274/1279] Inconsistent setting of '_cls' broke inherited document referencing Fixes #199 --- docs/changelog.rst | 15 +++++++------- mongoengine/base.py | 8 ++++---- mongoengine/fields.py | 2 +- tests/document.py | 48 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 12 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 11218e2..ea92623 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed issue with inconsitent setting of _cls breaking inherited referencing - Added help_text and verbose_name to fields to help with some form libs - Updated item_frequencies to handle embedded document lookups - Added delta tracking now only sets / unsets explicitly changed fields @@ -22,7 +23,7 @@ Changes in dev - Updated connection exception so it provides more info on the cause. - Added searching multiple levels deep in ``DictField`` - Added ``DictField`` entries containing strings to use matching operators -- Added ``MapField``, similar to ``DictField`` +- Added ``MapField``, similar to ``DictField`` - Added Abstract Base Classes - Added Custom Objects Managers - Added sliced subfields updating @@ -35,14 +36,14 @@ Changes in dev - Updated queryset to handle latest version of pymongo map_reduce now requires an output. - Added ``Document`` __hash__, __ne__ for pickling -- Added ``FileField`` optional size arg for read method +- Added ``FileField`` optional size arg for read method - Fixed ``FileField`` seek and tell methods for reading files -- Added ``QuerySet.clone`` to support copying querysets +- Added ``QuerySet.clone`` to support copying querysets - Fixed item_frequencies when using name thats the same as a native js function - Added reverse delete rules - Fixed issue with unset operation - Fixed Q-object bug -- Added ``QuerySet.all_fields`` resets previous .only() and .exlude() +- Added ``QuerySet.all_fields`` resets previous .only() and .exlude() - Added ``QuerySet.exclude`` - Added django style choices - Fixed order and filter issue @@ -82,7 +83,7 @@ Changes in v0.3 =============== - Added MapReduce support - Added ``contains``, ``startswith`` and ``endswith`` query operators (and - case-insensitive versions that are prefixed with 'i') + case-insensitive versions that are prefixed with 'i') - Deprecated fields' ``name`` parameter, replaced with ``db_field`` - Added ``QuerySet.only`` for only retrieving specific fields - Added ``QuerySet.in_bulk()`` for bulk querying using ids @@ -129,7 +130,7 @@ Changes in v0.2 =============== - Added ``Q`` class for building advanced queries - Added ``QuerySet`` methods for atomic updates to documents -- Fields may now specify ``unique=True`` to enforce uniqueness across a +- Fields may now specify ``unique=True`` to enforce uniqueness across a collection - Added option for default document ordering - Fixed bug in index definitions @@ -137,7 +138,7 @@ Changes in v0.2 Changes in v0.1.3 ================= - Added Django authentication backend -- Added ``Document.meta`` support for indexes, which are ensured just before +- Added ``Document.meta`` support for indexes, which are ensured just before querying takes place - A few minor bugfixes diff --git a/mongoengine/base.py b/mongoengine/base.py index d50cf95..6d34368 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -173,7 +173,7 @@ class ComplexBaseField(BaseField): for k,v in value_list.items(): if isinstance(v, dict) and '_cls' in v and '_ref' not in v: - value_list[k] = get_document(v['_cls'].split('.')[-1])._from_son(v) + value_list[k] = get_document(v['_cls'])._from_son(v) # Handle all dereferencing db = _get_db() @@ -401,6 +401,7 @@ class DocumentMetaclass(type): else: simple_class = False + doc_class_name = '.'.join(reversed(class_name)) meta = attrs.get('_meta', attrs.get('meta', {})) if 'allow_inheritance' not in meta: @@ -412,8 +413,7 @@ class DocumentMetaclass(type): raise ValueError('Only direct subclasses of Document may set ' '"allow_inheritance" to False') attrs['_meta'] = meta - - attrs['_class_name'] = '.'.join(reversed(class_name)) + attrs['_class_name'] = doc_class_name attrs['_superclasses'] = superclasses # Add the document's fields to the _fields attribute @@ -448,7 +448,7 @@ class DocumentMetaclass(type): new_class.add_to_class('MultipleObjectsReturned', exc) global _document_registry - _document_registry[name] = new_class + _document_registry[doc_class_name] = new_class return new_class diff --git a/mongoengine/fields.py b/mongoengine/fields.py index ca18255..2699920 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -652,7 +652,7 @@ class GenericReferenceField(BaseField): id_ = id_field.to_mongo(id_) collection = document._meta['collection'] ref = pymongo.dbref.DBRef(collection, id_) - return {'_cls': document.__class__.__name__, '_ref': ref} + return {'_cls': document._class_name, '_ref': ref} def prepare_query_value(self, op, value): return self.to_mongo(value) diff --git a/tests/document.py b/tests/document.py index 4f90ba2..3a5419d 100644 --- a/tests/document.py +++ b/tests/document.py @@ -116,6 +116,8 @@ class DocumentTest(unittest.TestCase): class Human(Mammal): pass class Dog(Mammal): pass + Animal.drop_collection() + Animal().save() Fish().save() Mammal().save() @@ -133,6 +135,52 @@ class DocumentTest(unittest.TestCase): Animal.drop_collection() + def test_polymorphic_references(self): + """Ensure that the correct subclasses are returned from a query when + using references / generic references + """ + class Animal(Document): pass + class Fish(Animal): pass + class Mammal(Animal): pass + class Human(Mammal): pass + class Dog(Mammal): pass + + class Zoo(Document): + animals = ListField(ReferenceField(Animal)) + + Zoo.drop_collection() + Animal.drop_collection() + + Animal().save() + Fish().save() + Mammal().save() + Human().save() + Dog().save() + + # Save a reference to each animal + zoo = Zoo(animals=Animal.objects) + zoo.save() + zoo.reload() + + classes = [a.__class__ for a in Zoo.objects.first().animals] + self.assertEqual(classes, [Animal, Fish, Mammal, Human, Dog]) + + Zoo.drop_collection() + + class Zoo(Document): + animals = ListField(GenericReferenceField(Animal)) + + # Save a reference to each animal + zoo = Zoo(animals=Animal.objects) + zoo.save() + zoo.reload() + + classes = [a.__class__ for a in Zoo.objects.first().animals] + self.assertEqual(classes, [Animal, Fish, Mammal, Human, Dog]) + + Zoo.drop_collection() + Animal.drop_collection() + def test_inheritance(self): """Ensure that document may inherit fields from a superclass document. """ From 22a7ee58852ae5218d2197c6fd472c04176b150e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 16 Jun 2011 09:47:44 +0100 Subject: [PATCH 0275/1279] Handle old named (referenced) docs Refs #199 --- mongoengine/base.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 6d34368..49efba6 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -23,13 +23,20 @@ class ValidationError(Exception): _document_registry = {} def get_document(name): - if name not in _document_registry: + doc = _document_registry.get(name, None) + if not doc: + # Possible old style names + end = ".%s" % name + possible_match = [k for k in _document_registry.keys() if k.endswith(end)] + if len(possible_match) == 1: + doc = _document_registry.get(possible_match.pop(), None) + if not doc: raise NotRegistered(""" `%s` has not been registered in the document registry. Importing the document class automatically registers it, has it been imported? """.strip() % name) - return _document_registry[name] + return doc class BaseField(object): From cae3f3eefffa3809dc396e65941f36f66c4bdb52 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 16 Jun 2011 12:50:45 +0100 Subject: [PATCH 0276/1279] Fixes pickling issue with choice fields Removes the dynamic __get_field_display partials before pickling --- mongoengine/base.py | 72 ++++++++++++++++++++++++++++++--------------- tests/document.py | 6 ++-- 2 files changed, 51 insertions(+), 27 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 49efba6..938808a 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -614,9 +614,6 @@ class BaseDocument(object): self._data = {} # Assign default values to instance for attr_name, field in self._fields.items(): - if field.choices: # dynamically adds a way to get the display value for a field with choices - setattr(self, 'get_%s_display' % attr_name, partial(self._get_FIELD_display, field=field)) - value = getattr(self, attr_name, None) setattr(self, attr_name, value) @@ -628,9 +625,29 @@ class BaseDocument(object): except AttributeError: pass + # Set any get_fieldname_display methods + self.__set_field_display() + signals.post_init.send(self.__class__, document=self) - def _get_FIELD_display(self, field): + def __getstate__(self): + self_dict = self.__dict__ + removals = ["get_%s_display" % k for k,v in self._fields.items() if v.choices] + for k in removals: + if hasattr(self, k): + delattr(self, k) + return self.__dict__ + + def __setstate__(self, __dict__): + self.__dict__ = __dict__ + self.__set_field_display() + + def __set_field_display(self): + for attr_name, field in self._fields.items(): + if field.choices: # dynamically adds a way to get the display value for a field with choices + setattr(self, 'get_%s_display' % attr_name, partial(self.__get_field_display, field=field)) + + def __get_field_display(self, field): """Returns the display value for a choice field""" value = getattr(self, field.name) return dict(field.choices).get(value, value) @@ -865,42 +882,46 @@ class BaseList(list): super(BaseList, self).__init__(list_items) def __setitem__(self, *args, **kwargs): - if hasattr(self, 'instance') and hasattr(self, 'name'): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() super(BaseList, self).__setitem__(*args, **kwargs) def __delitem__(self, *args, **kwargs): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() super(BaseList, self).__delitem__(*args, **kwargs) def append(self, *args, **kwargs): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() return super(BaseList, self).append(*args, **kwargs) def extend(self, *args, **kwargs): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() return super(BaseList, self).extend(*args, **kwargs) def insert(self, *args, **kwargs): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() return super(BaseList, self).insert(*args, **kwargs) def pop(self, *args, **kwargs): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() return super(BaseList, self).pop(*args, **kwargs) def remove(self, *args, **kwargs): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() return super(BaseList, self).remove(*args, **kwargs) def reverse(self, *args, **kwargs): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() return super(BaseList, self).reverse(*args, **kwargs) def sort(self, *args, **kwargs): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() return super(BaseList, self).sort(*args, **kwargs) + def _mark_as_changed(self): + """Marks a list as changed if has an instance and a name""" + if hasattr(self, 'instance') and hasattr(self, 'name'): + self.instance._mark_as_changed(self.name) + class BaseDict(dict): """A special dict so we can watch any changes @@ -912,39 +933,42 @@ class BaseDict(dict): super(BaseDict, self).__init__(dict_items) def __setitem__(self, *args, **kwargs): - if hasattr(self, 'instance') and hasattr(self, 'name'): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() super(BaseDict, self).__setitem__(*args, **kwargs) def __setattr__(self, *args, **kwargs): - if hasattr(self, 'instance') and hasattr(self, 'name'): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() super(BaseDict, self).__setattr__(*args, **kwargs) def __delete__(self, *args, **kwargs): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() super(BaseDict, self).__delete__(*args, **kwargs) def __delitem__(self, *args, **kwargs): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() super(BaseDict, self).__delitem__(*args, **kwargs) def __delattr__(self, *args, **kwargs): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() super(BaseDict, self).__delattr__(*args, **kwargs) def clear(self, *args, **kwargs): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() super(BaseDict, self).clear(*args, **kwargs) def pop(self, *args, **kwargs): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() super(BaseDict, self).clear(*args, **kwargs) def popitem(self, *args, **kwargs): - self.instance._mark_as_changed(self.name) + self._mark_as_changed() super(BaseDict, self).clear(*args, **kwargs) + def _mark_as_changed(self): + """Marks a dict as changed if has an instance and a name""" + if hasattr(self, 'instance') and hasattr(self, 'name'): + self.instance._mark_as_changed(self.name) + if sys.version_info < (2, 5): # Prior to Python 2.5, Exception was an old-style class import types diff --git a/tests/document.py b/tests/document.py index 3a5419d..b33f3fe 100644 --- a/tests/document.py +++ b/tests/document.py @@ -15,7 +15,7 @@ class PickleEmbedded(EmbeddedDocument): class PickleTest(Document): number = IntField() - string = StringField() + string = StringField(choices=(('One', '1'), ('Two', '2'))) embedded = EmbeddedDocumentField(PickleEmbedded) lists = ListField(StringField()) @@ -1516,7 +1516,7 @@ class DocumentTest(unittest.TestCase): def test_picklable(self): - pickle_doc = PickleTest(number=1, string="OH HAI", lists=['1', '2']) + pickle_doc = PickleTest(number=1, string="One", lists=['1', '2']) pickle_doc.embedded = PickleEmbedded() pickle_doc.save() @@ -1525,7 +1525,7 @@ class DocumentTest(unittest.TestCase): self.assertEquals(resurrected, pickle_doc) - resurrected.string = "Working" + resurrected.string = "Two" resurrected.save() pickle_doc.reload() From 5e8604967c5eabb9a71f0b3a87f39e37d6f589bb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 16 Jun 2011 15:00:10 +0100 Subject: [PATCH 0277/1279] Fixes for django Q query rendering bug Ensures that the QNodes haven't already been processed Fixes #185 --- mongoengine/queryset.py | 3 ++- tests/django_tests.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 76d4d1c..92229a1 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -251,7 +251,8 @@ class QCombination(QNode): def accept(self, visitor): for i in range(len(self.children)): - self.children[i] = self.children[i].accept(visitor) + if isinstance(self.children[i], QNode): + self.children[i] = self.children[i].accept(visitor) return visitor.visit_combination(self) diff --git a/tests/django_tests.py b/tests/django_tests.py index ee8084c..930cc11 100644 --- a/tests/django_tests.py +++ b/tests/django_tests.py @@ -53,4 +53,7 @@ class QuerySetTest(unittest.TestCase): t = Template("{% for o in ol %}{{ o.name }}-{{ o.age }}:{% endfor %}") d = {"ol": self.Person.objects.filter(Q(age=10) | Q(name="C"))} - self.assertEqual(t.render(Context(d)), u'D-10:C-30:') \ No newline at end of file + self.assertEqual(t.render(Context(d)), 'D-10:C-30:') + + # Check double rendering doesn't throw an error + self.assertEqual(t.render(Context(d)), 'D-10:C-30:') \ No newline at end of file From 5cc9188c5b2fa56de911458af3d280fc36a1d3ab Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 16 Jun 2011 15:25:09 +0100 Subject: [PATCH 0278/1279] Improved validation of (Generic)Reference fields --- mongoengine/fields.py | 14 ++++++++++++++ tests/fields.py | 21 +++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 2699920..e1b4366 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -604,6 +604,11 @@ class ReferenceField(BaseField): def validate(self, value): assert isinstance(value, (self.document_type, pymongo.dbref.DBRef)) + if isinstance(value, Document) and value.id is None: + raise ValidationError('You can only reference documents once ' + 'they have been saved to the database') + + def lookup_member(self, member_name): return self.document_type._fields.get(member_name) @@ -628,6 +633,15 @@ class GenericReferenceField(BaseField): return super(GenericReferenceField, self).__get__(instance, owner) + def validate(self, value): + if not isinstance(value, (Document, pymongo.dbref.DBRef)): + raise ValidationError('GenericReferences can only contain documents') + + # We need the id from the saved object to create the DBRef + if isinstance(value, Document) and value.id is None: + raise ValidationError('You can only reference documents once ' + 'they have been saved to the database') + def dereference(self, value): doc_cls = get_document(value['_cls']) reference = value['_ref'] diff --git a/tests/fields.py b/tests/fields.py index c13f9e3..2204930 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -377,6 +377,7 @@ class FieldTest(unittest.TestCase): comments = ListField(EmbeddedDocumentField(Comment)) tags = ListField(StringField()) authors = ListField(ReferenceField(User)) + generic = ListField(GenericReferenceField()) post = BlogPost(content='Went for a walk today...') post.validate() @@ -404,8 +405,28 @@ class FieldTest(unittest.TestCase): self.assertRaises(ValidationError, post.validate) post.authors = [User()] + self.assertRaises(ValidationError, post.validate) + + user = User() + user.save() + post.authors = [user] post.validate() + post.generic = [1, 2] + self.assertRaises(ValidationError, post.validate) + + post.generic = [User(), Comment()] + self.assertRaises(ValidationError, post.validate) + + post.generic = [Comment()] + self.assertRaises(ValidationError, post.validate) + + post.generic = [user] + post.validate() + + User.drop_collection() + BlogPost.drop_collection() + def test_sorted_list_sorting(self): """Ensure that a sorted list field properly sorts values. """ From 62c8823e6423d7445dfc011a74d9a748fe0c8d65 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 17 Jun 2011 11:39:53 +0100 Subject: [PATCH 0279/1279] Fixing requirements Test requirements are not install requirements! --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 37ec437..6877b62 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,7 @@ setup(name='mongoengine', long_description=LONG_DESCRIPTION, platforms=['any'], classifiers=CLASSIFIERS, - install_requires=['pymongo', 'blinker', 'django==1.3'], + install_requires=['pymongo'], test_suite='tests', + tests_require=['blinker', 'django==1.3'] ) From 5e7efcc8c2f4947d9d25bb3e40f0cfa1e759c419 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 17 Jun 2011 12:43:28 +0100 Subject: [PATCH 0280/1279] Added 'hint' support, telling Mongo the proper index to use for the query. Judicious use of hints can greatly improve query performance. When doing a query on multiple fields (at least one of which is indexed) pass the indexed field as a hint to the query. Hinting will not do anything if the corresponding index does not exist. The last hint applied to this cursor takes precedence over all others. Closes #203 --- docs/changelog.rst | 1 + mongoengine/queryset.py | 21 ++++++++++++++++++++- tests/document.py | 30 +++++++++++++++++++++++++++++- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ea92623..a9cfe32 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added hint() support, so cantell Mongo the proper index to use for the query - Fixed issue with inconsitent setting of _cls breaking inherited referencing - Added help_text and verbose_name to fields to help with some form libs - Updated item_frequencies to handle embedded document lookups diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 92229a1..bfa89b4 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -347,6 +347,7 @@ class QuerySet(object): self._cursor_obj = None self._limit = None self._skip = None + self._hint = -1 # Using -1 as None is a valid value for hint def clone(self): """Creates a copy of the current :class:`~mongoengine.queryset.QuerySet`""" @@ -354,7 +355,7 @@ class QuerySet(object): copy_props = ('_initial_query', '_query_obj', '_where_clause', '_loaded_fields', '_ordering', '_snapshot', - '_timeout', '_limit', '_skip', '_slave_okay') + '_timeout', '_limit', '_skip', '_slave_okay', '_hint') for prop in copy_props: val = getattr(self, prop) @@ -539,6 +540,9 @@ class QuerySet(object): if self._skip is not None: self._cursor_obj.skip(self._skip) + if self._hint != -1: + self._cursor_obj.hint(self._hint) + return self._cursor_obj @classmethod @@ -965,6 +969,21 @@ class QuerySet(object): self._skip = n return self + def hint(self, index=None): + """Added 'hint' support, telling Mongo the proper index to use for the + query. + + Judicious use of hints can greatly improve query performance. When doing + a query on multiple fields (at least one of which is indexed) pass the + indexed field as a hint to the query. + + Hinting will not do anything if the corresponding index does not exist. + The last hint applied to this cursor takes precedence over all others. + """ + self._cursor.hint(index) + self._hint = index + return self + def __getitem__(self, key): """Support skip and limit using getitem and slicing syntax. """ diff --git a/tests/document.py b/tests/document.py index b33f3fe..5d44ca2 100644 --- a/tests/document.py +++ b/tests/document.py @@ -513,7 +513,6 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() - def test_dictionary_indexes(self): """Ensure that indexes are used when meta[indexes] contains dictionaries instead of lists. @@ -546,6 +545,35 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + def test_hint(self): + + class BlogPost(Document): + tags = ListField(StringField()) + meta = { + 'indexes': [ + 'tags', + ], + } + + BlogPost.drop_collection() + + for i in xrange(0, 10): + tags = [("tag %i" % n) for n in xrange(0, i % 2)] + BlogPost(tags=tags).save() + + self.assertEquals(BlogPost.objects.count(), 10) + self.assertEquals(BlogPost.objects.hint().count(), 10) + self.assertEquals(BlogPost.objects.hint([('tags', 1)]).count(), 10) + + self.assertEquals(BlogPost.objects.hint([('ZZ', 1)]).count(), 10) + + def invalid_index(): + BlogPost.objects.hint('tags') + self.assertRaises(TypeError, invalid_index) + + def invalid_index_2(): + return BlogPost.objects.hint(('tags', 1)) + self.assertRaises(TypeError, invalid_index_2) def test_unique(self): """Ensure that uniqueness constraints are applied to fields. From f3d265bbe01159379062a110d4b1da420c57ff7c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 17 Jun 2011 10:34:29 +0100 Subject: [PATCH 0281/1279] Added to_dbref Thanks to Ankhbayar for the initial code Closes #202 --- mongoengine/document.py | 8 ++++++++ tests/document.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/mongoengine/document.py b/mongoengine/document.py index 69b19e2..0b408cc 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -167,6 +167,14 @@ class Document(BaseDocument): value._changed_fields = [] return value + def to_dbref(self): + """Returns an instance of :class:`~pymongo.dbref.DBRef` useful in + `__raw__` queries.""" + if not self.pk: + msg = "Only saved documents can have a valid dbref" + raise OperationError(msg) + return pymongo.dbref.DBRef(self.__class__._meta['collection'], self.pk) + @classmethod def register_delete_rule(cls, document_cls, field_name, rule): """This method registers the delete rules to apply when removing this diff --git a/tests/document.py b/tests/document.py index 5d44ca2..d414041 100644 --- a/tests/document.py +++ b/tests/document.py @@ -777,6 +777,14 @@ class DocumentTest(unittest.TestCase): self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 30) + def test_to_dbref(self): + """Ensure that you can get a dbref of a document""" + person = self.Person(name="Test User", age=30) + self.assertRaises(OperationError, person.to_dbref) + person.save() + + person.to_dbref() + def test_reload(self): """Ensure that attributes may be reloaded. """ From 99f923e27f365f245e259c24e0e7953b1b145011 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 17 Jun 2011 15:04:07 +0100 Subject: [PATCH 0282/1279] Fixed queryset repr mid iteration Closes #144 --- docs/changelog.rst | 1 + mongoengine/queryset.py | 5 ++++- tests/queryset.py | 12 ++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a9cfe32..48b5848 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed queryet __repr__ mid iteration - Added hint() support, so cantell Mongo the proper index to use for the query - Fixed issue with inconsitent setting of _cls breaking inherited referencing - Added help_text and verbose_name to fields to help with some form libs diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index bfa89b4..79d24bb 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1524,7 +1524,10 @@ class QuerySet(object): limit = REPR_OUTPUT_SIZE + 1 if self._limit is not None and self._limit < limit: limit = self._limit - data = list(self[self._skip:limit]) + try: + data = list(self[self._skip:limit]) + except pymongo.errors.InvalidOperation: + return ".. queryset mid-iteration .." if len(data) > REPR_OUTPUT_SIZE: data[-1] = "...(remaining elements truncated)..." return repr(data) diff --git a/tests/queryset.py b/tests/queryset.py index cc219fb..6f0098d 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -463,6 +463,18 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(people1, people2) + def test_repr_iteration(self): + """Ensure that QuerySet __repr__ can handle loops + """ + self.Person(name='Person 1').save() + self.Person(name='Person 2').save() + + queryset = self.Person.objects + self.assertEquals('[, ]', repr(queryset)) + for person in queryset: + self.assertEquals('.. queryset mid-iteration ..', repr(queryset)) + + def test_regex_query_shortcuts(self): """Ensure that contains, startswith, endswith, etc work. """ From c24bc77c17fcff8f2ad1f144f4bbe44ecc942971 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 17 Jun 2011 15:07:27 +0100 Subject: [PATCH 0283/1279] Fixes depreciation warnings in Django Auth. Closes #156 --- mongoengine/django/auth.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 41d307c..2711ee1 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -99,6 +99,10 @@ class MongoEngineBackend(object): """Authenticate using MongoEngine and mongoengine.django.auth.User. """ + supports_object_permissions = False + supports_anonymous_user = False + supports_inactive_user = False + def authenticate(self, username=None, password=None): user = User.objects(username=username).first() if user: From e04e5f42efc547c9c5704a07b0b6691ec26aefc4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 20 Jun 2011 08:46:40 +0100 Subject: [PATCH 0284/1279] Added test inheriting document from another file works Closes #28 --- tests/document.py | 58 ++++++++++++++++++++++++++++++++++++++--------- tests/fixtures.py | 26 +++++++++++++++++++++ 2 files changed, 73 insertions(+), 11 deletions(-) create mode 100644 tests/fixtures.py diff --git a/tests/document.py b/tests/document.py index d414041..8f9364f 100644 --- a/tests/document.py +++ b/tests/document.py @@ -4,22 +4,13 @@ import pymongo import pickle import weakref +from fixtures import Base, Mixin, PickleEmbedded, PickleTest + from mongoengine import * from mongoengine.base import BaseField from mongoengine.connection import _get_db -class PickleEmbedded(EmbeddedDocument): - date = DateTimeField(default=datetime.now) - - -class PickleTest(Document): - number = IntField() - string = StringField(choices=(('One', '1'), ('Two', '2'))) - embedded = EmbeddedDocumentField(PickleEmbedded) - lists = ListField(StringField()) - - class DocumentTest(unittest.TestCase): def setUp(self): @@ -108,6 +99,51 @@ class DocumentTest(unittest.TestCase): } self.assertEqual(Animal._get_subclasses(), animal_subclasses) + def test_external_super_and_sub_classes(self): + """Ensure that the correct list of sub and super classes is assembled. + when importing part of the model + """ + class Animal(Base): pass + class Fish(Animal): pass + class Mammal(Animal): pass + class Human(Mammal): pass + class Dog(Mammal): pass + + mammal_superclasses = {'Base': Base, 'Base.Animal': Animal} + self.assertEqual(Mammal._superclasses, mammal_superclasses) + + dog_superclasses = { + 'Base': Base, + 'Base.Animal': Animal, + 'Base.Animal.Mammal': Mammal, + } + self.assertEqual(Dog._superclasses, dog_superclasses) + + animal_subclasses = { + 'Base.Animal.Fish': Fish, + 'Base.Animal.Mammal': Mammal, + 'Base.Animal.Mammal.Dog': Dog, + 'Base.Animal.Mammal.Human': Human + } + self.assertEqual(Animal._get_subclasses(), animal_subclasses) + + mammal_subclasses = { + 'Base.Animal.Mammal.Dog': Dog, + 'Base.Animal.Mammal.Human': Human + } + self.assertEqual(Mammal._get_subclasses(), mammal_subclasses) + + Base.drop_collection() + + h = Human() + h.save() + + self.assertEquals(Human.objects.count(), 1) + self.assertEquals(Mammal.objects.count(), 1) + self.assertEquals(Animal.objects.count(), 1) + self.assertEquals(Base.objects.count(), 1) + Base.drop_collection() + def test_polymorphic_queries(self): """Ensure that the correct subclasses are returned from a query""" class Animal(Document): pass diff --git a/tests/fixtures.py b/tests/fixtures.py new file mode 100644 index 0000000..483b718 --- /dev/null +++ b/tests/fixtures.py @@ -0,0 +1,26 @@ +from datetime import datetime +import pymongo + +from mongoengine import * +from mongoengine.base import BaseField +from mongoengine.connection import _get_db + + +class PickleEmbedded(EmbeddedDocument): + date = DateTimeField(default=datetime.now) + + +class PickleTest(Document): + number = IntField() + string = StringField(choices=(('One', '1'), ('Two', '2'))) + embedded = EmbeddedDocumentField(PickleEmbedded) + lists = ListField(StringField()) + + +class Mixin(object): + number = IntField() + string = StringField(choices=(('One', '1'), ('Two', '2'))) + + +class Base(Document): + pass From 1b0323bc22a8281a2f3fdc7cde635772639eba89 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 20 Jun 2011 09:44:53 +0100 Subject: [PATCH 0285/1279] Added document mixin support For extendable / reusable documents Fixes #204 --- docs/changelog.rst | 1 + mongoengine/base.py | 4 ++++ tests/document.py | 30 ++++++++++++++++++++++++++++++ tests/fixtures.py | 3 +-- 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 48b5848..e3cd723 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added Document Mixin support - Fixed queryet __repr__ mid iteration - Added hint() support, so cantell Mongo the proper index to use for the query - Fixed issue with inconsitent setting of _cls breaking inherited referencing diff --git a/mongoengine/base.py b/mongoengine/base.py index 938808a..f8d415b 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -389,6 +389,7 @@ class DocumentMetaclass(type): class_name = [name] superclasses = {} simple_class = True + for base in bases: # Include all fields present in superclasses if hasattr(base, '_fields'): @@ -397,6 +398,9 @@ class DocumentMetaclass(type): # Get superclasses from superclass superclasses[base._class_name] = base superclasses.update(base._superclasses) + else: # Add any mixin fields + attrs.update(dict([(k,v) for k,v in base.__dict__.items() + if issubclass(v.__class__, BaseField)])) if hasattr(base, '_meta') and not base._meta.get('abstract'): # Ensure that the Document class may be subclassed - diff --git a/tests/document.py b/tests/document.py index 8f9364f..c5aa6e8 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1380,6 +1380,36 @@ class DocumentTest(unittest.TestCase): promoted_employee.reload() self.assertEqual(promoted_employee.details, None) + def test_mixins_dont_add_to_types(self): + + class Bob(Document): name = StringField() + + Bob.drop_collection() + + p = Bob(name="Rozza") + p.save() + Bob.drop_collection() + + class Person(Document, Mixin): + pass + + Person.drop_collection() + + p = Person(name="Rozza") + p.save() + self.assertEquals(p._fields.keys(), ['name', 'id']) + + collection = self.db[Person._meta['collection']] + obj = collection.find_one() + self.assertEquals(obj['_cls'], 'Person') + self.assertEquals(obj['_types'], ['Person']) + + + + self.assertEquals(Person.objects.count(), 1) + rozza = Person.objects.get(name="Rozza") + + Person.drop_collection() def test_save_reference(self): """Ensure that a document reference field may be saved in the database. diff --git a/tests/fixtures.py b/tests/fixtures.py index 483b718..5aaba55 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -18,8 +18,7 @@ class PickleTest(Document): class Mixin(object): - number = IntField() - string = StringField(choices=(('One', '1'), ('Two', '2'))) + name = StringField() class Base(Document): From f41c5217c6f0155d4aa7909fb52a556a79c67aba Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 20 Jun 2011 11:48:12 +0100 Subject: [PATCH 0286/1279] Added a cleaner way to get collection names Also handles dynamic collection naming - refs #180. --- mongoengine/base.py | 26 +++++++++++--- mongoengine/document.py | 46 ++++++++++++++++++++++-- mongoengine/fields.py | 8 +++-- mongoengine/queryset.py | 40 ++------------------- tests/document.py | 79 +++++++++++++++++++++++++++++------------ tests/queryset.py | 4 +-- 6 files changed, 130 insertions(+), 73 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index f8d415b..e59119e 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -22,6 +22,7 @@ class ValidationError(Exception): _document_registry = {} + def get_document(name): doc = _document_registry.get(name, None) if not doc: @@ -195,7 +196,7 @@ class ComplexBaseField(BaseField): elif isinstance(v, (dict, pymongo.son.SON)): if '_ref' in v: # generic reference - collection = get_document(v['_cls'])._meta['collection'] + collection = get_document(v['_cls'])._get_collection_name() collections.setdefault(collection, []).append((k,v)) else: # Use BaseDict so can watch any changes @@ -257,7 +258,7 @@ class ComplexBaseField(BaseField): if v.pk is None: raise ValidationError('You can only reference documents once ' 'they have been saved to the database') - collection = v._meta['collection'] + collection = v._get_collection_name() value_dict[k] = pymongo.dbref.DBRef(collection, v.pk) elif hasattr(v, 'to_python'): value_dict[k] = v.to_python() @@ -306,7 +307,7 @@ class ComplexBaseField(BaseField): from fields import GenericReferenceField value_dict[k] = GenericReferenceField().to_mongo(v) else: - collection = v._meta['collection'] + collection = v._get_collection_name() value_dict[k] = pymongo.dbref.DBRef(collection, v.pk) elif hasattr(v, 'to_mongo'): value_dict[k] = v.to_mongo() @@ -500,9 +501,14 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Subclassed documents inherit collection from superclass for base in bases: if hasattr(base, '_meta'): - if 'collection' in base._meta: - collection = base._meta['collection'] + if 'collection' in attrs.get('meta', {}) and not base._meta.get('abstract', False): + import warnings + msg = "Trying to set a collection on a subclass (%s)" % name + warnings.warn(msg, SyntaxWarning) + del(attrs['meta']['collection']) + if base._get_collection_name(): + collection = base._get_collection_name() # Propagate index options. for key in ('index_background', 'index_drop_dups', 'index_opts'): if key in base._meta: @@ -539,6 +545,10 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # DocumentMetaclass before instantiating CollectionManager object new_class = super_new(cls, name, bases, attrs) + collection = attrs['_meta'].get('collection', None) + if callable(collection): + new_class._meta['collection'] = collection(new_class) + # Provide a default queryset unless one has been manually provided manager = attrs.get('objects', QuerySetManager()) if hasattr(manager, 'queryset_class'): @@ -675,6 +685,12 @@ class BaseDocument(object): elif field.required: raise ValidationError('Field "%s" is required' % field.name) + @classmethod + def _get_collection_name(cls): + """Returns the collection name for this class. + """ + return cls._meta.get('collection', None) + @classmethod def _get_subclasses(cls): """Return a dictionary of all subclasses (found recursively). diff --git a/mongoengine/document.py b/mongoengine/document.py index 0b408cc..36bf401 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -6,7 +6,12 @@ from connection import _get_db import pymongo -__all__ = ['Document', 'EmbeddedDocument', 'ValidationError', 'OperationError'] +__all__ = ['Document', 'EmbeddedDocument', 'ValidationError', + 'OperationError', 'InvalidCollectionError'] + + +class InvalidCollectionError(Exception): + pass class EmbeddedDocument(BaseDocument): @@ -72,6 +77,41 @@ class Document(BaseDocument): """ __metaclass__ = TopLevelDocumentMetaclass + @classmethod + def _get_collection(self): + """Returns the collection for the document.""" + db = _get_db() + collection_name = self._get_collection_name() + + if not hasattr(self, '_collection') or self._collection is None: + # Create collection as a capped collection if specified + if self._meta['max_size'] or self._meta['max_documents']: + # Get max document limit and max byte size from meta + max_size = self._meta['max_size'] or 10000000 # 10MB default + max_documents = self._meta['max_documents'] + + if collection_name in db.collection_names(): + self._collection = db[collection_name] + # The collection already exists, check if its capped + # options match the specified capped options + options = self._collection.options() + if options.get('max') != max_documents or \ + options.get('size') != max_size: + msg = ('Cannot create collection "%s" as a capped ' + 'collection as it already exists') % self._collection + raise InvalidCollectionError(msg) + else: + # Create the collection as a capped collection + opts = {'capped': True, 'size': max_size} + if max_documents: + opts['max'] = max_documents + self._collection = db.create_collection( + collection_name, **opts + ) + else: + self._collection = db[collection_name] + return self._collection + def save(self, safe=True, force_insert=False, validate=True, write_options=None): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be @@ -173,7 +213,7 @@ class Document(BaseDocument): if not self.pk: msg = "Only saved documents can have a valid dbref" raise OperationError(msg) - return pymongo.dbref.DBRef(self.__class__._meta['collection'], self.pk) + return pymongo.dbref.DBRef(self.__class__._get_collection_name(), self.pk) @classmethod def register_delete_rule(cls, document_cls, field_name, rule): @@ -188,7 +228,7 @@ class Document(BaseDocument): :class:`~mongoengine.Document` type from the database. """ db = _get_db() - db.drop_collection(cls._meta['collection']) + db.drop_collection(cls._get_collection_name()) class MapReduceDocument(object): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e1b4366..50a30a1 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -252,7 +252,7 @@ class DateTimeField(BaseField): return datetime.datetime(value.year, value.month, value.day) # Attempt to parse a datetime: - #value = smart_str(value) + # value = smart_str(value) # split usecs, because they are not recognized by strptime. if '.' in value: try: @@ -278,6 +278,7 @@ class DateTimeField(BaseField): return None + class ComplexDateTimeField(StringField): """ ComplexDateTimeField handles microseconds exactly instead of rounding @@ -526,6 +527,7 @@ class MapField(DictField): super(MapField, self).__init__(field=field, *args, **kwargs) + class ReferenceField(BaseField): """A reference to a document that will be automatically dereferenced on access (lazily). @@ -595,7 +597,7 @@ class ReferenceField(BaseField): id_ = document id_ = id_field.to_mongo(id_) - collection = self.document_type._meta['collection'] + collection = self.document_type._get_collection_name() return pymongo.dbref.DBRef(collection, id_) def prepare_query_value(self, op, value): @@ -664,7 +666,7 @@ class GenericReferenceField(BaseField): id_ = document id_ = id_field.to_mongo(id_) - collection = document._meta['collection'] + collection = document._get_collection_name() ref = pymongo.dbref.DBRef(collection, id_) return {'_cls': document._class_name, '_ref': ref} diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 79d24bb..2a5d3ed 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -11,7 +11,7 @@ import itertools import operator __all__ = ['queryset_manager', 'Q', 'InvalidQueryError', - 'InvalidCollectionError', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY'] + 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY'] # The maximum number of items to display in a QuerySet.__repr__ @@ -40,10 +40,6 @@ class OperationError(Exception): pass -class InvalidCollectionError(Exception): - pass - - RE_TYPE = type(re.compile('')) @@ -1360,7 +1356,7 @@ class QuerySet(object): fields = [QuerySet._translate_field_name(self._document, f) for f in fields] - collection = self._document._meta['collection'] + collection = self._document._get_collection_name() scope = { 'collection': collection, @@ -1550,39 +1546,9 @@ class QuerySetManager(object): # Document class being used rather than a document object return self - db = _get_db() - collection = owner._meta['collection'] - if (db, collection) not in self._collections: - # Create collection as a capped collection if specified - if owner._meta['max_size'] or owner._meta['max_documents']: - # Get max document limit and max byte size from meta - max_size = owner._meta['max_size'] or 10000000 # 10MB default - max_documents = owner._meta['max_documents'] - - if collection in db.collection_names(): - self._collections[(db, collection)] = db[collection] - # The collection already exists, check if its capped - # options match the specified capped options - options = self._collections[(db, collection)].options() - if options.get('max') != max_documents or \ - options.get('size') != max_size: - msg = ('Cannot create collection "%s" as a capped ' - 'collection as it already exists') % collection - raise InvalidCollectionError(msg) - else: - # Create the collection as a capped collection - opts = {'capped': True, 'size': max_size} - if max_documents: - opts['max'] = max_documents - self._collections[(db, collection)] = db.create_collection( - collection, **opts - ) - else: - self._collections[(db, collection)] = db[collection] - # owner is the document that contains the QuerySetManager queryset_class = owner._meta['queryset_class'] or QuerySet - queryset = queryset_class(owner, self._collections[(db, collection)]) + queryset = queryset_class(owner, owner._get_collection()) if self.get_queryset: if self.get_queryset.func_code.co_argcount == 1: queryset = self.get_queryset(queryset) diff --git a/tests/document.py b/tests/document.py index c5aa6e8..c10c903 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1,5 +1,10 @@ +import pickle +import pymongo import unittest +import warnings + from datetime import datetime + import pymongo import pickle import weakref @@ -30,7 +35,7 @@ class DocumentTest(unittest.TestCase): """ self.Person(name='Test').save() - collection = self.Person._meta['collection'] + collection = self.Person._get_collection_name() self.assertTrue(collection in self.db.collection_names()) self.Person.drop_collection() @@ -57,6 +62,23 @@ class DocumentTest(unittest.TestCase): # Ensure Document isn't treated like an actual document self.assertFalse(hasattr(Document, '_fields')) + def test_dynamic_collection_naming(self): + + def create_collection_name(cls): + return "PERSON" + + class DynamicPerson(Document): + name = StringField() + age = IntField() + + meta = {'collection': create_collection_name} + + collection = DynamicPerson._get_collection_name() + self.assertEquals(collection, 'PERSON') + + DynamicPerson(name='Test User', age=30).save() + self.assertTrue(collection in self.db.collection_names()) + def test_get_superclasses(self): """Ensure that the correct list of superclasses is assembled. """ @@ -225,8 +247,8 @@ class DocumentTest(unittest.TestCase): self.assertTrue('name' in Employee._fields) self.assertTrue('salary' in Employee._fields) - self.assertEqual(Employee._meta['collection'], - self.Person._meta['collection']) + self.assertEqual(Employee._get_collection_name(), + self.Person._get_collection_name()) # Ensure that MRO error is not raised class A(Document): pass @@ -251,7 +273,7 @@ class DocumentTest(unittest.TestCase): # Check that _cls etc aren't present on simple documents dog = Animal(name='dog') dog.save() - collection = self.db[Animal._meta['collection']] + collection = self.db[Animal._get_collection_name()] obj = collection.find_one() self.assertFalse('_cls' in obj) self.assertFalse('_types' in obj) @@ -297,7 +319,7 @@ class DocumentTest(unittest.TestCase): # Check that _cls etc aren't present on simple documents dog = Animal(name='dog') dog.save() - collection = self.db[Animal._meta['collection']] + collection = self.db[Animal._get_collection_name()] obj = collection.find_one() self.assertFalse('_cls' in obj) self.assertFalse('_types' in obj) @@ -318,7 +340,7 @@ class DocumentTest(unittest.TestCase): dog = Animal(name='dog') dog.save() - collection = self.db[Animal._meta['collection']] + collection = self.db[Animal._get_collection_name()] obj = collection.find_one() self.assertTrue('_cls' in obj) self.assertTrue('_types' in obj) @@ -381,9 +403,12 @@ class DocumentTest(unittest.TestCase): self.assertFalse('collection' in Animal._meta) self.assertFalse('collection' in Mammal._meta) - self.assertEqual(Fish._meta['collection'], 'fish') - self.assertEqual(Guppy._meta['collection'], 'fish') - self.assertEqual(Human._meta['collection'], 'human') + self.assertEqual(Animal._get_collection_name(), None) + self.assertEqual(Mammal._get_collection_name(), None) + + self.assertEqual(Fish._get_collection_name(), 'fish') + self.assertEqual(Guppy._get_collection_name(), 'fish') + self.assertEqual(Human._get_collection_name(), 'human') def create_bad_abstract(): class EvilHuman(Human): @@ -434,14 +459,21 @@ class DocumentTest(unittest.TestCase): def test_inherited_collections(self): """Ensure that subclassed documents don't override parents' collections. """ - class Drink(Document): - name = StringField() + with warnings.catch_warnings(record=True) as w: + # Cause all warnings to always be triggered. + warnings.simplefilter("always") - class AlcoholicDrink(Drink): - meta = {'collection': 'booze'} + class Drink(Document): + name = StringField() - class Drinker(Document): - drink = GenericReferenceField() + class AlcoholicDrink(Drink): + meta = {'collection': 'booze'} + + class Drinker(Document): + drink = GenericReferenceField() + + # Confirm we triggered a SyntaxWarning + assert issubclass(w[0].category, SyntaxWarning) Drink.drop_collection() AlcoholicDrink.drop_collection() @@ -455,7 +487,6 @@ class DocumentTest(unittest.TestCase): beer = AlcoholicDrink(name='Beer') beer.save() - real_person = Drinker(drink=beer) real_person.save() @@ -936,7 +967,7 @@ class DocumentTest(unittest.TestCase): person = self.Person(name='Test User', age=30) person.save() # Ensure that the object is in the database - collection = self.db[self.Person._meta['collection']] + collection = self.db[self.Person._get_collection_name()] person_obj = collection.find_one({'name': 'Test User'}) self.assertEqual(person_obj['name'], 'Test User') self.assertEqual(person_obj['age'], 30) @@ -1279,7 +1310,7 @@ class DocumentTest(unittest.TestCase): id='497ce96f395f2f052a494fd4') person.save() # Ensure that the object is in the database with the correct _id - collection = self.db[self.Person._meta['collection']] + collection = self.db[self.Person._get_collection_name()] person_obj = collection.find_one({'name': 'Test User'}) self.assertEqual(str(person_obj['_id']), '497ce96f395f2f052a494fd4') @@ -1291,7 +1322,7 @@ class DocumentTest(unittest.TestCase): pk='497ce96f395f2f052a494fd4') person.save() # Ensure that the object is in the database with the correct _id - collection = self.db[self.Person._meta['collection']] + collection = self.db[self.Person._get_collection_name()] person_obj = collection.find_one({'name': 'Test User'}) self.assertEqual(str(person_obj['_id']), '497ce96f395f2f052a494fd4') @@ -1314,7 +1345,7 @@ class DocumentTest(unittest.TestCase): post.comments = comments post.save() - collection = self.db[BlogPost._meta['collection']] + collection = self.db[BlogPost._get_collection_name()] post_obj = collection.find_one() self.assertEqual(post_obj['tags'], tags) for comment_obj, comment in zip(post_obj['comments'], comments): @@ -1339,7 +1370,7 @@ class DocumentTest(unittest.TestCase): employee.save() # Ensure that the object is in the database - collection = self.db[self.Person._meta['collection']] + collection = self.db[self.Person._get_collection_name()] employee_obj = collection.find_one({'name': 'Test Employee'}) self.assertEqual(employee_obj['name'], 'Test Employee') self.assertEqual(employee_obj['age'], 50) @@ -1370,6 +1401,7 @@ class DocumentTest(unittest.TestCase): promoted_employee.reload() self.assertEqual(promoted_employee.name, 'Test Employee') self.assertEqual(promoted_employee.age, 50) + # Ensure that the 'details' embedded object saved correctly self.assertEqual(promoted_employee.details.position, 'Senior Developer') @@ -1399,7 +1431,7 @@ class DocumentTest(unittest.TestCase): p.save() self.assertEquals(p._fields.keys(), ['name', 'id']) - collection = self.db[Person._meta['collection']] + collection = self.db[Person._get_collection_name()] obj = collection.find_one() self.assertEquals(obj['_cls'], 'Person') self.assertEquals(obj['_types'], ['Person']) @@ -1492,6 +1524,9 @@ class DocumentTest(unittest.TestCase): text = StringField() post = ReferenceField(BlogPost, reverse_delete_rule=CASCADE) + self.Person.drop_collection() + BlogPost.drop_collection() + Comment.drop_collection() author = self.Person(name='Test User') author.save() diff --git a/tests/queryset.py b/tests/queryset.py index 6f0098d..c5f177c 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- - - import unittest import pymongo from datetime import datetime, timedelta @@ -27,7 +25,7 @@ class QuerySetTest(unittest.TestCase): """ self.assertTrue(isinstance(self.Person.objects, QuerySet)) self.assertEqual(self.Person.objects._collection.name, - self.Person._meta['collection']) + self.Person._get_collection_name()) self.assertTrue(isinstance(self.Person.objects._collection, pymongo.collection.Collection)) From e3cd398f70594693a4f3539b9796e0143b659992 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 20 Jun 2011 14:00:06 +0100 Subject: [PATCH 0287/1279] Changed default collection naming Also added upgrade text --- docs/changelog.rst | 1 + docs/index.rst | 5 ++-- docs/upgrade.rst | 73 +++++++++++++++++++++++++++++++++++++++++++++ mongoengine/base.py | 2 +- tests/document.py | 72 +++++++++++++++++++++++++++++++++++++------- 5 files changed, 139 insertions(+), 14 deletions(-) create mode 100644 docs/upgrade.rst diff --git a/docs/changelog.rst b/docs/changelog.rst index e3cd723..cfae79e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Updated default collection naming convention - Added Document Mixin support - Fixed queryet __repr__ mid iteration - Added hint() support, so cantell Mongo the proper index to use for the query diff --git a/docs/index.rst b/docs/index.rst index ccb7fbe..3b03656 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,7 +2,7 @@ MongoEngine User Documentation ============================== -MongoEngine is an Object-Document Mapper, written in Python for working with +MongoEngine is an Object-Document Mapper, written in Python for working with MongoDB. To install it, simply run .. code-block:: console @@ -15,7 +15,7 @@ To get help with using MongoEngine, use the `MongoEngine Users mailing list `_ or come chat on the `#mongoengine IRC channel `_. -If you are interested in contributing, join the developers' `mailing list +If you are interested in contributing, join the developers' `mailing list `_. .. toctree:: @@ -26,6 +26,7 @@ If you are interested in contributing, join the developers' `mailing list apireference django changelog + upgrading Indices and tables ================== diff --git a/docs/upgrade.rst b/docs/upgrade.rst new file mode 100644 index 0000000..f005e2e --- /dev/null +++ b/docs/upgrade.rst @@ -0,0 +1,73 @@ +========= +Upgrading +========= + +0.4 to 0.5 +=========== + +There have been the following backwards incompatibilities from 0.4 to 0.5: + +#. Default collection naming. + +Previously it was just lowercase, its now much more pythonic and readable as its +lowercase and underscores, previously :: + + class MyAceDocument(Document): + pass + + MyAceDocument._meta['collection'] == myacedocument + +In 0.5 this will change to :: + + class MyAceDocument(Document): + pass + + MyAceDocument._get_collection_name() == my_ace_document + +To upgrade use a Mixin class to set meta like so :: + + class BaseMixin(object): + meta = { + 'collection': lambda c: c.__name__.lower() + } + + class MyAceDocument(Document, BaseMixin): + pass + + MyAceDocument._get_collection_name() == myacedocument + +Alternatively, you can rename your collections eg :: + + from mongoengine.connection import _get_db + from mongoengine.base import _document_registry + + def rename_collections(): + db = _get_db() + + failure = False + + collection_names = [d._get_collection_name() for d in _document_registry.values()] + + for new_style_name in collection_names: + if not new_style_name: # embedded documents don't have collections + continue + old_style_name = new_style_name.replace('_', '') + + if old_style_name == new_style_name: + continue # Nothing to do + + existing = db.collection_names() + if old_style_name in existing: + if new_style_name in existing: + failure = True + print "FAILED to rename: %s to %s (already exists)" % ( + old_style_name, new_style_name) + else: + db[old_style_name].rename(new_style_name) + print "Renamed: %s to %s" % (old_style_name, new_style_name) + + if failure: + print "Upgrading collection names failed" + else: + print "Upgraded collection names" + diff --git a/mongoengine/base.py b/mongoengine/base.py index e59119e..94f00cb 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -492,7 +492,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): raise ValueError("Abstract document cannot have non-abstract base") return super_new(cls, name, bases, attrs) - collection = name.lower() + collection = ''.join('_%s' % c if c.isupper() else c for c in name).strip('_').lower() id_field = None base_indexes = [] diff --git a/tests/document.py b/tests/document.py index c10c903..28d6133 100644 --- a/tests/document.py +++ b/tests/document.py @@ -62,22 +62,72 @@ class DocumentTest(unittest.TestCase): # Ensure Document isn't treated like an actual document self.assertFalse(hasattr(Document, '_fields')) - def test_dynamic_collection_naming(self): + def test_collection_name(self): + """Ensure that a collection with a specified name may be used. + """ - def create_collection_name(cls): - return "PERSON" + class DefaultNamingTest(Document): + pass + self.assertEquals('default_naming_test', DefaultNamingTest._get_collection_name()) - class DynamicPerson(Document): - name = StringField() - age = IntField() + class CustomNamingTest(Document): + meta = {'collection': 'pimp_my_collection'} - meta = {'collection': create_collection_name} + self.assertEquals('pimp_my_collection', CustomNamingTest._get_collection_name()) - collection = DynamicPerson._get_collection_name() - self.assertEquals(collection, 'PERSON') + class DynamicNamingTest(Document): + meta = {'collection': lambda c: "DYNAMO"} + self.assertEquals('DYNAMO', DynamicNamingTest._get_collection_name()) - DynamicPerson(name='Test User', age=30).save() - self.assertTrue(collection in self.db.collection_names()) + # Use Abstract class to handle backwards compatibility + class BaseDocument(Document): + meta = { + 'abstract': True, + 'collection': lambda c: c.__name__.lower() + } + + class OldNamingConvention(BaseDocument): + pass + self.assertEquals('oldnamingconvention', OldNamingConvention._get_collection_name()) + + class InheritedAbstractNamingTest(BaseDocument): + meta = {'collection': 'wibble'} + self.assertEquals('wibble', InheritedAbstractNamingTest._get_collection_name()) + + with warnings.catch_warnings(record=True) as w: + # Cause all warnings to always be triggered. + warnings.simplefilter("always") + + class NonAbstractBase(Document): + pass + + class InheritedDocumentFailTest(NonAbstractBase): + meta = {'collection': 'fail'} + + self.assertTrue(issubclass(w[0].category, SyntaxWarning)) + self.assertEquals('non_abstract_base', InheritedDocumentFailTest._get_collection_name()) + + # Mixin tests + class BaseMixin(object): + meta = { + 'collection': lambda c: c.__name__.lower() + } + + class OldMixinNamingConvention(Document, BaseMixin): + pass + self.assertEquals('oldmixinnamingconvention', OldMixinNamingConvention._get_collection_name()) + + class BaseMixin(object): + meta = { + 'collection': lambda c: c.__name__.lower() + } + + class BaseDocument(Document, BaseMixin): + pass + + class MyDocument(BaseDocument): + pass + self.assertEquals('mydocument', OldMixinNamingConvention._get_collection_name()) def test_get_superclasses(self): """Ensure that the correct list of superclasses is assembled. From 08ba51f714731a6fc27340cae8cdb47bf1d60302 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 20 Jun 2011 15:41:23 +0100 Subject: [PATCH 0288/1279] Updated geo_index checking to be recursive Fixes #127 - Embedded Documents can declare geo indexes and have them created automatically --- docs/changelog.rst | 1 + mongoengine/base.py | 194 +++++++++++++++++++++------------------- mongoengine/queryset.py | 11 ++- tests/fields.py | 21 +++++ 4 files changed, 129 insertions(+), 98 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index cfae79e..0737171 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Updated geo index checking to be recursive and check in embedded documents - Updated default collection naming convention - Added Document Mixin support - Fixed queryet __repr__ mid iteration diff --git a/mongoengine/base.py b/mongoengine/base.py index 94f00cb..12c760a 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -644,28 +644,6 @@ class BaseDocument(object): signals.post_init.send(self.__class__, document=self) - def __getstate__(self): - self_dict = self.__dict__ - removals = ["get_%s_display" % k for k,v in self._fields.items() if v.choices] - for k in removals: - if hasattr(self, k): - delattr(self, k) - return self.__dict__ - - def __setstate__(self, __dict__): - self.__dict__ = __dict__ - self.__set_field_display() - - def __set_field_display(self): - for attr_name, field in self._fields.items(): - if field.choices: # dynamically adds a way to get the display value for a field with choices - setattr(self, 'get_%s_display' % attr_name, partial(self.__get_field_display, field=field)) - - def __get_field_display(self, field): - """Returns the display value for a choice field""" - value = getattr(self, field.name) - return dict(field.choices).get(value, value) - def validate(self): """Ensure that all fields' values are valid and that required fields are present. @@ -685,6 +663,33 @@ class BaseDocument(object): elif field.required: raise ValidationError('Field "%s" is required' % field.name) + @apply + def pk(): + """Primary key alias + """ + def fget(self): + return getattr(self, self._meta['id_field']) + def fset(self, value): + return setattr(self, self._meta['id_field'], value) + return property(fget, fset) + + def to_mongo(self): + """Return data dictionary ready for use with MongoDB. + """ + data = {} + for field_name, field in self._fields.items(): + value = getattr(self, field_name, None) + if value is not None: + data[field.db_field] = field.to_mongo(value) + # Only add _cls and _types if allow_inheritance is not False + if not (hasattr(self, '_meta') and + self._meta.get('allow_inheritance', True) == False): + data['_cls'] = self._class_name + data['_types'] = self._superclasses.keys() + [self._class_name] + if '_id' in data and data['_id'] is None: + del data['_id'] + return data + @classmethod def _get_collection_name(cls): """Returns the collection name for this class. @@ -706,76 +711,6 @@ class BaseDocument(object): all_subclasses.update(subclass._get_subclasses()) return all_subclasses - @apply - def pk(): - """Primary key alias - """ - def fget(self): - return getattr(self, self._meta['id_field']) - def fset(self, value): - return setattr(self, self._meta['id_field'], value) - return property(fget, fset) - - def __iter__(self): - return iter(self._fields) - - def __getitem__(self, name): - """Dictionary-style field access, return a field's value if present. - """ - try: - if name in self._fields: - return getattr(self, name) - except AttributeError: - pass - raise KeyError(name) - - def __setitem__(self, name, value): - """Dictionary-style field access, set a field's value. - """ - # Ensure that the field exists before settings its value - if name not in self._fields: - raise KeyError(name) - return setattr(self, name, value) - - def __contains__(self, name): - try: - val = getattr(self, name) - return val is not None - except AttributeError: - return False - - def __len__(self): - return len(self._data) - - def __repr__(self): - try: - u = unicode(self) - except (UnicodeEncodeError, UnicodeDecodeError): - u = '[Bad Unicode data]' - return u'<%s: %s>' % (self.__class__.__name__, u) - - def __str__(self): - if hasattr(self, '__unicode__'): - return unicode(self).encode('utf-8') - return '%s object' % self.__class__.__name__ - - def to_mongo(self): - """Return data dictionary ready for use with MongoDB. - """ - data = {} - for field_name, field in self._fields.items(): - value = getattr(self, field_name, None) - if value is not None: - data[field.db_field] = field.to_mongo(value) - # Only add _cls and _types if allow_inheritance is not False - if not (hasattr(self, '_meta') and - self._meta.get('allow_inheritance', True) == False): - data['_cls'] = self._class_name - data['_types'] = self._superclasses.keys() + [self._class_name] - if '_id' in data and data['_id'] is None: - del data['_id'] - return data - @classmethod def _from_son(cls, son): """Create an instance of a Document (subclass) from a PyMongo SON. @@ -874,6 +809,81 @@ class BaseDocument(object): unset_data[k] = 1 return set_data, unset_data + @classmethod + def _geo_indices(cls): + geo_indices = [] + for field in cls._fields.values(): + if hasattr(field, 'document_type'): + geo_indices += field.document_type._geo_indices() + elif field._geo_index: + geo_indices.append(field) + return geo_indices + + def __getstate__(self): + self_dict = self.__dict__ + removals = ["get_%s_display" % k for k,v in self._fields.items() if v.choices] + for k in removals: + if hasattr(self, k): + delattr(self, k) + return self.__dict__ + + def __setstate__(self, __dict__): + self.__dict__ = __dict__ + self.__set_field_display() + + def __set_field_display(self): + for attr_name, field in self._fields.items(): + if field.choices: # dynamically adds a way to get the display value for a field with choices + setattr(self, 'get_%s_display' % attr_name, partial(self.__get_field_display, field=field)) + + def __get_field_display(self, field): + """Returns the display value for a choice field""" + value = getattr(self, field.name) + return dict(field.choices).get(value, value) + + def __iter__(self): + return iter(self._fields) + + def __getitem__(self, name): + """Dictionary-style field access, return a field's value if present. + """ + try: + if name in self._fields: + return getattr(self, name) + except AttributeError: + pass + raise KeyError(name) + + def __setitem__(self, name, value): + """Dictionary-style field access, set a field's value. + """ + # Ensure that the field exists before settings its value + if name not in self._fields: + raise KeyError(name) + return setattr(self, name, value) + + def __contains__(self, name): + try: + val = getattr(self, name) + return val is not None + except AttributeError: + return False + + def __len__(self): + return len(self._data) + + def __repr__(self): + try: + u = unicode(self) + except (UnicodeEncodeError, UnicodeDecodeError): + u = '[Bad Unicode data]' + return u'<%s: %s>' % (self.__class__.__name__, u) + + def __str__(self): + if hasattr(self, '__unicode__'): + return unicode(self).encode('utf-8') + return '%s object' % self.__class__.__name__ + def __eq__(self, other): if isinstance(other, self.__class__) and hasattr(other, 'id'): if self.id == other.id: diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 2a5d3ed..e2947a0 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -494,12 +494,11 @@ class QuerySet(object): self._collection.ensure_index('_types', background=background, **index_opts) - # Ensure all needed field indexes are created - for field in self._document._fields.values(): - if field.__class__._geo_index: - index_spec = [(field.db_field, pymongo.GEO2D)] - self._collection.ensure_index(index_spec, - background=background, **index_opts) + # Add geo indicies + for field in self._document._geo_indices(): + index_spec = [(field.db_field, pymongo.GEO2D)] + self._collection.ensure_index(index_spec, + background=background, **index_opts) return self._collection_obj diff --git a/tests/fields.py b/tests/fields.py index 2204930..fe53d9e 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1312,6 +1312,27 @@ class FieldTest(unittest.TestCase): Event.drop_collection() + def test_geo_embedded_indexes(self): + """Ensure that indexes are created automatically for GeoPointFields on + embedded documents. + """ + class Venue(EmbeddedDocument): + location = GeoPointField() + name = StringField() + + class Event(Document): + title = StringField() + venue = EmbeddedDocumentField(Venue) + + Event.drop_collection() + venue = Venue(name="Double Door", location=[41.909889, -87.677137]) + event = Event(title="Coltrane Motion", venue=venue) + event.save() + + info = Event.objects._collection.index_information() + self.assertTrue(u'location_2d' in info) + self.assertTrue(info[u'location_2d']['key'] == [(u'location', u'2d')]) + def test_ensure_unique_default_instances(self): """Ensure that every field has it's own unique default instance.""" class D(Document): From 09c32a63cedcb2faa8f58e4f4444cd0788eb4df3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 21 Jun 2011 12:34:14 +0100 Subject: [PATCH 0289/1279] Fixes bug with appending post save - due to lists not being reset --- mongoengine/document.py | 13 ++++++++++++- tests/fields.py | 10 ++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 36bf401..e20500d 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -161,7 +161,18 @@ class Document(BaseDocument): raise OperationError(message % unicode(err)) id_field = self._meta['id_field'] self[id_field] = self._fields[id_field].to_python(object_id) - self._changed_fields = [] + + def reset_changed_fields(doc): + """Loop through and reset changed fields lists""" + if hasattr(doc, '_changed_fields'): + doc._changed_fields = [] + + for field_name in doc._fields: + field = getattr(doc, field_name) + if hasattr(field, '_changed_fields') and field != doc: + reset_changed_fields(field) + + reset_changed_fields(self) signals.post_save.send(self.__class__, document=self, created=created) def delete(self, safe=False): diff --git a/tests/fields.py b/tests/fields.py index fe53d9e..01280a1 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -879,7 +879,7 @@ class FieldTest(unittest.TestCase): name = StringField() children = ListField(EmbeddedDocumentField('self')) - Tree.drop_collection + Tree.drop_collection() tree = Tree(name="Tree") first_child = TreeNode(name="Child 1") @@ -887,9 +887,15 @@ class FieldTest(unittest.TestCase): second_child = TreeNode(name="Child 2") first_child.children.append(second_child) + tree.save() + + tree = Tree.objects.first() + self.assertEqual(len(tree.children), 1) + + self.assertEqual(len(tree.children[0].children), 1) third_child = TreeNode(name="Child 3") - first_child.children.append(third_child) + tree.children[0].children.append(third_child) tree.save() self.assertEqual(len(tree.children), 1) From 14be7ba2e202b8bc7c0f8b7bc729aeb59d3f3e0a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 21 Jun 2011 14:50:11 +0100 Subject: [PATCH 0290/1279] Added support for the $ positional operator closes #205 --- docs/changelog.rst | 1 + docs/guide/querying.rst | 34 ++++++++++++++------ mongoengine/queryset.py | 7 ++-- tests/queryset.py | 71 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 11 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0737171..4fb5d62 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added support for the positional operator - Updated geo index checking to be recursive and check in embedded documents - Updated default collection naming convention - Added Document Mixin support diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 1caed2d..4f36e96 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -23,7 +23,7 @@ fetch documents from the database:: Filtering queries ================= The query may be filtered by calling the -:class:`~mongoengine.queryset.QuerySet` object with field lookup keyword +:class:`~mongoengine.queryset.QuerySet` object with field lookup keyword arguments. The keys in the keyword arguments correspond to fields on the :class:`~mongoengine.Document` you are querying:: @@ -84,7 +84,7 @@ Available operators are as follows: * ``nin`` -- value is not in list (a list of values should be provided) * ``mod`` -- ``value % x == y``, where ``x`` and ``y`` are two provided values * ``all`` -- every item in list of values provided is in array -* ``size`` -- the size of the array is +* ``size`` -- the size of the array is * ``exists`` -- value for field exists The following operators are available as shortcuts to querying with regular @@ -163,9 +163,9 @@ To retrieve a result that should be unique in the collection, use and :class:`~mongoengine.queryset.MultipleObjectsReturned` if more than one document matched the query. -A variation of this method exists, +A variation of this method exists, :meth:`~mongoengine.queryset.Queryset.get_or_create`, that will create a new -document with the query arguments if no documents match the query. An +document with the query arguments if no documents match the query. An additional keyword argument, :attr:`defaults` may be provided, which will be used as default values for the new document, in the case that it should need to be created:: @@ -240,7 +240,7 @@ Javascript code that is executed on the database server. Counting results ---------------- Just as with limiting and skipping results, there is a method on -:class:`~mongoengine.queryset.QuerySet` objects -- +:class:`~mongoengine.queryset.QuerySet` objects -- :meth:`~mongoengine.queryset.QuerySet.count`, but there is also a more Pythonic way of achieving this:: @@ -309,11 +309,11 @@ Advanced queries ================ Sometimes calling a :class:`~mongoengine.queryset.QuerySet` object with keyword arguments can't fully express the query you want to use -- for example if you -need to combine a number of constraints using *and* and *or*. This is made +need to combine a number of constraints using *and* and *or*. This is made possible in MongoEngine through the :class:`~mongoengine.queryset.Q` class. A :class:`~mongoengine.queryset.Q` object represents part of a query, and can be initialised using the same keyword-argument syntax you use to query -documents. To build a complex query, you may combine +documents. To build a complex query, you may combine :class:`~mongoengine.queryset.Q` objects using the ``&`` (and) and ``|`` (or) operators. To use a :class:`~mongoengine.queryset.Q` object, pass it in as the first positional argument to :attr:`Document.objects` when you filter it by @@ -434,7 +434,7 @@ Atomic updates ============== Documents may be updated atomically by using the :meth:`~mongoengine.queryset.QuerySet.update_one` and -:meth:`~mongoengine.queryset.QuerySet.update` methods on a +:meth:`~mongoengine.queryset.QuerySet.update` methods on a :meth:`~mongoengine.queryset.QuerySet`. There are several different "modifiers" that you may use with these methods: @@ -450,7 +450,7 @@ that you may use with these methods: * ``pull_all`` -- remove several values from a list * ``add_to_set`` -- add value to a list only if its not in the list already -The syntax for atomic updates is similar to the querying syntax, but the +The syntax for atomic updates is similar to the querying syntax, but the modifier comes before the field, not after it:: >>> post = BlogPost(title='Test', page_views=0, tags=['database']) @@ -467,3 +467,19 @@ modifier comes before the field, not after it:: >>> post.reload() >>> post.tags ['database', 'nosql'] + +The positional operator allows you to update list items without knowing the +index position, therefore making the update a single atomic operation. As we +cannot use the `$` syntax in keyword arguments it has been mapped to `S`:: + + >>> post = BlogPost(title='Test', page_views=0, tags=['database', 'mongo']) + >>> post.save() + >>> BlogPost.objects(id=post.id, tags='mongo').update(set__tags__S='mongodb') + >>> post.reload() + >>> post.tags + ['database', 'mongodb'] + +.. note :: + Currently only top level lists are handled, future versions of mongodb / + pymongo plan to support nested positional operators. See `The $ positional + operator `_. \ No newline at end of file diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index e2947a0..82138fe 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1215,6 +1215,9 @@ class QuerySet(object): append_field = True for field in fields: if isinstance(field, str): + # Convert the S operator to $ + if field == 'S': + field = '$' parts.append(field) append_field = False else: @@ -1243,7 +1246,7 @@ class QuerySet(object): return mongo_update - def update(self, safe_update=True, upsert=False, write_options=None, **update): + def update(self, safe_update=True, upsert=False, multi=True, write_options=None, **update): """Perform an atomic update on the fields matched by the query. When ``safe_update`` is used, the number of affected documents is returned. @@ -1261,7 +1264,7 @@ class QuerySet(object): update = QuerySet._transform_update(self._document, **update) try: - ret = self._collection.update(self._query, update, multi=True, + ret = self._collection.update(self._query, update, multi=multi, upsert=upsert, safe=safe_update, **write_options) if ret is not None and 'n' in ret: diff --git a/tests/queryset.py b/tests/queryset.py index c5f177c..c0860b5 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -260,6 +260,77 @@ class QuerySetTest(unittest.TestCase): Blog.drop_collection() + def test_update_using_positional_operator(self): + """Ensure that the list fields can be updated using the positional + operator.""" + + class Comment(EmbeddedDocument): + by = StringField() + votes = IntField() + + class BlogPost(Document): + title = StringField() + comments = ListField(EmbeddedDocumentField(Comment)) + + BlogPost.drop_collection() + + c1 = Comment(by="joe", votes=3) + c2 = Comment(by="jane", votes=7) + + BlogPost(title="ABC", comments=[c1, c2]).save() + + BlogPost.objects(comments__by="joe").update(inc__comments__S__votes=1) + + post = BlogPost.objects.first() + self.assertEquals(post.comments[0].by, 'joe') + self.assertEquals(post.comments[0].votes, 4) + + # Currently the $ operator only applies to the first matched item in + # the query + + class Simple(Document): + x = ListField() + + Simple.drop_collection() + Simple(x=[1, 2, 3, 2]).save() + Simple.objects(x=2).update(inc__x__S=1) + + simple = Simple.objects.first() + self.assertEquals(simple.x, [1, 3, 3, 2]) + Simple.drop_collection() + + # You can set multiples + Simple.drop_collection() + Simple(x=[1, 2, 3, 4]).save() + Simple(x=[2, 3, 4, 5]).save() + Simple(x=[3, 4, 5, 6]).save() + Simple(x=[4, 5, 6, 7]).save() + Simple.objects(x=3).update(set__x__S=0) + + s = Simple.objects() + self.assertEquals(s[0].x, [1, 2, 0, 4]) + self.assertEquals(s[1].x, [2, 0, 4, 5]) + self.assertEquals(s[2].x, [0, 4, 5, 6]) + self.assertEquals(s[3].x, [4, 5, 6, 7]) + + # Using "$unset" with an expression like this "array.$" will result in + # the array item becoming None, not being removed. + Simple.drop_collection() + Simple(x=[1, 2, 3, 4, 3, 2, 3, 4]).save() + Simple.objects(x=3).update(unset__x__S=1) + simple = Simple.objects.first() + self.assertEquals(simple.x, [1, 2, None, 4, 3, 2, 3, 4]) + + # Nested updates arent supported yet.. + def update_nested(): + Simple.drop_collection() + Simple(x=[{'test': [1, 2, 3, 4]}]).save() + Simple.objects(x__test=2).update(set__x__S__test__S=3) + self.assertEquals(simple.x, [1, 2, 3, 4]) + + self.assertRaises(OperationError, update_nested) + Simple.drop_collection() + def test_mapfield_update(self): """Ensure that the MapField can be updated.""" class Member(EmbeddedDocument): From 87f486c4f13508215c5dec9cf945b5ed775394c1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 22 Jun 2011 15:45:25 +0100 Subject: [PATCH 0291/1279] Added select_related() and refactored dereferencing Added a dereference class to handle both select_related / recursive dereferencing and fetching dereference. Refs #206 --- mongoengine/base.py | 74 ++-------- mongoengine/dereference.py | 171 +++++++++++++++++++++++ mongoengine/document.py | 5 + mongoengine/queryset.py | 11 +- tests/dereference.py | 272 ++++++++++++++++++++++++++++++++++++- tests/document.py | 6 +- 6 files changed, 459 insertions(+), 80 deletions(-) create mode 100644 mongoengine/dereference.py diff --git a/mongoengine/base.py b/mongoengine/base.py index 12c760a..8101aa0 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -8,7 +8,7 @@ import weakref import sys import pymongo import pymongo.objectid -from operator import itemgetter +import operator from functools import partial @@ -163,70 +163,14 @@ class ComplexBaseField(BaseField): def __get__(self, instance, owner): """Descriptor to automatically dereference references. """ - from connection import _get_db - if instance is None: # Document class being used rather than a document object return self - # Get value from document instance if available - value_list = instance._data.get(self.name) - if not value_list or isinstance(value_list, basestring): - return super(ComplexBaseField, self).__get__(instance, owner) - - is_list = False - if not hasattr(value_list, 'items'): - is_list = True - value_list = dict([(k,v) for k,v in enumerate(value_list)]) - - for k,v in value_list.items(): - if isinstance(v, dict) and '_cls' in v and '_ref' not in v: - value_list[k] = get_document(v['_cls'])._from_son(v) - - # Handle all dereferencing - db = _get_db() - dbref = {} - collections = {} - for k,v in value_list.items(): - - # Save any DBRefs - if isinstance(v, (pymongo.dbref.DBRef)): - # direct reference (DBRef) - collections.setdefault(v.collection, []).append((k,v)) - elif isinstance(v, (dict, pymongo.son.SON)): - if '_ref' in v: - # generic reference - collection = get_document(v['_cls'])._get_collection_name() - collections.setdefault(collection, []).append((k,v)) - else: - # Use BaseDict so can watch any changes - dbref[k] = BaseDict(v, instance=instance, name=self.name) - else: - dbref[k] = v - - # For each collection get the references - for collection, dbrefs in collections.items(): - id_map = {} - for k,v in dbrefs: - if isinstance(v, (pymongo.dbref.DBRef)): - # direct reference (DBRef), has no _cls information - id_map[v.id] = (k, None) - elif isinstance(v, (dict, pymongo.son.SON)) and '_ref' in v: - # generic reference - includes _cls information - id_map[v['_ref'].id] = (k, get_document(v['_cls'])) - - references = db[collection].find({'_id': {'$in': id_map.keys()}}) - for ref in references: - key, doc_cls = id_map[ref['_id']] - if not doc_cls: # If no doc_cls get it from the referenced doc - doc_cls = get_document(ref['_cls']) - dbref[key] = doc_cls._from_son(ref) - - if is_list: - dbref = BaseList([v for k,v in sorted(dbref.items(), key=itemgetter(0))], instance=instance, name=self.name) - else: - dbref = BaseDict(dbref, instance=instance, name=self.name) - instance._data[self.name] = dbref + from dereference import dereference + instance._data[self.name] = dereference( + instance._data.get(self.name), max_depth=1, instance=instance, name=self.name, get=True + ) return super(ComplexBaseField, self).__get__(instance, owner) def to_python(self, value): @@ -266,7 +210,7 @@ class ComplexBaseField(BaseField): value_dict[k] = self.to_python(v) if is_list: # Convert back to a list - return [v for k,v in sorted(value_dict.items(), key=itemgetter(0))] + return [v for k,v in sorted(value_dict.items(), key=operator.itemgetter(0))] return value_dict def to_mongo(self, value): @@ -315,7 +259,7 @@ class ComplexBaseField(BaseField): value_dict[k] = self.to_mongo(v) if is_list: # Convert back to a list - return [v for k,v in sorted(value_dict.items(), key=itemgetter(0))] + return [v for k,v in sorted(value_dict.items(), key=operator.itemgetter(0))] return value_dict def validate(self, value): @@ -907,7 +851,7 @@ class BaseList(list): """ def __init__(self, list_items, instance, name): - self.instance = weakref.proxy(instance) + self.instance = instance self.name = name super(BaseList, self).__init__(list_items) @@ -958,7 +902,7 @@ class BaseDict(dict): """ def __init__(self, dict_items, instance, name): - self.instance = weakref.proxy(instance) + self.instance = instance self.name = name super(BaseDict, self).__init__(dict_items) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py new file mode 100644 index 0000000..9192901 --- /dev/null +++ b/mongoengine/dereference.py @@ -0,0 +1,171 @@ +import operator + +import pymongo + +from base import BaseDict, BaseList, get_document +from connection import _get_db +from queryset import QuerySet + + +class DeReference(object): + + def __call__(self, items, max_depth=1, instance=None, name=None, get=False): + """ + Cheaply dereferences the items to a set depth. + Also handles the convertion of complex data types. + + :param items: The iterable (dict, list, queryset) to be dereferenced. + :param max_depth: The maximum depth to recurse to + :param instance: The owning instance used for tracking changes by + :class:`~mongoengine.base.ComplexBaseField` + :param name: The name of the field, used for tracking changes by + :class:`~mongoengine.base.ComplexBaseField` + :param get: A boolean determining if being called by __get__ + """ + if items is None or isinstance(items, basestring): + return items + + # cheapest way to convert a queryset to a list + # list(queryset) uses a count() query to determine length + if isinstance(items, QuerySet): + items = [i for i in items] + + self.max_depth = max_depth + self.reference_map = self._find_references(items) + self.object_map = self._fetch_objects() + return self._attach_objects(items, 0, instance, name, get) + + def _find_references(self, items, depth=0): + """ + Recursively finds all db references to be dereferenced + + :param items: The iterable (dict, list, queryset) + :param depth: The current depth of recursion + """ + reference_map = {} + if not items: + return reference_map + + # Determine the iterator to use + if not hasattr(items, 'items'): + iterator = enumerate(items) + else: + iterator = items.iteritems() + + # Recursively find dbreferences + for k, item in iterator: + if hasattr(item, '_fields'): + for field_name, field in item._fields.iteritems(): + v = item._data.get(field_name, None) + if isinstance(v, (pymongo.dbref.DBRef)): + reference_map.setdefault(field.document_type, []).append(v.id) + elif isinstance(v, (dict, pymongo.son.SON)) and '_ref' in v: + reference_map.setdefault(get_document(v['_cls']), []).append(v['_ref'].id) + elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth: + field_cls = getattr(getattr(field, 'field', None), 'document_type', None) + references = self._find_references(v, depth) + for key, refs in references.iteritems(): + if field_cls: + key = field_cls + reference_map.setdefault(key, []).extend(refs) + elif isinstance(item, (pymongo.dbref.DBRef)): + reference_map.setdefault(item.collection, []).append(item.id) + elif isinstance(item, (dict, pymongo.son.SON)) and '_ref' in item: + reference_map.setdefault(get_document(item['_cls']), []).append(item['_ref'].id) + elif isinstance(item, (dict, list, tuple)) and depth <= self.max_depth: + references = self._find_references(item, depth) + for key, refs in references.iteritems(): + reference_map.setdefault(key, []).extend(refs) + depth += 1 + return reference_map + + def _fetch_objects(self): + """Fetch all references and convert to their document objects + """ + object_map = {} + for col, dbrefs in self.reference_map.iteritems(): + keys = object_map.keys() + refs = list(set([dbref for dbref in dbrefs if str(dbref) not in keys])) + if hasattr(col, 'objects'): # We have a document class for the refs + references = col.objects.in_bulk(refs) + for key, doc in references.iteritems(): + object_map[key] = doc + else: # Generic reference: use the refs data to convert to document + references = _get_db()[col].find({'_id': {'$in': refs}}) + for ref in references: + doc = get_document(ref['_cls'])._from_son(ref) + object_map[doc.id] = doc + return object_map + + def _attach_objects(self, items, depth=0, instance=None, name=None, get=False): + """ + Recursively finds all db references to be dereferenced + + :param items: The iterable (dict, list, queryset) + :param depth: The current depth of recursion + :param instance: The owning instance used for tracking changes by + :class:`~mongoengine.base.ComplexBaseField` + :param name: The name of the field, used for tracking changes by + :class:`~mongoengine.base.ComplexBaseField` + :param get: A boolean determining if being called by __get__ + """ + if not items: + if isinstance(items, (BaseDict, BaseList)): + return items + + if instance: + if isinstance(items, dict): + return BaseDict(items, instance=instance, name=name) + else: + return BaseList(items, instance=instance, name=name) + + if isinstance(items, (dict, pymongo.son.SON)): + if '_ref' in items: + return self.object_map.get(items['_ref'].id, items) + elif '_types' in items and '_cls' in items: + doc = get_document(items['_cls'])._from_son(items) + if not get: + doc._data = self._attach_objects(doc._data, depth, doc, name, get) + return doc + + if not hasattr(items, 'items'): + is_list = True + iterator = enumerate(items) + data = [] + else: + is_list = False + iterator = items.iteritems() + data = {} + + for k, v in iterator: + if is_list: + data.append(v) + else: + data[k] = v + + if k in self.object_map: + data[k] = self.object_map[k] + elif hasattr(v, '_fields'): + for field_name, field in v._fields.iteritems(): + v = data[k]._data.get(field_name, None) + if isinstance(v, (pymongo.dbref.DBRef)): + data[k]._data[field_name] = self.object_map.get(v.id, v) + elif isinstance(v, (dict, pymongo.son.SON)) and '_ref' in v: + data[k]._data[field_name] = self.object_map.get(v['_ref'].id, v) + elif isinstance(v, dict) and depth < self.max_depth: + data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name, get=get) + elif isinstance(v, (list, tuple)): + data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name, get=get) + elif isinstance(v, (dict, list, tuple)) and depth < self.max_depth: + data[k] = self._attach_objects(v, depth, instance=instance, name=name, get=get) + elif hasattr(v, 'id'): + data[k] = self.object_map.get(v.id, v) + + if instance and name: + if is_list: + return BaseList(data, instance=instance, name=name) + return BaseDict(data, instance=instance, name=name) + depth += 1 + return data + +dereference = DeReference() diff --git a/mongoengine/document.py b/mongoengine/document.py index e20500d..31a2530 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -193,6 +193,11 @@ class Document(BaseDocument): signals.post_delete.send(self.__class__, document=self) + def select_related(self, max_depth=1): + from dereference import dereference + self._data = dereference(self._data, max_depth) + return self + def reload(self): """Reloads all attributes from the database. diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 82138fe..6b110ff 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -801,13 +801,7 @@ class QuerySet(object): :param object_id: the value for the id of the document to look up """ - id_field = self._document._meta['id_field'] - object_id = self._document._fields[id_field].to_mongo(object_id) - - result = self._collection.find_one({'_id': object_id}, **self._cursor_args) - if result is not None: - result = self._document._from_son(result) - return result + return self._document.objects(pk=object_id).first() def in_bulk(self, object_ids): """Retrieve a set of documents by their ids. @@ -1530,6 +1524,9 @@ class QuerySet(object): data[-1] = "...(remaining elements truncated)..." return repr(data) + def select_related(self, max_depth=1): + from dereference import dereference + return dereference(self, max_depth=max_depth) class QuerySetManager(object): diff --git a/tests/dereference.py b/tests/dereference.py index 4040d5b..a98267f 100644 --- a/tests/dereference.py +++ b/tests/dereference.py @@ -30,6 +30,9 @@ class FieldTest(unittest.TestCase): group = Group(members=User.objects) group.save() + group = Group(members=User.objects) + group.save() + with query_counter() as q: self.assertEqual(q, 0) @@ -39,6 +42,24 @@ class FieldTest(unittest.TestCase): [m for m in group_obj.members] self.assertEqual(q, 2) + # Document select_related + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first().select_related() + self.assertEqual(q, 2) + [m for m in group_obj.members] + self.assertEqual(q, 2) + + # Queryset select_related + with query_counter() as q: + self.assertEqual(q, 0) + group_objs = Group.objects.select_related() + self.assertEqual(q, 2) + for group_obj in group_objs: + [m for m in group_obj.members] + self.assertEqual(q, 2) + User.drop_collection() Group.drop_collection() @@ -50,6 +71,8 @@ class FieldTest(unittest.TestCase): boss = ReferenceField('self') friends = ListField(ReferenceField('self')) + Employee.drop_collection() + bill = Employee(name='Bill Lumbergh') bill.save() @@ -63,6 +86,10 @@ class FieldTest(unittest.TestCase): peter = Employee(name='Peter Gibbons', boss=bill, friends=friends) peter.save() + Employee(name='Funky Gibbon', boss=bill, friends=friends).save() + Employee(name='Funky Gibbon', boss=bill, friends=friends).save() + Employee(name='Funky Gibbon', boss=bill, friends=friends).save() + with query_counter() as q: self.assertEqual(q, 0) @@ -75,6 +102,33 @@ class FieldTest(unittest.TestCase): peter.friends self.assertEqual(q, 3) + # Document select_related + with query_counter() as q: + self.assertEqual(q, 0) + + peter = Employee.objects.with_id(peter.id).select_related() + self.assertEqual(q, 2) + + self.assertEquals(peter.boss, bill) + self.assertEqual(q, 2) + + self.assertEquals(peter.friends, friends) + self.assertEqual(q, 2) + + # Queryset select_related + with query_counter() as q: + self.assertEqual(q, 0) + + employees = Employee.objects(boss=bill).select_related() + self.assertEqual(q, 2) + + for employee in employees: + self.assertEquals(employee.boss, bill) + self.assertEqual(q, 2) + + self.assertEquals(employee.friends, friends) + self.assertEqual(q, 2) + def test_generic_reference(self): class UserA(Document): @@ -110,6 +164,9 @@ class FieldTest(unittest.TestCase): group = Group(members=members) group.save() + group = Group(members=members) + group.save() + with query_counter() as q: self.assertEqual(q, 0) @@ -125,6 +182,39 @@ class FieldTest(unittest.TestCase): for m in group_obj.members: self.assertTrue('User' in m.__class__.__name__) + # Document select_related + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first().select_related() + self.assertEqual(q, 4) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + for m in group_obj.members: + self.assertTrue('User' in m.__class__.__name__) + + # Queryset select_related + with query_counter() as q: + self.assertEqual(q, 0) + + group_objs = Group.objects.select_related() + self.assertEqual(q, 4) + + for group_obj in group_objs: + [m for m in group_obj.members] + self.assertEqual(q, 4) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + for m in group_obj.members: + self.assertTrue('User' in m.__class__.__name__) + UserA.drop_collection() UserB.drop_collection() UserC.drop_collection() @@ -165,6 +255,9 @@ class FieldTest(unittest.TestCase): group = Group(members=members) group.save() + group = Group(members=members) + group.save() + with query_counter() as q: self.assertEqual(q, 0) @@ -180,6 +273,39 @@ class FieldTest(unittest.TestCase): for m in group_obj.members: self.assertTrue('User' in m.__class__.__name__) + # Document select_related + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first().select_related() + self.assertEqual(q, 4) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + for m in group_obj.members: + self.assertTrue('User' in m.__class__.__name__) + + # Queryset select_related + with query_counter() as q: + self.assertEqual(q, 0) + + group_objs = Group.objects.select_related() + self.assertEqual(q, 4) + + for group_obj in group_objs: + [m for m in group_obj.members] + self.assertEqual(q, 4) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + for m in group_obj.members: + self.assertTrue('User' in m.__class__.__name__) + UserA.drop_collection() UserB.drop_collection() UserC.drop_collection() @@ -205,6 +331,9 @@ class FieldTest(unittest.TestCase): group = Group(members=dict([(str(u.id), u) for u in members])) group.save() + group = Group(members=dict([(str(u.id), u) for u in members])) + group.save() + with query_counter() as q: self.assertEqual(q, 0) @@ -217,6 +346,33 @@ class FieldTest(unittest.TestCase): for k, m in group_obj.members.iteritems(): self.assertTrue(isinstance(m, User)) + # Document select_related + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first().select_related() + self.assertEqual(q, 2) + + [m for m in group_obj.members] + self.assertEqual(q, 2) + + for k, m in group_obj.members.iteritems(): + self.assertTrue(isinstance(m, User)) + + # Queryset select_related + with query_counter() as q: + self.assertEqual(q, 0) + + group_objs = Group.objects.select_related() + self.assertEqual(q, 2) + + for group_obj in group_objs: + [m for m in group_obj.members] + self.assertEqual(q, 2) + + for k, m in group_obj.members.iteritems(): + self.assertTrue(isinstance(m, User)) + User.drop_collection() Group.drop_collection() @@ -254,6 +410,8 @@ class FieldTest(unittest.TestCase): group = Group(members=dict([(str(u.id), u) for u in members])) group.save() + group = Group(members=dict([(str(u.id), u) for u in members])) + group.save() with query_counter() as q: self.assertEqual(q, 0) @@ -270,8 +428,41 @@ class FieldTest(unittest.TestCase): for k, m in group_obj.members.iteritems(): self.assertTrue('User' in m.__class__.__name__) - group.members = {} - group.save() + # Document select_related + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first().select_related() + self.assertEqual(q, 4) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + for k, m in group_obj.members.iteritems(): + self.assertTrue('User' in m.__class__.__name__) + + # Queryset select_related + with query_counter() as q: + self.assertEqual(q, 0) + + group_objs = Group.objects.select_related() + self.assertEqual(q, 4) + + for group_obj in group_objs: + [m for m in group_obj.members] + self.assertEqual(q, 4) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + for k, m in group_obj.members.iteritems(): + self.assertTrue('User' in m.__class__.__name__) + + Group.objects.delete() + Group().save() with query_counter() as q: self.assertEqual(q, 0) @@ -310,6 +501,9 @@ class FieldTest(unittest.TestCase): group = Group(members=dict([(str(u.id), u) for u in members])) group.save() + group = Group(members=dict([(str(u.id), u) for u in members])) + group.save() + with query_counter() as q: self.assertEqual(q, 0) @@ -325,6 +519,39 @@ class FieldTest(unittest.TestCase): for k, m in group_obj.members.iteritems(): self.assertTrue(isinstance(m, UserA)) + # Document select_related + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first().select_related() + self.assertEqual(q, 2) + + [m for m in group_obj.members] + self.assertEqual(q, 2) + + [m for m in group_obj.members] + self.assertEqual(q, 2) + + for k, m in group_obj.members.iteritems(): + self.assertTrue(isinstance(m, UserA)) + + # Queryset select_related + with query_counter() as q: + self.assertEqual(q, 0) + + group_objs = Group.objects.select_related() + self.assertEqual(q, 2) + + for group_obj in group_objs: + [m for m in group_obj.members] + self.assertEqual(q, 2) + + [m for m in group_obj.members] + self.assertEqual(q, 2) + + for k, m in group_obj.members.iteritems(): + self.assertTrue(isinstance(m, UserA)) + UserA.drop_collection() Group.drop_collection() @@ -362,6 +589,8 @@ class FieldTest(unittest.TestCase): group = Group(members=dict([(str(u.id), u) for u in members])) group.save() + group = Group(members=dict([(str(u.id), u) for u in members])) + group.save() with query_counter() as q: self.assertEqual(q, 0) @@ -378,8 +607,41 @@ class FieldTest(unittest.TestCase): for k, m in group_obj.members.iteritems(): self.assertTrue('User' in m.__class__.__name__) - group.members = {} - group.save() + # Document select_related + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first().select_related() + self.assertEqual(q, 4) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + for k, m in group_obj.members.iteritems(): + self.assertTrue('User' in m.__class__.__name__) + + # Queryset select_related + with query_counter() as q: + self.assertEqual(q, 0) + + group_objs = Group.objects.select_related() + self.assertEqual(q, 4) + + for group_obj in group_objs: + [m for m in group_obj.members] + self.assertEqual(q, 4) + + [m for m in group_obj.members] + self.assertEqual(q, 4) + + for k, m in group_obj.members.iteritems(): + self.assertTrue('User' in m.__class__.__name__) + + Group.objects.delete() + Group().save() with query_counter() as q: self.assertEqual(q, 0) @@ -393,4 +655,4 @@ class FieldTest(unittest.TestCase): UserA.drop_collection() UserB.drop_collection() UserC.drop_collection() - Group.drop_collection() \ No newline at end of file + Group.drop_collection() diff --git a/tests/document.py b/tests/document.py index 28d6133..82488cf 100644 --- a/tests/document.py +++ b/tests/document.py @@ -932,7 +932,7 @@ class DocumentTest(unittest.TestCase): list_field = ListField() embedded_field = EmbeddedDocumentField(Embedded) - Doc.drop_collection + Doc.drop_collection() doc = Doc() doc.dict_field = {'hello': 'world'} doc.list_field = ['1', 2, {'hello': 'world'}] @@ -1125,7 +1125,7 @@ class DocumentTest(unittest.TestCase): dict_field = DictField() list_field = ListField() - Doc.drop_collection + Doc.drop_collection() doc = Doc() doc.save() @@ -1180,7 +1180,7 @@ class DocumentTest(unittest.TestCase): list_field = ListField() embedded_field = EmbeddedDocumentField(Embedded) - Doc.drop_collection + Doc.drop_collection() doc = Doc() doc.save() From b039a2293fea59ab3f1f7c25b85755cbb1237e1f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 27 Jun 2011 12:42:26 +0100 Subject: [PATCH 0292/1279] Updated documentation about dereferencing Refs #206 --- docs/changelog.rst | 1 + docs/guide/querying.rst | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4fb5d62..cad1b68 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added select_related() support - Added support for the positional operator - Updated geo index checking to be recursive and check in embedded documents - Updated default collection naming convention diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 4f36e96..b23ea4d 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -175,6 +175,22 @@ to be created:: >>> a.name == b.name and a.age == b.age True +Dereferencing results +--------------------- +When iterating the results of :class:`~mongoengine.ListField` or +:class:`~mongoengine.DictField` we automatically dereference any +:class:`~pymongo.dbref.DBRef` objects as efficiently as possible, reducing the +number the queries to mongo. + +There are times when that efficiency is not enough, documents that have +:class:`~mongoengine.ReferenceField` objects or +:class:`~mongoengine.GenericReferenceField` objects at the top level are +expensive as the number of queries to MongoDB can quickly rise. + +To limit the number of queries use +:func:`~mongoengine.queryset.QuerySet.select_related` which converts the +QuerySet to a list and dereferences as efficiently as possible. + Default Document queries ======================== By default, the objects :attr:`~mongoengine.Document.objects` attribute on a From 4036e9fe3496b62cb2389022bd38e91a20567898 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 27 Jun 2011 13:17:41 +0100 Subject: [PATCH 0293/1279] Moved private method to make class more readable --- mongoengine/fields.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 50a30a1..bac312b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -775,11 +775,6 @@ class GridFSProxy(object): self.gridout = None self._mark_as_changed() - def _mark_as_changed(self): - """Inform the instance that `self.key` has been changed""" - if self.instance: - self.instance._mark_as_changed(self.key) - def replace(self, file_obj, **kwargs): self.delete() self.put(file_obj, **kwargs) @@ -788,6 +783,11 @@ class GridFSProxy(object): if self.newfile: self.newfile.close() + def _mark_as_changed(self): + """Inform the instance that `self.key` has been changed""" + if self.instance: + self.instance._mark_as_changed(self.key) + class FileField(BaseField): """A GridFS storage field. From 84e611b91e94ac32341460dc6cbb46396743cf58 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 27 Jun 2011 16:46:39 +0100 Subject: [PATCH 0294/1279] Tweak to dereferencing --- mongoengine/dereference.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 9192901..6bfabd9 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -2,9 +2,10 @@ import operator import pymongo -from base import BaseDict, BaseList, get_document +from base import BaseDict, BaseList, get_document, TopLevelDocumentMetaclass from connection import _get_db from queryset import QuerySet +from document import Document class DeReference(object): @@ -65,7 +66,7 @@ class DeReference(object): field_cls = getattr(getattr(field, 'field', None), 'document_type', None) references = self._find_references(v, depth) for key, refs in references.iteritems(): - if field_cls: + if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)): key = field_cls reference_map.setdefault(key, []).extend(refs) elif isinstance(item, (pymongo.dbref.DBRef)): From 3d15a3b3e2d362f617a2f1059b091f96dc99a54b Mon Sep 17 00:00:00 2001 From: Zak Johnson Date: Wed, 29 Jun 2011 20:48:39 -0700 Subject: [PATCH 0295/1279] Add GridFSProxy.__nonzero__ For documents that do not have a value set for a given field, most field types return None (or [] in the case of ListField). This makes it easy to test whether a field has been set using "if doc.field". FileFields, on the other hand, always return a GridFSProxy. Adding GridFSProxy.__nonzero__ which simply checks for a grid_id allows the same boolean-test pattern for FileFields, as well. --- mongoengine/fields.py | 5 ++++- tests/fields.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 50a30a1..3e3b5b1 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -723,6 +723,9 @@ class GridFSProxy(object): def __get__(self, instance, value): return self + def __nonzero__(self): + return bool(self.grid_id) + def get(self, id=None): if id: self.grid_id = id @@ -805,7 +808,7 @@ class FileField(BaseField): # Check if a file already exists for this model grid_file = instance._data.get(self.name) self.grid_file = grid_file - if self.grid_file: + if isinstance(self.grid_file, GridFSProxy): if not self.grid_file.key: self.grid_file.key = self.name self.grid_file.instance = instance diff --git a/tests/fields.py b/tests/fields.py index 01280a1..f2543fc 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1300,6 +1300,21 @@ class FieldTest(unittest.TestCase): TestFile.drop_collection() + def test_file_boolean(self): + """Ensure that a boolean test of a FileField indicates its presence + """ + class TestFile(Document): + file = FileField() + + testfile = TestFile() + self.assertFalse(bool(testfile.file)) + testfile.file = 'Hello, World!' + testfile.file.content_type = 'text/plain' + testfile.save() + self.assertTrue(bool(testfile.file)) + + TestFile.drop_collection() + def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """ From 8e1d701c277467f774ac8fb3857811bace29d5f7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 30 Jun 2011 10:32:05 +0100 Subject: [PATCH 0296/1279] Fixed infinite recursion bug in _geo_indices() Fixes #213 Thanks to joshink for the bug report --- mongoengine/base.py | 9 +++++++-- tests/document.py | 12 ++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 8101aa0..b83164a 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -754,11 +754,16 @@ class BaseDocument(object): return set_data, unset_data @classmethod - def _geo_indices(cls): + def _geo_indices(cls, inspected_classes=None): + inspected_classes = inspected_classes or [] geo_indices = [] + inspected_classes.append(cls) for field in cls._fields.values(): if hasattr(field, 'document_type'): - geo_indices += field.document_type._geo_indices() + field_cls = field.document_type + if field_cls in inspected_classes: + continue + geo_indices += field_cls._geo_indices(inspected_classes) elif field._geo_index: geo_indices.append(field) return geo_indices diff --git a/tests/document.py b/tests/document.py index 82488cf..c1abd46 100644 --- a/tests/document.py +++ b/tests/document.py @@ -662,6 +662,18 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + def test_geo_indexes_recursion(self): + + class User(Document): + channel = ReferenceField('Channel') + location = GeoPointField() + + class Channel(Document): + user = ReferenceField('User') + location = GeoPointField() + + self.assertEquals(len(User._geo_indices()), 2) + def test_hint(self): class BlogPost(Document): From 556e620c7a5b4cff3f9b93ce0361b39033e587f9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 1 Jul 2011 08:44:46 +0100 Subject: [PATCH 0297/1279] Fixes recursion error when resetting changed fields Fixes #214 - thanks to wpjunior for the test case --- mongoengine/document.py | 9 ++++++--- tests/document.py | 26 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 31a2530..c653c8f 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -162,15 +162,18 @@ class Document(BaseDocument): id_field = self._meta['id_field'] self[id_field] = self._fields[id_field].to_python(object_id) - def reset_changed_fields(doc): + def reset_changed_fields(doc, inspected_docs=None): """Loop through and reset changed fields lists""" + + inspected_docs = inspected_docs or [] + inspected_docs.append(doc) if hasattr(doc, '_changed_fields'): doc._changed_fields = [] for field_name in doc._fields: field = getattr(doc, field_name) - if hasattr(field, '_changed_fields') and field != doc: - reset_changed_fields(field) + if field not in inspected_docs and hasattr(field, '_changed_fields'): + reset_changed_fields(field, inspected_docs) reset_changed_fields(self) signals.post_save.send(self.__class__, document=self, created=created) diff --git a/tests/document.py b/tests/document.py index c1abd46..9498cfb 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1045,6 +1045,32 @@ class DocumentTest(unittest.TestCase): except ValidationError: self.fail() + def test_save_max_recursion_not_hit(self): + + class Person(Document): + name = StringField() + parent = ReferenceField('self') + friend = ReferenceField('self') + + Person.drop_collection() + + p1 = Person(name="Wilson Jr") + p1.parent = None + p1.save() + + p2 = Person(name="Wilson Jr2") + p2.parent = p1 + p2.save() + + p1.friend = p2 + p1.save() + + # Confirm can save and it resets the changed fields without hitting + # max recursion error + p0 = Person.objects.first() + p0.name = 'wpjunior' + p0.save() + def test_update(self): """Ensure that an existing document is updated instead of be overwritten. """ From 4e6f91ae77baf7cf757daf2dd8f8fe648f25795b Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Sat, 2 Jul 2011 19:48:21 +0400 Subject: [PATCH 0298/1279] Typo fixed in "Quering The Db" guide. --- docs/guide/querying.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index b23ea4d..c454b6e 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -5,8 +5,8 @@ Querying the database is used for accessing the objects in the database associated with the class. The :attr:`objects` attribute is actually a :class:`~mongoengine.queryset.QuerySetManager`, which creates and returns a new -a new :class:`~mongoengine.queryset.QuerySet` object on access. The -:class:`~mongoengine.queryset.QuerySet` object may may be iterated over to +:class:`~mongoengine.queryset.QuerySet` object on access. The +:class:`~mongoengine.queryset.QuerySet` object may be iterated over to fetch documents from the database:: # Prints out the names of all the users in the database @@ -498,4 +498,4 @@ cannot use the `$` syntax in keyword arguments it has been mapped to `S`:: .. note :: Currently only top level lists are handled, future versions of mongodb / pymongo plan to support nested positional operators. See `The $ positional - operator `_. \ No newline at end of file + operator `_. From dc5a613bc7a28148791f8fa42a7cebfe83fe5a5f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 11 Jul 2011 09:19:27 +0100 Subject: [PATCH 0299/1279] Fixes conversion of null genericreferences in querysets closes #211 --- mongoengine/fields.py | 3 +++ tests/fields.py | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index eb0825e..8804011 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -653,6 +653,9 @@ class GenericReferenceField(BaseField): return doc def to_mongo(self, document): + if document is None: + return None + id_field_name = document.__class__._meta['id_field'] id_field = document.__class__._fields[id_field_name] diff --git a/tests/fields.py b/tests/fields.py index f2543fc..7a75299 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1095,6 +1095,18 @@ class FieldTest(unittest.TestCase): Link.drop_collection() User.drop_collection() + def test_generic_reference_is_none(self): + + class Person(Document): + name = StringField() + city = GenericReferenceField() + + Person.drop_collection() + Person(name="Wilson Jr").save() + + self.assertEquals(repr(Person.objects(city=None)), + "[]") + def test_binary_fields(self): """Ensure that binary fields can be stored and retrieved. """ From 803164a993b272dd72860cf5cb5ea1b2464a1aee Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Mon, 11 Jul 2011 08:08:49 -0400 Subject: [PATCH 0300/1279] add unique index on User.username --- mongoengine/django/auth.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 2711ee1..9242490 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -32,6 +32,12 @@ class User(Document): last_login = DateTimeField(default=datetime.datetime.now) date_joined = DateTimeField(default=datetime.datetime.now) + meta = { + 'indexes': [ + {'fields': ['username'], 'unique': True} + ] + } + def __unicode__(self): return self.username From 859de712b493f47fc764e2a648f086e14da920c9 Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Mon, 11 Jul 2011 09:44:28 -0400 Subject: [PATCH 0301/1279] only create indexes on first collection access (fix #223) --- mongoengine/queryset.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6b110ff..a477e37 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -320,10 +320,11 @@ class QuerySet(object): providing :class:`~mongoengine.Document` objects as the results. """ + ALREADY_INDEXED = set() + def __init__(self, document, collection): self._document = document self._collection_obj = collection - self._accessed_collection = False self._mongo_query = None self._query_obj = Q() self._initial_query = {} @@ -467,8 +468,8 @@ class QuerySet(object): """Property that returns the collection object. This allows us to perform operations only if the collection is accessed. """ - if not self._accessed_collection: - self._accessed_collection = True + if self._document not in QuerySet.ALREADY_INDEXED: + QuerySet.ALREADY_INDEXED.add(self._document) background = self._document._meta.get('index_background', False) drop_dups = self._document._meta.get('index_drop_dups', False) From 0847687fd16ca435f5f16f88566a9b325e2f8cc6 Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Mon, 11 Jul 2011 10:15:55 -0400 Subject: [PATCH 0302/1279] don't create extra index on _types (fix #222) mongodb will use an index that begins with _types to service queries against _types, so the extra index is only needed if no other fields are indexed in the document. to be safe, we explicitly check all indexes to see if any begins with _types, and only then prevent creation of the additional index on _types. --- mongoengine/queryset.py | 21 +++++++++++++++++++-- tests/document.py | 11 +++++++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a477e37..69c78b2 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -476,22 +476,39 @@ class QuerySet(object): index_opts = self._document._meta.get('index_options', {}) index_types = self._document._meta.get('index_types', True) + # determine if an index which we are creating includes + # _type as its first field; if so, we can avoid creating + # an extra index on _type, as mongodb will use the existing + # index to service queries against _type + types_indexed = False + def includes_types(fields): + first_field = None + if len(fields): + if isinstance(fields[0], basestring): + first_field = fields[0] + elif isinstance(fields[0], (list, tuple)) and len(fields[0]): + first_field = fields[0][0] + return first_field == '_types' + # Ensure indexes created by uniqueness constraints for index in self._document._meta['unique_indexes']: + types_indexed = types_indexed or includes_types(index) self._collection.ensure_index(index, unique=True, background=background, drop_dups=drop_dups, **index_opts) # Ensure document-defined indexes are created if self._document._meta['indexes']: for spec in self._document._meta['indexes']: + types_indexed = types_indexed or includes_types(spec['fields']) opts = index_opts.copy() opts['unique'] = spec.get('unique', False) opts['sparse'] = spec.get('sparse', False) self._collection.ensure_index(spec['fields'], background=background, **opts) - # If _types is being used (for polymorphism), it needs an index - if index_types and '_types' in self._query: + # If _types is being used (for polymorphism), it needs an index, + # only if another index doesn't begin with _types + if index_types and '_types' in self._query and not types_indexed: self._collection.ensure_index('_types', background=background, **index_opts) diff --git a/tests/document.py b/tests/document.py index 9498cfb..0c056a1 100644 --- a/tests/document.py +++ b/tests/document.py @@ -397,7 +397,7 @@ class DocumentTest(unittest.TestCase): info = collection.index_information() info = [value['key'] for key, value in info.iteritems()] - self.assertEquals([[(u'_id', 1)], [(u'_types', 1)], [(u'_types', 1), (u'name', 1)]], info) + self.assertEquals([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], info) # Turn off inheritance class Animal(Document): @@ -415,7 +415,7 @@ class DocumentTest(unittest.TestCase): info = collection.index_information() info = [value['key'] for key, value in info.iteritems()] - self.assertEquals([[(u'_id', 1)], [(u'_types', 1)], [(u'_types', 1), (u'name', 1)]], info) + self.assertEquals([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], info) info = collection.index_information() indexes_to_drop = [key for key, value in info.iteritems() if '_types' in dict(value['key'])] @@ -601,8 +601,11 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() info = BlogPost.objects._collection.index_information() - # _id, types, '-date', 'tags', ('cat', 'date') - self.assertEqual(len(info), 5) + # _id, '-date', 'tags', ('cat', 'date') + # NB: there is no index on _types by itself, since + # the indices on -date and tags will both contain + # _types as first element in the key + self.assertEqual(len(info), 4) # Indexes are lazy so use list() to perform query list(BlogPost.objects) From 0fb629e24ccb6ce6f2b8c3bf92fd3c239ba5ef11 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 11 Jul 2011 16:01:48 +0100 Subject: [PATCH 0303/1279] Added cascading deletes Also ensured that unsetting works when not the default value of a field --- docs/changelog.rst | 1 + mongoengine/base.py | 19 ++++++++--- mongoengine/document.py | 17 +++++++++- tests/document.py | 70 ++++++++++++++++++++++++++++++++++++----- 4 files changed, 94 insertions(+), 13 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index cad1b68..1b4842e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added cascading saves - so changes to Referenced documents are saved on .save() - Added select_related() support - Added support for the positional operator - Updated geo index checking to be recursive and check in embedded documents diff --git a/mongoengine/base.py b/mongoengine/base.py index b83164a..25b049a 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -747,10 +747,21 @@ class BaseDocument(object): if '_id' in set_data: del(set_data['_id']) - for k,v in set_data.items(): - if not v: - del(set_data[k]) - unset_data[k] = 1 + # Determine if any changed items were actually unset. + for path, value in set_data.items(): + if value: + continue + + # If we've set a value that aint the default value save it. + if path in self._fields: + default = self._fields[path].default + if callable(default): + default = default() + if default != value: + continue + + del(set_data[path]) + unset_data[path] = 1 return set_data, unset_data @classmethod diff --git a/mongoengine/document.py b/mongoengine/document.py index c653c8f..6ccda99 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -112,7 +112,7 @@ class Document(BaseDocument): self._collection = db[collection_name] return self._collection - def save(self, safe=True, force_insert=False, validate=True, write_options=None): + def save(self, safe=True, force_insert=False, validate=True, write_options=None, _refs=None): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be created. @@ -131,6 +131,8 @@ class Document(BaseDocument): For example, ``save(..., w=2, fsync=True)`` will wait until at least two servers have recorded the write and will force an fsync on each server being written to. """ + from fields import ReferenceField, GenericReferenceField + signals.pre_save.send(self.__class__, document=self) if validate: @@ -140,6 +142,7 @@ class Document(BaseDocument): write_options = {} doc = self.to_mongo() + created = '_id' not in doc try: collection = self.__class__.objects._collection @@ -154,6 +157,18 @@ class Document(BaseDocument): collection.update({'_id': object_id}, {"$set": updates}, upsert=True, safe=safe, **write_options) if removals: collection.update({'_id': object_id}, {"$unset": removals}, upsert=True, safe=safe, **write_options) + + # Save any references / generic references + _refs = _refs or [] + for name, cls in self._fields.items(): + if isinstance(cls, (ReferenceField, GenericReferenceField)): + ref = getattr(self, name) + if ref and str(ref) not in _refs: + _refs.append(str(ref)) + ref.save(safe=safe, force_insert=force_insert, + validate=validate, write_options=write_options, + _refs=_refs) + except pymongo.errors.OperationFailure, err: message = 'Could not save document (%s)' if u'duplicate key' in unicode(err): diff --git a/tests/document.py b/tests/document.py index 9498cfb..81670eb 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1054,11 +1054,11 @@ class DocumentTest(unittest.TestCase): Person.drop_collection() - p1 = Person(name="Wilson Jr") + p1 = Person(name="Wilson Snr") p1.parent = None p1.save() - p2 = Person(name="Wilson Jr2") + p2 = Person(name="Wilson Jr") p2.parent = p1 p2.save() @@ -1071,6 +1071,51 @@ class DocumentTest(unittest.TestCase): p0.name = 'wpjunior' p0.save() + def test_save_cascades(self): + + class Person(Document): + name = StringField() + parent = ReferenceField('self') + + Person.drop_collection() + + p1 = Person(name="Wilson Snr") + p1.parent = None + p1.save() + + p2 = Person(name="Wilson Jr") + p2.parent = p1 + p2.save() + + p = Person.objects(name="Wilson Jr").get() + p.parent.name = "Daddy Wilson" + p.save() + + p1.reload() + self.assertEquals(p1.name, p.parent.name) + + def test_save_cascades_generically(self): + + class Person(Document): + name = StringField() + parent = GenericReferenceField() + + Person.drop_collection() + + p1 = Person(name="Wilson Snr") + p1.save() + + p2 = Person(name="Wilson Jr") + p2.parent = p1 + p2.save() + + p = Person.objects(name="Wilson Jr").get() + p.parent.name = "Daddy Wilson" + p.save() + + p1.reload() + self.assertEquals(p1.name, p.parent.name) + def test_update(self): """Ensure that an existing document is updated instead of be overwritten. """ @@ -1364,22 +1409,31 @@ class DocumentTest(unittest.TestCase): """Ensure save only sets / unsets changed fields """ + class User(self.Person): + active = BooleanField(default=True) + + + User.drop_collection() + # Create person object and save it to the database - person = self.Person(name='Test User', age=30) - person.save() - person.reload() + user = User(name='Test User', age=30, active=True) + user.save() + user.reload() + # Simulated Race condition same_person = self.Person.objects.get() + same_person.active = False + + user.age = 21 + user.save() - person.age = 21 same_person.name = 'User' - - person.save() same_person.save() person = self.Person.objects.get() self.assertEquals(person.name, 'User') self.assertEquals(person.age, 21) + self.assertEquals(person.active, False) def test_delete(self): """Ensure that document may be deleted using the delete method. From 1452d3fac5f2500cda0439a45294b9382c8c2d42 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 11 Jul 2011 16:50:31 +0100 Subject: [PATCH 0304/1279] Fixed item_frequency methods to handle null values [fixes #216] --- mongoengine/queryset.py | 13 ++++++++----- tests/queryset.py | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6b110ff..d533736 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1435,7 +1435,7 @@ class QuerySet(object): path = '{{~%(field)s}}'.split('.'); field = this; for (p in path) { field = field[path[p]]; } - if (field.constructor == Array) { + if (field && field.constructor == Array) { field.forEach(function(item) { emit(item, 1); }); @@ -1481,7 +1481,7 @@ class QuerySet(object): db[collection].find(query).forEach(function(doc) { field = doc; for (p in path) { field = field[path[p]]; } - if (field.constructor == Array) { + if (field && field.constructor == Array) { total += field.length; } else { total++; @@ -1497,7 +1497,7 @@ class QuerySet(object): db[collection].find(query).forEach(function(doc) { field = doc; for (p in path) { field = field[path[p]]; } - if (field.constructor == Array) { + if (field && field.constructor == Array) { field.forEach(function(item) { frequencies[item] = inc + (isNaN(frequencies[item]) ? 0: frequencies[item]); }); @@ -1509,8 +1509,11 @@ class QuerySet(object): return frequencies; } """ - - return self.exec_js(freq_func, field, normalize=normalize) + data = self.exec_js(freq_func, field, normalize=normalize) + if 'undefined' in data: + data[None] = data['undefined'] + del(data['undefined']) + return data def __repr__(self): limit = REPR_OUTPUT_SIZE + 1 diff --git a/tests/queryset.py b/tests/queryset.py index c0860b5..e21db0f 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1781,6 +1781,28 @@ class QuerySetTest(unittest.TestCase): test_assertions(exec_js) test_assertions(map_reduce) + def test_item_frequencies_null_values(self): + + class Person(Document): + name = StringField() + city = StringField() + + Person.drop_collection() + + Person(name="Wilson Snr", city="CRB").save() + Person(name="Wilson Jr").save() + + freq = Person.objects.item_frequencies('city') + self.assertEquals(freq, {'CRB': 1.0, None: 1.0}) + freq = Person.objects.item_frequencies('city', normalize=True) + self.assertEquals(freq, {'CRB': 0.5, None: 0.5}) + + + freq = Person.objects.item_frequencies('city', map_reduce=True) + self.assertEquals(freq, {'CRB': 1.0, None: 1.0}) + freq = Person.objects.item_frequencies('city', normalize=True, map_reduce=True) + self.assertEquals(freq, {'CRB': 0.5, None: 0.5}) + def test_average(self): """Ensure that field can be averaged correctly. """ From 2a8d0012136b83de78bff5f897670ad9cd3bc42a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 11 Jul 2011 17:02:23 +0100 Subject: [PATCH 0305/1279] Improvements to indexes and efficiencies Thanks to @dcrosta for the patches closes #225 --- mongoengine/queryset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index d82708c..de80a3d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -320,7 +320,7 @@ class QuerySet(object): providing :class:`~mongoengine.Document` objects as the results. """ - ALREADY_INDEXED = set() + __already_indexed = set() def __init__(self, document, collection): self._document = document @@ -468,8 +468,8 @@ class QuerySet(object): """Property that returns the collection object. This allows us to perform operations only if the collection is accessed. """ - if self._document not in QuerySet.ALREADY_INDEXED: - QuerySet.ALREADY_INDEXED.add(self._document) + if self._document not in QuerySet.__already_indexed: + QuerySet.__already_indexed.add(self._document) background = self._document._meta.get('index_background', False) drop_dups = self._document._meta.get('index_drop_dups', False) From cace665858f78782d6f0aaecf9cbd68d39c2f224 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 12 Jul 2011 10:20:36 +0100 Subject: [PATCH 0306/1279] _delta checking didn't handle db_field_names at all Fixed and added tests, thanks to @wpjunior and @iapain for initial test cases [fixes #226] --- mongoengine/base.py | 11 +- tests/document.py | 274 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 281 insertions(+), 4 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 25b049a..c2f4d21 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -381,6 +381,7 @@ class DocumentMetaclass(type): attr_value.db_field = attr_name doc_fields[attr_name] = attr_value attrs['_fields'] = doc_fields + attrs['_db_field_map'] = dict([(k, v.db_field) for k, v in doc_fields.items()]) new_class = super_new(cls, name, bases, attrs) for field in new_class._fields.values(): @@ -696,6 +697,7 @@ class BaseDocument(object): """ if not key: return + key = self._db_field_map.get(key, key) if hasattr(self, '_changed_fields') and key not in self._changed_fields: self._changed_fields.append(key) @@ -705,13 +707,13 @@ class BaseDocument(object): from mongoengine import EmbeddedDocument _changed_fields = [] _changed_fields += getattr(self, '_changed_fields', []) - for field_name in self._fields: - key = '%s.' % field_name + db_field_name = self._db_field_map.get(field_name, field_name) + key = '%s.' % db_field_name field = getattr(self, field_name, None) - if isinstance(field, EmbeddedDocument) and field_name not in _changed_fields: # Grab all embedded fields that have been changed + if isinstance(field, EmbeddedDocument) and db_field_name not in _changed_fields: # Grab all embedded fields that have been changed _changed_fields += ["%s%s" % (key, k) for k in field._get_changed_fields(key) if k] - elif isinstance(field, (list, tuple)) and field_name not in _changed_fields: # Loop list fields as they contain documents + elif isinstance(field, (list, tuple)) and db_field_name not in _changed_fields: # Loop list fields as they contain documents for index, value in enumerate(field): if not hasattr(value, '_get_changed_fields'): continue @@ -726,6 +728,7 @@ class BaseDocument(object): # Handles cases where not loaded from_son but has _id doc = self.to_mongo() set_fields = self._get_changed_fields() + set_data = {} unset_data = {} if hasattr(self, '_changed_fields'): diff --git a/tests/document.py b/tests/document.py index a816469..df3b4fa 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1203,6 +1203,59 @@ class DocumentTest(unittest.TestCase): self.assertEqual(person.name, None) self.assertEqual(person.age, None) + def test_embedded_update(self): + """ + Test update on `EmbeddedDocumentField` fields + """ + + class Page(EmbeddedDocument): + log_message = StringField(verbose_name="Log message", + required=True) + + class Site(Document): + page = EmbeddedDocumentField(Page) + + + Site.drop_collection() + site = Site(page=Page(log_message="Warning: Dummy message")) + site.save() + + # Update + site = Site.objects.first() + site.page.log_message = "Error: Dummy message" + site.save() + + site = Site.objects.first() + self.assertEqual(site.page.log_message, "Error: Dummy message") + + def test_embedded_update_db_field(self): + """ + Test update on `EmbeddedDocumentField` fields when db_field is other + than default. + """ + + class Page(EmbeddedDocument): + log_message = StringField(verbose_name="Log message", + db_field="page_log_message", + required=True) + + class Site(Document): + page = EmbeddedDocumentField(Page) + + + Site.drop_collection() + + site = Site(page=Page(log_message="Warning: Dummy message")) + site.save() + + # Update + site = Site.objects.first() + site.page.log_message = "Error: Dummy message" + site.save() + + site = Site.objects.first() + self.assertEqual(site.page.log_message, "Error: Dummy message") + def test_delta(self): class Doc(Document): @@ -1408,6 +1461,227 @@ class DocumentTest(unittest.TestCase): del(doc.embedded_field.list_field[2].list_field) self.assertEquals(doc._delta(), ({}, {'embedded_field.list_field.2.list_field': 1})) + def test_delta_db_field(self): + + class Doc(Document): + string_field = StringField(db_field='db_string_field') + int_field = IntField(db_field='db_int_field') + dict_field = DictField(db_field='db_dict_field') + list_field = ListField(db_field='db_list_field') + + Doc.drop_collection() + doc = Doc() + doc.save() + + doc = Doc.objects.first() + self.assertEquals(doc._get_changed_fields(), []) + self.assertEquals(doc._delta(), ({}, {})) + + doc.string_field = 'hello' + self.assertEquals(doc._get_changed_fields(), ['db_string_field']) + self.assertEquals(doc._delta(), ({'db_string_field': 'hello'}, {})) + + doc._changed_fields = [] + doc.int_field = 1 + self.assertEquals(doc._get_changed_fields(), ['db_int_field']) + self.assertEquals(doc._delta(), ({'db_int_field': 1}, {})) + + doc._changed_fields = [] + dict_value = {'hello': 'world', 'ping': 'pong'} + doc.dict_field = dict_value + self.assertEquals(doc._get_changed_fields(), ['db_dict_field']) + self.assertEquals(doc._delta(), ({'db_dict_field': dict_value}, {})) + + doc._changed_fields = [] + list_value = ['1', 2, {'hello': 'world'}] + doc.list_field = list_value + self.assertEquals(doc._get_changed_fields(), ['db_list_field']) + self.assertEquals(doc._delta(), ({'db_list_field': list_value}, {})) + + # Test unsetting + doc._changed_fields = [] + doc.dict_field = {} + self.assertEquals(doc._get_changed_fields(), ['db_dict_field']) + self.assertEquals(doc._delta(), ({}, {'db_dict_field': 1})) + + doc._changed_fields = [] + doc.list_field = [] + self.assertEquals(doc._get_changed_fields(), ['db_list_field']) + self.assertEquals(doc._delta(), ({}, {'db_list_field': 1})) + + # Test it saves that data + doc = Doc() + doc.save() + + doc.string_field = 'hello' + doc.int_field = 1 + doc.dict_field = {'hello': 'world'} + doc.list_field = ['1', 2, {'hello': 'world'}] + doc.save() + doc.reload() + + self.assertEquals(doc.string_field, 'hello') + self.assertEquals(doc.int_field, 1) + self.assertEquals(doc.dict_field, {'hello': 'world'}) + self.assertEquals(doc.list_field, ['1', 2, {'hello': 'world'}]) + + def test_delta_recursive_db_field(self): + + class Embedded(EmbeddedDocument): + string_field = StringField(db_field='db_string_field') + int_field = IntField(db_field='db_int_field') + dict_field = DictField(db_field='db_dict_field') + list_field = ListField(db_field='db_list_field') + + class Doc(Document): + string_field = StringField(db_field='db_string_field') + int_field = IntField(db_field='db_int_field') + dict_field = DictField(db_field='db_dict_field') + list_field = ListField(db_field='db_list_field') + embedded_field = EmbeddedDocumentField(Embedded, db_field='db_embedded_field') + + Doc.drop_collection() + doc = Doc() + doc.save() + + doc = Doc.objects.first() + self.assertEquals(doc._get_changed_fields(), []) + self.assertEquals(doc._delta(), ({}, {})) + + embedded_1 = Embedded() + embedded_1.string_field = 'hello' + embedded_1.int_field = 1 + embedded_1.dict_field = {'hello': 'world'} + embedded_1.list_field = ['1', 2, {'hello': 'world'}] + doc.embedded_field = embedded_1 + + self.assertEquals(doc._get_changed_fields(), ['db_embedded_field']) + + embedded_delta = { + '_types': ['Embedded'], + '_cls': 'Embedded', + 'db_string_field': 'hello', + 'db_int_field': 1, + 'db_dict_field': {'hello': 'world'}, + 'db_list_field': ['1', 2, {'hello': 'world'}] + } + self.assertEquals(doc.embedded_field._delta(), (embedded_delta, {})) + self.assertEquals(doc._delta(), ({'db_embedded_field': embedded_delta}, {})) + + doc.save() + doc.reload() + + doc.embedded_field.dict_field = {} + self.assertEquals(doc._get_changed_fields(), ['db_embedded_field.db_dict_field']) + self.assertEquals(doc.embedded_field._delta(), ({}, {'db_dict_field': 1})) + self.assertEquals(doc._delta(), ({}, {'db_embedded_field.db_dict_field': 1})) + doc.save() + doc.reload() + self.assertEquals(doc.embedded_field.dict_field, {}) + + doc.embedded_field.list_field = [] + self.assertEquals(doc._get_changed_fields(), ['db_embedded_field.db_list_field']) + self.assertEquals(doc.embedded_field._delta(), ({}, {'db_list_field': 1})) + self.assertEquals(doc._delta(), ({}, {'db_embedded_field.db_list_field': 1})) + doc.save() + doc.reload() + self.assertEquals(doc.embedded_field.list_field, []) + + embedded_2 = Embedded() + embedded_2.string_field = 'hello' + embedded_2.int_field = 1 + embedded_2.dict_field = {'hello': 'world'} + embedded_2.list_field = ['1', 2, {'hello': 'world'}] + + doc.embedded_field.list_field = ['1', 2, embedded_2] + self.assertEquals(doc._get_changed_fields(), ['db_embedded_field.db_list_field']) + self.assertEquals(doc.embedded_field._delta(), ({ + 'db_list_field': ['1', 2, { + '_cls': 'Embedded', + '_types': ['Embedded'], + 'db_string_field': 'hello', + 'db_dict_field': {'hello': 'world'}, + 'db_int_field': 1, + 'db_list_field': ['1', 2, {'hello': 'world'}], + }] + }, {})) + + self.assertEquals(doc._delta(), ({ + 'db_embedded_field.db_list_field': ['1', 2, { + '_cls': 'Embedded', + '_types': ['Embedded'], + 'db_string_field': 'hello', + 'db_dict_field': {'hello': 'world'}, + 'db_int_field': 1, + 'db_list_field': ['1', 2, {'hello': 'world'}], + }] + }, {})) + doc.save() + doc.reload() + + self.assertEquals(doc.embedded_field.list_field[0], '1') + self.assertEquals(doc.embedded_field.list_field[1], 2) + for k in doc.embedded_field.list_field[2]._fields: + self.assertEquals(doc.embedded_field.list_field[2][k], embedded_2[k]) + + doc.embedded_field.list_field[2].string_field = 'world' + self.assertEquals(doc._get_changed_fields(), ['db_embedded_field.db_list_field.2.db_string_field']) + self.assertEquals(doc.embedded_field._delta(), ({'db_list_field.2.db_string_field': 'world'}, {})) + self.assertEquals(doc._delta(), ({'db_embedded_field.db_list_field.2.db_string_field': 'world'}, {})) + doc.save() + doc.reload() + self.assertEquals(doc.embedded_field.list_field[2].string_field, 'world') + + # Test multiple assignments + doc.embedded_field.list_field[2].string_field = 'hello world' + doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] + self.assertEquals(doc._get_changed_fields(), ['db_embedded_field.db_list_field']) + self.assertEquals(doc.embedded_field._delta(), ({ + 'db_list_field': ['1', 2, { + '_types': ['Embedded'], + '_cls': 'Embedded', + 'db_string_field': 'hello world', + 'db_int_field': 1, + 'db_list_field': ['1', 2, {'hello': 'world'}], + 'db_dict_field': {'hello': 'world'}}]}, {})) + self.assertEquals(doc._delta(), ({ + 'db_embedded_field.db_list_field': ['1', 2, { + '_types': ['Embedded'], + '_cls': 'Embedded', + 'db_string_field': 'hello world', + 'db_int_field': 1, + 'db_list_field': ['1', 2, {'hello': 'world'}], + 'db_dict_field': {'hello': 'world'}} + ]}, {})) + doc.save() + doc.reload() + self.assertEquals(doc.embedded_field.list_field[2].string_field, 'hello world') + + # Test list native methods + doc.embedded_field.list_field[2].list_field.pop(0) + self.assertEquals(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [2, {'hello': 'world'}]}, {})) + doc.save() + doc.reload() + + doc.embedded_field.list_field[2].list_field.append(1) + self.assertEquals(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [2, {'hello': 'world'}, 1]}, {})) + doc.save() + doc.reload() + self.assertEquals(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) + + doc.embedded_field.list_field[2].list_field.sort() + doc.save() + doc.reload() + self.assertEquals(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) + + del(doc.embedded_field.list_field[2].list_field[2]['hello']) + self.assertEquals(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [1, 2, {}]}, {})) + doc.save() + doc.reload() + + del(doc.embedded_field.list_field[2].list_field) + self.assertEquals(doc._delta(), ({}, {'db_embedded_field.db_list_field.2.db_list_field': 1})) + def test_save_only_changed_fields(self): """Ensure save only sets / unsets changed fields """ From 7f0d3638bae2eb8685e55bafedfa6a11ac1b39b9 Mon Sep 17 00:00:00 2001 From: Leo Honkanen Date: Tue, 12 Jul 2011 16:10:47 +0300 Subject: [PATCH 0307/1279] guard against potentially destructive updates with no update parameters --- mongoengine/queryset.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index de80a3d..d55c5f7 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1268,6 +1268,9 @@ class QuerySet(object): .. versionadded:: 0.2 """ + if not update: + raise OperationError("No update parameters, would remove data") + if pymongo.version < '1.1.1': raise OperationError('update() method requires PyMongo 1.1.1+') @@ -1298,6 +1301,9 @@ class QuerySet(object): .. versionadded:: 0.2 """ + if not update: + raise OperationError("No update parameters, would remove data") + if not write_options: write_options = {} update = QuerySet._transform_update(self._document, **update) From e0799246322bedb4b197978f4d091e8734d383de Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 12 Jul 2011 14:43:21 +0100 Subject: [PATCH 0308/1279] Added extra test for update / update_one [closes #231] --- tests/document.py | 20 -------------------- tests/queryset.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/tests/document.py b/tests/document.py index df3b4fa..92aa3c2 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2087,26 +2087,6 @@ class DocumentTest(unittest.TestCase): pickle_doc.reload() self.assertEquals(resurrected, pickle_doc) - def test_write_options(self): - """Test that passing write_options works""" - - self.Person.drop_collection() - - write_options = {"fsync": True} - - author, created = self.Person.objects.get_or_create( - name='Test User', write_options=write_options) - author.save(write_options=write_options) - - self.Person.objects.update(set__name='Ross', write_options=write_options) - - author = self.Person.objects.first() - self.assertEquals(author.name, 'Ross') - - self.Person.objects.update_one(set__name='Test User', write_options=write_options) - author = self.Person.objects.first() - self.assertEquals(author.name, 'Test User') - if __name__ == '__main__': unittest.main() diff --git a/tests/queryset.py b/tests/queryset.py index e21db0f..a07ff92 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -211,6 +211,42 @@ class QuerySetTest(unittest.TestCase): Blog.drop_collection() + def test_update_write_options(self): + """Test that passing write_options works""" + + self.Person.drop_collection() + + write_options = {"fsync": True} + + author, created = self.Person.objects.get_or_create( + name='Test User', write_options=write_options) + author.save(write_options=write_options) + + self.Person.objects.update(set__name='Ross', write_options=write_options) + + author = self.Person.objects.first() + self.assertEquals(author.name, 'Ross') + + self.Person.objects.update_one(set__name='Test User', write_options=write_options) + author = self.Person.objects.first() + self.assertEquals(author.name, 'Test User') + + def test_update_update_has_a_value(self): + """Test to ensure that update is passed a value to update to""" + self.Person.drop_collection() + + author = self.Person(name='Test User') + author.save() + + def update_raises(): + self.Person.objects(pk=author.pk).update({}) + + def update_one_raises(): + self.Person.objects(pk=author.pk).update({}) + + self.assertRaises(OperationError, update_raises) + self.assertRaises(OperationError, update_one_raises) + def test_update_array_position(self): """Ensure that updating by array position works. From 7a3412dc13a6745247b723d330bfadb2fa10e025 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 13 Jul 2011 09:54:41 +0100 Subject: [PATCH 0309/1279] Added helper for reseting the index cache --- mongoengine/queryset.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index d55c5f7..11c7a80 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -122,7 +122,6 @@ class QueryTreeTransformerVisitor(QNodeVisitor): q_object = reduce(lambda a, b: a & b, and_parts, Q()) q_object = reduce(lambda a, b: a & b, or_group, q_object) clauses.append(q_object) - # Finally, $or the generated clauses in to one query. Each of the # clauses is sufficient for the query to succeed. return reduce(lambda a, b: a | b, clauses, Q()) @@ -431,6 +430,11 @@ class QuerySet(object): return spec + @classmethod + def _reset_already_indexed(cls): + """Helper to reset already indexed, can be useful for testing purposes""" + cls.__already_indexed = set() + def __call__(self, q_obj=None, class_check=True, slave_okay=False, **query): """Filter the selected documents by calling the :class:`~mongoengine.queryset.QuerySet` with a query. From a4c197a83cb366ccb6382538cbea98b6a7082a22 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 13 Jul 2011 14:15:46 +0100 Subject: [PATCH 0310/1279] Added update() convenience method to a document Thanks to @dcrosta for the initial code [closes #229] --- mongoengine/document.py | 12 ++++++++++++ tests/document.py | 23 +++++++++++++++++++++++ tests/queryset.py | 2 +- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 6ccda99..c41303d 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -193,6 +193,18 @@ class Document(BaseDocument): reset_changed_fields(self) signals.post_save.send(self.__class__, document=self, created=created) + def update(self, **kwargs): + """Performs an update on the :class:`~mongoengine.Document` + A convenience wrapper to :meth:`~mongoengine.QuerySet.update`. + + Raises :class:`OperationError` if called on an object that has not yet + been saved. + """ + if not self.pk: + raise OperationError('attempt to update a document not yet saved') + + return self.__class__.objects(pk=self.pk).update_one(**kwargs) + def delete(self, safe=False): """Delete the :class:`~mongoengine.Document` from the database. This will only take effect if the document has been previously saved. diff --git a/tests/document.py b/tests/document.py index 92aa3c2..e1c536e 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1203,6 +1203,29 @@ class DocumentTest(unittest.TestCase): self.assertEqual(person.name, None) self.assertEqual(person.age, None) + def test_document_update(self): + + def update_not_saved_raises(): + person = self.Person(name='dcrosta') + person.update(set__name='Dan Crosta') + + self.assertRaises(OperationError, update_not_saved_raises) + + author = self.Person(name='dcrosta') + author.save() + + author.update(set__name='Dan Crosta') + author.reload() + + p1 = self.Person.objects.first() + self.assertEquals(p1.name, author.name) + + def update_no_value_raises(): + person = self.Person.objects.first() + person.update() + + self.assertRaises(OperationError, update_no_value_raises) + def test_embedded_update(self): """ Test update on `EmbeddedDocumentField` fields diff --git a/tests/queryset.py b/tests/queryset.py index a07ff92..51c9511 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -242,7 +242,7 @@ class QuerySetTest(unittest.TestCase): self.Person.objects(pk=author.pk).update({}) def update_one_raises(): - self.Person.objects(pk=author.pk).update({}) + self.Person.objects(pk=author.pk).update_one({}) self.assertRaises(OperationError, update_raises) self.assertRaises(OperationError, update_one_raises) From 7395ce5b22f6de552069cee9b028530e0f7d2c1b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 13 Jul 2011 16:05:17 +0100 Subject: [PATCH 0311/1279] Updating changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1b4842e..246f917 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added update() convenience method to a document - Added cascading saves - so changes to Referenced documents are saved on .save() - Added select_related() support - Added support for the positional operator From 72995a4b3e32f85ad5298d99d0392566230cf144 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 13 Jul 2011 16:06:40 +0100 Subject: [PATCH 0312/1279] Fixed changing default values to False for embedded items --- mongoengine/base.py | 19 ++++++++++++++++++- tests/document.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index c2f4d21..6b11d23 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -755,9 +755,26 @@ class BaseDocument(object): if value: continue - # If we've set a value that aint the default value save it. + # If we've set a value that ain't the default value unset it. + default = None + if path in self._fields: default = self._fields[path].default + else: # Perform a full lookup for lists / embedded lookups + d = self + parts = path.split('.') + field_name = parts.pop() + for p in parts: + if p.isdigit(): + d = d[int(p)] + elif hasattr(d, '__getattribute__'): + d = getattr(d, p) + else: + d = d.get(p) + if hasattr(d, '_fields'): + default = d._fields[field_name].default + + if default is not None: if callable(default): default = default() if default != value: diff --git a/tests/document.py b/tests/document.py index e1c536e..146681b 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1735,6 +1735,47 @@ class DocumentTest(unittest.TestCase): self.assertEquals(person.age, 21) self.assertEquals(person.active, False) + def test_save_only_changed_fields_recursive(self): + """Ensure save only sets / unsets changed fields + """ + + class Comment(EmbeddedDocument): + published = BooleanField(default=True) + + class User(self.Person): + comments_dict = DictField() + comments = ListField(EmbeddedDocumentField(Comment)) + active = BooleanField(default=True) + + User.drop_collection() + + # Create person object and save it to the database + person = User(name='Test User', age=30, active=True) + person.comments.append(Comment()) + person.save() + person.reload() + + person = self.Person.objects.get() + self.assertTrue(person.comments[0].published) + + person.comments[0].published = False + person.save() + + person = self.Person.objects.get() + self.assertFalse(person.comments[0].published) + + # Simple dict w + person.comments_dict['first_post'] = Comment() + person.save() + + person = self.Person.objects.get() + self.assertTrue(person.comments_dict['first_post'].published) + + person.comments_dict['first_post'].published = False + person.save() + + person = self.Person.objects.get() + self.assertTrue(person.comments_dict['first_post'].published) def test_delete(self): """Ensure that document may be deleted using the delete method. """ From b3ef67a544e2e50859bfe254e9ebc5892bcdcc90 Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Thu, 14 Jul 2011 18:43:11 -0400 Subject: [PATCH 0313/1279] get_document_or_404 raises 404 if given an invalid ObjectId (and possibly on other errors, not sure what else raises ValidationError) --- mongoengine/django/shortcuts.py | 3 ++- tests/django_tests.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mongoengine/django/shortcuts.py b/mongoengine/django/shortcuts.py index 29bc17a..59a2074 100644 --- a/mongoengine/django/shortcuts.py +++ b/mongoengine/django/shortcuts.py @@ -1,6 +1,7 @@ from django.http import Http404 from mongoengine.queryset import QuerySet from mongoengine.base import BaseDocument +from mongoengine.base import ValidationError def _get_queryset(cls): """Inspired by django.shortcuts.*""" @@ -25,7 +26,7 @@ def get_document_or_404(cls, *args, **kwargs): queryset = _get_queryset(cls) try: return queryset.get(*args, **kwargs) - except queryset._document.DoesNotExist: + except (queryset._document.DoesNotExist, ValidationError): raise Http404('No %s matches the given query.' % queryset._document._class_name) def get_list_or_404(cls, *args, **kwargs): diff --git a/tests/django_tests.py b/tests/django_tests.py index 930cc11..9c7e328 100644 --- a/tests/django_tests.py +++ b/tests/django_tests.py @@ -3,7 +3,9 @@ import unittest from mongoengine import * +from mongoengine.django.shortcuts import get_document_or_404 +from django.http import Http404 from django.template import Context, Template from django.conf import settings settings.configure() @@ -56,4 +58,12 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(t.render(Context(d)), 'D-10:C-30:') # Check double rendering doesn't throw an error - self.assertEqual(t.render(Context(d)), 'D-10:C-30:') \ No newline at end of file + self.assertEqual(t.render(Context(d)), 'D-10:C-30:') + + def test_get_document_or_404(self): + p = self.Person(name="G404") + p.save() + + self.assertRaises(Http404, get_document_or_404, self.Person, pk='1234') + self.assertEqual(p, get_document_or_404(self.Person, pk=p.pk)) + From bbd3a6961ef5c731487f9e1a8dd2043f04639ffe Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 18 Jul 2011 08:35:29 +0100 Subject: [PATCH 0314/1279] Fixed typo in tutorial [closes #235] Thanks @mulka --- docs/tutorial.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 63f8fe9..6ce8d10 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -22,7 +22,7 @@ function. The only argument we need to provide is the name of the MongoDB database to use:: from mongoengine import * - + connect('tumblelog') For more information about connecting to MongoDB see :ref:`guide-connecting`. @@ -112,7 +112,7 @@ link table, we can just store a list of tags in each post. So, for both efficiency and simplicity's sake, we'll store the tags as strings directly within the post, rather than storing references to tags in a separate collection. Especially as tags are generally very short (often even shorter -than a document's id), this denormalisation won't impact very strongly on the +than a document's id), this denormalisation won't impact very strongly on the size of our database. So let's take a look that the code our modified :class:`Post` class:: @@ -265,5 +265,5 @@ the first matched by the query you provide. Aggregation functions may also be used on :class:`~mongoengine.queryset.QuerySet` objects:: num_posts = Post.objects(tags='mongodb').count() - print 'Found % posts with tag "mongodb"' % num_posts - + print 'Found %d posts with tag "mongodb"' % num_posts + From fa39789bac7e2e76280f17832e517a8cd378f48d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 18 Jul 2011 12:44:28 -0300 Subject: [PATCH 0315/1279] added SequenceField --- mongoengine/fields.py | 33 ++++++++++++++++++++++++++++++++- tests/fields.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 8804011..a89ec3e 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -20,7 +20,8 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'ObjectIdField', 'ReferenceField', 'ValidationError', 'MapField', 'DecimalField', 'ComplexDateTimeField', 'URLField', 'GenericReferenceField', 'FileField', 'BinaryField', - 'SortedListField', 'EmailField', 'GeoPointField'] + 'SortedListField', 'EmailField', 'GeoPointField', + 'SequenceField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -876,3 +877,33 @@ class GeoPointField(BaseField): if (not isinstance(value[0], (float, int)) and not isinstance(value[1], (float, int))): raise ValidationError('Both values in point must be float or int.') + + +class SequenceField(IntField): + def generate_new_value(self): + """ + Generate and Increment counter + """ + sequence_id = "{0}.{1}".format(self.owner_document._get_collection_name(), + self.name) + collection = _get_db()['mongoengine.counters'] + counter = collection.find_and_modify(query={"_id": sequence_id}, + update={"$inc" : {"next": 1}}, + new=True, + upsert=True) + return counter['next'] + + def __get__(self, instance, owner): + if not instance._data: + return + + if instance is None: + return self + + value = instance._data.get(self.name) + + if not value: + value = self.generate_new_value() + instance._data[self.name] = value + + return value diff --git a/tests/fields.py b/tests/fields.py index 7a75299..2ceda7d 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1380,5 +1380,42 @@ class FieldTest(unittest.TestCase): self.assertEqual(d2.data2, {}) + def test_sequence_field(self): + class Person(Document): + id = SequenceField(primary_key=True) + + self.db['mongoengine.counters'].drop() + Person.drop_collection() + p = Person() + p.save() + + p = Person.objects.first() + self.assertEqual(p.id, 1) + + def test_multiple_sequence_field(self): + class Person(Document): + id = SequenceField(primary_key=True) + name = StringField() + + self.db['mongoengine.counters'].drop() + Person.drop_collection() + + for x in xrange(10): + p = Person(name="Person %s" % x) + p.save() + + ids = [i.id for i in Person.objects] + self.assertEqual(ids, range(1, 11)) + + for x in xrange(10): + p = Person(name="Person %s" % x) + p.save() + + ids = [i.id for i in Person.objects] + self.assertEqual(ids, range(1, 21)) + + counter = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) + self.assertEqual(counter['next'], 20) + if __name__ == '__main__': unittest.main() From cb324595ef67c6a7e826aa738e23a9c37d4f41e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 19 Jul 2011 07:36:35 -0300 Subject: [PATCH 0316/1279] fixerrors --- mongoengine/base.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 6b11d23..04e13cf 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -382,6 +382,7 @@ class DocumentMetaclass(type): doc_fields[attr_name] = attr_value attrs['_fields'] = doc_fields attrs['_db_field_map'] = dict([(k, v.db_field) for k, v in doc_fields.items()]) + attrs['_reverse_db_field_map'] = dict([(v.db_field, k) for k, v in doc_fields.items()]) new_class = super_new(cls, name, bases, attrs) for field in new_class._fields.values(): @@ -763,17 +764,22 @@ class BaseDocument(object): else: # Perform a full lookup for lists / embedded lookups d = self parts = path.split('.') - field_name = parts.pop() + db_field_name = parts.pop() for p in parts: if p.isdigit(): d = d[int(p)] elif hasattr(d, '__getattribute__'): - d = getattr(d, p) + real_path = d._reverse_db_field_map.get(p, p) + d = getattr(d, real_path) else: d = d.get(p) + if hasattr(d, '_fields'): + field_name = d._reverse_db_field_map.get(db_field_name, + db_field_name) + default = d._fields[field_name].default - + if default is not None: if callable(default): default = default() From 5834fa840c867cf2c02fd66b0fcfd884dd2f482a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 19 Jul 2011 16:51:26 +0100 Subject: [PATCH 0317/1279] Tweaked SequenceField so that it doesn't increment on creation. [refs #238] --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/base.py | 9 ++--- mongoengine/fields.py | 36 ++++++++++++++------ tests/fields.py | 79 +++++++++++++++++++++++++++++++++---------- 5 files changed, 95 insertions(+), 31 deletions(-) diff --git a/AUTHORS b/AUTHORS index aecdcaa..b13af2b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -4,3 +4,4 @@ Deepak Thukral Florian Schlachter Steve Challis Ross Lawley +Wilson Júnior diff --git a/docs/changelog.rst b/docs/changelog.rst index 246f917..e2eccee 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added SequenceField - for creating sequential counters - Added update() convenience method to a document - Added cascading saves - so changes to Referenced documents are saved on .save() - Added select_related() support diff --git a/mongoengine/base.py b/mongoengine/base.py index 04e13cf..07f53c3 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -587,7 +587,8 @@ class BaseDocument(object): # Set any get_fieldname_display methods self.__set_field_display() - + # Flag initialised + self._initialised = True signals.post_init.send(self.__class__, document=self) def validate(self): @@ -773,13 +774,13 @@ class BaseDocument(object): d = getattr(d, real_path) else: d = d.get(p) - + if hasattr(d, '_fields'): field_name = d._reverse_db_field_map.get(db_field_name, db_field_name) - + default = d._fields[field_name].default - + if default is not None: if callable(default): default = default() diff --git a/mongoengine/fields.py b/mongoengine/fields.py index a89ec3e..3234160 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -880,30 +880,46 @@ class GeoPointField(BaseField): class SequenceField(IntField): + """Provides a sequental counter. + + ..note:: Although traditional databases often use increasing sequence + numbers for primary keys. In MongoDB, the preferred approach is to + use Object IDs instead. The concept is that in a very large + cluster of machines, it is easier to create an object ID than have + global, uniformly increasing sequence numbers. + + .. versionadded:: 0.5 + """ + def __init__(self, collection_name=None, *args, **kwargs): + self.collection_name = collection_name or 'mongoengine.counters' + return super(SequenceField, self).__init__(*args, **kwargs) + def generate_new_value(self): """ - Generate and Increment counter + Generate and Increment the counter """ sequence_id = "{0}.{1}".format(self.owner_document._get_collection_name(), self.name) - collection = _get_db()['mongoengine.counters'] + collection = _get_db()[self.collection_name] counter = collection.find_and_modify(query={"_id": sequence_id}, - update={"$inc" : {"next": 1}}, + update={"$inc": {"next": 1}}, new=True, upsert=True) return counter['next'] def __get__(self, instance, owner): - if not instance._data: - return - + if instance is None: return self - + if not instance._data: + return value = instance._data.get(self.name) - - if not value: + if not value and instance._initialised: value = self.generate_new_value() instance._data[self.name] = value - + return value + + def to_python(self, value): + if value is None: + value = self.generate_new_value() return value diff --git a/tests/fields.py b/tests/fields.py index 2ceda7d..1f070ae 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1379,20 +1379,7 @@ class FieldTest(unittest.TestCase): self.assertEqual(d2.data, {}) self.assertEqual(d2.data2, {}) - def test_sequence_field(self): - class Person(Document): - id = SequenceField(primary_key=True) - - self.db['mongoengine.counters'].drop() - Person.drop_collection() - p = Person() - p.save() - - p = Person.objects.first() - self.assertEqual(p.id, 1) - - def test_multiple_sequence_field(self): class Person(Document): id = SequenceField(primary_key=True) name = StringField() @@ -1404,18 +1391,76 @@ class FieldTest(unittest.TestCase): p = Person(name="Person %s" % x) p.save() + c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) + self.assertEqual(c['next'], 10) + ids = [i.id for i in Person.objects] self.assertEqual(ids, range(1, 11)) + c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) + self.assertEqual(c['next'], 10) + + def test_multiple_sequence_fields(self): + class Person(Document): + id = SequenceField(primary_key=True) + counter = SequenceField() + name = StringField() + + self.db['mongoengine.counters'].drop() + Person.drop_collection() + for x in xrange(10): p = Person(name="Person %s" % x) p.save() - ids = [i.id for i in Person.objects] - self.assertEqual(ids, range(1, 21)) + c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) + self.assertEqual(c['next'], 10) + + ids = [i.id for i in Person.objects] + self.assertEqual(ids, range(1, 11)) + + counters = [i.counter for i in Person.objects] + self.assertEqual(counters, range(1, 11)) + + c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) + self.assertEqual(c['next'], 10) + + def test_multiple_sequence_fields_on_docs(self): + + class Animal(Document): + id = SequenceField(primary_key=True) + + class Person(Document): + id = SequenceField(primary_key=True) + + self.db['mongoengine.counters'].drop() + Animal.drop_collection() + Person.drop_collection() + + for x in xrange(10): + a = Animal(name="Animal %s" % x) + a.save() + p = Person(name="Person %s" % x) + p.save() + + c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) + self.assertEqual(c['next'], 10) + + c = self.db['mongoengine.counters'].find_one({'_id': 'animal.id'}) + self.assertEqual(c['next'], 10) + + ids = [i.id for i in Person.objects] + self.assertEqual(ids, range(1, 11)) + + id = [i.id for i in Animal.objects] + self.assertEqual(id, range(1, 11)) + + c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) + self.assertEqual(c['next'], 10) + + c = self.db['mongoengine.counters'].find_one({'_id': 'animal.id'}) + self.assertEqual(c['next'], 10) - counter = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) - self.assertEqual(counter['next'], 20) if __name__ == '__main__': unittest.main() From 49764b51dc69b33857efce105d082bd80df2d97a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 19 Jul 2011 14:43:32 -0300 Subject: [PATCH 0318/1279] tweaks for _db_field_map --- mongoengine/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 07f53c3..b88a2b8 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -381,8 +381,8 @@ class DocumentMetaclass(type): attr_value.db_field = attr_name doc_fields[attr_name] = attr_value attrs['_fields'] = doc_fields - attrs['_db_field_map'] = dict([(k, v.db_field) for k, v in doc_fields.items()]) - attrs['_reverse_db_field_map'] = dict([(v.db_field, k) for k, v in doc_fields.items()]) + attrs['_db_field_map'] = dict([(k, v.db_field) for k, v in doc_fields.items() if k!=v]) + attrs['_reverse_db_field_map'] = dict([(v, k) for k, v in attrs['_db_field_map'].items()]) new_class = super_new(cls, name, bases, attrs) for field in new_class._fields.values(): From 273412fda183fc8c516ef681751a5b353de9db55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 19 Jul 2011 14:48:38 -0300 Subject: [PATCH 0319/1279] tweaks for _db_field_map --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index b88a2b8..525b8bc 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -381,7 +381,7 @@ class DocumentMetaclass(type): attr_value.db_field = attr_name doc_fields[attr_name] = attr_value attrs['_fields'] = doc_fields - attrs['_db_field_map'] = dict([(k, v.db_field) for k, v in doc_fields.items() if k!=v]) + attrs['_db_field_map'] = dict([(k, v.db_field) for k, v in doc_fields.items() if k!=v.db_field]) attrs['_reverse_db_field_map'] = dict([(v, k) for k, v in attrs['_db_field_map'].items()]) new_class = super_new(cls, name, bases, attrs) From 0d1804461dd302ceb4a04a5e4e0e0b545b86b3c1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 19 Jul 2011 22:12:09 +0100 Subject: [PATCH 0320/1279] Updated handling setting of object managers and inheritance --- mongoengine/base.py | 13 ++++++++---- tests/queryset.py | 50 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 07f53c3..18ee913 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -447,7 +447,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Subclassed documents inherit collection from superclass for base in bases: if hasattr(base, '_meta'): - if 'collection' in attrs.get('meta', {}) and not base._meta.get('abstract', False): import warnings msg = "Trying to set a collection on a subclass (%s)" % name @@ -465,14 +464,20 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Propagate 'allow_inheritance' if 'allow_inheritance' in base._meta: base_meta['allow_inheritance'] = base._meta['allow_inheritance'] + if 'queryset_class' in base._meta: + base_meta['queryset_class'] = base._meta['queryset_class'] + try: + base_meta['objects'] = base.__getattribute__(base, 'objects') + except AttributeError: + pass meta = { 'abstract': False, 'collection': collection, 'max_documents': None, 'max_size': None, - 'ordering': [], # default ordering applied at runtime - 'indexes': [], # indexes to be ensured at runtime + 'ordering': [], # default ordering applied at runtime + 'indexes': [], # indexes to be ensured at runtime 'id_field': id_field, 'index_background': False, 'index_drop_dups': False, @@ -496,7 +501,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): new_class._meta['collection'] = collection(new_class) # Provide a default queryset unless one has been manually provided - manager = attrs.get('objects', QuerySetManager()) + manager = attrs.get('objects', meta.get('objects', QuerySetManager())) if hasattr(manager, 'queryset_class'): meta['queryset_class'] = manager.queryset_class new_class.objects = manager diff --git a/tests/queryset.py b/tests/queryset.py index 51c9511..a21bae6 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -2318,6 +2318,56 @@ class QuerySetTest(unittest.TestCase): Post.drop_collection() + def test_custom_querysets_inherited(self): + """Ensure that custom QuerySet classes may be used. + """ + + class CustomQuerySet(QuerySet): + def not_empty(self): + return len(self) > 0 + + class Base(Document): + meta = {'abstract': True, 'queryset_class': CustomQuerySet} + + class Post(Base): + pass + + Post.drop_collection() + self.assertTrue(isinstance(Post.objects, CustomQuerySet)) + self.assertFalse(Post.objects.not_empty()) + + Post().save() + self.assertTrue(Post.objects.not_empty()) + + Post.drop_collection() + + def test_custom_querysets_inherited_direct(self): + """Ensure that custom QuerySet classes may be used. + """ + + class CustomQuerySet(QuerySet): + def not_empty(self): + return len(self) > 0 + + class CustomQuerySetManager(QuerySetManager): + queryset_class = CustomQuerySet + + class Base(Document): + meta = {'abstract': True} + objects = CustomQuerySetManager() + + class Post(Base): + pass + + Post.drop_collection() + self.assertTrue(isinstance(Post.objects, CustomQuerySet)) + self.assertFalse(Post.objects.not_empty()) + + Post().save() + self.assertTrue(Post.objects.not_empty()) + + Post.drop_collection() + def test_call_after_limits_set(self): """Ensure that re-filtering after slicing works """ From 72aa191e70a8b5006e30a9b41f59e3108fe124fb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 20 Jul 2011 11:58:13 +0100 Subject: [PATCH 0321/1279] Stop abstract classes being used in the document_registry --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 18ee913..0243387 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -339,7 +339,6 @@ class DocumentMetaclass(type): # Include all fields present in superclasses if hasattr(base, '_fields'): doc_fields.update(base._fields) - class_name.append(base._class_name) # Get superclasses from superclass superclasses[base._class_name] = base superclasses.update(base._superclasses) @@ -351,6 +350,7 @@ class DocumentMetaclass(type): # Ensure that the Document class may be subclassed - # inheritance may be disabled to remove dependency on # additional fields _cls and _types + class_name.append(base._class_name) if base._meta.get('allow_inheritance', True) == False: raise ValueError('Document %s may not be subclassed' % base.__name__) From 13afead9fb43ff6c1150bd21a5084b260116596a Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Wed, 20 Jul 2011 12:41:20 -0400 Subject: [PATCH 0322/1279] add where() method to QuerySet --- mongoengine/queryset.py | 5 +++++ tests/queryset.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 11c7a80..b1185ee 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1397,6 +1397,11 @@ class QuerySet(object): db = _get_db() return db.eval(code, *fields) + def where(self, where_clause): + where_clause = self._sub_js_fields(where_clause) + self._where_clause = where_clause + return self + def sum(self, field): """Sum over the values of the specified field. diff --git a/tests/queryset.py b/tests/queryset.py index a21bae6..ce64a00 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -2502,6 +2502,34 @@ class QuerySetTest(unittest.TestCase): for key, value in info.iteritems()] self.assertTrue(([('_types', 1), ('message', 1)], False, False) in info) + def test_where(self): + """Ensure that where clauses work. + """ + + class IntPair(Document): + fielda = IntField() + fieldb = IntField() + + IntPair.objects._collection.remove() + + a = IntPair(fielda=1, fieldb=1) + b = IntPair(fielda=1, fieldb=2) + c = IntPair(fielda=2, fieldb=1) + a.save() + b.save() + c.save() + + query = IntPair.objects.where('this[~fielda] >= this[~fieldb]') + self.assertEqual('this["fielda"] >= this["fieldb"]', query._where_clause) + results = list(query) + self.assertEqual(2, len(results)) + self.assertTrue(a in results) + self.assertTrue(c in results) + + query = IntPair.objects.where('this[~fielda] == this[~fieldb]') + results = list(query) + self.assertEqual(1, len(results)) + self.assertTrue(a in results) class QTest(unittest.TestCase): From ac72722e57fb376dba9e67391e98d8def5b17d60 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 22 Jul 2011 13:51:11 +0100 Subject: [PATCH 0323/1279] Fixing bug setting a value that equates to false --- mongoengine/base.py | 6 +++--- tests/document.py | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 0243387..909ed6c 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -762,7 +762,7 @@ class BaseDocument(object): if value: continue - # If we've set a value that ain't the default value unset it. + # If we've set a value that ain't the default value dont unset it. default = None if path in self._fields: @@ -789,8 +789,8 @@ class BaseDocument(object): if default is not None: if callable(default): default = default() - if default != value: - continue + if default != value: + continue del(set_data[path]) unset_data[path] = 1 diff --git a/tests/document.py b/tests/document.py index 146681b..5789e20 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1048,6 +1048,26 @@ class DocumentTest(unittest.TestCase): except ValidationError: self.fail() + def test_save_to_a_value_that_equates_to_false(self): + + class Thing(EmbeddedDocument): + count = IntField() + + class User(Document): + thing = EmbeddedDocumentField(Thing) + + User.drop_collection() + + user = User(thing=Thing(count=1)) + user.save() + user.reload() + + user.thing.count = 0 + user.save() + + user.reload() + self.assertEquals(user.thing.count, 0) + def test_save_max_recursion_not_hit(self): class Person(Document): From 130fb9916d21c8fb14ae2a31be7898f529aa549c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 22 Jul 2011 10:19:41 -0300 Subject: [PATCH 0324/1279] fixes for SequenceField --- mongoengine/base.py | 1 + mongoengine/fields.py | 12 ++++++++++++ tests/fields.py | 26 ++++++++++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index 565bf6b..79851da 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -577,6 +577,7 @@ class BaseDocument(object): signals.pre_init.send(self.__class__, document=self, values=values) self._data = {} + self._initialised = False # Assign default values to instance for attr_name, field in self._fields.items(): value = getattr(self, attr_name, None) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 3234160..b2f1e2a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -911,14 +911,26 @@ class SequenceField(IntField): if instance is None: return self + if not instance._data: return + value = instance._data.get(self.name) + if not value and instance._initialised: value = self.generate_new_value() instance._data[self.name] = value + instance._mark_as_changed(self.name) + return value + def __set__(self, instance, value): + + if value is None and instance._initialised: + value = self.generate_new_value() + + return super(SequenceField, self).__set__(instance, value) + def to_python(self, value): if value is None: value = self.generate_new_value() diff --git a/tests/fields.py b/tests/fields.py index 1f070ae..f8aeb86 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1425,6 +1425,32 @@ class FieldTest(unittest.TestCase): c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) + def test_sequence_fields_reload(self): + class Animal(Document): + counter = SequenceField() + type = StringField() + + self.db['mongoengine.counters'].drop() + Animal.drop_collection() + + a = Animal(type="Boi") + a.save() + + self.assertEqual(a.counter, 1) + a.reload() + self.assertEqual(a.counter, 1) + + a.counter = None + self.assertEqual(a.counter, 2) + a.save() + + self.assertEqual(a.counter, 2) + + a = Animal.objects.first() + self.assertEqual(a.counter, 2) + a.reload() + self.assertEqual(a.counter, 2) + def test_multiple_sequence_fields_on_docs(self): class Animal(Document): From 6471c6e133cbca62983b3f785191457732daa3c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Wed, 27 Jul 2011 08:45:15 -0300 Subject: [PATCH 0325/1279] added GenericEmbeddedDocumentField --- mongoengine/fields.py | 28 +++++++++++++++++++++++++++- tests/fields.py | 25 +++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index b2f1e2a..7d57d78 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -21,7 +21,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DecimalField', 'ComplexDateTimeField', 'URLField', 'GenericReferenceField', 'FileField', 'BinaryField', 'SortedListField', 'EmailField', 'GeoPointField', - 'SequenceField'] + 'SequenceField', 'GenericEmbeddedDocumentField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -420,6 +420,32 @@ class EmbeddedDocumentField(BaseField): def prepare_query_value(self, op, value): return self.to_mongo(value) +class GenericEmbeddedDocumentField(BaseField): + def prepare_query_value(self, op, value): + return self.to_mongo(value) + + def to_python(self, value): + if isinstance(value, dict): + doc_cls = get_document(value['_cls']) + value = doc_cls._from_son(value) + + return value + + def validate(self, value): + if not isinstance(value, EmbeddedDocument): + raise ValidationError('Invalid embedded document instance ' + 'provided to an GenericEmbeddedDocumentField') + + value.validate() + + def to_mongo(self, document): + if document is None: + return None + + data = document.to_mongo() + if not '_cls' in data: + data['_cls'] = document._class_name + return data class ListField(ComplexBaseField): """A list field that wraps a standard field, allowing multiple instances diff --git a/tests/fields.py b/tests/fields.py index f8aeb86..960e55c 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1488,5 +1488,30 @@ class FieldTest(unittest.TestCase): self.assertEqual(c['next'], 10) + def test_generic_embedded_document(self): + class Car(EmbeddedDocument): + name = StringField() + + class Dish(EmbeddedDocument): + food = StringField(required=True) + number = IntField() + + class Person(Document): + name = StringField() + like = GenericEmbeddedDocumentField() + + person = Person(name='Test User') + person.like = Car(name='Fiat') + person.save() + + person = Person.objects.first() + self.assertTrue(isinstance(person.like, Car)) + + person.like = Dish(food="arroz", number=15) + person.save() + + person = Person.objects.first() + self.assertTrue(isinstance(person.like, Dish)) + if __name__ == '__main__': unittest.main() From 3f3f93b0fa07d17960d5670d6783b7896612771c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 29 Jul 2011 15:48:29 +0100 Subject: [PATCH 0326/1279] Fixing delta bug for dict fields --- docs/changelog.rst | 2 +- mongoengine/base.py | 13 +++++++++---- tests/document.py | 15 ++++++++++++++- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e2eccee..f3a4b94 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -53,7 +53,7 @@ Changes in dev - Added reverse delete rules - Fixed issue with unset operation - Fixed Q-object bug -- Added ``QuerySet.all_fields`` resets previous .only() and .exlude() +- Added ``QuerySet.all_fields`` resets previous .only() and .exclude() - Added ``QuerySet.exclude`` - Added django style choices - Fixed order and filter issue diff --git a/mongoengine/base.py b/mongoengine/base.py index ea0f98a..e224367 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -721,12 +721,18 @@ class BaseDocument(object): field = getattr(self, field_name, None) if isinstance(field, EmbeddedDocument) and db_field_name not in _changed_fields: # Grab all embedded fields that have been changed _changed_fields += ["%s%s" % (key, k) for k in field._get_changed_fields(key) if k] - elif isinstance(field, (list, tuple)) and db_field_name not in _changed_fields: # Loop list fields as they contain documents - for index, value in enumerate(field): + elif isinstance(field, (list, tuple, dict)) and db_field_name not in _changed_fields: # Loop list / dict fields as they contain documents + # Determine the iterator to use + if not hasattr(field, 'items'): + iterator = enumerate(field) + else: + iterator = field.iteritems() + for index, value in iterator: if not hasattr(value, '_get_changed_fields'): continue list_key = "%s%s." % (key, index) _changed_fields += ["%s%s" % (list_key, k) for k in value._get_changed_fields(list_key) if k] + return _changed_fields def _delta(self): @@ -736,7 +742,6 @@ class BaseDocument(object): # Handles cases where not loaded from_son but has _id doc = self.to_mongo() set_fields = self._get_changed_fields() - set_data = {} unset_data = {} if hasattr(self, '_changed_fields'): @@ -775,7 +780,7 @@ class BaseDocument(object): for p in parts: if p.isdigit(): d = d[int(p)] - elif hasattr(d, '__getattribute__'): + elif hasattr(d, '__getattribute__') and not isinstance(d, dict): real_path = d._reverse_db_field_map.get(p, p) d = getattr(d, real_path) else: diff --git a/tests/document.py b/tests/document.py index 5789e20..1c9b90e 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1504,6 +1504,18 @@ class DocumentTest(unittest.TestCase): del(doc.embedded_field.list_field[2].list_field) self.assertEquals(doc._delta(), ({}, {'embedded_field.list_field.2.list_field': 1})) + doc.save() + doc.reload() + + doc.dict_field['Embedded'] = embedded_1 + doc.save() + doc.reload() + + doc.dict_field['Embedded'].string_field = 'Hello World' + self.assertEquals(doc._get_changed_fields(), ['dict_field.Embedded.string_field']) + self.assertEquals(doc._delta(), ({'dict_field.Embedded.string_field': 'Hello World'}, {})) + + def test_delta_db_field(self): class Doc(Document): @@ -1795,7 +1807,8 @@ class DocumentTest(unittest.TestCase): person.save() person = self.Person.objects.get() - self.assertTrue(person.comments_dict['first_post'].published) + self.assertFalse(person.comments_dict['first_post'].published) + def test_delete(self): """Ensure that document may be deleted using the delete method. """ From 7913ed1841abc7776b5efbb8362da3a76b1c35cf Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sat, 30 Jul 2011 00:52:37 +0300 Subject: [PATCH 0327/1279] Prevent double saving when doing a forced insert. When doing save(force_insert=True) on a document missing an _id field, the document was first getting inserted and then being saved a second time. Also refactatored the code a bit to make the intent (insert/update/delta-update) cleaner, especially since the `created` variable name was so confusing. --- mongoengine/document.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index c41303d..bd2bbda 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -143,13 +143,15 @@ class Document(BaseDocument): doc = self.to_mongo() - created = '_id' not in doc + created = '_id' in doc + creation_mode = force_insert or not created try: collection = self.__class__.objects._collection - if force_insert: - object_id = collection.insert(doc, safe=safe, **write_options) - if created: - object_id = collection.save(doc, safe=safe, **write_options) + if creation_mode: + if force_insert: + object_id = collection.insert(doc, safe=safe, **write_options) + else: + object_id = collection.save(doc, safe=safe, **write_options) else: object_id = doc['_id'] updates, removals = self._delta() @@ -191,7 +193,7 @@ class Document(BaseDocument): reset_changed_fields(field, inspected_docs) reset_changed_fields(self) - signals.post_save.send(self.__class__, document=self, created=created) + signals.post_save.send(self.__class__, document=self, created=creation_mode) def update(self, **kwargs): """Performs an update on the :class:`~mongoengine.Document` From 376ca717fa53bd010a834d1fbd30d535ec018529 Mon Sep 17 00:00:00 2001 From: John Arnfield Date: Sat, 30 Jul 2011 22:01:24 +0100 Subject: [PATCH 0328/1279] Added support for within_polygon for spatial queries --- mongoengine/queryset.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 0af8dea..5ceeea9 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -481,7 +481,7 @@ class QuerySet(object): """ operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', 'all', 'size', 'exists', 'not'] - geo_operators = ['within_distance', 'within_spherical_distance', 'within_box', 'near', 'near_sphere'] + geo_operators = ['within_distance', 'within_spherical_distance', 'within_box', 'within_polygon', 'near', 'near_sphere'] match_operators = ['contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith', 'exact', 'iexact'] @@ -527,6 +527,8 @@ class QuerySet(object): value = {'$within': {'$center': value}} elif op == "within_spherical_distance": value = {'$within': {'$centerSphere': value}} + elif op == "within_polygon": + value = {'$within': {'$polygon': value}} elif op == "near": value = {'$near': value} elif op == "near_sphere": From 63ee4fef1a5c81f453a69f355bcaddc61c5eff6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 5 Aug 2011 11:03:47 -0300 Subject: [PATCH 0329/1279] Translations for django/auth.py --- mongoengine/django/auth.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 9242490..38370cc 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -3,6 +3,7 @@ from mongoengine import * from django.utils.hashcompat import md5_constructor, sha_constructor from django.utils.encoding import smart_str from django.contrib.auth.models import AnonymousUser +from django.utils.translation import ugettext_lazy as _ import datetime @@ -21,16 +22,32 @@ class User(Document): """A User document that aims to mirror most of the API specified by Django at http://docs.djangoproject.com/en/dev/topics/auth/#users """ - username = StringField(max_length=30, required=True) - first_name = StringField(max_length=30) - last_name = StringField(max_length=30) - email = StringField() - password = StringField(max_length=128) - is_staff = BooleanField(default=False) - is_active = BooleanField(default=True) - is_superuser = BooleanField(default=False) - last_login = DateTimeField(default=datetime.datetime.now) - date_joined = DateTimeField(default=datetime.datetime.now) + username = StringField(max_length=30, required=True, + verbose_name=_('username'), + help_text=_("Required. 30 characters or fewer. Letters, numbers and @/./+/-/_ characters")) + + first_name = StringField(max_length=30, + verbose_name=_('first name')) + + last_name = StringField(max_length=30, + verbose_name=_('last name')) + email = EmailField(verbose_name=_('e-mail address')) + password = StringField(max_length=128, + verbose_name=_('password'), + help_text=_("Use '[algo]$[salt]$[hexdigest]' or use the change password form.")) + is_staff = BooleanField(default=False, + verbose_name=_('staff status'), + help_text=_("Designates whether the user can log into this admin site.")) + is_active = BooleanField(default=True, + verbose_name=_('active'), + help_text=_("Designates whether this user should be treated as active. Unselect this instead of deleting accounts.")) + is_superuser = BooleanField(default=False, + verbose_name=_('superuser status'), + help_text=_("Designates that this user has all permissions without explicitly assigning them.")) + last_login = DateTimeField(default=datetime.datetime.now, + verbose_name=_('last login')) + date_joined = DateTimeField(default=datetime.datetime.now, + verbose_name=_('date joined')) meta = { 'indexes': [ From 331f8b8ae7ef31badb0db3ddf4b7e843406ea807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 9 Aug 2011 14:31:26 -0300 Subject: [PATCH 0330/1279] fixes dereference for documents (allow_inheritance = False) --- mongoengine/dereference.py | 18 +++++++++++++++--- tests/document.py | 25 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 6bfabd9..7fe9ba2 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -3,6 +3,7 @@ import operator import pymongo from base import BaseDict, BaseList, get_document, TopLevelDocumentMetaclass +from fields import ReferenceField from connection import _get_db from queryset import QuerySet from document import Document @@ -32,8 +33,16 @@ class DeReference(object): items = [i for i in items] self.max_depth = max_depth + + doc_type = None + if instance and instance._fields: + doc_type = instance._fields[name].field + + if isinstance(doc_type, ReferenceField): + doc_type = doc_type.document_type + self.reference_map = self._find_references(items) - self.object_map = self._fetch_objects() + self.object_map = self._fetch_objects(doc_type=doc_type) return self._attach_objects(items, 0, instance, name, get) def _find_references(self, items, depth=0): @@ -80,7 +89,7 @@ class DeReference(object): depth += 1 return reference_map - def _fetch_objects(self): + def _fetch_objects(self, doc_type=None): """Fetch all references and convert to their document objects """ object_map = {} @@ -94,7 +103,10 @@ class DeReference(object): else: # Generic reference: use the refs data to convert to document references = _get_db()[col].find({'_id': {'$in': refs}}) for ref in references: - doc = get_document(ref['_cls'])._from_son(ref) + if '_cls' in ref: + doc = get_document(ref['_cls'])._from_son(ref) + else: + doc = doc_type._from_son(ref) object_map[doc.id] = doc return object_map diff --git a/tests/document.py b/tests/document.py index 1c9b90e..90a0bc2 100644 --- a/tests/document.py +++ b/tests/document.py @@ -289,6 +289,31 @@ class DocumentTest(unittest.TestCase): Zoo.drop_collection() Animal.drop_collection() + def test_reference_inheritance(self): + class Stats(Document): + created = DateTimeField(default=datetime.now) + + meta = {'allow_inheritance': False} + + class CompareStats(Document): + generated = DateTimeField(default=datetime.now) + stats = ListField(ReferenceField(Stats)) + + Stats.drop_collection() + CompareStats.drop_collection() + + list_stats = [] + + for i in xrange(10): + s = Stats() + s.save() + list_stats.append(s) + + cmp_stats = CompareStats(stats=list_stats) + cmp_stats.save() + + self.assertEqual(list_stats, CompareStats.objects.first().stats) + def test_inheritance(self): """Ensure that document may inherit fields from a superclass document. """ From 4abfcb0188257abb7955efec24e54a4ea3d12e7d Mon Sep 17 00:00:00 2001 From: Gareth Lloyd Date: Mon, 15 Aug 2011 10:01:48 +0100 Subject: [PATCH 0331/1279] check for presence of _geo_indices on field class before referencing --- mongoengine/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index e224367..6be5c3d 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -812,7 +812,8 @@ class BaseDocument(object): field_cls = field.document_type if field_cls in inspected_classes: continue - geo_indices += field_cls._geo_indices(inspected_classes) + if hasattr(field_cls, '_geo_indices'): + geo_indices += field_cls._geo_indices(inspected_classes) elif field._geo_index: geo_indices.append(field) return geo_indices From 81b69648efaf01e64aefe0260aac78cb8147d407 Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Mon, 15 Aug 2011 16:56:42 -0400 Subject: [PATCH 0332/1279] docstring for `where()` --- mongoengine/queryset.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b1185ee..c894514 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1398,6 +1398,10 @@ class QuerySet(object): return db.eval(code, *fields) def where(self, where_clause): + """Filter ``QuerySet`` results with a ``$where`` clause (a Javascript + expression). Performs automatic field name substitution like + :meth:`mongoengine.queryset.Queryset.exec_js`. + """ where_clause = self._sub_js_fields(where_clause) self._where_clause = where_clause return self From 3f301f6b0f5c56b731a37ae82442a5e90315972e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Aug 2011 10:32:21 +0100 Subject: [PATCH 0333/1279] Finishing touches to where implementation - thanks to dcrosta Refs #242 --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/fields.py | 4 ++-- mongoengine/queryset.py | 4 ++++ tests/queryset.py | 13 +++++++++++++ 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index b13af2b..fbf78cf 100644 --- a/AUTHORS +++ b/AUTHORS @@ -5,3 +5,4 @@ Florian Schlachter Steve Challis Ross Lawley Wilson Júnior +Dan Crosta https://github.com/dcrosta diff --git a/docs/changelog.rst b/docs/changelog.rst index f3a4b94..87247d5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added where() - filter to allowing users to specify query expressions as Javascript - Added SequenceField - for creating sequential counters - Added update() convenience method to a document - Added cascading saves - so changes to Referenced documents are saved on .save() diff --git a/mongoengine/fields.py b/mongoengine/fields.py index b2f1e2a..619b8c6 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -620,7 +620,7 @@ class GenericReferenceField(BaseField): """A reference to *any* :class:`~mongoengine.document.Document` subclass that will be automatically dereferenced on access (lazily). - note: Any documents used as a generic reference must be registered in the + ..note :: Any documents used as a generic reference must be registered in the document registry. Importing the model will automatically register it. .. versionadded:: 0.3 @@ -925,7 +925,7 @@ class SequenceField(IntField): return value def __set__(self, instance, value): - + if value is None and instance._initialised: value = self.generate_new_value() diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index c894514..303fcc1 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1401,6 +1401,10 @@ class QuerySet(object): """Filter ``QuerySet`` results with a ``$where`` clause (a Javascript expression). Performs automatic field name substitution like :meth:`mongoengine.queryset.Queryset.exec_js`. + + .. note:: When using this mode of query, the database will call your + function, or evaluate your predicate clause, for each object + in the collection. """ where_clause = self._sub_js_fields(where_clause) self._where_clause = where_clause diff --git a/tests/queryset.py b/tests/queryset.py index ce64a00..6ae1c10 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -2531,6 +2531,19 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(1, len(results)) self.assertTrue(a in results) + query = IntPair.objects.where('function() { return this[~fielda] >= this[~fieldb] }') + self.assertEqual('function() { return this["fielda"] >= this["fieldb"] }', query._where_clause) + results = list(query) + self.assertEqual(2, len(results)) + self.assertTrue(a in results) + self.assertTrue(c in results) + + def invalid_where(): + list(IntPair.objects.where(fielda__gte=3)) + + self.assertRaises(TypeError, invalid_where) + + class QTest(unittest.TestCase): def setUp(self): From 8bdb42827c3433d69757325113b7f14fe22509d9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Aug 2011 11:33:41 +0100 Subject: [PATCH 0334/1279] Updated AUTHORS Thanks to all those that have contributed to MongoEngine --- AUTHORS | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/AUTHORS b/AUTHORS index fbf78cf..ed022c2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,3 +1,5 @@ +The PRIMARY AUTHORS are (and/or have been): + Harry Marr Matt Dennewitz Deepak Thukral @@ -6,3 +8,61 @@ Steve Challis Ross Lawley Wilson Júnior Dan Crosta https://github.com/dcrosta + +CONTRIBUTORS + +Dervived from the git logs, inevitably incomplete but all of whom and others +have submitted patches, reported bugs and generally helped make MongoEngine +that much better: + + * Harry Marr + * Ross Lawley + * blackbrrr + * Florian Schlachter + * Vincent Driessen + * Steve Challis + * flosch + * Deepak Thukral + * Colin Howe + * Wilson Júnior + * Alistair Roche + * Dan Crosta + * Viktor Kerkez + * Stephan Jaekel + * Rached Ben Mustapha + * Greg Turner + * Daniel Hasselrot + * Mircea Pasoi + * Matt Chisholm + * James Punteney + * TimothéePeignier + * Stuart Rackham + * Serge Matveenko + * Matt Dennewitz + * Don Spaulding + * Ales Zoulek + * sshwsfc + * sib + * Samuel Clay + * Nick Vlku + * martin + * Flavio Amieiro + * Анхбаяр Лхагвадорж + * Zak Johnson + * Victor Farazdagi + * vandersonmota + * Theo Julienne + * sp + * Slavi Pantaleev + * Richard Henry + * Nicolas Perriault + * Nick Vlku Jr + * Michael Henson + * Leo Honkanen + * kuno + * Josh Ourisman + * Jaime + * Igor Ivanov + * Gregg Lind + * Gareth Lloyd + * Albert Choi From 5f058434035121335c375d0e4d8b4d7447327144 Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Tue, 16 Aug 2011 08:20:06 -0400 Subject: [PATCH 0335/1279] prefer to use map-reduce to db.eval where possible --- mongoengine/queryset.py | 72 ++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 23 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 303fcc1..a271608 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1416,16 +1416,26 @@ class QuerySet(object): :param field: the field to sum over; use dot-notation to refer to embedded document fields """ - sum_func = """ - function(sumField) { - var total = 0.0; - db[collection].find(query).forEach(function(doc) { - total += (doc[sumField] || 0.0); - }); - return total; + map_func = pymongo.code.Code(""" + function() { + emit(1, this[field] || 0); } - """ - return self.exec_js(sum_func, field) + """, scope={'field': field}) + + reduce_func = pymongo.code.Code(""" + function(key, values) { + var sum = 0; + for (var i in values) { + sum += values[i]; + } + return sum; + } + """) + + for result in self.map_reduce(map_func, reduce_func, output='inline'): + return result.value + else: + return 0 def average(self, field): """Average over the values of the specified field. @@ -1433,22 +1443,38 @@ class QuerySet(object): :param field: the field to average over; use dot-notation to refer to embedded document fields """ - average_func = """ - function(averageField) { - var total = 0.0; - var num = 0; - db[collection].find(query).forEach(function(doc) { - if (doc[averageField] !== undefined) { - total += doc[averageField]; - num += 1; - } - }); - return total / num; + map_func = pymongo.code.Code(""" + function() { + if (this.hasOwnProperty(field)) + emit(1, {t: this[field] || 0, c: 1}); } - """ - return self.exec_js(average_func, field) + """, scope={'field': field}) - def item_frequencies(self, field, normalize=False, map_reduce=False): + reduce_func = pymongo.code.Code(""" + function(key, values) { + var out = {t: 0, c: 0}; + for (var i in values) { + var value = values[i]; + out.t += value.t; + out.c += value.c; + } + return out; + } + """) + + finalize_func = pymongo.code.Code(""" + function(key, value) { + return value.t / value.c; + } + """) + + for result in self.map_reduce(map_func, reduce_func, finalize_f=finalize_func, output='inline'): + return result.value + else: + return 0 + + + def item_frequencies(self, field, normalize=False, map_reduce=True): """Returns a dictionary of all items present in a field across the whole queried set of documents, and their corresponding frequency. This is useful for generating tag clouds, or searching documents. From fd2e40d735126e788c688a296f8bf7bf45e34dc6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Aug 2011 15:24:37 +0100 Subject: [PATCH 0336/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 87247d5..551d3b2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Updated sum / average to use map_reduce as db.eval doesn't work in sharded environments - Added where() - filter to allowing users to specify query expressions as Javascript - Added SequenceField - for creating sequential counters - Added update() convenience method to a document From 2a8543b3b730974376324a27e5e46c33905a86fa Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Aug 2011 15:26:11 +0100 Subject: [PATCH 0337/1279] Updated changelog --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 551d3b2..787b9c9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -30,7 +30,7 @@ Changes in dev - Added insert method for bulk inserts - Added blinker signal support - Added query_counter context manager for tests -- Added optional map_reduce method item_frequencies +- Added map_reduce method item_frequencies and set as default (as db.eval doesn't work in sharded environments) - Added inline_map_reduce option to map_reduce - Updated connection exception so it provides more info on the cause. - Added searching multiple levels deep in ``DictField`` From 3c8cbcfee757334b4e58606dd2e64d2082fa1e7e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Aug 2011 16:50:50 +0100 Subject: [PATCH 0338/1279] Added tests for showing how to set embedded document indexes refs #257 --- tests/document.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/document.py b/tests/document.py index 90a0bc2..6984ef3 100644 --- a/tests/document.py +++ b/tests/document.py @@ -690,6 +690,57 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + def test_embedded_document_index(self): + """Tests settings an index on an embedded document + """ + class Date(EmbeddedDocument): + year = IntField(db_field='yr') + + class BlogPost(Document): + title = StringField() + date = EmbeddedDocumentField(Date) + + meta = { + 'indexes': [ + '-date.year' + ], + } + + BlogPost.drop_collection() + + info = BlogPost.objects._collection.index_information() + self.assertEqual(info.keys(), ['_types_1_date.yr_-1', '_id_']) + BlogPost.drop_collection() + + def test_list_embedded_document_index(self): + """Ensure list embedded documents can be indexed + """ + class Tag(EmbeddedDocument): + name = StringField(db_field='tag') + + class BlogPost(Document): + title = StringField() + tags = ListField(EmbeddedDocumentField(Tag)) + + meta = { + 'indexes': [ + 'tags.name' + ], + } + + BlogPost.drop_collection() + + info = BlogPost.objects._collection.index_information() + # we don't use _types in with list fields by default + self.assertEqual(info.keys(), ['_id_', '_types_1', 'tags.tag_1']) + + post1 = BlogPost(title="Embedded Indexes tests in place", + tags=[Tag(name="about"), Tag(name="time")] + ) + post1.save() + BlogPost.drop_collection() + + def test_geo_indexes_recursion(self): class User(Document): From b76590dc011d69ef591db45b18e92dfe1ea74939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Wed, 17 Aug 2011 09:32:04 -0300 Subject: [PATCH 0339/1279] more tests for embedded lists --- tests/document.py | 52 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/document.py b/tests/document.py index 6984ef3..cfeffb7 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1945,6 +1945,58 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + def test_list_search_by_embedded(self): + class User(Document): + username = StringField(required=True) + + meta = {'allow_inheritance': False} + + class Comment(EmbeddedDocument): + comment = StringField() + user = ReferenceField(User, + required=True) + + meta = {'allow_inheritance': False} + + class Page(Document): + comments = ListField(EmbeddedDocumentField(Comment)) + meta = {'allow_inheritance': False, + 'indexes': [ + {'fields': ['comments.user']} + ]} + + User.drop_collection() + Page.drop_collection() + + u1 = User(username="wilson") + u1.save() + + u2 = User(username="rozza") + u2.save() + + u3 = User(username="hmarr") + u3.save() + + p1 = Page(comments = [Comment(user=u1, comment="Its very good"), + Comment(user=u2, comment="Hello world"), + Comment(user=u3, comment="Ping Pong"), + Comment(user=u1, comment="I like a beer")]) + p1.save() + + p2 = Page(comments = [Comment(user=u1, comment="Its very good"), + Comment(user=u2, comment="Hello world")]) + p2.save() + + p3 = Page(comments = [Comment(user=u3, comment="Its very good")]) + p3.save() + + p4 = Page(comments = [Comment(user=u2, comment="Heavy Metal song")]) + p4.save() + + self.assertEqual([p1, p2], list(Page.objects.filter(comments__user=u1))) + self.assertEqual([p1, p2, p4], list(Page.objects.filter(comments__user=u2))) + self.assertEqual([p1, p3], list(Page.objects.filter(comments__user=u3))) + def test_save_embedded_document(self): """Ensure that a document with an embedded document field may be saved in the database. From 8071b23bff94a9818a4a8187896275dcd631a820 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 17 Aug 2011 14:17:06 +0100 Subject: [PATCH 0340/1279] Updated upgrade.rst --- docs/upgrade.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index f005e2e..ef44b96 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -7,6 +7,15 @@ Upgrading There have been the following backwards incompatibilities from 0.4 to 0.5: +# PyMongo / MongoDB + +map reduce now requires pymongo 1.11+ More methods now use map_reduce as db.eval +is not supported for sharding - the following have been changed: + + * sum + * average + * item_frequencies + #. Default collection naming. Previously it was just lowercase, its now much more pythonic and readable as its From ca3b004921fd9e69ab0a3d3b6b62a435826c26a5 Mon Sep 17 00:00:00 2001 From: John Arnfield Date: Wed, 17 Aug 2011 20:04:38 +0100 Subject: [PATCH 0341/1279] Added tests for polygon queries --- tests/queryset.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 72623b8..778d931 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1355,6 +1355,26 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(events.count(), 1) self.assertEqual(events[0].id, event2.id) + # check that polygon works + polygon = [ + (41.912114,-87.694445), + (41.919395,-87.69084), + (41.927186,-87.681742), + (41.911731,-87.654276), + (41.898061,-87.656164), + ] + events = Event.objects(location__within_polygon=polygon) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0].id, event1.id) + + polygon2 = [ + (54.033586,-1.742249), + (52.792797,-1.225891), + (53.389881,-4.40094) + ] + events = Event.objects(location__within_polygon=polygon2) + self.assertEqual(events.count(), 0) + Event.drop_collection() def test_spherical_geospatial_operators(self): From 88cb8f39638c99860b81cb299df9409d17a91624 Mon Sep 17 00:00:00 2001 From: John Arnfield Date: Wed, 17 Aug 2011 20:14:24 +0100 Subject: [PATCH 0342/1279] left some conflict markers in - oops --- mongoengine/queryset.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 397a153..a715b57 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -617,15 +617,9 @@ class QuerySet(object): """ operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', 'all', 'size', 'exists', 'not'] -<<<<<<< HEAD - geo_operators = ['within_distance', 'within_spherical_distance', 'within_box', 'near', 'near_sphere'] + geo_operators = ['within_distance', 'within_spherical_distance', 'within_box', 'within_polygon' 'near', 'near_sphere'] match_operators = ['contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith', -======= - geo_operators = ['within_distance', 'within_spherical_distance', 'within_box', 'within_polygon', 'near', 'near_sphere'] - match_operators = ['contains', 'icontains', 'startswith', - 'istartswith', 'endswith', 'iendswith', ->>>>>>> master 'exact', 'iexact'] mongo_query = {} From 10bc93dfa64e146e64077e12aa29412139bd436f Mon Sep 17 00:00:00 2001 From: John Arnfield Date: Wed, 17 Aug 2011 20:15:47 +0100 Subject: [PATCH 0343/1279] Commas help too :) --- mongoengine/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a715b57..93b4dec 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -617,7 +617,7 @@ class QuerySet(object): """ operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', 'all', 'size', 'exists', 'not'] - geo_operators = ['within_distance', 'within_spherical_distance', 'within_box', 'within_polygon' 'near', 'near_sphere'] + geo_operators = ['within_distance', 'within_spherical_distance', 'within_box', 'within_polygon', 'near', 'near_sphere'] match_operators = ['contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith', 'exact', 'iexact'] From b037fb3e21e7b9f4c535fe57b231caf692d0c762 Mon Sep 17 00:00:00 2001 From: John Arnfield Date: Wed, 17 Aug 2011 21:23:40 +0100 Subject: [PATCH 0344/1279] Added version check to the polygon test to ensure server version >= 1.9 --- tests/queryset.py | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/tests/queryset.py b/tests/queryset.py index 001d10b..5b0e658 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -7,6 +7,7 @@ from mongoengine.queryset import (QuerySet, QuerySetManager, MultipleObjectsReturned, DoesNotExist, QueryFieldList) from mongoengine import * +from mongoengine.connection import _get_connection from mongoengine.tests import query_counter @@ -14,7 +15,7 @@ class QuerySetTest(unittest.TestCase): def setUp(self): connect(db='mongoenginetest') - + class Person(Document): name = StringField() age = IntField() @@ -2197,25 +2198,30 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(events.count(), 1) self.assertEqual(events[0].id, event2.id) - # check that polygon works - polygon = [ - (41.912114,-87.694445), - (41.919395,-87.69084), - (41.927186,-87.681742), - (41.911731,-87.654276), - (41.898061,-87.656164), - ] - events = Event.objects(location__within_polygon=polygon) - self.assertEqual(events.count(), 1) - self.assertEqual(events[0].id, event1.id) - - polygon2 = [ - (54.033586,-1.742249), - (52.792797,-1.225891), - (53.389881,-4.40094) - ] - events = Event.objects(location__within_polygon=polygon2) - self.assertEqual(events.count(), 0) + # check that polygon works for users who have a server >= 1.9 + server_version = tuple( + _get_connection().server_info()['version'].split('.') + ) + required_version = tuple("1.9.0".split(".")) + if server_version >= required_version: + polygon = [ + (41.912114,-87.694445), + (41.919395,-87.69084), + (41.927186,-87.681742), + (41.911731,-87.654276), + (41.898061,-87.656164), + ] + events = Event.objects(location__within_polygon=polygon) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0].id, event1.id) + + polygon2 = [ + (54.033586,-1.742249), + (52.792797,-1.225891), + (53.389881,-4.40094) + ] + events = Event.objects(location__within_polygon=polygon2) + self.assertEqual(events.count(), 0) Event.drop_collection() From 97ac7e54767375e83d9df713d95da17510200060 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 17 Aug 2011 21:34:35 +0100 Subject: [PATCH 0345/1279] Remove old pymongo version checks Closes #264 --- mongoengine/queryset.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a271608..6e5a471 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1275,9 +1275,6 @@ class QuerySet(object): if not update: raise OperationError("No update parameters, would remove data") - if pymongo.version < '1.1.1': - raise OperationError('update() method requires PyMongo 1.1.1+') - if not write_options: write_options = {} @@ -1314,14 +1311,10 @@ class QuerySet(object): try: # Explicitly provide 'multi=False' to newer versions of PyMongo # as the default may change to 'True' - if pymongo.version >= '1.1.1': - ret = self._collection.update(self._query, update, multi=False, - upsert=upsert, safe=safe_update, - **write_options) - else: - # Older versions of PyMongo don't support 'multi' - ret = self._collection.update(self._query, update, - safe=safe_update) + ret = self._collection.update(self._query, update, multi=False, + upsert=upsert, safe=safe_update, + **write_options) + if ret is not None and 'n' in ret: return ret['n'] except pymongo.errors.OperationFailure, e: From 11621c6f5a0335fb3ab1b230c6db70c534d7b7ee Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 17 Aug 2011 21:38:46 +0100 Subject: [PATCH 0346/1279] Removed keeptemp from map_reduce as 0.5 requires pymongo 1.11 Closes #258 --- mongoengine/queryset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6e5a471..ca7cffb 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -872,7 +872,7 @@ class QuerySet(object): return self.count() def map_reduce(self, map_f, reduce_f, output, finalize_f=None, limit=None, - scope=None, keep_temp=False): + scope=None): """Perform a map/reduce query using the current query spec and ordering. While ``map_reduce`` respects ``QuerySet`` chaining, it must be the last call made, as it does not return a maleable @@ -920,7 +920,7 @@ class QuerySet(object): reduce_f_code = self._sub_js_fields(reduce_f) reduce_f = pymongo.code.Code(reduce_f_code, reduce_f_scope) - mr_args = {'query': self._query, 'keeptemp': keep_temp} + mr_args = {'query': self._query} if finalize_f: finalize_f_scope = {} @@ -937,7 +937,7 @@ class QuerySet(object): if limit: mr_args['limit'] = limit - if output == 'inline' or (not keep_temp and not self._ordering): + if output == 'inline' and not self._ordering: map_reduce_function = 'inline_map_reduce' else: map_reduce_function = 'map_reduce' @@ -1514,7 +1514,7 @@ class QuerySet(object): return total; } """ - values = self.map_reduce(map_func, reduce_func, 'inline', keep_temp=False) + values = self.map_reduce(map_func, reduce_func, 'inline') frequencies = {} for f in values: key = f.key From 10c30f2224cd7aa3fbc9fa252b770665e51800ce Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Wed, 17 Aug 2011 16:42:36 -0400 Subject: [PATCH 0347/1279] remove keep_temp from map_reduce fixes #258 --- mongoengine/queryset.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index ca7cffb..1dfc06a 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -900,6 +900,10 @@ class QuerySet(object): :meth:`~pymongo.collection.Collection.map_reduce` helper requires PyMongo version **>= 1.11**. + .. versionchanged:: 0.5 + - removed ``keep_temp`` keyword argument, which was only relevant + for MongoDB server versions older than 1.7.4 + .. versionadded:: 0.3 """ from document import MapReduceDocument From 91a0e499d9295bad1f52ff9337f4df4dde9a7e04 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 17 Aug 2011 21:48:41 +0100 Subject: [PATCH 0348/1279] Updated changelog and authors Refs #263 --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index ed022c2..b342830 100644 --- a/AUTHORS +++ b/AUTHORS @@ -66,3 +66,4 @@ that much better: * Gregg Lind * Gareth Lloyd * Albert Choi + * John Arnfield diff --git a/docs/changelog.rst b/docs/changelog.rst index 787b9c9..3a2a2c4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added within_polygon support - for those with mongodb 1.9 - Updated sum / average to use map_reduce as db.eval doesn't work in sharded environments - Added where() - filter to allowing users to specify query expressions as Javascript - Added SequenceField - for creating sequential counters From bda716ef9d721248a9f3d5501c1e3e5a2637c442 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 18 Aug 2011 08:30:52 +0100 Subject: [PATCH 0349/1279] Improved update in test case for removing inheritance --- tests/document.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/document.py b/tests/document.py index cfeffb7..d6d7028 100644 --- a/tests/document.py +++ b/tests/document.py @@ -431,7 +431,7 @@ class DocumentTest(unittest.TestCase): 'allow_inheritance': False, 'indexes': ['name'] } - collection.update({}, {"$unset": {"_types": 1, "_cls": 1}}, False, True) + collection.update({}, {"$unset": {"_types": 1, "_cls": 1}}, multi=True) # Confirm extra data is removed obj = collection.find_one() @@ -1948,14 +1948,14 @@ class DocumentTest(unittest.TestCase): def test_list_search_by_embedded(self): class User(Document): username = StringField(required=True) - + meta = {'allow_inheritance': False} - + class Comment(EmbeddedDocument): comment = StringField() user = ReferenceField(User, required=True) - + meta = {'allow_inheritance': False} class Page(Document): From dd49d1d4bbdbef4e68d9776cf07207f58f7884c1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 24 Aug 2011 13:37:20 +0100 Subject: [PATCH 0350/1279] Added choices note to upgrade docs --- docs/upgrade.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index ef44b96..7187adc 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -7,6 +7,12 @@ Upgrading There have been the following backwards incompatibilities from 0.4 to 0.5: +# Choice options: + +Are now expected to be an iterable of tuples, with the first element in each +tuple being the actual value to be stored. The second element is the +human-readable name for the option. + # PyMongo / MongoDB map reduce now requires pymongo 1.11+ More methods now use map_reduce as db.eval From 1631788ab6bfe66779f226f750b9f235c1f6c7ea Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 24 Aug 2011 13:37:39 +0100 Subject: [PATCH 0351/1279] Now Raise an exception if subclasses are missing at querytime. Beats returning None thanks to #aid for mentioning it on IRC --- mongoengine/base.py | 7 +++++-- tests/document.py | 30 +++++++++++++++++++++++++++--- tests/fields.py | 3 ++- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 6be5c3d..8d0c470 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -671,7 +671,6 @@ class BaseDocument(object): # get the class name from the document, falling back to the given # class if unavailable class_name = son.get(u'_cls', cls._class_name) - data = dict((str(key), value) for key, value in son.items()) if '_types' in data: @@ -686,7 +685,11 @@ class BaseDocument(object): if class_name not in subclasses: # Type of document is probably more generic than the class # that has been queried to return this SON - return None + raise NotRegistered(""" + `%s` has not been registered in the document registry. + Importing the document class automatically registers it, + has it been imported? + """.strip() % class_name) cls = subclasses[class_name] present_fields = data.keys() diff --git a/tests/document.py b/tests/document.py index d6d7028..b76b6f9 100644 --- a/tests/document.py +++ b/tests/document.py @@ -12,7 +12,7 @@ import weakref from fixtures import Base, Mixin, PickleEmbedded, PickleTest from mongoengine import * -from mongoengine.base import BaseField +from mongoengine.base import _document_registry, NotRegistered from mongoengine.connection import _get_db @@ -740,7 +740,6 @@ class DocumentTest(unittest.TestCase): post1.save() BlogPost.drop_collection() - def test_geo_indexes_recursion(self): class User(Document): @@ -799,7 +798,6 @@ class DocumentTest(unittest.TestCase): post2 = BlogPost(title='test2', slug='test') self.assertRaises(OperationError, post2.save) - def test_unique_with(self): """Ensure that unique_with constraints are applied to fields. """ @@ -978,6 +976,32 @@ class DocumentTest(unittest.TestCase): User.drop_collection() + + def test_document_not_registered(self): + + class Place(Document): + name = StringField() + + class NicePlace(Place): + pass + + Place.drop_collection() + + Place(name="London").save() + NicePlace(name="Buckingham Palace").save() + + # Mimic Place and NicePlace definitions being in a different file + # and the NicePlace model not being imported in at query time. + @classmethod + def _get_subclasses(cls): + return {} + Place._get_subclasses = _get_subclasses + + def query_without_importing_nice_place(): + print Place.objects.all() + self.assertRaises(NotRegistered, query_without_importing_nice_place) + + def test_creation(self): """Ensure that document may be created using keyword arguments. """ diff --git a/tests/fields.py b/tests/fields.py index f8aeb86..f973490 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1062,6 +1062,7 @@ class FieldTest(unittest.TestCase): Post.drop_collection() User.drop_collection() + def test_generic_reference_document_not_registered(self): """Ensure dereferencing out of the document registry throws a `NotRegistered` error. @@ -1445,7 +1446,7 @@ class FieldTest(unittest.TestCase): a.save() self.assertEqual(a.counter, 2) - + a = Animal.objects.first() self.assertEqual(a.counter, 2) a.reload() From bc9a09f52e5b64695e682b0bcd4997ca8c41f41f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 9 Sep 2011 04:21:32 -0700 Subject: [PATCH 0352/1279] Document updates --- mongoengine/base.py | 5 ++++- mongoengine/document.py | 10 ++++++++++ mongoengine/fields.py | 8 +++++++- mongoengine/queryset.py | 38 ++++++++++++++++++++++++++++++++++---- 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 8d0c470..6a94670 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -43,6 +43,8 @@ def get_document(name): class BaseField(object): """A base class for fields in a MongoDB document. Instances of this class may be added to subclasses of `Document` to define a document's schema. + + .. versionchanged:: 0.5 - added verbose and help text """ # Fields may have _types inserted into indexes by default @@ -156,6 +158,8 @@ class ComplexBaseField(BaseField): Allows for nesting of embedded documents inside complex types. Handles the lazy dereferencing of a queryset by lazily dereferencing all items in a list / dict rather than one at a time. + + .. versionadded:: 0.5 """ field = None @@ -896,7 +900,6 @@ class BaseDocument(object): return not self.__eq__(other) def __hash__(self): - """ For list, dict key """ if self.pk is None: # For new object return super(BaseDocument,self).__hash__() diff --git a/mongoengine/document.py b/mongoengine/document.py index bd2bbda..3ccc4dd 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -130,6 +130,11 @@ class Document(BaseDocument): which will be used as options for the resultant ``getLastError`` command. For example, ``save(..., w=2, fsync=True)`` will wait until at least two servers have recorded the write and will force an fsync on each server being written to. + + .. versionchanged:: 0.5 + In existing documents it only saves changed fields using set / unset + Saves are cascaded and any :class:`~pymongo.dbref.DBRef` objects + that have changes are saved as well. """ from fields import ReferenceField, GenericReferenceField @@ -226,6 +231,11 @@ class Document(BaseDocument): signals.post_delete.send(self.__class__, document=self) def select_related(self, max_depth=1): + """Handles dereferencing of :class:`~pymongo.dbref.DBRef` objects to + a maximum depth in order to cut down the number queries to mongodb. + + .. versionadded:: 0.5 + """ from dereference import dereference self._data = dereference(self._data, max_depth) return self diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 619b8c6..0514f1f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -279,7 +279,6 @@ class DateTimeField(BaseField): return None - class ComplexDateTimeField(StringField): """ ComplexDateTimeField handles microseconds exactly instead of rounding @@ -295,6 +294,8 @@ class ComplexDateTimeField(StringField): Where NNNNNN is the number of microseconds of the represented `datetime`. The `,` as the separator can be easily modified by passing the `separator` keyword when initializing the field. + + .. versionadded:: 0.5 """ def __init__(self, separator=',', **kwargs): @@ -478,6 +479,7 @@ class DictField(ComplexBaseField): similar to an embedded document, but the structure is not defined. .. versionadded:: 0.3 + .. versionchanged:: 0.5 - Can now handle complex / varying types of data """ def __init__(self, basecls=None, field=None, *args, **kwargs): @@ -542,6 +544,8 @@ class ReferenceField(BaseField): * NULLIFY - Updates the reference to null. * CASCADE - Deletes the documents associated with the reference. * DENY - Prevent the deletion of the reference object. + + .. versionchanged:: 0.5 added `reverse_delete_rule` """ def __init__(self, document_type, reverse_delete_rule=DO_NOTHING, **kwargs): @@ -708,6 +712,7 @@ class GridFSProxy(object): """Proxy object to handle writing and reading of files to and from GridFS .. versionadded:: 0.4 + .. versionchanged:: 0.5 - added optional size param to read """ def __init__(self, grid_id=None, key=None, instance=None): @@ -800,6 +805,7 @@ class FileField(BaseField): """A GridFS storage field. .. versionadded:: 0.4 + .. versionchanged:: 0.5 added optional size param for read """ def __init__(self, **kwargs): diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index bade2b4..a830150 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -346,7 +346,10 @@ class QuerySet(object): self._hint = -1 # Using -1 as None is a valid value for hint def clone(self): - """Creates a copy of the current :class:`~mongoengine.queryset.QuerySet`""" + """Creates a copy of the current :class:`~mongoengine.queryset.QuerySet` + + .. versionadded:: 0.5 + """ c = self.__class__(self._document, self._collection_obj) copy_props = ('_initial_query', '_query_obj', '_where_clause', @@ -996,6 +999,8 @@ class QuerySet(object): Hinting will not do anything if the corresponding index does not exist. The last hint applied to this cursor takes precedence over all others. + + .. versionadded:: 0.5 """ self._cursor.hint(index) self._hint = index @@ -1038,11 +1043,12 @@ class QuerySet(object): def only(self, *fields): """Load only a subset of this document's fields. :: - post = BlogPost.objects(...).only("title") + post = BlogPost.objects(...).only("title", "author.name") :param fields: fields to include .. versionadded:: 0.3 + .. versionchanged:: 0.5 - Added subfield support """ fields = dict([(f, QueryFieldList.ONLY) for f in fields]) return self.fields(**fields) @@ -1053,6 +1059,8 @@ class QuerySet(object): post = BlogPost.objects(...).exclude("comments") :param fields: fields to exclude + + .. versionadded:: 0.5 """ fields = dict([(f, QueryFieldList.EXCLUDE) for f in fields]) return self.fields(**fields) @@ -1098,6 +1106,8 @@ class QuerySet(object): """Include all fields. Reset all previously calls of .only() and .exclude(). :: post = BlogPost.objects(...).exclude("comments").only("title").all_fields() + + .. versionadded:: 0.5 """ self._loaded_fields = QueryFieldList(always_include=self._loaded_fields.always_include) return self @@ -1153,6 +1163,8 @@ class QuerySet(object): """Enable or disable snapshot mode when querying. :param enabled: whether or not snapshot mode is enabled + + ..versionchanged:: 0.5 - made chainable """ self._snapshot = enabled return self @@ -1161,6 +1173,8 @@ class QuerySet(object): """Enable or disable the default mongod timeout when querying. :param enabled: whether or not the timeout is used + + ..versionchanged:: 0.5 - made chainable """ self._timeout = enabled return self @@ -1404,6 +1418,8 @@ class QuerySet(object): .. note:: When using this mode of query, the database will call your function, or evaluate your predicate clause, for each object in the collection. + + .. versionadded:: 0.5 """ where_clause = self._sub_js_fields(where_clause) self._where_clause = where_clause @@ -1414,6 +1430,9 @@ class QuerySet(object): :param field: the field to sum over; use dot-notation to refer to embedded document fields + + .. versionchanged:: 0.5 - updated to map_reduce as db.eval doesnt work + with sharding. """ map_func = pymongo.code.Code(""" function() { @@ -1441,6 +1460,9 @@ class QuerySet(object): :param field: the field to average over; use dot-notation to refer to embedded document fields + + .. versionchanged:: 0.5 - updated to map_reduce as db.eval doesnt work + with sharding. """ map_func = pymongo.code.Code(""" function() { @@ -1472,7 +1494,6 @@ class QuerySet(object): else: return 0 - def item_frequencies(self, field, normalize=False, map_reduce=True): """Returns a dictionary of all items present in a field across the whole queried set of documents, and their corresponding frequency. @@ -1490,6 +1511,9 @@ class QuerySet(object): :param field: the field to use :param normalize: normalize the results so they add to 1.0 :param map_reduce: Use map_reduce over exec_js + + .. versionchanged:: 0.5 defaults to map_reduce and can handle embedded + document lookups """ if map_reduce: return self._item_frequencies_map_reduce(field, normalize=normalize) @@ -1532,7 +1556,7 @@ class QuerySet(object): if normalize: count = sum(frequencies.values()) - frequencies = dict([(k, v/count) for k,v in frequencies.items()]) + frequencies = dict([(k, v / count) for k, v in frequencies.items()]) return frequencies @@ -1594,9 +1618,15 @@ class QuerySet(object): return repr(data) def select_related(self, max_depth=1): + """Handles dereferencing of :class:`~pymongo.dbref.DBRef` objects to + a maximum depth in order to cut down the number queries to mongodb. + + .. versionadded:: 0.5 + """ from dereference import dereference return dereference(self, max_depth=max_depth) + class QuerySetManager(object): get_queryset = None From a6449a7b2c97cf669b887cb04f6cd61911e874c2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 9 Sep 2011 05:45:56 -0700 Subject: [PATCH 0353/1279] Updates to documentation in prep for 0.5 --- docs/guide/defining-documents.rst | 78 +++++++----- docs/guide/document-instances.rst | 10 +- docs/guide/gridfs.rst | 1 + docs/guide/installing.rst | 20 +-- docs/guide/querying.rst | 202 +++++++++++++++++------------- docs/guide/signals.rst | 4 +- docs/index.rst | 39 +++++- docs/upgrade.rst | 27 ++-- mongoengine/queryset.py | 14 ++- 9 files changed, 240 insertions(+), 155 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index a524520..00a7d09 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -4,14 +4,14 @@ Defining documents In MongoDB, a **document** is roughly equivalent to a **row** in an RDBMS. When working with relational databases, rows are stored in **tables**, which have a strict **schema** that the rows follow. MongoDB stores documents in -**collections** rather than tables - the principle difference is that no schema -is enforced at a database level. +**collections** rather than tables - the principle difference is that no schema +is enforced at a database level. Defining a document's schema ============================ MongoEngine allows you to define schemata for documents as this helps to reduce coding errors, and allows for utility methods to be defined on fields which may -be present. +be present. To define a schema for a document, create a class that inherits from :class:`~mongoengine.Document`. Fields are specified by adding **field @@ -19,7 +19,7 @@ objects** as class attributes to the document class:: from mongoengine import * import datetime - + class Page(Document): title = StringField(max_length=200, required=True) date_modified = DateTimeField(default=datetime.datetime.now) @@ -31,31 +31,34 @@ By default, fields are not required. To make a field mandatory, set the validation constraints available (such as :attr:`max_length` in the example above). Fields may also take default values, which will be used if a value is not provided. Default values may optionally be a callable, which will be called -to retrieve the value (such as in the above example). The field types available +to retrieve the value (such as in the above example). The field types available are as follows: * :class:`~mongoengine.StringField` * :class:`~mongoengine.URLField` +* :class:`~mongoengine.EmailField` * :class:`~mongoengine.IntField` * :class:`~mongoengine.FloatField` * :class:`~mongoengine.DecimalField` * :class:`~mongoengine.DateTimeField` +* :class:`~mongoengine.ComplexDateTimeField` * :class:`~mongoengine.ListField` +* :class:`~mongoengine.SortedListField` * :class:`~mongoengine.DictField` +* :class:`~mongoengine.MapField` * :class:`~mongoengine.ObjectIdField` -* :class:`~mongoengine.EmbeddedDocumentField` * :class:`~mongoengine.ReferenceField` * :class:`~mongoengine.GenericReferenceField` +* :class:`~mongoengine.EmbeddedDocumentField` * :class:`~mongoengine.BooleanField` * :class:`~mongoengine.FileField` -* :class:`~mongoengine.EmailField` -* :class:`~mongoengine.SortedListField` * :class:`~mongoengine.BinaryField` * :class:`~mongoengine.GeoPointField` +* :class:`~mongoengine.SequenceField` Field arguments --------------- -Each field type can be customized by keyword arguments. The following keyword +Each field type can be customized by keyword arguments. The following keyword arguments can be set on all fields: :attr:`db_field` (Default: None) @@ -74,7 +77,7 @@ arguments can be set on all fields: The definion of default parameters follow `the general rules on Python `__, - which means that some care should be taken when dealing with default mutable objects + which means that some care should be taken when dealing with default mutable objects (like in :class:`~mongoengine.ListField` or :class:`~mongoengine.DictField`):: class ExampleFirst(Document): @@ -89,7 +92,7 @@ arguments can be set on all fields: # This can make an .append call to add values to the default (and all the following objects), # instead to just an object values = ListField(IntField(), default=[1,2,3]) - + :attr:`unique` (Default: False) When True, no documents in the collection will have the same value for this @@ -104,7 +107,13 @@ arguments can be set on all fields: :attr:`choices` (Default: None) An iterable of choices to which the value of this field should be limited. - + +:attr:`help_text` (Default: None) + Optional help text to output with the field - used by form libraries + +:attr:`verbose` (Default: None) + Optional human-readable name for the field - used by form libraries + List fields ----------- @@ -121,7 +130,7 @@ Embedded documents MongoDB has the ability to embed documents within other documents. Schemata may be defined for these embedded documents, just as they may be for regular documents. To create an embedded document, just define a document as usual, but -inherit from :class:`~mongoengine.EmbeddedDocument` rather than +inherit from :class:`~mongoengine.EmbeddedDocument` rather than :class:`~mongoengine.Document`:: class Comment(EmbeddedDocument): @@ -144,7 +153,7 @@ Often, an embedded document may be used instead of a dictionary -- generally this is recommended as dictionaries don't support validation or custom field types. However, sometimes you will not know the structure of what you want to store; in this situation a :class:`~mongoengine.DictField` is appropriate:: - + class SurveyResponse(Document): date = DateTimeField() user = ReferenceField(User) @@ -152,16 +161,19 @@ store; in this situation a :class:`~mongoengine.DictField` is appropriate:: survey_response = SurveyResponse(date=datetime.now(), user=request.user) response_form = ResponseForm(request.POST) - survey_response.answers = response_form.cleaned_data() + survey_response.answers = response_form.cleaned_data() survey_response.save() +Dictionaries can store complex data, other dictionaries, lists, references to +other objects, so are the most flexible field type available. + Reference fields ---------------- References may be stored to other documents in the database using the :class:`~mongoengine.ReferenceField`. Pass in another document class as the first argument to the constructor, then simply assign document objects to the field:: - + class User(Document): name = StringField() @@ -235,13 +247,13 @@ Its value can take any of the following constants: in-memory, by the MongoEngine module, it is of the upmost importance that the module that declares the relationship is loaded **BEFORE** the delete is invoked. - + If, for example, the :class:`Employee` object lives in the :mod:`payroll` app, and the :class:`ProfilePage` in the :mod:`people` app, it is extremely important that the :mod:`people` app is loaded before any employee is removed, because otherwise, MongoEngine could never know this relationship exists. - + In Django, be sure to put all apps that have such delete rule declarations in their :file:`models.py` in the :const:`INSTALLED_APPS` tuple. @@ -250,15 +262,15 @@ Generic reference fields '''''''''''''''''''''''' A second kind of reference field also exists, :class:`~mongoengine.GenericReferenceField`. This allows you to reference any -kind of :class:`~mongoengine.Document`, and hence doesn't take a +kind of :class:`~mongoengine.Document`, and hence doesn't take a :class:`~mongoengine.Document` subclass as a constructor argument:: class Link(Document): url = StringField() - + class Post(Document): title = StringField() - + class Bookmark(Document): bookmark_object = GenericReferenceField() @@ -272,9 +284,10 @@ kind of :class:`~mongoengine.Document`, and hence doesn't take a Bookmark(bookmark_object=post).save() .. note:: + Using :class:`~mongoengine.GenericReferenceField`\ s is slightly less efficient than the standard :class:`~mongoengine.ReferenceField`\ s, so if - you will only be referencing one document type, prefer the standard + you will only be referencing one document type, prefer the standard :class:`~mongoengine.ReferenceField`. Uniqueness constraints @@ -282,7 +295,7 @@ Uniqueness constraints MongoEngine allows you to specify that a field should be unique across a collection by providing ``unique=True`` to a :class:`~mongoengine.Field`\ 's constructor. If you try to save a document that has the same value for a unique -field as a document that is already in the database, a +field as a document that is already in the database, a :class:`~mongoengine.OperationError` will be raised. You may also specify multi-field uniqueness constraints by using :attr:`unique_with`, which may be either a single field name, or a list or tuple of field names:: @@ -294,14 +307,14 @@ either a single field name, or a list or tuple of field names:: Skipping Document validation on save ------------------------------------ -You can also skip the whole document validation process by setting -``validate=False`` when caling the :meth:`~mongoengine.document.Document.save` +You can also skip the whole document validation process by setting +``validate=False`` when caling the :meth:`~mongoengine.document.Document.save` method:: class Recipient(Document): name = StringField() email = EmailField() - + recipient = Recipient(name='admin', email='root@localhost') recipient.save() # will raise a ValidationError while recipient.save(validate=False) # won't @@ -329,7 +342,7 @@ A :class:`~mongoengine.Document` may use a **Capped Collection** by specifying stored in the collection, and :attr:`max_size` is the maximum size of the collection in bytes. If :attr:`max_size` is not specified and :attr:`max_documents` is, :attr:`max_size` defaults to 10000000 bytes (10MB). -The following example shows a :class:`Log` document that will be limited to +The following example shows a :class:`Log` document that will be limited to 1000 entries and 2MB of disk space:: class Log(Document): @@ -369,9 +382,10 @@ If a dictionary is passed then the following options are available: Whether the index should be sparse. .. note:: - Geospatial indexes will be automatically created for all + + Geospatial indexes will be automatically created for all :class:`~mongoengine.GeoPointField`\ s - + Ordering ======== A default ordering can be specified for your @@ -393,7 +407,7 @@ subsequent calls to :meth:`~mongoengine.queryset.QuerySet.order_by`. :: blog_post_1 = BlogPost(title="Blog Post #1") blog_post_1.published_date = datetime(2010, 1, 5, 0, 0 ,0) - blog_post_2 = BlogPost(title="Blog Post #2") + blog_post_2 = BlogPost(title="Blog Post #2") blog_post_2.published_date = datetime(2010, 1, 6, 0, 0 ,0) blog_post_3 = BlogPost(title="Blog Post #3") @@ -405,7 +419,7 @@ subsequent calls to :meth:`~mongoengine.queryset.QuerySet.order_by`. :: # get the "first" BlogPost using default ordering # from BlogPost.meta.ordering - latest_post = BlogPost.objects.first() + latest_post = BlogPost.objects.first() assert latest_post.title == "Blog Post #3" # override default ordering, order BlogPosts by "published_date" @@ -434,7 +448,7 @@ Working with existing data To enable correct retrieval of documents involved in this kind of heirarchy, two extra attributes are stored on each document in the database: :attr:`_cls` and :attr:`_types`. These are hidden from the user through the MongoEngine -interface, but may not be present if you are trying to use MongoEngine with +interface, but may not be present if you are trying to use MongoEngine with an existing database. For this reason, you may disable this inheritance mechansim, removing the dependency of :attr:`_cls` and :attr:`_types`, enabling you to work with existing databases. To disable inheritance on a document diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index aeed7cd..317bfef 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -4,12 +4,12 @@ Documents instances To create a new document object, create an instance of the relevant document class, providing values for its fields as its constructor keyword arguments. You may provide values for any of the fields on the document:: - + >>> page = Page(title="Test Page") >>> page.title 'Test Page' -You may also assign values to the document's fields using standard object +You may also assign values to the document's fields using standard object attribute syntax:: >>> page.title = "Example Page" @@ -18,9 +18,9 @@ attribute syntax:: Saving and deleting documents ============================= -MongoEngine tracks changes to documents to provide efficient saving. To save +MongoEngine tracks changes to documents to provide efficient saving. To save the document to the database, call the :meth:`~mongoengine.Document.save` method. -If the document does not exist in the database, it will be created. If it does +If the document does not exist in the database, it will be created. If it does already exist, then any changes will be updated atomically. For example:: >>> page = Page(title="Test Page") @@ -29,6 +29,7 @@ already exist, then any changes will be updated atomically. For example:: >>> page.save() # Performs an atomic set on the title field. .. note:: + Changes to documents are tracked and on the whole perform `set` operations. * ``list_field.pop(0)`` - *sets* the resulting list @@ -78,6 +79,7 @@ is an alias to :attr:`id`:: >>> page.id == page.pk .. note:: + If you define your own primary key field, the field implicitly becomes required, so a :class:`ValidationError` will be thrown if you don't provide it. diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst index 0cd0653..3abad77 100644 --- a/docs/guide/gridfs.rst +++ b/docs/guide/gridfs.rst @@ -66,6 +66,7 @@ Deleting stored files is achieved with the :func:`delete` method:: marmot.photo.delete() .. note:: + The FileField in a Document actually only stores the ID of a file in a separate GridFS collection. This means that deleting a document with a defined FileField does not actually delete the file. You must be diff --git a/docs/guide/installing.rst b/docs/guide/installing.rst index 132f107..f15d3db 100644 --- a/docs/guide/installing.rst +++ b/docs/guide/installing.rst @@ -1,31 +1,31 @@ ====================== Installing MongoEngine ====================== + To use MongoEngine, you will need to download `MongoDB `_ and ensure it is running in an accessible location. You will also need `PyMongo `_ to use MongoEngine, but if you install MongoEngine using setuptools, then the dependencies will be handled for you. -MongoEngine is available on PyPI, so to use it you can use -:program:`easy_install`: - +MongoEngine is available on PyPI, so to use it you can use :program:`pip`: + .. code-block:: console - # easy_install mongoengine + $ pip install mongoengine -Alternatively, if you don't have setuptools installed, `download it from PyPi +Alternatively, if you don't have setuptools installed, `download it from PyPi `_ and run .. code-block:: console - # python setup.py install + $ python setup.py install To use the bleeding-edge version of MongoEngine, you can get the source from `GitHub `_ and install it as above: - + .. code-block:: console - # git clone git://github.com/hmarr/mongoengine - # cd mongoengine - # python setup.py install + $ git clone git://github.com/hmarr/mongoengine + $ cd mongoengine + $ python setup.py install diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index c454b6e..13e1110 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -14,6 +14,7 @@ fetch documents from the database:: print user.name .. note:: + Once the iteration finishes (when :class:`StopIteration` is raised), :meth:`~mongoengine.queryset.QuerySet.rewind` will be called so that the :class:`~mongoengine.queryset.QuerySet` may be iterated over again. The @@ -39,29 +40,6 @@ syntax:: # been written by a user whose 'country' field is set to 'uk' uk_pages = Page.objects(author__country='uk') -Querying lists --------------- -On most fields, this syntax will look up documents where the field specified -matches the given value exactly, but when the field refers to a -:class:`~mongoengine.ListField`, a single item may be provided, in which case -lists that contain that item will be matched:: - - class Page(Document): - tags = ListField(StringField()) - - # This will match all pages that have the word 'coding' as an item in the - # 'tags' list - Page.objects(tags='coding') - -Raw queries ------------ -It is possible to provide a raw PyMongo query as a query parameter, which will -be integrated directly into the query. This is done using the ``__raw__`` -keyword argument:: - - Page.objects(__raw__={'tags': 'coding'}) - -.. versionadded:: 0.4 Query operators =============== @@ -99,26 +77,67 @@ expressions: * ``endswith`` -- string field ends with value * ``iendswith`` -- string field ends with value (case insensitive) -.. versionadded:: 0.3 - There are a few special operators for performing geographical queries, that may used with :class:`~mongoengine.GeoPointField`\ s: * ``within_distance`` -- provide a list containing a point and a maximum distance (e.g. [(41.342, -87.653), 5]) +* ``within_spherical_distance`` -- Same as above but using the spherical geo model + (e.g. [(41.342, -87.653), 5/earth_radius]) +* ``near`` -- order the documents by how close they are to a given point +* ``near_sphere`` -- Same as above but using the spherical geo model * ``within_box`` -- filter documents to those within a given bounding box (e.g. [(35.0, -125.0), (40.0, -100.0)]) -* ``near`` -- order the documents by how close they are to a given point +* ``within_polygon`` -- filter documents to those within a given polygon (e.g. + [(41.91,-87.69), (41.92,-87.68), (41.91,-87.65), (41.89,-87.65)]). + .. note:: Requires Mongo Server 2.0 -.. versionadded:: 0.4 -Querying by position -==================== +Querying lists +-------------- +On most fields, this syntax will look up documents where the field specified +matches the given value exactly, but when the field refers to a +:class:`~mongoengine.ListField`, a single item may be provided, in which case +lists that contain that item will be matched:: + + class Page(Document): + tags = ListField(StringField()) + + # This will match all pages that have the word 'coding' as an item in the + # 'tags' list + Page.objects(tags='coding') + It is possible to query by position in a list by using a numerical value as a query operator. So if you wanted to find all pages whose first tag was ``db``, you could use the following query:: - BlogPost.objects(tags__0='db') + Page.objects(tags__0='db') + +If you only want to fetch part of a list eg: you want to paginate a list, then +the `slice` operator is required:: + + # comments - skip 5, limit 10 + Page.objects.fields(slice__comments=[5, 10]) + +For updating documents, if you don't know the position in a list, you can use +the $ positional operator :: + + Post.objects(comments__by="joe").update(**{'inc__comments__$__votes': 1}) + +However, this doesn't map well to the syntax so you can alos use a capital S instead :: + + Post.objects(comments__by="joe").update(inc__comments__S__votes=1) + + .. note:: Due to Mongo currently the $ operator only applies to the first matched item in the query. + + +Raw queries +----------- +It is possible to provide a raw PyMongo query as a query parameter, which will +be integrated directly into the query. This is done using the ``__raw__`` +keyword argument:: + + Page.objects(__raw__={'tags': 'coding'}) .. versionadded:: 0.4 @@ -270,6 +289,7 @@ You may sum over the values of a specific field on documents using yearly_expense = Employee.objects.sum('salary') .. note:: + If the field isn't present on a document, that document will be ignored from the sum. @@ -318,6 +338,11 @@ will be given:: >>> f.rating # default value 3 +.. note:: + + The :meth:`~mongoengine.queryset.QuerySet.exclude` is the opposite of + :meth:`~mongoengine.queryset.QuerySet.only` if you want to exclude a field. + If you later need the missing fields, just call :meth:`~mongoengine.Document.reload` on your document. @@ -341,6 +366,67 @@ calling it with keyword arguments:: # Get top posts Post.objects((Q(featured=True) & Q(hits__gte=1000)) | Q(hits__gte=5000)) +.. _guide-atomic-updates: + +Atomic updates +============== +Documents may be updated atomically by using the +:meth:`~mongoengine.queryset.QuerySet.update_one` and +:meth:`~mongoengine.queryset.QuerySet.update` methods on a +:meth:`~mongoengine.queryset.QuerySet`. There are several different "modifiers" +that you may use with these methods: + +* ``set`` -- set a particular value +* ``unset`` -- delete a particular value (since MongoDB v1.3+) +* ``inc`` -- increment a value by a given amount +* ``dec`` -- decrement a value by a given amount +* ``pop`` -- remove the last item from a list +* ``push`` -- append a value to a list +* ``push_all`` -- append several values to a list +* ``pop`` -- remove the first or last element of a list +* ``pull`` -- remove a value from a list +* ``pull_all`` -- remove several values from a list +* ``add_to_set`` -- add value to a list only if its not in the list already + +The syntax for atomic updates is similar to the querying syntax, but the +modifier comes before the field, not after it:: + + >>> post = BlogPost(title='Test', page_views=0, tags=['database']) + >>> post.save() + >>> BlogPost.objects(id=post.id).update_one(inc__page_views=1) + >>> post.reload() # the document has been changed, so we need to reload it + >>> post.page_views + 1 + >>> BlogPost.objects(id=post.id).update_one(set__title='Example Post') + >>> post.reload() + >>> post.title + 'Example Post' + >>> BlogPost.objects(id=post.id).update_one(push__tags='nosql') + >>> post.reload() + >>> post.tags + ['database', 'nosql'] + +.. note :: + + In version 0.5 the :meth:`~mongoengine.Document.save` runs atomic updates + on changed documents by tracking changes to that document. + +The positional operator allows you to update list items without knowing the +index position, therefore making the update a single atomic operation. As we +cannot use the `$` syntax in keyword arguments it has been mapped to `S`:: + + >>> post = BlogPost(title='Test', page_views=0, tags=['database', 'mongo']) + >>> post.save() + >>> BlogPost.objects(id=post.id, tags='mongo').update(set__tags__S='mongodb') + >>> post.reload() + >>> post.tags + ['database', 'mongodb'] + +.. note :: + Currently only top level lists are handled, future versions of mongodb / + pymongo plan to support nested positional operators. See `The $ positional + operator `_. + Server-side javascript execution ================================ Javascript functions may be written and sent to the server for execution. The @@ -443,59 +529,3 @@ following example shows how the substitutions are made:: return comments; } """) - -.. _guide-atomic-updates: - -Atomic updates -============== -Documents may be updated atomically by using the -:meth:`~mongoengine.queryset.QuerySet.update_one` and -:meth:`~mongoengine.queryset.QuerySet.update` methods on a -:meth:`~mongoengine.queryset.QuerySet`. There are several different "modifiers" -that you may use with these methods: - -* ``set`` -- set a particular value -* ``unset`` -- delete a particular value (since MongoDB v1.3+) -* ``inc`` -- increment a value by a given amount -* ``dec`` -- decrement a value by a given amount -* ``pop`` -- remove the last item from a list -* ``push`` -- append a value to a list -* ``push_all`` -- append several values to a list -* ``pop`` -- remove the first or last element of a list -* ``pull`` -- remove a value from a list -* ``pull_all`` -- remove several values from a list -* ``add_to_set`` -- add value to a list only if its not in the list already - -The syntax for atomic updates is similar to the querying syntax, but the -modifier comes before the field, not after it:: - - >>> post = BlogPost(title='Test', page_views=0, tags=['database']) - >>> post.save() - >>> BlogPost.objects(id=post.id).update_one(inc__page_views=1) - >>> post.reload() # the document has been changed, so we need to reload it - >>> post.page_views - 1 - >>> BlogPost.objects(id=post.id).update_one(set__title='Example Post') - >>> post.reload() - >>> post.title - 'Example Post' - >>> BlogPost.objects(id=post.id).update_one(push__tags='nosql') - >>> post.reload() - >>> post.tags - ['database', 'nosql'] - -The positional operator allows you to update list items without knowing the -index position, therefore making the update a single atomic operation. As we -cannot use the `$` syntax in keyword arguments it has been mapped to `S`:: - - >>> post = BlogPost(title='Test', page_views=0, tags=['database', 'mongo']) - >>> post.save() - >>> BlogPost.objects(id=post.id, tags='mongo').update(set__tags__S='mongodb') - >>> post.reload() - >>> post.tags - ['database', 'mongodb'] - -.. note :: - Currently only top level lists are handled, future versions of mongodb / - pymongo plan to support nested positional operators. See `The $ positional - operator `_. diff --git a/docs/guide/signals.rst b/docs/guide/signals.rst index 3c3159f..58b3d6e 100644 --- a/docs/guide/signals.rst +++ b/docs/guide/signals.rst @@ -41,9 +41,9 @@ Example usage:: logging.debug("Created") else: logging.debug("Updated") - + signals.pre_save.connect(Author.pre_save, sender=Author) signals.post_save.connect(Author.post_save, sender=Author) -.. _blinker: http://pypi.python.org/pypi/blinker \ No newline at end of file +.. _blinker: http://pypi.python.org/pypi/blinker diff --git a/docs/index.rst b/docs/index.rst index 3b03656..920ddf6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,35 +2,62 @@ MongoEngine User Documentation ============================== -MongoEngine is an Object-Document Mapper, written in Python for working with +**MongoEngine** is an Object-Document Mapper, written in Python for working with MongoDB. To install it, simply run .. code-block:: console # pip install -U mongoengine -The source is available on `GitHub `_. +:doc:`tutorial` + Start here for a quick overview. + +:doc:`guide/index` + The Full guide to MongoEngine + +:doc:`apireference` + The complete API documentation. + +:doc:`django` + Using MongoEngine and Django + +Community +--------- To get help with using MongoEngine, use the `MongoEngine Users mailing list `_ or come chat on the `#mongoengine IRC channel `_. -If you are interested in contributing, join the developers' `mailing list +Contributing +------------ + +The source is available on `GitHub `_ and +contributions are always encouraged. Contributions can be as simple as +minor tweaks to this documentation. To contribute, fork the project on +`GitHub `_ and send a +pull request. + +Also, you can join the developers' `mailing list `_. +Changes +------- +See the :doc:`changelog` for a full list of changes to MongoEngine. + .. toctree:: - :maxdepth: 2 + :hidden: tutorial guide/index apireference django changelog - upgrading + upgrade Indices and tables -================== +------------------ * :ref:`genindex` +* :ref:`modindex` * :ref:`search` diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 7187adc..c684c1a 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -5,24 +5,33 @@ Upgrading 0.4 to 0.5 =========== -There have been the following backwards incompatibilities from 0.4 to 0.5: +There have been the following backwards incompatibilities from 0.4 to 0.5. The +main areas of changed are: choices in fields, map_reduce and collection names. -# Choice options: +Choice options: +-------------- Are now expected to be an iterable of tuples, with the first element in each tuple being the actual value to be stored. The second element is the human-readable name for the option. -# PyMongo / MongoDB -map reduce now requires pymongo 1.11+ More methods now use map_reduce as db.eval -is not supported for sharding - the following have been changed: +PyMongo / MongoDB +----------------- - * sum - * average - * item_frequencies +map reduce now requires pymongo 1.11+- The pymongo merge_output and reduce_output +parameters, have been depreciated. -#. Default collection naming. +More methods now use map_reduce as db.eval is not supported for sharding as such +the following have been changed: + + * :meth:`~mongoengine.queryset.QuerySet.sum` + * :meth:`~mongoengine.queryset.QuerySet.average` + * :meth:`~mongoengine.queryset.QuerySet.item_frequencies` + + +Default collection naming +------------------------- Previously it was just lowercase, its now much more pythonic and readable as its lowercase and underscores, previously :: diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a830150..a662685 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -744,7 +744,7 @@ class QuerySet(object): :param write_options: optional extra keyword arguments used if we have to create a new document. - Passes any write_options onto :meth:`~mongoengine.document.Document.save` + Passes any write_options onto :meth:`~mongoengine.Document.save` .. versionadded:: 0.3 """ @@ -901,9 +901,11 @@ class QuerySet(object): Returns an iterator yielding :class:`~mongoengine.document.MapReduceDocument`. - .. note:: Map/Reduce changed in server version **>= 1.7.4**. The PyMongo - :meth:`~pymongo.collection.Collection.map_reduce` helper requires - PyMongo version **>= 1.11**. + .. note:: + + Map/Reduce changed in server version **>= 1.7.4**. The PyMongo + :meth:`~pymongo.collection.Collection.map_reduce` helper requires + PyMongo version **>= 1.11**. .. versionchanged:: 0.5 - removed ``keep_temp`` keyword argument, which was only relevant @@ -1070,8 +1072,7 @@ class QuerySet(object): and `.exclude()` to manipulate which fields to retrieve. Fields also allows for a greater level of control for example: - Retrieving a Subrange of Array Elements - --------------------------------------- + Retrieving a Subrange of Array Elements: You can use the $slice operator to retrieve a subrange of elements in an array :: @@ -1500,6 +1501,7 @@ class QuerySet(object): This is useful for generating tag clouds, or searching documents. .. note:: + Can only do direct simple mappings and cannot map across :class:`~mongoengine.ReferenceField` or :class:`~mongoengine.GenericReferenceField` for more complex From ee7d370751dd93ac80cd2084f0269d377e1d79bd Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 9 Sep 2011 05:52:43 -0700 Subject: [PATCH 0354/1279] Bumped the version --- docs/conf.py | 2 +- mongoengine/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 2541f49..03ba047 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -38,7 +38,7 @@ master_doc = 'index' # General information about the project. project = u'MongoEngine' -copyright = u'2009-2010, Harry Marr' +copyright = u'2009-2011, Harry Marr' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index de635f9..0d27178 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -14,7 +14,8 @@ __all__ = (document.__all__ + fields.__all__ + connection.__all__ + __author__ = 'Harry Marr' -VERSION = (0, 4, 0) +VERSION = (0, 4, 1) + def get_version(): version = '%s.%s' % (VERSION[0], VERSION[1]) @@ -23,4 +24,3 @@ def get_version(): return version __version__ = get_version() - From b8a5791de6148e22785c4f75d55ecca96b764807 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 9 Sep 2011 14:33:27 +0100 Subject: [PATCH 0355/1279] Updates to documents [#245] --- docs/guide/defining-documents.rst | 1 + mongoengine/fields.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 00a7d09..fd005e4 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -50,6 +50,7 @@ are as follows: * :class:`~mongoengine.ReferenceField` * :class:`~mongoengine.GenericReferenceField` * :class:`~mongoengine.EmbeddedDocumentField` +* :class:`~mongoengine.GenericEmbeddedDocumentField` * :class:`~mongoengine.BooleanField` * :class:`~mongoengine.FileField` * :class:`~mongoengine.BinaryField` diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 9bb414b..c573443 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -374,8 +374,8 @@ class ComplexDateTimeField(StringField): class EmbeddedDocumentField(BaseField): - """An embedded document field. Only valid values are subclasses of - :class:`~mongoengine.EmbeddedDocument`. + """An embedded document field - with a declared document_type. + Only valid values are subclasses of :class:`~mongoengine.EmbeddedDocument`. """ def __init__(self, document_type, **kwargs): @@ -421,7 +421,14 @@ class EmbeddedDocumentField(BaseField): def prepare_query_value(self, op, value): return self.to_mongo(value) + class GenericEmbeddedDocumentField(BaseField): + """A generic embedded document field - allows any + :class:`~mongoengine.EmbeddedDocument` to be stored. + + Only valid values are subclasses of :class:`~mongoengine.EmbeddedDocument`. + """ + def prepare_query_value(self, op, value): return self.to_mongo(value) @@ -448,6 +455,7 @@ class GenericEmbeddedDocumentField(BaseField): data['_cls'] = document._class_name return data + class ListField(ComplexBaseField): """A list field that wraps a standard field, allowing multiple instances of the field to be used as a list in the database. From 60f0491f6230a3aa71fdee340c0f8f1cdab3bd0d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 9 Sep 2011 17:35:44 +0100 Subject: [PATCH 0356/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3a2a2c4..1073ba4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added GenericEmbeddedDocument - So you can embed any type of embeddable document - Added within_polygon support - for those with mongodb 1.9 - Updated sum / average to use map_reduce as db.eval doesn't work in sharded environments - Added where() - filter to allowing users to specify query expressions as Javascript From 050542c29b420128b0e89f6dfb1cf54d6b03b698 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 9 Sep 2011 17:36:40 +0100 Subject: [PATCH 0357/1279] Added InvalidDocumentError Ensures defined documents are valid and users don't override core methods by accident. fixes #275 --- docs/changelog.rst | 3 ++- mongoengine/base.py | 8 ++++++++ tests/document.py | 11 ++++++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1073ba4..04235db 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,7 +5,8 @@ Changelog Changes in dev ============== -- Added GenericEmbeddedDocument - So you can embed any type of embeddable document +- Added InvalidDocumentError - so Document core methods can't be overwritten +- Added GenericEmbeddedDocument - so you can embed any type of embeddable document - Added within_polygon support - for those with mongodb 1.9 - Updated sum / average to use map_reduce as db.eval doesn't work in sharded environments - Added where() - filter to allowing users to specify query expressions as Javascript diff --git a/mongoengine/base.py b/mongoengine/base.py index 6a94670..c4bcee1 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -16,6 +16,9 @@ class NotRegistered(Exception): pass +class InvalidDocumentError(Exception): + pass + class ValidationError(Exception): pass @@ -388,6 +391,8 @@ class DocumentMetaclass(type): attrs['_db_field_map'] = dict([(k, v.db_field) for k, v in doc_fields.items() if k!=v.db_field]) attrs['_reverse_db_field_map'] = dict([(v, k) for k, v in attrs['_db_field_map'].items()]) + from mongoengine import Document + new_class = super_new(cls, name, bases, attrs) for field in new_class._fields.values(): field.owner_document = new_class @@ -396,6 +401,9 @@ class DocumentMetaclass(type): field.document_type.register_delete_rule(new_class, field.name, delete_rule) + if field.name and hasattr(Document, field.name): + raise InvalidDocumentError("%s is a document method and not a valid field name" % field.name) + module = attrs.get('__module__') base_excs = tuple(base.DoesNotExist for base in bases diff --git a/tests/document.py b/tests/document.py index b76b6f9..95f3774 100644 --- a/tests/document.py +++ b/tests/document.py @@ -12,7 +12,7 @@ import weakref from fixtures import Base, Mixin, PickleEmbedded, PickleTest from mongoengine import * -from mongoengine.base import _document_registry, NotRegistered +from mongoengine.base import _document_registry, NotRegistered, InvalidDocumentError from mongoengine.connection import _get_db @@ -2336,6 +2336,15 @@ class DocumentTest(unittest.TestCase): pickle_doc.reload() self.assertEquals(resurrected, pickle_doc) + def throw_invalid_document_error(self): + + # test handles people trying to upsert + def throw_invalid_document_error(): + class Blog(Document): + validate = DictField() + + self.assertRaises(InvalidDocumentError, throw_invalid_document_error) + if __name__ == '__main__': unittest.main() From b91db87ae08b0035ab017223f827a393eab7f76c Mon Sep 17 00:00:00 2001 From: Colin Howe Date: Fri, 9 Sep 2011 19:17:40 +0100 Subject: [PATCH 0358/1279] Pre and post bulk-insert signals --- docs/changelog.rst | 1 + docs/guide/signals.rst | 4 ++- mongoengine/queryset.py | 6 +++++ mongoengine/signals.py | 2 ++ tests/signals.py | 56 ++++++++++++++++++++++++++++++++++++++--- 5 files changed, 65 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 04235db..efdc197 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -67,6 +67,7 @@ Changes in dev way the user has specified them - Fixed various errors - Added many tests +- Added pre and post bulk-insert signals Changes in v0.4 =============== diff --git a/docs/guide/signals.rst b/docs/guide/signals.rst index 58b3d6e..47ba06b 100644 --- a/docs/guide/signals.rst +++ b/docs/guide/signals.rst @@ -9,7 +9,7 @@ Signal support is provided by the excellent `blinker`_ library and will gracefully fall back if it is not available. -The following document signals exist in MongoEngine and are pretty self explaintary: +The following document signals exist in MongoEngine and are pretty self explanatory: * `mongoengine.signals.pre_init` * `mongoengine.signals.post_init` @@ -17,6 +17,8 @@ The following document signals exist in MongoEngine and are pretty self explaint * `mongoengine.signals.post_save` * `mongoengine.signals.pre_delete` * `mongoengine.signals.post_delete` + * `mongoengine.signals.pre_bulk_insert` + * `mongoengine.signals.post_bulk_insert` Example usage:: diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a662685..96dc2ca 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1,4 +1,5 @@ from connection import _get_db +from mongoengine import signals import pprint import pymongo @@ -812,15 +813,20 @@ class QuerySet(object): raise OperationError(msg) raw.append(doc.to_mongo()) + signals.pre_bulk_insert.send(self._document, documents=docs) ids = self._collection.insert(raw) if not load_bulk: + signals.post_bulk_insert.send( + self._document, documents=docs, loaded=False) return return_one and ids[0] or ids documents = self.in_bulk(ids) results = [] for obj_id in ids: results.append(documents.get(obj_id)) + signals.post_bulk_insert.send( + self._document, documents=results, loaded=True) return return_one and results[0] or results def with_id(self, object_id): diff --git a/mongoengine/signals.py b/mongoengine/signals.py index 0a69753..52ef312 100644 --- a/mongoengine/signals.py +++ b/mongoengine/signals.py @@ -42,3 +42,5 @@ pre_save = _signals.signal('pre_save') post_save = _signals.signal('post_save') pre_delete = _signals.signal('pre_delete') post_delete = _signals.signal('post_delete') +pre_bulk_insert = _signals.signal('pre_bulk_insert') +post_bulk_insert = _signals.signal('post_bulk_insert') diff --git a/tests/signals.py b/tests/signals.py index 9c41337..fd282b2 100644 --- a/tests/signals.py +++ b/tests/signals.py @@ -17,6 +17,7 @@ class SignalTests(unittest.TestCase): global signal_output signal_output = [] fn(*args, **kwargs) + print signal_output return signal_output def setUp(self): @@ -56,6 +57,18 @@ class SignalTests(unittest.TestCase): @classmethod def post_delete(cls, sender, document, **kwargs): signal_output.append('post_delete signal, %s' % document) + + @classmethod + def pre_bulk_insert(cls, sender, documents, **kwargs): + signal_output.append('pre_bulk_insert signal, %s' % documents) + + @classmethod + def post_bulk_insert(cls, sender, documents, **kwargs): + signal_output.append('post_bulk_insert signal, %s' % documents) + if kwargs.get('loaded', False): + signal_output.append('Is loaded') + else: + signal_output.append('Not loaded') self.Author = Author @@ -104,7 +117,9 @@ class SignalTests(unittest.TestCase): len(signals.pre_save.receivers), len(signals.post_save.receivers), len(signals.pre_delete.receivers), - len(signals.post_delete.receivers) + len(signals.post_delete.receivers), + len(signals.pre_bulk_insert.receivers), + len(signals.post_bulk_insert.receivers), ) signals.pre_init.connect(Author.pre_init, sender=Author) @@ -113,6 +128,8 @@ class SignalTests(unittest.TestCase): signals.post_save.connect(Author.post_save, sender=Author) signals.pre_delete.connect(Author.pre_delete, sender=Author) signals.post_delete.connect(Author.post_delete, sender=Author) + signals.pre_bulk_insert.connect(Author.pre_bulk_insert, sender=Author) + signals.post_bulk_insert.connect(Author.post_bulk_insert, sender=Author) signals.pre_init.connect(Another.pre_init, sender=Another) signals.post_init.connect(Another.post_init, sender=Another) @@ -128,6 +145,8 @@ class SignalTests(unittest.TestCase): signals.pre_delete.disconnect(self.Author.pre_delete) signals.post_save.disconnect(self.Author.post_save) signals.pre_save.disconnect(self.Author.pre_save) + signals.pre_bulk_insert.disconnect(self.Author.pre_bulk_insert) + signals.post_bulk_insert.disconnect(self.Author.post_bulk_insert) signals.pre_init.disconnect(self.Another.pre_init) signals.post_init.disconnect(self.Another.post_init) @@ -143,7 +162,9 @@ class SignalTests(unittest.TestCase): len(signals.pre_save.receivers), len(signals.post_save.receivers), len(signals.pre_delete.receivers), - len(signals.post_delete.receivers) + len(signals.post_delete.receivers), + len(signals.pre_bulk_insert.receivers), + len(signals.post_bulk_insert.receivers), ) self.assertEqual(self.pre_signals, post_signals) @@ -154,6 +175,14 @@ class SignalTests(unittest.TestCase): def create_author(): a1 = self.Author(name='Bill Shakespeare') + def bulk_create_author_with_load(): + a1 = self.Author(name='Bill Shakespeare') + self.Author.objects.insert([a1], load_bulk=True) + + def bulk_create_author_without_load(): + a1 = self.Author(name='Bill Shakespeare') + self.Author.objects.insert([a1], load_bulk=False) + self.assertEqual(self.get_signal_output(create_author), [ "pre_init signal, Author", "{'name': 'Bill Shakespeare'}", @@ -178,4 +207,25 @@ class SignalTests(unittest.TestCase): self.assertEqual(self.get_signal_output(a1.delete), [ 'pre_delete signal, William Shakespeare', 'post_delete signal, William Shakespeare', - ]) \ No newline at end of file + ]) + + signal_output = self.get_signal_output(bulk_create_author_with_load) + + # The output of this signal is not entirely deterministic. The reloaded + # object will have an object ID. Hence, we only check part of the output + self.assertEquals(signal_output[3], + "pre_bulk_insert signal, []") + self.assertEquals(signal_output[-2:], + ["post_bulk_insert signal, []", + "Is loaded",]) + + self.assertEqual(self.get_signal_output(bulk_create_author_without_load), [ + "pre_init signal, Author", + "{'name': 'Bill Shakespeare'}", + "post_init signal, Bill Shakespeare", + "pre_bulk_insert signal, []", + "post_bulk_insert signal, []", + "Not loaded", + ]) + + self.Author.objects.delete() From 88b1a2971944e9e762910bd583fef56da2cc82a6 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sat, 10 Sep 2011 11:54:43 +0200 Subject: [PATCH 0359/1279] Typo fix --- docs/guide/querying.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 13e1110..13a374c 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -124,7 +124,7 @@ the $ positional operator :: Post.objects(comments__by="joe").update(**{'inc__comments__$__votes': 1}) -However, this doesn't map well to the syntax so you can alos use a capital S instead :: +However, this doesn't map well to the syntax so you can also use a capital S instead :: Post.objects(comments__by="joe").update(inc__comments__S__votes=1) From 66c53f949b01bb57289b0a829cf9360157ffeec8 Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Sat, 10 Sep 2011 11:47:16 +0100 Subject: [PATCH 0360/1279] Version bump to 0.5 --- mongoengine/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 0d27178..dd3c517 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -14,7 +14,7 @@ __all__ = (document.__all__ + fields.__all__ + connection.__all__ + __author__ = 'Harry Marr' -VERSION = (0, 4, 1) +VERSION = (0, 5, 0) def get_version(): From 8105bfd8b3edd9c6405243ed480ea13a820dfa55 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Sat, 10 Sep 2011 13:00:34 +0100 Subject: [PATCH 0361/1279] Updated changelog for 0.5 release --- docs/changelog.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 04235db..b7904bd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,8 +2,8 @@ Changelog ========= -Changes in dev -============== +Changes in v0.5 +=============== - Added InvalidDocumentError - so Document core methods can't be overwritten - Added GenericEmbeddedDocument - so you can embed any type of embeddable document From 89c44cd14eb77d6f778789260344f9de7659c2be Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Sun, 11 Sep 2011 08:09:16 +0100 Subject: [PATCH 0362/1279] Added missing fields to the api documentation --- docs/apireference.rst | 35 ++++++++++------------------------- mongoengine/fields.py | 6 ++++-- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 2442803..57472dc 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -15,12 +15,12 @@ Documents .. attribute:: objects - A :class:`~mongoengine.queryset.QuerySet` object that is created lazily + A :class:`~mongoengine.queryset.QuerySet` object that is created lazily on access. .. autoclass:: mongoengine.EmbeddedDocument :members: - + .. autoclass:: mongoengine.document.MapReduceDocument :members: @@ -31,46 +31,31 @@ Querying :members: .. automethod:: mongoengine.queryset.QuerySet.__call__ - + .. autofunction:: mongoengine.queryset.queryset_manager Fields ====== .. autoclass:: mongoengine.StringField - .. autoclass:: mongoengine.URLField - .. autoclass:: mongoengine.EmailField - .. autoclass:: mongoengine.IntField - .. autoclass:: mongoengine.FloatField - .. autoclass:: mongoengine.DecimalField - -.. autoclass:: mongoengine.BooleanField - .. autoclass:: mongoengine.DateTimeField - .. autoclass:: mongoengine.ComplexDateTimeField - -.. autoclass:: mongoengine.EmbeddedDocumentField - -.. autoclass:: mongoengine.DictField - .. autoclass:: mongoengine.ListField - .. autoclass:: mongoengine.SortedListField - -.. autoclass:: mongoengine.BinaryField - +.. autoclass:: mongoengine.DictField +.. autoclass:: mongoengine.MapField .. autoclass:: mongoengine.ObjectIdField - .. autoclass:: mongoengine.ReferenceField - .. autoclass:: mongoengine.GenericReferenceField - +.. autoclass:: mongoengine.EmbeddedDocumentField +.. autoclass:: mongoengine.GenericEmbeddedDocumentField +.. autoclass:: mongoengine.BooleanField .. autoclass:: mongoengine.FileField - +.. autoclass:: mongoengine.BinaryField .. autoclass:: mongoengine.GeoPointField +.. autoclass:: mongoengine.SequenceField diff --git a/mongoengine/fields.py b/mongoengine/fields.py index c573443..fa99352 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -920,9 +920,11 @@ class GeoPointField(BaseField): class SequenceField(IntField): - """Provides a sequental counter. + """Provides a sequental counter (see http://www.mongodb.org/display/DOCS/Object+IDs#ObjectIDs-SequenceNumbers) - ..note:: Although traditional databases often use increasing sequence + .. note:: + + Although traditional databases often use increasing sequence numbers for primary keys. In MongoDB, the preferred approach is to use Object IDs instead. The concept is that in a very large cluster of machines, it is easier to create an object ID than have From adb7bbeea0b95975eca21cbe090eca08b4dd9315 Mon Sep 17 00:00:00 2001 From: Karim Allah Date: Sun, 18 Sep 2011 19:48:33 +0200 Subject: [PATCH 0363/1279] Being compatible with non-django style chioces --- mongoengine/base.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index c4bcee1..92a5b71 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -139,9 +139,14 @@ class BaseField(object): def _validate(self, value): # check choices if self.choices is not None: - option_keys = [option_key for option_key, option_value in self.choices] - if value not in option_keys: - raise ValidationError("Value must be one of %s." % unicode(option_keys)) + if type(choices[0]) is tuple: + option_keys = [option_key for option_key, option_value in self.choices] + if value not in option_keys: + raise ValidationError("Value must be one of %s." % unicode(option_keys)) + else: + if value not in self.choices: + raise ValidationError("Value must be one of %s." % unicode(self.choices)) + # check validation argument if self.validation is not None: From f7fbb3d2f6029f25aa294f9f49adcb7db3d1359b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 20 Sep 2011 03:45:11 -0700 Subject: [PATCH 0364/1279] Relaxed field name checking on embedded documents --- mongoengine/base.py | 4 ++-- tests/document.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index c4bcee1..02ad8bb 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -391,7 +391,7 @@ class DocumentMetaclass(type): attrs['_db_field_map'] = dict([(k, v.db_field) for k, v in doc_fields.items() if k!=v.db_field]) attrs['_reverse_db_field_map'] = dict([(v, k) for k, v in attrs['_db_field_map'].items()]) - from mongoengine import Document + from mongoengine import Document, EmbeddedDocument new_class = super_new(cls, name, bases, attrs) for field in new_class._fields.values(): @@ -401,7 +401,7 @@ class DocumentMetaclass(type): field.document_type.register_delete_rule(new_class, field.name, delete_rule) - if field.name and hasattr(Document, field.name): + if field.name and hasattr(Document, field.name) and EmbeddedDocument not in new_class.mro(): raise InvalidDocumentError("%s is a document method and not a valid field name" % field.name) module = attrs.get('__module__') diff --git a/tests/document.py b/tests/document.py index 95f3774..1eeda46 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2336,7 +2336,7 @@ class DocumentTest(unittest.TestCase): pickle_doc.reload() self.assertEquals(resurrected, pickle_doc) - def throw_invalid_document_error(self): + def test_throw_invalid_document_error(self): # test handles people trying to upsert def throw_invalid_document_error(): From a9cacd2e0646423de5dee036387a1025f66d487e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 20 Sep 2011 08:56:30 -0300 Subject: [PATCH 0365/1279] fixed embedded null item_frequencies --- mongoengine/queryset.py | 21 ++++++++++++++++++--- tests/queryset.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a662685..a46581e 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1526,7 +1526,12 @@ class QuerySet(object): function() { path = '{{~%(field)s}}'.split('.'); field = this; - for (p in path) { field = field[path[p]]; } + for (p in path) { + if (field) + field = field[path[p]]; + else + break; + } if (field && field.constructor == Array) { field.forEach(function(item) { emit(item, 1); @@ -1572,7 +1577,12 @@ class QuerySet(object): var total = 0.0; db[collection].find(query).forEach(function(doc) { field = doc; - for (p in path) { field = field[path[p]]; } + for (p in path) { + if (field) + field = field[path[p]]; + else + break; + } if (field && field.constructor == Array) { total += field.length; } else { @@ -1588,7 +1598,12 @@ class QuerySet(object): } db[collection].find(query).forEach(function(doc) { field = doc; - for (p in path) { field = field[path[p]]; } + for (p in path) { + if (field) + field = field[path[p]]; + else + break; + } if (field && field.constructor == Array) { field.forEach(function(item) { frequencies[item] = inc + (isNaN(frequencies[item]) ? 0: frequencies[item]); diff --git a/tests/queryset.py b/tests/queryset.py index f093f6a..de02387 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1840,6 +1840,36 @@ class QuerySetTest(unittest.TestCase): freq = Person.objects.item_frequencies('city', normalize=True, map_reduce=True) self.assertEquals(freq, {'CRB': 0.5, None: 0.5}) + + def test_item_frequencies_with_null_embedded(self): + class Data(EmbeddedDocument): + name = StringField() + + class Extra(EmbeddedDocument): + tag = StringField() + + class Person(Document): + data = EmbeddedDocumentField(Data, required=True) + extra = EmbeddedDocumentField(Extra) + + + Person.drop_collection() + + p = Person() + p.data = Data(name="Wilson Jr") + p.save() + + p = Person() + p.data = Data(name="Wesley") + p.extra = Extra(tag="friend") + p.save() + + ot = Person.objects.item_frequencies('extra.tag', map_reduce=False) + self.assertEquals(ot, {None: 1.0, u'friend': 1.0}) + + ot = Person.objects.item_frequencies('extra.tag', map_reduce=True) + self.assertEquals(ot, {None: 1.0, u'friend': 1.0}) + def test_average(self): """Ensure that field can be averaged correctly. """ From 2ca66482270e98af2447dbb5774d3bbcb20ae623 Mon Sep 17 00:00:00 2001 From: kuno Date: Tue, 20 Sep 2011 21:30:20 +0800 Subject: [PATCH 0366/1279] fixed indentation error in signal docs --- docs/guide/signals.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guide/signals.rst b/docs/guide/signals.rst index 58b3d6e..4e13603 100644 --- a/docs/guide/signals.rst +++ b/docs/guide/signals.rst @@ -42,8 +42,8 @@ Example usage:: else: logging.debug("Updated") - signals.pre_save.connect(Author.pre_save, sender=Author) - signals.post_save.connect(Author.post_save, sender=Author) + signals.pre_save.connect(Author.pre_save, sender=Author) + signals.post_save.connect(Author.post_save, sender=Author) .. _blinker: http://pypi.python.org/pypi/blinker From c081aca79431e7e060e66c2e4d4da88cc66ed8ae Mon Sep 17 00:00:00 2001 From: Karim Allah Date: Sun, 25 Sep 2011 18:58:40 +0200 Subject: [PATCH 0367/1279] Fixing dereferencing when the dereferenced-document wasn't found. --- mongoengine/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index 92a5b71..d02848f 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -10,6 +10,7 @@ import pymongo import pymongo.objectid import operator from functools import partial +from bson.dbref import DBRef class NotRegistered(Exception): @@ -97,6 +98,9 @@ class BaseField(object): # Get value from document instance if available, if not use default value = instance._data.get(self.name) + if isinstance(value, DBRef): + raise ValueError("Can't dereference from the given DBRef (%s)" % str(value)) + if value is None: value = self.default # Allow callable default values From a7edd8602cd38f6ad39fd274e3cbb8a737d7d1b9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 21 Sep 2011 01:42:52 -0700 Subject: [PATCH 0368/1279] Added support for expando style dynamic documents. Added two new classes: DynamicDocument and DynamicEmbeddedDocument for handling expando style setting of attributes. [closes #112] --- docs/apireference.rst | 6 + docs/changelog.rst | 5 + docs/guide/defining-documents.rst | 28 ++ docs/upgrade.rst | 5 + mongoengine/base.py | 148 +++++++++-- mongoengine/document.py | 47 +++- mongoengine/queryset.py | 10 +- tests/dynamic_document.py | 413 ++++++++++++++++++++++++++++++ 8 files changed, 641 insertions(+), 21 deletions(-) create mode 100644 tests/dynamic_document.py diff --git a/docs/apireference.rst b/docs/apireference.rst index 57472dc..932152f 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -21,6 +21,12 @@ Documents .. autoclass:: mongoengine.EmbeddedDocument :members: +.. autoclass:: mongoengine.DynamicDocument + :members: + +.. autoclass:: mongoengine.DynamicEmbeddedDocument + :members: + .. autoclass:: mongoengine.document.MapReduceDocument :members: diff --git a/docs/changelog.rst b/docs/changelog.rst index b7904bd..904eb20 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,11 @@ Changelog ========= +Changes in dev +============== + +- Added DynamicDocument and EmbeddedDynamicDocument classes for expando schemas + Changes in v0.5 =============== diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index fd005e4..6367d95 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -24,6 +24,34 @@ objects** as class attributes to the document class:: title = StringField(max_length=200, required=True) date_modified = DateTimeField(default=datetime.datetime.now) +Dynamic document schemas +======================== +One of the benefits of MongoDb is dynamic schemas for a collection, whilst data +should be planned and organised (after all explicit is better than implicit!) +there are scenarios where having dynamic / expando style documents is desirable. + +:class:`~mongoengine.DynamicDocument` documents work in the same way as +:class:`~mongoengine.Document` but any data / attributes set to them will also +be saved :: + + from mongoengine import * + + class Page(DynamicDocument): + title = StringField(max_length=200, required=True) + + # Create a new page and add tags + >>> page = Page(title='Using MongoEngine') + >>> page.tags = ['mongodb', 'mongoengine'] + >>> page.save() + + >>> Page.objects(tags='mongoengine').count() + >>> 1 + +..note:: + + There is one caveat on Dynamic Documents: fields cannot start with `_` + + Fields ====== By default, fields are not required. To make a field mandatory, set the diff --git a/docs/upgrade.rst b/docs/upgrade.rst index c684c1a..671b930 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -2,6 +2,11 @@ Upgrading ========= +0.5 to 0.6 +========== + +TBC + 0.4 to 0.5 =========== diff --git a/mongoengine/base.py b/mongoengine/base.py index 02ad8bb..40953f1 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -301,6 +301,40 @@ class ComplexBaseField(BaseField): owner_document = property(_get_owner_document, _set_owner_document) +class BaseDynamicField(BaseField): + """Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data""" + + def to_mongo(self, value): + """Convert a Python type to a MongoDBcompatible type. + """ + + if isinstance(value, basestring): + return value + + if hasattr(value, 'to_mongo'): + return value.to_mongo() + + if not isinstance(value, (dict, list, tuple)): + return value + + is_list = False + if not hasattr(value, 'items'): + is_list = True + value = dict([(k, v) for k, v in enumerate(value)]) + + data = {} + for k, v in value.items(): + data[k] = self.to_mongo(v) + + if is_list: # Convert back to a list + value = [v for k, v in sorted(data.items(), key=operator.itemgetter(0))] + else: + value = data + return value + + def lookup_member(self, member_name): + return member_name + class ObjectIdField(BaseField): """An field wrapper around MongoDB's ObjectIds. """ @@ -585,30 +619,98 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): class BaseDocument(object): + _dynamic = False + def __init__(self, **values): signals.pre_init.send(self.__class__, document=self, values=values) self._data = {} self._initialised = False + # Assign default values to instance for attr_name, field in self._fields.items(): value = getattr(self, attr_name, None) setattr(self, attr_name, value) - # Assign initial values to instance - for attr_name in values.keys(): - try: - value = values.pop(attr_name) - setattr(self, attr_name, value) - except AttributeError: - pass + # Set passed values after initialisation + if self._dynamic: + self._dynamic_fields = {} + dynamic_data = {} + for key, value in values.items(): + if key in self._fields or key == '_id': + setattr(self, key, value) + elif self._dynamic: + dynamic_data[key] = value + else: + for key, value in values.items(): + setattr(self, key, value) - # Set any get_fieldname_display methods + # Set any get_fieldname_display methodsF self.__set_field_display() # Flag initialised self._initialised = True + + if self._dynamic: + for key, value in dynamic_data.items(): + setattr(self, key, value) signals.post_init.send(self.__class__, document=self) + def __setattr__(self, name, value): + # Handle dynamic data only if an intialised dynamic document + if self._dynamic and getattr(self, '_initialised', False): + + field = None + if not hasattr(self, name) and not name.startswith('_'): + field = BaseDynamicField(db_field=name) + field.name = name + self._dynamic_fields[name] = field + + if not name.startswith('_'): + value = self.__expand_dynamic_values(name, value) + + # Handle marking data as changed + if name in self._dynamic_fields: + self._data[name] = value + if hasattr(self, '_changed_fields'): + self._mark_as_changed(name) + + super(BaseDocument, self).__setattr__(name, value) + + def __expand_dynamic_values(self, name, value): + """expand any dynamic values to their correct types / values""" + if not isinstance(value, (dict, list, tuple)): + return value + + is_list = False + if not hasattr(value, 'items'): + is_list = True + value = dict([(k, v) for k, v in enumerate(value)]) + + if not is_list and '_cls' in value: + cls = get_document(value['_cls']) + value = cls(**value) + value._dynamic = True + value._changed_fields = [] + return value + + data = {} + for k, v in value.items(): + key = name if is_list else k + data[k] = self.__expand_dynamic_values(key, v) + + if is_list: # Convert back to a list + value = [v for k, v in sorted(data.items(), key=operator.itemgetter(0))] + else: + value = data + + # Convert lists / values so we can watch for any changes on them + if isinstance(value, (list, tuple)) and not isinstance(value, BaseList): + value = BaseList(value, instance=self, name=name) + elif isinstance(value, dict) and not isinstance(value, BaseDict): + value = BaseDict(value, instance=self, name=name) + + return value + def validate(self): """Ensure that all fields' values are valid and that required fields are present. @@ -653,6 +755,12 @@ class BaseDocument(object): data['_types'] = self._superclasses.keys() + [self._class_name] if '_id' in data and data['_id'] is None: del data['_id'] + + if not self._dynamic: + return data + + for name, field in self._dynamic_fields.items(): + data[name] = field.to_mongo(self._data.get(name, None)) return data @classmethod @@ -727,14 +835,19 @@ class BaseDocument(object): def _get_changed_fields(self, key=''): """Returns a list of all fields that have explicitly been changed. """ - from mongoengine import EmbeddedDocument + from mongoengine import EmbeddedDocument, DynamicEmbeddedDocument _changed_fields = [] _changed_fields += getattr(self, '_changed_fields', []) - for field_name in self._fields: + + field_list = self._fields.copy() + if self._dynamic: + field_list.update(self._dynamic_fields) + + for field_name in field_list: db_field_name = self._db_field_map.get(field_name, field_name) key = '%s.' % db_field_name field = getattr(self, field_name, None) - if isinstance(field, EmbeddedDocument) and db_field_name not in _changed_fields: # Grab all embedded fields that have been changed + if isinstance(field, (EmbeddedDocument, DynamicEmbeddedDocument)) and db_field_name not in _changed_fields: # Grab all embedded fields that have been changed _changed_fields += ["%s%s" % (key, k) for k in field._get_changed_fields(key) if k] elif isinstance(field, (list, tuple, dict)) and db_field_name not in _changed_fields: # Loop list / dict fields as they contain documents # Determine the iterator to use @@ -747,7 +860,6 @@ class BaseDocument(object): continue list_key = "%s%s." % (key, index) _changed_fields += ["%s%s" % (list_key, k) for k in value._get_changed_fields(list_key) if k] - return _changed_fields def _delta(self): @@ -785,8 +897,11 @@ class BaseDocument(object): # If we've set a value that ain't the default value dont unset it. default = None - - if path in self._fields: + if self._dynamic and parts[0] in self._dynamic_fields: + del(set_data[path]) + unset_data[path] = 1 + continue + elif path in self._fields: default = self._fields[path].default else: # Perform a full lookup for lists / embedded lookups d = self @@ -805,7 +920,10 @@ class BaseDocument(object): field_name = d._reverse_db_field_map.get(db_field_name, db_field_name) - default = d._fields[field_name].default + if field_name in d._fields: + default = d._fields.get(field_name).default + else: + default = None if default is not None: if callable(default): diff --git a/mongoengine/document.py b/mongoengine/document.py index 3ccc4dd..81c288e 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,13 +1,14 @@ +import operator from mongoengine import signals from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, - ValidationError, BaseDict, BaseList) + ValidationError, BaseDict, BaseList, BaseDynamicField) from queryset import OperationError from connection import _get_db import pymongo -__all__ = ['Document', 'EmbeddedDocument', 'ValidationError', - 'OperationError', 'InvalidCollectionError'] +__all__ = ['Document', 'EmbeddedDocument', 'DynamicDocument', 'DynamicEmbeddedDocument', + 'ValidationError', 'OperationError', 'InvalidCollectionError'] class InvalidCollectionError(Exception): @@ -198,6 +199,7 @@ class Document(BaseDocument): reset_changed_fields(field, inspected_docs) reset_changed_fields(self) + self._changed_fields = [] signals.post_save.send(self.__class__, document=self, created=creation_mode) def update(self, **kwargs): @@ -247,8 +249,12 @@ class Document(BaseDocument): """ id_field = self._meta['id_field'] obj = self.__class__.objects(**{id_field: self[id_field]}).first() + for field in self._fields: setattr(self, field, self._reload(field, obj[field])) + if self._dynamic: + for name in self._dynamic_fields.keys(): + setattr(self, name, self._reload(name, obj._data[name])) self._changed_fields = [] def _reload(self, key, value): @@ -261,7 +267,7 @@ class Document(BaseDocument): elif isinstance(value, BaseList): value = [self._reload(key, v) for v in value] value = BaseList(value, instance=self, name=key) - elif isinstance(value, EmbeddedDocument): + elif isinstance(value, (EmbeddedDocument, DynamicEmbeddedDocument)): value._changed_fields = [] return value @@ -289,6 +295,39 @@ class Document(BaseDocument): db.drop_collection(cls._get_collection_name()) +class DynamicDocument(Document): + """A Dynamic Document class allowing flexible, expandable and uncontrolled + schemas. As a :class:`~mongoengine.Document` subclass, acts in the same + way as an ordinary document but has expando style properties. Any data + passed or set against the :class:`~mongoengine.DynamicDocument` that is + not a field is automatically converted into a + :class:`~mongoengine.BaseDynamicField` and data can be attributed to that + field. + + ..note:: + + There is one caveat on Dynamic Documents: fields cannot start with `_` + """ + __metaclass__ = TopLevelDocumentMetaclass + _dynamic = True + + +class DynamicEmbeddedDocument(EmbeddedDocument): + """A Dynamic Embedded Document class allowing flexible, expandable and + uncontrolled schemas. See :class:`~mongoengine.DynamicDocument` for more + information about dynamic documents. + """ + + __metaclass__ = DocumentMetaclass + _dynamic = True + + def __delattr__(self, *args, **kwargs): + """Deletes the attribute by setting to None and allowing _delta to unset + it""" + field_name = args[0] + setattr(self, field_name, None) + + class MapReduceDocument(object): """A document returned from a map/reduce query. diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a46581e..2b44dc4 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -590,7 +590,14 @@ class QuerySet(object): if field_name == 'pk': # Deal with "primary key" alias field_name = document._meta['id_field'] - field = document._fields[field_name] + if field_name in document._fields: + field = document._fields[field_name] + elif document._dynamic: + from base import BaseDynamicField + field = BaseDynamicField(db_field=field_name) + else: + raise InvalidQueryError('Cannot resolve field "%s"' + % field_name) else: # Look up subfield on the previous field new_field = field.lookup_member(field_name) @@ -603,7 +610,6 @@ class QuerySet(object): % field_name) field = new_field # update field to the new field type fields.append(field) - return fields @classmethod diff --git a/tests/dynamic_document.py b/tests/dynamic_document.py new file mode 100644 index 0000000..d76b196 --- /dev/null +++ b/tests/dynamic_document.py @@ -0,0 +1,413 @@ +import unittest + +from mongoengine import * +from mongoengine.connection import _get_db + +class DynamicDocTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = _get_db() + + class Person(DynamicDocument): + name = StringField() + + Person.drop_collection() + + self.Person = Person + + def test_simple_dynamic_document(self): + """Ensures simple dynamic documents are saved correctly""" + + p = self.Person() + p.name = "James" + p.age = 34 + + self.assertEquals(p.to_mongo(), + {"_types": ["Person"], "_cls": "Person", + "name": "James", "age": 34} + ) + + p.save() + + self.assertEquals(self.Person.objects.first().age, 34) + + # Confirm no changes to self.Person + self.assertFalse(hasattr(self.Person, 'age')) + + def test_change_scope_of_variable(self): + """Test changing the scope of a dynamic field has no adverse effects""" + p = self.Person() + p.name = "Dean" + p.misc = 22 + p.save() + + p = self.Person.objects.get() + p.misc = {'hello': 'world'} + p.save() + + p = self.Person.objects.get() + self.assertEquals(p.misc, {'hello': 'world'}) + + def test_dynamic_document_queries(self): + """Ensure we can query dynamic fields""" + p = self.Person() + p.name = "Dean" + p.age = 22 + p.save() + + self.assertEquals(1, self.Person.objects(age=22).count()) + p = self.Person.objects(age=22) + p = p.get() + self.assertEquals(22, p.age) + + def test_complex_data_lookups(self): + """Ensure you can query dynamic document dynamic fields""" + p = self.Person() + p.misc = {'hello': 'world'} + p.save() + + self.assertEquals(1, self.Person.objects(misc__hello='world').count()) + + def test_inheritance(self): + """Ensure that dynamic document plays nice with inheritance""" + class Employee(self.Person): + salary = IntField() + + Employee.drop_collection() + + self.assertTrue('name' in Employee._fields) + self.assertTrue('salary' in Employee._fields) + self.assertEqual(Employee._get_collection_name(), + self.Person._get_collection_name()) + + joe_bloggs = Employee() + joe_bloggs.name = "Joe Bloggs" + joe_bloggs.salary = 10 + joe_bloggs.age = 20 + joe_bloggs.save() + + self.assertEquals(1, self.Person.objects(age=20).count()) + self.assertEquals(1, Employee.objects(age=20).count()) + + joe_bloggs = self.Person.objects.first() + self.assertTrue(isinstance(joe_bloggs, Employee)) + + def test_embedded_dynamic_document(self): + """Test dynamic embedded documents""" + class Embedded(DynamicEmbeddedDocument): + pass + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + + embedded_1 = Embedded() + embedded_1.string_field = 'hello' + embedded_1.int_field = 1 + embedded_1.dict_field = {'hello': 'world'} + embedded_1.list_field = ['1', 2, {'hello': 'world'}] + doc.embedded_field = embedded_1 + + self.assertEquals(doc.to_mongo(), {"_types": ['Doc'], "_cls": "Doc", + "embedded_field": { + "_types": ['Embedded'], "_cls": "Embedded", + "string_field": "hello", + "int_field": 1, + "dict_field": {"hello": "world"}, + "list_field": ['1', 2, {'hello': 'world'}] + } + }) + doc.save() + + doc = Doc.objects.first() + self.assertEquals(doc.embedded_field.__class__, Embedded) + self.assertEquals(doc.embedded_field.string_field, "hello") + self.assertEquals(doc.embedded_field.int_field, 1) + self.assertEquals(doc.embedded_field.dict_field, {'hello': 'world'}) + self.assertEquals(doc.embedded_field.list_field, ['1', 2, {'hello': 'world'}]) + + def test_complex_embedded_documents(self): + """Test complex dynamic embedded documents setups""" + class Embedded(DynamicEmbeddedDocument): + pass + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + + embedded_1 = Embedded() + embedded_1.string_field = 'hello' + embedded_1.int_field = 1 + embedded_1.dict_field = {'hello': 'world'} + + embedded_2 = Embedded() + embedded_2.string_field = 'hello' + embedded_2.int_field = 1 + embedded_2.dict_field = {'hello': 'world'} + embedded_2.list_field = ['1', 2, {'hello': 'world'}] + + embedded_1.list_field = ['1', 2, embedded_2] + doc.embedded_field = embedded_1 + + self.assertEquals(doc.to_mongo(), {"_types": ['Doc'], "_cls": "Doc", + "embedded_field": { + "_types": ['Embedded'], "_cls": "Embedded", + "string_field": "hello", + "int_field": 1, + "dict_field": {"hello": "world"}, + "list_field": ['1', 2, + {"_types": ['Embedded'], "_cls": "Embedded", + "string_field": "hello", + "int_field": 1, + "dict_field": {"hello": "world"}, + "list_field": ['1', 2, {'hello': 'world'}]} + ] + } + }) + doc.save() + doc = Doc.objects.first() + self.assertEquals(doc.embedded_field.__class__, Embedded) + self.assertEquals(doc.embedded_field.string_field, "hello") + self.assertEquals(doc.embedded_field.int_field, 1) + self.assertEquals(doc.embedded_field.dict_field, {'hello': 'world'}) + self.assertEquals(doc.embedded_field.list_field[0], '1') + self.assertEquals(doc.embedded_field.list_field[1], 2) + + embedded_field = doc.embedded_field.list_field[2] + + self.assertEquals(embedded_field.__class__, Embedded) + self.assertEquals(embedded_field.string_field, "hello") + self.assertEquals(embedded_field.int_field, 1) + self.assertEquals(embedded_field.dict_field, {'hello': 'world'}) + self.assertEquals(embedded_field.list_field, ['1', 2, {'hello': 'world'}]) + + def test_delta_for_dynamic_documents(self): + p = self.Person() + p.name = "Dean" + p.age = 22 + p.save() + + p.age = 24 + self.assertEquals(p.age, 24) + self.assertEquals(p._get_changed_fields(), ['age']) + self.assertEquals(p._delta(), ({'age': 24}, {})) + + p = self.Person.objects(age=22).get() + p.age = 24 + self.assertEquals(p.age, 24) + self.assertEquals(p._get_changed_fields(), ['age']) + self.assertEquals(p._delta(), ({'age': 24}, {})) + + p.save() + self.assertEquals(1, self.Person.objects(age=24).count()) + + def test_delta(self): + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + doc.save() + + doc = Doc.objects.first() + self.assertEquals(doc._get_changed_fields(), []) + self.assertEquals(doc._delta(), ({}, {})) + + doc.string_field = 'hello' + self.assertEquals(doc._get_changed_fields(), ['string_field']) + self.assertEquals(doc._delta(), ({'string_field': 'hello'}, {})) + + doc._changed_fields = [] + doc.int_field = 1 + self.assertEquals(doc._get_changed_fields(), ['int_field']) + self.assertEquals(doc._delta(), ({'int_field': 1}, {})) + + doc._changed_fields = [] + dict_value = {'hello': 'world', 'ping': 'pong'} + doc.dict_field = dict_value + self.assertEquals(doc._get_changed_fields(), ['dict_field']) + self.assertEquals(doc._delta(), ({'dict_field': dict_value}, {})) + + doc._changed_fields = [] + list_value = ['1', 2, {'hello': 'world'}] + doc.list_field = list_value + self.assertEquals(doc._get_changed_fields(), ['list_field']) + self.assertEquals(doc._delta(), ({'list_field': list_value}, {})) + + # Test unsetting + doc._changed_fields = [] + doc.dict_field = {} + self.assertEquals(doc._get_changed_fields(), ['dict_field']) + self.assertEquals(doc._delta(), ({}, {'dict_field': 1})) + + doc._changed_fields = [] + doc.list_field = [] + self.assertEquals(doc._get_changed_fields(), ['list_field']) + self.assertEquals(doc._delta(), ({}, {'list_field': 1})) + + def test_delta_recursive(self): + """Testing deltaing works with dynamic documents""" + class Embedded(DynamicEmbeddedDocument): + pass + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + doc.save() + + doc = Doc.objects.first() + self.assertEquals(doc._get_changed_fields(), []) + self.assertEquals(doc._delta(), ({}, {})) + + embedded_1 = Embedded() + embedded_1.string_field = 'hello' + embedded_1.int_field = 1 + embedded_1.dict_field = {'hello': 'world'} + embedded_1.list_field = ['1', 2, {'hello': 'world'}] + doc.embedded_field = embedded_1 + + self.assertEquals(doc._get_changed_fields(), ['embedded_field']) + + embedded_delta = { + '_types': ['Embedded'], + '_cls': 'Embedded', + 'string_field': 'hello', + 'int_field': 1, + 'dict_field': {'hello': 'world'}, + 'list_field': ['1', 2, {'hello': 'world'}] + } + self.assertEquals(doc.embedded_field._delta(), (embedded_delta, {})) + self.assertEquals(doc._delta(), ({'embedded_field': embedded_delta}, {})) + + doc.save() + doc.reload() + + doc.embedded_field.dict_field = {} + self.assertEquals(doc._get_changed_fields(), ['embedded_field.dict_field']) + self.assertEquals(doc.embedded_field._delta(), ({}, {'dict_field': 1})) + + self.assertEquals(doc._delta(), ({}, {'embedded_field.dict_field': 1})) + doc.save() + doc.reload() + + doc.embedded_field.list_field = [] + self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field']) + self.assertEquals(doc.embedded_field._delta(), ({}, {'list_field': 1})) + self.assertEquals(doc._delta(), ({}, {'embedded_field.list_field': 1})) + doc.save() + doc.reload() + + embedded_2 = Embedded() + embedded_2.string_field = 'hello' + embedded_2.int_field = 1 + embedded_2.dict_field = {'hello': 'world'} + embedded_2.list_field = ['1', 2, {'hello': 'world'}] + + doc.embedded_field.list_field = ['1', 2, embedded_2] + self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field']) + self.assertEquals(doc.embedded_field._delta(), ({ + 'list_field': ['1', 2, { + '_cls': 'Embedded', + '_types': ['Embedded'], + 'string_field': 'hello', + 'dict_field': {'hello': 'world'}, + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], + }] + }, {})) + + self.assertEquals(doc._delta(), ({ + 'embedded_field.list_field': ['1', 2, { + '_cls': 'Embedded', + '_types': ['Embedded'], + 'string_field': 'hello', + 'dict_field': {'hello': 'world'}, + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], + }] + }, {})) + doc.save() + doc.reload() + + self.assertEquals(doc.embedded_field.list_field[2]._changed_fields, []) + self.assertEquals(doc.embedded_field.list_field[0], '1') + self.assertEquals(doc.embedded_field.list_field[1], 2) + for k in doc.embedded_field.list_field[2]._fields: + self.assertEquals(doc.embedded_field.list_field[2][k], embedded_2[k]) + + doc.embedded_field.list_field[2].string_field = 'world' + self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field.2.string_field']) + self.assertEquals(doc.embedded_field._delta(), ({'list_field.2.string_field': 'world'}, {})) + self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.string_field': 'world'}, {})) + doc.save() + doc.reload() + self.assertEquals(doc.embedded_field.list_field[2].string_field, 'world') + + # Test multiple assignments + doc.embedded_field.list_field[2].string_field = 'hello world' + doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] + self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field']) + self.assertEquals(doc.embedded_field._delta(), ({ + 'list_field': ['1', 2, { + '_types': ['Embedded'], + '_cls': 'Embedded', + 'string_field': 'hello world', + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], + 'dict_field': {'hello': 'world'}}]}, {})) + self.assertEquals(doc._delta(), ({ + 'embedded_field.list_field': ['1', 2, { + '_types': ['Embedded'], + '_cls': 'Embedded', + 'string_field': 'hello world', + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], + 'dict_field': {'hello': 'world'}} + ]}, {})) + doc.save() + doc.reload() + self.assertEquals(doc.embedded_field.list_field[2].string_field, 'hello world') + + # Test list native methods + doc.embedded_field.list_field[2].list_field.pop(0) + self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}]}, {})) + doc.save() + doc.reload() + + doc.embedded_field.list_field[2].list_field.append(1) + self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}, 1]}, {})) + doc.save() + doc.reload() + self.assertEquals(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) + + doc.embedded_field.list_field[2].list_field.sort() + doc.save() + doc.reload() + self.assertEquals(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) + + del(doc.embedded_field.list_field[2].list_field[2]['hello']) + self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) + doc.save() + doc.reload() + + del(doc.embedded_field.list_field[2].list_field) + self.assertEquals(doc._delta(), ({}, {'embedded_field.list_field.2.list_field': 1})) + + doc.save() + doc.reload() + + doc.dict_field = {'embedded': embedded_1} + doc.save() + doc.reload() + + doc.dict_field['embedded'].string_field = 'Hello World' + self.assertEquals(doc._get_changed_fields(), ['dict_field.embedded.string_field']) + self.assertEquals(doc._delta(), ({'dict_field.embedded.string_field': 'Hello World'}, {})) From 823cf421fac4bd1a20b3b10bd4347b74e7bcf857 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 29 Sep 2011 14:07:30 -0700 Subject: [PATCH 0369/1279] Fixes to circular references. Removes infinite looping refs #294 --- mongoengine/dereference.py | 6 ++-- tests/dereference.py | 59 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 7fe9ba2..949bb2f 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -33,13 +33,15 @@ class DeReference(object): items = [i for i in items] self.max_depth = max_depth - + doc_type = None if instance and instance._fields: doc_type = instance._fields[name].field - + if isinstance(doc_type, ReferenceField): doc_type = doc_type.document_type + if all([i.__class__ == doc_type for i in items]): + return items self.reference_map = self._find_references(items) self.object_map = self._fetch_objects(doc_type=doc_type) diff --git a/tests/dereference.py b/tests/dereference.py index a98267f..b85ca17 100644 --- a/tests/dereference.py +++ b/tests/dereference.py @@ -129,6 +129,65 @@ class FieldTest(unittest.TestCase): self.assertEquals(employee.friends, friends) self.assertEqual(q, 2) + def test_circular_reference(self): + """Ensure you can handle circular references + """ + class Person(Document): + name = StringField() + relations = ListField(EmbeddedDocumentField('Relation')) + + def __repr__(self): + return "" % self.name + + class Relation(EmbeddedDocument): + name = StringField() + person = ReferenceField('Person') + + Person.drop_collection() + mother = Person(name="Mother") + daughter = Person(name="Daughter") + + mother.save() + daughter.save() + + daughter_rel = Relation(name="Daughter", person=daughter) + mother.relations.append(daughter_rel) + mother.save() + + mother_rel = Relation(name="Daughter", person=mother) + self_rel = Relation(name="Self", person=daughter) + daughter.relations.append(mother_rel) + daughter.relations.append(self_rel) + daughter.save() + + self.assertEquals("[, ]", "%s" % Person.objects()) + + def test_circular_reference_on_self(self): + """Ensure you can handle circular references + """ + class Person(Document): + name = StringField() + relations = ListField(ReferenceField('self')) + + def __repr__(self): + return "" % self.name + + Person.drop_collection() + mother = Person(name="Mother") + daughter = Person(name="Daughter") + + mother.save() + daughter.save() + + mother.relations.append(daughter) + mother.save() + + daughter.relations.append(mother) + daughter.relations.append(daughter) + daughter.save() + + self.assertEquals("[, ]", "%s" % Person.objects()) + def test_generic_reference(self): class UserA(Document): From 08288e591cf39c635c26d5f566ecd467f600b258 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Fri, 30 Sep 2011 08:22:33 +0300 Subject: [PATCH 0370/1279] small optimizing fix --- mongoengine/document.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 81c288e..e9b8871 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -81,10 +81,10 @@ class Document(BaseDocument): @classmethod def _get_collection(self): """Returns the collection for the document.""" - db = _get_db() - collection_name = self._get_collection_name() if not hasattr(self, '_collection') or self._collection is None: + db = _get_db() + collection_name = self._get_collection_name() # Create collection as a capped collection if specified if self._meta['max_size'] or self._meta['max_documents']: # Get max document limit and max byte size from meta From 1a2c74391cedcd257f767050f75023c4e4680767 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Oct 2011 05:18:25 -0700 Subject: [PATCH 0371/1279] Added grubberr to AUTHORS [Refs #296] --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index b342830..0332f12 100644 --- a/AUTHORS +++ b/AUTHORS @@ -67,3 +67,4 @@ that much better: * Gareth Lloyd * Albert Choi * John Arnfield + * grubberr From 9b4d0f6450b4bea8a2607334be046837eb2d4090 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Fri, 30 Sep 2011 16:58:36 +0200 Subject: [PATCH 0372/1279] Make sure that ListFields are not strings --- mongoengine/fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index fa99352..705bf3a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -472,7 +472,8 @@ class ListField(ComplexBaseField): def validate(self, value): """Make sure that a list of valid fields is being used. """ - if not isinstance(value, (list, tuple)): + if (not isinstance(value, (list, tuple)) or + isinstance(value, basestring)): raise ValidationError('Only lists and tuples may be used in a ' 'list field') super(ListField, self).validate(value) From 60b6ad3fcfc89f66ab2dacb02542d416947cd5a9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Oct 2011 05:30:23 -0700 Subject: [PATCH 0373/1279] Added test for listfield fix Added Pau Aliagas to authors [closes #299] --- AUTHORS | 1 + docs/changelog.rst | 1 + tests/fields.py | 14 ++++++++++++++ 3 files changed, 16 insertions(+) diff --git a/AUTHORS b/AUTHORS index 0332f12..8af1813 100644 --- a/AUTHORS +++ b/AUTHORS @@ -68,3 +68,4 @@ that much better: * Albert Choi * John Arnfield * grubberr + * Paul Aliagas diff --git a/docs/changelog.rst b/docs/changelog.rst index 904eb20..b2ef802 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed ListField so it doesnt accept strings - Added DynamicDocument and EmbeddedDynamicDocument classes for expando schemas Changes in v0.5 diff --git a/tests/fields.py b/tests/fields.py index dc53eae..fd99316 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -515,6 +515,20 @@ class FieldTest(unittest.TestCase): Simple.drop_collection() + + def test_list_field_rejects_strings(self): + """Strings aren't valid list field data types""" + + class Simple(Document): + mapping = ListField() + + Simple.drop_collection() + e = Simple() + e.mapping = 'hello world' + + self.assertRaises(ValidationError, e.save) + + def test_list_field_complex(self): """Ensure that the list fields can handle the complex types.""" From d99c7c20cc049f230fa619e42119b7eada5ba4fd Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Mon, 3 Oct 2011 16:42:10 +0200 Subject: [PATCH 0374/1279] Don't allow empty lists when they are required When using ListField, an empty list is added as the default value. But when you mark this field as required, you expect it not to be empty, so this patch makes sure that this is duly checked. --- mongoengine/fields.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 705bf3a..933ab4d 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -476,6 +476,10 @@ class ListField(ComplexBaseField): isinstance(value, basestring)): raise ValidationError('Only lists and tuples may be used in a ' 'list field') + # don't allow empty lists when they are required + if self.required and not value: + raise ValidationError('Field "%s" is required and cannot be empty' % + self.name) super(ListField, self).validate(value) def prepare_query_value(self, op, value): From bec6805296de8f85d5062d6540b8f6439383332a Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Sat, 1 Oct 2011 10:17:50 +0200 Subject: [PATCH 0375/1279] Add UUIDField --- mongoengine/fields.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 705bf3a..a200822 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -13,6 +13,7 @@ import pymongo.binary import datetime, time import decimal import gridfs +import uuid __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', @@ -21,7 +22,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DecimalField', 'ComplexDateTimeField', 'URLField', 'GenericReferenceField', 'FileField', 'BinaryField', 'SortedListField', 'EmailField', 'GeoPointField', - 'SequenceField', 'GenericEmbeddedDocumentField'] + 'SequenceField', 'UUIDField', 'GenericEmbeddedDocumentField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -978,3 +979,30 @@ class SequenceField(IntField): if value is None: value = self.generate_new_value() return value + + +class UUIDField(BaseField): + """A UUID field. + + .. versionadded:: 0.6 + """ + + def __init__(self, **kwargs): + super(UUIDField, self).__init__(**kwargs) + + def to_python(self, value): + if not isinstance(value, basestring): + value = unicode(value) + return uuid.UUID(value) + + def to_mongo(self, value): + return unicode(value) + + def validate(self, value): + if not isinstance(value, uuid.UUID): + if not isinstance(value, basestring): + value = str(value) + try: + value = uuid.UUID(value) + except Exception, exc: + raise ValidationError('Could not convert to UUID: %s' % exc) From 09f9c59b3d1a036eda0bf704de20c460c56b9ad9 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 4 Oct 2011 10:24:44 +0200 Subject: [PATCH 0376/1279] Add some more files to ignore in .gitignore --- .gitignore | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 315674f..0300bc7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ .* !.gitignore -*.pyc -.*.swp +*~ +*.py[co] +.*.sw[po] *.egg docs/.build docs/_build From 2f26f7a827ad2be25e872a46cba8110607cbfefb Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 4 Oct 2011 10:33:26 +0200 Subject: [PATCH 0377/1279] Add dependencies to spec file Add spec file for rpm-based systems --- python-mongoengine.spec | 53 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 python-mongoengine.spec diff --git a/python-mongoengine.spec b/python-mongoengine.spec new file mode 100644 index 0000000..054993d --- /dev/null +++ b/python-mongoengine.spec @@ -0,0 +1,53 @@ +# sitelib for noarch packages, sitearch for others (remove the unneeded one) +%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} +%{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} + +%define srcname mongoengine + +Name: python-%{srcname} +Version: 0.5.0 +Release: 2%{?dist} +Summary: A Python Document-Object Mapper for working with MongoDB + +Group: Development/Libraries +License: MIT +URL: https://github.com/namlook/mongoengine +Source0: %{srcname}-%{version}.tar.bz2 + +BuildRequires: python-devel +BuildRequires: python-setuptools + +Requires: mongodb +Requires: pymongo +Requires: python-blinker + +%description +MongoEngine is an ORM-like layer on top of PyMongo. + +%prep +%setup -q -n %{srcname}-%{version} + + +%build +# Remove CFLAGS=... for noarch packages (unneeded) +CFLAGS="$RPM_OPT_FLAGS" %{__python} setup.py build + + +%install +rm -rf $RPM_BUILD_ROOT +%{__python} setup.py install -O1 --skip-build --root $RPM_BUILD_ROOT + +%clean +rm -rf $RPM_BUILD_ROOT + +%files +%defattr(-,root,root,-) +%doc docs AUTHORS LICENSE README.rst +# For noarch packages: sitelib + %{python_sitelib}/* +# For arch-specific packages: sitearch +# %{python_sitearch}/* + +%changelog +* Fri Sep 23 2011 Pau Aliagas 0.5.0-1 +- Initial version From 17728d4e7423780f02a959716fe3d50db9cb6b75 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Oct 2011 02:57:50 -0700 Subject: [PATCH 0378/1279] Added tests for empty lists --- tests/fields.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/fields.py b/tests/fields.py index fd99316..802b02d 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -485,7 +485,6 @@ class FieldTest(unittest.TestCase): post.info = [{'test': 3}] post.save() - self.assertEquals(BlogPost.objects.count(), 3) self.assertEquals(BlogPost.objects.filter(info__exact='test').count(), 1) self.assertEquals(BlogPost.objects.filter(info__0__test='test').count(), 1) @@ -515,7 +514,6 @@ class FieldTest(unittest.TestCase): Simple.drop_collection() - def test_list_field_rejects_strings(self): """Strings aren't valid list field data types""" @@ -528,6 +526,17 @@ class FieldTest(unittest.TestCase): self.assertRaises(ValidationError, e.save) + def test_list_field_required(self): + """Ensure required cant be None / Empty""" + + class Simple(Document): + mapping = ListField(required=True) + + Simple.drop_collection() + e = Simple() + e.mapping = [] + + self.assertRaises(ValidationError, e.save) def test_list_field_complex(self): """Ensure that the list fields can handle the complex types.""" From 94ae1388b15d2b3d253b804b30e6057145062c79 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Oct 2011 02:59:00 -0700 Subject: [PATCH 0379/1279] Updated .gitignore --- .gitignore | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 315674f..0300bc7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ .* !.gitignore -*.pyc -.*.swp +*~ +*.py[co] +.*.sw[po] *.egg docs/.build docs/_build From 6d70ef1a08755e576cf605df6600071d1bff9c68 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Oct 2011 03:18:39 -0700 Subject: [PATCH 0380/1279] Updated changelog [#304] --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b2ef802..830c019 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added spec file for rpm-based distributions - Fixed ListField so it doesnt accept strings - Added DynamicDocument and EmbeddedDynamicDocument classes for expando schemas From 6961a9494f156769c8e3a5d20977eff98f15b0c2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Oct 2011 04:26:56 -0700 Subject: [PATCH 0381/1279] Updates to ComplexFields Required now means they cannot be empty [#302] --- docs/changelog.rst | 1 + mongoengine/base.py | 5 +++++ mongoengine/fields.py | 10 ++++++---- tests/fields.py | 11 ++++++++++- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b2ef802..e76b441 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Updated ComplexFields so if required they won't accept empty lists / dicts - Fixed ListField so it doesnt accept strings - Added DynamicDocument and EmbeddedDynamicDocument classes for expando schemas diff --git a/mongoengine/base.py b/mongoengine/base.py index 40953f1..adf5eee 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -282,6 +282,11 @@ class ComplexBaseField(BaseField): raise ValidationError('Invalid %s item (%s)' % ( self.field.__class__.__name__, str(v))) + # Don't allow empty values if required + if self.required and not value: + raise ValidationError('Field "%s" is required and cannot be empty' % + self.name) + def prepare_query_value(self, op, value): return self.to_mongo(value) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 933ab4d..e75890f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -459,6 +459,9 @@ class GenericEmbeddedDocumentField(BaseField): class ListField(ComplexBaseField): """A list field that wraps a standard field, allowing multiple instances of the field to be used as a list in the database. + + .. note:: + Required means it cannot be empty - as the default for ListFields is [] """ # ListFields cannot be indexed with _types - MongoDB doesn't support this @@ -476,10 +479,6 @@ class ListField(ComplexBaseField): isinstance(value, basestring)): raise ValidationError('Only lists and tuples may be used in a ' 'list field') - # don't allow empty lists when they are required - if self.required and not value: - raise ValidationError('Field "%s" is required and cannot be empty' % - self.name) super(ListField, self).validate(value) def prepare_query_value(self, op, value): @@ -517,6 +516,9 @@ class DictField(ComplexBaseField): """A dictionary field that wraps a standard Python dictionary. This is similar to an embedded document, but the structure is not defined. + .. note:: + Required means it cannot be empty - as the default for ListFields is [] + .. versionadded:: 0.3 .. versionchanged:: 0.5 - Can now handle complex / varying types of data """ diff --git a/tests/fields.py b/tests/fields.py index 802b02d..b182502 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -526,7 +526,7 @@ class FieldTest(unittest.TestCase): self.assertRaises(ValidationError, e.save) - def test_list_field_required(self): + def test_complex_field_required(self): """Ensure required cant be None / Empty""" class Simple(Document): @@ -538,6 +538,15 @@ class FieldTest(unittest.TestCase): self.assertRaises(ValidationError, e.save) + class Simple(Document): + mapping = DictField(required=True) + + Simple.drop_collection() + e = Simple() + e.mapping = {} + + self.assertRaises(ValidationError, e.save) + def test_list_field_complex(self): """Ensure that the list fields can handle the complex types.""" From dca135190a7a04b7d1254b66b779c7d45a1911cf Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Oct 2011 04:28:30 -0700 Subject: [PATCH 0382/1279] Fixed changelog --- docs/changelog.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a4a7fa9..71f8e51 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,11 +5,8 @@ Changelog Changes in dev ============== -<<<<<<< HEAD -- Added spec file for rpm-based distributions -======= - Updated ComplexFields so if required they won't accept empty lists / dicts ->>>>>>> listfield +- Added spec file for rpm-based distributions - Fixed ListField so it doesnt accept strings - Added DynamicDocument and EmbeddedDynamicDocument classes for expando schemas From 0a03f9a31ad7ab412e6ff5ab251aadb4d3296db7 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 4 Oct 2011 15:59:56 +0200 Subject: [PATCH 0383/1279] Add unit tests for UUIDField --- tests/fields.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/fields.py b/tests/fields.py index fd99316..6c183b4 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1,9 +1,7 @@ import unittest import datetime from decimal import Decimal - -import pymongo -import gridfs +import uuid from mongoengine import * from mongoengine.connection import _get_db @@ -175,6 +173,26 @@ class FieldTest(unittest.TestCase): person.admin = 'Yes' self.assertRaises(ValidationError, person.validate) + def test_uuid_validation(self): + """Ensure that invalid values cannot be assigned to UUID fields. + """ + class Person(Document): + api_key = UUIDField() + + person = Person() + # any uuid type is valid + person.api_key = uuid.uuid4() + person.validate() + person.api_key = uuid.uuid1() + person.validate() + + # last g cannot belong to an hex number + person.api_key = '9d159858-549b-4975-9f98-dd2f987c113g' + self.assertRaises(ValidationError, person.validate) + # short strings don't validate + person.api_key = '9d159858-549b-4975-9f98-dd2f987c113' + self.assertRaises(ValidationError, person.validate) + def test_datetime_validation(self): """Ensure that invalid values cannot be assigned to datetime fields. """ From 3aa2233b5de23a00a096a5e7cc87a964245bca0d Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 4 Oct 2011 18:35:32 +0200 Subject: [PATCH 0384/1279] Add field name to exception messages --- mongoengine/base.py | 26 +++++++------ mongoengine/fields.py | 85 ++++++++++++++++++++++++++++--------------- 2 files changed, 70 insertions(+), 41 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index adf5eee..c51aa97 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -141,16 +141,18 @@ class BaseField(object): if self.choices is not None: option_keys = [option_key for option_key, option_value in self.choices] if value not in option_keys: - raise ValidationError("Value must be one of %s." % unicode(option_keys)) + raise ValidationError('Value must be one of %s ("%s")' % + (unicode(option_keys), self.name)) # check validation argument if self.validation is not None: if callable(self.validation): if not self.validation(value): - raise ValidationError('Value does not match custom' \ - 'validation method.') + raise ValidationError('Value does not match custom ' + 'validation method ("%s")' % self.name) else: - raise ValueError('validation argument must be a callable.') + raise ValueError('validation argument for "%s" must be a ' + 'callable.' % self.name) self.validate(value) @@ -207,8 +209,9 @@ class ComplexBaseField(BaseField): if isinstance(v, Document): # We need the id from the saved object to create the DBRef if v.pk is None: - raise ValidationError('You can only reference documents once ' - 'they have been saved to the database') + raise ValidationError('You can only reference ' + 'documents once they have been saved ' + 'to the database ("%s")' % self.name) collection = v._get_collection_name() value_dict[k] = pymongo.dbref.DBRef(collection, v.pk) elif hasattr(v, 'to_python'): @@ -247,8 +250,9 @@ class ComplexBaseField(BaseField): if isinstance(v, Document): # We need the id from the saved object to create the DBRef if v.pk is None: - raise ValidationError('You can only reference documents once ' - 'they have been saved to the database') + raise ValidationError('You can only reference ' + 'documents once they have been saved ' + 'to the database ("%s")' % self.name) # If its a document that is not inheritable it won't have # _types / _cls data so make it a generic reference allows @@ -279,8 +283,8 @@ class ComplexBaseField(BaseField): else: [self.field.validate(v) for v in value] except Exception, err: - raise ValidationError('Invalid %s item (%s)' % ( - self.field.__class__.__name__, str(v))) + raise ValidationError('Invalid %s item (%s) ("%s")' % ( + self.field.__class__.__name__, str(v), self.name)) # Don't allow empty values if required if self.required and not value: @@ -363,7 +367,7 @@ class ObjectIdField(BaseField): try: pymongo.objectid.ObjectId(unicode(value)) except: - raise ValidationError('Invalid Object ID') + raise ValidationError('Invalid Object ID ("%s")' % self.name) class DocumentMetaclass(type): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e75890f..937bcb3 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -43,13 +43,16 @@ class StringField(BaseField): assert isinstance(value, (str, unicode)) if self.max_length is not None and len(value) > self.max_length: - raise ValidationError('String value is too long') + raise ValidationError('String value is too long ("%s")' % + self.name) if self.min_length is not None and len(value) < self.min_length: - raise ValidationError('String value is too short') + raise ValidationError('String value is too short ("%s")' % + self.name) if self.regex is not None and self.regex.match(value) is None: - message = 'String value did not match validation regex' + message = 'String value did not match validation regex ("%s")' % \ + self.name raise ValidationError(message) def lookup_member(self, member_name): @@ -100,7 +103,8 @@ class URLField(StringField): def validate(self, value): if not URLField.URL_REGEX.match(value): - raise ValidationError('Invalid URL: %s' % value) + raise ValidationError('Invalid URL: %s ("%s")' % (value, + self.name)) if self.verify_exists: import urllib2 @@ -108,7 +112,8 @@ class URLField(StringField): request = urllib2.Request(value) response = urllib2.urlopen(request) except Exception, e: - message = 'This URL appears to be a broken link: %s' % e + message = 'This URL appears to be a broken link: %s ("%s")' % ( + e, self.name) raise ValidationError(message) @@ -126,7 +131,8 @@ class EmailField(StringField): def validate(self, value): if not EmailField.EMAIL_REGEX.match(value): - raise ValidationError('Invalid Mail-address: %s' % value) + raise ValidationError('Invalid Mail-address: %s ("%s")' % (value, + self.name)) class IntField(BaseField): @@ -144,13 +150,16 @@ class IntField(BaseField): try: value = int(value) except: - raise ValidationError('%s could not be converted to int' % value) + raise ValidationError('%s could not be converted to int ("%s")' % ( + value, self.name)) if self.min_value is not None and value < self.min_value: - raise ValidationError('Integer value is too small') + raise ValidationError('Integer value is too small ("%s")' % + self.name) if self.max_value is not None and value > self.max_value: - raise ValidationError('Integer value is too large') + raise ValidationError('Integer value is too large ("%s")' % + self.name) def prepare_query_value(self, op, value): return int(value) @@ -173,10 +182,12 @@ class FloatField(BaseField): assert isinstance(value, float) if self.min_value is not None and value < self.min_value: - raise ValidationError('Float value is too small') + raise ValidationError('Float value is too small ("%s")' % + self.name) if self.max_value is not None and value > self.max_value: - raise ValidationError('Float value is too large') + raise ValidationError('Float value is too large ("%s")' % + self.name) def prepare_query_value(self, op, value): return float(value) @@ -207,13 +218,16 @@ class DecimalField(BaseField): try: value = decimal.Decimal(value) except Exception, exc: - raise ValidationError('Could not convert to decimal: %s' % exc) + raise ValidationError('Could not convert value to decimal: %s' + '("%s")' % (exc, self.name)) if self.min_value is not None and value < self.min_value: - raise ValidationError('Decimal value is too small') + raise ValidationError('Decimal value is too small ("%s")' % + self.name) if self.max_value is not None and value > self.max_value: - raise ValidationError('Decimal value is too large') + raise ValidationError('Decimal value is too large ("%s")' % + self.name) class BooleanField(BaseField): @@ -360,8 +374,8 @@ class ComplexDateTimeField(StringField): def validate(self, value): if not isinstance(value, datetime.datetime): - raise ValidationError('Only datetime objects may used in a \ - ComplexDateTimeField') + raise ValidationError('Only datetime objects may used in a ' + 'ComplexDateTimeField ("%s")' % self.name) def to_python(self, value): return self._convert_from_string(value) @@ -382,7 +396,8 @@ class EmbeddedDocumentField(BaseField): if not isinstance(document_type, basestring): if not issubclass(document_type, EmbeddedDocument): raise ValidationError('Invalid embedded document class ' - 'provided to an EmbeddedDocumentField') + 'provided to an EmbeddedDocumentField ' + '("%s")' % self.name) self.document_type_obj = document_type super(EmbeddedDocumentField, self).__init__(**kwargs) @@ -412,7 +427,8 @@ class EmbeddedDocumentField(BaseField): # Using isinstance also works for subclasses of self.document if not isinstance(value, self.document_type): raise ValidationError('Invalid embedded document instance ' - 'provided to an EmbeddedDocumentField') + 'provided to an EmbeddedDocumentField ' + '("%s")' % self.name) self.document_type.validate(value) def lookup_member(self, member_name): @@ -442,7 +458,8 @@ class GenericEmbeddedDocumentField(BaseField): def validate(self, value): if not isinstance(value, EmbeddedDocument): raise ValidationError('Invalid embedded document instance ' - 'provided to an GenericEmbeddedDocumentField') + 'provided to an GenericEmbeddedDocumentField ' + '("%s")' % self.name) value.validate() @@ -478,7 +495,7 @@ class ListField(ComplexBaseField): if (not isinstance(value, (list, tuple)) or isinstance(value, basestring)): raise ValidationError('Only lists and tuples may be used in a ' - 'list field') + 'list field ("%s")' % self.name) super(ListField, self).validate(value) def prepare_query_value(self, op, value): @@ -535,11 +552,12 @@ class DictField(ComplexBaseField): """ if not isinstance(value, dict): raise ValidationError('Only dictionaries may be used in a ' - 'DictField') + 'DictField ("%s")' % self.name) if any(('.' in k or '$' in k) for k in value): raise ValidationError('Invalid dictionary key name - keys may not ' - 'contain "." or "$" characters') + 'contain "." or "$" characters ("%s")' % + self.name) super(DictField, self).validate(value) def lookup_member(self, member_name): @@ -654,7 +672,8 @@ class ReferenceField(BaseField): if isinstance(value, Document) and value.id is None: raise ValidationError('You can only reference documents once ' - 'they have been saved to the database') + 'they have been saved to the database ' + '("%s")' % self.name) def lookup_member(self, member_name): @@ -683,12 +702,14 @@ class GenericReferenceField(BaseField): def validate(self, value): if not isinstance(value, (Document, pymongo.dbref.DBRef)): - raise ValidationError('GenericReferences can only contain documents') + raise ValidationError('GenericReferences can only contain ' + 'documents ("%s")' % self.name) # We need the id from the saved object to create the DBRef if isinstance(value, Document) and value.id is None: raise ValidationError('You can only reference documents once ' - 'they have been saved to the database') + 'they have been saved to the database ' + '("%s")' % self.name) def dereference(self, value): doc_cls = get_document(value['_cls']) @@ -710,7 +731,8 @@ class GenericReferenceField(BaseField): id_ = document.id if id_ is None: raise ValidationError('You can only reference documents once ' - 'they have been saved to the database') + 'they have been saved to the database ' + '("%s")' % self.name) else: id_ = document @@ -742,7 +764,8 @@ class BinaryField(BaseField): assert isinstance(value, str) if self.max_bytes is not None and len(value) > self.max_bytes: - raise ValidationError('Binary value is too long') + raise ValidationError('Binary value is too long ("%s")' % + self.name) class GridFSError(Exception): @@ -917,13 +940,15 @@ class GeoPointField(BaseField): """ if not isinstance(value, (list, tuple)): raise ValidationError('GeoPointField can only accept tuples or ' - 'lists of (x, y)') + 'lists of (x, y) ("%s")' % self.name) if not len(value) == 2: - raise ValidationError('Value must be a two-dimensional point.') + raise ValidationError('Value must be a two-dimensional point ' + '("%s")' % self.name) if (not isinstance(value[0], (float, int)) and not isinstance(value[1], (float, int))): - raise ValidationError('Both values in point must be float or int.') + raise ValidationError('Both values in point must be float or int ' + '("%s")' % self.name) class SequenceField(IntField): From 219d316b4980dbec3192b82cd3cd9cf366b3f3ad Mon Sep 17 00:00:00 2001 From: Marc Tamlyn Date: Wed, 5 Oct 2011 13:26:57 +0100 Subject: [PATCH 0385/1279] Fix iteration on querysets. If iteration of a queryset was interrupted (by a break, or a caught error), the next iterator would start from the second element as the cursor had already moved to the first. This is fixed by adding a rewind into the __iter__ method. --- mongoengine/queryset.py | 1 + tests/queryset.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a662685..a90d759 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1342,6 +1342,7 @@ class QuerySet(object): raise OperationError(u'Update failed [%s]' % unicode(e)) def __iter__(self): + self.rewind() return self def _sub_js_fields(self, code): diff --git a/tests/queryset.py b/tests/queryset.py index f093f6a..5c7afe6 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -567,7 +567,13 @@ class QuerySetTest(unittest.TestCase): people1 = [person for person in queryset] people2 = [person for person in queryset] + # Check that it still works even if iteration is interrupted. + for person in queryset: + break + people3 = [person for person in queryset] + self.assertEqual(people1, people2) + self.assertEqual(people1, people3) def test_repr_iteration(self): """Ensure that QuerySet __repr__ can handle loops From 9a0a0b1bd4059db3369e2e3a2cf2bec4b503b10f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Sat, 8 Oct 2011 07:29:12 -0700 Subject: [PATCH 0386/1279] Ported bugfix for circular references regression Refs #294 --- docs/changelog.rst | 5 ++++ mongoengine/dereference.py | 6 ++-- tests/dereference.py | 59 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b7904bd..0b93c74 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,11 @@ Changelog ========= +Changes in v0.5.1 +================= + +- Circular reference bugfix + Changes in v0.5 =============== diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 7fe9ba2..949bb2f 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -33,13 +33,15 @@ class DeReference(object): items = [i for i in items] self.max_depth = max_depth - + doc_type = None if instance and instance._fields: doc_type = instance._fields[name].field - + if isinstance(doc_type, ReferenceField): doc_type = doc_type.document_type + if all([i.__class__ == doc_type for i in items]): + return items self.reference_map = self._find_references(items) self.object_map = self._fetch_objects(doc_type=doc_type) diff --git a/tests/dereference.py b/tests/dereference.py index a98267f..b85ca17 100644 --- a/tests/dereference.py +++ b/tests/dereference.py @@ -129,6 +129,65 @@ class FieldTest(unittest.TestCase): self.assertEquals(employee.friends, friends) self.assertEqual(q, 2) + def test_circular_reference(self): + """Ensure you can handle circular references + """ + class Person(Document): + name = StringField() + relations = ListField(EmbeddedDocumentField('Relation')) + + def __repr__(self): + return "" % self.name + + class Relation(EmbeddedDocument): + name = StringField() + person = ReferenceField('Person') + + Person.drop_collection() + mother = Person(name="Mother") + daughter = Person(name="Daughter") + + mother.save() + daughter.save() + + daughter_rel = Relation(name="Daughter", person=daughter) + mother.relations.append(daughter_rel) + mother.save() + + mother_rel = Relation(name="Daughter", person=mother) + self_rel = Relation(name="Self", person=daughter) + daughter.relations.append(mother_rel) + daughter.relations.append(self_rel) + daughter.save() + + self.assertEquals("[, ]", "%s" % Person.objects()) + + def test_circular_reference_on_self(self): + """Ensure you can handle circular references + """ + class Person(Document): + name = StringField() + relations = ListField(ReferenceField('self')) + + def __repr__(self): + return "" % self.name + + Person.drop_collection() + mother = Person(name="Mother") + daughter = Person(name="Daughter") + + mother.save() + daughter.save() + + mother.relations.append(daughter) + mother.save() + + daughter.relations.append(mother) + daughter.relations.append(daughter) + daughter.save() + + self.assertEquals("[, ]", "%s" % Person.objects()) + def test_generic_reference(self): class UserA(Document): From 591149b1f0c936b87a0ff7f75b4ee5353c8dddea Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Sat, 8 Oct 2011 07:31:24 -0700 Subject: [PATCH 0387/1279] Bumped version for 0.5.1 release --- mongoengine/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index dd3c517..48d2742 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -14,7 +14,7 @@ __all__ = (document.__all__ + fields.__all__ + connection.__all__ + __author__ = 'Harry Marr' -VERSION = (0, 5, 0) +VERSION = (0, 5, 1) def get_version(): From 268908b3b271a58090792a3d0ea10e63883caf15 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Sat, 8 Oct 2011 08:06:23 -0700 Subject: [PATCH 0388/1279] Improvements to .get() efficiency Closes #307 and #290 --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/queryset.py | 16 +++++++++------- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/AUTHORS b/AUTHORS index 8af1813..f3f0743 100644 --- a/AUTHORS +++ b/AUTHORS @@ -69,3 +69,4 @@ that much better: * John Arnfield * grubberr * Paul Aliagas + * Paul Cunnane diff --git a/docs/changelog.rst b/docs/changelog.rst index c458ec6..41432b3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Changes in dev ============== +- Improved efficiency of .get() - Updated ComplexFields so if required they won't accept empty lists / dicts - Added spec file for rpm-based distributions - Fixed ListField so it doesnt accept strings diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 2b44dc4..e3d2b47 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -728,15 +728,17 @@ class QuerySet(object): .. versionadded:: 0.3 """ self.__call__(*q_objs, **query) - count = self.count() - if count == 1: - return self[0] - elif count > 1: - message = u'%d items returned, instead of 1' % count - raise self._document.MultipleObjectsReturned(message) - else: + try: + result1 = self[0] + except IndexError: raise self._document.DoesNotExist("%s matching query does not exist." % self._document._class_name) + try: + result2 = self[1] + except IndexError: + return result1 + message = u'%d items returned, instead of 1' % self.count() + raise self._document.MultipleObjectsReturned(message) def get_or_create(self, write_options=None, *q_objs, **query): """Retrieve unique object or create, if it doesn't exist. Returns a tuple of From 87975656066ed8d5059060f42aba89483550d1e3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Sat, 8 Oct 2011 08:13:53 -0700 Subject: [PATCH 0389/1279] UPdated changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 41432b3..1533101 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,8 @@ Changelog Changes in dev ============== +- Add field name to validation exception messages +- Added UUID field - Improved efficiency of .get() - Updated ComplexFields so if required they won't accept empty lists / dicts - Added spec file for rpm-based distributions From 7b1860d17b0d60bd11aaace9052bd58a2d702cbf Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 10 Oct 2011 09:16:32 -0700 Subject: [PATCH 0390/1279] Fixes tree based circular references Thanks to jpfarias for the fix. Also normalised the other circular checks. --- docs/changelog.rst | 3 ++- mongoengine/base.py | 27 +++++++++++++++++-------- mongoengine/document.py | 10 ++++----- tests/dereference.py | 45 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 14 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1533101..abbc1e4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Changes in dev ============== +- Fixed tree based circular reference bug - Add field name to validation exception messages - Added UUID field - Improved efficiency of .get() @@ -17,7 +18,7 @@ Changes in dev Changes in v0.5.1 ================= -- Circular reference bugfix +- Fixed simple circular reference bug Changes in v0.5 =============== diff --git a/mongoengine/base.py b/mongoengine/base.py index adf5eee..e8a3fe5 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -837,13 +837,19 @@ class BaseDocument(object): if hasattr(self, '_changed_fields') and key not in self._changed_fields: self._changed_fields.append(key) - def _get_changed_fields(self, key=''): + def _get_changed_fields(self, key='', inspected=[]): """Returns a list of all fields that have explicitly been changed. """ from mongoengine import EmbeddedDocument, DynamicEmbeddedDocument _changed_fields = [] _changed_fields += getattr(self, '_changed_fields', []) + inspected = inspected or [] + if hasattr(self, 'id'): + if self.id in inspected: + return _changed_fields + inspected.append(self.id) + field_list = self._fields.copy() if self._dynamic: field_list.update(self._dynamic_fields) @@ -852,8 +858,13 @@ class BaseDocument(object): db_field_name = self._db_field_map.get(field_name, field_name) key = '%s.' % db_field_name field = getattr(self, field_name, None) + if hasattr(field, 'id'): + if field.id in inspected: + continue + inspected.append(field.id) + if isinstance(field, (EmbeddedDocument, DynamicEmbeddedDocument)) and db_field_name not in _changed_fields: # Grab all embedded fields that have been changed - _changed_fields += ["%s%s" % (key, k) for k in field._get_changed_fields(key) if k] + _changed_fields += ["%s%s" % (key, k) for k in field._get_changed_fields(key, inspected) if k] elif isinstance(field, (list, tuple, dict)) and db_field_name not in _changed_fields: # Loop list / dict fields as they contain documents # Determine the iterator to use if not hasattr(field, 'items'): @@ -864,7 +875,7 @@ class BaseDocument(object): if not hasattr(value, '_get_changed_fields'): continue list_key = "%s%s." % (key, index) - _changed_fields += ["%s%s" % (list_key, k) for k in value._get_changed_fields(list_key) if k] + _changed_fields += ["%s%s" % (list_key, k) for k in value._get_changed_fields(list_key, inspected) if k] return _changed_fields def _delta(self): @@ -941,17 +952,17 @@ class BaseDocument(object): return set_data, unset_data @classmethod - def _geo_indices(cls, inspected_classes=None): - inspected_classes = inspected_classes or [] + def _geo_indices(cls, inspected=None): + inspected = inspected or [] geo_indices = [] - inspected_classes.append(cls) + inspected.append(cls) for field in cls._fields.values(): if hasattr(field, 'document_type'): field_cls = field.document_type - if field_cls in inspected_classes: + if field_cls in inspected: continue if hasattr(field_cls, '_geo_indices'): - geo_indices += field_cls._geo_indices(inspected_classes) + geo_indices += field_cls._geo_indices(inspected) elif field._geo_index: geo_indices.append(field) return geo_indices diff --git a/mongoengine/document.py b/mongoengine/document.py index e9b8871..ce001d2 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -185,18 +185,18 @@ class Document(BaseDocument): id_field = self._meta['id_field'] self[id_field] = self._fields[id_field].to_python(object_id) - def reset_changed_fields(doc, inspected_docs=None): + def reset_changed_fields(doc, inspected=None): """Loop through and reset changed fields lists""" - inspected_docs = inspected_docs or [] - inspected_docs.append(doc) + inspected = inspected or [] + inspected.append(doc) if hasattr(doc, '_changed_fields'): doc._changed_fields = [] for field_name in doc._fields: field = getattr(doc, field_name) - if field not in inspected_docs and hasattr(field, '_changed_fields'): - reset_changed_fields(field, inspected_docs) + if field not in inspected and hasattr(field, '_changed_fields'): + reset_changed_fields(field, inspected) reset_changed_fields(self) self._changed_fields = [] diff --git a/tests/dereference.py b/tests/dereference.py index b85ca17..088db98 100644 --- a/tests/dereference.py +++ b/tests/dereference.py @@ -188,6 +188,51 @@ class FieldTest(unittest.TestCase): self.assertEquals("[, ]", "%s" % Person.objects()) + def test_circular_tree_reference(self): + """Ensure you can handle circular references with more than one level + """ + class Other(EmbeddedDocument): + name = StringField() + friends = ListField(ReferenceField('Person')) + + class Person(Document): + name = StringField() + other = EmbeddedDocumentField(Other, default=lambda: Other()) + + def __repr__(self): + return "" % self.name + + Person.drop_collection() + paul = Person(name="Paul") + paul.save() + maria = Person(name="Maria") + maria.save() + julia = Person(name='Julia') + julia.save() + anna = Person(name='Anna') + anna.save() + + paul.other.friends = [maria, julia, anna] + paul.other.name = "Paul's friends" + paul.save() + + maria.other.friends = [paul, julia, anna] + maria.other.name = "Maria's friends" + maria.save() + + julia.other.friends = [paul, maria, anna] + julia.other.name = "Julia's friends" + julia.save() + + anna.other.friends = [paul, maria, julia] + anna.other.name = "Anna's friends" + anna.save() + + self.assertEquals( + "[, , , ]", + "%s" % Person.objects() + ) + def test_generic_reference(self): class UserA(Document): From 0624cdd6e419bf717ceb3c5c3bb108c82a953cc8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 11 Oct 2011 02:26:33 -0700 Subject: [PATCH 0391/1279] Fixes collection creation post drop_collection Thanks to Julien Rebetez for the original patch closes [#285] --- docs/changelog.rst | 1 + mongoengine/document.py | 4 ++-- mongoengine/queryset.py | 13 +++++++++++-- tests/document.py | 15 +++++++++++++++ tests/queryset.py | 16 +++++++++------- 5 files changed, 38 insertions(+), 11 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index abbc1e4..fab2041 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changelog Changes in dev ============== +- Fixed calling a queryset after drop_collection now recreates the collection - Fixed tree based circular reference bug - Add field name to validation exception messages - Added UUID field diff --git a/mongoengine/document.py b/mongoengine/document.py index ce001d2..a87f460 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -36,7 +36,6 @@ class EmbeddedDocument(BaseDocument): super(EmbeddedDocument, self).__delattr__(*args, **kwargs) - class Document(BaseDocument): """The base class used for defining the structure and properties of collections of documents stored in MongoDB. Inherit from this class, and @@ -81,7 +80,6 @@ class Document(BaseDocument): @classmethod def _get_collection(self): """Returns the collection for the document.""" - if not hasattr(self, '_collection') or self._collection is None: db = _get_db() collection_name = self._get_collection_name() @@ -291,8 +289,10 @@ class Document(BaseDocument): """Drops the entire collection associated with this :class:`~mongoengine.Document` type from the database. """ + from mongoengine.queryset import QuerySet db = _get_db() db.drop_collection(cls._get_collection_name()) + QuerySet._reset_already_indexed(cls) class DynamicDocument(Document): diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index e3d2b47..4c88ba2 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -434,9 +434,11 @@ class QuerySet(object): return spec @classmethod - def _reset_already_indexed(cls): + def _reset_already_indexed(cls, document=None): """Helper to reset already indexed, can be useful for testing purposes""" - cls.__already_indexed = set() + if document: + cls.__already_indexed.discard(document) + cls.__already_indexed.clear() def __call__(self, q_obj=None, class_check=True, slave_okay=False, **query): """Filter the selected documents by calling the @@ -476,6 +478,13 @@ class QuerySet(object): perform operations only if the collection is accessed. """ if self._document not in QuerySet.__already_indexed: + + # Ensure collection exists + db = _get_db() + if self._collection_obj.name not in db.collection_names(): + self._document._collection = None + self._collection_obj = self._document._get_collection() + QuerySet.__already_indexed.add(self._document) background = self._document._meta.get('index_background', False) diff --git a/tests/document.py b/tests/document.py index 1eeda46..816bb49 100644 --- a/tests/document.py +++ b/tests/document.py @@ -41,6 +41,21 @@ class DocumentTest(unittest.TestCase): self.Person.drop_collection() self.assertFalse(collection in self.db.collection_names()) + def test_queryset_resurrects_dropped_collection(self): + + self.Person.objects().item_frequencies('name') + self.Person.drop_collection() + + self.assertEqual({}, self.Person.objects().item_frequencies('name')) + + class Actor(self.Person): + pass + + # Ensure works correctly with inhertited classes + Actor.objects().item_frequencies('name') + self.Person.drop_collection() + self.assertEqual({}, Actor.objects().item_frequencies('name')) + def test_definition(self): """Ensure that document may be defined using fields. """ diff --git a/tests/queryset.py b/tests/queryset.py index de02387..92be1d3 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -15,7 +15,7 @@ class QuerySetTest(unittest.TestCase): def setUp(self): connect(db='mongoenginetest') - + class Person(Document): name = StringField() age = IntField() @@ -455,6 +455,9 @@ class QuerySetTest(unittest.TestCase): Blog.drop_collection() + # Recreates the collection + self.assertEqual(0, Blog.objects.count()) + with query_counter() as q: self.assertEqual(q, 0) @@ -468,10 +471,10 @@ class QuerySetTest(unittest.TestCase): blogs.append(Blog(title="post %s" % i, posts=[post1, post2])) Blog.objects.insert(blogs, load_bulk=False) - self.assertEqual(q, 2) # 1 for the inital connection and 1 for the insert + self.assertEqual(q, 1) # 1 for the insert Blog.objects.insert(blogs) - self.assertEqual(q, 4) # 1 for insert, and 1 for in bulk + self.assertEqual(q, 3) # 1 for insert, and 1 for in bulk fetch (3 in total) Blog.drop_collection() @@ -1840,7 +1843,6 @@ class QuerySetTest(unittest.TestCase): freq = Person.objects.item_frequencies('city', normalize=True, map_reduce=True) self.assertEquals(freq, {'CRB': 0.5, None: 0.5}) - def test_item_frequencies_with_null_embedded(self): class Data(EmbeddedDocument): name = StringField() @@ -2227,7 +2229,7 @@ class QuerySetTest(unittest.TestCase): events = Event.objects(location__within_box=box) self.assertEqual(events.count(), 1) self.assertEqual(events[0].id, event2.id) - + # check that polygon works for users who have a server >= 1.9 server_version = tuple( _get_connection().server_info()['version'].split('.') @@ -2244,7 +2246,7 @@ class QuerySetTest(unittest.TestCase): events = Event.objects(location__within_polygon=polygon) self.assertEqual(events.count(), 1) self.assertEqual(events[0].id, event1.id) - + polygon2 = [ (54.033586,-1.742249), (52.792797,-1.225891), @@ -2252,7 +2254,7 @@ class QuerySetTest(unittest.TestCase): ] events = Event.objects(location__within_polygon=polygon2) self.assertEqual(events.count(), 0) - + Event.drop_collection() def test_spherical_geospatial_operators(self): From 53598781b843980c1caa6ee8c05327e0c38df2dd Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 11 Oct 2011 12:44:41 +0200 Subject: [PATCH 0392/1279] Facepalm - mutable default argument in method.. --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 0cf57ec..f70d094 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -841,7 +841,7 @@ class BaseDocument(object): if hasattr(self, '_changed_fields') and key not in self._changed_fields: self._changed_fields.append(key) - def _get_changed_fields(self, key='', inspected=[]): + def _get_changed_fields(self, key='', inspected=None): """Returns a list of all fields that have explicitly been changed. """ from mongoengine import EmbeddedDocument, DynamicEmbeddedDocument From c4b0002ddb4bd6f287a488f541e38401fd8c6401 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 11 Oct 2011 14:59:58 +0200 Subject: [PATCH 0393/1279] Fixed typo --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index f70d094..a42ec44 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -654,7 +654,7 @@ class BaseDocument(object): for key, value in values.items(): setattr(self, key, value) - # Set any get_fieldname_display methodsF + # Set any get_fieldname_display methods self.__set_field_display() # Flag initialised self._initialised = True From 269e6e29d69587515f020288eb85e2e7b24f0a9d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 12 Oct 2011 00:18:12 -0700 Subject: [PATCH 0394/1279] Updated Authors --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index f3f0743..42ca1d5 100644 --- a/AUTHORS +++ b/AUTHORS @@ -70,3 +70,4 @@ that much better: * grubberr * Paul Aliagas * Paul Cunnane + * Julien Rebetez From 452bbcc19b2f003efb9050227455d2489e5182d2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 12 Oct 2011 00:30:12 -0700 Subject: [PATCH 0395/1279] Ported fix for Circular Reference bug to Master Ready for a 0.5.2 release --- AUTHORS | 2 ++ docs/changelog.rst | 5 +++++ mongoengine/base.py | 25 ++++++++++++++++++------ tests/dereference.py | 45 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 6 deletions(-) diff --git a/AUTHORS b/AUTHORS index b342830..dd86cc5 100644 --- a/AUTHORS +++ b/AUTHORS @@ -67,3 +67,5 @@ that much better: * Gareth Lloyd * Albert Choi * John Arnfield + * Julien Rebetez + diff --git a/docs/changelog.rst b/docs/changelog.rst index 0b93c74..aa48b5c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,11 @@ Changelog ========= +Changes in v0.5.2 +================= + +- A Robust Circular reference bugfix + Changes in v0.5.1 ================= diff --git a/mongoengine/base.py b/mongoengine/base.py index c4bcee1..113cab6 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -724,18 +724,32 @@ class BaseDocument(object): if hasattr(self, '_changed_fields') and key not in self._changed_fields: self._changed_fields.append(key) - def _get_changed_fields(self, key=''): + def _get_changed_fields(self, key='', inspected=None): """Returns a list of all fields that have explicitly been changed. """ from mongoengine import EmbeddedDocument _changed_fields = [] _changed_fields += getattr(self, '_changed_fields', []) - for field_name in self._fields: + + inspected = inspected or set() + if hasattr(self, 'id'): + if self.id in inspected: + return _changed_fields + inspected.add(self.id) + + field_list = self._fields.copy() + + for field_name in field_list: db_field_name = self._db_field_map.get(field_name, field_name) key = '%s.' % db_field_name field = getattr(self, field_name, None) - if isinstance(field, EmbeddedDocument) and db_field_name not in _changed_fields: # Grab all embedded fields that have been changed - _changed_fields += ["%s%s" % (key, k) for k in field._get_changed_fields(key) if k] + if hasattr(field, 'id'): + if field.id in inspected: + continue + inspected.add(field.id) + + if isinstance(field, (EmbeddedDocument,)) and db_field_name not in _changed_fields: # Grab all embedded fields that have been changed + _changed_fields += ["%s%s" % (key, k) for k in field._get_changed_fields(key, inspected) if k] elif isinstance(field, (list, tuple, dict)) and db_field_name not in _changed_fields: # Loop list / dict fields as they contain documents # Determine the iterator to use if not hasattr(field, 'items'): @@ -746,8 +760,7 @@ class BaseDocument(object): if not hasattr(value, '_get_changed_fields'): continue list_key = "%s%s." % (key, index) - _changed_fields += ["%s%s" % (list_key, k) for k in value._get_changed_fields(list_key) if k] - + _changed_fields += ["%s%s" % (list_key, k) for k in value._get_changed_fields(list_key, inspected) if k] return _changed_fields def _delta(self): diff --git a/tests/dereference.py b/tests/dereference.py index b85ca17..088db98 100644 --- a/tests/dereference.py +++ b/tests/dereference.py @@ -188,6 +188,51 @@ class FieldTest(unittest.TestCase): self.assertEquals("[, ]", "%s" % Person.objects()) + def test_circular_tree_reference(self): + """Ensure you can handle circular references with more than one level + """ + class Other(EmbeddedDocument): + name = StringField() + friends = ListField(ReferenceField('Person')) + + class Person(Document): + name = StringField() + other = EmbeddedDocumentField(Other, default=lambda: Other()) + + def __repr__(self): + return "" % self.name + + Person.drop_collection() + paul = Person(name="Paul") + paul.save() + maria = Person(name="Maria") + maria.save() + julia = Person(name='Julia') + julia.save() + anna = Person(name='Anna') + anna.save() + + paul.other.friends = [maria, julia, anna] + paul.other.name = "Paul's friends" + paul.save() + + maria.other.friends = [paul, julia, anna] + maria.other.name = "Maria's friends" + maria.save() + + julia.other.friends = [paul, maria, anna] + julia.other.name = "Julia's friends" + julia.save() + + anna.other.friends = [paul, maria, julia] + anna.other.name = "Anna's friends" + anna.save() + + self.assertEquals( + "[, , , ]", + "%s" % Person.objects() + ) + def test_generic_reference(self): class UserA(Document): From 4d5f602ee77bf03711bb43a6330a12399f0cb6d6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 12 Oct 2011 00:31:02 -0700 Subject: [PATCH 0396/1279] Bumped the version --- mongoengine/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 48d2742..b28c374 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -14,7 +14,7 @@ __all__ = (document.__all__ + fields.__all__ + connection.__all__ + __author__ = 'Harry Marr' -VERSION = (0, 5, 1) +VERSION = (0, 5, 2) def get_version(): From 79ecf027dd79e642baf623a54a961205a6c34783 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 4 Oct 2011 10:33:26 +0200 Subject: [PATCH 0397/1279] Add dependencies to spec file Add spec file for rpm-based systems --- python-mongoengine.spec | 53 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 python-mongoengine.spec diff --git a/python-mongoengine.spec b/python-mongoengine.spec new file mode 100644 index 0000000..054993d --- /dev/null +++ b/python-mongoengine.spec @@ -0,0 +1,53 @@ +# sitelib for noarch packages, sitearch for others (remove the unneeded one) +%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")} +%{!?python_sitearch: %global python_sitearch %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))")} + +%define srcname mongoengine + +Name: python-%{srcname} +Version: 0.5.0 +Release: 2%{?dist} +Summary: A Python Document-Object Mapper for working with MongoDB + +Group: Development/Libraries +License: MIT +URL: https://github.com/namlook/mongoengine +Source0: %{srcname}-%{version}.tar.bz2 + +BuildRequires: python-devel +BuildRequires: python-setuptools + +Requires: mongodb +Requires: pymongo +Requires: python-blinker + +%description +MongoEngine is an ORM-like layer on top of PyMongo. + +%prep +%setup -q -n %{srcname}-%{version} + + +%build +# Remove CFLAGS=... for noarch packages (unneeded) +CFLAGS="$RPM_OPT_FLAGS" %{__python} setup.py build + + +%install +rm -rf $RPM_BUILD_ROOT +%{__python} setup.py install -O1 --skip-build --root $RPM_BUILD_ROOT + +%clean +rm -rf $RPM_BUILD_ROOT + +%files +%defattr(-,root,root,-) +%doc docs AUTHORS LICENSE README.rst +# For noarch packages: sitelib + %{python_sitelib}/* +# For arch-specific packages: sitearch +# %{python_sitearch}/* + +%changelog +* Fri Sep 23 2011 Pau Aliagas 0.5.0-1 +- Initial version From 181e191feef7a2db0d5b180b57146c01eb8caba5 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Tue, 4 Oct 2011 10:24:44 +0200 Subject: [PATCH 0398/1279] Add some more files to ignore in .gitignore --- .gitignore | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 315674f..0300bc7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ .* !.gitignore -*.pyc -.*.swp +*~ +*.py[co] +.*.sw[po] *.egg docs/.build docs/_build From 3d817f145c0a5f719fc05497ff500ec5f6a2345e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Wed, 12 Oct 2011 18:28:40 -0300 Subject: [PATCH 0399/1279] fixes for #315 issue --- mongoengine/base.py | 10 ---------- mongoengine/document.py | 10 ++++++++++ setup.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 881dd32..ed14c74 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -739,16 +739,6 @@ class BaseDocument(object): elif field.required: raise ValidationError('Field "%s" is required' % field.name) - @apply - def pk(): - """Primary key alias - """ - def fget(self): - return getattr(self, self._meta['id_field']) - def fset(self, value): - return setattr(self, self._meta['id_field'], value) - return property(fget, fset) - def to_mongo(self): """Return data dictionary ready for use with MongoDB. """ diff --git a/mongoengine/document.py b/mongoengine/document.py index a87f460..82b94a3 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -76,6 +76,16 @@ class Document(BaseDocument): by setting index_types to False on the meta dictionary for the document. """ __metaclass__ = TopLevelDocumentMetaclass + + @apply + def pk(): + """Primary key alias + """ + def fget(self): + return getattr(self, self._meta['id_field']) + def fset(self, value): + return setattr(self, self._meta['id_field'], value) + return property(fget, fset) @classmethod def _get_collection(self): diff --git a/setup.py b/setup.py index 6877b62..d4dd5ab 100644 --- a/setup.py +++ b/setup.py @@ -47,5 +47,5 @@ setup(name='mongoengine', classifiers=CLASSIFIERS, install_requires=['pymongo'], test_suite='tests', - tests_require=['blinker', 'django==1.3'] + tests_require=['blinker', 'django>=1.3'] ) From 202d6e414f20b31d628403cad21c6275b82409d9 Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Wed, 12 Oct 2011 11:03:56 +0200 Subject: [PATCH 0400/1279] Update spec file --- python-mongoengine.spec | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 054993d..dea5e60 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,8 +5,8 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.5.0 -Release: 2%{?dist} +Version: 0.5.2 +Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB Group: Development/Libraries @@ -49,5 +49,8 @@ rm -rf $RPM_BUILD_ROOT # %{python_sitearch}/* %changelog +* Wed Oct 12 2011 Pau Aliagas 0.5.2-1 +- Update to latest version + * Fri Sep 23 2011 Pau Aliagas 0.5.0-1 - Initial version From b09c52fc7ec9b8cc5379797583bb1bb0e7f11b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Wed, 19 Oct 2011 06:30:41 -0200 Subject: [PATCH 0401/1279] fixes for #325 issue --- mongoengine/fields.py | 6 ++++++ setup.py | 2 +- tests/fields.py | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index abea212..a1638bf 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -666,6 +666,9 @@ class ReferenceField(BaseField): return pymongo.dbref.DBRef(collection, id_) def prepare_query_value(self, op, value): + if value is None: + return None + return self.to_mongo(value) def validate(self, value): @@ -743,6 +746,9 @@ class GenericReferenceField(BaseField): return {'_cls': document._class_name, '_ref': ref} def prepare_query_value(self, op, value): + if value is None: + return None + return self.to_mongo(value) diff --git a/setup.py b/setup.py index 6877b62..d4dd5ab 100644 --- a/setup.py +++ b/setup.py @@ -47,5 +47,5 @@ setup(name='mongoengine', classifiers=CLASSIFIERS, install_requires=['pymongo'], test_suite='tests', - tests_require=['blinker', 'django==1.3'] + tests_require=['blinker', 'django>=1.3'] ) diff --git a/tests/fields.py b/tests/fields.py index c95b544..4f1f13d 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -997,10 +997,16 @@ class FieldTest(unittest.TestCase): mongodb = Product(name='MongoDB', company=ten_gen) mongodb.save() + me = Product(name='MongoEngine') + me.save() + obj = Product.objects(company=ten_gen).first() self.assertEqual(obj, mongodb) self.assertEqual(obj.company, ten_gen) + obj = Product.objects(company=None).first() + self.assertEqual(obj, me) + def test_reference_query_conversion(self): """Ensure that ReferenceFields can be queried using objects and values of the type of the primary key of the referenced object. From 7f2b686ab54104549dc25872ccbb5950eea61182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Wed, 19 Oct 2011 06:55:05 -0200 Subject: [PATCH 0402/1279] added drop_collection for test --- tests/fields.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/fields.py b/tests/fields.py index 4f1f13d..2304671 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -992,6 +992,9 @@ class FieldTest(unittest.TestCase): class Company(Document): name = StringField() + Product.drop_collection() + Company.drop_collection() + ten_gen = Company(name='10gen') ten_gen.save() mongodb = Product(name='MongoDB', company=ten_gen) From f842c90007631c7f9d1a424e05c0543c50bb95c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Wed, 19 Oct 2011 06:57:39 -0200 Subject: [PATCH 0403/1279] Merge branches 'master' and 'fixes-325' into fixes-325 From 8e87648d53eac1caf786e47197cc79e79c2b3045 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Wed, 19 Oct 2011 09:44:49 -0200 Subject: [PATCH 0404/1279] added tests for get_or_create --- tests/fields.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/fields.py b/tests/fields.py index 2304671..80a343e 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1010,6 +1010,11 @@ class FieldTest(unittest.TestCase): obj = Product.objects(company=None).first() self.assertEqual(obj, me) + obj, created = Product.objects.get_or_create(company=None) + + self.assertEqual(created, False) + self.assertEqual(obj, me) + def test_reference_query_conversion(self): """Ensure that ReferenceFields can be queried using objects and values of the type of the primary key of the referenced object. From 3d7b30da777bd93affa81d9ed343e428ffb74bd3 Mon Sep 17 00:00:00 2001 From: Nicolas Perriault Date: Mon, 24 Oct 2011 01:02:31 +0200 Subject: [PATCH 0405/1279] first version of BC validation schema --- mongoengine/base.py | 131 +++++++++++++++++++++++++++--------------- mongoengine/fields.py | 120 +++++++++++++++----------------------- tests/fields.py | 61 +++++++++++++++----- 3 files changed, 177 insertions(+), 135 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 881dd32..96b0a58 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -4,7 +4,6 @@ from queryset import DO_NOTHING from mongoengine import signals -import weakref import sys import pymongo import pymongo.objectid @@ -19,9 +18,27 @@ class NotRegistered(Exception): class InvalidDocumentError(Exception): pass -class ValidationError(Exception): - pass +class ValidationError(Exception): + errors = {} + field_name = None + + def __init__(self, message, **kwargs): + self.field_name = kwargs.get('field_name') + super(ValidationError, self).__init__(message) + + def __str__(self): + return self.message + + def __repr__(self): + return '%s(%s,)' % (self.__class__.__name__, self.message) + + def __getattribute__(self, name): + message = super(ValidationError, self).__getattribute__(name) + if name == 'message' and self.field_name: + return message + ' ("%s")' % self.field_name + else: + return message _document_registry = {} @@ -50,6 +67,8 @@ class BaseField(object): .. versionchanged:: 0.5 - added verbose and help text """ + name = None + # Fields may have _types inserted into indexes by default _index_with_types = True _geo_index = False @@ -116,6 +135,14 @@ class BaseField(object): instance._data[self.name] = value instance._mark_as_changed(self.name) + def error(self, message, errors=None, field_name=None): + """Raises a ValidationError. + """ + field_name = field_name if field_name else self.name + error = ValidationError(message, field_name=field_name) + error.errors = errors if errors else {} + raise error + def to_python(self, value): """Convert a MongoDB-compatible type to a Python type. """ @@ -141,15 +168,13 @@ class BaseField(object): if self.choices is not None: option_keys = [option_key for option_key, option_value in self.choices] if value not in option_keys: - raise ValidationError('Value must be one of %s ("%s")' % - (unicode(option_keys), self.name)) + self.error('Value must be one of %s' % unicode(option_keys)) # check validation argument if self.validation is not None: if callable(self.validation): if not self.validation(value): - raise ValidationError('Value does not match custom ' - 'validation method ("%s")' % self.name) + self.error('Value does not match custom validation method') else: raise ValueError('validation argument for "%s" must be a ' 'callable.' % self.name) @@ -197,7 +222,7 @@ class ComplexBaseField(BaseField): if not hasattr(value, 'items'): try: is_list = True - value = dict([(k,v) for k,v in enumerate(value)]) + value = dict([(k, v) for k, v in enumerate(value)]) except TypeError: # Not iterable return the value return value @@ -205,13 +230,12 @@ class ComplexBaseField(BaseField): value_dict = dict([(key, self.field.to_python(item)) for key, item in value.items()]) else: value_dict = {} - for k,v in value.items(): + for k, v in value.items(): if isinstance(v, Document): # We need the id from the saved object to create the DBRef if v.pk is None: - raise ValidationError('You can only reference ' - 'documents once they have been saved ' - 'to the database ("%s")' % self.name) + self.error('You can only reference documents once they' + ' have been saved to the database') collection = v._get_collection_name() value_dict[k] = pymongo.dbref.DBRef(collection, v.pk) elif hasattr(v, 'to_python'): @@ -220,7 +244,7 @@ class ComplexBaseField(BaseField): value_dict[k] = self.to_python(v) if is_list: # Convert back to a list - return [v for k,v in sorted(value_dict.items(), key=operator.itemgetter(0))] + return [v for k, v in sorted(value_dict.items(), key=operator.itemgetter(0))] return value_dict def to_mongo(self, value): @@ -238,7 +262,7 @@ class ComplexBaseField(BaseField): if not hasattr(value, 'items'): try: is_list = True - value = dict([(k,v) for k,v in enumerate(value)]) + value = dict([(k, v) for k, v in enumerate(value)]) except TypeError: # Not iterable return the value return value @@ -246,13 +270,12 @@ class ComplexBaseField(BaseField): value_dict = dict([(key, self.field.to_mongo(item)) for key, item in value.items()]) else: value_dict = {} - for k,v in value.items(): + for k, v in value.items(): if isinstance(v, Document): # We need the id from the saved object to create the DBRef if v.pk is None: - raise ValidationError('You can only reference ' - 'documents once they have been saved ' - 'to the database ("%s")' % self.name) + self.error('You can only reference documents once they' + ' have been saved to the database') # If its a document that is not inheritable it won't have # _types / _cls data so make it a generic reference allows @@ -270,26 +293,33 @@ class ComplexBaseField(BaseField): value_dict[k] = self.to_mongo(v) if is_list: # Convert back to a list - return [v for k,v in sorted(value_dict.items(), key=operator.itemgetter(0))] + return [v for k, v in sorted(value_dict.items(), key=operator.itemgetter(0))] return value_dict def validate(self, value): - """If field provided ensure the value is valid. + """If field is provided ensure the value is valid. """ + errors = {} if self.field: - try: - if hasattr(value, 'iteritems'): - [self.field.validate(v) for k,v in value.iteritems()] - else: - [self.field.validate(v) for v in value] - except Exception, err: - raise ValidationError('Invalid %s item (%s) ("%s")' % ( - self.field.__class__.__name__, str(v), self.name)) - + if hasattr(value, 'iteritems'): + sequence = value.iteritems() + else: + sequence = enumerate(value) + for k, v in sequence: + try: + self.field.validate(v) + except (ValidationError, AssertionError), error: + if hasattr(error, 'errors'): + errors[k] = error.errors + else: + errors[k] = error + if errors: + field_class = self.field.__class__.__name__ + self.error('Invalid %s item (%s)' % (field_class, value), + errors=errors) # Don't allow empty values if required if self.required and not value: - raise ValidationError('Field "%s" is required and cannot be empty' % - self.name) + self.error('Field is required and cannot be empty') def prepare_query_value(self, op, value): return self.to_mongo(value) @@ -344,6 +374,7 @@ class BaseDynamicField(BaseField): def lookup_member(self, member_name): return member_name + class ObjectIdField(BaseField): """An field wrapper around MongoDB's ObjectIds. """ @@ -356,8 +387,8 @@ class ObjectIdField(BaseField): try: return pymongo.objectid.ObjectId(unicode(value)) except Exception, e: - #e.message attribute has been deprecated since Python 2.6 - raise ValidationError(unicode(e)) + # e.message attribute has been deprecated since Python 2.6 + self.error(unicode(e)) return value def prepare_query_value(self, op, value): @@ -367,7 +398,7 @@ class ObjectIdField(BaseField): try: pymongo.objectid.ObjectId(unicode(value)) except: - raise ValidationError('Invalid Object ID ("%s")' % self.name) + self.error('Invalid Object ID') class DocumentMetaclass(type): @@ -393,7 +424,7 @@ class DocumentMetaclass(type): superclasses[base._class_name] = base superclasses.update(base._superclasses) else: # Add any mixin fields - attrs.update(dict([(k,v) for k,v in base.__dict__.items() + attrs.update(dict([(k, v) for k, v in base.__dict__.items() if issubclass(v.__class__, BaseField)])) if hasattr(base, '_meta') and not base._meta.get('abstract'): @@ -488,7 +519,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): ('meta' in attrs and attrs['meta'].get('abstract', False))): # Make sure no base class was non-abstract non_abstract_bases = [b for b in bases - if hasattr(b,'_meta') and not b._meta.get('abstract', False)] + if hasattr(b, '_meta') and not b._meta.get('abstract', False)] if non_abstract_bases: raise ValueError("Abstract document cannot have non-abstract base") return super_new(cls, name, bases, attrs) @@ -665,7 +696,7 @@ class BaseDocument(object): signals.post_init.send(self.__class__, document=self) def __setattr__(self, name, value): - # Handle dynamic data only if an intialised dynamic document + # Handle dynamic data only if an initialised dynamic document if self._dynamic and getattr(self, '_initialised', False): field = None @@ -708,7 +739,8 @@ class BaseDocument(object): data[k] = self.__expand_dynamic_values(key, v) if is_list: # Convert back to a list - value = [v for k, v in sorted(data.items(), key=operator.itemgetter(0))] + data_items = sorted(data.items(), key=operator.itemgetter(0)) + value = [v for k, v in data_items] else: value = data @@ -729,15 +761,22 @@ class BaseDocument(object): for name, field in self._fields.items()] # Ensure that each field is matched to a valid value + errors = {} for field, value in fields: if value is not None: try: field._validate(value) - except (ValueError, AttributeError, AssertionError), e: - raise ValidationError('Invalid value for field named "%s" of type "%s": %s' - % (field.name, field.__class__.__name__, value)) + except ValidationError, error: + errors[field.name] = error.errors or error + except (ValueError, AttributeError, AssertionError), error: + errors[field.name] = error elif field.required: - raise ValidationError('Field "%s" is required' % field.name) + errors[field.name] = ValidationError('Field is required', + field_name=field.name) + if errors: + error = ValidationError('Errors encountered validating document') + error.errors = errors + raise error @apply def pk(): @@ -745,8 +784,10 @@ class BaseDocument(object): """ def fget(self): return getattr(self, self._meta['id_field']) + def fset(self, value): return setattr(self, self._meta['id_field'], value) + return property(fget, fset) def to_mongo(self): @@ -821,7 +862,6 @@ class BaseDocument(object): """.strip() % class_name) cls = subclasses[class_name] - present_fields = data.keys() for field_name, field in cls._fields.items(): if field.db_field in data: value = data[field.db_field] @@ -972,8 +1012,7 @@ class BaseDocument(object): return geo_indices def __getstate__(self): - self_dict = self.__dict__ - removals = ["get_%s_display" % k for k,v in self._fields.items() if v.choices] + removals = ["get_%s_display" % k for k, v in self._fields.items() if v.choices] for k in removals: if hasattr(self, k): delattr(self, k) @@ -1048,7 +1087,7 @@ class BaseDocument(object): def __hash__(self): if self.pk is None: # For new object - return super(BaseDocument,self).__hash__() + return super(BaseDocument, self).__hash__() else: return hash(self.pk) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index abea212..0bfe4fb 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -44,17 +44,13 @@ class StringField(BaseField): assert isinstance(value, (str, unicode)) if self.max_length is not None and len(value) > self.max_length: - raise ValidationError('String value is too long ("%s")' % - self.name) + self.error('String value is too long') if self.min_length is not None and len(value) < self.min_length: - raise ValidationError('String value is too short ("%s")' % - self.name) + self.error('String value is too short') if self.regex is not None and self.regex.match(value) is None: - message = 'String value did not match validation regex ("%s")' % \ - self.name - raise ValidationError(message) + self.error('String value did not match validation regex') def lookup_member(self, member_name): return None @@ -111,11 +107,9 @@ class URLField(StringField): import urllib2 try: request = urllib2.Request(value) - response = urllib2.urlopen(request) + urllib2.urlopen(request) except Exception, e: - message = 'This URL appears to be a broken link: %s ("%s")' % ( - e, self.name) - raise ValidationError(message) + self.error('This URL appears to be a broken link: %s' % e) class EmailField(StringField): @@ -132,8 +126,7 @@ class EmailField(StringField): def validate(self, value): if not EmailField.EMAIL_REGEX.match(value): - raise ValidationError('Invalid Mail-address: %s ("%s")' % (value, - self.name)) + self.error('Invalid Mail-address: %s' % value) class IntField(BaseField): @@ -151,16 +144,13 @@ class IntField(BaseField): try: value = int(value) except: - raise ValidationError('%s could not be converted to int ("%s")' % ( - value, self.name)) + raise self.error('%s could not be converted to int' % value) if self.min_value is not None and value < self.min_value: - raise ValidationError('Integer value is too small ("%s")' % - self.name) + self.error('Integer value is too small') if self.max_value is not None and value > self.max_value: - raise ValidationError('Integer value is too large ("%s")' % - self.name) + self.error('Integer value is too large') def prepare_query_value(self, op, value): return int(value) @@ -183,12 +173,10 @@ class FloatField(BaseField): assert isinstance(value, float) if self.min_value is not None and value < self.min_value: - raise ValidationError('Float value is too small ("%s")' % - self.name) + self.error('Float value is too small') if self.max_value is not None and value > self.max_value: - raise ValidationError('Float value is too large ("%s")' % - self.name) + self.error('Float value is too large') def prepare_query_value(self, op, value): return float(value) @@ -219,16 +207,13 @@ class DecimalField(BaseField): try: value = decimal.Decimal(value) except Exception, exc: - raise ValidationError('Could not convert value to decimal: %s' - '("%s")' % (exc, self.name)) + self.error('Could not convert value to decimal: %s' % exc) if self.min_value is not None and value < self.min_value: - raise ValidationError('Decimal value is too small ("%s")' % - self.name) + self.error('Decimal value is too small') if self.max_value is not None and value > self.max_value: - raise ValidationError('Decimal value is too large ("%s")' % - self.name) + self.error('Decimal value is too large') class BooleanField(BaseField): @@ -375,8 +360,8 @@ class ComplexDateTimeField(StringField): def validate(self, value): if not isinstance(value, datetime.datetime): - raise ValidationError('Only datetime objects may used in a ' - 'ComplexDateTimeField ("%s")' % self.name) + self.error('Only datetime objects may used in a ' + 'ComplexDateTimeField') def to_python(self, value): return self._convert_from_string(value) @@ -396,9 +381,8 @@ class EmbeddedDocumentField(BaseField): def __init__(self, document_type, **kwargs): if not isinstance(document_type, basestring): if not issubclass(document_type, EmbeddedDocument): - raise ValidationError('Invalid embedded document class ' - 'provided to an EmbeddedDocumentField ' - '("%s")' % self.name) + self.error('Invalid embedded document class provided to an ' + 'EmbeddedDocumentField') self.document_type_obj = document_type super(EmbeddedDocumentField, self).__init__(**kwargs) @@ -427,9 +411,8 @@ class EmbeddedDocumentField(BaseField): """ # Using isinstance also works for subclasses of self.document if not isinstance(value, self.document_type): - raise ValidationError('Invalid embedded document instance ' - 'provided to an EmbeddedDocumentField ' - '("%s")' % self.name) + self.error('Invalid embedded document instance provided to an ' + 'EmbeddedDocumentField') self.document_type.validate(value) def lookup_member(self, member_name): @@ -458,9 +441,8 @@ class GenericEmbeddedDocumentField(BaseField): def validate(self, value): if not isinstance(value, EmbeddedDocument): - raise ValidationError('Invalid embedded document instance ' - 'provided to an GenericEmbeddedDocumentField ' - '("%s")' % self.name) + self.error('Invalid embedded document instance provided to an ' + 'GenericEmbeddedDocumentField') value.validate() @@ -495,8 +477,7 @@ class ListField(ComplexBaseField): """ if (not isinstance(value, (list, tuple)) or isinstance(value, basestring)): - raise ValidationError('Only lists and tuples may be used in a ' - 'list field ("%s")' % self.name) + self.error('Only lists and tuples may be used in a list field') super(ListField, self).validate(value) def prepare_query_value(self, op, value): @@ -552,13 +533,11 @@ class DictField(ComplexBaseField): """Make sure that a list of valid fields is being used. """ if not isinstance(value, dict): - raise ValidationError('Only dictionaries may be used in a ' - 'DictField ("%s")' % self.name) + self.error('Only dictionaries may be used in a DictField') if any(('.' in k or '$' in k) for k in value): - raise ValidationError('Invalid dictionary key name - keys may not ' - 'contain "." or "$" characters ("%s")' % - self.name) + self.error('Invalid dictionary key name - keys may not contain "."' + ' or "$" characters') super(DictField, self).validate(value) def lookup_member(self, member_name): @@ -585,12 +564,11 @@ class MapField(DictField): def __init__(self, field=None, *args, **kwargs): if not isinstance(field, BaseField): - raise ValidationError('Argument to MapField constructor must be ' - 'a valid field') + self.error('Argument to MapField constructor must be a valid ' + 'field') super(MapField, self).__init__(field=field, *args, **kwargs) - class ReferenceField(BaseField): """A reference to a document that will be automatically dereferenced on access (lazily). @@ -616,8 +594,8 @@ class ReferenceField(BaseField): """ if not isinstance(document_type, basestring): if not issubclass(document_type, (Document, basestring)): - raise ValidationError('Argument to ReferenceField constructor ' - 'must be a document class or a string') + self.error('Argument to ReferenceField constructor must be a ' + 'document class or a string') self.document_type_obj = document_type self.reverse_delete_rule = reverse_delete_rule super(ReferenceField, self).__init__(**kwargs) @@ -656,8 +634,8 @@ class ReferenceField(BaseField): # We need the id from the saved object to create the DBRef id_ = document.id if id_ is None: - raise ValidationError('You can only reference documents once ' - 'they have been saved to the database') + self.error('You can only reference documents once they have' + ' been saved to the database') else: id_ = document @@ -672,10 +650,8 @@ class ReferenceField(BaseField): assert isinstance(value, (self.document_type, pymongo.dbref.DBRef)) if isinstance(value, Document) and value.id is None: - raise ValidationError('You can only reference documents once ' - 'they have been saved to the database ' - '("%s")' % self.name) - + raise self.error('You can only reference documents once they have' + ' been saved to the database') def lookup_member(self, member_name): return self.document_type._fields.get(member_name) @@ -703,14 +679,12 @@ class GenericReferenceField(BaseField): def validate(self, value): if not isinstance(value, (Document, pymongo.dbref.DBRef)): - raise ValidationError('GenericReferences can only contain ' - 'documents ("%s")' % self.name) + raise self.error('GenericReferences can only contain documents') # We need the id from the saved object to create the DBRef if isinstance(value, Document) and value.id is None: - raise ValidationError('You can only reference documents once ' - 'they have been saved to the database ' - '("%s")' % self.name) + self.error('You can only reference documents once they have been' + ' saved to the database') def dereference(self, value): doc_cls = get_document(value['_cls']) @@ -731,9 +705,8 @@ class GenericReferenceField(BaseField): # We need the id from the saved object to create the DBRef id_ = document.id if id_ is None: - raise ValidationError('You can only reference documents once ' - 'they have been saved to the database ' - '("%s")' % self.name) + self.error('You can only reference documents once they have' + ' been saved to the database') else: id_ = document @@ -765,8 +738,7 @@ class BinaryField(BaseField): assert isinstance(value, str) if self.max_bytes is not None and len(value) > self.max_bytes: - raise ValidationError('Binary value is too long ("%s")' % - self.name) + self.error('Binary value is too long') class GridFSError(Exception): @@ -940,16 +912,14 @@ class GeoPointField(BaseField): """Make sure that a geo-value is of type (x, y) """ if not isinstance(value, (list, tuple)): - raise ValidationError('GeoPointField can only accept tuples or ' - 'lists of (x, y) ("%s")' % self.name) + self.error('GeoPointField can only accept tuples or lists ' + 'of (x, y)') if not len(value) == 2: - raise ValidationError('Value must be a two-dimensional point ' - '("%s")' % self.name) + raise self.error('Value must be a two-dimensional point') if (not isinstance(value[0], (float, int)) and not isinstance(value[1], (float, int))): - raise ValidationError('Both values in point must be float or int ' - '("%s")' % self.name) + self.error('Both values in point must be float or int') class SequenceField(IntField): @@ -1036,4 +1006,4 @@ class UUIDField(BaseField): try: value = uuid.UUID(value) except Exception, exc: - raise ValidationError('Could not convert to UUID: %s' % exc) + self.error('Could not convert to UUID: %s' % exc) diff --git a/tests/fields.py b/tests/fields.py index c95b544..4525349 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -355,27 +355,27 @@ class FieldTest(unittest.TestCase): logs = LogEntry.objects.order_by("date") count = logs.count() i = 0 - while i == count-1: - self.assertTrue(logs[i].date <= logs[i+1].date) - i +=1 + while i == count - 1: + self.assertTrue(logs[i].date <= logs[i + 1].date) + i += 1 logs = LogEntry.objects.order_by("-date") count = logs.count() i = 0 - while i == count-1: - self.assertTrue(logs[i].date >= logs[i+1].date) - i +=1 + while i == count - 1: + self.assertTrue(logs[i].date >= logs[i + 1].date) + i += 1 # Test searching - logs = LogEntry.objects.filter(date__gte=datetime.datetime(1980,1,1)) + logs = LogEntry.objects.filter(date__gte=datetime.datetime(1980, 1, 1)) self.assertEqual(logs.count(), 30) - logs = LogEntry.objects.filter(date__lte=datetime.datetime(1980,1,1)) + logs = LogEntry.objects.filter(date__lte=datetime.datetime(1980, 1, 1)) self.assertEqual(logs.count(), 30) logs = LogEntry.objects.filter( - date__lte=datetime.datetime(2011,1,1), - date__gte=datetime.datetime(2000,1,1), + date__lte=datetime.datetime(2011, 1, 1), + date__gte=datetime.datetime(2000, 1, 1), ) self.assertEqual(logs.count(), 10) @@ -1112,7 +1112,6 @@ class FieldTest(unittest.TestCase): Post.drop_collection() User.drop_collection() - def test_generic_reference_document_not_registered(self): """Ensure dereferencing out of the document registry throws a `NotRegistered` error. @@ -1139,7 +1138,7 @@ class FieldTest(unittest.TestCase): user = User.objects.first() try: user.bookmarks - raise AssertionError, "Link was removed from the registry" + raise AssertionError("Link was removed from the registry") except NotRegistered: pass @@ -1339,7 +1338,7 @@ class FieldTest(unittest.TestCase): # Make sure FileField is optional and not required class DemoFile(Document): file = FileField() - d = DemoFile.objects.create() + DemoFile.objects.create() def test_file_uniqueness(self): """Ensure that each instance of a FileField is unique @@ -1538,7 +1537,6 @@ class FieldTest(unittest.TestCase): c = self.db['mongoengine.counters'].find_one({'_id': 'animal.id'}) self.assertEqual(c['next'], 10) - def test_generic_embedded_document(self): class Car(EmbeddedDocument): name = StringField() @@ -1564,5 +1562,40 @@ class FieldTest(unittest.TestCase): person = Person.objects.first() self.assertTrue(isinstance(person.like, Dish)) + def test_recursive_validation(self): + """Ensure that a validation result schema is available. + """ + class Author(EmbeddedDocument): + name = StringField(required=True) + + class Comment(EmbeddedDocument): + author = EmbeddedDocumentField(Author, required=True) + content = StringField(required=True) + + class Post(Document): + title = StringField(required=True) + comments = ListField(EmbeddedDocumentField(Comment)) + + bob = Author(name='Bob') + post = Post(title='hello world') + post.comments.append(Comment(content='hello', author=bob)) + post.comments.append(Comment(author=bob)) + + try: + post.validate() + except ValidationError, error: + pass + + self.assertTrue(hasattr(error, 'errors')) + self.assertTrue(isinstance(error.errors, dict)) + self.assertTrue('comments' in error.errors) + self.assertTrue(1 in error.errors['comments']) + self.assertTrue(isinstance(error.errors['comments'][1]['content'], + ValidationError)) + + post.comments[1].content = 'here we go' + post.validate() + + if __name__ == '__main__': unittest.main() From 62480fe9403481a9571cd897a2f497805e70b5d2 Mon Sep 17 00:00:00 2001 From: Nicolas Perriault Date: Mon, 24 Oct 2011 17:15:34 +0200 Subject: [PATCH 0406/1279] added a ValidatorError.schema properties which contains a dict representation of the whole validation error schema --- mongoengine/base.py | 22 +++++++++++++++++++++ tests/fields.py | 48 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index 96b0a58..204a743 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -20,10 +20,13 @@ class InvalidDocumentError(Exception): class ValidationError(Exception): + """Validation exception. + """ errors = {} field_name = None def __init__(self, message, **kwargs): + self.errors = kwargs.get('errors', {}) self.field_name = kwargs.get('field_name') super(ValidationError, self).__init__(message) @@ -40,6 +43,25 @@ class ValidationError(Exception): else: return message + @property + def schema(self): + def get_schema(source): + errors_dict = {} + if not source: + return errors_dict + if isinstance(source, dict): + for field_name, error in source.iteritems(): + errors_dict[field_name] = get_schema(error) + elif isinstance(source, ValidationError) and source.errors: + return get_schema(source.errors) + else: + return unicode(source) + return errors_dict + if not self.errors: + return {} + return get_schema(self.errors) + + _document_registry = {} diff --git a/tests/fields.py b/tests/fields.py index 4525349..75c1966 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1586,6 +1586,7 @@ class FieldTest(unittest.TestCase): except ValidationError, error: pass + # ValidationError.errors property self.assertTrue(hasattr(error, 'errors')) self.assertTrue(isinstance(error.errors, dict)) self.assertTrue('comments' in error.errors) @@ -1593,9 +1594,56 @@ class FieldTest(unittest.TestCase): self.assertTrue(isinstance(error.errors['comments'][1]['content'], ValidationError)) + # ValidationError.schema property + schema = error.schema + self.assertTrue(isinstance(schema, dict)) + self.assertTrue('comments' in schema) + self.assertTrue(1 in schema['comments']) + self.assertTrue('content' in schema['comments'][1]) + self.assertEquals(schema['comments'][1]['content'], + u'Field is required ("content")') + post.comments[1].content = 'here we go' post.validate() +class ValidatorErrorTest(unittest.TestCase): + + def test_schema(self): + """Ensure a ValidationError handles error schema correctly. + """ + error = ValidationError('root') + self.assertEquals(error.schema, {}) + + # 1st level error schema + error.errors = {'1st': ValidationError('bad 1st'), } + self.assertTrue('1st' in error.schema) + self.assertEquals(error.schema['1st'], 'bad 1st') + + # 2nd level error schema + error.errors = {'1st': ValidationError('bad 1st', errors={ + '2nd': ValidationError('bad 2nd'), + })} + self.assertTrue('1st' in error.schema) + self.assertTrue(isinstance(error.schema['1st'], dict)) + self.assertTrue('2nd' in error.schema['1st']) + self.assertEquals(error.schema['1st']['2nd'], 'bad 2nd') + + # moar levels + error.errors = {'1st': ValidationError('bad 1st', errors={ + '2nd': ValidationError('bad 2nd', errors={ + '3rd': ValidationError('bad 3rd', errors={ + '4th': ValidationError('Inception'), + }), + }), + })} + self.assertTrue('1st' in error.schema) + self.assertTrue('2nd' in error.schema['1st']) + self.assertTrue('3rd' in error.schema['1st']['2nd']) + self.assertTrue('4th' in error.schema['1st']['2nd']['3rd']) + self.assertEquals(error.schema['1st']['2nd']['3rd']['4th'], + 'Inception') + + if __name__ == '__main__': unittest.main() From 7db5335420ecf6afe9d20c38b7b7a34ed844e52c Mon Sep 17 00:00:00 2001 From: Nicolas Perriault Date: Tue, 25 Oct 2011 10:53:58 +0200 Subject: [PATCH 0407/1279] fixed URLField.validate() wasn't using BaseField.error() to raise a ValidationError --- mongoengine/fields.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 0bfe4fb..180a74a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -100,8 +100,7 @@ class URLField(StringField): def validate(self, value): if not URLField.URL_REGEX.match(value): - raise ValidationError('Invalid URL: %s ("%s")' % (value, - self.name)) + self.error('Invalid URL: %s' % value) if self.verify_exists: import urllib2 From b8e2bdc99f612074551a6faad79bda55de396b78 Mon Sep 17 00:00:00 2001 From: Nicolas Perriault Date: Tue, 25 Oct 2011 20:04:39 +0200 Subject: [PATCH 0408/1279] simpler raising of ValidatioError --- mongoengine/base.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 204a743..9722820 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -161,9 +161,7 @@ class BaseField(object): """Raises a ValidationError. """ field_name = field_name if field_name else self.name - error = ValidationError(message, field_name=field_name) - error.errors = errors if errors else {} - raise error + raise ValidationError(message, errors=errors, field_name=field_name) def to_python(self, value): """Convert a MongoDB-compatible type to a Python type. From a1db437c426477cd5ba9dbdde195883453d86ffb Mon Sep 17 00:00:00 2001 From: Nicolas Perriault Date: Tue, 25 Oct 2011 22:38:43 +0200 Subject: [PATCH 0409/1279] got rid of assert for validation; ValidationError now extends AssertionError for BC purpose --- mongoengine/base.py | 11 +++++------ mongoengine/fields.py | 37 +++++++++++++++++++++++-------------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 9722820..a99b026 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -19,13 +19,13 @@ class InvalidDocumentError(Exception): pass -class ValidationError(Exception): +class ValidationError(AssertionError): """Validation exception. """ errors = {} field_name = None - def __init__(self, message, **kwargs): + def __init__(self, message="", **kwargs): self.errors = kwargs.get('errors', {}) self.field_name = kwargs.get('field_name') super(ValidationError, self).__init__(message) @@ -157,7 +157,7 @@ class BaseField(object): instance._data[self.name] = value instance._mark_as_changed(self.name) - def error(self, message, errors=None, field_name=None): + def error(self, message="", errors=None, field_name=None): """Raises a ValidationError. """ field_name = field_name if field_name else self.name @@ -794,9 +794,8 @@ class BaseDocument(object): errors[field.name] = ValidationError('Field is required', field_name=field.name) if errors: - error = ValidationError('Errors encountered validating document') - error.errors = errors - raise error + raise ValidationError('Errors encountered validating document', + errors=errors) @apply def pk(): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 180a74a..24de509 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -41,7 +41,8 @@ class StringField(BaseField): return unicode(value) def validate(self, value): - assert isinstance(value, (str, unicode)) + if not isinstance(value, (str, unicode)): + self.error('StringField only accepts string values') if self.max_length is not None and len(value) > self.max_length: self.error('String value is too long') @@ -143,7 +144,7 @@ class IntField(BaseField): try: value = int(value) except: - raise self.error('%s could not be converted to int' % value) + self.error('%s could not be converted to int' % value) if self.min_value is not None and value < self.min_value: self.error('Integer value is too small') @@ -169,7 +170,8 @@ class FloatField(BaseField): def validate(self, value): if isinstance(value, int): value = float(value) - assert isinstance(value, float) + if not isinstance(value, float): + self.error('FoatField only accepts float values') if self.min_value is not None and value < self.min_value: self.error('Float value is too small') @@ -225,7 +227,8 @@ class BooleanField(BaseField): return bool(value) def validate(self, value): - assert isinstance(value, bool) + if not isinstance(value, bool): + self.error('BooleanField only accepts boolean values') class DateTimeField(BaseField): @@ -238,7 +241,8 @@ class DateTimeField(BaseField): """ def validate(self, value): - assert isinstance(value, (datetime.datetime, datetime.date)) + if not isinstance(value, (datetime.datetime, datetime.date)): + self.error(u'cannot parse date "%s"' % value) def to_mongo(self, value): return self.prepare_query_value(None, value) @@ -524,7 +528,8 @@ class DictField(ComplexBaseField): def __init__(self, basecls=None, field=None, *args, **kwargs): self.field = field self.basecls = basecls or BaseField - assert issubclass(self.basecls, BaseField) + if not issubclass(self.basecls, BaseField): + self.error('DictField only accepts dict values') kwargs.setdefault('default', lambda: {}) super(DictField, self).__init__(*args, **kwargs) @@ -646,11 +651,12 @@ class ReferenceField(BaseField): return self.to_mongo(value) def validate(self, value): - assert isinstance(value, (self.document_type, pymongo.dbref.DBRef)) + if not isinstance(value, (self.document_type, pymongo.dbref.DBRef)): + self.error('A ReferenceField only accepts DBRef') if isinstance(value, Document) and value.id is None: - raise self.error('You can only reference documents once they have' - ' been saved to the database') + self.error('You can only reference documents once they have been ' + 'saved to the database') def lookup_member(self, member_name): return self.document_type._fields.get(member_name) @@ -678,7 +684,7 @@ class GenericReferenceField(BaseField): def validate(self, value): if not isinstance(value, (Document, pymongo.dbref.DBRef)): - raise self.error('GenericReferences can only contain documents') + self.error('GenericReferences can only contain documents') # We need the id from the saved object to create the DBRef if isinstance(value, Document) and value.id is None: @@ -734,7 +740,8 @@ class BinaryField(BaseField): return str(value) def validate(self, value): - assert isinstance(value, str) + if not isinstance(value, str): + self.error('BinaryField only accepts string values') if self.max_bytes is not None and len(value) > self.max_bytes: self.error('Binary value is too long') @@ -895,8 +902,10 @@ class FileField(BaseField): def validate(self, value): if value.grid_id is not None: - assert isinstance(value, GridFSProxy) - assert isinstance(value.grid_id, pymongo.objectid.ObjectId) + if not isinstance(value, GridFSProxy): + self.error('FileField only accepts GridFSProxy values') + if not isinstance(value.grid_id, pymongo.objectid.ObjectId): + self.error('Invalid GridFSProxy value') class GeoPointField(BaseField): @@ -915,7 +924,7 @@ class GeoPointField(BaseField): 'of (x, y)') if not len(value) == 2: - raise self.error('Value must be a two-dimensional point') + self.error('Value must be a two-dimensional point') if (not isinstance(value[0], (float, int)) and not isinstance(value[1], (float, int))): self.error('Both values in point must be float or int') From 634e1f661fd040794c84359704e3287033a648a0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 27 Oct 2011 00:31:48 -0700 Subject: [PATCH 0410/1279] Updated docs / upgrade notes --- docs/changelog.rst | 1 + docs/upgrade.rst | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 81bc198..4b3916d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed accessing pk on an embedded document - Fixed calling a queryset after drop_collection now recreates the collection - Add field name to validation exception messages - Added UUID field diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 671b930..9151cf3 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -5,7 +5,8 @@ Upgrading 0.5 to 0.6 ========== -TBC +Embedded Documents - if you had a `pk` field you will have to rename it from `_id` +to `pk` as pk is no longer a property of Embedded Documents. 0.4 to 0.5 =========== From 165cdc8840a3da68373ba09841a6b3f6a38de74a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 27 Oct 2011 00:35:34 -0700 Subject: [PATCH 0411/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4b3916d..bd598a8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed Reference Fields can be None in get_or_create / queries - Fixed accessing pk on an embedded document - Fixed calling a queryset after drop_collection now recreates the collection - Add field name to validation exception messages From 56d1139d7192f1c5509f6037f1740cf11bace5fe Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 27 Oct 2011 00:58:47 -0700 Subject: [PATCH 0412/1279] Added ImageField Support Thanks to @wpjunior for the patch Closes [#298] --- docs/changelog.rst | 1 + mongoengine/base.py | 1 + mongoengine/fields.py | 217 ++++++++++++++++++++++++++++++++++++++---- setup.py | 2 +- tests/fields.py | 71 +++++++++++++- tests/mongoengine.png | Bin 0 -> 8313 bytes 6 files changed, 269 insertions(+), 23 deletions(-) create mode 100644 tests/mongoengine.png diff --git a/docs/changelog.rst b/docs/changelog.rst index bd598a8..43e900e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added ImageField - requires PIL - Fixed Reference Fields can be None in get_or_create / queries - Fixed accessing pk on an embedded document - Fixed calling a queryset after drop_collection now recreates the collection diff --git a/mongoengine/base.py b/mongoengine/base.py index ed14c74..20388e9 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -19,6 +19,7 @@ class NotRegistered(Exception): class InvalidDocumentError(Exception): pass + class ValidationError(Exception): pass diff --git a/mongoengine/fields.py b/mongoengine/fields.py index a1638bf..5700fe4 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1,3 +1,14 @@ +import datetime +import time +import decimal +import gridfs +import pymongo +import pymongo.binary +import pymongo.dbref +import pymongo.son +import re +import uuid + from base import (BaseField, ComplexBaseField, ObjectIdField, ValidationError, get_document) from queryset import DO_NOTHING @@ -5,15 +16,17 @@ from document import Document, EmbeddedDocument from connection import _get_db from operator import itemgetter -import re -import pymongo -import pymongo.dbref -import pymongo.son -import pymongo.binary -import datetime, time -import decimal -import gridfs -import uuid + +try: + from PIL import Image, ImageOps +except ImportError: + Image = None + ImageOps = None + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', @@ -21,7 +34,7 @@ __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'ObjectIdField', 'ReferenceField', 'ValidationError', 'MapField', 'DecimalField', 'ComplexDateTimeField', 'URLField', 'GenericReferenceField', 'FileField', 'BinaryField', - 'SortedListField', 'EmailField', 'GeoPointField', + 'SortedListField', 'EmailField', 'GeoPointField', 'ImageField', 'SequenceField', 'UUIDField', 'GenericEmbeddedDocumentField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -784,10 +797,12 @@ class GridFSProxy(object): .. versionadded:: 0.4 .. versionchanged:: 0.5 - added optional size param to read + .. versionchanged:: 0.6 - added collection name param """ - def __init__(self, grid_id=None, key=None, instance=None): - self.fs = gridfs.GridFS(_get_db()) # Filesystem instance + def __init__(self, grid_id=None, key=None, + instance=None, collection_name='fs'): + self.fs = gridfs.GridFS(_get_db(), collection_name) # Filesystem instance self.newfile = None # Used for partial writes self.grid_id = grid_id # Store GridFS id for file self.gridout = None @@ -878,9 +893,11 @@ class FileField(BaseField): .. versionadded:: 0.4 .. versionchanged:: 0.5 added optional size param for read """ + proxy_class = GridFSProxy - def __init__(self, **kwargs): + def __init__(self, collection_name="fs", **kwargs): super(FileField, self).__init__(**kwargs) + self.collection_name = collection_name def __get__(self, instance, owner): if instance is None: @@ -889,12 +906,13 @@ class FileField(BaseField): # Check if a file already exists for this model grid_file = instance._data.get(self.name) self.grid_file = grid_file - if isinstance(self.grid_file, GridFSProxy): + if isinstance(self.grid_file, self.proxy_class): if not self.grid_file.key: self.grid_file.key = self.name self.grid_file.instance = instance return self.grid_file - return GridFSProxy(key=self.name, instance=instance) + return self.proxy_class(key=self.name, instance=instance, + collection_name=self.collection_name) def __set__(self, instance, value): key = self.name @@ -911,7 +929,8 @@ class FileField(BaseField): grid_file.put(value) else: # Create a new proxy object as we don't already have one - instance._data[key] = GridFSProxy(key=key, instance=instance) + instance._data[key] = self.proxy_class(key=key, instance=instance, + collection_name=self.collection_name) instance._data[key].put(value) else: instance._data[key] = value @@ -920,20 +939,180 @@ class FileField(BaseField): def to_mongo(self, value): # Store the GridFS file id in MongoDB - if isinstance(value, GridFSProxy) and value.grid_id is not None: + if isinstance(value, self.proxy_class) and value.grid_id is not None: return value.grid_id return None def to_python(self, value): if value is not None: - return GridFSProxy(value) + return self.proxy_class(value, + collection_name=self.collection_name) def validate(self, value): if value.grid_id is not None: - assert isinstance(value, GridFSProxy) + assert isinstance(value, self.proxy_class) assert isinstance(value.grid_id, pymongo.objectid.ObjectId) +class ImageGridFsProxy(GridFSProxy): + """ + Proxy for ImageField + + versionadded: 0.6 + """ + def put(self, file_obj, **kwargs): + """ + Insert a image in database + applying field properties (size, thumbnail_size) + """ + field = self.instance._fields[self.key] + + try: + img = Image.open(file_obj) + except: + raise ValidationError('Invalid image') + + if (field.size and (img.size[0] > field.size['width'] or + img.size[1] > field.size['height'])): + size = field.size + + if size['force']: + img = ImageOps.fit(img, + (size['width'], + size['height']), + Image.ANTIALIAS) + else: + img.thumbnail((size['width'], + size['height']), + Image.ANTIALIAS) + + thumbnail = None + if field.thumbnail_size: + size = field.thumbnail_size + + if size['force']: + thumbnail = ImageOps.fit(img, + (size['width'], + size['height']), + Image.ANTIALIAS) + else: + thumbnail = img.copy() + thumbnail.thumbnail((size['width'], + size['height']), + Image.ANTIALIAS) + + if thumbnail: + thumb_id = self._put_thumbnail(thumbnail, + img.format) + else: + thumb_id = None + + w, h = img.size + + io = StringIO() + img.save(io, img.format) + io.seek(0) + + return super(ImageGridFsProxy, self).put(io, + width=w, + height=h, + format=img.format, + thumbnail_id=thumb_id, + **kwargs) + + def delete(self, *args, **kwargs): + #deletes thumbnail + out = self.get() + if out and out.thumbnail_id: + self.fs.delete(out.thumbnail_id) + + return super(ImageGridFsProxy, self).delete(*args, **kwargs) + + def _put_thumbnail(self, thumbnail, format, **kwargs): + w, h = thumbnail.size + + io = StringIO() + thumbnail.save(io, format) + io.seek(0) + + return self.fs.put(io, width=w, + height=h, + format=format, + **kwargs) + @property + def size(self): + """ + return a width, height of image + """ + out = self.get() + if out: + return out.width, out.height + + @property + def format(self): + """ + return format of image + ex: PNG, JPEG, GIF, etc + """ + out = self.get() + if out: + return out.format + + @property + def thumbnail(self): + """ + return a gridfs.grid_file.GridOut + representing a thumbnail of Image + """ + out = self.get() + if out and out.thumbnail_id: + return self.fs.get(out.thumbnail_id) + + def write(self, *args, **kwargs): + raise RuntimeError("Please use \"put\" method instead") + + def writelines(self, *args, **kwargs): + raise RuntimeError("Please use \"put\" method instead") + + +class ImproperlyConfigured(Exception): + pass + + +class ImageField(FileField): + """ + A Image File storage field. + + @size (width, height, force): + max size to store images, if larger will be automatically resized + ex: size=(800, 600, True) + + @thumbnail (width, height, force): + size to generate a thumbnail + + .. versionadded:: 0.6 + """ + proxy_class = ImageGridFsProxy + + def __init__(self, size=None, thumbnail_size=None, + collection_name='images', **kwargs): + if not Image: + raise ImproperlyConfigured("PIL library was not found") + + params_size = ('width', 'height', 'force') + extra_args = dict(size=size, thumbnail_size=thumbnail_size) + for att_name, att in extra_args.items(): + if att and (isinstance(att, tuple) or isinstance(att, list)): + setattr(self, att_name, dict( + map(None, params_size, att))) + else: + setattr(self, att_name, None) + + super(ImageField, self).__init__( + collection_name=collection_name, + **kwargs) + + class GeoPointField(BaseField): """A list storing a latitude and longitude. diff --git a/setup.py b/setup.py index d4dd5ab..b0c29bf 100644 --- a/setup.py +++ b/setup.py @@ -47,5 +47,5 @@ setup(name='mongoengine', classifiers=CLASSIFIERS, install_requires=['pymongo'], test_suite='tests', - tests_require=['blinker', 'django>=1.3'] + tests_require=['blinker', 'django>=1.3', 'PIL'] ) diff --git a/tests/fields.py b/tests/fields.py index 80a343e..20cdf19 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1,12 +1,16 @@ -import unittest import datetime -from decimal import Decimal +import os +import unittest import uuid +from decimal import Decimal + from mongoengine import * from mongoengine.connection import _get_db from mongoengine.base import _document_registry, NotRegistered +TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') + class FieldTest(unittest.TestCase): @@ -1011,7 +1015,7 @@ class FieldTest(unittest.TestCase): self.assertEqual(obj, me) obj, created = Product.objects.get_or_create(company=None) - + self.assertEqual(created, False) self.assertEqual(obj, me) @@ -1392,6 +1396,67 @@ class FieldTest(unittest.TestCase): TestFile.drop_collection() + def test_image_field(self): + + class TestImage(Document): + image = ImageField() + + TestImage.drop_collection() + + t = TestImage() + t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.save() + + t = TestImage.objects.first() + + self.assertEquals(t.image.format, 'PNG') + + w, h = t.image.size + self.assertEquals(w, 371) + self.assertEquals(h, 76) + + t.image.delete() + + def test_image_field_resize(self): + + class TestImage(Document): + image = ImageField(size=(185, 37)) + + TestImage.drop_collection() + + t = TestImage() + t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.save() + + t = TestImage.objects.first() + + self.assertEquals(t.image.format, 'PNG') + w, h = t.image.size + + self.assertEquals(w, 185) + self.assertEquals(h, 37) + + t.image.delete() + + def test_image_field_thumbnail(self): + + class TestImage(Document): + image = ImageField(thumbnail_size=(92, 18)) + + TestImage.drop_collection() + + t = TestImage() + t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.save() + + t = TestImage.objects.first() + + self.assertEquals(t.image.thumbnail.format, 'PNG') + self.assertEquals(t.image.thumbnail.width, 92) + self.assertEquals(t.image.thumbnail.height, 18) + + t.image.delete() + def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """ diff --git a/tests/mongoengine.png b/tests/mongoengine.png new file mode 100644 index 0000000000000000000000000000000000000000..56acb96dbebb029feb884d3ffa4f9f001a0b28cb GIT binary patch literal 8313 zcmV-*bq z{QJ#$Px9r<-rs!Ay(f3S&$Ba}Z`$Af)_(U|Yp=a_s;sO`EG*h1N=izkqM}0P&Yf!; zBPtFVAaeC6k%s2htVLf{&QJwpHd2)ys&wM98dX-Q^0pMEW=KoASk~7Mla$;p#i@;X ztz{ZQSEOpxDDucuk;%T~eujQY0g!szK4Z;`ZvM z$JPH8k*D%o+Z6wJZkEXMs>E+^$=t%iqR`M)l^?5eohsct&n!~p+1MhoLXr4}dZisJ z^|&}SVXLN`PjpapJLQv{5V<9D3k!=v!zfiot8%cf7MxOlY*O0zvTBjK<_xi;q@_IX zweC=?>Y+${dS%WYVaeRW!lF=csVWarGxyfUThbYAd`)de125j{QmIlzUzyihqdxU& zFnM#4NOMc>9%JRh!os2eaGNTB_a$r$-EPJ#d?uh zOLP7xOXe097SXY^N)f2ifO~e%l?&k%RqpinF(~3*RV%VQtBn`wMwi9vr-^xMn_^Xm z%?%>2FUYukat!Ncv9Pd+j=Cn1g=;bse92>1$xL<@`D)}V6^Sn+k*Bg+_zt0m_o`Bz zmv$-QvtOOBte3j{Jkm;|g@r{dy!B}&{4bSd(xi&qpvqpx=cX8)0&RSy(#ngS{GDY3B+x|4K=sG>%tX$DutR9& zJ1|+=@pq~&bFXwma`iJIov~78RjT2mBCnsplDUP2MSQ&ZiAZxg$wif46|CO$(#oNc zuddZhbnbw=)CKRywiMt4$HJeIV)JP@wq%3I#1VSvPvq}QI(=W{nq&zYD@Qk|XD(VTvSOpi=6dt_@R8>*p4^hTg@r|K z@$77o1Nw;c>n73|EAiG~RSq%|`NoEfR$i2Hv8dbqky6Z?zC<00HUnA0N!?G^ip*W1 zH173r-^-G@g@r`{ph;%Z;&b-clb5Ck`i&`xc7Esu6tJcJL-=G z;aRaL6WRz3{M~Yqigg9i{FcluEG&}4Y^A|Y`Xu9SscnpUmwk5lC98N>Dk87XE(mgK zV%7wO+EYM%2wv|2e9 z1tD?PRzJC$$hZFcSL%#maBR<6j#V z!KH#tB{nvAg&`sx;gzb~6I}N%&iM3Snw!Zno=z8hb6YaEu&~Gt5XSe+tc*q-&fT(j z%b&QUDV=ezZ2e&eOjYFsRmKFqI|i3EivORTDN?>}r)h3W<`xzfxx>ebEMJ%}vP(yi zE~O%!OEXa;-dG~oh#3RGx;mvnH<78*PEubZ8Mn%bs!TK9DZ6U3BMa(Kxj|&&Y<2IN zovNvAG>e6WMQ-uZg3Rhk5NjtzrYG+$(z{D0IhP(t{{4Z}HyzKf6E0A9Wn^YiT zrCgsJQ8I2699ye2@WqPvYZ|s4!z#}Drk$>tEty+bSmYL%WSrtiIyGy#>~hv{>D#Nb zG;5Z{#+*WpU~8(#VJtN!2iJ{C-uYYoD`q>AhsM?-^=L_-tLq&C?EG$=t%i zBEML%Nm?G7Do2m#EnPd5NV9A5F@^sY$$|6x^ptl8^`6sER5VV~>2d1Uh1&bU9Rm_? zYR>q;KU$Vqg5S8gdl4OV~7Ytvup`lxS!+9_5J8(fu zc9Ui|;7)H74wnihHdk*FnJ`mi(!zF4Xq%|qP`{=0=v*QVO=+oZv@Vn|WZqkq9-3nq3RkI8?Q@ULsvN7z>5|Dq z+@05AHUf6hE2{jTWG1kK;z(7>R2iX4FUf2pRmb|gtI8Xy%uTp0Fd2NPDu=0ZxGH^A z>B`^h9Pfj7nJJlt4MBl-=&AV~lDdNT_XFzo1yv5=H|xafk(Z)UmDgFfRSC5Rc6&Km zm6KE%z_!45z}jl8zMG-STY_!5%;?6t>L%J(vhG2#FTV?9t-mVD&fbz;uYvbj!up_I zx!QoSle8|)q=%7B2&a`Rc59y>?R&*v#M&FbbVzh-_4C(*p@mD zTgSfZhvBLmL-|7A@50}!DXWuJd0Von29m?)DN}fMKlYhyKSO)Y=CeOAy!%$9%F4PZqsx)kjE9Od|j2}#6Lmva#fyD zg8IOjmEn&C@>j?!#|$Dmz)H0QIRRdA-bf z83T?>N=l@nqC)1*og0L}E?4DDDj0rfA&Fw8;-entp?((_t_Nv6ugV?l*lzMdKe>nl zoFA}a=W%|M4H4zyEBqE^{Jufne=OfI3k)`QQ!aLsaN9q(aF8C){^TUKi`dpW4qL-P z{#J1&@Y;s6*p^=u-R?#^%GSSC`MqRv!bC+ca@{!`7=w67pd36!4&7`b^9K!N?!-iX z%&g<;Kz80yAzuOwex-)|hNv={oi+@h(O%^en4o|Kf!q#12q2tZz+m!oRbKTt2ju!= zpZh#Ug3DqA2f$+@)ag@_#D^iS>Bi3f?TGC#iQg95;4P}$>2p2?s3$!%PK2lZRCF5^ z1V;tQ<1Z1(yp%F>y}^170t_nO;Wu&PX#<%%aS!W|1qQW01sqg#gm7L+%Vl>}{;0}V zBfkH0?4xTa&(}zNTt`C9;^F`t@Gh;)ItQUbk^M%5=hyN*{vP4l^Z8AOMtt{~>>E>b zohHay*Dulp$O_Y>2t*ECrphl|za7F+{L~2dgQoU3$t2qegCnTD-4v6^vo0aHl76d7 z2M?{n52tW|$!f=ML?m)xFKQ5bhUl;_#UOH_3+Pq4WR41Oz9|f0C<*X?V-mUW{YR5D zcMo|jtb%`0GwuA3&T_b^E@T^zh*+0{C>Og1Jb$1n|A|55LhbW;w*PSn zxhBpDC`;~sy0=5dUXMuRLf6`ukndB%LGhUwM4tVIuj#%?5&biMD5l`SZ#f}C$8h2& zj+Yt64?QHt{j!bJC4{;7T8s|ir7f5`N{}}0%6Ea;{7;&panRqTy?8x$e2js^iD4XE zdj&XG=SLq-@+eFA+g!l=4hwkx01nV!@;MJKoJzX1HhG*kgx2kl2y)SdeeYOmf}LZw z#qa=DQ`VI;KB0aB#qZ$|ebf(oxLhJZ7eC^>p5&Z^V%DLEyc5Lr2PN+bRletQUKM2t zGPT}kFnN?p_I5;w7@u{Hj;-?>HPHZj$56Zf`M+haYo;*TUVsa8iVZ7&kup+4I61o{M>6UUnz zZQRZ9e9Qy%Wt(&}97j9Pm&{~S7YgKX9}NbBgV!YGoorhZ)=?XrLm|u7b12Q$12t{X z;^7vnas1nDO}ywcu;Uu|Kc~@!vXL?|n1pex$G089`$JAX^XVw#DZ@G}Wk-6Bu9Hpd z*ciZ$rulumLCYP>=Qlg#JS!JH4CG!*`G4F%9FVgGEVYy!SONQZyd&mB?xRc0gx~pC z-)m^i!AG@>@6?~)5tg!h9_?%o4Z51&*15T7k%t9Q;v(=`wqu&#uZxE@_-8|($@=7H z@w;^3;6GB#KJA0p$DZ=xoj-fH(>9anpJCr#&${&B{r`uq8J!Cd@`QkmE%TZ1nRnyM zrzwz6YdYvk3ix+jzfYv!3I3Y~*QkZc|3UP#%x|+}{YsR-|$~GQM^W8m&eL~W_+V%Sa5}YH1Pvhhu35(o{uP|Tg89!7z4^B6xI|wOr3+sUK8_V zUNvRnPaM$1gRs$lmwLF*rc(3YXz-bj_HZ|$Eq^KA5kp0^KJPMUj0hTd!13OlT&^*< zvh*Q4`Ngi^r&BH_$d3N_F4Wq7?J?N@l;2i|FR_1$L8-W(va!s={dqp+VqYJ*c$ouW zjiIkSsTrWlIat&>p4Wr@E30(6T_Bt28oAc>`)c<2$0%EMnm%$n6*L{tXT4m+-xYJ< z2l(?wADI@qR30=~>wn|myoJo&2hnLRbN$xn7!926l6wRCQQaCxGe4G-3}=(5=lI<7 zRf8E0O>I94o6cm!1jy$M_k4yXP?)g3bJIDe$ ze?=cvw16bn5U?IuE#(RcKEczK+R2$7=R80n)8Ub& z+>6AnA<+lL9sG7~d`3Cdg`>W!(S~a|*g6M>+c|*O2XIHcC;GXEw?seJRKjZxHCPGf zd$f@oyHj!TnnSsXgO15cTB32=xyjs(Uvsc>b6CoyWxqnQ^_vh*@tsf89NWMFbA{&E zP-^;JL-q$9PNpSPs_CmxRA&_h^zW9sZgO(~A_pdkKFXumMdTl|Bh3kL&$|rfT0b#Q zm|fw~vH!u5FKm4Fvvk{-e5H4LxP^)gg4@A0SY8`|$g}4910EfEcY}#|iborVnW?X) z*6r0POr|QdugeX`rgPw)7V_JC$m>k8-Y{K{)O?fM4d>qp~EDR_py5{|+k`S&ci&di@duhn2Xj=PygJ$euJg zPp1%}kMD1|?nVQVzpk+m{O~%J9vzolX(AOrj3-eA#j8HY8)z0+2E1R^m+nSK(NA!7 z*0_hVn8RaH6Fj^rnXoGz26@n1=vFF>}x9ni9Ilkhu+W4b}{Y=hk z;eEq^0wwmKuw?#-ct-;0fJqY5vShdr?u+nJD-6GPG-#kYyvT-40>3l}$Nr*^FZOdI z)kdg8lLwJ@cm2P=!R>^1m>8oS&|zN-asIgmlNb|1V-xx|kXhXhH67CF^qJJmI~rUc zHR8R3(+?kWaMoFEeIv+NltDJT3et^~MlLY84mDUwuf%8v$mNtaLgtT(-u)HyJ#^@1 z(3VZwbyPHqd9rlxFp0LHVf-aR1sZv+?pc@|SrXx%CLdf%Lt-Zy&V4XJ0l^F!d|Ib# zj-BB#qPqc?!gDd|VQQBm%}Cx|b6%zBU07i-GsL+k8GavQ&}b#1{K3_5o#q(G{9cmS zAOneEGWF1i&o*s0+?iw^R54mit0yXI0?Gx#8Fj8VXoGS0VVmJ$h2;w^G5dXWz;;C6 zI@}-+ICfTyI&?SuJ}?ZKPm^d#W?IDTlb#Txg9ZWI3;hi4q7M=llvfR8zJG{&d=S&^ z6SQ4Si+UDHI6tDvO%w!}I2vIP`2QsQZT@W_^CA3uKLd#kj5)Y^VW4=&|2y$|%)u?F zf3|l51GEXBwgIgiO4pni*9T4Eh=7CL49eyoa`bLJ!?;n&)@x5QfZ??qDl(a8I3PuEVMvJgdr}0Xfs8wiSUl z<&a6jwb2xC&^N6%UMk6@Yju7bor&Cugx*>5k1&&}D_VJgNrC{|!UX8@R&K{!q9Kf% zCFu^fQiA0f&IM`ZVsvgInY#zb5($=r1E|TajAB^Ra$k0ds+X$u?SY|7JfO!5RJCl|YIpdYOX)4)lOtPOc+R8#KeE;)Es zC&Ndq3GuG^+E&H%Ee9oV?;ZgOy@BuRZnK`$*y|JiHk}QfYqJN@V;e^Vl zm^$r#2@q+}3Ez!u7q>~ehlGpt2!dV)(-(vNG7pUn1I=7%KQOr;4@uzOd$*%@+#vc$ zCMXpPB}Xo&qIMvhIwHj)qCFFS)`^DSKWT&QQehx$3`m1x-aBZTS;Ri;e8HW?KRHua z9AYr@SF#;UgZ@4ee$iJUV~IyQ^on`!q3x(0*NQ$W4Q+T(t{%FxlR3;9-HIu6>{Anb zj@SJ@Y=iAmF8UBcXys?dym!#N=on;8TW_78@cWvU&(GrDGen;QXNW}Ggr!h1Tl5QK z5bbkvjQe(yWb)+~3RW8|OK9q+TQZLZ_{d&29Q(E;Hz||dd|LPXSK45^U{cStkm-sT z?XrJJ^81emnfFQPU2z{%VyovJ7Tjz@=exWekr1>=Q;6-=G44A`yz{HKC*Jv5uB^SH zcF?fMo1*jgV^{2}61YEn@1|^f*j4-EHrg)Z48J3n{kJ0C^LhyvUkftbU@%92me70N zVsH__6n>fiPJ6Tm^41?B?FXif7O-0P+=%zQLE0Zo3nGW+!18Q2WDZT^CBrc+hHuuC2O8mcST6nL5YNGc5EAzM!f+fB5vK%7b z`&nvZhikM-_|->}aB*I@h;=|j%Z-L((2B7O6!Jd@b>HB;pK>rVttY_7AFDv*jg8wy zr-IB)xjRmw932+&{jh!8o0JFaRC7rIC_fmOO?hJwI{FIbe}wEfzk-452r5aBNNyGj zl}hrl+PSc}UZVV(c;}4>$_k=ku;DwlP=x#M7H5atnSdQPuu>I~35XG%DcfSIujR{s zUUS_*nwS41c})@hs=GmSC%|;{ikpSMw=KmdU&eO9J3$q7CiMCt%|V^kiNPNJbL`B6 zMJ}JQJ(0O>Pthl+LJX~G0hI~p^vB7L+ZARz?r+%c_tE01GH5$z&^A4YgRU14o^Gwet|Wg#sfi9T zTz?)5V#DY0qoA{55zE0ei+cLh4cgsr+gQ7l$@crEg6y(ev0>1;m>PsBM+pRUU-6Is z-aw7v#t3!4hV6Ws;rJ1Bbs#2uAqVPeS_B7kVCfgY64`FoDTkXS3+%&r6$x9{o}D*iLX1y%tgl=cER8Xd7bAata<#U(^XJKId zGhAPVO6EaxRk+~J^Lm%gH-B6Y@Lb`Zv67!Dd`9RQ3E!^hy3)CRT#PH)GZu~-O zdc_gzejYWHuX~&~kmU;@?|(lBnINeZ9^?D`RlNP=`%AbK#{gEFew1gq2(pa5Q6m=< zNHPxv7${9x%K_uW2-h|Q{9a9Ms2z2@^FI666XHETw?_ z67OV^+EaO!iy+xaP5$$LQ|lcc;knt3wWmw8IhD-;`+B}6L+8b~_5*(V@d?%kyo2Ba z%qi6r8B8ZJm^M*O6P@`UCrs-l3Feb7_8^TPv2ovsQ4dVs-J!}+;=gfqkasVR`ElVp z3$g*SX%NS?SF-bt7VlI9X}UVD=lj2zppgES_k2pCPjo@k#};9us9{F|G^(Q|v!U~{ z81FNO1IkGb_f{CWh^uss5q}@Omp>bFZ~^tR9*p8P=@J(EFSW1~ZQZoK_=-Yzj;6>D+38LB zgVzVpMq#dFvI;wCpCJA@BsQ^g!hiduM9Vde5q?+A?;BzY8#*20!!hZC$zIb+Grx{K zl?CmJ+4CWincV6?_rofOoVcwU(=)?n{zDsMvIjnB_#EN#I>2YZt7j*He|DV2+f~M? z9p1^o?W=SpjS&AZMReSEC?hXOuw-kq4TKtZvh+jBIl|e78wQQK04|NUSh}$dFsZj2 z$+L#az(TfrIv|nywaYX7_Lz_zY`Bm6J69MuEN(t?v=3Z?-)BEFZSu0l=RRX8pJ&q& zJuHA5tBT(d^7NeU8L6_eGMy%!GH7B^&6%Qv?-x{a3HyvLB(LmbS|18(6$Nz(HI0gx z_02^ahiR8wE@ZleX4E*=oqw1j5*xywrO@WQzkk|gh6Lv8Av9M?Briym5I_342gnsyvCQ=n%`>tl)J zW>=`4SuRExelO2K-3zT9p#;2^tC7XRA_|~YoMs^6g|>OxEik|zsren6=cl%La%(17 zSVRF9q3C?c(54sJ=4rP;$gXZu67k`yZ1Y$$w`f~HE|hBx$Hqziwm!3HTR;?#Y2CmR zws|a>TOU5$d7{1t2fxmvboqz;cAHBKoTh|Yxhc*i#Yr>wuuW8h#W$kH$o#ZK{t_df#?d; z{<*tLW}y(4?t5)i0@q88ZDLF27RllohbDF^3rHN3Q%gz4ScHRcFRbuH4wWHd&OegH zSjps#uqaGi$-lAQ;2f4($^~M!kq^F}7Cz=m`cW?Yc8R)2+Fm+fOXd~@4`|qz7)NW zA=)e8_2rV8On+R=JJnkx2wiR#3yW;72;^QKKy$kj&0{1+Le3H7jF`v4psEmGGDdEe z_KJR#i|pnbkc*C#3uy32fVZ4-fsB*Yw#o8`{}*5YY#%T9D4-Dk00000NkvXXu0mjf DWUWgQ literal 0 HcmV?d00001 From 8074094568add900be0126970591fba15582a5cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Thu, 27 Oct 2011 06:02:59 -0200 Subject: [PATCH 0413/1279] optimizations for get_or_create --- mongoengine/queryset.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 8ea7ca3..3f4f8fc 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -739,18 +739,20 @@ class QuerySet(object): """ self.__call__(*q_objs, **query) try: - result1 = self[0] - except IndexError: + result1 = self.next() + except StopIteration: raise self._document.DoesNotExist("%s matching query does not exist." % self._document._class_name) try: - result2 = self[1] - except IndexError: + result2 = self.next() + except StopIteration: return result1 + + self.rewind() message = u'%d items returned, instead of 1' % self.count() raise self._document.MultipleObjectsReturned(message) - def get_or_create(self, write_options=None, *q_objs, **query): + def get_or_create(self, write_options=None, auto_save=True, *q_objs, **query): """Retrieve unique object or create, if it doesn't exist. Returns a tuple of ``(object, created)``, where ``object`` is the retrieved or created object and ``created`` is a boolean specifying whether a new object was created. Raises @@ -765,23 +767,25 @@ class QuerySet(object): Passes any write_options onto :meth:`~mongoengine.Document.save` .. versionadded:: 0.3 + + :param auto_save: if the object is to be saved automatically if not found. + + .. versionadded:: 0.6 """ defaults = query.get('defaults', {}) if 'defaults' in query: del query['defaults'] - self.__call__(*q_objs, **query) - count = self.count() - if count == 0: + try: + doc = self.get(*q_objs, **query) + return doc, False + except self._document.DoesNotExist: query.update(defaults) doc = self._document(**query) - doc.save(write_options=write_options) + + if auto_save: + doc.save(write_options=write_options) return doc, True - elif count == 1: - return self.first(), False - else: - message = u'%d items returned, instead of 1' % count - raise self._document.MultipleObjectsReturned(message) def create(self, **kwargs): """Create new object. Returns the saved object instance. From 5eb63cfa304185e82fde84b82a631ec244a9b4a0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 27 Oct 2011 01:14:51 -0700 Subject: [PATCH 0414/1279] Updated changelog --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6659f8d..d381a82 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added pre and post bulk-insert signals - Added ImageField - requires PIL - Fixed Reference Fields can be None in get_or_create / queries - Fixed accessing pk on an embedded document @@ -93,7 +94,6 @@ Changes in v0.5 way the user has specified them - Fixed various errors - Added many tests -- Added pre and post bulk-insert signals Changes in v0.4 =============== From 7cd22aaf83e2cbec85c276db8b581d55cbdcc2ab Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 27 Oct 2011 01:18:32 -0700 Subject: [PATCH 0415/1279] Removed debug print --- tests/signals.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/signals.py b/tests/signals.py index fd282b2..4dc683e 100644 --- a/tests/signals.py +++ b/tests/signals.py @@ -17,7 +17,6 @@ class SignalTests(unittest.TestCase): global signal_output signal_output = [] fn(*args, **kwargs) - print signal_output return signal_output def setUp(self): @@ -213,9 +212,9 @@ class SignalTests(unittest.TestCase): # The output of this signal is not entirely deterministic. The reloaded # object will have an object ID. Hence, we only check part of the output - self.assertEquals(signal_output[3], + self.assertEquals(signal_output[3], "pre_bulk_insert signal, []") - self.assertEquals(signal_output[-2:], + self.assertEquals(signal_output[-2:], ["post_bulk_insert signal, []", "Is loaded",]) From 1acdb880fc7dfe3cc62b5615e7c82d84daae4ff5 Mon Sep 17 00:00:00 2001 From: Sergey Chvalyuk Date: Fri, 28 Oct 2011 00:23:13 +0300 Subject: [PATCH 0416/1279] fixing #336 --- mongoengine/queryset.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a662685..bb423af 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1609,10 +1609,16 @@ class QuerySet(object): def __repr__(self): limit = REPR_OUTPUT_SIZE + 1 - if self._limit is not None and self._limit < limit: - limit = self._limit + start = ( 0 if self._skip is None else self._skip ) + if self._limit is None: + stop = start + limit + if self._limit is not None: + if self._limit - start > limit: + stop = start + limit + else: + stop = self._limit try: - data = list(self[self._skip:limit]) + data = list(self[start:stop]) except pymongo.errors.InvalidOperation: return ".. queryset mid-iteration .." if len(data) > REPR_OUTPUT_SIZE: From ecdf2ae5c7ad23e92df080da6c47b0a6fd8d72b2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 1 Nov 2011 01:20:47 -0700 Subject: [PATCH 0417/1279] Updated docs and Authors --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 587c74d..200182b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -71,4 +71,5 @@ that much better: * Paul Aliagas * Paul Cunnane * Julien Rebetez + * Marc Tamlyn diff --git a/docs/changelog.rst b/docs/changelog.rst index d381a82..9f109cc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed breaking during queryset iteration - Added pre and post bulk-insert signals - Added ImageField - requires PIL - Fixed Reference Fields can be None in get_or_create / queries From 3ee60affa940a59b867a5d2e47811d2ec93c8202 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 1 Nov 2011 01:51:58 -0700 Subject: [PATCH 0418/1279] Renamed schema for errors Now is `to_dict()` as is more explicit [refs #344 #328] --- mongoengine/base.py | 11 +++++------ tests/fields.py | 42 +++++++++++++++++++++--------------------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 8eaa05b..f22374b 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -52,23 +52,22 @@ class ValidationError(AssertionError): message = property(_get_message, _set_message) - @property - def schema(self): - def get_schema(source): + def to_dict(self): + def build_dict(source): errors_dict = {} if not source: return errors_dict if isinstance(source, dict): for field_name, error in source.iteritems(): - errors_dict[field_name] = get_schema(error) + errors_dict[field_name] = build_dict(error) elif isinstance(source, ValidationError) and source.errors: - return get_schema(source.errors) + return build_dict(source.errors) else: return unicode(source) return errors_dict if not self.errors: return {} - return get_schema(self.errors) + return build_dict(self.errors) _document_registry = {} diff --git a/tests/fields.py b/tests/fields.py index 6920648..dd68cb5 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1642,7 +1642,7 @@ class FieldTest(unittest.TestCase): self.assertTrue(isinstance(person.like, Dish)) def test_recursive_validation(self): - """Ensure that a validation result schema is available. + """Ensure that a validation result to_dict is available. """ class Author(EmbeddedDocument): name = StringField(required=True) @@ -1674,12 +1674,12 @@ class FieldTest(unittest.TestCase): ValidationError)) # ValidationError.schema property - schema = error.schema - self.assertTrue(isinstance(schema, dict)) - self.assertTrue('comments' in schema) - self.assertTrue(1 in schema['comments']) - self.assertTrue('content' in schema['comments'][1]) - self.assertEquals(schema['comments'][1]['content'], + error_dict = error.to_dict() + self.assertTrue(isinstance(error_dict, dict)) + self.assertTrue('comments' in error_dict) + self.assertTrue(1 in error_dict['comments']) + self.assertTrue('content' in error_dict['comments'][1]) + self.assertEquals(error_dict['comments'][1]['content'], u'Field is required ("content")') post.comments[1].content = 'here we go' @@ -1688,25 +1688,25 @@ class FieldTest(unittest.TestCase): class ValidatorErrorTest(unittest.TestCase): - def test_schema(self): - """Ensure a ValidationError handles error schema correctly. + def test_to_dict(self): + """Ensure a ValidationError handles error to_dict correctly. """ error = ValidationError('root') - self.assertEquals(error.schema, {}) + self.assertEquals(error.to_dict(), {}) # 1st level error schema error.errors = {'1st': ValidationError('bad 1st'), } - self.assertTrue('1st' in error.schema) - self.assertEquals(error.schema['1st'], 'bad 1st') + self.assertTrue('1st' in error.to_dict()) + self.assertEquals(error.to_dict()['1st'], 'bad 1st') # 2nd level error schema error.errors = {'1st': ValidationError('bad 1st', errors={ '2nd': ValidationError('bad 2nd'), })} - self.assertTrue('1st' in error.schema) - self.assertTrue(isinstance(error.schema['1st'], dict)) - self.assertTrue('2nd' in error.schema['1st']) - self.assertEquals(error.schema['1st']['2nd'], 'bad 2nd') + self.assertTrue('1st' in error.to_dict()) + self.assertTrue(isinstance(error.to_dict()['1st'], dict)) + self.assertTrue('2nd' in error.to_dict()['1st']) + self.assertEquals(error.to_dict()['1st']['2nd'], 'bad 2nd') # moar levels error.errors = {'1st': ValidationError('bad 1st', errors={ @@ -1716,11 +1716,11 @@ class ValidatorErrorTest(unittest.TestCase): }), }), })} - self.assertTrue('1st' in error.schema) - self.assertTrue('2nd' in error.schema['1st']) - self.assertTrue('3rd' in error.schema['1st']['2nd']) - self.assertTrue('4th' in error.schema['1st']['2nd']['3rd']) - self.assertEquals(error.schema['1st']['2nd']['3rd']['4th'], + self.assertTrue('1st' in error.to_dict()) + self.assertTrue('2nd' in error.to_dict()['1st']) + self.assertTrue('3rd' in error.to_dict()['1st']['2nd']) + self.assertTrue('4th' in error.to_dict()['1st']['2nd']['3rd']) + self.assertEquals(error.to_dict()['1st']['2nd']['3rd']['4th'], 'Inception') From 59bd72a8881dacabb1bb39ed0a23909744987c25 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 1 Nov 2011 02:15:31 -0700 Subject: [PATCH 0419/1279] Added tests for __repr__ fix --- docs/changelog.rst | 1 + tests/queryset.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 64142fd..facb1b9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed __repr__ of a sliced queryset - Added recursive validation error of documents / complex fields - Fixed breaking during queryset iteration - Added pre and post bulk-insert signals diff --git a/tests/queryset.py b/tests/queryset.py index 67f1ea2..5f5e78b 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -110,6 +110,16 @@ class QuerySetTest(unittest.TestCase): people = list(self.Person.objects[80000:80001]) self.assertEqual(len(people), 0) + # Test larger slice __repr__ + self.Person.objects.delete() + for i in xrange(55): + self.Person(name='A%s' % i, age=i).save() + + self.assertEqual(len(self.Person.objects), 55) + self.assertEqual("Person object", "%s" % self.Person.objects[0]) + self.assertEqual("[, ]", "%s" % self.Person.objects[1:3]) + self.assertEqual("[, ]", "%s" % self.Person.objects[51:53]) + def test_find_one(self): """Ensure that a query using find_one returns a valid result. """ From a9fc476fb8c7e632aa2df12b98a4ea793d96a51c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Wed, 2 Nov 2011 09:38:26 -0200 Subject: [PATCH 0420/1279] fixed errors in repr if unicode string is found --- mongoengine/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index f22374b..56c7259 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1080,10 +1080,10 @@ class BaseDocument(object): def __repr__(self): try: - u = unicode(self) + u = unicode(self).encode('utf-8') except (UnicodeEncodeError, UnicodeDecodeError): u = '[Bad Unicode data]' - return u'<%s: %s>' % (self.__class__.__name__, u) + return '<%s: %s>' % (self.__class__.__name__, u) def __str__(self): if hasattr(self, '__unicode__'): From 4e44198bbdcfe71c8109ac70f9ae0cbed5b8b933 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 4 Nov 2011 01:45:44 -0700 Subject: [PATCH 0421/1279] Clean up of choices code and added tests [#284] [#314] --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/base.py | 18 +++++++++-------- tests/fields.py | 47 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/AUTHORS b/AUTHORS index 200182b..9ebc005 100644 --- a/AUTHORS +++ b/AUTHORS @@ -72,4 +72,5 @@ that much better: * Paul Cunnane * Julien Rebetez * Marc Tamlyn + * Karim Allah diff --git a/docs/changelog.rst b/docs/changelog.rst index facb1b9..dbb4a72 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added Non-Django Style choices back (you can have either) - Fixed __repr__ of a sliced queryset - Added recursive validation error of documents / complex fields - Fixed breaking during queryset iteration diff --git a/mongoengine/base.py b/mongoengine/base.py index ab6ccbe..24a3797 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -195,14 +195,14 @@ class BaseField(object): def _validate(self, value): # check choices - if self.choices is not None: - if type(choices[0]) is tuple: - option_keys = [option_key for option_key, option_value in self.choices] - if value not in option_keys: - self.error('Value must be one of %s' % unicode(option_keys)) - else: - if value not in self.choices: + if self.choices: + if isinstance(self.choices[0], (list, tuple)): + option_keys = [option_key for option_key, option_value in self.choices] + if value not in option_keys: self.error('Value must be one of %s' % unicode(option_keys)) + else: + if value not in self.choices: + self.error('Value must be one of %s' % unicode(self.choices)) # check validation argument if self.validation is not None: @@ -1051,7 +1051,9 @@ class BaseDocument(object): def __get_field_display(self, field): """Returns the display value for a choice field""" value = getattr(self, field.name) - return dict(field.choices).get(value, value) + if field.choices and isinstance(field.choices[0], (list, tuple)): + return dict(field.choices).get(value, value) + return value def __iter__(self): return iter(self._fields) diff --git a/tests/fields.py b/tests/fields.py index dd68cb5..3b69791 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1280,6 +1280,53 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() + def test_simple_choices_validation(self): + """Ensure that value is in a container of allowed values. + """ + class Shirt(Document): + size = StringField(max_length=3, choices=('S', 'M', 'L', 'XL', 'XXL')) + + Shirt.drop_collection() + + shirt = Shirt() + shirt.validate() + + shirt.size = "S" + shirt.validate() + + shirt.size = "XS" + self.assertRaises(ValidationError, shirt.validate) + + Shirt.drop_collection() + + def test_simple_choices_get_field_display(self): + """Test dynamic helper for returning the display value of a choices field. + """ + class Shirt(Document): + size = StringField(max_length=3, choices=('S', 'M', 'L', 'XL', 'XXL')) + style = StringField(max_length=3, choices=('Small', 'Baggy', 'wide'), default='Small') + + Shirt.drop_collection() + + shirt = Shirt() + + self.assertEqual(shirt.get_size_display(), None) + self.assertEqual(shirt.get_style_display(), 'Small') + + shirt.size = "XXL" + shirt.style = "Baggy" + self.assertEqual(shirt.get_size_display(), 'XXL') + self.assertEqual(shirt.get_style_display(), 'Baggy') + + # Set as Z - an invalid choice + shirt.size = "Z" + shirt.style = "Z" + self.assertEqual(shirt.get_size_display(), 'Z') + self.assertEqual(shirt.get_style_display(), 'Z') + self.assertRaises(ValidationError, shirt.validate) + + Shirt.drop_collection() + def test_file_fields(self): """Ensure that file fields can be written to and their data retrieved """ From 4c1509a62a83167637c8b3fb885334c600ebd7d9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 4 Nov 2011 01:54:30 -0700 Subject: [PATCH 0422/1279] Updated docs re choices [#284] [#314] --- docs/guide/defining-documents.rst | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 6367d95..730d180 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -135,7 +135,28 @@ arguments can be set on all fields: When True, use this field as a primary key for the collection. :attr:`choices` (Default: None) - An iterable of choices to which the value of this field should be limited. + An iterable (e.g. a list or tuple) of choices to which the value of this + field should be limited. + + Can be either be a nested tuples of value (stored in mongo) and a + human readable key :: + + SIZE = (('S', 'Small'), + ('M', 'Medium'), + ('L', 'Large'), + ('XL', 'Extra Large'), + ('XXL', 'Extra Extra Large')) + + + class Shirt(Document): + size = StringField(max_length=3, choices=SIZE) + + Or a flat iterable just containing values :: + + SIZE = ('S', 'M', 'L', 'XL', 'XXL') + + class Shirt(Document): + size = StringField(max_length=3, choices=SIZE) :attr:`help_text` (Default: None) Optional help text to output with the field - used by form libraries From 5aeee9deb2e7df841bf2dd01b317f096e460415d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 4 Nov 2011 01:55:46 -0700 Subject: [PATCH 0423/1279] Added PIL to spec file [#314] --- python-mongoengine.spec | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 054993d..3609a54 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,8 +5,8 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.5.0 -Release: 2%{?dist} +Version: 0.5.3 +Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB Group: Development/Libraries @@ -20,6 +20,7 @@ BuildRequires: python-setuptools Requires: mongodb Requires: pymongo Requires: python-blinker +Requires: python-imaging %description MongoEngine is an ORM-like layer on top of PyMongo. @@ -49,5 +50,12 @@ rm -rf $RPM_BUILD_ROOT # %{python_sitearch}/* %changelog +* Thu Oct 27 2011 Pau Aliagas 0.5.3-1 +- Update to latest dev version +* Add PIL dependency for ImageField + +* Wed Oct 12 2011 Pau Aliagas 0.5.2-1 +- Update version + * Fri Sep 23 2011 Pau Aliagas 0.5.0-1 - Initial version From 34646a414c15b016a10bbc95f4c1bd33baa88b97 Mon Sep 17 00:00:00 2001 From: Adam Parrish Date: Thu, 10 Nov 2011 12:03:51 -0800 Subject: [PATCH 0424/1279] Fixes bug using positional operator to update embedded documents. append_field wasn't getting reset to True in the loop, so fields wouldn't get appended to clean_fields after str was encountered [#354] --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/queryset.py | 2 +- tests/queryset.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 9ebc005..4f68be2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -73,4 +73,5 @@ that much better: * Julien Rebetez * Marc Tamlyn * Karim Allah + * Adam Parrish diff --git a/docs/changelog.rst b/docs/changelog.rst index dbb4a72..5394312 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed positional operator when replacing embedded documents - Added Non-Django Style choices back (you can have either) - Fixed __repr__ of a sliced queryset - Added recursive validation error of documents / complex fields diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 8ebebe3..6f01b76 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1276,8 +1276,8 @@ class QuerySet(object): parts = [] cleaned_fields = [] - append_field = True for field in fields: + append_field = True if isinstance(field, str): # Convert the S operator to $ if field == 'S': diff --git a/tests/queryset.py b/tests/queryset.py index 5f5e78b..60adae3 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -378,6 +378,34 @@ class QuerySetTest(unittest.TestCase): self.assertRaises(OperationError, update_nested) Simple.drop_collection() + def test_update_using_positional_operator_embedded_document(self): + """Ensure that the embedded documents can be updated using the positional + operator.""" + + class Vote(EmbeddedDocument): + score = IntField() + + class Comment(EmbeddedDocument): + by = StringField() + votes = EmbeddedDocumentField(Vote) + + class BlogPost(Document): + title = StringField() + comments = ListField(EmbeddedDocumentField(Comment)) + + BlogPost.drop_collection() + + c1 = Comment(by="joe", votes=Vote(score=3)) + c2 = Comment(by="jane", votes=Vote(score=7)) + + BlogPost(title="ABC", comments=[c1, c2]).save() + + BlogPost.objects(comments__by="joe").update(set__comments__S__votes=Vote(score=4)) + + post = BlogPost.objects.first() + self.assertEquals(post.comments[0].by, 'joe') + self.assertEquals(post.comments[0].votes.score, 4) + def test_mapfield_update(self): """Ensure that the MapField can be updated.""" class Member(EmbeddedDocument): From 63c5a4dd65498e3ff150b3bc12dc053f1d402b4a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Nov 2011 07:34:08 -0800 Subject: [PATCH 0425/1279] Fixes saving document schemas that have changed Ensures that form defaults which are documents are automatically marked as changed, so schemas can evolve without migration issues. [#360] --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/base.py | 9 ++++++++- tests/document.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 4f68be2..573ddb1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -74,4 +74,5 @@ that much better: * Marc Tamlyn * Karim Allah * Adam Parrish + * jpfarias diff --git a/docs/changelog.rst b/docs/changelog.rst index 5394312..6ce81e5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed document mutation saving issue - Fixed positional operator when replacing embedded documents - Added Non-Django Style choices back (you can have either) - Fixed __repr__ of a sliced queryset diff --git a/mongoengine/base.py b/mongoengine/base.py index b021eac..4198e8b 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -883,14 +883,21 @@ class BaseDocument(object): """.strip() % class_name) cls = subclasses[class_name] + changed_fields = [] for field_name, field in cls._fields.items(): if field.db_field in data: value = data[field.db_field] data[field_name] = (value if value is None else field.to_python(value)) + elif field.default: + default = field.default + if callable(default): + default = default() + if isinstance(default, BaseDocument): + changed_fields.append(field_name) obj = cls(**data) - obj._changed_fields = [] + obj._changed_fields = changed_fields return obj def _mark_as_changed(self, key): diff --git a/tests/document.py b/tests/document.py index 816bb49..9da886c 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2360,6 +2360,36 @@ class DocumentTest(unittest.TestCase): self.assertRaises(InvalidDocumentError, throw_invalid_document_error) + def test_mutating_documents(self): + + class B(EmbeddedDocument): + field1 = StringField(default='field1') + + class A(Document): + b = EmbeddedDocumentField(B, default=lambda: B()) + + A.drop_collection() + a = A() + a.save() + a.reload() + self.assertEquals(a.b.field1, 'field1') + + class C(EmbeddedDocument): + c_field = StringField(default='cfield') + + class B(EmbeddedDocument): + field1 = StringField(default='field1') + field2 = EmbeddedDocumentField(C, default=lambda: C()) + + class A(Document): + b = EmbeddedDocumentField(B, default=lambda: B()) + + a = A.objects()[0] + a.b.field2.c_field = 'new value' + a.save() + + a.reload() + self.assertEquals(a.b.field2.c_field, 'new value') if __name__ == '__main__': unittest.main() From fa4b820931743dbc76455cd5d8da7bc17440c04e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 22 Nov 2011 13:40:01 -0200 Subject: [PATCH 0426/1279] added support for db_alias in FileFields --- mongoengine/connection.py | 3 ++- mongoengine/fields.py | 18 +++++++++++++----- tests/fields.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index c7d8f89..07b730b 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,7 +1,8 @@ from pymongo import Connection -__all__ = ['ConnectionError', 'connect', 'register_connection'] +__all__ = ['ConnectionError', 'connect', 'register_connection', + 'DEFAULT_CONNECTION_NAME'] DEFAULT_CONNECTION_NAME = 'default' diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 3e12296..23d9e45 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -13,7 +13,7 @@ from base import (BaseField, ComplexBaseField, ObjectIdField, ValidationError, get_document) from queryset import DO_NOTHING from document import Document, EmbeddedDocument -from connection import get_db +from connection import get_db, DEFAULT_CONNECTION_NAME from operator import itemgetter @@ -779,8 +779,10 @@ class GridFSProxy(object): """ def __init__(self, grid_id=None, key=None, - instance=None, collection_name='fs'): - self.fs = gridfs.GridFS(get_db(), collection_name) # Filesystem instance + instance=None, + db_alias=DEFAULT_CONNECTION_NAME, + collection_name='fs'): + self.fs = gridfs.GridFS(get_db(db_alias), collection_name) # Filesystem instance self.newfile = None # Used for partial writes self.grid_id = grid_id # Store GridFS id for file self.gridout = None @@ -870,12 +872,16 @@ class FileField(BaseField): .. versionadded:: 0.4 .. versionchanged:: 0.5 added optional size param for read + .. versionchanged:: 0.6 added db_alias for multidb support """ proxy_class = GridFSProxy - def __init__(self, collection_name="fs", **kwargs): + def __init__(self, + db_alias=DEFAULT_CONNECTION_NAME, + collection_name="fs", **kwargs): super(FileField, self).__init__(**kwargs) self.collection_name = collection_name + self.db_alias = db_alias def __get__(self, instance, owner): if instance is None: @@ -890,6 +896,7 @@ class FileField(BaseField): self.grid_file.instance = instance return self.grid_file return self.proxy_class(key=self.name, instance=instance, + db_alias=self.db_alias, collection_name=self.collection_name) def __set__(self, instance, value): @@ -924,7 +931,8 @@ class FileField(BaseField): def to_python(self, value): if value is not None: return self.proxy_class(value, - collection_name=self.collection_name) + collection_name=self.collection_name, + db_alias=self.db_alias) def validate(self, value): if value.grid_id is not None: diff --git a/tests/fields.py b/tests/fields.py index 768f18d..5f19e33 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1503,6 +1503,34 @@ class FieldTest(unittest.TestCase): t.image.delete() + + def test_file_multidb(self): + register_connection('testfiles', 'testfiles') + class TestFile(Document): + name = StringField() + file = FileField(db_alias="testfiles", + collection_name="macumba") + + TestFile.drop_collection() + + # delete old filesystem + get_db("testfiles").macumba.files.drop() + get_db("testfiles").macumba.chunks.drop() + + # First instance + testfile = TestFile() + testfile.name = "Hello, World!" + testfile.file.put('Hello, World!', + name="hello.txt") + testfile.save() + + data = get_db("testfiles").macumba.files.find_one() + self.assertEquals(data.get('name'), 'hello.txt') + + testfile = TestFile.objects.first() + self.assertEquals(testfile.file.read(), + 'Hello, World!') + def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """ From e80144e9f2399814760a343ee1c34a4e3e785c26 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 18 Nov 2011 07:22:37 -0800 Subject: [PATCH 0427/1279] Added multidb support No change required to upgrade to multiple databases. Aliases are used to describe the database and these can be manually registered or fall through to a default alias using connect. Made get_connection and get_db first class members of the connection class. Old style _get_connection and _get_db still supported. Refs: #84 #87 #93 #215 --- docs/apireference.rst | 1 + docs/guide/connecting.rst | 12 +++ mongoengine/connection.py | 152 +++++++++++++++++++++---------------- mongoengine/dereference.py | 6 +- mongoengine/document.py | 14 ++-- mongoengine/fields.py | 10 +-- mongoengine/queryset.py | 6 +- mongoengine/tests.py | 4 +- tests/connection.py | 48 ++++++++++++ tests/dereference.py | 4 +- tests/document.py | 10 +-- tests/dynamic_document.py | 5 +- tests/fields.py | 4 +- tests/fixtures.py | 3 - tests/queryset.py | 4 +- 15 files changed, 180 insertions(+), 103 deletions(-) create mode 100644 tests/connection.py diff --git a/docs/apireference.rst b/docs/apireference.rst index 932152f..9e6ed47 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -6,6 +6,7 @@ Connecting ========== .. autofunction:: mongoengine.connect +.. autofunction:: mongoengine.register_connection Documents ========= diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index 64e7666..a51d68e 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -3,6 +3,7 @@ ===================== Connecting to MongoDB ===================== + To connect to a running instance of :program:`mongod`, use the :func:`~mongoengine.connect` function. The first argument is the name of the database to connect to. If the database does not exist, it will be created. If @@ -18,3 +19,14 @@ provide :attr:`host` and :attr:`port` arguments to :func:`~mongoengine.connect`:: connect('project1', host='192.168.1.35', port=12345) + + +Multiple Databases +================== + +Multiple database support was added in MongoEngine 0.6. To use multiple +databases you can use :func:`~mongoengine.connect` and provide an `alias` name +for the connection - if no `alias` is provided then "default" is used. + +In the background this uses :func:`~mongoengine.register_connection` to +store the data and you can register all aliases up front if required. diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 7b5cd21..c7d8f89 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,82 +1,106 @@ from pymongo import Connection -import multiprocessing -import threading - -__all__ = ['ConnectionError', 'connect'] -_connection_defaults = { - 'host': 'localhost', - 'port': 27017, -} -_connection = {} -_connection_settings = _connection_defaults.copy() +__all__ = ['ConnectionError', 'connect', 'register_connection'] -_db_name = None -_db_username = None -_db_password = None -_db = {} + +DEFAULT_CONNECTION_NAME = 'default' class ConnectionError(Exception): pass -def _get_connection(reconnect=False): - """Handles the connection to the database +_connection_settings = {} +_connections = {} +_dbs = {} + + +def register_connection(alias, name, host='localhost', port=27017, + is_slave=False, slaves=None, username=None, + password=None): + """Add a connection. + + :param alias: the name that will be used to refer to this connection + throughout MongoEngine + :param name: the name of the specific database to use + :param host: the host name of the :program:`mongod` instance to connect to + :param port: the port that the :program:`mongod` instance is running on + :param is_slave: whether the connection can act as a slave + :param slaves: a list of aliases of slave connections; each of these must + be a registered connection that has :attr:`is_slave` set to ``True`` + :param username: username to authenticate with + :param password: password to authenticate with """ - global _connection - identity = get_identity() + global _connection_settings + _connection_settings[alias] = { + 'name': name, + 'host': host, + 'port': port, + 'is_slave': is_slave, + 'slaves': slaves or [], + 'username': username, + 'password': password, + } + + +def get_connection(alias=DEFAULT_CONNECTION_NAME): + global _connections # Connect to the database if not already connected - if _connection.get(identity) is None or reconnect: + if alias not in _connections: + if alias not in _connection_settings: + msg = 'Connection with alias "%s" has not been defined' + if alias == DEFAULT_CONNECTION_NAME: + msg = 'You have not defined a default connection' + raise ConnectionError(msg) + conn_settings = _connection_settings[alias].copy() + + # Get all the slave connections + slaves = [] + for slave_alias in conn_settings['slaves']: + slaves.append(get_connection(slave_alias)) + conn_settings['slaves'] = slaves + try: - _connection[identity] = Connection(**_connection_settings) + _connections[alias] = Connection(**conn_settings) except Exception, e: - raise ConnectionError("Cannot connect to the database:\n%s" % e) - return _connection[identity] + raise e + raise ConnectionError('Cannot connect to database %s' % alias) + return _connections[alias] -def _get_db(reconnect=False): - """Handles database connections and authentication based on the current - identity + +def get_db(alias=DEFAULT_CONNECTION_NAME): + global _dbs + if alias not in _dbs: + conn = get_connection(alias) + conn_settings = _connection_settings[alias] + _dbs[alias] = conn[conn_settings['name']] + + # Authenticate if necessary + if conn_settings['username'] and conn_settings['password']: + _dbs[alias].authenticate(conn_settings['username'], + conn_settings['password']) + return _dbs[alias] + + +def connect(db, alias=DEFAULT_CONNECTION_NAME, **kwargs): + """Connect to the database specified by the 'db' argument. + + Connection settings may be provided here as well if the database is not + running on the default port on localhost. If authentication is needed, + provide username and password arguments as well. + + Multiple databases are supported by using aliases. Provide a separate + `alias` to connect to a different instance of :program:`mongod`. + + .. versionchanged:: 0.6 - added multiple database support. """ - global _db, _connection - identity = get_identity() - # Connect if not already connected - if _connection.get(identity) is None or reconnect: - _connection[identity] = _get_connection(reconnect=reconnect) + global _connections + if alias not in _connections: + register_connection(alias, db, **kwargs) - if _db.get(identity) is None or reconnect: - # _db_name will be None if the user hasn't called connect() - if _db_name is None: - raise ConnectionError('Not connected to the database') - - # Get DB from current connection and authenticate if necessary - _db[identity] = _connection[identity][_db_name] - if _db_username and _db_password: - _db[identity].authenticate(_db_username, _db_password) - - return _db[identity] - -def get_identity(): - """Creates an identity key based on the current process and thread - identity. - """ - identity = multiprocessing.current_process()._identity - identity = 0 if not identity else identity[0] - - identity = (identity, threading.current_thread().ident) - return identity - -def connect(db, username=None, password=None, **kwargs): - """Connect to the database specified by the 'db' argument. Connection - settings may be provided here as well if the database is not running on - the default port on localhost. If authentication is needed, provide - username and password arguments as well. - """ - global _connection_settings, _db_name, _db_username, _db_password, _db - _connection_settings = dict(_connection_defaults, **kwargs) - _db_name = db - _db_username = username - _db_password = password - return _get_db(reconnect=True) + return get_connection(alias) +# Support old naming convention +_get_connection = get_connection +_get_db = get_db diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 949bb2f..d817a03 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -1,10 +1,8 @@ -import operator - import pymongo from base import BaseDict, BaseList, get_document, TopLevelDocumentMetaclass from fields import ReferenceField -from connection import _get_db +from connection import get_db from queryset import QuerySet from document import Document @@ -103,7 +101,7 @@ class DeReference(object): for key, doc in references.iteritems(): object_map[key] = doc else: # Generic reference: use the refs data to convert to document - references = _get_db()[col].find({'_id': {'$in': refs}}) + references = get_db()[col].find({'_id': {'$in': refs}}) for ref in references: if '_cls' in ref: doc = get_document(ref['_cls'])._from_son(ref) diff --git a/mongoengine/document.py b/mongoengine/document.py index 82b94a3..f3893dd 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,14 +1,14 @@ -import operator from mongoengine import signals from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, ValidationError, BaseDict, BaseList, BaseDynamicField) from queryset import OperationError -from connection import _get_db +from connection import get_db import pymongo -__all__ = ['Document', 'EmbeddedDocument', 'DynamicDocument', 'DynamicEmbeddedDocument', - 'ValidationError', 'OperationError', 'InvalidCollectionError'] +__all__ = ['Document', 'EmbeddedDocument', 'DynamicDocument', + 'DynamicEmbeddedDocument', 'ValidationError', 'OperationError', + 'InvalidCollectionError'] class InvalidCollectionError(Exception): @@ -76,7 +76,7 @@ class Document(BaseDocument): by setting index_types to False on the meta dictionary for the document. """ __metaclass__ = TopLevelDocumentMetaclass - + @apply def pk(): """Primary key alias @@ -91,7 +91,7 @@ class Document(BaseDocument): def _get_collection(self): """Returns the collection for the document.""" if not hasattr(self, '_collection') or self._collection is None: - db = _get_db() + db = get_db() collection_name = self._get_collection_name() # Create collection as a capped collection if specified if self._meta['max_size'] or self._meta['max_documents']: @@ -300,7 +300,7 @@ class Document(BaseDocument): :class:`~mongoengine.Document` type from the database. """ from mongoengine.queryset import QuerySet - db = _get_db() + db = get_db() db.drop_collection(cls._get_collection_name()) QuerySet._reset_already_indexed(cls) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 0bfd54d..3e12296 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -13,7 +13,7 @@ from base import (BaseField, ComplexBaseField, ObjectIdField, ValidationError, get_document) from queryset import DO_NOTHING from document import Document, EmbeddedDocument -from connection import _get_db +from connection import get_db from operator import itemgetter @@ -637,7 +637,7 @@ class ReferenceField(BaseField): value = instance._data.get(self.name) # Dereference DBRefs if isinstance(value, (pymongo.dbref.DBRef)): - value = _get_db().dereference(value) + value = get_db().dereference(value) if value is not None: instance._data[self.name] = self.document_type._from_son(value) @@ -710,7 +710,7 @@ class GenericReferenceField(BaseField): def dereference(self, value): doc_cls = get_document(value['_cls']) reference = value['_ref'] - doc = _get_db().dereference(reference) + doc = get_db().dereference(reference) if doc is not None: doc = doc_cls._from_son(doc) return doc @@ -780,7 +780,7 @@ class GridFSProxy(object): def __init__(self, grid_id=None, key=None, instance=None, collection_name='fs'): - self.fs = gridfs.GridFS(_get_db(), collection_name) # Filesystem instance + self.fs = gridfs.GridFS(get_db(), collection_name) # Filesystem instance self.newfile = None # Used for partial writes self.grid_id = grid_id # Store GridFS id for file self.gridout = None @@ -1138,7 +1138,7 @@ class SequenceField(IntField): """ sequence_id = "{0}.{1}".format(self.owner_document._get_collection_name(), self.name) - collection = _get_db()[self.collection_name] + collection = get_db()[self.collection_name] counter = collection.find_and_modify(query={"_id": sequence_id}, update={"$inc": {"next": 1}}, new=True, diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6f01b76..4185e39 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1,4 +1,4 @@ -from connection import _get_db +from connection import get_db from mongoengine import signals import pprint @@ -481,7 +481,7 @@ class QuerySet(object): if self._document not in QuerySet.__already_indexed: # Ensure collection exists - db = _get_db() + db = get_db() if self._collection_obj.name not in db.collection_names(): self._document._collection = None self._collection_obj = self._document._get_collection() @@ -1436,7 +1436,7 @@ class QuerySet(object): scope['query'] = query code = pymongo.code.Code(code, scope=scope) - db = _get_db() + db = get_db() return db.eval(code, *fields) def where(self, where_clause): diff --git a/mongoengine/tests.py b/mongoengine/tests.py index 9584bc7..6866377 100644 --- a/mongoengine/tests.py +++ b/mongoengine/tests.py @@ -1,4 +1,4 @@ -from mongoengine.connection import _get_db +from mongoengine.connection import get_db class query_counter(object): @@ -7,7 +7,7 @@ class query_counter(object): def __init__(self): """ Construct the query_counter. """ self.counter = 0 - self.db = _get_db() + self.db = get_db() def __enter__(self): """ On every with block we need to drop the profile collection. """ diff --git a/tests/connection.py b/tests/connection.py new file mode 100644 index 0000000..e017b38 --- /dev/null +++ b/tests/connection.py @@ -0,0 +1,48 @@ +import unittest +import pymongo + +import mongoengine.connection + +from mongoengine import * +from mongoengine.connection import get_db, get_connection + + +class ConnectionTest(unittest.TestCase): + + def tearDown(self): + mongoengine.connection._connection_settings = {} + mongoengine.connection._connections = {} + mongoengine.connection._dbs = {} + + def test_connect(self): + """Ensure that the connect() method works properly. + """ + connect('mongoenginetest') + + conn = get_connection() + self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + + db = get_db() + self.assertTrue(isinstance(db, pymongo.database.Database)) + self.assertEqual(db.name, 'mongoenginetest') + + connect('mongoenginetest2', alias='testdb') + conn = get_connection('testdb') + self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + + def test_register_connection(self): + """Ensure that connections with different aliases may be registered. + """ + register_connection('testdb', 'mongoenginetest2') + + self.assertRaises(ConnectionError, get_connection) + conn = get_connection('testdb') + self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + + db = get_db('testdb') + self.assertTrue(isinstance(db, pymongo.database.Database)) + self.assertEqual(db.name, 'mongoenginetest2') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/dereference.py b/tests/dereference.py index 088db98..2e24a61 100644 --- a/tests/dereference.py +++ b/tests/dereference.py @@ -1,7 +1,7 @@ import unittest from mongoengine import * -from mongoengine.connection import _get_db +from mongoengine.connection import get_db from mongoengine.tests import query_counter @@ -9,7 +9,7 @@ class FieldTest(unittest.TestCase): def setUp(self): connect(db='mongoenginetest') - self.db = _get_db() + self.db = get_db() def test_list_item_dereference(self): """Ensure that DBRef items in ListFields are dereferenced. diff --git a/tests/document.py b/tests/document.py index 9da886c..dca5fed 100644 --- a/tests/document.py +++ b/tests/document.py @@ -5,22 +5,18 @@ import warnings from datetime import datetime -import pymongo -import pickle -import weakref - from fixtures import Base, Mixin, PickleEmbedded, PickleTest from mongoengine import * -from mongoengine.base import _document_registry, NotRegistered, InvalidDocumentError -from mongoengine.connection import _get_db +from mongoengine.base import NotRegistered, InvalidDocumentError +from mongoengine.connection import get_db class DocumentTest(unittest.TestCase): def setUp(self): connect(db='mongoenginetest') - self.db = _get_db() + self.db = get_db() class Person(Document): name = StringField() diff --git a/tests/dynamic_document.py b/tests/dynamic_document.py index d76b196..19cd466 100644 --- a/tests/dynamic_document.py +++ b/tests/dynamic_document.py @@ -1,13 +1,14 @@ import unittest from mongoengine import * -from mongoengine.connection import _get_db +from mongoengine.connection import get_db + class DynamicDocTest(unittest.TestCase): def setUp(self): connect(db='mongoenginetest') - self.db = _get_db() + self.db = get_db() class Person(DynamicDocument): name = StringField() diff --git a/tests/fields.py b/tests/fields.py index 3b69791..768f18d 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -6,7 +6,7 @@ import uuid from decimal import Decimal from mongoengine import * -from mongoengine.connection import _get_db +from mongoengine.connection import get_db from mongoengine.base import _document_registry, NotRegistered TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') @@ -16,7 +16,7 @@ class FieldTest(unittest.TestCase): def setUp(self): connect(db='mongoenginetest') - self.db = _get_db() + self.db = get_db() def test_default_values(self): """Ensure that default field values are used when creating a document. diff --git a/tests/fixtures.py b/tests/fixtures.py index 5aaba55..32081fe 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,9 +1,6 @@ from datetime import datetime -import pymongo from mongoengine import * -from mongoengine.base import BaseField -from mongoengine.connection import _get_db class PickleEmbedded(EmbeddedDocument): diff --git a/tests/queryset.py b/tests/queryset.py index 60adae3..7c1c8dc 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -7,7 +7,7 @@ from mongoengine.queryset import (QuerySet, QuerySetManager, MultipleObjectsReturned, DoesNotExist, QueryFieldList) from mongoengine import * -from mongoengine.connection import _get_connection +from mongoengine.connection import get_connection from mongoengine.tests import query_counter @@ -2276,7 +2276,7 @@ class QuerySetTest(unittest.TestCase): # check that polygon works for users who have a server >= 1.9 server_version = tuple( - _get_connection().server_info()['version'].split('.') + get_connection().server_info()['version'].split('.') ) required_version = tuple("1.9.0".split(".")) if server_version >= required_version: From 5e553ffaf7028db8d322283de53ad3f878f9d29e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Nov 2011 00:59:43 -0800 Subject: [PATCH 0428/1279] Added reconnect back into the syntax forces a disconnect. --- mongoengine/connection.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 07b730b..1c0504a 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -45,9 +45,23 @@ def register_connection(alias, name, host='localhost', port=27017, } -def get_connection(alias=DEFAULT_CONNECTION_NAME): +def disconnect(alias=DEFAULT_CONNECTION_NAME): + global _connections + global _dbs + + if alias in _connections: + get_connection(alias=alias).disconnect() + del _connections[alias] + if alias in _dbs: + del _dbs[alias] + + +def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): global _connections # Connect to the database if not already connected + if reconnect: + disconnect(alias) + if alias not in _connections: if alias not in _connection_settings: msg = 'Connection with alias "%s" has not been defined' @@ -70,8 +84,11 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME): return _connections[alias] -def get_db(alias=DEFAULT_CONNECTION_NAME): +def get_db(alias=DEFAULT_CONNECTION_NAME, reconnect=False): global _dbs + if reconnect: + disconnect(alias) + if alias not in _dbs: conn = get_connection(alias) conn_settings = _connection_settings[alias] From 83fff80b0fa45a81a7fa3b219bbb91984884902d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 25 Nov 2011 08:28:20 -0800 Subject: [PATCH 0429/1279] Cleaned up dereferencing Dereferencing now respects max_depth, so should be more performant. Reload is chainable and can be passed a max_depth for dereferencing Added an Observer for ComplexBaseFields. Refs #324 #323 #289 Closes #320 --- docs/changelog.rst | 1 + mongoengine/base.py | 95 ++++++++++++++++++++++---------------- mongoengine/dereference.py | 45 +++++++++--------- mongoengine/document.py | 24 ++++++---- mongoengine/queryset.py | 2 + tests/document.py | 52 ++++++++++----------- 6 files changed, 122 insertions(+), 97 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6ce81e5..df897f3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed dereferencing - max_depth now taken into account - Fixed document mutation saving issue - Fixed positional operator when replacing embedded documents - Added Non-Django Style choices back (you can have either) diff --git a/mongoengine/base.py b/mongoengine/base.py index 4198e8b..c801c2e 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -155,9 +155,11 @@ class BaseField(object): # Convert lists / values so we can watch for any changes on them if isinstance(value, (list, tuple)) and not isinstance(value, BaseList): - value = BaseList(value, instance=instance, name=self.name) + observer = DataObserver(instance, self.name) + value = BaseList(value, observer) elif isinstance(value, dict) and not isinstance(value, BaseDict): - value = BaseDict(value, instance=instance, name=self.name) + observer = DataObserver(instance, self.name) + value = BaseDict(value, observer) return value def __set__(self, instance, value): @@ -237,7 +239,7 @@ class ComplexBaseField(BaseField): from dereference import dereference instance._data[self.name] = dereference( - instance._data.get(self.name), max_depth=1, instance=instance, name=self.name, get=True + instance._data.get(self.name), max_depth=1, instance=instance, name=self.name ) return super(ComplexBaseField, self).__get__(instance, owner) @@ -780,9 +782,11 @@ class BaseDocument(object): # Convert lists / values so we can watch for any changes on them if isinstance(value, (list, tuple)) and not isinstance(value, BaseList): - value = BaseList(value, instance=self, name=name) + observer = DataObserver(self, name) + value = BaseList(value, observer) elif isinstance(value, dict) and not isinstance(value, BaseDict): - value = BaseDict(value, instance=self, name=name) + observer = DataObserver(self, name) + value = BaseDict(value, observer) return value @@ -1122,102 +1126,113 @@ class BaseDocument(object): return hash(self.pk) +class DataObserver(object): + + __slots__ = ["instance", "name"] + + def __init__(self, instance, name): + self.instance = instance + self.name = name + + def updated(self): + self.instance._mark_as_changed(self.name) + + class BaseList(list): """A special list so we can watch any changes """ - def __init__(self, list_items, instance, name): - self.instance = instance - self.name = name + def __init__(self, list_items, observer): + self.observer = observer super(BaseList, self).__init__(list_items) def __setitem__(self, *args, **kwargs): - self._mark_as_changed() + self.observer.updated() super(BaseList, self).__setitem__(*args, **kwargs) def __delitem__(self, *args, **kwargs): - self._mark_as_changed() + self.observer.updated() super(BaseList, self).__delitem__(*args, **kwargs) + def __getstate__(self): + self.observer = None + return self + + def __setstate__(self, state): + self = state + def append(self, *args, **kwargs): - self._mark_as_changed() + self.observer.updated() return super(BaseList, self).append(*args, **kwargs) def extend(self, *args, **kwargs): - self._mark_as_changed() + self.observer.updated() return super(BaseList, self).extend(*args, **kwargs) def insert(self, *args, **kwargs): - self._mark_as_changed() + self.observer.updated() return super(BaseList, self).insert(*args, **kwargs) def pop(self, *args, **kwargs): - self._mark_as_changed() + self.observer.updated() return super(BaseList, self).pop(*args, **kwargs) def remove(self, *args, **kwargs): - self._mark_as_changed() + self.observer.updated() return super(BaseList, self).remove(*args, **kwargs) def reverse(self, *args, **kwargs): - self._mark_as_changed() + self.observer.updated() return super(BaseList, self).reverse(*args, **kwargs) def sort(self, *args, **kwargs): - self._mark_as_changed() + self.observer.updated() return super(BaseList, self).sort(*args, **kwargs) - def _mark_as_changed(self): - """Marks a list as changed if has an instance and a name""" - if hasattr(self, 'instance') and hasattr(self, 'name'): - self.instance._mark_as_changed(self.name) - class BaseDict(dict): """A special dict so we can watch any changes """ - def __init__(self, dict_items, instance, name): - self.instance = instance - self.name = name + def __init__(self, dict_items, observer): + self.observer = observer super(BaseDict, self).__init__(dict_items) def __setitem__(self, *args, **kwargs): - self._mark_as_changed() + self.observer.updated() super(BaseDict, self).__setitem__(*args, **kwargs) - def __setattr__(self, *args, **kwargs): - self._mark_as_changed() - super(BaseDict, self).__setattr__(*args, **kwargs) - def __delete__(self, *args, **kwargs): - self._mark_as_changed() + self.observer.updated() super(BaseDict, self).__delete__(*args, **kwargs) def __delitem__(self, *args, **kwargs): - self._mark_as_changed() + self.observer.updated() super(BaseDict, self).__delitem__(*args, **kwargs) def __delattr__(self, *args, **kwargs): - self._mark_as_changed() + self.observer.updated() super(BaseDict, self).__delattr__(*args, **kwargs) + def __getstate__(self): + self.observer = None + return self + + def __setstate__(self, state): + self = state + def clear(self, *args, **kwargs): - self._mark_as_changed() + self.observer.updated() super(BaseDict, self).clear(*args, **kwargs) def pop(self, *args, **kwargs): - self._mark_as_changed() + self.observer.updated() super(BaseDict, self).clear(*args, **kwargs) def popitem(self, *args, **kwargs): - self._mark_as_changed() + self.observer.updated() super(BaseDict, self).clear(*args, **kwargs) - def _mark_as_changed(self): - """Marks a dict as changed if has an instance and a name""" - if hasattr(self, 'instance') and hasattr(self, 'name'): - self.instance._mark_as_changed(self.name) if sys.version_info < (2, 5): # Prior to Python 2.5, Exception was an old-style class diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index d817a03..4e595b1 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -1,6 +1,7 @@ import pymongo -from base import BaseDict, BaseList, get_document, TopLevelDocumentMetaclass +from base import (BaseDict, BaseList, DataObserver, + TopLevelDocumentMetaclass, get_document) from fields import ReferenceField from connection import get_db from queryset import QuerySet @@ -9,7 +10,7 @@ from document import Document class DeReference(object): - def __call__(self, items, max_depth=1, instance=None, name=None, get=False): + def __call__(self, items, max_depth=1, instance=None, name=None): """ Cheaply dereferences the items to a set depth. Also handles the convertion of complex data types. @@ -43,7 +44,7 @@ class DeReference(object): self.reference_map = self._find_references(items) self.object_map = self._fetch_objects(doc_type=doc_type) - return self._attach_objects(items, 0, instance, name, get) + return self._attach_objects(items, 0, instance, name) def _find_references(self, items, depth=0): """ @@ -53,7 +54,7 @@ class DeReference(object): :param depth: The current depth of recursion """ reference_map = {} - if not items: + if not items or depth >= self.max_depth: return reference_map # Determine the iterator to use @@ -63,6 +64,7 @@ class DeReference(object): iterator = items.iteritems() # Recursively find dbreferences + depth += 1 for k, item in iterator: if hasattr(item, '_fields'): for field_name, field in item._fields.iteritems(): @@ -82,11 +84,11 @@ class DeReference(object): reference_map.setdefault(item.collection, []).append(item.id) elif isinstance(item, (dict, pymongo.son.SON)) and '_ref' in item: reference_map.setdefault(get_document(item['_cls']), []).append(item['_ref'].id) - elif isinstance(item, (dict, list, tuple)) and depth <= self.max_depth: - references = self._find_references(item, depth) + elif isinstance(item, (dict, list, tuple)) and depth - 1 <= self.max_depth: + references = self._find_references(item, depth - 1) for key, refs in references.iteritems(): reference_map.setdefault(key, []).extend(refs) - depth += 1 + return reference_map def _fetch_objects(self, doc_type=None): @@ -110,7 +112,7 @@ class DeReference(object): object_map[doc.id] = doc return object_map - def _attach_objects(self, items, depth=0, instance=None, name=None, get=False): + def _attach_objects(self, items, depth=0, instance=None, name=None): """ Recursively finds all db references to be dereferenced @@ -120,25 +122,24 @@ class DeReference(object): :class:`~mongoengine.base.ComplexBaseField` :param name: The name of the field, used for tracking changes by :class:`~mongoengine.base.ComplexBaseField` - :param get: A boolean determining if being called by __get__ """ if not items: if isinstance(items, (BaseDict, BaseList)): return items if instance: + observer = DataObserver(instance, name) if isinstance(items, dict): - return BaseDict(items, instance=instance, name=name) + return BaseDict(items, observer) else: - return BaseList(items, instance=instance, name=name) + return BaseList(items, observer) if isinstance(items, (dict, pymongo.son.SON)): if '_ref' in items: return self.object_map.get(items['_ref'].id, items) elif '_types' in items and '_cls' in items: doc = get_document(items['_cls'])._from_son(items) - if not get: - doc._data = self._attach_objects(doc._data, depth, doc, name, get) + doc._data = self._attach_objects(doc._data, depth, doc, name) return doc if not hasattr(items, 'items'): @@ -150,6 +151,7 @@ class DeReference(object): iterator = items.iteritems() data = {} + depth += 1 for k, v in iterator: if is_list: data.append(v) @@ -165,19 +167,20 @@ class DeReference(object): data[k]._data[field_name] = self.object_map.get(v.id, v) elif isinstance(v, (dict, pymongo.son.SON)) and '_ref' in v: data[k]._data[field_name] = self.object_map.get(v['_ref'].id, v) - elif isinstance(v, dict) and depth < self.max_depth: - data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name, get=get) - elif isinstance(v, (list, tuple)): - data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name, get=get) - elif isinstance(v, (dict, list, tuple)) and depth < self.max_depth: - data[k] = self._attach_objects(v, depth, instance=instance, name=name, get=get) + elif isinstance(v, dict) and depth <= self.max_depth: + data[k]._data[field_name] = self._attach_objects(v, depth - 1, instance=instance, name=name) + elif isinstance(v, (list, tuple)) and depth <= self.max_depth: + data[k]._data[field_name] = self._attach_objects(v, depth - 1, instance=instance, name=name) + elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth: + data[k] = self._attach_objects(v, depth - 1, instance=instance, name=name) elif hasattr(v, 'id'): data[k] = self.object_map.get(v.id, v) if instance and name: + observer = DataObserver(instance, name) if is_list: - return BaseList(data, instance=instance, name=name) - return BaseDict(data, instance=instance, name=name) + return BaseList(data, observer) + return BaseDict(data, observer) depth += 1 return data diff --git a/mongoengine/document.py b/mongoengine/document.py index f3893dd..76685ae 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,14 +1,13 @@ from mongoengine import signals from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, - ValidationError, BaseDict, BaseList, BaseDynamicField) + BaseDict, BaseList, DataObserver) from queryset import OperationError from connection import get_db import pymongo __all__ = ['Document', 'EmbeddedDocument', 'DynamicDocument', - 'DynamicEmbeddedDocument', 'ValidationError', 'OperationError', - 'InvalidCollectionError'] + 'DynamicEmbeddedDocument', 'OperationError', 'InvalidCollectionError'] class InvalidCollectionError(Exception): @@ -250,31 +249,36 @@ class Document(BaseDocument): self._data = dereference(self._data, max_depth) return self - def reload(self): + def reload(self, max_depth=1): """Reloads all attributes from the database. .. versionadded:: 0.1.2 + .. versionchanged:: 0.6 Now chainable """ id_field = self._meta['id_field'] - obj = self.__class__.objects(**{id_field: self[id_field]}).first() - + obj = self.__class__.objects( + **{id_field: self[id_field]} + ).first().select_related(max_depth=max_depth) for field in self._fields: setattr(self, field, self._reload(field, obj[field])) if self._dynamic: for name in self._dynamic_fields.keys(): setattr(self, name, self._reload(name, obj._data[name])) - self._changed_fields = [] + self._changed_fields = obj._changed_fields + return obj def _reload(self, key, value): """Used by :meth:`~mongoengine.Document.reload` to ensure the correct instance is linked to self. """ if isinstance(value, BaseDict): - value = [(k, self._reload(k,v)) for k,v in value.items()] - value = BaseDict(value, instance=self, name=key) + value = [(k, self._reload(k, v)) for k, v in value.items()] + observer = DataObserver(self, key) + value = BaseDict(value, observer) elif isinstance(value, BaseList): value = [self._reload(key, v) for v in value] - value = BaseList(value, instance=self, name=key) + observer = DataObserver(self, key) + value = BaseList(value, observer) elif isinstance(value, (EmbeddedDocument, DynamicEmbeddedDocument)): value._changed_fields = [] return value diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 4185e39..a9b3ea9 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1675,6 +1675,8 @@ class QuerySet(object): .. versionadded:: 0.5 """ from dereference import dereference + # Make select related work the same for querysets + max_depth += 1 return dereference(self, max_depth=max_depth) diff --git a/tests/document.py b/tests/document.py index dca5fed..a1cfcf4 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1069,7 +1069,7 @@ class DocumentTest(unittest.TestCase): doc.embedded_field = embedded_1 doc.save() - doc.reload() + doc = doc.reload(10) doc.list_field.append(1) doc.dict_field['woot'] = "woot" doc.embedded_field.list_field.append(1) @@ -1080,7 +1080,7 @@ class DocumentTest(unittest.TestCase): 'embedded_field.dict_field']) doc.save() - doc.reload() + doc = doc.reload(10) self.assertEquals(doc._get_changed_fields(), []) self.assertEquals(len(doc.list_field), 4) self.assertEquals(len(doc.dict_field), 2) @@ -1502,14 +1502,14 @@ class DocumentTest(unittest.TestCase): self.assertEquals(doc._delta(), ({'embedded_field': embedded_delta}, {})) doc.save() - doc.reload() + doc = doc.reload(10) doc.embedded_field.dict_field = {} self.assertEquals(doc._get_changed_fields(), ['embedded_field.dict_field']) self.assertEquals(doc.embedded_field._delta(), ({}, {'dict_field': 1})) self.assertEquals(doc._delta(), ({}, {'embedded_field.dict_field': 1})) doc.save() - doc.reload() + doc = doc.reload(10) self.assertEquals(doc.embedded_field.dict_field, {}) doc.embedded_field.list_field = [] @@ -1517,7 +1517,7 @@ class DocumentTest(unittest.TestCase): self.assertEquals(doc.embedded_field._delta(), ({}, {'list_field': 1})) self.assertEquals(doc._delta(), ({}, {'embedded_field.list_field': 1})) doc.save() - doc.reload() + doc = doc.reload(10) self.assertEquals(doc.embedded_field.list_field, []) embedded_2 = Embedded() @@ -1550,7 +1550,7 @@ class DocumentTest(unittest.TestCase): }] }, {})) doc.save() - doc.reload() + doc = doc.reload(10) self.assertEquals(doc.embedded_field.list_field[0], '1') self.assertEquals(doc.embedded_field.list_field[1], 2) @@ -1562,7 +1562,7 @@ class DocumentTest(unittest.TestCase): self.assertEquals(doc.embedded_field._delta(), ({'list_field.2.string_field': 'world'}, {})) self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.string_field': 'world'}, {})) doc.save() - doc.reload() + doc = doc.reload(10) self.assertEquals(doc.embedded_field.list_field[2].string_field, 'world') # Test multiple assignments @@ -1587,40 +1587,40 @@ class DocumentTest(unittest.TestCase): 'dict_field': {'hello': 'world'}} ]}, {})) doc.save() - doc.reload() + doc = doc.reload(10) self.assertEquals(doc.embedded_field.list_field[2].string_field, 'hello world') # Test list native methods doc.embedded_field.list_field[2].list_field.pop(0) self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}]}, {})) doc.save() - doc.reload() + doc = doc.reload(10) doc.embedded_field.list_field[2].list_field.append(1) self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}, 1]}, {})) doc.save() - doc.reload() + doc = doc.reload(10) self.assertEquals(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) doc.embedded_field.list_field[2].list_field.sort() doc.save() - doc.reload() + doc = doc.reload(10) self.assertEquals(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) del(doc.embedded_field.list_field[2].list_field[2]['hello']) self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) doc.save() - doc.reload() + doc = doc.reload(10) del(doc.embedded_field.list_field[2].list_field) self.assertEquals(doc._delta(), ({}, {'embedded_field.list_field.2.list_field': 1})) doc.save() - doc.reload() + doc = doc.reload(10) doc.dict_field['Embedded'] = embedded_1 doc.save() - doc.reload() + doc = doc.reload(10) doc.dict_field['Embedded'].string_field = 'Hello World' self.assertEquals(doc._get_changed_fields(), ['dict_field.Embedded.string_field']) @@ -1684,7 +1684,7 @@ class DocumentTest(unittest.TestCase): doc.dict_field = {'hello': 'world'} doc.list_field = ['1', 2, {'hello': 'world'}] doc.save() - doc.reload() + doc = doc.reload(10) self.assertEquals(doc.string_field, 'hello') self.assertEquals(doc.int_field, 1) @@ -1735,14 +1735,14 @@ class DocumentTest(unittest.TestCase): self.assertEquals(doc._delta(), ({'db_embedded_field': embedded_delta}, {})) doc.save() - doc.reload() + doc = doc.reload(10) doc.embedded_field.dict_field = {} self.assertEquals(doc._get_changed_fields(), ['db_embedded_field.db_dict_field']) self.assertEquals(doc.embedded_field._delta(), ({}, {'db_dict_field': 1})) self.assertEquals(doc._delta(), ({}, {'db_embedded_field.db_dict_field': 1})) doc.save() - doc.reload() + doc = doc.reload(10) self.assertEquals(doc.embedded_field.dict_field, {}) doc.embedded_field.list_field = [] @@ -1750,7 +1750,7 @@ class DocumentTest(unittest.TestCase): self.assertEquals(doc.embedded_field._delta(), ({}, {'db_list_field': 1})) self.assertEquals(doc._delta(), ({}, {'db_embedded_field.db_list_field': 1})) doc.save() - doc.reload() + doc = doc.reload(10) self.assertEquals(doc.embedded_field.list_field, []) embedded_2 = Embedded() @@ -1783,7 +1783,7 @@ class DocumentTest(unittest.TestCase): }] }, {})) doc.save() - doc.reload() + doc = doc.reload(10) self.assertEquals(doc.embedded_field.list_field[0], '1') self.assertEquals(doc.embedded_field.list_field[1], 2) @@ -1795,7 +1795,7 @@ class DocumentTest(unittest.TestCase): self.assertEquals(doc.embedded_field._delta(), ({'db_list_field.2.db_string_field': 'world'}, {})) self.assertEquals(doc._delta(), ({'db_embedded_field.db_list_field.2.db_string_field': 'world'}, {})) doc.save() - doc.reload() + doc = doc.reload(10) self.assertEquals(doc.embedded_field.list_field[2].string_field, 'world') # Test multiple assignments @@ -1820,30 +1820,30 @@ class DocumentTest(unittest.TestCase): 'db_dict_field': {'hello': 'world'}} ]}, {})) doc.save() - doc.reload() + doc = doc.reload(10) self.assertEquals(doc.embedded_field.list_field[2].string_field, 'hello world') # Test list native methods doc.embedded_field.list_field[2].list_field.pop(0) self.assertEquals(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [2, {'hello': 'world'}]}, {})) doc.save() - doc.reload() + doc = doc.reload(10) doc.embedded_field.list_field[2].list_field.append(1) self.assertEquals(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [2, {'hello': 'world'}, 1]}, {})) doc.save() - doc.reload() + doc = doc.reload(10) self.assertEquals(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) doc.embedded_field.list_field[2].list_field.sort() doc.save() - doc.reload() + doc = doc.reload(10) self.assertEquals(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) del(doc.embedded_field.list_field[2].list_field[2]['hello']) self.assertEquals(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [1, 2, {}]}, {})) doc.save() - doc.reload() + doc = doc.reload(10) del(doc.embedded_field.list_field[2].list_field) self.assertEquals(doc._delta(), ({}, {'db_embedded_field.db_list_field.2.db_list_field': 1})) @@ -2344,7 +2344,7 @@ class DocumentTest(unittest.TestCase): resurrected.string = "Two" resurrected.save() - pickle_doc.reload() + pickle_doc = pickle_doc.reload() self.assertEquals(resurrected, pickle_doc) def test_throw_invalid_document_error(self): From 0075c0a1e85929a4b418b8b73f3451173653ed5f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Nov 2011 05:54:03 -0800 Subject: [PATCH 0430/1279] Gracefully handle when self.observer is absent After pickles / deepcopying etc.. --- mongoengine/base.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index c801c2e..5a10cc7 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1135,7 +1135,8 @@ class DataObserver(object): self.name = name def updated(self): - self.instance._mark_as_changed(self.name) + if hasattr(self.instance, '_mark_as_changed'): + self.instance._mark_as_changed(self.name) class BaseList(list): @@ -1146,6 +1147,11 @@ class BaseList(list): self.observer = observer super(BaseList, self).__init__(list_items) + def __getattribute__(self, name): + if name == 'observer' and not hasattr(self, 'observer'): + self.observer = DataObserver(None, None) + return super(BaseList, self).__getattribute__(name) + def __setitem__(self, *args, **kwargs): self.observer.updated() super(BaseList, self).__setitem__(*args, **kwargs) @@ -1198,6 +1204,11 @@ class BaseDict(dict): self.observer = observer super(BaseDict, self).__init__(dict_items) + def __getattribute__(self, name): + if name == 'observer' and not hasattr(self, 'observer'): + self.observer = DataObserver(None, None) + return super(BaseList, self).__getattribute__(name) + def __setitem__(self, *args, **kwargs): self.observer.updated() super(BaseDict, self).__setitem__(*args, **kwargs) From aa5c776f3d630b70d8b5a5f3bc226eaeee1ec01a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Nov 2011 06:21:45 -0800 Subject: [PATCH 0431/1279] Copy and paste == brainless --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 5a10cc7..18a831d 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1207,7 +1207,7 @@ class BaseDict(dict): def __getattribute__(self, name): if name == 'observer' and not hasattr(self, 'observer'): self.observer = DataObserver(None, None) - return super(BaseList, self).__getattribute__(name) + return super(BaseDict, self).__getattribute__(name) def __setitem__(self, *args, **kwargs): self.observer.updated() From 4607b08be58e6299194a84b424453ad3b3ccb397 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Nov 2011 06:35:19 -0800 Subject: [PATCH 0432/1279] Making BaseDict / List more robust --- mongoengine/base.py | 53 +++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 18a831d..625d1a1 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1147,17 +1147,12 @@ class BaseList(list): self.observer = observer super(BaseList, self).__init__(list_items) - def __getattribute__(self, name): - if name == 'observer' and not hasattr(self, 'observer'): - self.observer = DataObserver(None, None) - return super(BaseList, self).__getattribute__(name) - def __setitem__(self, *args, **kwargs): - self.observer.updated() + self._updated() super(BaseList, self).__setitem__(*args, **kwargs) def __delitem__(self, *args, **kwargs): - self.observer.updated() + self._updated() super(BaseList, self).__delitem__(*args, **kwargs) def __getstate__(self): @@ -1168,33 +1163,39 @@ class BaseList(list): self = state def append(self, *args, **kwargs): - self.observer.updated() + self._updated() return super(BaseList, self).append(*args, **kwargs) def extend(self, *args, **kwargs): - self.observer.updated() + self._updated() return super(BaseList, self).extend(*args, **kwargs) def insert(self, *args, **kwargs): - self.observer.updated() + self._updated() return super(BaseList, self).insert(*args, **kwargs) def pop(self, *args, **kwargs): - self.observer.updated() + self._updated() return super(BaseList, self).pop(*args, **kwargs) def remove(self, *args, **kwargs): - self.observer.updated() + self._updated() return super(BaseList, self).remove(*args, **kwargs) def reverse(self, *args, **kwargs): - self.observer.updated() + self._updated() return super(BaseList, self).reverse(*args, **kwargs) def sort(self, *args, **kwargs): - self.observer.updated() + self._updated() return super(BaseList, self).sort(*args, **kwargs) + def _updated(self): + try: + self.observer.updated() + except AttributeError: + pass + class BaseDict(dict): """A special dict so we can watch any changes @@ -1204,25 +1205,20 @@ class BaseDict(dict): self.observer = observer super(BaseDict, self).__init__(dict_items) - def __getattribute__(self, name): - if name == 'observer' and not hasattr(self, 'observer'): - self.observer = DataObserver(None, None) - return super(BaseDict, self).__getattribute__(name) - def __setitem__(self, *args, **kwargs): - self.observer.updated() + self._updated() super(BaseDict, self).__setitem__(*args, **kwargs) def __delete__(self, *args, **kwargs): - self.observer.updated() + self._updated() super(BaseDict, self).__delete__(*args, **kwargs) def __delitem__(self, *args, **kwargs): - self.observer.updated() + self._updated() super(BaseDict, self).__delitem__(*args, **kwargs) def __delattr__(self, *args, **kwargs): - self.observer.updated() + self._updated() super(BaseDict, self).__delattr__(*args, **kwargs) def __getstate__(self): @@ -1233,17 +1229,22 @@ class BaseDict(dict): self = state def clear(self, *args, **kwargs): - self.observer.updated() + self._updated() super(BaseDict, self).clear(*args, **kwargs) def pop(self, *args, **kwargs): - self.observer.updated() + self._updated() super(BaseDict, self).clear(*args, **kwargs) def popitem(self, *args, **kwargs): - self.observer.updated() + self._updated() super(BaseDict, self).clear(*args, **kwargs) + def _updated(self): + try: + self.observer.updated() + except AttributeError: + pass if sys.version_info < (2, 5): # Prior to Python 2.5, Exception was an old-style class From e1bb453f321f349ae9e5e9c1a403feaea67dae0d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Nov 2011 05:17:19 -0800 Subject: [PATCH 0433/1279] Configurable cascading saves Updated cascading save logic - can now add meta or pass cascade to save(). Also Cleaned up reset changed fields logic as well, so less looping Refs: #370 #349 --- mongoengine/document.py | 58 ++++++++++++++++++++++------------------- tests/document.py | 29 +++++++++++++++++++++ 2 files changed, 60 insertions(+), 27 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 76685ae..a9ab103 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -120,7 +120,8 @@ class Document(BaseDocument): self._collection = db[collection_name] return self._collection - def save(self, safe=True, force_insert=False, validate=True, write_options=None, _refs=None): + def save(self, safe=True, force_insert=False, validate=True, write_options=None, + cascade=None, _refs=None): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be created. @@ -138,14 +139,19 @@ class Document(BaseDocument): which will be used as options for the resultant ``getLastError`` command. For example, ``save(..., w=2, fsync=True)`` will wait until at least two servers have recorded the write and will force an fsync on each server being written to. + :param cascade: Sets the flag for cascading saves. You can set a default by setting + "cascade" in the document __meta__ + :param _refs: A list of processed references used in cascading saves .. versionchanged:: 0.5 In existing documents it only saves changed fields using set / unset Saves are cascaded and any :class:`~pymongo.dbref.DBRef` objects that have changes are saved as well. - """ - from fields import ReferenceField, GenericReferenceField + .. versionchanged:: 0.6 + Cascade saves are optional = defaults to True, if you want fine grain + control then you can turn off using document meta['cascade'] = False + """ signals.pre_save.send(self.__class__, document=self) if validate: @@ -173,16 +179,11 @@ class Document(BaseDocument): if removals: collection.update({'_id': object_id}, {"$unset": removals}, upsert=True, safe=safe, **write_options) - # Save any references / generic references - _refs = _refs or [] - for name, cls in self._fields.items(): - if isinstance(cls, (ReferenceField, GenericReferenceField)): - ref = getattr(self, name) - if ref and str(ref) not in _refs: - _refs.append(str(ref)) - ref.save(safe=safe, force_insert=force_insert, - validate=validate, write_options=write_options, - _refs=_refs) + cascade = self._meta.get('cascade', True) if cascade is None else cascade + if cascade: + self.cascade_save(safe=safe, force_insert=force_insert, + validate=validate, write_options=write_options, + cascade=cascade, _refs=_refs) except pymongo.errors.OperationFailure, err: message = 'Could not save document (%s)' @@ -192,23 +193,26 @@ class Document(BaseDocument): id_field = self._meta['id_field'] self[id_field] = self._fields[id_field].to_python(object_id) - def reset_changed_fields(doc, inspected=None): - """Loop through and reset changed fields lists""" - - inspected = inspected or [] - inspected.append(doc) - if hasattr(doc, '_changed_fields'): - doc._changed_fields = [] - - for field_name in doc._fields: - field = getattr(doc, field_name) - if field not in inspected and hasattr(field, '_changed_fields'): - reset_changed_fields(field, inspected) - - reset_changed_fields(self) self._changed_fields = [] signals.post_save.send(self.__class__, document=self, created=creation_mode) + def cascade_save(self, *args, **kwargs): + """Recursively saves any references / generic references on an object""" + from fields import ReferenceField, GenericReferenceField + _refs = kwargs.get('_refs', []) or [] + for name, cls in self._fields.items(): + if not isinstance(cls, (ReferenceField, GenericReferenceField)): + continue + ref = getattr(self, name) + if not ref: + continue + ref_id = "%s,%s" % (ref.__class__.__name__, str(ref._data)) + if ref and ref_id not in _refs: + _refs.append(ref_id) + kwargs["_refs"] = _refs + ref.save(**kwargs) + ref._changed_fields = [] + def update(self, **kwargs): """Performs an update on the :class:`~mongoengine.Document` A convenience wrapper to :meth:`~mongoengine.QuerySet.update`. diff --git a/tests/document.py b/tests/document.py index a1cfcf4..75a7702 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1228,6 +1228,35 @@ class DocumentTest(unittest.TestCase): p1.reload() self.assertEquals(p1.name, p.parent.name) + def test_save_cascade_meta(self): + + class Person(Document): + name = StringField() + parent = ReferenceField('self') + + meta = {'cascade': False} + + Person.drop_collection() + + p1 = Person(name="Wilson Snr") + p1.parent = None + p1.save() + + p2 = Person(name="Wilson Jr") + p2.parent = p1 + p2.save() + + p = Person.objects(name="Wilson Jr").get() + p.parent.name = "Daddy Wilson" + p.save() + + p1.reload() + self.assertNotEquals(p1.name, p.parent.name) + + p.save(cascade=True) + p1.reload() + self.assertEquals(p1.name, p.parent.name) + def test_save_cascades_generically(self): class Person(Document): From 208a467b24a5a0437f0a9a09e9b9d9caceef3cd9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Nov 2011 07:05:54 -0800 Subject: [PATCH 0434/1279] Added dictfield check for Int keys Fixes #371 --- mongoengine/fields.py | 4 +++- tests/fields.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 23d9e45..a0dcdb2 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -552,7 +552,9 @@ class DictField(ComplexBaseField): if not isinstance(value, dict): self.error('Only dictionaries may be used in a DictField') - if any(('.' in k or '$' in k) for k in value): + if any(k for k in value.keys() if not isinstance(k, basestring)): + self.error('Invalid dictionary key - documents must have only string keys') + if any(('.' in k or '$' in k) for k in value.keys()): self.error('Invalid dictionary key name - keys may not contain "."' ' or "$" characters') super(DictField, self).validate(value) diff --git a/tests/fields.py b/tests/fields.py index 5f19e33..56d216a 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -636,6 +636,9 @@ class FieldTest(unittest.TestCase): post.info = {'the.title': 'test'} self.assertRaises(ValidationError, post.validate) + post.info = {1: 'test'} + self.assertRaises(ValidationError, post.validate) + post.info = {'title': 'test'} post.save() From 4e73566c117673dd8d8add6e8891b34ad0f8d375 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Nov 2011 07:06:56 -0800 Subject: [PATCH 0435/1279] Updated changelog - optional cascasde saves --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index df897f3..c78e8e6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added optional cascade saving - Fixed dereferencing - max_depth now taken into account - Fixed document mutation saving issue - Fixed positional operator when replacing embedded documents From d00859ecfd1a034335471119b8c99a1e52103222 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Nov 2011 07:07:26 -0800 Subject: [PATCH 0436/1279] Updated changelog - DictField fix --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index c78e8e6..408277d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed validation of DictField Int keys - Added optional cascade saving - Fixed dereferencing - max_depth now taken into account - Fixed document mutation saving issue From 083f00be8465c1d7a1d56f19fa84fac73e506e00 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Nov 2011 08:09:17 -0800 Subject: [PATCH 0437/1279] Fixes passed in Constructor data for complexfields Fixes #355 --- mongoengine/base.py | 3 ++- tests/fields.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 625d1a1..b06cacb 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -157,9 +157,11 @@ class BaseField(object): if isinstance(value, (list, tuple)) and not isinstance(value, BaseList): observer = DataObserver(instance, self.name) value = BaseList(value, observer) + instance._data[self.name] = value elif isinstance(value, dict) and not isinstance(value, BaseDict): observer = DataObserver(instance, self.name) value = BaseDict(value, observer) + instance._data[self.name] = value return value def __set__(self, instance, value): @@ -749,7 +751,6 @@ class BaseDocument(object): self._data[name] = value if hasattr(self, '_changed_fields'): self._mark_as_changed(name) - super(BaseDocument, self).__setattr__(name, value) def __expand_dynamic_values(self, name, value): diff --git a/tests/fields.py b/tests/fields.py index 56d216a..3a2e7a3 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -516,6 +516,21 @@ class FieldTest(unittest.TestCase): self.assertEquals(BlogPost.objects.filter(info__100__test__exact='test').count(), 0) BlogPost.drop_collection() + def test_list_field_passed_in_value(self): + class Foo(Document): + bars = ListField(ReferenceField("Bar")) + + class Bar(Document): + text = StringField() + + bar = Bar(text="hi") + bar.save() + + foo = Foo(bars=[]) + foo.bars.append(bar) + self.assertEquals(repr(foo.bars), '[]') + + def test_list_field_strict(self): """Ensure that list field handles validation if provided a strict field type.""" From 700e2cd93d79746ac302da4dd528055f9f1a41f3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Nov 2011 08:16:36 -0800 Subject: [PATCH 0438/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 408277d..a87d343 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed passing ComplexField defaults to constructor for ReferenceFields - Fixed validation of DictField Int keys - Added optional cascade saving - Fixed dereferencing - max_depth now taken into account From c775c0a80ce1c7b4389d735ec9b4790deddb9c25 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Nov 2011 08:23:28 -0800 Subject: [PATCH 0439/1279] Circular references with EmbeddedDocumentField fix Fixes #345 --- docs/changelog.rst | 2 ++ mongoengine/base.py | 2 +- tests/document.py | 11 +++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a87d343..977a568 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,8 @@ Changelog Changes in dev ============== +- Fixed issue creating indexes with recursive embedded documents +- Fixed recursive lookup in _unique_with_indexes - Fixed passing ComplexField defaults to constructor for ReferenceFields - Fixed validation of DictField Int keys - Added optional cascade saving diff --git a/mongoengine/base.py b/mongoengine/base.py index b06cacb..58fcbd6 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -687,7 +687,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): unique_indexes.append(index) # Grab any embedded document field unique indexes - if field.__class__.__name__ == "EmbeddedDocumentField": + if field.__class__.__name__ == "EmbeddedDocumentField" and field.document_type != new_class: field_namespace = "%s." % field_name unique_indexes += cls._unique_with_indexes(field.document_type, field_namespace) diff --git a/tests/document.py b/tests/document.py index 75a7702..a4dffe7 100644 --- a/tests/document.py +++ b/tests/document.py @@ -751,6 +751,17 @@ class DocumentTest(unittest.TestCase): post1.save() BlogPost.drop_collection() + def test_recursive_embedded_objects_dont_break_indexes(self): + + class RecursiveObject(EmbeddedDocument): + obj = EmbeddedDocumentField('self') + + class RecursiveDocument(Document): + recursive_obj = EmbeddedDocumentField(RecursiveObject) + + info = RecursiveDocument.objects._collection.index_information() + self.assertEqual(info.keys(), ['_id_', '_types_1']) + def test_geo_indexes_recursion(self): class User(Document): From d9005ac2fc7ecf56bcbeb9a00259db5b9a677edb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 28 Nov 2011 14:45:57 -0200 Subject: [PATCH 0440/1279] added elemMatch support --- mongoengine/queryset.py | 14 +++++++++++--- tests/queryset.py | 23 +++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a9b3ea9..6025dd9 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -640,6 +640,7 @@ class QuerySet(object): match_operators = ['contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith', 'exact', 'iexact'] + custom_operators = ['match'] mongo_query = {} for key, value in query.items(): @@ -652,7 +653,7 @@ class QuerySet(object): parts = [part for part in parts if not part.isdigit()] # Check for an operator and transform to mongo-style if there is op = None - if parts[-1] in operators + match_operators + geo_operators: + if parts[-1] in operators + match_operators + geo_operators + custom_operators: op = parts.pop() negate = False @@ -685,7 +686,7 @@ class QuerySet(object): if isinstance(field, basestring): if op in match_operators and isinstance(value, basestring): from mongoengine import StringField - value = StringField().prepare_query_value(op, value) + value = StringField.prepare_query_value(op, value) else: value = field else: @@ -693,7 +694,8 @@ class QuerySet(object): elif op in ('in', 'nin', 'all', 'near'): # 'in', 'nin' and 'all' require a list of values value = [field.prepare_query_value(op, v) for v in value] - + + # if op and op not in match_operators: if op: if op in geo_operators: @@ -712,6 +714,12 @@ class QuerySet(object): else: raise NotImplementedError("Geo method '%s' has not " "been implemented" % op) + elif op in custom_operators: + if op == 'match': + value = {"$elemMatch": value} + else: + NotImplementedError("Custom method '%s' has not " + "been implemented" % op) elif op not in match_operators: value = {'$' + op: value} diff --git a/tests/queryset.py b/tests/queryset.py index 7c1c8dc..d978cf2 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -2866,6 +2866,29 @@ class QueryFieldListTest(unittest.TestCase): q += QueryFieldList(fields=['a'], value={"$slice": 5}) self.assertEqual(q.as_dict(), {'a': {"$slice": 5}}) + def test_elem_match(self): + class Foo(EmbeddedDocument): + shape = StringField() + color = StringField() + trick = BooleanField() + meta = {'allow_inheritance': False} + + class Bar(Document): + foo = ListField(EmbeddedDocumentField(Foo)) + meta = {'allow_inheritance': False} + + Bar.drop_collection() + + b1 = Bar(foo=[Foo(shape= "square", color ="purple", thick = False), + Foo(shape= "circle", color ="red", thick = True)]) + b1.save() + + b2 = Bar(foo=[Foo(shape= "square", color ="red", thick = True), + Foo(shape= "circle", color ="purple", thick = False)]) + b2.save() + + ak = list(Bar.objects(foo__match={'shape': "square", "color": "purple"})) + self.assertEqual([b1], ak) if __name__ == '__main__': unittest.main() From 8d7291506e84736a1859e53ef46bc0e64a43b34e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 29 Nov 2011 01:44:23 -0800 Subject: [PATCH 0441/1279] Updated Authors --- AUTHORS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 573ddb1..3955a55 100644 --- a/AUTHORS +++ b/AUTHORS @@ -75,4 +75,4 @@ that much better: * Karim Allah * Adam Parrish * jpfarias - + * jonrscott From a8d91a56bf250c8e29e6c8c8d45df9b29e76c2a6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 29 Nov 2011 03:43:49 -0800 Subject: [PATCH 0442/1279] Fixes circular list references The depth deduciton for _fields was over zealous now max_depth is honoured/ Fixes #373 --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/dereference.py | 4 ++-- tests/dereference.py | 23 +++++++++++++++++++++++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 573ddb1..9276c6c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -75,4 +75,5 @@ that much better: * Karim Allah * Adam Parrish * jpfarias + * Alice Zoë Bevan-McGregor diff --git a/docs/changelog.rst b/docs/changelog.rst index 977a568..4acc8c1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed dereferencing - multi directional list dereferencing - Fixed issue creating indexes with recursive embedded documents - Fixed recursive lookup in _unique_with_indexes - Fixed passing ComplexField defaults to constructor for ReferenceFields diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 4e595b1..0d8d7f1 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -168,9 +168,9 @@ class DeReference(object): elif isinstance(v, (dict, pymongo.son.SON)) and '_ref' in v: data[k]._data[field_name] = self.object_map.get(v['_ref'].id, v) elif isinstance(v, dict) and depth <= self.max_depth: - data[k]._data[field_name] = self._attach_objects(v, depth - 1, instance=instance, name=name) + data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name) elif isinstance(v, (list, tuple)) and depth <= self.max_depth: - data[k]._data[field_name] = self._attach_objects(v, depth - 1, instance=instance, name=name) + data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name) elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth: data[k] = self._attach_objects(v, depth - 1, instance=instance, name=name) elif hasattr(v, 'id'): diff --git a/tests/dereference.py b/tests/dereference.py index 2e24a61..88cf17e 100644 --- a/tests/dereference.py +++ b/tests/dereference.py @@ -760,3 +760,26 @@ class FieldTest(unittest.TestCase): UserB.drop_collection() UserC.drop_collection() Group.drop_collection() + + def test_multidirectional_lists(self): + + class Asset(Document): + name = StringField(max_length=250, required=True) + parent = GenericReferenceField(default=None) + parents = ListField(GenericReferenceField()) + children = ListField(GenericReferenceField()) + + Asset.drop_collection() + + root = Asset(name='', path="/", title="Site Root") + root.save() + + company = Asset(name='company', title='Company', parent=root, parents=[root]) + company.save() + + root.children = [company] + root.save() + + root = root.reload() + self.assertEquals(root.children, [company]) + self.assertEquals(company.parents, [root]) From 6cef571bfb62ef7c78a81090e2728ecd85911552 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 30 Nov 2011 02:15:47 -0800 Subject: [PATCH 0443/1279] Added Reverse option to SortedLists Thanks Stephen Young for the patch closes #364 --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/fields.py | 8 ++++++-- tests/fields.py | 25 +++++++++++++++++++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 274fe1e..d36d3f2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -77,3 +77,4 @@ that much better: * jpfarias * jonrscott * Alice Zoë Bevan-McGregor + * Stephen Young diff --git a/docs/changelog.rst b/docs/changelog.rst index 4acc8c1..c040f0b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added reverse option to SortedListFields - Fixed dereferencing - multi directional list dereferencing - Fixed issue creating indexes with recursive embedded documents - Fixed recursive lookup in _unique_with_indexes diff --git a/mongoengine/fields.py b/mongoengine/fields.py index a0dcdb2..cf72e78 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -511,20 +511,24 @@ class SortedListField(ListField): retrieved. .. versionadded:: 0.4 + .. versionchanged:: 0.6 - added reverse keyword """ _ordering = None + _order_reverse = False def __init__(self, field, **kwargs): if 'ordering' in kwargs.keys(): self._ordering = kwargs.pop('ordering') + if 'reverse' in kwargs.keys(): + self._order_reverse = kwargs.pop('reverse') super(SortedListField, self).__init__(field, **kwargs) def to_mongo(self, value): value = super(SortedListField, self).to_mongo(value) if self._ordering is not None: - return sorted(value, key=itemgetter(self._ordering)) - return sorted(value) + return sorted(value, key=itemgetter(self._ordering), reverse=self._order_reverse) + return sorted(value, reverse=self._order_reverse) class DictField(ComplexBaseField): diff --git a/tests/fields.py b/tests/fields.py index 3a2e7a3..0bc15a3 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -481,6 +481,31 @@ class FieldTest(unittest.TestCase): BlogPost.drop_collection() + def test_reverse_list_sorting(self): + '''Ensure that a reverse sorted list field properly sorts values''' + + class Category(EmbeddedDocument): + count = IntField() + name = StringField() + + class CategoryList(Document): + categories = SortedListField(EmbeddedDocumentField(Category), ordering='count', reverse=True) + name = StringField() + + catlist = CategoryList(name="Top categories") + cat1 = Category(name='posts', count=10) + cat2 = Category(name='food', count=100) + cat3 = Category(name='drink', count=40) + catlist.categories = [cat1, cat2, cat3] + catlist.save() + catlist.reload() + + self.assertEqual(catlist.categories[0].name, cat2.name) + self.assertEqual(catlist.categories[1].name, cat3.name) + self.assertEqual(catlist.categories[2].name, cat1.name) + + CategoryList.drop_collection() + def test_list_field(self): """Ensure that list types work as expected. """ From 8b97808931ad76729a494f0647c1a128c9fef60f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 30 Nov 2011 02:30:29 -0800 Subject: [PATCH 0444/1279] Added docs for elemMatch --- docs/changelog.rst | 1 + docs/guide/querying.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index c040f0b..bd5cf56 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added support for the $elementMatch operator - Added reverse option to SortedListFields - Fixed dereferencing - multi directional list dereferencing - Fixed issue creating indexes with recursive embedded documents diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 13a374c..9ebf37f 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -76,6 +76,7 @@ expressions: * ``istartswith`` -- string field starts with value (case insensitive) * ``endswith`` -- string field ends with value * ``iendswith`` -- string field ends with value (case insensitive) +* ``match`` -- performs an $elemMatch so you can match an entire document within an array There are a few special operators for performing geographical queries, that may used with :class:`~mongoengine.GeoPointField`\ s: From fdc385ea3389210f2da1eff5a1eb6a2cdf911dcd Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 30 Nov 2011 03:06:46 -0800 Subject: [PATCH 0445/1279] Allow dynamic data to be deleted Fixes #374 --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/document.py | 9 +++++++++ tests/dynamic_document.py | 27 +++++++++++++++++++++++++++ 4 files changed, 38 insertions(+) diff --git a/AUTHORS b/AUTHORS index d36d3f2..7e7c332 100644 --- a/AUTHORS +++ b/AUTHORS @@ -78,3 +78,4 @@ that much better: * jonrscott * Alice Zoë Bevan-McGregor * Stephen Young + * tkloc diff --git a/docs/changelog.rst b/docs/changelog.rst index bd5cf56..055cf92 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed deletion of dynamic data - Added support for the $elementMatch operator - Added reverse option to SortedListFields - Fixed dereferencing - multi directional list dereferencing diff --git a/mongoengine/document.py b/mongoengine/document.py index a9ab103..e22e36f 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -329,6 +329,15 @@ class DynamicDocument(Document): __metaclass__ = TopLevelDocumentMetaclass _dynamic = True + def __delattr__(self, *args, **kwargs): + """Deletes the attribute by setting to None and allowing _delta to unset + it""" + field_name = args[0] + if field_name in self._dynamic_fields: + setattr(self, field_name, None) + else: + super(DynamicDocument, self).__delattr__(*args, **kwargs) + class DynamicEmbeddedDocument(EmbeddedDocument): """A Dynamic Embedded Document class allowing flexible, expandable and diff --git a/tests/dynamic_document.py b/tests/dynamic_document.py index 19cd466..3d808a1 100644 --- a/tests/dynamic_document.py +++ b/tests/dynamic_document.py @@ -50,6 +50,33 @@ class DynamicDocTest(unittest.TestCase): p = self.Person.objects.get() self.assertEquals(p.misc, {'hello': 'world'}) + def test_delete_dynamic_field(self): + """Test deleting a dynamic field works""" + self.Person.drop_collection() + p = self.Person() + p.name = "Dean" + p.misc = 22 + p.save() + + p = self.Person.objects.get() + p.misc = {'hello': 'world'} + p.save() + + p = self.Person.objects.get() + self.assertEquals(p.misc, {'hello': 'world'}) + collection = self.db[self.Person._get_collection_name()] + obj = collection.find_one() + self.assertEquals(sorted(obj.keys()), ['_cls', '_id', '_types', 'misc', 'name']) + + del(p.misc) + p.save() + + p = self.Person.objects.get() + self.assertFalse(hasattr(p, 'misc')) + + obj = collection.find_one() + self.assertEquals(sorted(obj.keys()), ['_cls', '_id', '_types', 'name']) + def test_dynamic_document_queries(self): """Ensure we can query dynamic fields""" p = self.Person() From beacfae400296ba364c3b6f0839005a6e601650b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 30 Nov 2011 07:55:33 -0800 Subject: [PATCH 0446/1279] Removed use of _get_subclasses favouring get_document _get_subclasses not actually required and causes issues where Base Classes aren't imported but dont actually need to be. Fixes #271 --- AUTHORS | 1 + docs/changelog.rst | 2 ++ docs/upgrade.rst | 2 ++ mongoengine/base.py | 26 +------------------------ mongoengine/document.py | 7 +++---- tests/document.py | 43 ++--------------------------------------- 6 files changed, 11 insertions(+), 70 deletions(-) diff --git a/AUTHORS b/AUTHORS index 7e7c332..28b57bb 100644 --- a/AUTHORS +++ b/AUTHORS @@ -79,3 +79,4 @@ that much better: * Alice Zoë Bevan-McGregor * Stephen Young * tkloc + * aid diff --git a/docs/changelog.rst b/docs/changelog.rst index 055cf92..d510531 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,8 @@ Changelog Changes in dev ============== +- Removed Document._get_subclasses() - no longer required +- Fixed bug requiring subclasses when not actually needed - Fixed deletion of dynamic data - Added support for the $elementMatch operator - Added reverse option to SortedListFields diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 9151cf3..b44ecad 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -8,6 +8,8 @@ Upgrading Embedded Documents - if you had a `pk` field you will have to rename it from `_id` to `pk` as pk is no longer a property of Embedded Documents. +Document._get_subclasses - Is no longer used and the class method has been removed. + 0.4 to 0.5 =========== diff --git a/mongoengine/base.py b/mongoengine/base.py index 58fcbd6..402f191 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -845,21 +845,6 @@ class BaseDocument(object): """ return cls._meta.get('collection', None) - @classmethod - def _get_subclasses(cls): - """Return a dictionary of all subclasses (found recursively). - """ - try: - subclasses = cls.__subclasses__() - except: - subclasses = cls.__subclasses__(cls) - - all_subclasses = {} - for subclass in subclasses: - all_subclasses[subclass._class_name] = subclass - all_subclasses.update(subclass._get_subclasses()) - return all_subclasses - @classmethod def _from_son(cls, son): """Create an instance of a Document (subclass) from a PyMongo SON. @@ -877,16 +862,7 @@ class BaseDocument(object): # Return correct subclass for document type if class_name != cls._class_name: - subclasses = cls._get_subclasses() - if class_name not in subclasses: - # Type of document is probably more generic than the class - # that has been queried to return this SON - raise NotRegistered(""" - `%s` has not been registered in the document registry. - Importing the document class automatically registers it, - has it been imported? - """.strip() % class_name) - cls = subclasses[class_name] + cls = get_document(class_name) changed_fields = [] for field_name, field in cls._fields.items(): diff --git a/mongoengine/document.py b/mongoengine/document.py index e22e36f..0eee4af 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -162,11 +162,10 @@ class Document(BaseDocument): doc = self.to_mongo() - created = '_id' in doc - creation_mode = force_insert or not created + created = force_insert or '_id' not in doc try: collection = self.__class__.objects._collection - if creation_mode: + if created: if force_insert: object_id = collection.insert(doc, safe=safe, **write_options) else: @@ -194,7 +193,7 @@ class Document(BaseDocument): self[id_field] = self._fields[id_field].to_python(object_id) self._changed_fields = [] - signals.post_save.send(self.__class__, document=self, created=creation_mode) + signals.post_save.send(self.__class__, document=self, created=created) def cascade_save(self, *args, **kwargs): """Recursively saves any references / generic references on an object""" diff --git a/tests/document.py b/tests/document.py index a4dffe7..ca42c09 100644 --- a/tests/document.py +++ b/tests/document.py @@ -158,29 +158,6 @@ class DocumentTest(unittest.TestCase): } self.assertEqual(Dog._superclasses, dog_superclasses) - def test_get_subclasses(self): - """Ensure that the correct list of subclasses is retrieved by the - _get_subclasses method. - """ - class Animal(Document): pass - class Fish(Animal): pass - class Mammal(Animal): pass - class Human(Mammal): pass - class Dog(Mammal): pass - - mammal_subclasses = { - 'Animal.Mammal.Dog': Dog, - 'Animal.Mammal.Human': Human - } - self.assertEqual(Mammal._get_subclasses(), mammal_subclasses) - - animal_subclasses = { - 'Animal.Fish': Fish, - 'Animal.Mammal': Mammal, - 'Animal.Mammal.Dog': Dog, - 'Animal.Mammal.Human': Human - } - self.assertEqual(Animal._get_subclasses(), animal_subclasses) def test_external_super_and_sub_classes(self): """Ensure that the correct list of sub and super classes is assembled. @@ -202,20 +179,6 @@ class DocumentTest(unittest.TestCase): } self.assertEqual(Dog._superclasses, dog_superclasses) - animal_subclasses = { - 'Base.Animal.Fish': Fish, - 'Base.Animal.Mammal': Mammal, - 'Base.Animal.Mammal.Dog': Dog, - 'Base.Animal.Mammal.Human': Human - } - self.assertEqual(Animal._get_subclasses(), animal_subclasses) - - mammal_subclasses = { - 'Base.Animal.Mammal.Dog': Dog, - 'Base.Animal.Mammal.Human': Human - } - self.assertEqual(Mammal._get_subclasses(), mammal_subclasses) - Base.drop_collection() h = Human() @@ -1014,10 +977,8 @@ class DocumentTest(unittest.TestCase): # Mimic Place and NicePlace definitions being in a different file # and the NicePlace model not being imported in at query time. - @classmethod - def _get_subclasses(cls): - return {} - Place._get_subclasses = _get_subclasses + from mongoengine.base import _document_registry + del(_document_registry['Place.NicePlace']) def query_without_importing_nice_place(): print Place.objects.all() From 0187a0e1130976202598a79e698795b6432544ed Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 30 Nov 2011 08:12:32 -0800 Subject: [PATCH 0447/1279] Handle updating and getting None values Fixes updating a field to None, so it works in a similar fashion as unsetting it via save() Updated to handle null data from the database Fixes #362 --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/base.py | 7 ++++ mongoengine/queryset.py | 6 ++-- tests/fields.py | 75 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 87 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index 28b57bb..bcf6129 100644 --- a/AUTHORS +++ b/AUTHORS @@ -80,3 +80,4 @@ that much better: * Stephen Young * tkloc * aid + * yamaneko1212 diff --git a/docs/changelog.rst b/docs/changelog.rst index d510531..602294a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed Handle None values for non-required fields. - Removed Document._get_subclasses() - no longer required - Fixed bug requiring subclasses when not actually needed - Fixed deletion of dynamic data diff --git a/mongoengine/base.py b/mongoengine/base.py index 402f191..ad2ebca 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -751,6 +751,13 @@ class BaseDocument(object): self._data[name] = value if hasattr(self, '_changed_fields'): self._mark_as_changed(name) + + # Handle None values for required fields + if value is None and name in getattr(self, '_fields', {}): + self._data[name] = value + if hasattr(self, '_changed_fields'): + self._mark_as_changed(name) + return super(BaseDocument, self).__setattr__(name, value) def __expand_dynamic_values(self, name, value): diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6025dd9..308bdf7 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -694,8 +694,7 @@ class QuerySet(object): elif op in ('in', 'nin', 'all', 'near'): # 'in', 'nin' and 'all' require a list of values value = [field.prepare_query_value(op, v) for v in value] - - + # if op and op not in match_operators: if op: if op in geo_operators: @@ -1301,7 +1300,8 @@ class QuerySet(object): field = cleaned_fields[-1] if op in (None, 'set', 'push', 'pull', 'addToSet'): - value = field.prepare_query_value(op, value) + if field.required or value is not None: + value = field.prepare_query_value(op, value) elif op in ('pushAll', 'pullAll'): value = [field.prepare_query_value(op, v) for v in value] diff --git a/tests/fields.py b/tests/fields.py index 0bc15a3..37972fd 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -46,6 +46,81 @@ class FieldTest(unittest.TestCase): person = Person(age=30) self.assertRaises(ValidationError, person.validate) + def test_not_required_handles_none_in_update(self): + """Ensure that every fields should accept None if required is False. + """ + + class HandleNoneFields(Document): + str_fld = StringField() + int_fld = IntField() + flt_fld = FloatField() + comp_dt_fld = ComplexDateTimeField() + + HandleNoneFields.drop_collection() + + doc = HandleNoneFields() + doc.str_fld = u'spam ham egg' + doc.int_fld = 42 + doc.flt_fld = 4.2 + doc.com_dt_fld = datetime.datetime.utcnow() + doc.save() + + res = HandleNoneFields.objects(id=doc.id).update( + set__str_fld=None, + set__int_fld=None, + set__flt_fld=None, + set__comp_dt_fld=None, + ) + self.assertEqual(res, 1) + + # Retrive data from db and verify it. + ret = HandleNoneFields.objects.all()[0] + + self.assertEqual(ret.str_fld, None) + self.assertEqual(ret.int_fld, None) + self.assertEqual(ret.flt_fld, None) + + # Return current time if retrived value is None. + self.assertTrue(isinstance(ret.comp_dt_fld, datetime.datetime)) + + def test_not_required_handles_none_from_database(self): + """Ensure that every fields can handle null values from the database. + """ + + class HandleNoneFields(Document): + str_fld = StringField(required=True) + int_fld = IntField(required=True) + flt_fld = FloatField(required=True) + comp_dt_fld = ComplexDateTimeField(required=True) + + HandleNoneFields.drop_collection() + + doc = HandleNoneFields() + doc.str_fld = u'spam ham egg' + doc.int_fld = 42 + doc.flt_fld = 4.2 + doc.com_dt_fld = datetime.datetime.utcnow() + doc.save() + + collection = self.db[HandleNoneFields._get_collection_name()] + obj = collection.update({"_id": doc.id}, {"$unset": { + "str_fld": 1, + "int_fld": 1, + "flt_fld": 1, + "comp_dt_fld": 1} + }) + + # Retrive data from db and verify it. + ret = HandleNoneFields.objects.all()[0] + + self.assertEqual(ret.str_fld, None) + self.assertEqual(ret.int_fld, None) + self.assertEqual(ret.flt_fld, None) + # Return current time if retrived value is None. + self.assert_(isinstance(ret.comp_dt_fld, datetime.datetime)) + + self.assertRaises(ValidationError, ret.validate) + def test_object_id_validation(self): """Ensure that invalid values cannot be assigned to string fields. """ From 9188f9bf62c63d23537e69911d5aef1411e66008 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 30 Nov 2011 08:54:33 -0800 Subject: [PATCH 0448/1279] Added custom cascade kwarg options Allows the user to overwrite any default kwargs Closes #295 --- AUTHORS | 1 + docs/changelog.rst | 3 ++- mongoengine/document.py | 19 +++++++++++++++---- tests/document.py | 25 ++++++++++++++++++++++++- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/AUTHORS b/AUTHORS index bcf6129..0dce8c4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -81,3 +81,4 @@ that much better: * tkloc * aid * yamaneko1212 + * dave mankoff diff --git a/docs/changelog.rst b/docs/changelog.rst index 602294a..b0fe680 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,7 +5,8 @@ Changelog Changes in dev ============== -- Fixed Handle None values for non-required fields. +- Added customisable cascade kwarg options +- Fixed Handle None values for non-required fields - Removed Document._get_subclasses() - no longer required - Fixed bug requiring subclasses when not actually needed - Fixed deletion of dynamic data diff --git a/mongoengine/document.py b/mongoengine/document.py index 0eee4af..3f4c9d7 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -121,7 +121,7 @@ class Document(BaseDocument): return self._collection def save(self, safe=True, force_insert=False, validate=True, write_options=None, - cascade=None, _refs=None): + cascade=None, cascade_kwargs=None, _refs=None): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be created. @@ -141,6 +141,7 @@ class Document(BaseDocument): have recorded the write and will force an fsync on each server being written to. :param cascade: Sets the flag for cascading saves. You can set a default by setting "cascade" in the document __meta__ + :param cascade_kwargs: optional kwargs dictionary to be passed throw to cascading saves :param _refs: A list of processed references used in cascading saves .. versionchanged:: 0.5 @@ -150,6 +151,8 @@ class Document(BaseDocument): .. versionchanged:: 0.6 Cascade saves are optional = defaults to True, if you want fine grain control then you can turn off using document meta['cascade'] = False + Also you can pass different kwargs to the cascade save using cascade_kwargs + which overwrites the existing kwargs with custom values """ signals.pre_save.send(self.__class__, document=self) @@ -180,9 +183,17 @@ class Document(BaseDocument): cascade = self._meta.get('cascade', True) if cascade is None else cascade if cascade: - self.cascade_save(safe=safe, force_insert=force_insert, - validate=validate, write_options=write_options, - cascade=cascade, _refs=_refs) + kwargs = { + "safe": safe, + "force_insert": force_insert, + "validate": validate, + "write_options": write_options, + "cascade": cascade + } + if cascade_kwargs: # Allow granular control over cascades + kwargs.update(cascade_kwargs) + kwargs['_refs'] = _refs + self.cascade_save(**kwargs) except pymongo.errors.OperationFailure, err: message = 'Could not save document (%s)' diff --git a/tests/document.py b/tests/document.py index ca42c09..f12affd 100644 --- a/tests/document.py +++ b/tests/document.py @@ -159,7 +159,7 @@ class DocumentTest(unittest.TestCase): self.assertEqual(Dog._superclasses, dog_superclasses) - def test_external_super_and_sub_classes(self): + def test_external_superclasses(self): """Ensure that the correct list of sub and super classes is assembled. when importing part of the model """ @@ -1200,6 +1200,29 @@ class DocumentTest(unittest.TestCase): p1.reload() self.assertEquals(p1.name, p.parent.name) + def test_save_cascade_kwargs(self): + + class Person(Document): + name = StringField() + parent = ReferenceField('self') + + Person.drop_collection() + + p1 = Person(name="Wilson Snr") + p1.parent = None + p1.save() + + p2 = Person(name="Wilson Jr") + p2.parent = p1 + p2.save(force_insert=True, cascade_kwargs={"force_insert": False}) + + p = Person.objects(name="Wilson Jr").get() + p.parent.name = "Daddy Wilson" + p.save() + + p1.reload() + self.assertEquals(p1.name, p.parent.name) + def test_save_cascade_meta(self): class Person(Document): From 8a44232bfc5d26b69d9ebfde7e57510d84fb58e5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 1 Dec 2011 07:57:24 -0800 Subject: [PATCH 0449/1279] Added Reverse Delete Rule support to ListFields DictFields and MapFields aren't supported and raise an InvalidDocument Error Closes #254 --- mongoengine/base.py | 13 ++++++++++--- tests/document.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index ad2ebca..2fd40ec 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -503,15 +503,22 @@ class DocumentMetaclass(type): attrs['_db_field_map'] = dict([(k, v.db_field) for k, v in doc_fields.items() if k!=v.db_field]) attrs['_reverse_db_field_map'] = dict([(v, k) for k, v in attrs['_db_field_map'].items()]) - from mongoengine import Document, EmbeddedDocument + from mongoengine import Document, EmbeddedDocument, DictField new_class = super_new(cls, name, bases, attrs) for field in new_class._fields.values(): field.owner_document = new_class + delete_rule = getattr(field, 'reverse_delete_rule', DO_NOTHING) + f = field + if isinstance(f, ComplexBaseField) and hasattr(f, 'field'): + delete_rule = getattr(f.field, 'reverse_delete_rule', DO_NOTHING) + if isinstance(f, DictField) and delete_rule != DO_NOTHING: + raise InvalidDocumentError("Reverse delete rules are not supported for %s (field: %s)" % (field.__class__.__name__, field.name)) + f = field.field + if delete_rule != DO_NOTHING: - field.document_type.register_delete_rule(new_class, field.name, - delete_rule) + f.document_type.register_delete_rule(new_class, field.name, delete_rule) if field.name and hasattr(Document, field.name) and EmbeddedDocument not in new_class.mro(): raise InvalidDocumentError("%s is a document method and not a valid field name" % field.name) diff --git a/tests/document.py b/tests/document.py index f12affd..391c3e2 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2214,6 +2214,46 @@ class DocumentTest(unittest.TestCase): author.delete() self.assertEqual(len(BlogPost.objects), 0) + def test_reverse_delete_rule_cascade_and_nullify_complex_field(self): + """Ensure that a referenced document is also deleted upon deletion. + """ + + class BlogPost(Document): + content = StringField() + authors = ListField(ReferenceField(self.Person, reverse_delete_rule=CASCADE)) + reviewers = ListField(ReferenceField(self.Person, reverse_delete_rule=NULLIFY)) + + self.Person.drop_collection() + BlogPost.drop_collection() + + author = self.Person(name='Test User') + author.save() + + reviewer = self.Person(name='Re Viewer') + reviewer.save() + + post = BlogPost(content= 'Watched some TV') + post.authors = [author] + post.reviewers = [reviewer] + post.save() + + reviewer.delete() + self.assertEqual(len(BlogPost.objects), 1) # No effect on the BlogPost + self.assertEqual(BlogPost.objects.get().reviewers, []) + + # Delete the Person, which should lead to deletion of the BlogPost, too + author.delete() + self.assertEqual(len(BlogPost.objects), 0) + + def throw_invalid_document_error(): + class Blog(Document): + content = StringField() + authors = MapField(ReferenceField(self.Person, reverse_delete_rule=CASCADE)) + reviewers = DictField(field=ReferenceField(self.Person, reverse_delete_rule=NULLIFY)) + + self.assertRaises(InvalidDocumentError, throw_invalid_document_error) + + def test_reverse_delete_rule_cascade_recurs(self): """Ensure that a chain of documents is also deleted upon cascaded deletion. From 391f659af1b40856564a6f64c1f2613092f44e84 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 1 Dec 2011 08:16:13 -0800 Subject: [PATCH 0450/1279] Updated docs re: reverse delete rules refs #254 --- docs/changelog.rst | 1 + docs/tutorial.rst | 5 +++++ docs/upgrade.rst | 3 +++ 3 files changed, 9 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b0fe680..cbb20e3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added Reverse Delete Rule support to ListFields - MapFields aren't supported - Added customisable cascade kwarg options - Fixed Handle None values for non-required fields - Removed Document._get_subclasses() - no longer required diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 6ce8d10..a5284c8 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -167,6 +167,11 @@ To delete all the posts if a user is deleted set the rule:: See :class:`~mongoengine.ReferenceField` for more information. +..note:: + MapFields and DictFields currently don't support automatic handling of + deleted references + + Adding data to our Tumblelog ============================ Now that we've defined how our documents will be structured, let's start adding diff --git a/docs/upgrade.rst b/docs/upgrade.rst index b44ecad..1b25c07 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -10,6 +10,9 @@ to `pk` as pk is no longer a property of Embedded Documents. Document._get_subclasses - Is no longer used and the class method has been removed. +Reverse Delete Rules on MapFields and DictFields now throw a InvalidDocument error +as they aren't supported. + 0.4 to 0.5 =========== From 071562d75534b95a4f7ed415290659b8e4a4042e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 2 Dec 2011 00:11:25 -0800 Subject: [PATCH 0451/1279] Fixed issue with dynamic documents deltas Closes #377 --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/base.py | 3 ++- tests/dynamic_document.py | 9 +++++++++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 0dce8c4..d543394 100644 --- a/AUTHORS +++ b/AUTHORS @@ -82,3 +82,4 @@ that much better: * aid * yamaneko1212 * dave mankoff + * Alexander G. Morano diff --git a/docs/changelog.rst b/docs/changelog.rst index cbb20e3..30d6707 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed issue with dynamic documents deltas - Added Reverse Delete Rule support to ListFields - MapFields aren't supported - Added customisable cascade kwarg options - Fixed Handle None values for non-required fields diff --git a/mongoengine/base.py b/mongoengine/base.py index 2fd40ec..29be876 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -954,6 +954,7 @@ class BaseDocument(object): set_fields = self._get_changed_fields() set_data = {} unset_data = {} + parts = [] if hasattr(self, '_changed_fields'): set_data = {} # Fetch each set item from its path @@ -980,7 +981,7 @@ class BaseDocument(object): # If we've set a value that ain't the default value dont unset it. default = None - if self._dynamic and parts[0] in self._dynamic_fields: + if self._dynamic and len(parts) and parts[0] in self._dynamic_fields: del(set_data[path]) unset_data[path] = 1 continue diff --git a/tests/dynamic_document.py b/tests/dynamic_document.py index 3d808a1..b40ba06 100644 --- a/tests/dynamic_document.py +++ b/tests/dynamic_document.py @@ -36,6 +36,15 @@ class DynamicDocTest(unittest.TestCase): # Confirm no changes to self.Person self.assertFalse(hasattr(self.Person, 'age')) + def test_dynamic_document_delta(self): + """Ensures simple dynamic documents can delta correctly""" + p = self.Person(name="James", age=34) + self.assertEquals(p._delta(), ({'_types': ['Person'], 'age': 34, 'name': 'James', '_cls': 'Person'}, {})) + + p.doc = 123 + del(p.doc) + self.assertEquals(p._delta(), ({'_types': ['Person'], 'age': 34, 'name': 'James', '_cls': 'Person'}, {'doc': 1})) + def test_change_scope_of_variable(self): """Test changing the scope of a dynamic field has no adverse effects""" p = self.Person() From d06c5f036bf5b2f6c5c7f7fb5ac6129e91f1d3f5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 2 Dec 2011 00:37:32 -0800 Subject: [PATCH 0452/1279] Cleaned up _transform_query Refs #354 #376 --- mongoengine/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 308bdf7..90547e2 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -667,8 +667,8 @@ class QuerySet(object): parts = [] cleaned_fields = [] - append_field = True for field in fields: + append_field = True if isinstance(field, str): parts.append(field) append_field = False From e231f71b4a373c103084636f8647e1328843f527 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 2 Dec 2011 02:46:55 -0800 Subject: [PATCH 0453/1279] EmbeddedDocuments dont support Reverse Delete Rules Now throws an InvalidDocumentError Refs #227 --- docs/upgrade.rst | 5 +++-- mongoengine/base.py | 14 ++++++++++++++ tests/document.py | 8 ++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 1b25c07..cb72b35 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -8,10 +8,11 @@ Upgrading Embedded Documents - if you had a `pk` field you will have to rename it from `_id` to `pk` as pk is no longer a property of Embedded Documents. +Reverse Delete Rules in Embedded Documents, MapFields and DictFields now throw +an InvalidDocument error as they aren't currently supported. + Document._get_subclasses - Is no longer used and the class method has been removed. -Reverse Delete Rules on MapFields and DictFields now throw a InvalidDocument error -as they aren't supported. 0.4 to 0.5 =========== diff --git a/mongoengine/base.py b/mongoengine/base.py index 29be876..00dd2ee 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -518,6 +518,8 @@ class DocumentMetaclass(type): f = field.field if delete_rule != DO_NOTHING: + if issubclass(new_class, EmbeddedDocument): + raise InvalidDocumentError("Reverse delete rules are not supported for EmbeddedDocuments (field: %s)" % field.name) f.document_type.register_delete_rule(new_class, field.name, delete_rule) if field.name and hasattr(Document, field.name) and EmbeddedDocument not in new_class.mro(): @@ -663,6 +665,18 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): return new_class + def __instancecheck__(cls, inst): + """Custom instance check for isinstance() as registering delete rules or + calling a cls method in __new__ seems to change the cls so vanilla + isinstance() fails""" + is_instance = super(DocumentMetaclass, cls).__instancecheck__(inst) + if hasattr(cls, '_meta') and 'delete_rules' in cls._meta and not is_instance: + try: + is_instance = get_document(cls.__name__) == get_document(inst.__class__.__name__) + except NotRegistered: + pass + return is_instance + @classmethod def _unique_with_indexes(cls, new_class, namespace=""): unique_indexes = [] diff --git a/tests/document.py b/tests/document.py index 391c3e2..a4f0992 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2245,6 +2245,8 @@ class DocumentTest(unittest.TestCase): author.delete() self.assertEqual(len(BlogPost.objects), 0) + def test_invalid_reverse_delete_rules_raise_errors(self): + def throw_invalid_document_error(): class Blog(Document): content = StringField() @@ -2253,6 +2255,12 @@ class DocumentTest(unittest.TestCase): self.assertRaises(InvalidDocumentError, throw_invalid_document_error) + def throw_invalid_document_error_embedded(): + class Parents(EmbeddedDocument): + father = ReferenceField('Person', reverse_delete_rule=DENY) + mother = ReferenceField('Person', reverse_delete_rule=DENY) + + self.assertRaises(InvalidDocumentError, throw_invalid_document_error_embedded) def test_reverse_delete_rule_cascade_recurs(self): """Ensure that a chain of documents is also deleted upon cascaded From 939bd2bb1f519a99d906433d9f4e87fc017b8874 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 2 Dec 2011 02:49:16 -0800 Subject: [PATCH 0454/1279] Updated Documentation --- docs/index.rst | 3 ++- mongoengine/fields.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 920ddf6..a574f29 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -42,7 +42,8 @@ Also, you can join the developers' `mailing list Changes ------- -See the :doc:`changelog` for a full list of changes to MongoEngine. +See the :doc:`changelog` for a full list of changes to MongoEngine and +:doc:`upgrade` for upgrade information. .. toctree:: :hidden: diff --git a/mongoengine/fields.py b/mongoengine/fields.py index cf72e78..68d7ace 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -597,7 +597,9 @@ class ReferenceField(BaseField): access (lazily). Use the `reverse_delete_rule` to handle what should happen if the document - the field is referencing is deleted. + the field is referencing is deleted. EmbeddedDocuments, DictFields and + MapFields do not support reverse_delete_rules and an `InvalidDocumentError` + will be raised if trying to set on one of these Document / Field types. The options are: From ba59e498def28e7535ba35b21c39ad576fd89649 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 2 Dec 2011 02:52:06 -0800 Subject: [PATCH 0455/1279] Custom __instancecheck__ no longer needed Would be needed if calling a classmethod in __new__ but as we dont support reverse_delete_rules on embedded documents there is no longer the need for it. Refs #227 --- mongoengine/base.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 00dd2ee..88324bb 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -665,18 +665,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): return new_class - def __instancecheck__(cls, inst): - """Custom instance check for isinstance() as registering delete rules or - calling a cls method in __new__ seems to change the cls so vanilla - isinstance() fails""" - is_instance = super(DocumentMetaclass, cls).__instancecheck__(inst) - if hasattr(cls, '_meta') and 'delete_rules' in cls._meta and not is_instance: - try: - is_instance = get_document(cls.__name__) == get_document(inst.__class__.__name__) - except NotRegistered: - pass - return is_instance - @classmethod def _unique_with_indexes(cls, new_class, namespace=""): unique_indexes = [] From fc460b775e966d294776120baafa36d94ce99f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 2 Dec 2011 09:46:51 -0200 Subject: [PATCH 0456/1279] Small improvements for item_frequencies --- mongoengine/queryset.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 90547e2..0da1718 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1572,8 +1572,10 @@ class QuerySet(object): field.forEach(function(item) { emit(item, 1); }); - } else { + } else if (field) { emit(field, 1); + } else { + emit(null, 1); } } """ % dict(field=field) From 6419a8d09adea4b3ad7a1f293c3da92ad517a3af Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 2 Dec 2011 06:03:15 -0800 Subject: [PATCH 0457/1279] Fixed False BooleanField marked as unset by _delta() Closes #282 --- docs/changelog.rst | 1 + mongoengine/base.py | 2 +- tests/document.py | 13 +++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 30d6707..157409e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed issue saving False booleans - Fixed issue with dynamic documents deltas - Added Reverse Delete Rule support to ListFields - MapFields aren't supported - Added customisable cascade kwarg options diff --git a/mongoengine/base.py b/mongoengine/base.py index 88324bb..6755150 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -978,7 +978,7 @@ class BaseDocument(object): # Determine if any changed items were actually unset. for path, value in set_data.items(): - if value: + if value or isinstance(value, bool): continue # If we've set a value that ain't the default value dont unset it. diff --git a/tests/document.py b/tests/document.py index a4f0992..bf8946f 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2459,5 +2459,18 @@ class DocumentTest(unittest.TestCase): a.reload() self.assertEquals(a.b.field2.c_field, 'new value') + def test_can_save_false_values(self): + class Doc(Document): + foo = StringField() + archived = BooleanField(default=False, required=True) + + Doc.drop_collection() + d = Doc() + d.save() + d.archived = False + d.save() + + self.assertEquals(Doc.objects(archived=False).count(), 1) + if __name__ == '__main__': unittest.main() From 153538cef9e37ac0031639281bb8a4ec4647d2a8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 2 Dec 2011 06:34:51 -0800 Subject: [PATCH 0458/1279] Added test for saving false on dynamic documents Refs #282 Closes #311 --- tests/document.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/document.py b/tests/document.py index bf8946f..28679e5 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2460,6 +2460,7 @@ class DocumentTest(unittest.TestCase): self.assertEquals(a.b.field2.c_field, 'new value') def test_can_save_false_values(self): + """Ensures you can save False values on save""" class Doc(Document): foo = StringField() archived = BooleanField(default=False, required=True) @@ -2472,5 +2473,19 @@ class DocumentTest(unittest.TestCase): self.assertEquals(Doc.objects(archived=False).count(), 1) + + def test_can_save_false_values_dynamic(self): + """Ensures you can save False values on dynamic docs""" + class Doc(DynamicDocument): + foo = StringField() + + Doc.drop_collection() + d = Doc() + d.save() + d.archived = False + d.save() + + self.assertEquals(Doc.objects(archived=False).count(), 1) + if __name__ == '__main__': unittest.main() From 403977cd496b0628c61ad52a590fd642d74bacc9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 2 Dec 2011 06:40:57 -0800 Subject: [PATCH 0459/1279] Added test for saving references unnecessarily. Refs #359 --- AUTHORS | 1 + tests/document.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/AUTHORS b/AUTHORS index d543394..5dbd4d7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -83,3 +83,4 @@ that much better: * yamaneko1212 * dave mankoff * Alexander G. Morano + * jwilder diff --git a/tests/document.py b/tests/document.py index 28679e5..78c3827 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2487,5 +2487,37 @@ class DocumentTest(unittest.TestCase): self.assertEquals(Doc.objects(archived=False).count(), 1) + def test_do_not_save_unchanged_references(self): + """Ensures cascading saves dont auto update""" + class Job(Document): + name = StringField() + + class Person(Document): + name = StringField() + age = IntField() + job = ReferenceField(Job) + + Job.drop_collection() + Person.drop_collection() + + job = Job(name="Job 1") + # job should not have any changed fields after the save + job.save() + + person = Person(name="name", age=10, job=job) + + from pymongo.collection import Collection + orig_update = Collection.update + try: + def fake_update(*args, **kwargs): + self.fail("Unexpected update for %s" % args[0].name) + return orig_update(*args, **kwargs) + + Collection.update = fake_update + person.save() + finally: + Collection.update = orig_update + + if __name__ == '__main__': unittest.main() From a6948771d8b95920ddaf0639f7662c65eaf43376 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 2 Dec 2011 06:47:58 -0800 Subject: [PATCH 0460/1279] Added ReferencField handling with .distinct() Closes #356 --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/queryset.py | 4 +++- tests/queryset.py | 24 +++++++++++++++++++++--- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/AUTHORS b/AUTHORS index 5dbd4d7..055a2b9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -84,3 +84,4 @@ that much better: * dave mankoff * Alexander G. Morano * jwilder + * Joe Shaw diff --git a/docs/changelog.rst b/docs/changelog.rst index 157409e..e42e390 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added support for DBRefs in distinct() - Fixed issue saving False booleans - Fixed issue with dynamic documents deltas - Added Reverse Delete Rule support to ListFields - MapFields aren't supported diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 0da1718..a7a3798 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1073,8 +1073,10 @@ class QuerySet(object): :param field: the field to select distinct values from .. versionadded:: 0.4 + .. versionchanged:: 0.5 - Fixed handling references """ - return self._cursor.distinct(field) + from dereference import dereference + return dereference(self._cursor.distinct(field), 1) def only(self, *fields): """Load only a subset of this document's fields. :: diff --git a/tests/queryset.py b/tests/queryset.py index d978cf2..c128240 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1958,6 +1958,24 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(set(self.Person.objects(age=30).distinct('name')), set(['Mr Orange', 'Mr Pink'])) + def test_distinct_handles_references(self): + class Foo(Document): + bar = ReferenceField("Bar") + + class Bar(Document): + text = StringField() + + Bar.drop_collection() + Foo.drop_collection() + + bar = Bar(text="hi") + bar.save() + + foo = Foo(bar=bar) + foo.save() + + self.assertEquals(Foo.objects.distinct("bar"), [bar]) + def test_custom_manager(self): """Ensure that custom QuerySetManager instances work as expected. """ @@ -2870,7 +2888,7 @@ class QueryFieldListTest(unittest.TestCase): class Foo(EmbeddedDocument): shape = StringField() color = StringField() - trick = BooleanField() + trick = BooleanField() meta = {'allow_inheritance': False} class Bar(Document): @@ -2878,7 +2896,7 @@ class QueryFieldListTest(unittest.TestCase): meta = {'allow_inheritance': False} Bar.drop_collection() - + b1 = Bar(foo=[Foo(shape= "square", color ="purple", thick = False), Foo(shape= "circle", color ="red", thick = True)]) b1.save() @@ -2886,7 +2904,7 @@ class QueryFieldListTest(unittest.TestCase): b2 = Bar(foo=[Foo(shape= "square", color ="red", thick = True), Foo(shape= "circle", color ="purple", thick = False)]) b2.save() - + ak = list(Bar.objects(foo__match={'shape': "square", "color": "purple"})) self.assertEqual([b1], ak) From e9d73532944c38057135947af6bcabfde2331fe1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 2 Dec 2011 07:11:06 -0800 Subject: [PATCH 0461/1279] Updated with_id to raise Error if used with a filter. Closes #365 --- docs/changelog.rst | 1 + docs/upgrade.rst | 1 + mongoengine/queryset.py | 8 +++++++- tests/queryset.py | 2 ++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e42e390..2e5f1a3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added InvalidQueryError when calling with_id with a filter - Added support for DBRefs in distinct() - Fixed issue saving False booleans - Fixed issue with dynamic documents deltas diff --git a/docs/upgrade.rst b/docs/upgrade.rst index cb72b35..d406688 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -13,6 +13,7 @@ an InvalidDocument error as they aren't currently supported. Document._get_subclasses - Is no longer used and the class method has been removed. +Document.objects.with_id - now raises an InvalidQueryError if used with a filter. 0.4 to 0.5 =========== diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a7a3798..ce9b175 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -858,10 +858,16 @@ class QuerySet(object): return return_one and results[0] or results def with_id(self, object_id): - """Retrieve the object matching the id provided. + """Retrieve the object matching the id provided. Uses `object_id` only + and raises InvalidQueryError if a filter has been applied. :param object_id: the value for the id of the document to look up + + .. versionchanged:: 0.6 Raises InvalidQueryError if filter has been set """ + if not self._query_obj.empty: + raise InvalidQueryError("Cannot use a filter whilst using `with_id`") + return self._document.objects(pk=object_id).first() def in_bulk(self, object_ids): diff --git a/tests/queryset.py b/tests/queryset.py index c128240..37fa524 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -154,6 +154,8 @@ class QuerySetTest(unittest.TestCase): person = self.Person.objects.with_id(person1.id) self.assertEqual(person.name, "User A") + self.assertRaises(InvalidQueryError, self.Person.objects(name="User A").with_id, person1.id) + def test_find_only_one(self): """Ensure that a query using ``get`` returns at most one result. """ From 9bfc838029571b54f3e7d85de6f8735425ca7c1d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 2 Dec 2011 08:14:25 -0800 Subject: [PATCH 0462/1279] Updated Docs and bumped version Hopefully nearer 0.6 closes #368 --- docs/guide/document-instances.rst | 18 +++++++++--- docs/guide/querying.rst | 47 ++++++++++++++++++++----------- docs/index.rst | 3 ++ mongoengine/__init__.py | 2 +- 4 files changed, 48 insertions(+), 22 deletions(-) diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index 317bfef..4e799ec 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -35,13 +35,23 @@ already exist, then any changes will be updated atomically. For example:: * ``list_field.pop(0)`` - *sets* the resulting list * ``del(list_field)`` - *unsets* whole list -To delete a document, call the :meth:`~mongoengine.Document.delete` method. -Note that this will only work if the document exists in the database and has a -valide :attr:`id`. - .. seealso:: :ref:`guide-atomic-updates` +Cascading Saves +--------------- +If your document contains :class:`~mongoengine.ReferenceField` or +:class:`~mongoengine.GenericReferenceField` objects, then by default the +:meth:`~mongoengine.Document.save` method will automatically save any changes to +those objects as well. If this is not desired passing :attr:`cascade` as False +to the save method turns this feature off. + +Deleting documents +------------------ +To delete a document, call the :meth:`~mongoengine.Document.delete` method. +Note that this will only work if the document exists in the database and has a +valid :attr:`id`. + Document IDs ============ Each document in the database has a unique id. This may be accessed through the diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 9ebf37f..a9567e2 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -195,22 +195,6 @@ to be created:: >>> a.name == b.name and a.age == b.age True -Dereferencing results ---------------------- -When iterating the results of :class:`~mongoengine.ListField` or -:class:`~mongoengine.DictField` we automatically dereference any -:class:`~pymongo.dbref.DBRef` objects as efficiently as possible, reducing the -number the queries to mongo. - -There are times when that efficiency is not enough, documents that have -:class:`~mongoengine.ReferenceField` objects or -:class:`~mongoengine.GenericReferenceField` objects at the top level are -expensive as the number of queries to MongoDB can quickly rise. - -To limit the number of queries use -:func:`~mongoengine.queryset.QuerySet.select_related` which converts the -QuerySet to a list and dereferences as efficiently as possible. - Default Document queries ======================== By default, the objects :attr:`~mongoengine.Document.objects` attribute on a @@ -313,8 +297,16 @@ would be generating "tag-clouds":: from operator import itemgetter top_tags = sorted(tag_freqs.items(), key=itemgetter(1), reverse=True)[:10] + +Query efficiency and performance +================================ + +There are a couple of methods to improve efficiency when querying, reducing the +information returned by the query or efficient dereferencing . + Retrieving a subset of fields -============================= +----------------------------- + Sometimes a subset of fields on a :class:`~mongoengine.Document` is required, and for efficiency only these should be retrieved from the database. This issue is especially important for MongoDB, as fields may often be extremely large @@ -347,6 +339,27 @@ will be given:: If you later need the missing fields, just call :meth:`~mongoengine.Document.reload` on your document. +Getting related data +-------------------- + +When iterating the results of :class:`~mongoengine.ListField` or +:class:`~mongoengine.DictField` we automatically dereference any +:class:`~pymongo.dbref.DBRef` objects as efficiently as possible, reducing the +number the queries to mongo. + +There are times when that efficiency is not enough, documents that have +:class:`~mongoengine.ReferenceField` objects or +:class:`~mongoengine.GenericReferenceField` objects at the top level are +expensive as the number of queries to MongoDB can quickly rise. + +To limit the number of queries use +:func:`~mongoengine.queryset.QuerySet.select_related` which converts the +QuerySet to a list and dereferences as efficiently as possible. By default +:func:`~mongoengine.queryset.QuerySet.select_related` only dereferences any +references to the depth of 1 level. If you have more complicated documents and +want to dereference more of the object at once then increasing the :attr:`max_depth` +will dereference more levels of the document. + Advanced queries ================ Sometimes calling a :class:`~mongoengine.queryset.QuerySet` object with keyword diff --git a/docs/index.rst b/docs/index.rst index a574f29..ef225e4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,6 +18,9 @@ MongoDB. To install it, simply run :doc:`apireference` The complete API documentation. +:doc:`upgrade` + The Upgrade guide. + :doc:`django` Using MongoEngine and Django diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index b28c374..d10d113 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -14,7 +14,7 @@ __all__ = (document.__all__ + fields.__all__ + connection.__all__ + __author__ = 'Harry Marr' -VERSION = (0, 5, 2) +VERSION = (0, 5, 3) def get_version(): From 700bc1b4bbc5c68bbb1a117ed89168f1bb4d3ed5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 2 Dec 2011 08:44:15 -0800 Subject: [PATCH 0463/1279] Multiple fields with the same db_field now raises Exception Closes #329 --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/base.py | 8 +++++++- tests/document.py | 10 ++++++++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 055a2b9..76f9b70 100644 --- a/AUTHORS +++ b/AUTHORS @@ -85,3 +85,4 @@ that much better: * Alexander G. Morano * jwilder * Joe Shaw + * Adam Flynn diff --git a/docs/changelog.rst b/docs/changelog.rst index 2e5f1a3..6dd9dbc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added Now raises an InvalidDocumentError when declaring multiple fields with the same db_field - Added InvalidQueryError when calling with_id with a filter - Added support for DBRefs in distinct() - Fixed issue saving False booleans diff --git a/mongoengine/base.py b/mongoengine/base.py index 6755150..c11b627 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -492,6 +492,7 @@ class DocumentMetaclass(type): attrs['_superclasses'] = superclasses # Add the document's fields to the _fields attribute + field_names = {} for attr_name, attr_value in attrs.items(): if hasattr(attr_value, "__class__") and \ issubclass(attr_value.__class__, BaseField): @@ -499,8 +500,13 @@ class DocumentMetaclass(type): if not attr_value.db_field: attr_value.db_field = attr_name doc_fields[attr_name] = attr_value + field_names[attr_value.db_field] = field_names.get(attr_value.db_field, 0) + 1 + + duplicate_db_fields = [k for k, v in field_names.items() if v > 1] + if duplicate_db_fields: + raise InvalidDocumentError("Multiple db_fields defined for: %s " % ", ".join(duplicate_db_fields)) attrs['_fields'] = doc_fields - attrs['_db_field_map'] = dict([(k, v.db_field) for k, v in doc_fields.items() if k!=v.db_field]) + attrs['_db_field_map'] = dict([(k, v.db_field) for k, v in doc_fields.items() if k != v.db_field]) attrs['_reverse_db_field_map'] = dict([(v, k) for k, v in attrs['_db_field_map'].items()]) from mongoengine import Document, EmbeddedDocument, DictField diff --git a/tests/document.py b/tests/document.py index 78c3827..5f3bc63 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2182,6 +2182,16 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + def test_duplicate_db_fields_raise_invalid_document_error(self): + """Ensure a InvalidDocumentError is thrown if duplicate fields + declare the same db_field""" + + def throw_invalid_document_error(): + class Foo(Document): + name = StringField() + name2 = StringField(db_field='name') + + self.assertRaises(InvalidDocumentError, throw_invalid_document_error) def test_reverse_delete_rule_cascade_and_nullify(self): """Ensure that a referenced document is also deleted upon deletion. From 84f9e44b6cb6a3d8ccc1534a4d5bdec2c8e578eb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 5 Dec 2011 04:16:57 -0800 Subject: [PATCH 0464/1279] Fixed GridFS documents can now be pickled Refs #135 #381 --- docs/changelog.rst | 1 + mongoengine/fields.py | 26 ++++++++++++++++++++++---- tests/fixtures.py | 1 + 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6dd9dbc..81b3430 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed GridFS documents can now be pickled - Added Now raises an InvalidDocumentError when declaring multiple fields with the same db_field - Added InvalidQueryError when calling with_id with a filter - Added support for DBRefs in distinct() diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 68d7ace..914cb92 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -786,18 +786,25 @@ class GridFSProxy(object): .. versionchanged:: 0.6 - added collection name param """ + _fs = None + def __init__(self, grid_id=None, key=None, instance=None, db_alias=DEFAULT_CONNECTION_NAME, collection_name='fs'): - self.fs = gridfs.GridFS(get_db(db_alias), collection_name) # Filesystem instance - self.newfile = None # Used for partial writes - self.grid_id = grid_id # Store GridFS id for file - self.gridout = None + self.grid_id = grid_id # Store GridFS id for file self.key = key self.instance = instance + self.db_alias = db_alias + self.collection_name = collection_name + self.newfile = None # Used for partial writes + self.gridout = None def __getattr__(self, name): + attrs = ('_fs', 'grid_id', 'key', 'instance', 'db_alias', + 'collection_name', 'newfile', 'gridout') + if name in attrs: + return self.__getattribute__(name) obj = self.get() if name in dir(obj): return getattr(obj, name) @@ -809,6 +816,17 @@ class GridFSProxy(object): def __nonzero__(self): return bool(self.grid_id) + def __getstate__(self): + self_dict = self.__dict__ + self_dict['_fs'] = None + return self_dict + + @property + def fs(self): + if not self._fs: + self._fs = gridfs.GridFS(get_db(self.db_alias), self.collection_name) + return self._fs + def get(self, id=None): if id: self.grid_id = id diff --git a/tests/fixtures.py b/tests/fixtures.py index 32081fe..a97e8eb 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -12,6 +12,7 @@ class PickleTest(Document): string = StringField(choices=(('One', '1'), ('Two', '2'))) embedded = EmbeddedDocumentField(PickleEmbedded) lists = ListField(StringField()) + photo = FileField() class Mixin(object): From 45b5bf73fe4fc2e5b0cb7976c22dad9cd2441fea Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 5 Dec 2011 04:17:51 -0800 Subject: [PATCH 0465/1279] Added Jan to the contributors list Refs #135 #381 --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 76f9b70..095c400 100644 --- a/AUTHORS +++ b/AUTHORS @@ -86,3 +86,4 @@ that much better: * jwilder * Joe Shaw * Adam Flynn + * Jan Schrewe From be78209f94637319a4e293e77816f8dc3ef24103 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 5 Dec 2011 04:44:40 -0800 Subject: [PATCH 0466/1279] Added test showing you can add index for dynamic docs --- tests/dynamic_document.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/dynamic_document.py b/tests/dynamic_document.py index b40ba06..6363672 100644 --- a/tests/dynamic_document.py +++ b/tests/dynamic_document.py @@ -448,3 +448,31 @@ class DynamicDocTest(unittest.TestCase): doc.dict_field['embedded'].string_field = 'Hello World' self.assertEquals(doc._get_changed_fields(), ['dict_field.embedded.string_field']) self.assertEquals(doc._delta(), ({'dict_field.embedded.string_field': 'Hello World'}, {})) + + def test_indexes(self): + """Ensure that indexes are used when meta[indexes] is specified. + """ + class BlogPost(DynamicDocument): + meta = { + 'indexes': [ + '-date', + ('category', '-date') + ], + } + + BlogPost.drop_collection() + + info = BlogPost.objects._collection.index_information() + # _id, '-date', ('cat', 'date') + # NB: there is no index on _types by itself, since + # the indices on -date and tags will both contain + # _types as first element in the key + self.assertEqual(len(info), 3) + + # Indexes are lazy so use list() to perform query + list(BlogPost.objects) + info = BlogPost.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] + self.assertTrue([('_types', 1), ('category', 1), ('date', -1)] + in info) + self.assertTrue([('_types', 1), ('date', -1)] in info) From cf4a45da118e7adf3cecf6084159f80cce1d9bb6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 6 Dec 2011 06:38:25 -0800 Subject: [PATCH 0467/1279] Dynamic Documents now support string query lookups --- mongoengine/base.py | 6 ++++++ tests/dynamic_document.py | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index c11b627..38e528f 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -412,6 +412,12 @@ class BaseDynamicField(BaseField): def lookup_member(self, member_name): return member_name + def prepare_query_value(self, op, value): + if isinstance(value, basestring): + from mongoengine.fields import StringField + return StringField().prepare_query_value(op, value) + return self.to_mongo(value) + class ObjectIdField(BaseField): """An field wrapper around MongoDB's ObjectIds. diff --git a/tests/dynamic_document.py b/tests/dynamic_document.py index 6363672..0eeedfa 100644 --- a/tests/dynamic_document.py +++ b/tests/dynamic_document.py @@ -98,6 +98,27 @@ class DynamicDocTest(unittest.TestCase): p = p.get() self.assertEquals(22, p.age) + def test_complex_dynamic_document_queries(self): + class Person(DynamicDocument): + name = StringField() + + Person.drop_collection() + + p = Person(name="test") + p.age = "ten" + p.save() + + p1 = Person(name="test1") + p1.age = "less then ten and a half" + p1.save() + + p2 = Person(name="test2") + p2.age = 10 + p2.save() + + self.assertEquals(Person.objects(age__icontains='ten').count(), 2) + self.assertEquals(Person.objects(age__gte=10).count(), 1) + def test_complex_data_lookups(self): """Ensure you can query dynamic document dynamic fields""" p = self.Person() From 8d2bc444bb64265f78f5bf716f773742dddd56c1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Dec 2011 01:14:02 -0800 Subject: [PATCH 0468/1279] db_alias using in model, queryset, reference fields, derefrence. --- mongoengine/dereference.py | 18 ++++-- mongoengine/document.py | 11 +++- mongoengine/fields.py | 9 +-- mongoengine/queryset.py | 4 +- tests/document.py | 116 +++++++++++++++++++++++++++++++++++++ 5 files changed, 143 insertions(+), 15 deletions(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 0d8d7f1..3dac92b 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -103,13 +103,19 @@ class DeReference(object): for key, doc in references.iteritems(): object_map[key] = doc else: # Generic reference: use the refs data to convert to document - references = get_db()[col].find({'_id': {'$in': refs}}) - for ref in references: - if '_cls' in ref: - doc = get_document(ref['_cls'])._from_son(ref) - else: + if doc_type: + references = doc_type._get_db()[col].find({'_id': {'$in': refs}}) + for ref in references: doc = doc_type._from_son(ref) - object_map[doc.id] = doc + object_map[doc.id] = doc + else: + references = get_db()[col].find({'_id': {'$in': refs}}) + for ref in references: + if '_cls' in ref: + doc = get_document(ref["_cls"])._from_son(ref) + else: + doc = doc_type._from_son(ref) + object_map[doc.id] = doc return object_map def _attach_objects(self, items, depth=0, instance=None, name=None): diff --git a/mongoengine/document.py b/mongoengine/document.py index 3f4c9d7..e56e4ab 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -2,7 +2,7 @@ from mongoengine import signals from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, BaseDict, BaseList, DataObserver) from queryset import OperationError -from connection import get_db +from connection import get_db, DEFAULT_CONNECTION_NAME import pymongo @@ -86,11 +86,16 @@ class Document(BaseDocument): return setattr(self, self._meta['id_field'], value) return property(fget, fset) + @classmethod + def _get_db(self): + """Some Model using other db_alias""" + return get_db(self._meta.get("db_alias", DEFAULT_CONNECTION_NAME )) + @classmethod def _get_collection(self): """Returns the collection for the document.""" if not hasattr(self, '_collection') or self._collection is None: - db = get_db() + db = self._get_db() collection_name = self._get_collection_name() # Create collection as a capped collection if specified if self._meta['max_size'] or self._meta['max_documents']: @@ -318,7 +323,7 @@ class Document(BaseDocument): :class:`~mongoengine.Document` type from the database. """ from mongoengine.queryset import QuerySet - db = get_db() + db = cls._get_db() db.drop_collection(cls._get_collection_name()) QuerySet._reset_already_indexed(cls) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 914cb92..2f5911f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -645,7 +645,7 @@ class ReferenceField(BaseField): value = instance._data.get(self.name) # Dereference DBRefs if isinstance(value, (pymongo.dbref.DBRef)): - value = get_db().dereference(value) + value = self.document_type._get_db().dereference(value) if value is not None: instance._data[self.name] = self.document_type._from_son(value) @@ -718,7 +718,7 @@ class GenericReferenceField(BaseField): def dereference(self, value): doc_cls = get_document(value['_cls']) reference = value['_ref'] - doc = get_db().dereference(reference) + doc = doc_cls._get_db().dereference(reference) if doc is not None: doc = doc_cls._from_son(doc) return doc @@ -1162,8 +1162,9 @@ class SequenceField(IntField): .. versionadded:: 0.5 """ - def __init__(self, collection_name=None, *args, **kwargs): + def __init__(self, collection_name=None, db_alias = None, *args, **kwargs): self.collection_name = collection_name or 'mongoengine.counters' + self.db_alias = db_alias or DEFAULT_CONNECTION_NAME return super(SequenceField, self).__init__(*args, **kwargs) def generate_new_value(self): @@ -1172,7 +1173,7 @@ class SequenceField(IntField): """ sequence_id = "{0}.{1}".format(self.owner_document._get_collection_name(), self.name) - collection = get_db()[self.collection_name] + collection = get_db(alias = self.db_alias )[self.collection_name] counter = collection.find_and_modify(query={"_id": sequence_id}, update={"$inc": {"next": 1}}, new=True, diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index ce9b175..0c39253 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -481,7 +481,7 @@ class QuerySet(object): if self._document not in QuerySet.__already_indexed: # Ensure collection exists - db = get_db() + db = self._document._get_db() if self._collection_obj.name not in db.collection_names(): self._document._collection = None self._collection_obj = self._document._get_collection() @@ -1452,7 +1452,7 @@ class QuerySet(object): scope['query'] = query code = pymongo.code.Code(code, scope=scope) - db = get_db() + db = self._document._get_db() return db.eval(code, *fields) def where(self, where_clause): diff --git a/tests/document.py b/tests/document.py index 5f3bc63..0d9f8d7 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2528,6 +2528,122 @@ class DocumentTest(unittest.TestCase): finally: Collection.update = orig_update + def test_db_alias_tests(self): + """ DB Alias tests """ + # mongoenginetest - Is default connection alias from setUp() + # Register Aliases + register_connection('testdb-1', "mongoenginetest2" ) + register_connection('testdb-2', "mongoenginetest3" ) + register_connection('testdb-3', 'mongoenginetest4') + + class User(Document): + name = StringField() + class Book(Document): + name = StringField() + meta = {"db_alias" : "testdb-1" } + + # Drops + User.drop_collection() + Book.drop_collection() + + # Create + bob = User.objects.create(name = "Bob") + hp = Book.objects.create(name = "Harry Potter") + + # Selects + self.assertEqual( User.objects.first(), bob) + self.assertEqual( Book.objects.first(), hp) + + # DeRefecence + class AuthorBooks(Document): + author = ReferenceField(User) + book = ReferenceField(Book) + meta = {"db_alias" : "testdb-2" } + # Drops + AuthorBooks.drop_collection() + + ab = AuthorBooks.objects.create( author = bob, book = hp) + # select + self.assertEqual( AuthorBooks.objects.first(), ab) + self.assertEqual( AuthorBooks.objects.first().book, hp) + self.assertEqual( AuthorBooks.objects.first().author, bob) + + + + + + + + + def test_db_ref_usage(self): + """ DB Ref usage in __raw__ queries """ + class User(Document): + name = StringField() + + class Book(Document): + name = StringField() + author = ReferenceField(User) + extra = DictField() + meta = { + 'ordering': ['+name'] + } + + def __unicode__(self): + return self.name + def __str__(self): + return self.name + + + # Drops + User.drop_collection() + Book.drop_collection() + + # Authors + bob = User.objects.create(name = "Bob") + jon = User.objects.create(name = "Jon") + + # Redactors + karl = User.objects.create( name = "Karl") + susan = User.objects.create( name = "Susan") + peter = User.objects.create( name = "Peter") + + # Bob + Book.objects.create( name = "1", author = bob, extra = {"a": bob.to_dbref(), "b" : [karl.to_dbref(), susan.to_dbref(),] } ) + Book.objects.create( name = "2", author = bob, extra = {"a": bob.to_dbref(), "b" : karl.to_dbref()} ) + Book.objects.create( name = "3", author = bob, extra = {"a": bob.to_dbref(), "c" : [jon.to_dbref(), peter.to_dbref() ] }) + Book.objects.create( name = "4", author = bob,) + + # Jon + Book.objects.create( name = "5", author = jon,) + Book.objects.create( name = "6", author = peter,) + Book.objects.create( name = "7", author = jon,) + Book.objects.create( name = "8", author = jon,) + Book.objects.create( name = "9", author = jon, extra = {"a": peter.to_dbref() }) + + # Checks + self.assertEqual(u",".join([str(b) for b in Book.objects.all()] ) , "1,2,3,4,5,6,7,8,9" ) + # bob related books + self.assertEqual(u",".join([str(b) for b in Book.objects.filter( + Q(extra__a = bob ) | + Q(author = bob) | + Q(extra__b = bob ) )] ) , + "1,2,3,4" ) + # Susan & Karl related books + self.assertEqual(u",".join([str(b) for b in Book.objects.filter( + Q(extra__a__all = [karl, susan] ) | + Q(author__all = [karl, susan ] ) | + Q(extra__b__all = [karl.to_dbref(), susan.to_dbref()] ) + ) ] ) , "1" ) + + # $Where + self.assertEqual(u",".join([str(b) for b in Book.objects.filter( + __raw__ = { + "$where" : """function(){ return this.name == '1' || this.name == '2'; } """ + } + ) ] ) , "1,2" ) + + + if __name__ == '__main__': unittest.main() From fbe1901e65a11c10821a79ee5b4f91e6eeda86e7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Dec 2011 01:14:43 -0800 Subject: [PATCH 0469/1279] Added some tests #384 --- tests/document.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/document.py b/tests/document.py index 0d9f8d7..3c48cfa 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2559,15 +2559,29 @@ class DocumentTest(unittest.TestCase): author = ReferenceField(User) book = ReferenceField(Book) meta = {"db_alias" : "testdb-2" } + # Drops AuthorBooks.drop_collection() ab = AuthorBooks.objects.create( author = bob, book = hp) + # select self.assertEqual( AuthorBooks.objects.first(), ab) self.assertEqual( AuthorBooks.objects.first().book, hp) self.assertEqual( AuthorBooks.objects.first().author, bob) + self.assertEqual( AuthorBooks.objects.filter(author = bob).first() , ab) + self.assertEqual( AuthorBooks.objects.filter(book = hp).first() , ab) + + # DB Alias + self.assertEqual( User._get_db() , get_db()) + self.assertEqual( Book._get_db() , get_db("testdb-1")) + self.assertEqual( AuthorBooks._get_db() , get_db("testdb-2")) + + # Collections + self.assertEqual( User._get_collection() , get_db()[User._get_collection_name()] ) + self.assertEqual( Book._get_collection() , get_db("testdb-2")[Book._get_collection_name()] ) + self.assertEqual( AuthorBooks._get_collection() , get_db("testdb-2")[AuthorBooks._get_collection_name()] ) From 216f15602b02cca1688fef5e4d0b87421052fa5a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Dec 2011 01:16:09 -0800 Subject: [PATCH 0470/1279] Fixing test --- tests/document.py | 80 ++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 43 deletions(-) diff --git a/tests/document.py b/tests/document.py index 3c48cfa..a4ce342 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2532,62 +2532,57 @@ class DocumentTest(unittest.TestCase): """ DB Alias tests """ # mongoenginetest - Is default connection alias from setUp() # Register Aliases - register_connection('testdb-1', "mongoenginetest2" ) - register_connection('testdb-2', "mongoenginetest3" ) + register_connection('testdb-1', 'mongoenginetest2') + register_connection('testdb-2', 'mongoenginetest3') register_connection('testdb-3', 'mongoenginetest4') class User(Document): name = StringField() + meta = {"db_alias": "testdb-1"} + class Book(Document): name = StringField() - meta = {"db_alias" : "testdb-1" } + meta = {"db_alias": "testdb-2"} # Drops User.drop_collection() Book.drop_collection() # Create - bob = User.objects.create(name = "Bob") - hp = Book.objects.create(name = "Harry Potter") + bob = User.objects.create(name="Bob") + hp = Book.objects.create(name="Harry Potter") # Selects - self.assertEqual( User.objects.first(), bob) - self.assertEqual( Book.objects.first(), hp) + self.assertEqual(User.objects.first(), bob) + self.assertEqual(Book.objects.first(), hp) # DeRefecence class AuthorBooks(Document): author = ReferenceField(User) book = ReferenceField(Book) - meta = {"db_alias" : "testdb-2" } + meta = {"db_alias": "testdb-3"} # Drops AuthorBooks.drop_collection() - ab = AuthorBooks.objects.create( author = bob, book = hp) + ab = AuthorBooks.objects.create(author=bob, book=hp) # select - self.assertEqual( AuthorBooks.objects.first(), ab) - self.assertEqual( AuthorBooks.objects.first().book, hp) - self.assertEqual( AuthorBooks.objects.first().author, bob) - - self.assertEqual( AuthorBooks.objects.filter(author = bob).first() , ab) - self.assertEqual( AuthorBooks.objects.filter(book = hp).first() , ab) + self.assertEqual(AuthorBooks.objects.first(), ab) + self.assertEqual(AuthorBooks.objects.first().book, hp) + self.assertEqual(AuthorBooks.objects.first().author, bob) + self.assertEqual(AuthorBooks.objects.filter(author=bob).first(), ab) + self.assertEqual(AuthorBooks.objects.filter(book=hp).first(), ab) # DB Alias - self.assertEqual( User._get_db() , get_db()) - self.assertEqual( Book._get_db() , get_db("testdb-1")) - self.assertEqual( AuthorBooks._get_db() , get_db("testdb-2")) + self.assertEqual(User._get_db(), get_db("testdb-1")) + self.assertEqual(Book._get_db(), get_db("testdb-2")) + self.assertEqual(AuthorBooks._get_db(), get_db("testdb-3")) # Collections - self.assertEqual( User._get_collection() , get_db()[User._get_collection_name()] ) - self.assertEqual( Book._get_collection() , get_db("testdb-2")[Book._get_collection_name()] ) - self.assertEqual( AuthorBooks._get_collection() , get_db("testdb-2")[AuthorBooks._get_collection_name()] ) - - - - - - + self.assertEqual(User._get_collection(), get_db("testdb-1")[User._get_collection_name()]) + self.assertEqual(Book._get_collection(), get_db("testdb-2")[Book._get_collection_name()]) + self.assertEqual(AuthorBooks._get_collection(), get_db("testdb-3")[AuthorBooks._get_collection_name()]) def test_db_ref_usage(self): """ DB Ref usage in __raw__ queries """ @@ -2597,17 +2592,17 @@ class DocumentTest(unittest.TestCase): class Book(Document): name = StringField() author = ReferenceField(User) - extra = DictField() + extra = DictField() meta = { 'ordering': ['+name'] } def __unicode__(self): return self.name + def __str__(self): return self.name - # Drops User.drop_collection() Book.drop_collection() @@ -2617,22 +2612,22 @@ class DocumentTest(unittest.TestCase): jon = User.objects.create(name = "Jon") # Redactors - karl = User.objects.create( name = "Karl") - susan = User.objects.create( name = "Susan") - peter = User.objects.create( name = "Peter") + karl = User.objects.create(name = "Karl") + susan = User.objects.create(name = "Susan") + peter = User.objects.create(name = "Peter") # Bob - Book.objects.create( name = "1", author = bob, extra = {"a": bob.to_dbref(), "b" : [karl.to_dbref(), susan.to_dbref(),] } ) - Book.objects.create( name = "2", author = bob, extra = {"a": bob.to_dbref(), "b" : karl.to_dbref()} ) - Book.objects.create( name = "3", author = bob, extra = {"a": bob.to_dbref(), "c" : [jon.to_dbref(), peter.to_dbref() ] }) - Book.objects.create( name = "4", author = bob,) + Book.objects.create(name = "1", author = bob, extra = {"a": bob.to_dbref(), "b" : [karl.to_dbref(), susan.to_dbref()]} ) + Book.objects.create(name = "2", author = bob, extra = {"a": bob.to_dbref(), "b" : karl.to_dbref()} ) + Book.objects.create(name = "3", author = bob, extra = {"a": bob.to_dbref(), "c" : [jon.to_dbref(), peter.to_dbref()] }) + Book.objects.create(name = "4", author = bob) # Jon - Book.objects.create( name = "5", author = jon,) - Book.objects.create( name = "6", author = peter,) - Book.objects.create( name = "7", author = jon,) - Book.objects.create( name = "8", author = jon,) - Book.objects.create( name = "9", author = jon, extra = {"a": peter.to_dbref() }) + Book.objects.create(name = "5", author = jon) + Book.objects.create(name = "6", author = peter) + Book.objects.create(name = "7", author = jon) + Book.objects.create(name = "8", author = jon) + Book.objects.create(name = "9", author = jon, extra = {"a": peter.to_dbref()}) # Checks self.assertEqual(u",".join([str(b) for b in Book.objects.all()] ) , "1,2,3,4,5,6,7,8,9" ) @@ -2642,6 +2637,7 @@ class DocumentTest(unittest.TestCase): Q(author = bob) | Q(extra__b = bob ) )] ) , "1,2,3,4" ) + # Susan & Karl related books self.assertEqual(u",".join([str(b) for b in Book.objects.filter( Q(extra__a__all = [karl, susan] ) | @@ -2657,7 +2653,5 @@ class DocumentTest(unittest.TestCase): ) ] ) , "1,2" ) - - if __name__ == '__main__': unittest.main() From 112e921ce29c1d5bbb66c4862edeb43c0eab1bb2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Dec 2011 01:34:36 -0800 Subject: [PATCH 0471/1279] Syntax cleaning --- mongoengine/document.py | 30 +++++++++++------------ tests/document.py | 53 ++++++++++++++++++++++------------------- 2 files changed, 43 insertions(+), 40 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index e56e4ab..6b0d828 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -87,43 +87,43 @@ class Document(BaseDocument): return property(fget, fset) @classmethod - def _get_db(self): + def _get_db(cls): """Some Model using other db_alias""" - return get_db(self._meta.get("db_alias", DEFAULT_CONNECTION_NAME )) + return get_db(cls._meta.get("db_alias", DEFAULT_CONNECTION_NAME )) @classmethod - def _get_collection(self): + def _get_collection(cls): """Returns the collection for the document.""" - if not hasattr(self, '_collection') or self._collection is None: - db = self._get_db() - collection_name = self._get_collection_name() + if not hasattr(cls, '_collection') or cls._collection is None: + db = cls._get_db() + collection_name = cls._get_collection_name() # Create collection as a capped collection if specified - if self._meta['max_size'] or self._meta['max_documents']: + if cls._meta['max_size'] or cls._meta['max_documents']: # Get max document limit and max byte size from meta - max_size = self._meta['max_size'] or 10000000 # 10MB default - max_documents = self._meta['max_documents'] + max_size = cls._meta['max_size'] or 10000000 # 10MB default + max_documents = cls._meta['max_documents'] if collection_name in db.collection_names(): - self._collection = db[collection_name] + cls._collection = db[collection_name] # The collection already exists, check if its capped # options match the specified capped options - options = self._collection.options() + options = cls._collection.options() if options.get('max') != max_documents or \ options.get('size') != max_size: msg = ('Cannot create collection "%s" as a capped ' - 'collection as it already exists') % self._collection + 'collection as it already exists') % cls._collection raise InvalidCollectionError(msg) else: # Create the collection as a capped collection opts = {'capped': True, 'size': max_size} if max_documents: opts['max'] = max_documents - self._collection = db.create_collection( + cls._collection = db.create_collection( collection_name, **opts ) else: - self._collection = db[collection_name] - return self._collection + cls._collection = db[collection_name] + return cls._collection def save(self, safe=True, force_insert=False, validate=True, write_options=None, cascade=None, cascade_kwargs=None, _refs=None): diff --git a/tests/document.py b/tests/document.py index a4ce342..faedb42 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2586,6 +2586,7 @@ class DocumentTest(unittest.TestCase): def test_db_ref_usage(self): """ DB Ref usage in __raw__ queries """ + class User(Document): name = StringField() @@ -2608,50 +2609,52 @@ class DocumentTest(unittest.TestCase): Book.drop_collection() # Authors - bob = User.objects.create(name = "Bob") - jon = User.objects.create(name = "Jon") + bob = User.objects.create(name="Bob") + jon = User.objects.create(name="Jon") # Redactors - karl = User.objects.create(name = "Karl") - susan = User.objects.create(name = "Susan") - peter = User.objects.create(name = "Peter") + karl = User.objects.create(name="Karl") + susan = User.objects.create(name="Susan") + peter = User.objects.create(name="Peter") # Bob - Book.objects.create(name = "1", author = bob, extra = {"a": bob.to_dbref(), "b" : [karl.to_dbref(), susan.to_dbref()]} ) - Book.objects.create(name = "2", author = bob, extra = {"a": bob.to_dbref(), "b" : karl.to_dbref()} ) - Book.objects.create(name = "3", author = bob, extra = {"a": bob.to_dbref(), "c" : [jon.to_dbref(), peter.to_dbref()] }) - Book.objects.create(name = "4", author = bob) + Book.objects.create(name="1", author=bob, extra={"a": bob.to_dbref(), "b": [karl.to_dbref(), susan.to_dbref()]}) + Book.objects.create(name="2", author=bob, extra={"a": bob.to_dbref(), "b": karl.to_dbref()} ) + Book.objects.create(name="3", author=bob, extra={"a": bob.to_dbref(), "c": [jon.to_dbref(), peter.to_dbref()]}) + Book.objects.create(name="4", author=bob) # Jon - Book.objects.create(name = "5", author = jon) - Book.objects.create(name = "6", author = peter) - Book.objects.create(name = "7", author = jon) - Book.objects.create(name = "8", author = jon) - Book.objects.create(name = "9", author = jon, extra = {"a": peter.to_dbref()}) + Book.objects.create(name="5", author=jon) + Book.objects.create(name="6", author=peter) + Book.objects.create(name="7", author=jon) + Book.objects.create(name="8", author=jon) + Book.objects.create(name="9", author=jon, extra={"a": peter.to_dbref()}) # Checks self.assertEqual(u",".join([str(b) for b in Book.objects.all()] ) , "1,2,3,4,5,6,7,8,9" ) # bob related books self.assertEqual(u",".join([str(b) for b in Book.objects.filter( - Q(extra__a = bob ) | - Q(author = bob) | - Q(extra__b = bob ) )] ) , - "1,2,3,4" ) + Q(extra__a=bob ) | + Q(author=bob) | + Q(extra__b=bob))]) , + "1,2,3,4") # Susan & Karl related books self.assertEqual(u",".join([str(b) for b in Book.objects.filter( - Q(extra__a__all = [karl, susan] ) | - Q(author__all = [karl, susan ] ) | - Q(extra__b__all = [karl.to_dbref(), susan.to_dbref()] ) + Q(extra__a__all=[karl, susan] ) | + Q(author__all=[karl, susan ] ) | + Q(extra__b__all=[karl.to_dbref(), susan.to_dbref()] ) ) ] ) , "1" ) # $Where self.assertEqual(u",".join([str(b) for b in Book.objects.filter( - __raw__ = { - "$where" : """function(){ return this.name == '1' || this.name == '2'; } """ + __raw__={ + "$where": """ + function(){ + return this.name == '1' || + this.name == '2';}""" } - ) ] ) , "1,2" ) - + ) ]), "1,2") if __name__ == '__main__': unittest.main() From e7bcb5e3665d42f46f7ce8e73dcd40d6fb3b7c20 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Dec 2011 01:46:11 -0800 Subject: [PATCH 0472/1279] Updated docs and Authors list re: db_alias --- AUTHORS | 1 + docs/changelog.rst | 1 + docs/guide/connecting.rst | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/AUTHORS b/AUTHORS index 095c400..1ac192c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -86,4 +86,5 @@ that much better: * jwilder * Joe Shaw * Adam Flynn + * Ankhbayar * Jan Schrewe diff --git a/docs/changelog.rst b/docs/changelog.rst index 81b3430..003a59f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added db_alias support to individual documents - Fixed GridFS documents can now be pickled - Added Now raises an InvalidDocumentError when declaring multiple fields with the same db_field - Added InvalidQueryError when calling with_id with a filter diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index a51d68e..470dcb8 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -30,3 +30,24 @@ for the connection - if no `alias` is provided then "default" is used. In the background this uses :func:`~mongoengine.register_connection` to store the data and you can register all aliases up front if required. + +Individual documents can also support multiple databases by providing a +`db_alias` in their meta data. This allows :class:`~pymongo.dbref.DBRef` objects +to point across databases and collections. Below is an example schema, using +3 different databases to store data:: + + class User(Document): + name = StringField() + + meta = {"db_alias": "user-db"} + + class Book(Document): + name = StringField() + + meta = {"db_alias": "book-db"} + + class AuthorBooks(Document): + author = ReferenceField(User) + book = ReferenceField(Book) + + meta = {"db_alias": "users-books-db"} From 3e2f035400b4923db7086f676a97d3cf953bd568 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Dec 2011 02:15:50 -0800 Subject: [PATCH 0473/1279] Abstract documents can now declare indexes fixes #380 --- docs/changelog.rst | 1 + mongoengine/base.py | 9 +++++++-- tests/document.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 003a59f..7c9055d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed Abstract documents can now declare indexes - Added db_alias support to individual documents - Fixed GridFS documents can now be pickled - Added Now raises an InvalidDocumentError when declaring multiple fields with the same db_field diff --git a/mongoengine/base.py b/mongoengine/base.py index 38e528f..5a07bdf 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -586,6 +586,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): collection = ''.join('_%s' % c if c.isupper() else c for c in name).strip('_').lower() id_field = None + abstract_base_indexes = [] base_indexes = [] base_meta = {} @@ -605,7 +606,10 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): base_meta[key] = base._meta[key] id_field = id_field or base._meta.get('id_field') - base_indexes += base._meta.get('indexes', []) + if base._meta.get('abstract', False): + abstract_base_indexes += base._meta.get('indexes', []) + else: + base_indexes += base._meta.get('indexes', []) # Propagate 'allow_inheritance' if 'allow_inheritance' in base._meta: base_meta['allow_inheritance'] = base._meta['allow_inheritance'] @@ -651,8 +655,9 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): meta['queryset_class'] = manager.queryset_class new_class.objects = manager + indicies = meta['indexes'] + abstract_base_indexes user_indexes = [QuerySet._build_index_spec(new_class, spec) - for spec in meta['indexes']] + base_indexes + for spec in indicies] + base_indexes new_class._meta['indexes'] = user_indexes unique_indexes = cls._unique_with_indexes(new_class) diff --git a/tests/document.py b/tests/document.py index faedb42..021addc 100644 --- a/tests/document.py +++ b/tests/document.py @@ -664,6 +664,35 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + def test_abstract_index_inheritance(self): + + class UserBase(Document): + meta = { + 'abstract': True, + 'indexes': ['user_guid'] + } + + user_guid = StringField(required=True) + + + class Person(UserBase): + meta = { + 'indexes': ['name'], + } + + name = StringField() + + Person.drop_collection() + + p = Person(name="test", user_guid='123') + p.save() + + self.assertEquals(1, Person.objects.count()) + info = Person.objects._collection.index_information() + self.assertEqual(info.keys(), ['_types_1_user_guid_1', '_id_', '_types_1_name_1']) + Person.drop_collection() + + def test_embedded_document_index(self): """Tests settings an index on an embedded document """ From 2531ade3bbfb03b1e79ffce2df6103a4b45c0072 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Dec 2011 03:01:56 -0800 Subject: [PATCH 0474/1279] Added David to Authors refs #380 --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 1ac192c..627e4a5 100644 --- a/AUTHORS +++ b/AUTHORS @@ -88,3 +88,4 @@ that much better: * Adam Flynn * Ankhbayar * Jan Schrewe + * David Koblas From 56c42921647c550dc7e610d392a93c0d77032854 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Wed, 7 Dec 2011 13:36:35 -0200 Subject: [PATCH 0475/1279] added custom db_alias support for MongoEngine DjangoSession --- mongoengine/django/sessions.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index 7405c85..2f0e17f 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -5,16 +5,22 @@ from django.utils.encoding import force_unicode from mongoengine.document import Document from mongoengine import fields from mongoengine.queryset import OperationError - +from mongoengine.connection import DEFAULT_CONNECTION_NAME +from django.conf import settings from datetime import datetime +MONGOENGINE_SESSION_DB_ALIAS = getattr( + settings, 'MONGOENGINE_SESSION_DB_ALIAS', + DEFAULT_CONNECTION_NAME) class MongoSession(Document): session_key = fields.StringField(primary_key=True, max_length=40) session_data = fields.StringField() expire_date = fields.DateTimeField() - meta = {'collection': 'django_session', 'allow_inheritance': False} + meta = {'collection': 'django_session', + 'db_alias': MONGOENGINE_SESSION_DB_ALIAS, + 'allow_inheritance': False} class SessionStore(SessionBase): From 83e3c5c7d8545d5ced30ce679fef800aeecbd60c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 9 Dec 2011 08:26:39 -0800 Subject: [PATCH 0476/1279] Updated connection for pymongo 2.1 support closes #378 --- mongoengine/connection.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 1c0504a..1284e69 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,4 +1,4 @@ -from pymongo import Connection +from pymongo import Connection, version_tuple __all__ = ['ConnectionError', 'connect', 'register_connection', @@ -18,8 +18,8 @@ _dbs = {} def register_connection(alias, name, host='localhost', port=27017, - is_slave=False, slaves=None, username=None, - password=None): + is_slave=False, read_preference=False, slaves=None, + username=None, password=None): """Add a connection. :param alias: the name that will be used to refer to this connection @@ -27,11 +27,13 @@ def register_connection(alias, name, host='localhost', port=27017, :param name: the name of the specific database to use :param host: the host name of the :program:`mongod` instance to connect to :param port: the port that the :program:`mongod` instance is running on - :param is_slave: whether the connection can act as a slave + :param is_slave: whether the connection can act as a slave ** Depreciated pymongo 2.0.1+ + :param read_preference: The read preference for the collection ** Added pymongo 2.1 :param slaves: a list of aliases of slave connections; each of these must be a registered connection that has :attr:`is_slave` set to ``True`` :param username: username to authenticate with :param password: password to authenticate with + """ global _connection_settings _connection_settings[alias] = { @@ -42,6 +44,7 @@ def register_connection(alias, name, host='localhost', port=27017, 'slaves': slaves or [], 'username': username, 'password': password, + 'read_preference': read_preference } @@ -70,11 +73,19 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): raise ConnectionError(msg) conn_settings = _connection_settings[alias].copy() - # Get all the slave connections - slaves = [] - for slave_alias in conn_settings['slaves']: - slaves.append(get_connection(slave_alias)) - conn_settings['slaves'] = slaves + if version_tuple[0] >= 2 and version_tuple [1] > 0: + conn_settings.pop('name') + conn_settings.pop('slaves') + conn_settings.pop('is_slave') + conn_settings.pop('username') + conn_settings.pop('password') + else: + # Get all the slave connections + slaves = [] + for slave_alias in conn_settings['slaves']: + slaves.append(get_connection(slave_alias)) + conn_settings['slaves'] = slaves + conn_settings.pop('read_preference') try: _connections[alias] = Connection(**conn_settings) From febb3d7e3d5c261b53a14a7704c63d2d839cecc5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 9 Dec 2011 08:39:50 -0800 Subject: [PATCH 0477/1279] Updated connection - so handles < pymongo 2.1 Updated docs Refs #378 --- docs/changelog.rst | 1 + mongoengine/connection.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7c9055d..0447118 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added pymongo 2.1 support - Fixed Abstract documents can now declare indexes - Added db_alias support to individual documents - Fixed GridFS documents can now be pickled diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 1284e69..8dbe718 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,4 +1,4 @@ -from pymongo import Connection, version_tuple +from pymongo import Connection, version __all__ = ['ConnectionError', 'connect', 'register_connection', @@ -73,6 +73,7 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): raise ConnectionError(msg) conn_settings = _connection_settings[alias].copy() + version_tuple = [int(v) for v in version.split('.')] if version_tuple[0] >= 2 and version_tuple [1] > 0: conn_settings.pop('name') conn_settings.pop('slaves') From 9b3899476c43fc4f9b6e6eceb50a02fe21f2cf2b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 12 Dec 2011 09:14:50 +0000 Subject: [PATCH 0478/1279] Allow arbitary kwargs to be passed to pymongo Fix pymongo 2.1+ check Closes #390 closes #378 --- mongoengine/connection.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 8dbe718..462c895 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,4 +1,5 @@ -from pymongo import Connection, version +import pymongo +from pymongo import Connection __all__ = ['ConnectionError', 'connect', 'register_connection', @@ -19,7 +20,7 @@ _dbs = {} def register_connection(alias, name, host='localhost', port=27017, is_slave=False, read_preference=False, slaves=None, - username=None, password=None): + username=None, password=None, **kwargs): """Add a connection. :param alias: the name that will be used to refer to this connection @@ -33,6 +34,7 @@ def register_connection(alias, name, host='localhost', port=27017, be a registered connection that has :attr:`is_slave` set to ``True`` :param username: username to authenticate with :param password: password to authenticate with + :param kwargs: allow ad-hoc parameters to be passed into the pymongo driver """ global _connection_settings @@ -46,7 +48,7 @@ def register_connection(alias, name, host='localhost', port=27017, 'password': password, 'read_preference': read_preference } - + _connection_settings[alias].update(kwargs) def disconnect(alias=DEFAULT_CONNECTION_NAME): global _connections @@ -73,8 +75,7 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): raise ConnectionError(msg) conn_settings = _connection_settings[alias].copy() - version_tuple = [int(v) for v in version.split('.')] - if version_tuple[0] >= 2 and version_tuple [1] > 0: + if hasattr(pymongo, 'version_tuple'): # Support for 2.1+ conn_settings.pop('name') conn_settings.pop('slaves') conn_settings.pop('is_slave') From 4a269eb2c4bafc808cfc8d290ff5d34ed8228316 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 12 Dec 2011 13:39:37 -0200 Subject: [PATCH 0479/1279] added .select method --- mongoengine/queryset.py | 104 ++++++++++++++++++++++++++++------------ tests/queryset.py | 45 +++++++++++++++++ 2 files changed, 119 insertions(+), 30 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 0c39253..1b3257a 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -314,6 +314,27 @@ class QueryFieldList(object): def __nonzero__(self): return bool(self.fields) +class SelectResult(object): + """ + Used for .select method in QuerySet + """ + def __init__(self, cursor, fields): + self._cursor = cursor + self._fields = fields + + def next(self): + try: + data = self._cursor.next() + return [data.get(f) for f in self._fields] + except StopIteration, e: + self.rewind() + raise e + + def rewind(self): + self._cursor.rewind() + + def __iter__(self): + return self class QuerySet(object): """A set of results returned from a query. Wraps a MongoDB cursor, @@ -548,33 +569,38 @@ class QuerySet(object): cursor_args['fields'] = self._loaded_fields.as_dict() return cursor_args + def _build_cursor(self, **cursor_args): + obj = self._collection.find(self._query, + **cursor_args) + # Apply where clauses to cursor + if self._where_clause: + obj.where(self._where_clause) + + # apply default ordering + if self._ordering: + obj.sort(self._ordering) + elif self._document._meta['ordering']: + self._ordering = self._get_order_key_list( + *self._document._meta['ordering']) + obj.sort(self._ordering) + + if self._limit is not None: + obj.limit(self._limit) + + if self._skip is not None: + obj.skip(self._skip) + + if self._hint != -1: + obj.hint(self._hint) + + return obj + @property def _cursor(self): if self._cursor_obj is None: - - self._cursor_obj = self._collection.find(self._query, - **self._cursor_args) - # Apply where clauses to cursor - if self._where_clause: - self._cursor_obj.where(self._where_clause) - - # apply default ordering - if self._ordering: - self._cursor_obj.sort(self._ordering) - elif self._document._meta['ordering']: - self.order_by(*self._document._meta['ordering']) - - if self._limit is not None: - self._cursor_obj.limit(self._limit) - - if self._skip is not None: - self._cursor_obj.skip(self._skip) - - if self._hint != -1: - self._cursor_obj.hint(self._hint) - + self._cursor_obj = self._build_cursor(**self._cursor_args) + return self._cursor_obj - @classmethod def _lookup_field(cls, document, parts): """Lookup a field based on its attribute and return a list containing @@ -803,6 +829,16 @@ class QuerySet(object): doc.save() return doc + def select(self, *fields): + """ + Select a field and make a tuple of element + """ + cursor_args = self._cursor_args + cursor_args['fields'] = self._fields_to_dbfields(fields) + cursor = self._build_cursor(**cursor_args) + + return SelectResult(cursor, fields) + def first(self): """Retrieve the first object matching the query. """ @@ -1163,13 +1199,9 @@ class QuerySet(object): ret.append(field) return ret - def order_by(self, *keys): - """Order the :class:`~mongoengine.queryset.QuerySet` by the keys. The - order may be specified by prepending each of the keys by a + or a -. - Ascending order is assumed. - - :param keys: fields to order the query results by; keys may be - prefixed with **+** or **-** to determine the ordering direction + def _get_order_key_list(self, *keys): + """ + Build order list for query """ key_list = [] for key in keys: @@ -1186,6 +1218,18 @@ class QuerySet(object): pass key_list.append((key, direction)) + return key_list + + def order_by(self, *keys): + """Order the :class:`~mongoengine.queryset.QuerySet` by the keys. The + order may be specified by prepending each of the keys by a + or a -. + Ascending order is assumed. + + :param keys: fields to order the query results by; keys may be + prefixed with **+** or **-** to determine the ordering direction + """ + + key_list = self._get_order_key_list(*keys) self._ordering = key_list self._cursor.sort(key_list) return self diff --git a/tests/queryset.py b/tests/queryset.py index 37fa524..0f5d84c 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -2909,6 +2909,51 @@ class QueryFieldListTest(unittest.TestCase): ak = list(Bar.objects(foo__match={'shape': "square", "color": "purple"})) self.assertEqual([b1], ak) + + def test_select(self): + class TestDoc(Document): + x = IntField() + y = BooleanField() + + TestDoc.drop_collection() + + TestDoc(x=10, y=True).save() + TestDoc(x=20, y=False).save() + TestDoc(x=30, y=True).save() + + plist = list(TestDoc.objects.select('x', 'y')) + + self.assertEqual(len(plist), 3) + self.assertEqual(plist[0], [10, True]) + self.assertEqual(plist[1], [20, False]) + self.assertEqual(plist[2], [30, True]) + + class UserDoc(Document): + name = StringField() + age = IntField() + + UserDoc.drop_collection() + + UserDoc(name="Wilson Jr", age=19).save() + UserDoc(name="Wilson", age=43).save() + UserDoc(name="Eliana", age=37).save() + UserDoc(name="Tayza", age=15).save() + + ulist = list(UserDoc.objects.select('name', 'age')) + + self.assertEqual(ulist, [ + [u'Wilson Jr', 19], + [u'Wilson', 43], + [u'Eliana', 37], + [u'Tayza', 15]]) + + ulist = list(UserDoc.objects.order_by('age').select('name')) + + self.assertEqual(ulist, [ + [u'Tayza'], + [u'Wilson Jr'], + [u'Eliana'], + [u'Wilson']]) if __name__ == '__main__': unittest.main() From 11daf706df34c5710c7ff94d762ee00774566fb9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 12 Dec 2011 16:13:19 +0000 Subject: [PATCH 0480/1279] Added Sharding support Added shard_key meta, so save() and update() passes shard keys to the pymongo query. Also made shard key fields immutable. Closes #388 and #389 --- AUTHORS | 3 +++ docs/changelog.rst | 1 + docs/guide/defining-documents.rst | 21 +++++++++++++++++++++ mongoengine/base.py | 8 ++++++++ mongoengine/document.py | 24 ++++++++++++++++++------ 5 files changed, 51 insertions(+), 6 deletions(-) diff --git a/AUTHORS b/AUTHORS index 627e4a5..f1a65b9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -89,3 +89,6 @@ that much better: * Ankhbayar * Jan Schrewe * David Koblas + * Crittercism + * Alvin Liang + * andrewmlevy diff --git a/docs/changelog.rst b/docs/changelog.rst index 0447118..49139ca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added sharding support - Added pymongo 2.1 support - Fixed Abstract documents can now declare indexes - Added db_alias support to individual documents diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 730d180..d749303 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -476,8 +476,29 @@ subsequent calls to :meth:`~mongoengine.queryset.QuerySet.order_by`. :: first_post = BlogPost.objects.order_by("+published_date").first() assert first_post.title == "Blog Post #1" +Shard keys +========== + +If your collection is sharded, then you need to specify the shard key as a tuple, +using the :attr:`shard_key` attribute of :attr:`-mongoengine.Document.meta`. +This ensures that the shard key is sent with the query when calling the +:meth:`~mongoengine.document.Document.save` or +:meth:`~mongoengine.document.Document.update` method on an existing +:class:`-mongoengine.Document` instance:: + + class LogEntry(Document): + machine = StringField() + app = StringField() + timestamp = DateTimeField() + data = StringField() + + meta = { + 'shard_key': ('machine', 'timestamp',) + } + Document inheritance ==================== + To create a specialised type of a :class:`~mongoengine.Document` you have defined, you may subclass it and add any extra fields or methods you may need. As this is new class is not a direct subclass of diff --git a/mongoengine/base.py b/mongoengine/base.py index 5a07bdf..99a71dc 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -724,6 +724,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): class BaseDocument(object): _dynamic = False + _created = True def __init__(self, **values): signals.pre_init.send(self.__class__, document=self, values=values) @@ -757,6 +758,7 @@ class BaseDocument(object): if self._dynamic: for key, value in dynamic_data.items(): setattr(self, key, value) + signals.post_init.send(self.__class__, document=self) def __setattr__(self, name, value): @@ -784,6 +786,11 @@ class BaseDocument(object): if hasattr(self, '_changed_fields'): self._mark_as_changed(name) return + + if not self._created and name in self._meta.get('shard_key', tuple()): + from queryset import OperationError + raise OperationError("Shard Keys are immutable. Tried to update %s" % name) + super(BaseDocument, self).__setattr__(name, value) def __expand_dynamic_values(self, name, value): @@ -912,6 +919,7 @@ class BaseDocument(object): obj = cls(**data) obj._changed_fields = changed_fields + obj._created = False return obj def _mark_as_changed(self, key): diff --git a/mongoengine/document.py b/mongoengine/document.py index 6b0d828..5996143 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -171,6 +171,7 @@ class Document(BaseDocument): doc = self.to_mongo() created = force_insert or '_id' not in doc + try: collection = self.__class__.objects._collection if created: @@ -181,10 +182,18 @@ class Document(BaseDocument): else: object_id = doc['_id'] updates, removals = self._delta() + + # Need to add shard key to query, or you get an error + select_dict = {'_id': object_id} + shard_key = self.__class__._meta.get('shard_key', tuple()) + for k in shard_key: + actual_key = self._db_field_map.get(k, k) + select_dict[actual_key] = doc[actual_key] + if updates: - collection.update({'_id': object_id}, {"$set": updates}, upsert=True, safe=safe, **write_options) + collection.update(select_dict, {"$set": updates}, upsert=True, safe=safe, **write_options) if removals: - collection.update({'_id': object_id}, {"$unset": removals}, upsert=True, safe=safe, **write_options) + collection.update(select_dict, {"$unset": removals}, upsert=True, safe=safe, **write_options) cascade = self._meta.get('cascade', True) if cascade is None else cascade if cascade: @@ -238,7 +247,12 @@ class Document(BaseDocument): if not self.pk: raise OperationError('attempt to update a document not yet saved') - return self.__class__.objects(pk=self.pk).update_one(**kwargs) + # Need to add shard key to query, or you get an error + select_dict = {'pk': self.pk} + shard_key = self.__class__._meta.get('shard_key', tuple()) + for k in shard_key: + select_dict[k] = getattr(self, k) + return self.__class__.objects(**select_dict).update_one(**kwargs) def delete(self, safe=False): """Delete the :class:`~mongoengine.Document` from the database. This @@ -248,10 +262,8 @@ class Document(BaseDocument): """ signals.pre_delete.send(self.__class__, document=self) - id_field = self._meta['id_field'] - object_id = self._fields[id_field].to_mongo(self[id_field]) try: - self.__class__.objects(**{id_field: object_id}).delete(safe=safe) + self.__class__.objects(pk=self.pk).delete(safe=safe) except pymongo.errors.OperationFailure, err: message = u'Could not delete document (%s)' % err.message raise OperationError(message) From 2b3b3bf6520e61b453fe80de872b3afeab843786 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 12 Dec 2011 16:26:10 +0000 Subject: [PATCH 0481/1279] Prelim PyPy support Refs: #392 --- mongoengine/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index 99a71dc..05271b7 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -617,6 +617,8 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): base_meta['queryset_class'] = base._meta['queryset_class'] try: base_meta['objects'] = base.__getattribute__(base, 'objects') + except TypeError: + pass except AttributeError: pass From ed5fba6b0f8794c5d0c2b66a4ede2fdd6346ec75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 13 Dec 2011 07:46:49 -0200 Subject: [PATCH 0482/1279] support for embedded fields --- mongoengine/queryset.py | 14 ++++++++++++-- tests/queryset.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 1b3257a..1a2ff1c 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -320,12 +320,22 @@ class SelectResult(object): """ def __init__(self, cursor, fields): self._cursor = cursor - self._fields = fields + self._fields = [f.split('.') for f in fields] + + def _get_value(self, keys, data): + for key in keys: + if data: + data = data.get(key) + else: + break + + return data def next(self): try: data = self._cursor.next() - return [data.get(f) for f in self._fields] + return [self._get_value(f, data) + for f in self._fields] except StopIteration, e: self.rewind() raise e diff --git a/tests/queryset.py b/tests/queryset.py index 0f5d84c..991fb00 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -2955,5 +2955,46 @@ class QueryFieldListTest(unittest.TestCase): [u'Eliana'], [u'Wilson']]) + def test_select_embedded(self): + class Profile(EmbeddedDocument): + name = StringField() + age = IntField() + + class Locale(EmbeddedDocument): + city = StringField() + country = StringField() + + class Person(Document): + profile = EmbeddedDocumentField(Profile) + locale = EmbeddedDocumentField(Locale) + + Person.drop_collection() + + Person(profile=Profile(name="Wilson Jr", age=19), + locale=Locale(city="Corumba-GO", country="Brazil")).save() + + Person(profile=Profile(name="Gabriel Falcao", age=23), + locale=Locale(city="New York", country="USA")).save() + + Person(profile=Profile(name="Lincoln de souza", age=28), + locale=Locale(city="Belo Horizonte", country="Brazil")).save() + + Person(profile=Profile(name="Walter cruz", age=30), + locale=Locale(city="Brasilia", country="Brazil")).save() + + self.assertEqual( + list(Person.objects.order_by('profile.age').select('profile.name')), + [[u'Wilson Jr'], [u'Gabriel Falcao'], + [u'Lincoln de souza'], [u'Walter cruz']]) + + ulist = list(Person.objects.order_by('locale.city') + .select('profile.name', 'profile.age', 'locale.city')) + self.assertEqual(ulist, + [[u'Lincoln de souza', 28, u'Belo Horizonte'], + [u'Walter cruz', 30, u'Brasilia'], + [u'Wilson Jr', 19, u'Corumba-GO'], + [u'Gabriel Falcao', 23, u'New York']]) + + if __name__ == '__main__': unittest.main() From ca7b2371fbbba4c705b6aaa12025b54eab329d16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 13 Dec 2011 11:54:19 -0200 Subject: [PATCH 0483/1279] added support for dereferences --- mongoengine/queryset.py | 45 +++++++++++++++++++++++++++++++++-------- tests/queryset.py | 31 ++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 1a2ff1c..80bff6f 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -318,24 +318,51 @@ class SelectResult(object): """ Used for .select method in QuerySet """ - def __init__(self, cursor, fields): + def __init__(self, document_type, cursor, fields, dbfields): + from base import BaseField + from fields import ReferenceField + # Caches for optimization + self.ReferenceField = ReferenceField + self._cursor = cursor - self._fields = [f.split('.') for f in fields] - def _get_value(self, keys, data): + f = [] + for field, dbfield in itertools.izip(fields, dbfields): + + p = document_type + for path in field.split('.'): + if p and isinstance(p, BaseField): + p = p.lookup_member(path) + elif p: + p = getattr(p, path) + else: + break + + f.append((dbfield.split('.'), p)) + + self._fields = f + + def _get_value(self, keys, field_type, data): for key in keys: if data: data = data.get(key) else: break - return data + if isinstance(field_type, self.ReferenceField): + doc_type = field_type.document_type + data = doc_type._get_db().dereference(data) + + if data: + return doc_type._from_son(data) + + return field_type.to_python(data) def next(self): try: data = self._cursor.next() - return [self._get_value(f, data) - for f in self._fields] + return [self._get_value(k, t, data) + for k, t in self._fields] except StopIteration, e: self.rewind() raise e @@ -843,11 +870,13 @@ class QuerySet(object): """ Select a field and make a tuple of element """ + dbfields = self._fields_to_dbfields(fields) + cursor_args = self._cursor_args - cursor_args['fields'] = self._fields_to_dbfields(fields) + cursor_args['fields'] = dbfields cursor = self._build_cursor(**cursor_args) - return SelectResult(cursor, fields) + return SelectResult(self._document, cursor, fields, dbfields) def first(self): """Retrieve the first object matching the query. diff --git a/tests/queryset.py b/tests/queryset.py index 991fb00..02e931e 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -2994,7 +2994,38 @@ class QueryFieldListTest(unittest.TestCase): [u'Walter cruz', 30, u'Brasilia'], [u'Wilson Jr', 19, u'Corumba-GO'], [u'Gabriel Falcao', 23, u'New York']]) + + def test_select_decimal(self): + from decimal import Decimal + class Person(Document): + name = StringField() + rating = DecimalField() + Person.drop_collection() + Person(name="Wilson Jr", rating=Decimal('1.0')).save() + + ulist = list(Person.objects.select('name', 'rating')) + self.assertEqual(ulist, [[u'Wilson Jr', Decimal('1.0')]]) + + + def test_select_reference_field(self): + class State(Document): + name = StringField() + + class Person(Document): + name = StringField() + state = ReferenceField(State) + + State.drop_collection() + Person.drop_collection() + + s1 = State(name="Goias") + s1.save() + + Person(name="Wilson JR", state=s1).save() + + plist = list(Person.objects.select('name', 'state')) + self.assertEqual(plist, [[u'Wilson JR', s1]]) if __name__ == '__main__': unittest.main() From 7c1afd00313c5e0eaad64079efa8be127a0a4f1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Tue, 13 Dec 2011 11:56:35 -0200 Subject: [PATCH 0484/1279] tests for db_field --- tests/queryset.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 02e931e..4b61956 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -3027,5 +3027,24 @@ class QueryFieldListTest(unittest.TestCase): plist = list(Person.objects.select('name', 'state')) self.assertEqual(plist, [[u'Wilson JR', s1]]) + + def test_select_db_field(self): + class TestDoc(Document): + x = IntField(db_field="y") + y = BooleanField(db_field="x") + + TestDoc.drop_collection() + + TestDoc(x=10, y=True).save() + TestDoc(x=20, y=False).save() + TestDoc(x=30, y=True).save() + + plist = list(TestDoc.objects.select('x', 'y')) + + self.assertEqual(len(plist), 3) + self.assertEqual(plist[0], [10, True]) + self.assertEqual(plist[1], [20, False]) + self.assertEqual(plist[2], [30, True]) + if __name__ == '__main__': unittest.main() From 7614b9219729833ec1871c266825572ca0b9f88a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 15 Dec 2011 09:16:35 +0000 Subject: [PATCH 0485/1279] Fixes super in BaseDict Closes #395 --- AUTHORS | 1 + mongoengine/base.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index f1a65b9..d73afc8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -92,3 +92,4 @@ that much better: * Crittercism * Alvin Liang * andrewmlevy + * Chris Faulkner diff --git a/mongoengine/base.py b/mongoengine/base.py index 05271b7..b421401 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1255,11 +1255,11 @@ class BaseDict(dict): def pop(self, *args, **kwargs): self._updated() - super(BaseDict, self).clear(*args, **kwargs) + super(BaseDict, self).pop(*args, **kwargs) def popitem(self, *args, **kwargs): self._updated() - super(BaseDict, self).clear(*args, **kwargs) + super(BaseDict, self).popitem(*args, **kwargs) def _updated(self): try: From 6d9bfff19cd3ddfe360cf34501a87471f3370105 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 16 Dec 2011 12:41:42 +0000 Subject: [PATCH 0486/1279] Started work on performance Added an initial benchmark.py Much more performant than 0.5.2 but still work todo. --- benchmark.py | 182 +++++++++++++++++++++++++++++++++++++ mongoengine/base.py | 148 ++++++++++++++++-------------- mongoengine/dereference.py | 13 +-- mongoengine/document.py | 8 +- mongoengine/fields.py | 4 +- 5 files changed, 274 insertions(+), 81 deletions(-) create mode 100644 benchmark.py diff --git a/benchmark.py b/benchmark.py new file mode 100644 index 0000000..247baeb --- /dev/null +++ b/benchmark.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python + +import timeit + + +def cprofile_main(): + from pymongo import Connection + connection = Connection() + connection.drop_database('timeit_test') + connection.disconnect() + + from mongoengine import Document, DictField, connect + connect("timeit_test") + + class Noddy(Document): + fields = DictField() + + for i in xrange(1): + noddy = Noddy() + for j in range(20): + noddy.fields["key" + str(j)] = "value " + str(j) + noddy.save() + + +def main(): + """ + 0.4 Performance Figures ... + + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - Pymongo + 1.1141769886 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine + 2.37724113464 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, safe=False, validate=False + 1.92479610443 + + 0.5.X + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - Pymongo + 1.10552310944 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine + 16.5169169903 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, safe=False, validate=False + 14.9446101189 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False + 14.912801981 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, force=True + 14.9617750645 + + Performance + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - Pymongo + 1.10072994232 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine + 5.27341103554 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, safe=False, validate=False + 4.49365401268 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False + 4.43459296227 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, force=True + 4.40114378929 + """ + + setup = """ +from pymongo import Connection +connection = Connection() +connection.drop_database('timeit_test') +""" + + stmt = """ +from pymongo import Connection +connection = Connection() + +db = connection.timeit_test +noddy = db.noddy + +for i in xrange(10000): + example = {'fields': {}} + for j in range(20): + example['fields']["key"+str(j)] = "value "+str(j) + + noddy.insert(example) + +myNoddys = noddy.find() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - Pymongo""" + t = timeit.Timer(stmt=stmt, setup=setup) + print t.timeit(1) + + setup = """ +from pymongo import Connection +connection = Connection() +connection.drop_database('timeit_test') +connection.disconnect() + +from mongoengine import Document, DictField, connect +connect("timeit_test") + +class Noddy(Document): + fields = DictField() +""" + + stmt = """ +for i in xrange(10000): + noddy = Noddy() + for j in range(20): + noddy.fields["key"+str(j)] = "value "+str(j) + noddy.save() + +myNoddys = Noddy.objects() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - MongoEngine""" + t = timeit.Timer(stmt=stmt, setup=setup) + print t.timeit(1) + + stmt = """ +for i in xrange(10000): + noddy = Noddy() + for j in range(20): + noddy.fields["key"+str(j)] = "value "+str(j) + noddy.save(safe=False, validate=False) + +myNoddys = Noddy.objects() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - MongoEngine, safe=False, validate=False""" + t = timeit.Timer(stmt=stmt, setup=setup) + print t.timeit(1) + + + stmt = """ +for i in xrange(10000): + noddy = Noddy() + for j in range(20): + noddy.fields["key"+str(j)] = "value "+str(j) + noddy.save(safe=False, validate=False, cascade=False) + +myNoddys = Noddy.objects() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False""" + t = timeit.Timer(stmt=stmt, setup=setup) + print t.timeit(1) + + stmt = """ +for i in xrange(10000): + noddy = Noddy() + for j in range(20): + noddy.fields["key"+str(j)] = "value "+str(j) + noddy.save(force_insert=True, safe=False, validate=False, cascade=False) + +myNoddys = Noddy.objects() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - MongoEngine, force=True""" + t = timeit.Timer(stmt=stmt, setup=setup) + print t.timeit(1) + +if __name__ == "__main__": + main() diff --git a/mongoengine/base.py b/mongoengine/base.py index b421401..a95339e 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -153,15 +153,6 @@ class BaseField(object): if callable(value): value = value() - # Convert lists / values so we can watch for any changes on them - if isinstance(value, (list, tuple)) and not isinstance(value, BaseList): - observer = DataObserver(instance, self.name) - value = BaseList(value, observer) - instance._data[self.name] = value - elif isinstance(value, dict) and not isinstance(value, BaseDict): - observer = DataObserver(instance, self.name) - value = BaseDict(value, observer) - instance._data[self.name] = value return value def __set__(self, instance, value): @@ -231,6 +222,7 @@ class ComplexBaseField(BaseField): """ field = None + _dereference = False def __get__(self, instance, owner): """Descriptor to automatically dereference references. @@ -239,11 +231,39 @@ class ComplexBaseField(BaseField): # Document class being used rather than a document object return self - from dereference import dereference - instance._data[self.name] = dereference( - instance._data.get(self.name), max_depth=1, instance=instance, name=self.name - ) - return super(ComplexBaseField, self).__get__(instance, owner) + if not self._dereference and instance._initialised: + from dereference import dereference + self._dereference = dereference # Cached + instance._data[self.name] = self._dereference( + instance._data.get(self.name), max_depth=1, instance=instance, + name=self.name + ) + + value = super(ComplexBaseField, self).__get__(instance, owner) + + # Convert lists / values so we can watch for any changes on them + if isinstance(value, (list, tuple)) and not isinstance(value, BaseList): + value = BaseList(value, instance, self.name) + instance._data[self.name] = value + elif isinstance(value, dict) and not isinstance(value, BaseDict): + value = BaseDict(value, instance, self.name) + instance._data[self.name] = value + + if self._dereference and instance._initialised and \ + isinstance(value, (BaseList, BaseDict)) and not value._dereferenced: + value = self._dereference( + value, max_depth=1, instance=instance, name=self.name + ) + value._dereferenced = True + instance._data[self.name] = value + + return value + + def __set__(self, instance, value): + """Descriptor for assigning a value to a field in a document. + """ + instance._data[self.name] = value + instance._mark_as_changed(self.name) def to_python(self, value): """Convert a MongoDB-compatible type to a Python type. @@ -727,12 +747,13 @@ class BaseDocument(object): _dynamic = False _created = True + _dynamic_lock = True + _initialised = False def __init__(self, **values): signals.pre_init.send(self.__class__, document=self, values=values) self._data = {} - self._initialised = False # Assign default values to instance for attr_name, field in self._fields.items(): @@ -754,18 +775,19 @@ class BaseDocument(object): # Set any get_fieldname_display methods self.__set_field_display() - # Flag initialised - self._initialised = True if self._dynamic: + self._dynamic_lock = False for key, value in dynamic_data.items(): setattr(self, key, value) + # Flag initialised + self._initialised = True signals.post_init.send(self.__class__, document=self) def __setattr__(self, name, value): # Handle dynamic data only if an initialised dynamic document - if self._dynamic and getattr(self, '_initialised', False): + if self._dynamic and not self._dynamic_lock: field = None if not hasattr(self, name) and not name.startswith('_'): @@ -825,11 +847,9 @@ class BaseDocument(object): # Convert lists / values so we can watch for any changes on them if isinstance(value, (list, tuple)) and not isinstance(value, BaseList): - observer = DataObserver(self, name) - value = BaseList(value, observer) + value = BaseList(value, self, name) elif isinstance(value, dict) and not isinstance(value, BaseDict): - observer = DataObserver(self, name) - value = BaseDict(value, observer) + value = BaseDict(value, self, name) return value @@ -1147,33 +1167,25 @@ class BaseDocument(object): return hash(self.pk) -class DataObserver(object): - - __slots__ = ["instance", "name"] - - def __init__(self, instance, name): - self.instance = instance - self.name = name - - def updated(self): - if hasattr(self.instance, '_mark_as_changed'): - self.instance._mark_as_changed(self.name) - - class BaseList(list): """A special list so we can watch any changes """ - def __init__(self, list_items, observer): - self.observer = observer + _dereferenced = False + _instance = None + _name = None + + def __init__(self, list_items, instance, name): + self._instance = instance + self._name = name super(BaseList, self).__init__(list_items) def __setitem__(self, *args, **kwargs): - self._updated() + self._mark_as_changed() super(BaseList, self).__setitem__(*args, **kwargs) def __delitem__(self, *args, **kwargs): - self._updated() + self._mark_as_changed() super(BaseList, self).__delitem__(*args, **kwargs) def __getstate__(self): @@ -1182,90 +1194,94 @@ class BaseList(list): def __setstate__(self, state): self = state + return self def append(self, *args, **kwargs): - self._updated() + self._mark_as_changed() return super(BaseList, self).append(*args, **kwargs) def extend(self, *args, **kwargs): - self._updated() + self._mark_as_changed() return super(BaseList, self).extend(*args, **kwargs) def insert(self, *args, **kwargs): - self._updated() + self._mark_as_changed() return super(BaseList, self).insert(*args, **kwargs) def pop(self, *args, **kwargs): - self._updated() + self._mark_as_changed() return super(BaseList, self).pop(*args, **kwargs) def remove(self, *args, **kwargs): - self._updated() + self._mark_as_changed() return super(BaseList, self).remove(*args, **kwargs) def reverse(self, *args, **kwargs): - self._updated() + self._mark_as_changed() return super(BaseList, self).reverse(*args, **kwargs) def sort(self, *args, **kwargs): - self._updated() + self._mark_as_changed() return super(BaseList, self).sort(*args, **kwargs) - def _updated(self): - try: - self.observer.updated() - except AttributeError: - pass + def _mark_as_changed(self): + if hasattr(self._instance, '_mark_as_changed'): + self._instance._mark_as_changed(self._name) class BaseDict(dict): """A special dict so we can watch any changes """ - def __init__(self, dict_items, observer): - self.observer = observer + _dereferenced = False + _instance = None + _name = None + + def __init__(self, dict_items, instance, name): + self._instance = instance + self._name = name super(BaseDict, self).__init__(dict_items) def __setitem__(self, *args, **kwargs): - self._updated() + self._mark_as_changed() super(BaseDict, self).__setitem__(*args, **kwargs) def __delete__(self, *args, **kwargs): - self._updated() + self._mark_as_changed() super(BaseDict, self).__delete__(*args, **kwargs) def __delitem__(self, *args, **kwargs): - self._updated() + self._mark_as_changed() super(BaseDict, self).__delitem__(*args, **kwargs) def __delattr__(self, *args, **kwargs): - self._updated() + self._mark_as_changed() super(BaseDict, self).__delattr__(*args, **kwargs) def __getstate__(self): - self.observer = None + self.instance = None + self._dereferenced = False return self def __setstate__(self, state): self = state + return self def clear(self, *args, **kwargs): - self._updated() + self._mark_as_changed() super(BaseDict, self).clear(*args, **kwargs) def pop(self, *args, **kwargs): - self._updated() + self._mark_as_changed() super(BaseDict, self).pop(*args, **kwargs) def popitem(self, *args, **kwargs): - self._updated() + self._mark_as_changed() super(BaseDict, self).popitem(*args, **kwargs) - def _updated(self): - try: - self.observer.updated() - except AttributeError: - pass + def _mark_as_changed(self): + if hasattr(self._instance, '_mark_as_changed'): + self._instance._mark_as_changed(self._name) if sys.version_info < (2, 5): # Prior to Python 2.5, Exception was an old-style class diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 3dac92b..40ceb49 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -1,7 +1,6 @@ import pymongo -from base import (BaseDict, BaseList, DataObserver, - TopLevelDocumentMetaclass, get_document) +from base import (BaseDict, BaseList, TopLevelDocumentMetaclass, get_document) from fields import ReferenceField from connection import get_db from queryset import QuerySet @@ -134,11 +133,10 @@ class DeReference(object): return items if instance: - observer = DataObserver(instance, name) if isinstance(items, dict): - return BaseDict(items, observer) + return BaseDict(items, instance, name) else: - return BaseList(items, observer) + return BaseList(items, instance, name) if isinstance(items, (dict, pymongo.son.SON)): if '_ref' in items: @@ -183,10 +181,9 @@ class DeReference(object): data[k] = self.object_map.get(v.id, v) if instance and name: - observer = DataObserver(instance, name) if is_list: - return BaseList(data, observer) - return BaseDict(data, observer) + return BaseList(data, instance, name) + return BaseDict(data, instance, name) depth += 1 return data diff --git a/mongoengine/document.py b/mongoengine/document.py index 5996143..686945c 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,6 +1,6 @@ from mongoengine import signals from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, - BaseDict, BaseList, DataObserver) + BaseDict, BaseList) from queryset import OperationError from connection import get_db, DEFAULT_CONNECTION_NAME @@ -304,12 +304,10 @@ class Document(BaseDocument): """ if isinstance(value, BaseDict): value = [(k, self._reload(k, v)) for k, v in value.items()] - observer = DataObserver(self, key) - value = BaseDict(value, observer) + value = BaseDict(value, self, key) elif isinstance(value, BaseList): value = [self._reload(key, v) for v in value] - observer = DataObserver(self, key) - value = BaseList(value, observer) + value = BaseList(value, self, key) elif isinstance(value, (EmbeddedDocument, DynamicEmbeddedDocument)): value._changed_fields = [] return value diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 2f5911f..2e4411a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -11,7 +11,7 @@ import uuid from base import (BaseField, ComplexBaseField, ObjectIdField, ValidationError, get_document) -from queryset import DO_NOTHING +from queryset import DO_NOTHING, QuerySet from document import Document, EmbeddedDocument from connection import get_db, DEFAULT_CONNECTION_NAME from operator import itemgetter @@ -491,7 +491,7 @@ class ListField(ComplexBaseField): def validate(self, value): """Make sure that a list of valid fields is being used. """ - if (not isinstance(value, (list, tuple)) or + if (not isinstance(value, (list, tuple, QuerySet)) or isinstance(value, basestring)): self.error('Only lists and tuples may be used in a list field') super(ListField, self).validate(value) From 62219d96480dd54004c5563ec926ad07d127bd9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 16 Dec 2011 11:07:38 -0200 Subject: [PATCH 0487/1279] changed name --- mongoengine/queryset.py | 19 ++++++++++++------ tests/queryset.py | 44 +++++++++++++++++++++++++++++------------ 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 80bff6f..23e581b 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -314,15 +314,17 @@ class QueryFieldList(object): def __nonzero__(self): return bool(self.fields) -class SelectResult(object): +class ListResult(object): """ - Used for .select method in QuerySet + Used for .values_list method in QuerySet """ def __init__(self, document_type, cursor, fields, dbfields): from base import BaseField - from fields import ReferenceField + from fields import ReferenceField, GenericReferenceField # Caches for optimization + self.ReferenceField = ReferenceField + self.GenericReferenceField = GenericReferenceField self._cursor = cursor @@ -356,6 +358,10 @@ class SelectResult(object): if data: return doc_type._from_son(data) + elif isinstance(field_type, self.GenericReferenceField): + if data and isinstance(data, (dict, pymongo.dbref.DBRef)): + return field_type.dereference(data) + return field_type.to_python(data) def next(self): @@ -866,9 +872,10 @@ class QuerySet(object): doc.save() return doc - def select(self, *fields): + def values_list(self, *fields): """ - Select a field and make a tuple of element + make a list of elements + .. versionadded:: 0.6 """ dbfields = self._fields_to_dbfields(fields) @@ -876,7 +883,7 @@ class QuerySet(object): cursor_args['fields'] = dbfields cursor = self._build_cursor(**cursor_args) - return SelectResult(self._document, cursor, fields, dbfields) + return ListResult(self._document, cursor, fields, dbfields) def first(self): """Retrieve the first object matching the query. diff --git a/tests/queryset.py b/tests/queryset.py index 4b61956..b749340 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -2910,7 +2910,7 @@ class QueryFieldListTest(unittest.TestCase): ak = list(Bar.objects(foo__match={'shape': "square", "color": "purple"})) self.assertEqual([b1], ak) - def test_select(self): + def test_values_list(self): class TestDoc(Document): x = IntField() y = BooleanField() @@ -2921,7 +2921,7 @@ class QueryFieldListTest(unittest.TestCase): TestDoc(x=20, y=False).save() TestDoc(x=30, y=True).save() - plist = list(TestDoc.objects.select('x', 'y')) + plist = list(TestDoc.objects.values_list('x', 'y')) self.assertEqual(len(plist), 3) self.assertEqual(plist[0], [10, True]) @@ -2939,7 +2939,7 @@ class QueryFieldListTest(unittest.TestCase): UserDoc(name="Eliana", age=37).save() UserDoc(name="Tayza", age=15).save() - ulist = list(UserDoc.objects.select('name', 'age')) + ulist = list(UserDoc.objects.values_list('name', 'age')) self.assertEqual(ulist, [ [u'Wilson Jr', 19], @@ -2947,7 +2947,7 @@ class QueryFieldListTest(unittest.TestCase): [u'Eliana', 37], [u'Tayza', 15]]) - ulist = list(UserDoc.objects.order_by('age').select('name')) + ulist = list(UserDoc.objects.order_by('age').values_list('name')) self.assertEqual(ulist, [ [u'Tayza'], @@ -2955,7 +2955,7 @@ class QueryFieldListTest(unittest.TestCase): [u'Eliana'], [u'Wilson']]) - def test_select_embedded(self): + def test_values_list_embedded(self): class Profile(EmbeddedDocument): name = StringField() age = IntField() @@ -2983,19 +2983,19 @@ class QueryFieldListTest(unittest.TestCase): locale=Locale(city="Brasilia", country="Brazil")).save() self.assertEqual( - list(Person.objects.order_by('profile.age').select('profile.name')), + list(Person.objects.order_by('profile.age').values_list('profile.name')), [[u'Wilson Jr'], [u'Gabriel Falcao'], [u'Lincoln de souza'], [u'Walter cruz']]) ulist = list(Person.objects.order_by('locale.city') - .select('profile.name', 'profile.age', 'locale.city')) + .values_list('profile.name', 'profile.age', 'locale.city')) self.assertEqual(ulist, [[u'Lincoln de souza', 28, u'Belo Horizonte'], [u'Walter cruz', 30, u'Brasilia'], [u'Wilson Jr', 19, u'Corumba-GO'], [u'Gabriel Falcao', 23, u'New York']]) - def test_select_decimal(self): + def test_values_list_decimal(self): from decimal import Decimal class Person(Document): name = StringField() @@ -3004,11 +3004,11 @@ class QueryFieldListTest(unittest.TestCase): Person.drop_collection() Person(name="Wilson Jr", rating=Decimal('1.0')).save() - ulist = list(Person.objects.select('name', 'rating')) + ulist = list(Person.objects.values_list('name', 'rating')) self.assertEqual(ulist, [[u'Wilson Jr', Decimal('1.0')]]) - def test_select_reference_field(self): + def test_values_list_reference_field(self): class State(Document): name = StringField() @@ -3024,11 +3024,29 @@ class QueryFieldListTest(unittest.TestCase): Person(name="Wilson JR", state=s1).save() - plist = list(Person.objects.select('name', 'state')) + plist = list(Person.objects.values_list('name', 'state')) self.assertEqual(plist, [[u'Wilson JR', s1]]) + def test_values_list_generic_reference_field(self): + class State(Document): + name = StringField() - def test_select_db_field(self): + class Person(Document): + name = StringField() + state = GenericReferenceField() + + State.drop_collection() + Person.drop_collection() + + s1 = State(name="Goias") + s1.save() + + Person(name="Wilson JR", state=s1).save() + + plist = list(Person.objects.values_list('name', 'state')) + self.assertEqual(plist, [[u'Wilson JR', s1]]) + + def test_values_list_db_field(self): class TestDoc(Document): x = IntField(db_field="y") y = BooleanField(db_field="x") @@ -3039,7 +3057,7 @@ class QueryFieldListTest(unittest.TestCase): TestDoc(x=20, y=False).save() TestDoc(x=30, y=True).save() - plist = list(TestDoc.objects.select('x', 'y')) + plist = list(TestDoc.objects.values_list('x', 'y')) self.assertEqual(len(plist), 3) self.assertEqual(plist[0], [10, True]) From 5ee4b4a5ac4193dc8add4168c0262f0786844a4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 16 Dec 2011 11:49:20 -0200 Subject: [PATCH 0488/1279] added count/len for ListResult --- mongoengine/queryset.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 23e581b..2f90227 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -376,6 +376,15 @@ class ListResult(object): def rewind(self): self._cursor.rewind() + def count(self): + """ + Count the selected elements in the query. + """ + return self._cursor.count(with_limit_and_skip=True) + + def __len__(self): + return self.count() + def __iter__(self): return self From 0d867a108d278cbdca65ee94e61c1b452e6f345d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 19 Dec 2011 11:31:42 -0200 Subject: [PATCH 0489/1279] mixin inheritance --- mongoengine/base.py | 20 +++++++++++++++++--- tests/document.py | 24 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index a95339e..8028ca9 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -468,8 +468,23 @@ class ObjectIdField(BaseField): class DocumentMetaclass(type): """Metaclass for all documents. """ - + + def __new__(cls, name, bases, attrs): + def _get_mixin_fields(base): + attrs = {} + attrs.update(dict([(k, v) for k, v in base.__dict__.items() + if issubclass(v.__class__, BaseField)])) + + for p_base in base.__bases__: + #optimize :-) + if p_base in (object, BaseDocument): + continue + + attrs.update(_get_mixin_fields(p_base)) + + return attrs + metaclass = attrs.get('__metaclass__') super_new = super(DocumentMetaclass, cls).__new__ if metaclass and issubclass(metaclass, DocumentMetaclass): @@ -488,8 +503,7 @@ class DocumentMetaclass(type): superclasses[base._class_name] = base superclasses.update(base._superclasses) else: # Add any mixin fields - attrs.update(dict([(k, v) for k, v in base.__dict__.items() - if issubclass(v.__class__, BaseField)])) + attrs.update(_get_mixin_fields(base)) if hasattr(base, '_meta') and not base._meta.get('abstract'): # Ensure that the Document class may be subclassed - diff --git a/tests/document.py b/tests/document.py index 021addc..dabc27e 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2175,6 +2175,30 @@ class DocumentTest(unittest.TestCase): Person.drop_collection() + def test_mixin_inheritance(self): + class BaseMixIn(object): + count = IntField() + data = StringField() + + class DoubleMixIn(BaseMixIn): + comment = StringField() + + class TestDoc(Document, DoubleMixIn): + age = IntField() + + TestDoc.drop_collection() + t = TestDoc(count=12, data="test", + comment="great!", age=19) + + t.save() + + t = TestDoc.objects.first() + + self.assertEquals(t.age, 19) + self.assertEquals(t.comment, "great!") + self.assertEquals(t.data, "test") + self.assertEquals(t.count, 12) + def test_save_reference(self): """Ensure that a document reference field may be saved in the database. """ From 0018674b621bbdae4c09ce459fdaf8893be4a4c1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 3 Jan 2012 20:37:33 +0000 Subject: [PATCH 0490/1279] Update docs/changelog.rst --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 49139ca..21556ff 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Improved Inheritance / Mixin - Added sharding support - Added pymongo 2.1 support - Fixed Abstract documents can now declare indexes From de8da7804229604f86d4ba0efaaf04631fb84676 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 3 Jan 2012 20:42:24 +0000 Subject: [PATCH 0491/1279] Update docs/changelog.rst --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 21556ff..ab4d949 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added values_list for returning a list of data - Improved Inheritance / Mixin - Added sharding support - Added pymongo 2.1 support From d8855a4a0f01d22559ca289d19c7a25ab1fc69c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Thu, 5 Jan 2012 13:35:32 -0200 Subject: [PATCH 0492/1279] fixes for None values in QuerySet.values_list --- mongoengine/queryset.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 2f90227..84b3d90 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -361,7 +361,10 @@ class ListResult(object): elif isinstance(field_type, self.GenericReferenceField): if data and isinstance(data, (dict, pymongo.dbref.DBRef)): return field_type.dereference(data) - + + if data is None: + return + return field_type.to_python(data) def next(self): From 7a4115517879c8156372102d608aedbee6209057 Mon Sep 17 00:00:00 2001 From: Ashwin Purohit Date: Sat, 14 Jan 2012 23:21:43 -0800 Subject: [PATCH 0493/1279] typo in signals guide --- docs/guide/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/signals.rst b/docs/guide/signals.rst index 58b3d6e..2bcb8b7 100644 --- a/docs/guide/signals.rst +++ b/docs/guide/signals.rst @@ -9,7 +9,7 @@ Signal support is provided by the excellent `blinker`_ library and will gracefully fall back if it is not available. -The following document signals exist in MongoEngine and are pretty self explaintary: +The following document signals exist in MongoEngine and are pretty self-explanatory: * `mongoengine.signals.pre_init` * `mongoengine.signals.post_init` From 1afe7240f4982a0d714cf69a2ee34ca2998bf0e3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 16 Jan 2012 09:03:12 +0000 Subject: [PATCH 0494/1279] Fixed pagination limit / skip bug fixes #398 --- docs/changelog.rst | 1 + mongoengine/queryset.py | 24 ++++++++++++------------ tests/django_tests.py | 21 +++++++++++++++++++++ 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ab4d949..0215335 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Fixed limit skip bug - Added values_list for returning a list of data - Improved Inheritance / Mixin - Added sharding support diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 84b3d90..34c920e 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -327,10 +327,10 @@ class ListResult(object): self.GenericReferenceField = GenericReferenceField self._cursor = cursor - + f = [] for field, dbfield in itertools.izip(fields, dbfields): - + p = document_type for path in field.split('.'): if p and isinstance(p, BaseField): @@ -343,7 +343,7 @@ class ListResult(object): f.append((dbfield.split('.'), p)) self._fields = f - + def _get_value(self, keys, field_type, data): for key in keys: if data: @@ -354,17 +354,17 @@ class ListResult(object): if isinstance(field_type, self.ReferenceField): doc_type = field_type.document_type data = doc_type._get_db().dereference(data) - + if data: return doc_type._from_son(data) elif isinstance(field_type, self.GenericReferenceField): if data and isinstance(data, (dict, pymongo.dbref.DBRef)): return field_type.dereference(data) - + if data is None: return - + return field_type.to_python(data) def next(self): @@ -638,13 +638,13 @@ class QuerySet(object): self._ordering = self._get_order_key_list( *self._document._meta['ordering']) obj.sort(self._ordering) - + if self._limit is not None: - obj.limit(self._limit) - + obj.limit(self._limit - (self._skip or 0)) + if self._skip is not None: obj.skip(self._skip) - + if self._hint != -1: obj.hint(self._hint) @@ -654,7 +654,7 @@ class QuerySet(object): def _cursor(self): if self._cursor_obj is None: self._cursor_obj = self._build_cursor(**self._cursor_args) - + return self._cursor_obj @classmethod def _lookup_field(cls, document, parts): @@ -1286,7 +1286,7 @@ class QuerySet(object): :param keys: fields to order the query results by; keys may be prefixed with **+** or **-** to determine the ordering direction """ - + key_list = self._get_order_key_list(*keys) self._ordering = key_list self._cursor.sort(key_list) diff --git a/tests/django_tests.py b/tests/django_tests.py index 9c7e328..3341eb1 100644 --- a/tests/django_tests.py +++ b/tests/django_tests.py @@ -8,6 +8,8 @@ from mongoengine.django.shortcuts import get_document_or_404 from django.http import Http404 from django.template import Context, Template from django.conf import settings +from django.core.paginator import Paginator + settings.configure() class QuerySetTest(unittest.TestCase): @@ -67,3 +69,22 @@ class QuerySetTest(unittest.TestCase): self.assertRaises(Http404, get_document_or_404, self.Person, pk='1234') self.assertEqual(p, get_document_or_404(self.Person, pk=p.pk)) + def test_pagination(self): + """Ensure that Pagination works as expected + """ + class Page(Document): + name = StringField() + + Page.drop_collection() + + for i in xrange(1, 11): + Page(name=str(i)).save() + + paginator = Paginator(Page.objects.all(), 2) + + t = Template("{% for i in page.object_list %}{{ i.name }}:{% endfor %}") + for p in paginator.page_range: + d = {"page": paginator.page(p)} + end = p * 2 + start = end - 1 + self.assertEqual(t.render(Context(d)), u'%d:%d:' % (start, end)) From 02b1aa7355934e1a54ae7b156c7073aaeb5a43ab Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 16 Jan 2012 09:06:18 +0000 Subject: [PATCH 0495/1279] Added Ashwin Purohit to authors Refs #410 --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index d73afc8..97f8674 100644 --- a/AUTHORS +++ b/AUTHORS @@ -93,3 +93,4 @@ that much better: * Alvin Liang * andrewmlevy * Chris Faulkner + * Ashwin Purohit \ No newline at end of file From 12f884e3ac91cde7cb8e6b0b328088fb44035fd1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 16 Jan 2012 09:11:00 +0000 Subject: [PATCH 0496/1279] Fixes typo in documents - thanks Shalabh Closes #406 --- AUTHORS | 3 ++- docs/guide/defining-documents.rst | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/AUTHORS b/AUTHORS index 97f8674..d7e1003 100644 --- a/AUTHORS +++ b/AUTHORS @@ -93,4 +93,5 @@ that much better: * Alvin Liang * andrewmlevy * Chris Faulkner - * Ashwin Purohit \ No newline at end of file + * Ashwin Purohit + * Shalabh Aggarwal \ No newline at end of file diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index d749303..6b9fcdd 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -161,7 +161,7 @@ arguments can be set on all fields: :attr:`help_text` (Default: None) Optional help text to output with the field - used by form libraries -:attr:`verbose` (Default: None) +:attr:`verbose_name` (Default: None) Optional human-readable name for the field - used by form libraries @@ -481,9 +481,9 @@ Shard keys If your collection is sharded, then you need to specify the shard key as a tuple, using the :attr:`shard_key` attribute of :attr:`-mongoengine.Document.meta`. -This ensures that the shard key is sent with the query when calling the -:meth:`~mongoengine.document.Document.save` or -:meth:`~mongoengine.document.Document.update` method on an existing +This ensures that the shard key is sent with the query when calling the +:meth:`~mongoengine.document.Document.save` or +:meth:`~mongoengine.document.Document.update` method on an existing :class:`-mongoengine.Document` instance:: class LogEntry(Document): From 50d9b0b7964a6aff685018feefd3c6e680b638ff Mon Sep 17 00:00:00 2001 From: Chris Faulkner Date: Mon, 16 Jan 2012 19:13:03 +0800 Subject: [PATCH 0497/1279] Add dict.update() support to BaseDict. --- mongoengine/base.py | 4 ++++ tests/fields.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index 8028ca9..163a198 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1293,6 +1293,10 @@ class BaseDict(dict): self._mark_as_changed() super(BaseDict, self).popitem(*args, **kwargs) + def update(self, *args, **kwargs): + self._mark_as_changed() + super(BaseDict, self).update(*args, **kwargs) + def _mark_as_changed(self): if hasattr(self._instance, '_mark_as_changed'): self._instance._mark_as_changed(self._name) diff --git a/tests/fields.py b/tests/fields.py index 37972fd..04ef3d9 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -772,6 +772,13 @@ class FieldTest(unittest.TestCase): # Confirm handles non strings or non existing keys self.assertEquals(BlogPost.objects.filter(info__details__test__exact=5).count(), 0) self.assertEquals(BlogPost.objects.filter(info__made_up__test__exact='test').count(), 0) + + post = BlogPost.objects.create(info={'title': 'original'}) + post.info.update({'title': 'updated'}) + post.save() + post.reload() + self.assertEquals('updated', post.info['title']) + BlogPost.drop_collection() def test_dictfield_strict(self): From 9a190eb00d046777a93c972f81456d46b827a344 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 25 Jan 2012 17:06:58 -0500 Subject: [PATCH 0498/1279] Added ability to have scalar return values instead of partially-populated Document instances. --- mongoengine/queryset.py | 48 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 34c920e..c36390b 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -411,6 +411,7 @@ class QuerySet(object): self._timeout = True self._class_check = True self._slave_okay = False + self._scalar = [] # If inheritance is allowed, only return instances and instances of # subclasses of the class being used @@ -977,8 +978,13 @@ class QuerySet(object): docs = self._collection.find({'_id': {'$in': object_ids}}, **self._cursor_args) - for doc in docs: - doc_map[doc['_id']] = self._document._from_son(doc) + if self._scalar: + for doc in docs: + doc_map[doc['_id']] = self._get_scalar( + self._document._from_son(doc)) + else: + for doc in docs: + doc_map[doc['_id']] = self._document._from_son(doc) return doc_map @@ -988,6 +994,9 @@ class QuerySet(object): try: if self._limit == 0: raise StopIteration + if self._scalar: + return self._get_scalar(self._document._from_son( + self._cursor.next())) return self._document._from_son(self._cursor.next()) except StopIteration, e: self.rewind() @@ -1164,6 +1173,9 @@ class QuerySet(object): return self # Integer index provided elif isinstance(key, int): + if self._scalar: + return self._get_scalar(self._document._from_son( + self._cursor[key])) return self._document._from_son(self._cursor[key]) raise AttributeError @@ -1490,6 +1502,38 @@ class QuerySet(object): self.rewind() return self + def _get_scalar(self, doc): + def lookup(obj, name): + chunks = name.split('__') + for chunk in chunks: + obj = getattr(obj, chunk) + return obj + + data = [lookup(doc, n) for n in self._scalar] + + if len(data) == 1: + return data[0] + + return tuple(data) + + def scalar(self, *fields): + """Instead of returning Document instances, return either a specific + value or a tuple of values in order. + + This effects all results and can be unset by calling ``scalar`` + without arguments. Calls ``only`` automatically. + + :param fields: One or more fields to return instead of a Document. + """ + self._scalar = list(fields) + + if fields: + self.only(*fields) + else: + self.all_fields() + + return self + def _sub_js_fields(self, code): """When fields are specified with [~fieldname] syntax, where *fieldname* is the Python name of a field, *fieldname* will be From f60a49d6f6fc4ad213597667c957a69607fbcd44 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Jan 2012 11:41:42 +0000 Subject: [PATCH 0499/1279] Added .scalar to Queryset More efficient than the previous .values_list implementation Ref #393 Reverted some of the .values_list code thats no longer needed. Closes #415 --- AUTHORS | 2 +- docs/changelog.rst | 2 +- mongoengine/base.py | 6 +- mongoengine/queryset.py | 180 ++++++++++------------------------------ setup.py | 4 +- tests/queryset.py | 116 ++++++++++++++++---------- 6 files changed, 123 insertions(+), 187 deletions(-) diff --git a/AUTHORS b/AUTHORS index d7e1003..2b52ecb 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,11 +1,11 @@ The PRIMARY AUTHORS are (and/or have been): +Ross Lawley Harry Marr Matt Dennewitz Deepak Thukral Florian Schlachter Steve Challis -Ross Lawley Wilson Júnior Dan Crosta https://github.com/dcrosta diff --git a/docs/changelog.rst b/docs/changelog.rst index 0215335..75230a2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,8 +5,8 @@ Changelog Changes in dev ============== +- Added scalar for efficiently returning partial data values (aliased to values_list) - Fixed limit skip bug -- Added values_list for returning a list of data - Improved Inheritance / Mixin - Added sharding support - Added pymongo 2.1 support diff --git a/mongoengine/base.py b/mongoengine/base.py index 163a198..369c10f 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -468,14 +468,14 @@ class ObjectIdField(BaseField): class DocumentMetaclass(type): """Metaclass for all documents. """ - - + + def __new__(cls, name, bases, attrs): def _get_mixin_fields(base): attrs = {} attrs.update(dict([(k, v) for k, v in base.__dict__.items() if issubclass(v.__class__, BaseField)])) - + for p_base in base.__bases__: #optimize :-) if p_base in (object, BaseDocument): diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index c36390b..c4ac637 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -314,82 +314,6 @@ class QueryFieldList(object): def __nonzero__(self): return bool(self.fields) -class ListResult(object): - """ - Used for .values_list method in QuerySet - """ - def __init__(self, document_type, cursor, fields, dbfields): - from base import BaseField - from fields import ReferenceField, GenericReferenceField - # Caches for optimization - - self.ReferenceField = ReferenceField - self.GenericReferenceField = GenericReferenceField - - self._cursor = cursor - - f = [] - for field, dbfield in itertools.izip(fields, dbfields): - - p = document_type - for path in field.split('.'): - if p and isinstance(p, BaseField): - p = p.lookup_member(path) - elif p: - p = getattr(p, path) - else: - break - - f.append((dbfield.split('.'), p)) - - self._fields = f - - def _get_value(self, keys, field_type, data): - for key in keys: - if data: - data = data.get(key) - else: - break - - if isinstance(field_type, self.ReferenceField): - doc_type = field_type.document_type - data = doc_type._get_db().dereference(data) - - if data: - return doc_type._from_son(data) - - elif isinstance(field_type, self.GenericReferenceField): - if data and isinstance(data, (dict, pymongo.dbref.DBRef)): - return field_type.dereference(data) - - if data is None: - return - - return field_type.to_python(data) - - def next(self): - try: - data = self._cursor.next() - return [self._get_value(k, t, data) - for k, t in self._fields] - except StopIteration, e: - self.rewind() - raise e - - def rewind(self): - self._cursor.rewind() - - def count(self): - """ - Count the selected elements in the query. - """ - return self._cursor.count(with_limit_and_skip=True) - - def __len__(self): - return self.count() - - def __iter__(self): - return self class QuerySet(object): """A set of results returned from a query. Wraps a MongoDB cursor, @@ -625,38 +549,33 @@ class QuerySet(object): cursor_args['fields'] = self._loaded_fields.as_dict() return cursor_args - def _build_cursor(self, **cursor_args): - obj = self._collection.find(self._query, - **cursor_args) - # Apply where clauses to cursor - if self._where_clause: - obj.where(self._where_clause) - - # apply default ordering - if self._ordering: - obj.sort(self._ordering) - elif self._document._meta['ordering']: - self._ordering = self._get_order_key_list( - *self._document._meta['ordering']) - obj.sort(self._ordering) - - if self._limit is not None: - obj.limit(self._limit - (self._skip or 0)) - - if self._skip is not None: - obj.skip(self._skip) - - if self._hint != -1: - obj.hint(self._hint) - - return obj - @property def _cursor(self): if self._cursor_obj is None: - self._cursor_obj = self._build_cursor(**self._cursor_args) + + self._cursor_obj = self._collection.find(self._query, + **self._cursor_args) + # Apply where clauses to cursor + if self._where_clause: + self._cursor_obj.where(self._where_clause) + + # apply default ordering + if self._ordering: + self._cursor_obj.sort(self._ordering) + elif self._document._meta['ordering']: + self.order_by(*self._document._meta['ordering']) + + if self._limit is not None: + self._cursor_obj.limit(self._limit - (self._skip or 0)) + + if self._skip is not None: + self._cursor_obj.skip(self._skip) + + if self._hint != -1: + self._cursor_obj.hint(self._hint) return self._cursor_obj + @classmethod def _lookup_field(cls, document, parts): """Lookup a field based on its attribute and return a list containing @@ -885,19 +804,6 @@ class QuerySet(object): doc.save() return doc - def values_list(self, *fields): - """ - make a list of elements - .. versionadded:: 0.6 - """ - dbfields = self._fields_to_dbfields(fields) - - cursor_args = self._cursor_args - cursor_args['fields'] = dbfields - cursor = self._build_cursor(**cursor_args) - - return ListResult(self._document, cursor, fields, dbfields) - def first(self): """Retrieve the first object matching the query. """ @@ -1269,9 +1175,13 @@ class QuerySet(object): ret.append(field) return ret - def _get_order_key_list(self, *keys): - """ - Build order list for query + def order_by(self, *keys): + """Order the :class:`~mongoengine.queryset.QuerySet` by the keys. The + order may be specified by prepending each of the keys by a + or a -. + Ascending order is assumed. + + :param keys: fields to order the query results by; keys may be + prefixed with **+** or **-** to determine the ordering direction """ key_list = [] for key in keys: @@ -1288,18 +1198,6 @@ class QuerySet(object): pass key_list.append((key, direction)) - return key_list - - def order_by(self, *keys): - """Order the :class:`~mongoengine.queryset.QuerySet` by the keys. The - order may be specified by prepending each of the keys by a + or a -. - Ascending order is assumed. - - :param keys: fields to order the query results by; keys may be - prefixed with **+** or **-** to determine the ordering direction - """ - - key_list = self._get_order_key_list(*keys) self._ordering = key_list self._cursor.sort(key_list) return self @@ -1503,37 +1401,43 @@ class QuerySet(object): return self def _get_scalar(self, doc): + def lookup(obj, name): chunks = name.split('__') for chunk in chunks: + if hasattr(obj, '_db_field_map'): + chunk = obj._db_field_map.get(chunk, chunk) obj = getattr(obj, chunk) return obj - + data = [lookup(doc, n) for n in self._scalar] - if len(data) == 1: return data[0] - + return tuple(data) def scalar(self, *fields): """Instead of returning Document instances, return either a specific value or a tuple of values in order. - + This effects all results and can be unset by calling ``scalar`` without arguments. Calls ``only`` automatically. - + :param fields: One or more fields to return instead of a Document. """ self._scalar = list(fields) - + if fields: self.only(*fields) else: self.all_fields() - + return self + def values_list(self, *fields): + """An alias for scalar""" + return self.scalar(*fields) + def _sub_js_fields(self, code): """When fields are specified with [~fieldname] syntax, where *fieldname* is the Python name of a field, *fieldname* will be diff --git a/setup.py b/setup.py index b0c29bf..7a69d83 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,9 @@ setup(name='mongoengine', packages=find_packages(), author='Harry Marr', author_email='harry.marr@{nospam}gmail.com', - url='http://hmarr.com/mongoengine/', + maintainer="Ross Lawley", + maintainer_email="ross.lawley@gmail.com", + url='http://mongoengine.org/', license='MIT', include_package_data=True, description=DESCRIPTION, diff --git a/tests/queryset.py b/tests/queryset.py index b749340..a84fd5b 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -2909,8 +2909,38 @@ class QueryFieldListTest(unittest.TestCase): ak = list(Bar.objects(foo__match={'shape': "square", "color": "purple"})) self.assertEqual([b1], ak) - - def test_values_list(self): + + def test_scalar(self): + + class Organization(Document): + id = ObjectIdField('_id') + name = StringField() + + class User(Document): + id = ObjectIdField('_id') + name = StringField() + organization = ObjectIdField() + + User.drop_collection() + Organization.drop_collection() + + whitehouse = Organization(name="White House") + whitehouse.save() + User(name="Bob Dole", organization=whitehouse.id).save() + + # Efficient way to get all unique organization names for a given + # set of users (Pretend this has additional filtering.) + user_orgs = set(User.objects.scalar('organization')) + orgs = Organization.objects(id__in=user_orgs).scalar('name') + self.assertEqual(list(orgs), ['White House']) + + # Efficient for generating listings, too. + orgs = Organization.objects.scalar('name').in_bulk(list(user_orgs)) + user_map = User.objects.scalar('name', 'organization') + user_listing = [(user, orgs[org]) for user, org in user_map] + self.assertEqual([("Bob Dole", "White House")], user_listing) + + def test_scalar_simple(self): class TestDoc(Document): x = IntField() y = BooleanField() @@ -2921,12 +2951,12 @@ class QueryFieldListTest(unittest.TestCase): TestDoc(x=20, y=False).save() TestDoc(x=30, y=True).save() - plist = list(TestDoc.objects.values_list('x', 'y')) + plist = list(TestDoc.objects.scalar('x', 'y')) self.assertEqual(len(plist), 3) - self.assertEqual(plist[0], [10, True]) - self.assertEqual(plist[1], [20, False]) - self.assertEqual(plist[2], [30, True]) + self.assertEqual(plist[0], (10, True)) + self.assertEqual(plist[1], (20, False)) + self.assertEqual(plist[2], (30, True)) class UserDoc(Document): name = StringField() @@ -2939,23 +2969,23 @@ class QueryFieldListTest(unittest.TestCase): UserDoc(name="Eliana", age=37).save() UserDoc(name="Tayza", age=15).save() - ulist = list(UserDoc.objects.values_list('name', 'age')) + ulist = list(UserDoc.objects.scalar('name', 'age')) self.assertEqual(ulist, [ - [u'Wilson Jr', 19], - [u'Wilson', 43], - [u'Eliana', 37], - [u'Tayza', 15]]) + (u'Wilson Jr', 19), + (u'Wilson', 43), + (u'Eliana', 37), + (u'Tayza', 15)]) - ulist = list(UserDoc.objects.order_by('age').values_list('name')) + ulist = list(UserDoc.objects.scalar('name').order_by('age')) self.assertEqual(ulist, [ - [u'Tayza'], - [u'Wilson Jr'], - [u'Eliana'], - [u'Wilson']]) + (u'Tayza'), + (u'Wilson Jr'), + (u'Eliana'), + (u'Wilson')]) - def test_values_list_embedded(self): + def test_scalar_embedded(self): class Profile(EmbeddedDocument): name = StringField() age = IntField() @@ -2983,32 +3013,31 @@ class QueryFieldListTest(unittest.TestCase): locale=Locale(city="Brasilia", country="Brazil")).save() self.assertEqual( - list(Person.objects.order_by('profile.age').values_list('profile.name')), - [[u'Wilson Jr'], [u'Gabriel Falcao'], - [u'Lincoln de souza'], [u'Walter cruz']]) + list(Person.objects.order_by('profile__age').scalar('profile__name')), + [u'Wilson Jr', u'Gabriel Falcao', u'Lincoln de souza', u'Walter cruz']) ulist = list(Person.objects.order_by('locale.city') - .values_list('profile.name', 'profile.age', 'locale.city')) + .scalar('profile__name', 'profile__age', 'locale__city')) self.assertEqual(ulist, - [[u'Lincoln de souza', 28, u'Belo Horizonte'], - [u'Walter cruz', 30, u'Brasilia'], - [u'Wilson Jr', 19, u'Corumba-GO'], - [u'Gabriel Falcao', 23, u'New York']]) + [(u'Lincoln de souza', 28, u'Belo Horizonte'), + (u'Walter cruz', 30, u'Brasilia'), + (u'Wilson Jr', 19, u'Corumba-GO'), + (u'Gabriel Falcao', 23, u'New York')]) - def test_values_list_decimal(self): + def test_scalar_decimal(self): from decimal import Decimal class Person(Document): name = StringField() rating = DecimalField() - + Person.drop_collection() Person(name="Wilson Jr", rating=Decimal('1.0')).save() - ulist = list(Person.objects.values_list('name', 'rating')) - self.assertEqual(ulist, [[u'Wilson Jr', Decimal('1.0')]]) + ulist = list(Person.objects.scalar('name', 'rating')) + self.assertEqual(ulist, [(u'Wilson Jr', Decimal('1.0'))]) - def test_values_list_reference_field(self): + def test_scalar_reference_field(self): class State(Document): name = StringField() @@ -3024,10 +3053,10 @@ class QueryFieldListTest(unittest.TestCase): Person(name="Wilson JR", state=s1).save() - plist = list(Person.objects.values_list('name', 'state')) - self.assertEqual(plist, [[u'Wilson JR', s1]]) + plist = list(Person.objects.scalar('name', 'state')) + self.assertEqual(plist, [(u'Wilson JR', s1)]) - def test_values_list_generic_reference_field(self): + def test_scalar_generic_reference_field(self): class State(Document): name = StringField() @@ -3043,13 +3072,14 @@ class QueryFieldListTest(unittest.TestCase): Person(name="Wilson JR", state=s1).save() - plist = list(Person.objects.values_list('name', 'state')) - self.assertEqual(plist, [[u'Wilson JR', s1]]) + plist = list(Person.objects.scalar('name', 'state')) + self.assertEqual(plist, [(u'Wilson JR', s1)]) + + def test_scalar_db_field(self): - def test_values_list_db_field(self): class TestDoc(Document): - x = IntField(db_field="y") - y = BooleanField(db_field="x") + x = IntField() + y = BooleanField() TestDoc.drop_collection() @@ -3057,12 +3087,12 @@ class QueryFieldListTest(unittest.TestCase): TestDoc(x=20, y=False).save() TestDoc(x=30, y=True).save() - plist = list(TestDoc.objects.values_list('x', 'y')) - + plist = list(TestDoc.objects.scalar('x', 'y')) self.assertEqual(len(plist), 3) - self.assertEqual(plist[0], [10, True]) - self.assertEqual(plist[1], [20, False]) - self.assertEqual(plist[2], [30, True]) + self.assertEqual(plist[0], (10, True)) + self.assertEqual(plist[1], (20, False)) + self.assertEqual(plist[2], (30, True)) + if __name__ == '__main__': unittest.main() From f59aa922ea886a42e2b4467059e51f06a405834f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Jan 2012 12:20:47 +0000 Subject: [PATCH 0500/1279] Added more .scalar tests --- mongoengine/queryset.py | 3 +- tests/queryset.py | 442 +++++++++++++++++++++++----------------- 2 files changed, 260 insertions(+), 185 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index c4ac637..0c0b11f 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -868,8 +868,7 @@ class QuerySet(object): """ if not self._query_obj.empty: raise InvalidQueryError("Cannot use a filter whilst using `with_id`") - - return self._document.objects(pk=object_id).first() + return self.filter(pk=object_id).first() def in_bulk(self, object_ids): """Retrieve a set of documents by their ids. diff --git a/tests/queryset.py b/tests/queryset.py index a84fd5b..555e7d0 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -2665,6 +2665,265 @@ class QuerySetTest(unittest.TestCase): self.assertRaises(TypeError, invalid_where) + def test_scalar(self): + + class Organization(Document): + id = ObjectIdField('_id') + name = StringField() + + class User(Document): + id = ObjectIdField('_id') + name = StringField() + organization = ObjectIdField() + + User.drop_collection() + Organization.drop_collection() + + whitehouse = Organization(name="White House") + whitehouse.save() + User(name="Bob Dole", organization=whitehouse.id).save() + + # Efficient way to get all unique organization names for a given + # set of users (Pretend this has additional filtering.) + user_orgs = set(User.objects.scalar('organization')) + orgs = Organization.objects(id__in=user_orgs).scalar('name') + self.assertEqual(list(orgs), ['White House']) + + # Efficient for generating listings, too. + orgs = Organization.objects.scalar('name').in_bulk(list(user_orgs)) + user_map = User.objects.scalar('name', 'organization') + user_listing = [(user, orgs[org]) for user, org in user_map] + self.assertEqual([("Bob Dole", "White House")], user_listing) + + def test_scalar_simple(self): + class TestDoc(Document): + x = IntField() + y = BooleanField() + + TestDoc.drop_collection() + + TestDoc(x=10, y=True).save() + TestDoc(x=20, y=False).save() + TestDoc(x=30, y=True).save() + + plist = list(TestDoc.objects.scalar('x', 'y')) + + self.assertEqual(len(plist), 3) + self.assertEqual(plist[0], (10, True)) + self.assertEqual(plist[1], (20, False)) + self.assertEqual(plist[2], (30, True)) + + class UserDoc(Document): + name = StringField() + age = IntField() + + UserDoc.drop_collection() + + UserDoc(name="Wilson Jr", age=19).save() + UserDoc(name="Wilson", age=43).save() + UserDoc(name="Eliana", age=37).save() + UserDoc(name="Tayza", age=15).save() + + ulist = list(UserDoc.objects.scalar('name', 'age')) + + self.assertEqual(ulist, [ + (u'Wilson Jr', 19), + (u'Wilson', 43), + (u'Eliana', 37), + (u'Tayza', 15)]) + + ulist = list(UserDoc.objects.scalar('name').order_by('age')) + + self.assertEqual(ulist, [ + (u'Tayza'), + (u'Wilson Jr'), + (u'Eliana'), + (u'Wilson')]) + + def test_scalar_embedded(self): + class Profile(EmbeddedDocument): + name = StringField() + age = IntField() + + class Locale(EmbeddedDocument): + city = StringField() + country = StringField() + + class Person(Document): + profile = EmbeddedDocumentField(Profile) + locale = EmbeddedDocumentField(Locale) + + Person.drop_collection() + + Person(profile=Profile(name="Wilson Jr", age=19), + locale=Locale(city="Corumba-GO", country="Brazil")).save() + + Person(profile=Profile(name="Gabriel Falcao", age=23), + locale=Locale(city="New York", country="USA")).save() + + Person(profile=Profile(name="Lincoln de souza", age=28), + locale=Locale(city="Belo Horizonte", country="Brazil")).save() + + Person(profile=Profile(name="Walter cruz", age=30), + locale=Locale(city="Brasilia", country="Brazil")).save() + + self.assertEqual( + list(Person.objects.order_by('profile__age').scalar('profile__name')), + [u'Wilson Jr', u'Gabriel Falcao', u'Lincoln de souza', u'Walter cruz']) + + ulist = list(Person.objects.order_by('locale.city') + .scalar('profile__name', 'profile__age', 'locale__city')) + self.assertEqual(ulist, + [(u'Lincoln de souza', 28, u'Belo Horizonte'), + (u'Walter cruz', 30, u'Brasilia'), + (u'Wilson Jr', 19, u'Corumba-GO'), + (u'Gabriel Falcao', 23, u'New York')]) + + def test_scalar_decimal(self): + from decimal import Decimal + class Person(Document): + name = StringField() + rating = DecimalField() + + Person.drop_collection() + Person(name="Wilson Jr", rating=Decimal('1.0')).save() + + ulist = list(Person.objects.scalar('name', 'rating')) + self.assertEqual(ulist, [(u'Wilson Jr', Decimal('1.0'))]) + + + def test_scalar_reference_field(self): + class State(Document): + name = StringField() + + class Person(Document): + name = StringField() + state = ReferenceField(State) + + State.drop_collection() + Person.drop_collection() + + s1 = State(name="Goias") + s1.save() + + Person(name="Wilson JR", state=s1).save() + + plist = list(Person.objects.scalar('name', 'state')) + self.assertEqual(plist, [(u'Wilson JR', s1)]) + + def test_scalar_generic_reference_field(self): + class State(Document): + name = StringField() + + class Person(Document): + name = StringField() + state = GenericReferenceField() + + State.drop_collection() + Person.drop_collection() + + s1 = State(name="Goias") + s1.save() + + Person(name="Wilson JR", state=s1).save() + + plist = list(Person.objects.scalar('name', 'state')) + self.assertEqual(plist, [(u'Wilson JR', s1)]) + + def test_scalar_db_field(self): + + class TestDoc(Document): + x = IntField() + y = BooleanField() + + TestDoc.drop_collection() + + TestDoc(x=10, y=True).save() + TestDoc(x=20, y=False).save() + TestDoc(x=30, y=True).save() + + plist = list(TestDoc.objects.scalar('x', 'y')) + self.assertEqual(len(plist), 3) + self.assertEqual(plist[0], (10, True)) + self.assertEqual(plist[1], (20, False)) + self.assertEqual(plist[2], (30, True)) + + def test_scalar_cursor_behaviour(self): + """Ensure that a query returns a valid set of results. + """ + person1 = self.Person(name="User A", age=20) + person1.save() + person2 = self.Person(name="User B", age=30) + person2.save() + + # Find all people in the collection + people = self.Person.objects.scalar('name') + self.assertEqual(len(people), 2) + results = list(people) + self.assertEqual(results[0], "User A") + self.assertEqual(results[1], "User B") + + # Use a query to filter the people found to just person1 + people = self.Person.objects(age=20).scalar('name') + self.assertEqual(len(people), 1) + person = people.next() + self.assertEqual(person, "User A") + + # Test limit + people = list(self.Person.objects.limit(1).scalar('name')) + self.assertEqual(len(people), 1) + self.assertEqual(people[0], 'User A') + + # Test skip + people = list(self.Person.objects.skip(1).scalar('name')) + self.assertEqual(len(people), 1) + self.assertEqual(people[0], 'User B') + + person3 = self.Person(name="User C", age=40) + person3.save() + + # Test slice limit + people = list(self.Person.objects[:2].scalar('name')) + self.assertEqual(len(people), 2) + self.assertEqual(people[0], 'User A') + self.assertEqual(people[1], 'User B') + + # Test slice skip + people = list(self.Person.objects[1:].scalar('name')) + self.assertEqual(len(people), 2) + self.assertEqual(people[0], 'User B') + self.assertEqual(people[1], 'User C') + + # Test slice limit and skip + people = list(self.Person.objects[1:2].scalar('name')) + self.assertEqual(len(people), 1) + self.assertEqual(people[0], 'User B') + + people = list(self.Person.objects[1:1].scalar('name')) + self.assertEqual(len(people), 0) + + # Test slice out of range + people = list(self.Person.objects.scalar('name')[80000:80001]) + self.assertEqual(len(people), 0) + + # Test larger slice __repr__ + self.Person.objects.delete() + for i in xrange(55): + self.Person(name='A%s' % i, age=i).save() + + self.assertEqual(len(self.Person.objects.scalar('name')), 55) + self.assertEqual("A0", "%s" % self.Person.objects.order_by('name').scalar('name').first()) + self.assertEqual("A0", "%s" % self.Person.objects.scalar('name').order_by('name')[0]) + self.assertEqual("[u'A1', u'A2']", "%s" % self.Person.objects.order_by('age').scalar('name')[1:3]) + self.assertEqual("[u'A51', u'A52']", "%s" % self.Person.objects.order_by('age').scalar('name')[51:53]) + + # with_id and in_bulk + person = self.Person.objects.order_by('name').first() + self.assertEqual("A0", "%s" % self.Person.objects.scalar('name').with_id(person.id)) + + pks = self.Person.objects.order_by('age').scalar('pk')[1:3] + self.assertEqual("[u'A1', u'A2']", "%s" % sorted(self.Person.objects.scalar('name').in_bulk(list(pks)).values())) + class QTest(unittest.TestCase): @@ -2910,189 +3169,6 @@ class QueryFieldListTest(unittest.TestCase): ak = list(Bar.objects(foo__match={'shape': "square", "color": "purple"})) self.assertEqual([b1], ak) - def test_scalar(self): - - class Organization(Document): - id = ObjectIdField('_id') - name = StringField() - - class User(Document): - id = ObjectIdField('_id') - name = StringField() - organization = ObjectIdField() - - User.drop_collection() - Organization.drop_collection() - - whitehouse = Organization(name="White House") - whitehouse.save() - User(name="Bob Dole", organization=whitehouse.id).save() - - # Efficient way to get all unique organization names for a given - # set of users (Pretend this has additional filtering.) - user_orgs = set(User.objects.scalar('organization')) - orgs = Organization.objects(id__in=user_orgs).scalar('name') - self.assertEqual(list(orgs), ['White House']) - - # Efficient for generating listings, too. - orgs = Organization.objects.scalar('name').in_bulk(list(user_orgs)) - user_map = User.objects.scalar('name', 'organization') - user_listing = [(user, orgs[org]) for user, org in user_map] - self.assertEqual([("Bob Dole", "White House")], user_listing) - - def test_scalar_simple(self): - class TestDoc(Document): - x = IntField() - y = BooleanField() - - TestDoc.drop_collection() - - TestDoc(x=10, y=True).save() - TestDoc(x=20, y=False).save() - TestDoc(x=30, y=True).save() - - plist = list(TestDoc.objects.scalar('x', 'y')) - - self.assertEqual(len(plist), 3) - self.assertEqual(plist[0], (10, True)) - self.assertEqual(plist[1], (20, False)) - self.assertEqual(plist[2], (30, True)) - - class UserDoc(Document): - name = StringField() - age = IntField() - - UserDoc.drop_collection() - - UserDoc(name="Wilson Jr", age=19).save() - UserDoc(name="Wilson", age=43).save() - UserDoc(name="Eliana", age=37).save() - UserDoc(name="Tayza", age=15).save() - - ulist = list(UserDoc.objects.scalar('name', 'age')) - - self.assertEqual(ulist, [ - (u'Wilson Jr', 19), - (u'Wilson', 43), - (u'Eliana', 37), - (u'Tayza', 15)]) - - ulist = list(UserDoc.objects.scalar('name').order_by('age')) - - self.assertEqual(ulist, [ - (u'Tayza'), - (u'Wilson Jr'), - (u'Eliana'), - (u'Wilson')]) - - def test_scalar_embedded(self): - class Profile(EmbeddedDocument): - name = StringField() - age = IntField() - - class Locale(EmbeddedDocument): - city = StringField() - country = StringField() - - class Person(Document): - profile = EmbeddedDocumentField(Profile) - locale = EmbeddedDocumentField(Locale) - - Person.drop_collection() - - Person(profile=Profile(name="Wilson Jr", age=19), - locale=Locale(city="Corumba-GO", country="Brazil")).save() - - Person(profile=Profile(name="Gabriel Falcao", age=23), - locale=Locale(city="New York", country="USA")).save() - - Person(profile=Profile(name="Lincoln de souza", age=28), - locale=Locale(city="Belo Horizonte", country="Brazil")).save() - - Person(profile=Profile(name="Walter cruz", age=30), - locale=Locale(city="Brasilia", country="Brazil")).save() - - self.assertEqual( - list(Person.objects.order_by('profile__age').scalar('profile__name')), - [u'Wilson Jr', u'Gabriel Falcao', u'Lincoln de souza', u'Walter cruz']) - - ulist = list(Person.objects.order_by('locale.city') - .scalar('profile__name', 'profile__age', 'locale__city')) - self.assertEqual(ulist, - [(u'Lincoln de souza', 28, u'Belo Horizonte'), - (u'Walter cruz', 30, u'Brasilia'), - (u'Wilson Jr', 19, u'Corumba-GO'), - (u'Gabriel Falcao', 23, u'New York')]) - - def test_scalar_decimal(self): - from decimal import Decimal - class Person(Document): - name = StringField() - rating = DecimalField() - - Person.drop_collection() - Person(name="Wilson Jr", rating=Decimal('1.0')).save() - - ulist = list(Person.objects.scalar('name', 'rating')) - self.assertEqual(ulist, [(u'Wilson Jr', Decimal('1.0'))]) - - - def test_scalar_reference_field(self): - class State(Document): - name = StringField() - - class Person(Document): - name = StringField() - state = ReferenceField(State) - - State.drop_collection() - Person.drop_collection() - - s1 = State(name="Goias") - s1.save() - - Person(name="Wilson JR", state=s1).save() - - plist = list(Person.objects.scalar('name', 'state')) - self.assertEqual(plist, [(u'Wilson JR', s1)]) - - def test_scalar_generic_reference_field(self): - class State(Document): - name = StringField() - - class Person(Document): - name = StringField() - state = GenericReferenceField() - - State.drop_collection() - Person.drop_collection() - - s1 = State(name="Goias") - s1.save() - - Person(name="Wilson JR", state=s1).save() - - plist = list(Person.objects.scalar('name', 'state')) - self.assertEqual(plist, [(u'Wilson JR', s1)]) - - def test_scalar_db_field(self): - - class TestDoc(Document): - x = IntField() - y = BooleanField() - - TestDoc.drop_collection() - - TestDoc(x=10, y=True).save() - TestDoc(x=20, y=False).save() - TestDoc(x=30, y=True).save() - - plist = list(TestDoc.objects.scalar('x', 'y')) - self.assertEqual(len(plist), 3) - self.assertEqual(plist[0], (10, True)) - self.assertEqual(plist[1], (20, False)) - self.assertEqual(plist[2], (30, True)) - if __name__ == '__main__': unittest.main() From 0301135f96be83ee613664e875409a9d4b590137 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 30 Jan 2012 10:24:45 +0000 Subject: [PATCH 0501/1279] Added uri style connection handling --- docs/changelog.rst | 1 + docs/guide/connecting.rst | 7 +++++++ mongoengine/connection.py | 26 ++++++++++++++++++++------ tests/connection.py | 13 +++++++++++++ 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 75230a2..e4d693b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added uri support for connections - Added scalar for efficiently returning partial data values (aliased to values_list) - Fixed limit skip bug - Improved Inheritance / Mixin diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index 470dcb8..50eb270 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -20,6 +20,13 @@ provide :attr:`host` and :attr:`port` arguments to connect('project1', host='192.168.1.35', port=12345) +Uri style connections are also supported as long as you include the database +name - just supply the uri as the :attr:`host` to +:func:`~mongoengine.connect`:: + + connect('project1', host='mongodb://localhost/database_name') + + Multiple Databases ================== diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 462c895..822c604 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,5 +1,5 @@ import pymongo -from pymongo import Connection +from pymongo import Connection, uri_parser __all__ = ['ConnectionError', 'connect', 'register_connection', @@ -38,6 +38,17 @@ def register_connection(alias, name, host='localhost', port=27017, """ global _connection_settings + + # Handle uri style connections + if "://" in host: + uri_dict = uri_parser.parse_uri(host) + if 'database' not in uri_dict: + raise ConnectionError("If using URI style connection include "\ + "database name in string") + uri_dict['name'] = uri_dict.get('database') + _connection_settings[alias] = uri_dict + return + _connection_settings[alias] = { 'name': name, 'host': host, @@ -48,8 +59,10 @@ def register_connection(alias, name, host='localhost', port=27017, 'password': password, 'read_preference': read_preference } + _connection_settings[alias].update(kwargs) + def disconnect(alias=DEFAULT_CONNECTION_NAME): global _connections global _dbs @@ -83,11 +96,12 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): conn_settings.pop('password') else: # Get all the slave connections - slaves = [] - for slave_alias in conn_settings['slaves']: - slaves.append(get_connection(slave_alias)) - conn_settings['slaves'] = slaves - conn_settings.pop('read_preference') + if 'slaves' in conn_settings: + slaves = [] + for slave_alias in conn_settings['slaves']: + slaves.append(get_connection(slave_alias)) + conn_settings['slaves'] = slaves + conn_settings.pop('read_preference') try: _connections[alias] = Connection(**conn_settings) diff --git a/tests/connection.py b/tests/connection.py index e017b38..7ff0998 100644 --- a/tests/connection.py +++ b/tests/connection.py @@ -30,6 +30,19 @@ class ConnectionTest(unittest.TestCase): conn = get_connection('testdb') self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + def test_connect_uri(self): + """Ensure that the connect() method works properly with uri's + """ + + connect("testdb_uri", host='mongodb://username:password@localhost/mongoenginetest') + + conn = get_connection() + self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + + db = get_db() + self.assertTrue(isinstance(db, pymongo.database.Database)) + self.assertEqual(db.name, 'mongoenginetest') + def test_register_connection(self): """Ensure that connections with different aliases may be registered. """ From d3962c4f7dde7fe1e8d4a89c0f0df222e8583b5c Mon Sep 17 00:00:00 2001 From: Robert Kajic Date: Tue, 31 Jan 2012 22:31:24 +0100 Subject: [PATCH 0502/1279] Added support for creating a geo2d index by prefixing the field name with a * --- mongoengine/queryset.py | 10 ++++++---- tests/document.py | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a662685..56ec523 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -399,12 +399,14 @@ class QuerySet(object): index_list = [] use_types = doc_cls._meta.get('allow_inheritance', True) for key in spec['fields']: - # Get direction from + or - + # Get ASCENDING direction from +, DESCENDING from -, and GEO2D from * direction = pymongo.ASCENDING if key.startswith("-"): direction = pymongo.DESCENDING - if key.startswith(("+", "-")): - key = key[1:] + elif key.startswith("*"): + direction = pymongo.GEO2D + if key.startswith(("+", "-", "*")): + key = key[1:] # Use real field name, do it manually because we need field # objects for the next part (list field checking) @@ -421,7 +423,7 @@ class QuerySet(object): # If _types is being used, prepend it to every specified index index_types = doc_cls._meta.get('index_types', True) allow_inheritance = doc_cls._meta.get('allow_inheritance') - if spec.get('types', index_types) and allow_inheritance and use_types: + if spec.get('types', index_types) and allow_inheritance and use_types and direction is not pymongo.GEO2D: index_list.insert(0, ('_types', 1)) spec['fields'] = index_list diff --git a/tests/document.py b/tests/document.py index 95f3774..0a26710 100644 --- a/tests/document.py +++ b/tests/document.py @@ -658,6 +658,26 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + def test_explicit_geo2d_index(self): + """Ensure that geo2d indexes work when created via meta[indexes] + """ + class Place(Document): + location = DictField() + meta = { + 'indexes': [ + '*location.point', + ], + } + Place.drop_collection() + + info = Place.objects._collection.index_information() + # Indexes are lazy so use list() to perform query + list(Place.objects) + info = Place.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] + + self.assertTrue([('location.point', '2d')] in info) + def test_dictionary_indexes(self): """Ensure that indexes are used when meta[indexes] contains dictionaries instead of lists. From cb2cb851e2931afa30adab89c5c87385609ea64e Mon Sep 17 00:00:00 2001 From: Pau Aliagas Date: Wed, 1 Feb 2012 12:29:05 +0100 Subject: [PATCH 0503/1279] Fix formatting typo in changelog entry --- python-mongoengine.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-mongoengine.spec b/python-mongoengine.spec index cc7b043..d528d7a 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -53,7 +53,7 @@ rm -rf $RPM_BUILD_ROOT %changelog * Thu Oct 27 2011 Pau Aliagas 0.5.3-1 - Update to latest dev version -* Add PIL dependency for ImageField +- Add PIL dependency for ImageField * Wed Oct 12 2011 Pau Aliagas 0.5.2-1 - Update version * Fri Sep 23 2011 Pau Aliagas 0.5.0-1 From 4a9ed5f2f2ba5e1205682ab73dc6c7c4239af695 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BD=D1=85=D0=B1=D0=B0=D1=8F=D1=80=20=D0=9B=D1=85?= =?UTF-8?q?=D0=B0=D0=B3=D0=B2=D0=B0=D0=B4=D0=BE=D1=80=D0=B6?= Date: Thu, 2 Feb 2012 18:33:12 +0800 Subject: [PATCH 0504/1279] Fix derefcence failed some case. --- mongoengine/base.py | 2 +- mongoengine/dereference.py | 4 ++-- tests/dereference.py | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 369c10f..3190de6 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -339,7 +339,7 @@ class ComplexBaseField(BaseField): # _types / _cls data so make it a generic reference allows # us to dereference meta = getattr(v, 'meta', getattr(v, '_meta', {})) - if meta and not meta['allow_inheritance'] and not self.field: + if meta and not meta.get('allow_inheritance', True) and not self.field: from fields import GenericReferenceField value_dict[k] = GenericReferenceField().to_mongo(v) else: diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 40ceb49..ac01ac7 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -1,7 +1,7 @@ import pymongo from base import (BaseDict, BaseList, TopLevelDocumentMetaclass, get_document) -from fields import ReferenceField +from fields import (ReferenceField, ListField, DictField, MapField) from connection import get_db from queryset import QuerySet from document import Document @@ -102,7 +102,7 @@ class DeReference(object): for key, doc in references.iteritems(): object_map[key] = doc else: # Generic reference: use the refs data to convert to document - if doc_type: + if doc_type and not isinstance(doc_type, (ListField, DictField, MapField,) ): references = doc_type._get_db()[col].find({'_id': {'$in': refs}}) for ref in references: doc = doc_type._from_son(ref) diff --git a/tests/dereference.py b/tests/dereference.py index 88cf17e..a23d78f 100644 --- a/tests/dereference.py +++ b/tests/dereference.py @@ -783,3 +783,41 @@ class FieldTest(unittest.TestCase): root = root.reload() self.assertEquals(root.children, [company]) self.assertEquals(company.parents, [root]) + + def test_dict_in_dbref_instance(self): + + class Person(Document): + name = StringField(max_length=250, required=True) + + meta = { "ordering" : "name" } + + class Room(Document): + number = StringField(max_length=250, required=True) + staffs_with_position = ListField(DictField()) + + meta = { "ordering" : "number" } + + + + Person.drop_collection() + + # 201 + bob = Person.objects.create(name='Bob') + bob.save() + keven = Person.objects.create(name='Keven') + keven.save() + sarah = Person.objects.create(name='Sarah') + sarah.save() + + room_201 = Room.objects.create( number = "201") + room_201.staffs_with_position = [ {'position_key': 'window', 'staff' : sarah.to_dbref() }, + { 'position_key': 'door', 'staff': bob.to_dbref() }, + { 'position_key': 'center' , 'staff' : keven.to_dbref() } ] + room_201.save() + + room = Room.objects.first().select_related() + room.to_mongo() + + + + From 734986c1b59ac4a8ee6af4821db32fa891fa58a1 Mon Sep 17 00:00:00 2001 From: Robert Kajic Date: Thu, 16 Feb 2012 10:41:47 +0100 Subject: [PATCH 0505/1279] Documentation on geospatial indexes and how to create them explicitly --- docs/guide/defining-documents.rst | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index fd005e4..004f7d8 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -382,10 +382,25 @@ If a dictionary is passed then the following options are available: :attr:`unique` (Default: False) Whether the index should be sparse. -.. note:: - Geospatial indexes will be automatically created for all - :class:`~mongoengine.GeoPointField`\ s +Geospatial indexes +--------------------------- +Geospatial indexes will be automatically created for all +:class:`~mongoengine.GeoPointField`\ s + +It is also possible to explicitly define geospatial indexes. This is +useful if you need to define a geospatial index on a subfield of a +:class:`~mongoengine.DictField` or a custom field that contains a +point. To create a geospatial index you must prefix the field with the +***** sign. :: + + class Place(Document): + location = DictField() + meta = { + 'indexes': [ + '*location.point', + ], + } Ordering ======== From df65f3fc3fb1784a53c0b2f42e56b3807d00d016 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 17 Feb 2012 09:41:01 +0000 Subject: [PATCH 0506/1279] base no longer expects meta to have allow_inheritance Closes #430 #431 --- AUTHORS | 3 ++- mongoengine/base.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 2b52ecb..e7597b3 100644 --- a/AUTHORS +++ b/AUTHORS @@ -94,4 +94,5 @@ that much better: * andrewmlevy * Chris Faulkner * Ashwin Purohit - * Shalabh Aggarwal \ No newline at end of file + * Shalabh Aggarwal + * Chris Williams \ No newline at end of file diff --git a/mongoengine/base.py b/mongoengine/base.py index 369c10f..3190de6 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -339,7 +339,7 @@ class ComplexBaseField(BaseField): # _types / _cls data so make it a generic reference allows # us to dereference meta = getattr(v, 'meta', getattr(v, '_meta', {})) - if meta and not meta['allow_inheritance'] and not self.field: + if meta and not meta.get('allow_inheritance', True) and not self.field: from fields import GenericReferenceField value_dict[k] = GenericReferenceField().to_mongo(v) else: From a59b518cf2ca53da5699ce34a95e15cc07561a6d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 17 Feb 2012 11:18:25 +0000 Subject: [PATCH 0507/1279] Updates to imports for future pymongo 2.2 --- mongoengine/base.py | 14 ++++++------ mongoengine/connection.py | 21 ++++++++++-------- mongoengine/dereference.py | 16 +++++++------- mongoengine/document.py | 15 +++++++------ mongoengine/fields.py | 22 +++++++++---------- mongoengine/queryset.py | 44 ++++++++++++++++++-------------------- tests/connection.py | 11 +++++++++- tests/document.py | 3 ++- tests/queryset.py | 4 ++-- 9 files changed, 81 insertions(+), 69 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 3190de6..be60db9 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -6,9 +6,11 @@ from mongoengine import signals import sys import pymongo -import pymongo.objectid +from bson import ObjectId import operator + from functools import partial +from bson.dbref import DBRef class NotRegistered(Exception): @@ -295,7 +297,7 @@ class ComplexBaseField(BaseField): self.error('You can only reference documents once they' ' have been saved to the database') collection = v._get_collection_name() - value_dict[k] = pymongo.dbref.DBRef(collection, v.pk) + value_dict[k] = DBRef(collection, v.pk) elif hasattr(v, 'to_python'): value_dict[k] = v.to_python() else: @@ -344,7 +346,7 @@ class ComplexBaseField(BaseField): value_dict[k] = GenericReferenceField().to_mongo(v) else: collection = v._get_collection_name() - value_dict[k] = pymongo.dbref.DBRef(collection, v.pk) + value_dict[k] = DBRef(collection, v.pk) elif hasattr(v, 'to_mongo'): value_dict[k] = v.to_mongo() else: @@ -447,9 +449,9 @@ class ObjectIdField(BaseField): return value def to_mongo(self, value): - if not isinstance(value, pymongo.objectid.ObjectId): + if not isinstance(value, ObjectId): try: - return pymongo.objectid.ObjectId(unicode(value)) + return ObjectId(unicode(value)) except Exception, e: # e.message attribute has been deprecated since Python 2.6 self.error(unicode(e)) @@ -460,7 +462,7 @@ class ObjectIdField(BaseField): def validate(self, value): try: - pymongo.objectid.ObjectId(unicode(value)) + ObjectId(unicode(value)) except: self.error('Invalid Object ID') diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 822c604..b6b716f 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -46,7 +46,12 @@ def register_connection(alias, name, host='localhost', port=27017, raise ConnectionError("If using URI style connection include "\ "database name in string") uri_dict['name'] = uri_dict.get('database') - _connection_settings[alias] = uri_dict + _connection_settings[alias] = { + 'host': host, + 'name': uri_dict.get('database'), + 'username': uri_dict.get('username'), + 'password': uri_dict.get('password') + } return _connection_settings[alias] = { @@ -89,11 +94,11 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): conn_settings = _connection_settings[alias].copy() if hasattr(pymongo, 'version_tuple'): # Support for 2.1+ - conn_settings.pop('name') - conn_settings.pop('slaves') - conn_settings.pop('is_slave') - conn_settings.pop('username') - conn_settings.pop('password') + conn_settings.pop('name', None) + conn_settings.pop('slaves', None) + conn_settings.pop('is_slave', None) + conn_settings.pop('username', None) + conn_settings.pop('password', None) else: # Get all the slave connections if 'slaves' in conn_settings: @@ -106,8 +111,7 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): try: _connections[alias] = Connection(**conn_settings) except Exception, e: - raise e - raise ConnectionError('Cannot connect to database %s' % alias) + raise ConnectionError("Cannot connect to database %s :\n%s" % (alias, e)) return _connections[alias] @@ -120,7 +124,6 @@ def get_db(alias=DEFAULT_CONNECTION_NAME, reconnect=False): conn = get_connection(alias) conn_settings = _connection_settings[alias] _dbs[alias] = conn[conn_settings['name']] - # Authenticate if necessary if conn_settings['username'] and conn_settings['password']: _dbs[alias].authenticate(conn_settings['username'], diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 40ceb49..a5ad916 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -1,4 +1,4 @@ -import pymongo +from bson import DBRef, SON from base import (BaseDict, BaseList, TopLevelDocumentMetaclass, get_document) from fields import ReferenceField @@ -68,9 +68,9 @@ class DeReference(object): if hasattr(item, '_fields'): for field_name, field in item._fields.iteritems(): v = item._data.get(field_name, None) - if isinstance(v, (pymongo.dbref.DBRef)): + if isinstance(v, (DBRef)): reference_map.setdefault(field.document_type, []).append(v.id) - elif isinstance(v, (dict, pymongo.son.SON)) and '_ref' in v: + elif isinstance(v, (dict, SON)) and '_ref' in v: reference_map.setdefault(get_document(v['_cls']), []).append(v['_ref'].id) elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth: field_cls = getattr(getattr(field, 'field', None), 'document_type', None) @@ -79,9 +79,9 @@ class DeReference(object): if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)): key = field_cls reference_map.setdefault(key, []).extend(refs) - elif isinstance(item, (pymongo.dbref.DBRef)): + elif isinstance(item, (DBRef)): reference_map.setdefault(item.collection, []).append(item.id) - elif isinstance(item, (dict, pymongo.son.SON)) and '_ref' in item: + elif isinstance(item, (dict, SON)) and '_ref' in item: reference_map.setdefault(get_document(item['_cls']), []).append(item['_ref'].id) elif isinstance(item, (dict, list, tuple)) and depth - 1 <= self.max_depth: references = self._find_references(item, depth - 1) @@ -138,7 +138,7 @@ class DeReference(object): else: return BaseList(items, instance, name) - if isinstance(items, (dict, pymongo.son.SON)): + if isinstance(items, (dict, SON)): if '_ref' in items: return self.object_map.get(items['_ref'].id, items) elif '_types' in items and '_cls' in items: @@ -167,9 +167,9 @@ class DeReference(object): elif hasattr(v, '_fields'): for field_name, field in v._fields.iteritems(): v = data[k]._data.get(field_name, None) - if isinstance(v, (pymongo.dbref.DBRef)): + if isinstance(v, (DBRef)): data[k]._data[field_name] = self.object_map.get(v.id, v) - elif isinstance(v, (dict, pymongo.son.SON)) and '_ref' in v: + elif isinstance(v, (dict, SON)) and '_ref' in v: data[k]._data[field_name] = self.object_map.get(v['_ref'].id, v) elif isinstance(v, dict) and depth <= self.max_depth: data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=name) diff --git a/mongoengine/document.py b/mongoengine/document.py index 686945c..6d9d314 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,11 +1,12 @@ +import pymongo +from bson.dbref import DBRef + from mongoengine import signals from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, BaseDict, BaseList) from queryset import OperationError from connection import get_db, DEFAULT_CONNECTION_NAME -import pymongo - __all__ = ['Document', 'EmbeddedDocument', 'DynamicDocument', 'DynamicEmbeddedDocument', 'OperationError', 'InvalidCollectionError'] @@ -151,7 +152,7 @@ class Document(BaseDocument): .. versionchanged:: 0.5 In existing documents it only saves changed fields using set / unset - Saves are cascaded and any :class:`~pymongo.dbref.DBRef` objects + Saves are cascaded and any :class:`~bson.dbref.DBRef` objects that have changes are saved as well. .. versionchanged:: 0.6 Cascade saves are optional = defaults to True, if you want fine grain @@ -271,7 +272,7 @@ class Document(BaseDocument): signals.post_delete.send(self.__class__, document=self) def select_related(self, max_depth=1): - """Handles dereferencing of :class:`~pymongo.dbref.DBRef` objects to + """Handles dereferencing of :class:`~bson.dbref.DBRef` objects to a maximum depth in order to cut down the number queries to mongodb. .. versionadded:: 0.5 @@ -313,12 +314,12 @@ class Document(BaseDocument): return value def to_dbref(self): - """Returns an instance of :class:`~pymongo.dbref.DBRef` useful in + """Returns an instance of :class:`~bson.dbref.DBRef` useful in `__raw__` queries.""" if not self.pk: msg = "Only saved documents can have a valid dbref" raise OperationError(msg) - return pymongo.dbref.DBRef(self.__class__._get_collection_name(), self.pk) + return DBRef(self.__class__._get_collection_name(), self.pk) @classmethod def register_delete_rule(cls, document_cls, field_name, rule): @@ -385,7 +386,7 @@ class MapReduceDocument(object): :param collection: An instance of :class:`~pymongo.Collection` :param key: Document/result key, often an instance of - :class:`~pymongo.objectid.ObjectId`. If supplied as + :class:`~bson.objectid.ObjectId`. If supplied as an ``ObjectId`` found in the given ``collection``, the object can be accessed via the ``object`` property. :param value: The result(s) for this key. diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 2e4411a..33b0ed5 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -2,13 +2,11 @@ import datetime import time import decimal import gridfs -import pymongo -import pymongo.binary -import pymongo.dbref -import pymongo.son import re import uuid +from bson import Binary, DBRef, SON, ObjectId + from base import (BaseField, ComplexBaseField, ObjectIdField, ValidationError, get_document) from queryset import DO_NOTHING, QuerySet @@ -644,7 +642,7 @@ class ReferenceField(BaseField): # Get value from document instance if available value = instance._data.get(self.name) # Dereference DBRefs - if isinstance(value, (pymongo.dbref.DBRef)): + if isinstance(value, (DBRef)): value = self.document_type._get_db().dereference(value) if value is not None: instance._data[self.name] = self.document_type._from_son(value) @@ -666,7 +664,7 @@ class ReferenceField(BaseField): id_ = id_field.to_mongo(id_) collection = self.document_type._get_collection_name() - return pymongo.dbref.DBRef(collection, id_) + return DBRef(collection, id_) def prepare_query_value(self, op, value): if value is None: @@ -675,7 +673,7 @@ class ReferenceField(BaseField): return self.to_mongo(value) def validate(self, value): - if not isinstance(value, (self.document_type, pymongo.dbref.DBRef)): + if not isinstance(value, (self.document_type, DBRef)): self.error('A ReferenceField only accepts DBRef') if isinstance(value, Document) and value.id is None: @@ -701,13 +699,13 @@ class GenericReferenceField(BaseField): return self value = instance._data.get(self.name) - if isinstance(value, (dict, pymongo.son.SON)): + if isinstance(value, (dict, SON)): instance._data[self.name] = self.dereference(value) return super(GenericReferenceField, self).__get__(instance, owner) def validate(self, value): - if not isinstance(value, (Document, pymongo.dbref.DBRef)): + if not isinstance(value, (Document, DBRef)): self.error('GenericReferences can only contain documents') # We need the id from the saved object to create the DBRef @@ -741,7 +739,7 @@ class GenericReferenceField(BaseField): id_ = id_field.to_mongo(id_) collection = document._get_collection_name() - ref = pymongo.dbref.DBRef(collection, id_) + ref = DBRef(collection, id_) return {'_cls': document._class_name, '_ref': ref} def prepare_query_value(self, op, value): @@ -760,7 +758,7 @@ class BinaryField(BaseField): super(BinaryField, self).__init__(**kwargs) def to_mongo(self, value): - return pymongo.binary.Binary(value) + return Binary(value) def to_python(self, value): # Returns str not unicode as this is binary data @@ -964,7 +962,7 @@ class FileField(BaseField): if value.grid_id is not None: if not isinstance(value, self.proxy_class): self.error('FileField only accepts GridFSProxy values') - if not isinstance(value.grid_id, pymongo.objectid.ObjectId): + if not isinstance(value.grid_id, ObjectId): self.error('Invalid GridFSProxy value') diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 0c0b11f..9032fc7 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1,16 +1,14 @@ -from connection import get_db -from mongoengine import signals - import pprint -import pymongo -import pymongo.code -import pymongo.dbref -import pymongo.objectid import re import copy import itertools import operator +import pymongo +from bson.code import Code + +from mongoengine import signals + __all__ = ['queryset_manager', 'Q', 'InvalidQueryError', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY'] @@ -935,9 +933,9 @@ class QuerySet(object): and :meth:`~mongoengine.tests.QuerySetTest.test_map_advanced` tests in ``tests.queryset.QuerySetTest`` for usage examples. - :param map_f: map function, as :class:`~pymongo.code.Code` or string + :param map_f: map function, as :class:`~bson.code.Code` or string :param reduce_f: reduce function, as - :class:`~pymongo.code.Code` or string + :class:`~bson.code.Code` or string :param output: output collection name, if set to 'inline' will try to use :class:`~pymongo.collection.Collection.inline_map_reduce` :param finalize_f: finalize function, an optional function that @@ -967,27 +965,27 @@ class QuerySet(object): raise NotImplementedError("Requires MongoDB >= 1.7.1") map_f_scope = {} - if isinstance(map_f, pymongo.code.Code): + if isinstance(map_f, Code): map_f_scope = map_f.scope map_f = unicode(map_f) - map_f = pymongo.code.Code(self._sub_js_fields(map_f), map_f_scope) + map_f = Code(self._sub_js_fields(map_f), map_f_scope) reduce_f_scope = {} - if isinstance(reduce_f, pymongo.code.Code): + if isinstance(reduce_f, Code): reduce_f_scope = reduce_f.scope reduce_f = unicode(reduce_f) reduce_f_code = self._sub_js_fields(reduce_f) - reduce_f = pymongo.code.Code(reduce_f_code, reduce_f_scope) + reduce_f = Code(reduce_f_code, reduce_f_scope) mr_args = {'query': self._query} if finalize_f: finalize_f_scope = {} - if isinstance(finalize_f, pymongo.code.Code): + if isinstance(finalize_f, Code): finalize_f_scope = finalize_f.scope finalize_f = unicode(finalize_f) finalize_f_code = self._sub_js_fields(finalize_f) - finalize_f = pymongo.code.Code(finalize_f_code, finalize_f_scope) + finalize_f = Code(finalize_f_code, finalize_f_scope) mr_args['finalize'] = finalize_f if scope: @@ -1499,7 +1497,7 @@ class QuerySet(object): query['$where'] = self._where_clause scope['query'] = query - code = pymongo.code.Code(code, scope=scope) + code = Code(code, scope=scope) db = self._document._get_db() return db.eval(code, *fields) @@ -1528,13 +1526,13 @@ class QuerySet(object): .. versionchanged:: 0.5 - updated to map_reduce as db.eval doesnt work with sharding. """ - map_func = pymongo.code.Code(""" + map_func = Code(""" function() { emit(1, this[field] || 0); } """, scope={'field': field}) - reduce_func = pymongo.code.Code(""" + reduce_func = Code(""" function(key, values) { var sum = 0; for (var i in values) { @@ -1558,14 +1556,14 @@ class QuerySet(object): .. versionchanged:: 0.5 - updated to map_reduce as db.eval doesnt work with sharding. """ - map_func = pymongo.code.Code(""" + map_func = Code(""" function() { if (this.hasOwnProperty(field)) emit(1, {t: this[field] || 0, c: 1}); } """, scope={'field': field}) - reduce_func = pymongo.code.Code(""" + reduce_func = Code(""" function(key, values) { var out = {t: 0, c: 0}; for (var i in values) { @@ -1577,7 +1575,7 @@ class QuerySet(object): } """) - finalize_func = pymongo.code.Code(""" + finalize_func = Code(""" function(key, value) { return value.t / value.c; } @@ -1719,7 +1717,7 @@ class QuerySet(object): def __repr__(self): limit = REPR_OUTPUT_SIZE + 1 - start = ( 0 if self._skip is None else self._skip ) + start = (0 if self._skip is None else self._skip) if self._limit is None: stop = start + limit if self._limit is not None: @@ -1736,7 +1734,7 @@ class QuerySet(object): return repr(data) def select_related(self, max_depth=1): - """Handles dereferencing of :class:`~pymongo.dbref.DBRef` objects to + """Handles dereferencing of :class:`~bson.dbref.DBRef` objects to a maximum depth in order to cut down the number queries to mongodb. .. versionadded:: 0.5 diff --git a/tests/connection.py b/tests/connection.py index 7ff0998..ce20a54 100644 --- a/tests/connection.py +++ b/tests/connection.py @@ -4,7 +4,7 @@ import pymongo import mongoengine.connection from mongoengine import * -from mongoengine.connection import get_db, get_connection +from mongoengine.connection import get_db, get_connection, ConnectionError class ConnectionTest(unittest.TestCase): @@ -33,6 +33,15 @@ class ConnectionTest(unittest.TestCase): def test_connect_uri(self): """Ensure that the connect() method works properly with uri's """ + c = connect(db='mongoenginetest', alias='admin') + c.admin.system.users.remove({}) + c.mongoenginetest.system.users.remove({}) + + c.admin.add_user("admin", "password") + c.admin.authenticate("admin", "password") + c.mongoenginetest.add_user("username", "password") + + self.assertRaises(ConnectionError, connect, "testdb_uri_bad", host='mongodb://test:password@localhost/mongoenginetest') connect("testdb_uri", host='mongodb://username:password@localhost/mongoenginetest') diff --git a/tests/document.py b/tests/document.py index dabc27e..b06731d 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1,5 +1,6 @@ import pickle import pymongo +import bson import unittest import warnings @@ -2222,7 +2223,7 @@ class DocumentTest(unittest.TestCase): # Test laziness self.assertTrue(isinstance(post_obj._data['author'], - pymongo.dbref.DBRef)) + bson.DBRef)) self.assertTrue(isinstance(post_obj.author, self.Person)) self.assertEqual(post_obj.author.name, 'Test User') diff --git a/tests/queryset.py b/tests/queryset.py index 555e7d0..06b65bf 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import unittest import pymongo +from bson import ObjectId from datetime import datetime, timedelta from mongoengine.queryset import (QuerySet, QuerySetManager, @@ -59,8 +60,7 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(len(people), 2) results = list(people) self.assertTrue(isinstance(results[0], self.Person)) - self.assertTrue(isinstance(results[0].id, (pymongo.objectid.ObjectId, - str, unicode))) + self.assertTrue(isinstance(results[0].id, (ObjectId, str, unicode))) self.assertEqual(results[0].name, "User A") self.assertEqual(results[0].age, 20) self.assertEqual(results[1].name, "User B") From 589a720162dd2b1cf894efa85a37cc47b49a0ffd Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 17 Feb 2012 17:09:14 +0000 Subject: [PATCH 0508/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e4d693b..4c55486 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Updated deprecated imports from pymongo (safe for pymongo 2.2) - Added uri support for connections - Added scalar for efficiently returning partial data values (aliased to values_list) - Fixed limit skip bug From ece8d251875aca723df5199d1503cbcc83f57a50 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 17 Feb 2012 17:09:48 +0000 Subject: [PATCH 0509/1279] Added replicaset connection support Provide replicaSet=NAME in your connection. fixes #423 --- docs/changelog.rst | 1 + mongoengine/connection.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4c55486..ab30f6a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added replicaSet connection support - Updated deprecated imports from pymongo (safe for pymongo 2.2) - Added uri support for connections - Added scalar for efficiently returning partial data values (aliased to values_list) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index b6b716f..79c5b5a 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,5 +1,5 @@ import pymongo -from pymongo import Connection, uri_parser +from pymongo import Connection, ReplicaSetConnection, uri_parser __all__ = ['ConnectionError', 'connect', 'register_connection', @@ -108,8 +108,11 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): conn_settings['slaves'] = slaves conn_settings.pop('read_preference') + connection_class = Connection + if 'replicaSet' in conn_settings: + connection_class = ReplicaSetConnection try: - _connections[alias] = Connection(**conn_settings) + _connections[alias] = connection_class(**conn_settings) except Exception, e: raise ConnectionError("Cannot connect to database %s :\n%s" % (alias, e)) return _connections[alias] From 855933ab2abeea28ab9ec717387aee1836994581 Mon Sep 17 00:00:00 2001 From: Robert Kajic Date: Fri, 24 Feb 2012 00:07:57 +0100 Subject: [PATCH 0510/1279] uri_dict will have the 'database' key even if the database wasn't present in the uri. We must check it's value as well. --- mongoengine/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 79c5b5a..70eac1e 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -42,7 +42,7 @@ def register_connection(alias, name, host='localhost', port=27017, # Handle uri style connections if "://" in host: uri_dict = uri_parser.parse_uri(host) - if 'database' not in uri_dict: + if uri_dict.get('database') is None: raise ConnectionError("If using URI style connection include "\ "database name in string") uri_dict['name'] = uri_dict.get('database') From 3300f409ba1971171f83a18059e6f0c5d8aeed07 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Feb 2012 10:34:51 +0000 Subject: [PATCH 0511/1279] Update Authors and changelist --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index e7597b3..41d9c66 100644 --- a/AUTHORS +++ b/AUTHORS @@ -95,4 +95,5 @@ that much better: * Chris Faulkner * Ashwin Purohit * Shalabh Aggarwal - * Chris Williams \ No newline at end of file + * Chris Williams + * Robert Kajic diff --git a/docs/changelog.rst b/docs/changelog.rst index ab30f6a..981f1e2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Added custom 2D index declarations - Added replicaSet connection support - Updated deprecated imports from pymongo (safe for pymongo 2.2) - Added uri support for connections From 6a229cfbc5a26b87a25ca8d842f8f9bf76f3b3dd Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Feb 2012 15:48:32 +0000 Subject: [PATCH 0512/1279] Updates can now take raw queries --- mongoengine/queryset.py | 3 +++ tests/queryset.py | 27 +++++++++++++++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 64b29ff..6ebaaca 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1279,6 +1279,9 @@ class QuerySet(object): mongo_update = {} for key, value in update.items(): + if key == "__raw__": + mongo_update.update(value) + continue parts = key.split('__') # Check for an operator and transform to mongo-style if there is op = None diff --git a/tests/queryset.py b/tests/queryset.py index 06b65bf..3c98d9e 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1420,20 +1420,39 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() - def test_update_pull(self): + def test_update_push_and_pull(self): """Ensure that the 'pull' update operation works correctly. """ class BlogPost(Document): slug = StringField() tags = ListField(StringField()) - post = BlogPost(slug="test", tags=['code', 'mongodb', 'code']) + BlogPost.drop_collection() + + post = BlogPost(slug="test") post.save() + BlogPost.objects.filter(id=post.id).update(push__tags="code") + post.reload() + self.assertEqual(post.tags, ["code"]) + + BlogPost.objects.filter(id=post.id).update(push_all__tags=["mongodb", "code"]) + post.reload() + self.assertEqual(post.tags, ["code", "mongodb", "code"]) + BlogPost.objects(slug="test").update(pull__tags="code") post.reload() - self.assertTrue('code' not in post.tags) - self.assertEqual(len(post.tags), 1) + self.assertEqual(post.tags, ["mongodb"]) + + + BlogPost.objects(slug="test").update(pull_all__tags=["mongodb", "code"]) + post.reload() + self.assertEqual(post.tags, []) + + BlogPost.objects(slug="test").update(__raw__={"$addToSet": {"tags": {"$each": ["code", "mongodb", "code"]}}}) + post.reload() + self.assertEqual(post.tags, ["code", "mongodb"]) + def test_update_one_pop_generic_reference(self): From e9b8093dac736d0237a0ddf77755a6210ad35428 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 29 Feb 2012 10:09:16 +0000 Subject: [PATCH 0513/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 981f1e2..7c63a16 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Updates can now take __raw__ queries - Added custom 2D index declarations - Added replicaSet connection support - Updated deprecated imports from pymongo (safe for pymongo 2.2) From 2a391f0f16c389ff2039643b3b4c511914cd51a6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 29 Feb 2012 10:10:51 +0000 Subject: [PATCH 0514/1279] Raise an error if trying to perform a join You can't join across reference fields, so raise an error if someone tries to. --- docs/changelog.rst | 1 + mongoengine/queryset.py | 3 +++ tests/document.py | 19 ++++++++++++++++++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7c63a16..6e1afaa 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Errors raised if trying to perform a join in a query - Updates can now take __raw__ queries - Added custom 2D index declarations - Added replicaSet connection support diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6ebaaca..3e4d474 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -610,6 +610,9 @@ class QuerySet(object): raise InvalidQueryError('Cannot resolve field "%s"' % field_name) else: + from mongoengine.fields import ReferenceField, GenericReferenceField + if isinstance(field, (ReferenceField, GenericReferenceField)): + raise InvalidQueryError('Cannot perform join in mongoDB: %s' % '__'.join(parts)) # Look up subfield on the previous field new_field = field.lookup_member(field_name) from base import ComplexBaseField diff --git a/tests/document.py b/tests/document.py index fc9350b..3573402 100644 --- a/tests/document.py +++ b/tests/document.py @@ -10,6 +10,7 @@ from fixtures import Base, Mixin, PickleEmbedded, PickleTest from mongoengine import * from mongoengine.base import NotRegistered, InvalidDocumentError +from mongoengine.queryset import InvalidQueryError from mongoengine.connection import get_db @@ -640,7 +641,7 @@ class DocumentTest(unittest.TestCase): location = DictField() meta = { 'indexes': [ - '*location.point', + '*location.point', ], } Place.drop_collection() @@ -2256,6 +2257,22 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + def test_cannot_perform_joins_references(self): + + class BlogPost(Document): + author = ReferenceField(self.Person) + author2 = GenericReferenceField() + + def test_reference(): + list(BlogPost.objects(author__name="test")) + + self.assertRaises(InvalidQueryError, test_reference) + + def test_generic_reference(): + list(BlogPost.objects(author2__name="test")) + + self.assertRaises(InvalidQueryError, test_generic_reference) + def test_duplicate_db_fields_raise_invalid_document_error(self): """Ensure a InvalidDocumentError is thrown if duplicate fields declare the same db_field""" From 1d7ea71c0d3444d80fc4a1b689b7fcb5fd36f637 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 29 Feb 2012 10:31:33 +0000 Subject: [PATCH 0515/1279] DeReference is now used in a thread safe manner No global / module instance is needed Fixes #399 --- mongoengine/base.py | 4 ++-- mongoengine/dereference.py | 2 -- mongoengine/document.py | 4 ++-- mongoengine/queryset.py | 8 ++++---- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index be60db9..aa79b40 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -234,8 +234,8 @@ class ComplexBaseField(BaseField): return self if not self._dereference and instance._initialised: - from dereference import dereference - self._dereference = dereference # Cached + from dereference import DeReference + self._dereference = DeReference() # Cached instance._data[self.name] = self._dereference( instance._data.get(self.name), max_depth=1, instance=instance, name=self.name diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index a5ad916..cb5079f 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -186,5 +186,3 @@ class DeReference(object): return BaseDict(data, instance, name) depth += 1 return data - -dereference = DeReference() diff --git a/mongoengine/document.py b/mongoengine/document.py index 6d9d314..5ef93e3 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -277,8 +277,8 @@ class Document(BaseDocument): .. versionadded:: 0.5 """ - from dereference import dereference - self._data = dereference(self._data, max_depth) + from dereference import DeReference + self._data = DeReference()(self._data, max_depth) return self def reload(self, max_depth=1): diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 3e4d474..0b73b87 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1095,8 +1095,8 @@ class QuerySet(object): .. versionadded:: 0.4 .. versionchanged:: 0.5 - Fixed handling references """ - from dereference import dereference - return dereference(self._cursor.distinct(field), 1) + from dereference import DeReference + return DeReference()(self._cursor.distinct(field), 1) def only(self, *fields): """Load only a subset of this document's fields. :: @@ -1747,10 +1747,10 @@ class QuerySet(object): .. versionadded:: 0.5 """ - from dereference import dereference + from dereference import DeReference # Make select related work the same for querysets max_depth += 1 - return dereference(self, max_depth=max_depth) + return DeReference()(self, max_depth=max_depth) class QuerySetManager(object): From 2afa2171f942ca211e6468ff53b89d4e7eafc86f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 29 Feb 2012 10:32:58 +0000 Subject: [PATCH 0516/1279] Updated Changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6e1afaa..18b94d3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- DeReferencing is now thread safe - Errors raised if trying to perform a join in a query - Updates can now take __raw__ queries - Added custom 2D index declarations From 44b9fb66e170c6a8d0a9d330e2f77117508eccf3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 29 Feb 2012 11:04:09 +0000 Subject: [PATCH 0517/1279] Updates must have an operation Closes #387 --- docs/changelog.rst | 1 + mongoengine/queryset.py | 5 ++++- tests/document.py | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 18b94d3..5baa761 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- Error raised if update doesn't have an operation - DeReferencing is now thread safe - Errors raised if trying to perform a join in a query - Updates can now take __raw__ queries diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 0b73b87..ade5279 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1332,11 +1332,14 @@ class QuerySet(object): key = '.'.join(parts) + if not op: + raise InvalidQueryError("Updates must supply an operation eg: set__FIELD=value") + if op: value = {key: value} key = '$' + op - if op is None or key not in mongo_update: + if key not in mongo_update: mongo_update[key] = value elif key in mongo_update and isinstance(mongo_update[key], dict): mongo_update[key].update(value) diff --git a/tests/document.py b/tests/document.py index 3573402..252393c 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1432,6 +1432,12 @@ class DocumentTest(unittest.TestCase): self.assertRaises(OperationError, update_no_value_raises) + def update_no_op_raises(): + person = self.Person.objects.first() + person.update(name="Dan") + + self.assertRaises(InvalidQueryError, update_no_op_raises) + def test_embedded_update(self): """ Test update on `EmbeddedDocumentField` fields From 5a1eaa0a98e802d9fec8b4e268f8da813eda67fa Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 29 Feb 2012 11:23:43 +0000 Subject: [PATCH 0518/1279] Updated new deref test --- tests/dereference.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/tests/dereference.py b/tests/dereference.py index a23d78f..8a4b310 100644 --- a/tests/dereference.py +++ b/tests/dereference.py @@ -783,41 +783,30 @@ class FieldTest(unittest.TestCase): root = root.reload() self.assertEquals(root.children, [company]) self.assertEquals(company.parents, [root]) - + def test_dict_in_dbref_instance(self): class Person(Document): name = StringField(max_length=250, required=True) - meta = { "ordering" : "name" } - class Room(Document): number = StringField(max_length=250, required=True) staffs_with_position = ListField(DictField()) - meta = { "ordering" : "number" } - - - Person.drop_collection() + Room.drop_collection() - # 201 bob = Person.objects.create(name='Bob') bob.save() - keven = Person.objects.create(name='Keven') - keven.save() sarah = Person.objects.create(name='Sarah') sarah.save() - room_201 = Room.objects.create( number = "201") - room_201.staffs_with_position = [ {'position_key': 'window', 'staff' : sarah.to_dbref() }, - { 'position_key': 'door', 'staff': bob.to_dbref() }, - { 'position_key': 'center' , 'staff' : keven.to_dbref() } ] - room_201.save() + room_101 = Room.objects.create(number="101") + room_101.staffs_with_position = [ + {'position_key': 'window', 'staff': sarah}, + {'position_key': 'door', 'staff': bob.to_dbref()}] + room_101.save() room = Room.objects.first().select_related() - room.to_mongo() - - - - + self.assertEquals(room.staffs_with_position[0]['staff'], sarah) + self.assertEquals(room.staffs_with_position[1]['staff'], bob) From a2eb876f8c3b2d72124f97cfeac07f9f43100889 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 29 Feb 2012 11:39:10 +0000 Subject: [PATCH 0519/1279] No longer always upsert on save closes #407 --- docs/changelog.rst | 1 + mongoengine/document.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5baa761..5c8f297 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in dev ============== +- No longer always upsert on save for items with a '_id' - Error raised if update doesn't have an operation - DeReferencing is now thread safe - Errors raised if trying to perform a join in a query diff --git a/mongoengine/document.py b/mongoengine/document.py index 5ef93e3..68668a7 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -191,10 +191,11 @@ class Document(BaseDocument): actual_key = self._db_field_map.get(k, k) select_dict[actual_key] = doc[actual_key] + upsert = self._created if updates: - collection.update(select_dict, {"$set": updates}, upsert=True, safe=safe, **write_options) + collection.update(select_dict, {"$set": updates}, upsert=upsert, safe=safe, **write_options) if removals: - collection.update(select_dict, {"$unset": removals}, upsert=True, safe=safe, **write_options) + collection.update(select_dict, {"$unset": removals}, upsert=upsert, safe=safe, **write_options) cascade = self._meta.get('cascade', True) if cascade is None else cascade if cascade: @@ -219,6 +220,7 @@ class Document(BaseDocument): self[id_field] = self._fields[id_field].to_python(object_id) self._changed_fields = [] + self._created = False signals.post_save.send(self.__class__, document=self, created=created) def cascade_save(self, *args, **kwargs): From ae20c785eacfdcf53b3d5833bf5db58ac52b079a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 29 Feb 2012 12:04:43 +0000 Subject: [PATCH 0520/1279] Updated docs --- docs/changelog.rst | 2 +- docs/guide/defining-documents.rst | 10 +++++++++- docs/index.rst | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5c8f297..e478bef 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,7 @@ Changelog ========= -Changes in dev +Changes in 0.6 ============== - No longer always upsert on save for items with a '_id' diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index a7d48f7..1565c50 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -431,6 +431,12 @@ If a dictionary is passed then the following options are available: :attr:`unique` (Default: False) Whether the index should be sparse. +.. warning:: + + + Inheritance adds extra indices. + If don't need inheritance for a document turn inheritance off - see :ref:`document-inheritance`. + Geospatial indexes --------------------------- @@ -447,7 +453,7 @@ point. To create a geospatial index you must prefix the field with the location = DictField() meta = { 'indexes': [ - '*location.point', + '*location.point', ], } @@ -511,6 +517,8 @@ This ensures that the shard key is sent with the query when calling the 'shard_key': ('machine', 'timestamp',) } +.. _document-inheritance: + Document inheritance ==================== diff --git a/docs/index.rst b/docs/index.rst index ef225e4..a701de2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,7 +19,7 @@ MongoDB. To install it, simply run The complete API documentation. :doc:`upgrade` - The Upgrade guide. + How to upgrade MongoEngine. :doc:`django` Using MongoEngine and Django From 16395762030eef2721a0bfe4c10eab5a9a0f3090 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 29 Feb 2012 12:05:47 +0000 Subject: [PATCH 0521/1279] Bumped version --- mongoengine/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index d10d113..decfbb3 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,9 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -__author__ = 'Harry Marr' - -VERSION = (0, 5, 3) +VERSION = (0, 6, 0) def get_version(): From bdf7187d5c481f5e68d2b9ce7e9b63087c30559c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 29 Feb 2012 14:57:24 +0000 Subject: [PATCH 0522/1279] Added a limit to .get --- mongoengine/queryset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index ade5279..3eac1af 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -748,6 +748,7 @@ class QuerySet(object): .. versionadded:: 0.3 """ + self.limit(2) self.__call__(*q_objs, **query) try: result1 = self.next() From 32fc4152a71f931f2e3a43ef215f8826488edb21 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 2 Mar 2012 13:42:24 +0000 Subject: [PATCH 0523/1279] Enable covered indexes for simple documents. Refs #444 --- docs/changelog.rst | 1 + mongoengine/queryset.py | 13 ++++++++++--- tests/document.py | 29 ++++++++++++++++++++++++++++- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e478bef..43eac27 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.6 ============== +- Added support for covered indexes when inheritance is off - No longer always upsert on save for items with a '_id' - Error raised if update doesn't have an operation - DeReferencing is now thread safe diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 3eac1af..85a5152 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -273,16 +273,20 @@ class Q(QNode): class QueryFieldList(object): """Object that handles combinations of .only() and .exclude() calls""" - ONLY = True - EXCLUDE = False + ONLY = 1 + EXCLUDE = 0 def __init__(self, fields=[], value=ONLY, always_include=[]): self.value = value self.fields = set(fields) self.always_include = set(always_include) + self._id = None def as_dict(self): - return dict((field, self.value) for field in self.fields) + field_list = dict((field, self.value) for field in self.fields) + if self._id is not None: + field_list['_id'] = self._id + return field_list def __add__(self, f): if not self.fields: @@ -298,6 +302,9 @@ class QueryFieldList(object): self.value = self.ONLY self.fields = f.fields - self.fields + if '_id' in f.fields: + self._id = f.value + if self.always_include: if self.value is self.ONLY and self.fields: self.fields = self.fields.union(self.always_include) diff --git a/tests/document.py b/tests/document.py index 252393c..b6c72c5 100644 --- a/tests/document.py +++ b/tests/document.py @@ -714,7 +714,6 @@ class DocumentTest(unittest.TestCase): self.assertEqual(info.keys(), ['_types_1_user_guid_1', '_id_', '_types_1_name_1']) Person.drop_collection() - def test_embedded_document_index(self): """Tests settings an index on an embedded document """ @@ -788,6 +787,34 @@ class DocumentTest(unittest.TestCase): self.assertEquals(len(User._geo_indices()), 2) + def test_covered_index(self): + """Ensure that covered indexes can be used + """ + + class Test(Document): + a = IntField() + + meta = { + 'indexes': ['a'], + 'allow_inheritance': False + } + + Test.drop_collection() + + obj = Test(a=1) + obj.save() + + # Need to be explicit about covered indexes as mongoDB doesn't know if + # the documents returned might have more keys in that here. + query_plan = Test.objects(id=obj.id).exclude('a').explain() + self.assertFalse(query_plan['indexOnly']) + + query_plan = Test.objects(id=obj.id).only('id').explain() + self.assertTrue(query_plan['indexOnly']) + + query_plan = Test.objects(a=1).only('a').exclude('id').explain() + self.assertTrue(query_plan['indexOnly']) + def test_hint(self): class BlogPost(Document): From 3c7bf500894f1475fedecc2627f8533b583dd1b9 Mon Sep 17 00:00:00 2001 From: Robert Kajic Date: Tue, 28 Feb 2012 23:18:22 +0100 Subject: [PATCH 0524/1279] Make uri style connections use parameters not specified in the uri, as well as other keyword arguments --- mongoengine/connection.py | 14 ++++++-------- tests/connection.py | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 70eac1e..815ffa0 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -45,14 +45,12 @@ def register_connection(alias, name, host='localhost', port=27017, if uri_dict.get('database') is None: raise ConnectionError("If using URI style connection include "\ "database name in string") - uri_dict['name'] = uri_dict.get('database') - _connection_settings[alias] = { - 'host': host, - 'name': uri_dict.get('database'), - 'username': uri_dict.get('username'), - 'password': uri_dict.get('password') - } - return + node = uri_dict['nodelist'][0] + host = node[0] + port = node[1] + name = uri_dict.get('database') + username = uri_dict.get('username') or username + password = uri_dict.get('password') or password _connection_settings[alias] = { 'name': name, diff --git a/tests/connection.py b/tests/connection.py index ce20a54..9aad142 100644 --- a/tests/connection.py +++ b/tests/connection.py @@ -41,7 +41,7 @@ class ConnectionTest(unittest.TestCase): c.admin.authenticate("admin", "password") c.mongoenginetest.add_user("username", "password") - self.assertRaises(ConnectionError, connect, "testdb_uri_bad", host='mongodb://test:password@localhost/mongoenginetest') + self.assertRaises(ConnectionError, connect, "testdb_uri_bad", host='mongodb://test:password@localhost') connect("testdb_uri", host='mongodb://username:password@localhost/mongoenginetest') From c61de6540a12c1365cff50b32c3c5db0be2a3580 Mon Sep 17 00:00:00 2001 From: Robert Kajic Date: Fri, 2 Mar 2012 11:37:45 +0100 Subject: [PATCH 0525/1279] Don't ignore kwargs for uri style connections --- mongoengine/connection.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 815ffa0..dbde9e6 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -45,12 +45,14 @@ def register_connection(alias, name, host='localhost', port=27017, if uri_dict.get('database') is None: raise ConnectionError("If using URI style connection include "\ "database name in string") - node = uri_dict['nodelist'][0] - host = node[0] - port = node[1] - name = uri_dict.get('database') - username = uri_dict.get('username') or username - password = uri_dict.get('password') or password + _connection_settings[alias] = { + 'host': host, + 'name': uri_dict.get('database'), + 'username': uri_dict.get('username'), + 'password': uri_dict.get('password') + } + _connection_settings[alias].update(kwargs) + return _connection_settings[alias] = { 'name': name, @@ -62,7 +64,6 @@ def register_connection(alias, name, host='localhost', port=27017, 'password': password, 'read_preference': read_preference } - _connection_settings[alias].update(kwargs) From c272b7901faf514e9456f27d641e0d5f605fac1a Mon Sep 17 00:00:00 2001 From: Robert Kajic Date: Fri, 2 Mar 2012 11:15:31 +0100 Subject: [PATCH 0526/1279] Fix for bug where changes to a a embedded document field are not recorded if the root document was just created+saved. --- mongoengine/document.py | 4 ++++ tests/document.py | 36 ++++++++++++++++++++++++++++++++---- tests/dynamic_document.py | 6 ++++-- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 68668a7..4b5506f 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -24,6 +24,10 @@ class EmbeddedDocument(BaseDocument): __metaclass__ = DocumentMetaclass + def __init__(self, *args, **kwargs): + super(EmbeddedDocument, self).__init__(*args, **kwargs) + self._changed_fields = [] + def __delattr__(self, *args, **kwargs): """Handle deletions of fields""" field_name = args[0] diff --git a/tests/document.py b/tests/document.py index b6c72c5..a9bb70c 100644 --- a/tests/document.py +++ b/tests/document.py @@ -1599,14 +1599,16 @@ class DocumentTest(unittest.TestCase): self.assertEquals(doc._get_changed_fields(), ['embedded_field']) embedded_delta = { - '_types': ['Embedded'], - '_cls': 'Embedded', 'string_field': 'hello', 'int_field': 1, 'dict_field': {'hello': 'world'}, 'list_field': ['1', 2, {'hello': 'world'}] } self.assertEquals(doc.embedded_field._delta(), (embedded_delta, {})) + embedded_delta.update({ + '_types': ['Embedded'], + '_cls': 'Embedded', + }) self.assertEquals(doc._delta(), ({'embedded_field': embedded_delta}, {})) doc.save() @@ -1832,14 +1834,16 @@ class DocumentTest(unittest.TestCase): self.assertEquals(doc._get_changed_fields(), ['db_embedded_field']) embedded_delta = { - '_types': ['Embedded'], - '_cls': 'Embedded', 'db_string_field': 'hello', 'db_int_field': 1, 'db_dict_field': {'hello': 'world'}, 'db_list_field': ['1', 2, {'hello': 'world'}] } self.assertEquals(doc.embedded_field._delta(), (embedded_delta, {})) + embedded_delta.update({ + '_types': ['Embedded'], + '_cls': 'Embedded', + }) self.assertEquals(doc._delta(), ({'db_embedded_field': embedded_delta}, {})) doc.save() @@ -2164,6 +2168,30 @@ class DocumentTest(unittest.TestCase): # Ensure that the 'details' embedded object saved correctly self.assertEqual(employee_obj['details']['position'], 'Developer') + def test_embedded_update_after_save(self): + """ + Test update of `EmbeddedDocumentField` attached to a newly saved + document. + """ + class Page(EmbeddedDocument): + log_message = StringField(verbose_name="Log message", + required=True) + + class Site(Document): + page = EmbeddedDocumentField(Page) + + + Site.drop_collection() + site = Site(page=Page(log_message="Warning: Dummy message")) + site.save() + + # Update + site.page.log_message = "Error: Dummy message" + site.save() + + site = Site.objects.first() + self.assertEqual(site.page.log_message, "Error: Dummy message") + def test_updating_an_embedded_document(self): """Ensure that a document with an embedded document field may be saved in the database. diff --git a/tests/dynamic_document.py b/tests/dynamic_document.py index 0eeedfa..ec0ee1d 100644 --- a/tests/dynamic_document.py +++ b/tests/dynamic_document.py @@ -335,14 +335,16 @@ class DynamicDocTest(unittest.TestCase): self.assertEquals(doc._get_changed_fields(), ['embedded_field']) embedded_delta = { - '_types': ['Embedded'], - '_cls': 'Embedded', 'string_field': 'hello', 'int_field': 1, 'dict_field': {'hello': 'world'}, 'list_field': ['1', 2, {'hello': 'world'}] } self.assertEquals(doc.embedded_field._delta(), (embedded_delta, {})) + embedded_delta.update({ + '_types': ['Embedded'], + '_cls': 'Embedded', + }) self.assertEquals(doc._delta(), ({'embedded_field': embedded_delta}, {})) doc.save() From b7d0d8f0ccf60c417f3b49da91e6fefb2da3bc3f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 5 Mar 2012 11:20:22 +0000 Subject: [PATCH 0527/1279] Added warning to SortedListField --- mongoengine/fields.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 33b0ed5..14fcca0 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -508,6 +508,12 @@ class SortedListField(ListField): the database in order to ensure that a sorted list is always retrieved. + .. warning:: + There is a potential race condition when handling lists. If you set / + save the whole list then other processes trying to save the whole list + as well could overwrite changes. The safest way to append to a list is + to perform a push operation. + .. versionadded:: 0.4 .. versionchanged:: 0.6 - added reverse keyword """ From 6ecdc7b59d787ba06413072a8951255812ea2a42 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 5 Mar 2012 11:25:13 +0000 Subject: [PATCH 0528/1279] Added FutureWarning for inherited classes not declaring allow_inheritance Refs #437 --- docs/changelog.rst | 1 + mongoengine/base.py | 15 +++++++++++++-- setup.py | 2 +- tests/document.py | 38 ++++++++++++++++++++++++++++++++++---- tests/dynamic_document.py | 1 + tests/fixtures.py | 2 +- tests/queryset.py | 1 + 7 files changed, 52 insertions(+), 8 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 43eac27..f0a95d8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.6 ============== +- Added FutureWarning to inherited classes not declaring 'allow_inheritance' as the default will change in 0.7 - Added support for covered indexes when inheritance is off - No longer always upsert on save for items with a '_id' - Error raised if update doesn't have an operation diff --git a/mongoengine/base.py b/mongoengine/base.py index aa79b40..04b3f1f 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1,3 +1,5 @@ +import warnings + from queryset import QuerySet, QuerySetManager from queryset import DoesNotExist, MultipleObjectsReturned from queryset import DO_NOTHING @@ -116,7 +118,6 @@ class BaseField(object): validation=None, choices=None, verbose_name=None, help_text=None): self.db_field = (db_field or name) if not primary_key else '_id' if name: - import warnings msg = "Fields' 'name' attribute deprecated in favour of 'db_field'" warnings.warn(msg, DeprecationWarning) self.name = None @@ -471,7 +472,6 @@ class DocumentMetaclass(type): """Metaclass for all documents. """ - def __new__(cls, name, bases, attrs): def _get_mixin_fields(base): attrs = {} @@ -512,6 +512,13 @@ class DocumentMetaclass(type): # inheritance may be disabled to remove dependency on # additional fields _cls and _types class_name.append(base._class_name) + if not base._meta.get('allow_inheritance_defined', True): + warnings.warn( + "%s uses inheritance, the default for allow_inheritance " + "is changing to off by default. Please add it to the " + "document meta." % name, + FutureWarning + ) if base._meta.get('allow_inheritance', True) == False: raise ValueError('Document %s may not be subclassed' % base.__name__) @@ -673,6 +680,10 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): 'delete_rules': {}, 'allow_inheritance': True } + + allow_inheritance_defined = ('allow_inheritance' in base_meta or + 'allow_inheritance'in attrs.get('meta', {})) + meta['allow_inheritance_defined'] = allow_inheritance_defined meta.update(base_meta) # Apply document-defined meta options diff --git a/setup.py b/setup.py index 7a69d83..d92c491 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ setup(name='mongoengine', author='Harry Marr', author_email='harry.marr@{nospam}gmail.com', maintainer="Ross Lawley", - maintainer_email="ross.lawley@gmail.com", + maintainer_email="ross.lawley@{nospam}gmail.com", url='http://mongoengine.org/', license='MIT', include_package_data=True, diff --git a/tests/document.py b/tests/document.py index a9bb70c..910a78f 100644 --- a/tests/document.py +++ b/tests/document.py @@ -23,11 +23,32 @@ class DocumentTest(unittest.TestCase): class Person(Document): name = StringField() age = IntField() + + meta = {'allow_inheritance': True} + self.Person = Person def tearDown(self): self.Person.drop_collection() + def test_future_warning(self): + """Add FutureWarning for future allow_inhertiance default change. + """ + + with warnings.catch_warnings(True) as errors: + + class SimpleBase(Document): + a = IntField() + + class InheritedClass(SimpleBase): + b = IntField() + + InheritedClass() + self.assertEquals(len(errors), 1) + warning = errors[0] + self.assertEquals(FutureWarning, warning.category) + self.assertTrue("InheritedClass" in warning.message.message) + def test_drop_collection(self): """Ensure that the collection may be dropped from the database. """ @@ -145,7 +166,8 @@ class DocumentTest(unittest.TestCase): def test_get_superclasses(self): """Ensure that the correct list of superclasses is assembled. """ - class Animal(Document): pass + class Animal(Document): + meta = {'allow_inheritance': True} class Fish(Animal): pass class Mammal(Animal): pass class Human(Mammal): pass @@ -194,7 +216,8 @@ class DocumentTest(unittest.TestCase): def test_polymorphic_queries(self): """Ensure that the correct subclasses are returned from a query""" - class Animal(Document): pass + class Animal(Document): + meta = {'allow_inheritance': True} class Fish(Animal): pass class Mammal(Animal): pass class Human(Mammal): pass @@ -223,7 +246,8 @@ class DocumentTest(unittest.TestCase): """Ensure that the correct subclasses are returned from a query when using references / generic references """ - class Animal(Document): pass + class Animal(Document): + meta = {'allow_inheritance': True} class Fish(Animal): pass class Mammal(Animal): pass class Human(Mammal): pass @@ -302,7 +326,8 @@ class DocumentTest(unittest.TestCase): self.Person._get_collection_name()) # Ensure that MRO error is not raised - class A(Document): pass + class A(Document): + meta = {'allow_inheritance': True} class B(A): pass class C(B): pass @@ -597,6 +622,7 @@ class DocumentTest(unittest.TestCase): 'tags', ('category', '-date') ], + 'allow_inheritance': True } BlogPost.drop_collection() @@ -996,6 +1022,8 @@ class DocumentTest(unittest.TestCase): username = StringField(primary_key=True) name = StringField() + meta = {'allow_inheritance': True} + User.drop_collection() self.assertEqual(User._fields['username'].db_field, '_id') @@ -1045,6 +1073,8 @@ class DocumentTest(unittest.TestCase): class Place(Document): name = StringField() + meta = {'allow_inheritance': True} + class NicePlace(Place): pass diff --git a/tests/dynamic_document.py b/tests/dynamic_document.py index ec0ee1d..6ff199a 100644 --- a/tests/dynamic_document.py +++ b/tests/dynamic_document.py @@ -12,6 +12,7 @@ class DynamicDocTest(unittest.TestCase): class Person(DynamicDocument): name = StringField() + meta = {'allow_inheritance': True} Person.drop_collection() diff --git a/tests/fixtures.py b/tests/fixtures.py index a97e8eb..fd9062e 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -20,4 +20,4 @@ class Mixin(object): class Base(Document): - pass + meta = {'allow_inheritance': True} diff --git a/tests/queryset.py b/tests/queryset.py index 3c98d9e..9704895 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -20,6 +20,7 @@ class QuerySetTest(unittest.TestCase): class Person(Document): name = StringField() age = IntField() + meta = {'allow_inheritance': True} self.Person = Person def test_initialisation(self): From 69251e5000e48ea03db0444ade937cf32017db7a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 5 Mar 2012 11:35:12 +0000 Subject: [PATCH 0529/1279] Updated docs --- docs/guide/defining-documents.rst | 5 +++++ docs/upgrade.rst | 3 +++ 2 files changed, 8 insertions(+) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 1565c50..f93b458 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -533,10 +533,15 @@ convenient and efficient retrieval of related documents:: class Page(Document): title = StringField(max_length=200, required=True) + meta = {'allow_inheritance': True} + # Also stored in the collection named 'page' class DatedPage(Page): date = DateTimeField() +.. note:: From 0.7 onwards you must declare `allow_inheritance` in the document +meta. + Working with existing data -------------------------- To enable correct retrieval of documents involved in this kind of heirarchy, diff --git a/docs/upgrade.rst b/docs/upgrade.rst index d406688..0dec8c8 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -15,6 +15,9 @@ Document._get_subclasses - Is no longer used and the class method has been remov Document.objects.with_id - now raises an InvalidQueryError if used with a filter. +FutureWarning - A future warning has been added to all inherited classes that +don't define `allow_inheritance` in their meta. + 0.4 to 0.5 =========== From 7564bbdee8a807014d3e0f67ccccdb163163b0d4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 5 Mar 2012 11:43:54 +0000 Subject: [PATCH 0530/1279] Fix docs --- docs/conf.py | 4 ++-- docs/guide/defining-documents.rst | 4 ++-- docs/upgrade.rst | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 03ba047..62fa150 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -38,7 +38,7 @@ master_doc = 'index' # General information about the project. project = u'MongoEngine' -copyright = u'2009-2011, Harry Marr' +copyright = u'2009-2012, MongoEngine Authors' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -121,7 +121,7 @@ html_theme_path = ['_themes'] # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index f93b458..1e404c0 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -539,8 +539,8 @@ convenient and efficient retrieval of related documents:: class DatedPage(Page): date = DateTimeField() -.. note:: From 0.7 onwards you must declare `allow_inheritance` in the document -meta. +.. note:: From 0.7 onwards you must declare `allow_inheritance` in the document meta. + Working with existing data -------------------------- diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 0dec8c8..1609bb8 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -25,7 +25,7 @@ There have been the following backwards incompatibilities from 0.4 to 0.5. The main areas of changed are: choices in fields, map_reduce and collection names. Choice options: --------------- +--------------- Are now expected to be an iterable of tuples, with the first element in each tuple being the actual value to be stored. The second element is the From 30d9347272ae561d9af066f8312d44b133cb0d86 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 5 Mar 2012 12:20:59 +0000 Subject: [PATCH 0531/1279] Updated readme --- README.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index f6866e3..0b0cb82 100644 --- a/README.rst +++ b/README.rst @@ -3,12 +3,13 @@ MongoEngine =========== :Info: MongoEngine is an ORM-like layer on top of PyMongo. :Author: Harry Marr (http://github.com/hmarr) +:Maintainer: Ross Lawley (http://github.com/rozza) About ===== -MongoEngine is a Python Object-Document Mapper for working with MongoDB. -Documentation available at http://hmarr.com/mongoengine/ - there is currently -a `tutorial `_, a `user guide +MongoEngine is a Python Object-Document Mapper for working with MongoDB. +Documentation available at http://hmarr.com/mongoengine/ - there is currently +a `tutorial `_, a `user guide `_ and an `API reference `_. @@ -84,9 +85,9 @@ the standard port, and run ``python setup.py test``. Community ========= -- `MongoEngine Users mailing list +- `MongoEngine Users mailing list `_ -- `MongoEngine Developers mailing list +- `MongoEngine Developers mailing list `_ - `#mongoengine IRC channel `_ From fcdb0eff8f9c4fd14db52842dec543b50077e7a1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 5 Mar 2012 12:21:53 +0000 Subject: [PATCH 0532/1279] Added for read the docs --- requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8c7d698 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +pymongo \ No newline at end of file From 61411bb2597f71a32384b64660f8f8a69602c2b1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 5 Mar 2012 12:26:44 +0000 Subject: [PATCH 0533/1279] Doc updates --- README.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 0b0cb82..a8588c8 100644 --- a/README.rst +++ b/README.rst @@ -8,16 +8,16 @@ MongoEngine About ===== MongoEngine is a Python Object-Document Mapper for working with MongoDB. -Documentation available at http://hmarr.com/mongoengine/ - there is currently -a `tutorial `_, a `user guide -`_ and an `API reference -`_. +Documentation available at http://mongoengine-odm.rtfd.org - there is currently +a `tutorial `_, a `user guide +`_ and an `API reference +`_. Installation ============ If you have `setuptools `_ you can use ``easy_install -U mongoengine``. Otherwise, you can download the -source from `GitHub `_ and run ``python +source from `GitHub `_ and run ``python setup.py install``. Dependencies @@ -93,6 +93,6 @@ Community Contributing ============ -The source is available on `GitHub `_ - to +The source is available on `GitHub `_ - to contribute to the project, fork it on GitHub and send a pull request, all contributions and suggestions are welcome! From 9451c9f3312520ed0439d6a2189c7ad8071e124a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 5 Mar 2012 13:14:36 +0000 Subject: [PATCH 0534/1279] Updated Readme - points to readthedocs --- README.rst | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index f6866e3..a8588c8 100644 --- a/README.rst +++ b/README.rst @@ -3,20 +3,21 @@ MongoEngine =========== :Info: MongoEngine is an ORM-like layer on top of PyMongo. :Author: Harry Marr (http://github.com/hmarr) +:Maintainer: Ross Lawley (http://github.com/rozza) About ===== -MongoEngine is a Python Object-Document Mapper for working with MongoDB. -Documentation available at http://hmarr.com/mongoengine/ - there is currently -a `tutorial `_, a `user guide -`_ and an `API reference -`_. +MongoEngine is a Python Object-Document Mapper for working with MongoDB. +Documentation available at http://mongoengine-odm.rtfd.org - there is currently +a `tutorial `_, a `user guide +`_ and an `API reference +`_. Installation ============ If you have `setuptools `_ you can use ``easy_install -U mongoengine``. Otherwise, you can download the -source from `GitHub `_ and run ``python +source from `GitHub `_ and run ``python setup.py install``. Dependencies @@ -84,14 +85,14 @@ the standard port, and run ``python setup.py test``. Community ========= -- `MongoEngine Users mailing list +- `MongoEngine Users mailing list `_ -- `MongoEngine Developers mailing list +- `MongoEngine Developers mailing list `_ - `#mongoengine IRC channel `_ Contributing ============ -The source is available on `GitHub `_ - to +The source is available on `GitHub `_ - to contribute to the project, fork it on GitHub and send a pull request, all contributions and suggestions are welcome! From 82446d641e83ea8d3497474c19e0302c8ab678d3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 5 Mar 2012 14:28:33 +0000 Subject: [PATCH 0535/1279] Updated the spec --- python-mongoengine.spec | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python-mongoengine.spec b/python-mongoengine.spec index d528d7a..d094082 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,13 +5,13 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.5.3 +Version: 0.6 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB Group: Development/Libraries License: MIT -URL: https://github.com/namlook/mongoengine +URL: https://github.com/MongoEngine/mongoengine Source0: %{srcname}-%{version}.tar.bz2 BuildRequires: python-devel @@ -51,6 +51,8 @@ rm -rf $RPM_BUILD_ROOT # %{python_sitearch}/* %changelog +* Mon Mar 05 2012 Ross Lawley 0.6 +- 0.6 released * Thu Oct 27 2011 Pau Aliagas 0.5.3-1 - Update to latest dev version - Add PIL dependency for ImageField From c34e79fad993946855232eb36f3491ac63e4eeb3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 5 Mar 2012 16:15:06 +0000 Subject: [PATCH 0536/1279] Fix replicaset connection --- mongoengine/connection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index dbde9e6..3efb7d3 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -109,6 +109,7 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): connection_class = Connection if 'replicaSet' in conn_settings: + conn_settings['hosts_or_uri'] = conn_settings.pop('host', None) connection_class = ReplicaSetConnection try: _connections[alias] = connection_class(**conn_settings) From 859e9b3cc4ffa749a1dd45db7e11a7840482811f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 5 Mar 2012 16:16:37 +0000 Subject: [PATCH 0537/1279] Version bump --- docs/changelog.rst | 4 ++-- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f0a95d8..7374884 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,8 +2,8 @@ Changelog ========= -Changes in 0.6 -============== +Changes in 0.6.x +================ - Added FutureWarning to inherited classes not declaring 'allow_inheritance' as the default will change in 0.7 - Added support for covered indexes when inheritance is off diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index decfbb3..7b76794 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 0) +VERSION = (0, 6, 1) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index d094082..8ce3ff6 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6 +Version: 0.6.1 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From ed2cc2a60bf50e5694066fc03ae177cd2a85d955 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 5 Mar 2012 16:20:37 +0000 Subject: [PATCH 0538/1279] Adding Jacob Peddicord to Authors --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 41d9c66..a5705ee 100644 --- a/AUTHORS +++ b/AUTHORS @@ -97,3 +97,4 @@ that much better: * Shalabh Aggarwal * Chris Williams * Robert Kajic + * Jacob Peddicord From 4c67cbb4b765cbbbcccaaad3a2f6512a1cb79c1f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 6 Mar 2012 12:31:14 +0000 Subject: [PATCH 0539/1279] Updated upgrade docs. Reported issues with older pymongo and sharding --- docs/upgrade.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 1609bb8..d6eabfe 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -18,6 +18,8 @@ Document.objects.with_id - now raises an InvalidQueryError if used with a filter FutureWarning - A future warning has been added to all inherited classes that don't define `allow_inheritance` in their meta. +You may need to update pyMongo to 2.0 for use with Sharding. + 0.4 to 0.5 =========== From 737cbf5f6096b6ee397ed8d3e38f36f3406e7d2e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 8 Mar 2012 12:39:25 +0000 Subject: [PATCH 0540/1279] Updated docs and added fix for _types and positional operator Bumped version to 0.6.2 --- docs/changelog.rst | 11 ++++++++++- docs/guide/connecting.rst | 5 +++++ mongoengine/queryset.py | 18 ++++++++++++++++-- python-mongoengine.spec | 2 +- tests/queryset.py | 6 +++--- 5 files changed, 35 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7374884..e4c6132 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,16 @@ Changelog ========= -Changes in 0.6.x +Changes in 0.6.2 +================ +- Updated documentation for ReplicaSet connections +- Hack round _types issue with SERVER-5247 - querying other arrays may also cause problems. + +Changes in 0.6.1 +================ +- Fix for replicaSet connections + +Changes in 0.6 ================ - Added FutureWarning to inherited classes not declaring 'allow_inheritance' as the default will change in 0.7 diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index 50eb270..bc45dbf 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -26,7 +26,12 @@ name - just supply the uri as the :attr:`host` to connect('project1', host='mongodb://localhost/database_name') +ReplicaSets +=========== +MongoEngine now supports :func:`~pymongo.replica_set_connection.ReplicaSetConnection` +to use them please use a URI style connection and provide the `replicaSet` name in the +connection kwargs. Multiple Databases ================== diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 85a5152..6d1c9c1 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1371,8 +1371,15 @@ class QuerySet(object): write_options = {} update = QuerySet._transform_update(self._document, **update) + query = self._query + + # SERVER-5247 hack + remove_types = "_types" in query and ".$." in unicode(update) + if remove_types: + del query["_types"] + try: - ret = self._collection.update(self._query, update, multi=multi, + ret = self._collection.update(query, update, multi=multi, upsert=upsert, safe=safe_update, **write_options) if ret is not None and 'n' in ret: @@ -1400,10 +1407,17 @@ class QuerySet(object): if not write_options: write_options = {} update = QuerySet._transform_update(self._document, **update) + query = self._query + + # SERVER-5247 hack + remove_types = "_types" in query and ".$." in unicode(update) + if remove_types: + del query["_types"] + try: # Explicitly provide 'multi=False' to newer versions of PyMongo # as the default may change to 'True' - ret = self._collection.update(self._query, update, multi=False, + ret = self._collection.update(query, update, multi=False, upsert=upsert, safe=safe_update, **write_options) diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 8ce3ff6..106243a 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.1 +Version: 0.6.2 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB diff --git a/tests/queryset.py b/tests/queryset.py index 9704895..ee3ab93 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -329,11 +329,11 @@ class QuerySetTest(unittest.TestCase): BlogPost(title="ABC", comments=[c1, c2]).save() - BlogPost.objects(comments__by="joe").update(inc__comments__S__votes=1) + BlogPost.objects(comments__by="jane").update(inc__comments__S__votes=1) post = BlogPost.objects.first() - self.assertEquals(post.comments[0].by, 'joe') - self.assertEquals(post.comments[0].votes, 4) + self.assertEquals(post.comments[1].by, 'jane') + self.assertEquals(post.comments[1].votes, 8) # Currently the $ operator only applies to the first matched item in # the query From 65591c772794dcc8782bb21e14fa8d44817780aa Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 8 Mar 2012 12:40:07 +0000 Subject: [PATCH 0541/1279] Version Bump --- mongoengine/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 7b76794..0cc74a4 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 1) +VERSION = (0, 6, 2) def get_version(): From 83eb4f6b16ea9e8731f83d802c8588b8b3151b6a Mon Sep 17 00:00:00 2001 From: Adam Reeve Date: Fri, 9 Mar 2012 16:07:59 +1300 Subject: [PATCH 0542/1279] Document the ValidationError class --- docs/apireference.rst | 3 +++ docs/guide/defining-documents.rst | 2 +- docs/guide/document-instances.rst | 4 ++-- mongoengine/base.py | 15 +++++++++++++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 9e6ed47..304755f 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -31,6 +31,9 @@ Documents .. autoclass:: mongoengine.document.MapReduceDocument :members: +.. autoclass:: mongoengine.ValidationError + :members: + Querying ======== diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 1e404c0..ba7a580 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -98,7 +98,7 @@ arguments can be set on all fields: :attr:`required` (Default: False) If set to True and the field is not set on the document instance, a - :class:`~mongoengine.base.ValidationError` will be raised when the document is + :class:`~mongoengine.ValidationError` will be raised when the document is validated. :attr:`default` (Default: None) diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index 4e799ec..54fa804 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -91,5 +91,5 @@ is an alias to :attr:`id`:: .. note:: If you define your own primary key field, the field implicitly becomes - required, so a :class:`ValidationError` will be thrown if you don't provide - it. + required, so a :class:`~mongoengine.ValidationError` will be thrown if + you don't provide it. diff --git a/mongoengine/base.py b/mongoengine/base.py index 04b3f1f..5a9dd88 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -25,7 +25,15 @@ class InvalidDocumentError(Exception): class ValidationError(AssertionError): """Validation exception. + + May represent an error validating a field or a + document containing fields with validation errors. + + :ivar errors: A dictionary of errors for fields within this + document or list, or None if the error is for an + individual field. """ + errors = {} field_name = None _message = None @@ -57,6 +65,13 @@ class ValidationError(AssertionError): message = property(_get_message, _set_message) def to_dict(self): + """Returns a dictionary of all errors within a document + + Keys are field names or list indices and values are the + validation error messages, or a nested dictionary of + errors for an embedded document or list. + """ + def build_dict(source): errors_dict = {} if not source: From 540a0cc59c57de51e89350bdc4add0ddfc93afa2 Mon Sep 17 00:00:00 2001 From: Adam Reeve Date: Fri, 9 Mar 2012 16:08:30 +1300 Subject: [PATCH 0543/1279] List all document errors when raising a ValidationError --- mongoengine/base.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 5a9dd88..e423558 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -51,10 +51,12 @@ class ValidationError(AssertionError): def __getattribute__(self, name): message = super(ValidationError, self).__getattribute__(name) - if name == 'message' and self.field_name: - return message + ' ("%s")' % self.field_name - else: - return message + if name == 'message': + if self.field_name: + message += ' ("%s")' % self.field_name + if self.errors: + message += ':\n' + self._format_errors() + return message def _get_message(self): return self._message @@ -88,6 +90,20 @@ class ValidationError(AssertionError): return {} return build_dict(self.errors) + def _format_errors(self): + """Returns a string listing all errors within a document""" + + def format_error(field, value, prefix=''): + if isinstance(value, dict): + new_prefix = (prefix + '.' if prefix else '') + str(field) + return '\n'.join( + [format_error(k, value[k], new_prefix) for k in value]) + else: + return (prefix + ": " if prefix else '') + str(value) + + return '\n'.join( + [format_error(k, v) for k, v in self.to_dict().items()]) + _document_registry = {} From bd1572f11a12b88a3c5fb3e36478142e7e5f804d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 12 Mar 2012 10:31:51 +0000 Subject: [PATCH 0544/1279] Fixed upgrade docs and instructions --- docs/upgrade.rst | 2 +- mongoengine/base.py | 11 +++++++++-- tests/document.py | 7 ++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index d6eabfe..bfb5cc5 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -76,7 +76,7 @@ To upgrade use a Mixin class to set meta like so :: class MyAceDocument(Document, BaseMixin): pass - MyAceDocument._get_collection_name() == myacedocument + MyAceDocument._get_collection_name() == "myacedocument" Alternatively, you can rename your collections eg :: diff --git a/mongoengine/base.py b/mongoengine/base.py index 04b3f1f..adb40d8 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -478,13 +478,18 @@ class DocumentMetaclass(type): attrs.update(dict([(k, v) for k, v in base.__dict__.items() if issubclass(v.__class__, BaseField)])) + # Handle simple mixin's with meta + if hasattr(base, 'meta') and not isinstance(base, DocumentMetaclass): + meta = attrs.get('meta', {}) + meta.update(base.meta) + attrs['meta'] = meta + for p_base in base.__bases__: #optimize :-) if p_base in (object, BaseDocument): continue attrs.update(_get_mixin_fields(p_base)) - return attrs metaclass = attrs.get('__metaclass__') @@ -498,6 +503,7 @@ class DocumentMetaclass(type): simple_class = True for base in bases: + # Include all fields present in superclasses if hasattr(base, '_fields'): doc_fields.update(base._fields) @@ -526,7 +532,8 @@ class DocumentMetaclass(type): simple_class = False doc_class_name = '.'.join(reversed(class_name)) - meta = attrs.get('_meta', attrs.get('meta', {})) + meta = attrs.get('_meta', {}) + meta.update(attrs.get('meta', {})) if 'allow_inheritance' not in meta: meta['allow_inheritance'] = True diff --git a/tests/document.py b/tests/document.py index 910a78f..20f3924 100644 --- a/tests/document.py +++ b/tests/document.py @@ -96,7 +96,7 @@ class DocumentTest(unittest.TestCase): # Ensure Document isn't treated like an actual document self.assertFalse(hasattr(Document, '_fields')) - def test_collection_name(self): + def test_collection_naming(self): """Ensure that a collection with a specified name may be used. """ @@ -157,11 +157,12 @@ class DocumentTest(unittest.TestCase): } class BaseDocument(Document, BaseMixin): - pass + meta = {'allow_inheritance': True} class MyDocument(BaseDocument): pass - self.assertEquals('mydocument', OldMixinNamingConvention._get_collection_name()) + + self.assertEquals('basedocument', MyDocument._get_collection_name()) def test_get_superclasses(self): """Ensure that the correct list of superclasses is assembled. From 58635b24ba2488af907e3af302eadc33acff0950 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 12 Mar 2012 10:35:02 +0000 Subject: [PATCH 0545/1279] Updated changelog --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e4c6132..58d0430 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.6.X +================ +- Bug fix for collection naming and mixins + Changes in 0.6.2 ================ - Updated documentation for ReplicaSet connections From 64860c628706cfd2682912ce3ff8b9e3b4674ef5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 15 Mar 2012 16:27:31 +0000 Subject: [PATCH 0546/1279] Fix signals documentation --- docs/guide/signals.rst | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/guide/signals.rst b/docs/guide/signals.rst index f60e4d7..4d0b0df 100644 --- a/docs/guide/signals.rst +++ b/docs/guide/signals.rst @@ -5,15 +5,13 @@ Signals .. versionadded:: 0.5 -Signal support is provided by the excellent `blinker`_ library and -will gracefully fall back if it is not available. +..note :: + + Signal support is provided by the excellent `blinker`_ library and + will gracefully fall back if it is not available. -<<<<<<< HEAD -The following document signals exist in MongoEngine and are pretty self explanatory: -======= The following document signals exist in MongoEngine and are pretty self-explanatory: ->>>>>>> master * `mongoengine.signals.pre_init` * `mongoengine.signals.post_init` From fd18a4860840e5fe34e01113e0f64b7b4215ab6f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 15 Mar 2012 16:36:04 +0000 Subject: [PATCH 0547/1279] Rst fix --- docs/guide/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/signals.rst b/docs/guide/signals.rst index 4d0b0df..c25c178 100644 --- a/docs/guide/signals.rst +++ b/docs/guide/signals.rst @@ -5,7 +5,7 @@ Signals .. versionadded:: 0.5 -..note :: +.. note:: Signal support is provided by the excellent `blinker`_ library and will gracefully fall back if it is not available. From 7e376b40bbe745a1a0fe15c596e6da306176c446 Mon Sep 17 00:00:00 2001 From: Colin Howe Date: Mon, 19 Mar 2012 20:27:08 +0000 Subject: [PATCH 0548/1279] Add new meta option to Document: allow_index_creation. Defaults to True. If set to False then MongoEngine will not ensure indexes exist --- mongoengine/document.py | 5 +++ mongoengine/queryset.py | 97 +++++++++++++++++++++-------------------- tests/document.py | 22 ++++++++++ 3 files changed, 77 insertions(+), 47 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 4b5506f..fdf9d28 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -74,6 +74,11 @@ class Document(BaseDocument): names. Index direction may be specified by prefixing the field names with a **+** or **-** sign. + Index creation can be disabled by specifying :attr:`index_allow_creation` in + the :attr:`meta` dictionary. If this is set to True then indexes will not be + created by MongoEngine. This is useful in production systems where index + creation is performed as part of a deployment system. + By default, _types will be added to the start of every index (that doesn't contain a list) if allow_inheritence is True. This can be disabled by either setting types to False on the specific index or diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 85a5152..c9a5fb5 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -481,13 +481,60 @@ class QuerySet(object): """Returns all documents.""" return self.__call__() + def _ensure_indexes_exist(self): + background = self._document._meta.get('index_background', False) + drop_dups = self._document._meta.get('index_drop_dups', False) + index_opts = self._document._meta.get('index_options', {}) + index_types = self._document._meta.get('index_types', True) + + # determine if an index which we are creating includes + # _type as its first field; if so, we can avoid creating + # an extra index on _type, as mongodb will use the existing + # index to service queries against _type + types_indexed = False + def includes_types(fields): + first_field = None + if len(fields): + if isinstance(fields[0], basestring): + first_field = fields[0] + elif isinstance(fields[0], (list, tuple)) and len(fields[0]): + first_field = fields[0][0] + return first_field == '_types' + + # Ensure indexes created by uniqueness constraints + for index in self._document._meta['unique_indexes']: + types_indexed = types_indexed or includes_types(index) + self._collection.ensure_index(index, unique=True, + background=background, drop_dups=drop_dups, **index_opts) + + # Ensure document-defined indexes are created + if self._document._meta['indexes']: + for spec in self._document._meta['indexes']: + types_indexed = types_indexed or includes_types(spec['fields']) + opts = index_opts.copy() + opts['unique'] = spec.get('unique', False) + opts['sparse'] = spec.get('sparse', False) + self._collection.ensure_index(spec['fields'], + background=background, **opts) + + # If _types is being used (for polymorphism), it needs an index, + # only if another index doesn't begin with _types + if index_types and '_types' in self._query and not types_indexed: + self._collection.ensure_index('_types', + background=background, **index_opts) + + # Add geo indicies + for field in self._document._geo_indices(): + index_spec = [(field.db_field, pymongo.GEO2D)] + self._collection.ensure_index(index_spec, + background=background, **index_opts) + @property def _collection(self): """Property that returns the collection object. This allows us to perform operations only if the collection is accessed. """ if self._document not in QuerySet.__already_indexed: - # Ensure collection exists db = self._document._get_db() if self._collection_obj.name not in db.collection_names(): @@ -496,52 +543,8 @@ class QuerySet(object): QuerySet.__already_indexed.add(self._document) - background = self._document._meta.get('index_background', False) - drop_dups = self._document._meta.get('index_drop_dups', False) - index_opts = self._document._meta.get('index_options', {}) - index_types = self._document._meta.get('index_types', True) - - # determine if an index which we are creating includes - # _type as its first field; if so, we can avoid creating - # an extra index on _type, as mongodb will use the existing - # index to service queries against _type - types_indexed = False - def includes_types(fields): - first_field = None - if len(fields): - if isinstance(fields[0], basestring): - first_field = fields[0] - elif isinstance(fields[0], (list, tuple)) and len(fields[0]): - first_field = fields[0][0] - return first_field == '_types' - - # Ensure indexes created by uniqueness constraints - for index in self._document._meta['unique_indexes']: - types_indexed = types_indexed or includes_types(index) - self._collection.ensure_index(index, unique=True, - background=background, drop_dups=drop_dups, **index_opts) - - # Ensure document-defined indexes are created - if self._document._meta['indexes']: - for spec in self._document._meta['indexes']: - types_indexed = types_indexed or includes_types(spec['fields']) - opts = index_opts.copy() - opts['unique'] = spec.get('unique', False) - opts['sparse'] = spec.get('sparse', False) - self._collection.ensure_index(spec['fields'], - background=background, **opts) - - # If _types is being used (for polymorphism), it needs an index, - # only if another index doesn't begin with _types - if index_types and '_types' in self._query and not types_indexed: - self._collection.ensure_index('_types', - background=background, **index_opts) - - # Add geo indicies - for field in self._document._geo_indices(): - index_spec = [(field.db_field, pymongo.GEO2D)] - self._collection.ensure_index(index_spec, - background=background, **index_opts) + if self._document._meta.get('index_allow_creation', True): + self._ensure_indexes_exist() return self._collection_obj diff --git a/tests/document.py b/tests/document.py index 910a78f..c88164e 100644 --- a/tests/document.py +++ b/tests/document.py @@ -740,6 +740,28 @@ class DocumentTest(unittest.TestCase): self.assertEqual(info.keys(), ['_types_1_user_guid_1', '_id_', '_types_1_name_1']) Person.drop_collection() + def test_disable_index_creation(self): + """Tests setting allow_index_creation to False on the connection will + disable any index generation. + """ + class User(Document): + meta = { + 'indexes': ['user_guid'], + 'index_allow_creation': False + } + user_guid = StringField(required=True) + + + User.drop_collection() + + u = User(user_guid='123') + u.save() + + self.assertEquals(1, User.objects.count()) + info = User.objects._collection.index_information() + self.assertEqual(info.keys(), ['_id_']) + User.drop_collection() + def test_embedded_document_index(self): """Tests settings an index on an embedded document """ From 520051af25934b577647bd88fd869a275cf65b13 Mon Sep 17 00:00:00 2001 From: Adam Parrish Date: Wed, 11 Jan 2012 16:41:39 -0800 Subject: [PATCH 0549/1279] preparing values in a ListField won't mangle embedded documents any more --- mongoengine/fields.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 14fcca0..13b7ed8 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -8,7 +8,7 @@ import uuid from bson import Binary, DBRef, SON, ObjectId from base import (BaseField, ComplexBaseField, ObjectIdField, - ValidationError, get_document) + ValidationError, get_document, BaseDocument) from queryset import DO_NOTHING, QuerySet from document import Document, EmbeddedDocument from connection import get_db, DEFAULT_CONNECTION_NAME @@ -497,6 +497,7 @@ class ListField(ComplexBaseField): def prepare_query_value(self, op, value): if self.field: if op in ('set', 'unset') and (not isinstance(value, basestring) + and not isinstance(value, BaseDocument) and hasattr(value, '__iter__')): return [self.field.prepare_query_value(op, v) for v in value] return self.field.prepare_query_value(op, value) From 0f420abc8ea48a61a003ddb8083c26327673f153 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 22 Mar 2012 15:44:22 +0000 Subject: [PATCH 0550/1279] Added test for listfields containing embedded documents Added Adam to the authors - thanks for the patch fixes #466 --- AUTHORS | 1 + tests/queryset.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/AUTHORS b/AUTHORS index a5705ee..68b3ecf 100644 --- a/AUTHORS +++ b/AUTHORS @@ -98,3 +98,4 @@ that much better: * Chris Williams * Robert Kajic * Jacob Peddicord + * Adam Parrish diff --git a/tests/queryset.py b/tests/queryset.py index ee3ab93..0d5aaab 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1518,6 +1518,37 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() + + def test_set_list_embedded_documents(self): + + class Author(EmbeddedDocument): + name = StringField() + + class Message(Document): + title = StringField() + authors = ListField(EmbeddedDocumentField('Author')) + + Message.drop_collection() + + message = Message(title="hello", authors=[Author(name="Harry")]) + message.save() + + Message.objects(authors__name="Harry").update_one( + set__authors__S=Author(name="Ross")) + + message = message.reload() + self.assertEquals(message.authors[0].name, "Ross") + + Message.objects(authors__name="Ross").update_one( + set__authors=[Author(name="Harry"), + Author(name="Ross"), + Author(name="Adam")]) + + message = message.reload() + self.assertEquals(message.authors[0].name, "Harry") + self.assertEquals(message.authors[1].name, "Ross") + self.assertEquals(message.authors[2].name, "Adam") + def test_order_by(self): """Ensure that QuerySets may be ordered. """ From f1e7b97a931168c3cb3b1b5a831fe349cebdbd90 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 22 Mar 2012 15:48:54 +0000 Subject: [PATCH 0551/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 58d0430..a458be8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Bug fix for updates where listfields contain embedded documents - Bug fix for collection naming and mixins Changes in 0.6.2 From e2bef076d3161035daaa174600f73206dd3d6c30 Mon Sep 17 00:00:00 2001 From: Nils Hasenbanck Date: Sat, 24 Mar 2012 11:07:37 +0100 Subject: [PATCH 0552/1279] Fixed the session backend for django 1.4 Signed-off-by: Nils Hasenbanck --- mongoengine/django/sessions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index 2f0e17f..d3d2d3b 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -41,7 +41,7 @@ class SessionStore(SessionBase): def create(self): while True: - self.session_key = self._get_new_session_key() + self._session_key = self._get_new_session_key() try: self.save(must_create=True) except CreateError: From 3af6d0dbfdf0ce76c6792f0a7b1a2e71ad49e05b Mon Sep 17 00:00:00 2001 From: Nils Hasenbanck Date: Sat, 24 Mar 2012 11:08:00 +0100 Subject: [PATCH 0553/1279] Replaces deprecated hasher with new django 1.4 hasher This way we can even use the new hasher configuration django 1.4 provides. Signed-off-by: Nils Hasenbanck --- mongoengine/django/auth.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 38370cc..156daf7 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -1,23 +1,14 @@ from mongoengine import * -from django.utils.hashcompat import md5_constructor, sha_constructor from django.utils.encoding import smart_str from django.contrib.auth.models import AnonymousUser +from django.contrib.auth.hashers import check_password, make_password from django.utils.translation import ugettext_lazy as _ import datetime REDIRECT_FIELD_NAME = 'next' -def get_hexdigest(algorithm, salt, raw_password): - raw_password, salt = smart_str(raw_password), smart_str(salt) - if algorithm == 'md5': - return md5_constructor(salt + raw_password).hexdigest() - elif algorithm == 'sha1': - return sha_constructor(salt + raw_password).hexdigest() - raise ValueError('Got unknown password algorithm type in password') - - class User(Document): """A User document that aims to mirror most of the API specified by Django at http://docs.djangoproject.com/en/dev/topics/auth/#users @@ -34,7 +25,7 @@ class User(Document): email = EmailField(verbose_name=_('e-mail address')) password = StringField(max_length=128, verbose_name=_('password'), - help_text=_("Use '[algo]$[salt]$[hexdigest]' or use the change password form.")) + help_text=_("Use '[algo]$[iterations]$[salt]$[hexdigest]' or use the change password form.")) is_staff = BooleanField(default=False, verbose_name=_('staff status'), help_text=_("Designates whether the user can log into this admin site.")) @@ -75,11 +66,7 @@ class User(Document): assigning to :attr:`~mongoengine.django.auth.User.password` as the password is hashed before storage. """ - from random import random - algo = 'sha1' - salt = get_hexdigest(algo, str(random()), str(random()))[:5] - hash = get_hexdigest(algo, salt, raw_password) - self.password = '%s$%s$%s' % (algo, salt, hash) + self.password = make_password(raw_password) self.save() return self @@ -89,8 +76,7 @@ class User(Document): :attr:`~mongoengine.django.auth.User.password` as the password is hashed before storage. """ - algo, salt, hash = self.password.split('$') - return hash == get_hexdigest(algo, salt, raw_password) + return check_password(raw_password, self.password) @classmethod def create_user(cls, username, password, email=None): From 8fe4a7029977a0870b137edab1aa173a7570ca77 Mon Sep 17 00:00:00 2001 From: Nils Hasenbanck Date: Sat, 24 Mar 2012 19:24:42 +0100 Subject: [PATCH 0554/1279] Fixed the exception when saving a new session Signed-off-by: Nils Hasenbanck --- mongoengine/django/sessions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index d3d2d3b..3686ecf 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -51,7 +51,7 @@ class SessionStore(SessionBase): return def save(self, must_create=False): - s = MongoSession(session_key=self.session_key) + s = MongoSession(session_key=self._session_key) s.session_data = self.encode(self._get_session(no_load=must_create)) s.expire_date = self.get_expiry_date() try: From 421f324f9eb8ab7eb3d1783746fef0407834c677 Mon Sep 17 00:00:00 2001 From: Nils Hasenbanck Date: Sat, 24 Mar 2012 19:24:42 +0100 Subject: [PATCH 0555/1279] Fixed the exception when saving a new session The session was not created for some reason. Now it is. Signed-off-by: Nils Hasenbanck --- mongoengine/django/sessions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index d3d2d3b..ca35962 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -51,7 +51,9 @@ class SessionStore(SessionBase): return def save(self, must_create=False): - s = MongoSession(session_key=self.session_key) + if self._session_key is None: + self.create() + s = MongoSession(session_key=self._session_key) s.session_data = self.encode(self._get_session(no_load=must_create)) s.expire_date = self.get_expiry_date() try: From 6bd2ccc9bf2f0311ed2af516c72cc08032cc4917 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Sat, 24 Mar 2012 19:03:24 +0000 Subject: [PATCH 0556/1279] UPdated authors --- AUTHORS | 1 - 1 file changed, 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 68b3ecf..a5705ee 100644 --- a/AUTHORS +++ b/AUTHORS @@ -98,4 +98,3 @@ that much better: * Chris Williams * Robert Kajic * Jacob Peddicord - * Adam Parrish From bbdd15161a41343e638873b617bdd6f43d81c2ee Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Sat, 24 Mar 2012 19:06:08 +0000 Subject: [PATCH 0557/1279] 0.6.3 release --- AUTHORS | 1 + docs/changelog.rst | 3 ++- docs/django.rst | 16 +++++++++------- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/AUTHORS b/AUTHORS index a5705ee..112b7f3 100644 --- a/AUTHORS +++ b/AUTHORS @@ -98,3 +98,4 @@ that much better: * Chris Williams * Robert Kajic * Jacob Peddicord + * Nils Hasenbanck \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index a458be8..0c962cd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,8 +2,9 @@ Changelog ========= -Changes in 0.6.X +Changes in 0.6.3 ================ +- Updated sessions for Django 1.4 - Bug fix for updates where listfields contain embedded documents - Bug fix for collection naming and mixins diff --git a/docs/django.rst b/docs/django.rst index 4478b94..144baab 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -2,19 +2,21 @@ Using MongoEngine with Django ============================= +.. note :: Updated to support Django 1.4 + Connecting ========== In your **settings.py** file, ignore the standard database settings (unless you -also plan to use the ORM in your project), and instead call +also plan to use the ORM in your project), and instead call :func:`~mongoengine.connect` somewhere in the settings module. Authentication ============== MongoEngine includes a Django authentication backend, which uses MongoDB. The -:class:`~mongoengine.django.auth.User` model is a MongoEngine -:class:`~mongoengine.Document`, but implements most of the methods and +:class:`~mongoengine.django.auth.User` model is a MongoEngine +:class:`~mongoengine.Document`, but implements most of the methods and attributes that the standard Django :class:`User` model does - so the two are -moderately compatible. Using this backend will allow you to store users in +moderately compatible. Using this backend will allow you to store users in MongoDB but still use many of the Django authentication infrastucture (such as the :func:`login_required` decorator and the :func:`authenticate` function). To enable the MongoEngine auth backend, add the following to you **settings.py** @@ -24,7 +26,7 @@ file:: 'mongoengine.django.auth.MongoEngineBackend', ) -The :mod:`~mongoengine.django.auth` module also contains a +The :mod:`~mongoengine.django.auth` module also contains a :func:`~mongoengine.django.auth.get_user` helper function, that takes a user's :attr:`id` and returns a :class:`~mongoengine.django.auth.User` object. @@ -49,9 +51,9 @@ Storage ======= With MongoEngine's support for GridFS via the :class:`~mongoengine.FileField`, it is useful to have a Django file storage backend that wraps this. The new -storage module is called :class:`~mongoengine.django.storage.GridFSStorage`. +storage module is called :class:`~mongoengine.django.storage.GridFSStorage`. Using it is very similar to using the default FileSystemStorage.:: - + from mongoengine.django.storage import GridFSStorage fs = GridFSStorage() diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 0cc74a4..9d0a757 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 2) +VERSION = (0, 6, 3) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 106243a..164f5af 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.2 +Version: 0.6.3 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 66d215c9c148572c4ebd9370453f7406da89abaf Mon Sep 17 00:00:00 2001 From: mostlystatic Date: Sat, 24 Mar 2012 20:01:40 +0000 Subject: [PATCH 0558/1279] Fix for unknown connection alias error message. --- mongoengine/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 3efb7d3..385d738 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -86,7 +86,7 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): if alias not in _connections: if alias not in _connection_settings: - msg = 'Connection with alias "%s" has not been defined' + msg = 'Connection with alias "%s" has not been defined' % alias if alias == DEFAULT_CONNECTION_NAME: msg = 'You have not defined a default connection' raise ConnectionError(msg) From ad2e11928275fa6bd86d541392a9dbed8c5b9218 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Mon, 26 Mar 2012 16:48:37 -0700 Subject: [PATCH 0559/1279] The port is defaulted in to conn_settings, so discard the port since hosts_or_uri must be used. --- mongoengine/connection.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 3efb7d3..7ecc3de 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -110,6 +110,8 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): connection_class = Connection if 'replicaSet' in conn_settings: conn_settings['hosts_or_uri'] = conn_settings.pop('host', None) + # Discard port since it can't be used on ReplicaSetConnection + conn_settings.pop('port') connection_class = ReplicaSetConnection try: _connections[alias] = connection_class(**conn_settings) From 98e5daa0e088a006f7699780fb8effd339d55f73 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Mar 2012 00:49:34 +0100 Subject: [PATCH 0560/1279] Added mostlystatic to the AUTHORS --- AUTHORS | 3 ++- docs/changelog.rst | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 112b7f3..1c367ff 100644 --- a/AUTHORS +++ b/AUTHORS @@ -98,4 +98,5 @@ that much better: * Chris Williams * Robert Kajic * Jacob Peddicord - * Nils Hasenbanck \ No newline at end of file + * Nils Hasenbanck + * mostlystatic \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 0c962cd..8900671 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,11 @@ Changelog ========= +Changes in 0.6.X +================ + +- bug fix for unknown connection alias error message + Changes in 0.6.3 ================ - Updated sessions for Django 1.4 From 2a34358abc40ad84a17014e41d9bd8af7c0fd071 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Mar 2012 01:47:17 +0100 Subject: [PATCH 0561/1279] Updated connection refs #474 --- docs/changelog.rst | 1 + mongoengine/connection.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8900671..8674e9e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.6.X ================ +- updated replicasetconnection - pop port if exists - bug fix for unknown connection alias error message Changes in 0.6.3 diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 0b7daa8..9cf8264 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -105,13 +105,13 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): for slave_alias in conn_settings['slaves']: slaves.append(get_connection(slave_alias)) conn_settings['slaves'] = slaves - conn_settings.pop('read_preference') + conn_settings.pop('read_preference', None) connection_class = Connection if 'replicaSet' in conn_settings: conn_settings['hosts_or_uri'] = conn_settings.pop('host', None) # Discard port since it can't be used on ReplicaSetConnection - conn_settings.pop('port') + conn_settings.pop('port', None) connection_class = ReplicaSetConnection try: _connections[alias] = connection_class(**conn_settings) From a1d43fecd962ea856e361fe59bf0913269f8eb84 Mon Sep 17 00:00:00 2001 From: Greg Banks Date: Wed, 11 Apr 2012 16:37:22 -0700 Subject: [PATCH 0562/1279] fix for issue 473 --- mongoengine/dereference.py | 3 +++ tests/dereference.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index b1529d3..b67c2d2 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -113,6 +113,9 @@ class DeReference(object): if '_cls' in ref: doc = get_document(ref["_cls"])._from_son(ref) else: + if doc_type is None: + doc_type = get_document( + ''.join(x.capitalize() for x in col.split('_'))) doc = doc_type._from_son(ref) object_map[doc.id] = doc return object_map diff --git a/tests/dereference.py b/tests/dereference.py index 8a4b310..0ed64e6 100644 --- a/tests/dereference.py +++ b/tests/dereference.py @@ -810,3 +810,22 @@ class FieldTest(unittest.TestCase): room = Room.objects.first().select_related() self.assertEquals(room.staffs_with_position[0]['staff'], sarah) self.assertEquals(room.staffs_with_position[1]['staff'], bob) + + def test_document_reload_no_inheritance(self): + class Foo(Document): + meta = {'allow_inheritance': False} + bar = ReferenceField('Bar') + + class Bar(Document): + meta = {'allow_inheritance': False} + msg = StringField(required=True, default='Blammo!') + + Foo.drop_collection() + Bar.drop_collection() + + bar = Bar() + bar.save() + foo = Foo() + foo.bar = bar + foo.save() + foo.reload() From 49a66ba81a4353445ec0917173f50a65880ad7bc Mon Sep 17 00:00:00 2001 From: Greg Banks Date: Thu, 12 Apr 2012 11:42:10 -0700 Subject: [PATCH 0563/1279] whoops, don't dereference all references as the first type encountered --- mongoengine/dereference.py | 7 ++++--- tests/dereference.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index b67c2d2..8928d1e 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -112,10 +112,11 @@ class DeReference(object): for ref in references: if '_cls' in ref: doc = get_document(ref["_cls"])._from_son(ref) + elif doc_type is None: + doc = get_document( + ''.join(x.capitalize() + for x in col.split('_')))._from_son(ref) else: - if doc_type is None: - doc_type = get_document( - ''.join(x.capitalize() for x in col.split('_'))) doc = doc_type._from_son(ref) object_map[doc.id] = doc return object_map diff --git a/tests/dereference.py b/tests/dereference.py index 0ed64e6..9f0d433 100644 --- a/tests/dereference.py +++ b/tests/dereference.py @@ -815,17 +815,29 @@ class FieldTest(unittest.TestCase): class Foo(Document): meta = {'allow_inheritance': False} bar = ReferenceField('Bar') + baz = ReferenceField('Baz') class Bar(Document): meta = {'allow_inheritance': False} msg = StringField(required=True, default='Blammo!') + class Baz(Document): + meta = {'allow_inheritance': False} + msg = StringField(required=True, default='Kaboom!') + Foo.drop_collection() Bar.drop_collection() + Baz.drop_collection() bar = Bar() bar.save() + baz = Baz() + baz.save() foo = Foo() foo.bar = bar + foo.baz = baz foo.save() foo.reload() + + self.assertEquals(type(foo.bar), Bar) + self.assertEquals(type(foo.baz), Baz) From bfae93e57ef725ec1ac9db1dc483a90f7c780201 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 13 Apr 2012 04:56:20 -0300 Subject: [PATCH 0564/1279] small fixes for ReferenceField --- mongoengine/fields.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 13b7ed8..831186d 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -657,6 +657,9 @@ class ReferenceField(BaseField): return super(ReferenceField, self).__get__(instance, owner) def to_mongo(self, document): + if isinstance(document, DBRef): + return document + id_field_name = self.document_type._meta['id_field'] id_field = self.document_type._fields[id_field_name] From 0376910f3392ee58fa57558458e9f0aa363e2cbc Mon Sep 17 00:00:00 2001 From: Dan Crosta Date: Tue, 17 Apr 2012 19:47:54 -0400 Subject: [PATCH 0565/1279] refactor get_connection In the previous version, the requested ReadPreference was ignored in the case that the user specified a MongoDB URI. This rearranges the code to ensure that only those values which we explicitly parse out of the URI override values set as keyword arguments. This leaves open the possibility of conflicts between the URI and the kwargs -- we should consider whether to raise an exception if, e.g., username is specified as a kwarg *and* in the URI. --- mongoengine/connection.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 9cf8264..96b8100 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -39,22 +39,7 @@ def register_connection(alias, name, host='localhost', port=27017, """ global _connection_settings - # Handle uri style connections - if "://" in host: - uri_dict = uri_parser.parse_uri(host) - if uri_dict.get('database') is None: - raise ConnectionError("If using URI style connection include "\ - "database name in string") - _connection_settings[alias] = { - 'host': host, - 'name': uri_dict.get('database'), - 'username': uri_dict.get('username'), - 'password': uri_dict.get('password') - } - _connection_settings[alias].update(kwargs) - return - - _connection_settings[alias] = { + conn_settings = { 'name': name, 'host': host, 'port': port, @@ -64,7 +49,22 @@ def register_connection(alias, name, host='localhost', port=27017, 'password': password, 'read_preference': read_preference } - _connection_settings[alias].update(kwargs) + + # Handle uri style connections + if "://" in host: + uri_dict = uri_parser.parse_uri(host) + if uri_dict.get('database') is None: + raise ConnectionError("If using URI style connection include "\ + "database name in string") + conn_settings.update({ + 'host': host, + 'name': uri_dict.get('database'), + 'username': uri_dict.get('username'), + 'password': uri_dict.get('password'), + 'read_preference': read_preference, + }) + + _connection_settings[alias] = conn_settings def disconnect(alias=DEFAULT_CONNECTION_NAME): From 2d71eb8a18f50dca717da04f7cf72c2de9ba2816 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 18 Apr 2012 10:22:26 +0100 Subject: [PATCH 0566/1279] Added support back for Django 1.3 as well as 1.4 --- docs/changelog.rst | 3 ++- mongoengine/django/auth.py | 28 ++++++++++++++++++++++++++-- mongoengine/django/sessions.py | 15 +++++++++------ 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8674e9e..5a52160 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,8 +5,9 @@ Changelog Changes in 0.6.X ================ -- updated replicasetconnection - pop port if exists +- refactored connection / fixed replicasetconnection - bug fix for unknown connection alias error message +- Sessions support Django 1.3 and Django 1.4 Changes in 0.6.3 ================ diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 156daf7..000e352 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -1,11 +1,35 @@ +import datetime + from mongoengine import * from django.utils.encoding import smart_str from django.contrib.auth.models import AnonymousUser -from django.contrib.auth.hashers import check_password, make_password from django.utils.translation import ugettext_lazy as _ -import datetime +try: + from django.contrib.auth.hashers import check_password, make_password +except ImportError: + """Handle older versions of Django""" + + def get_hexdigest(algorithm, salt, raw_password): + raw_password, salt = smart_str(raw_password), smart_str(salt) + if algorithm == 'md5': + return md5_constructor(salt + raw_password).hexdigest() + elif algorithm == 'sha1': + return sha_constructor(salt + raw_password).hexdigest() + raise ValueError('Got unknown password algorithm type in password') + + def check_password(raw_password, password): + algo, salt, hash = password.split('$') + return hash == get_hexdigest(algo, salt, raw_password) + + def make_password(raw_password): + from random import random + algo = 'sha1' + salt = get_hexdigest(algo, str(random()), str(random()))[:5] + hash = get_hexdigest(algo, salt, raw_password) + return '%s$%s$%s' % (algo, salt, hash) + REDIRECT_FIELD_NAME = 'next' diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index ca35962..667cf24 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -1,3 +1,6 @@ +from datetime import datetime + +from django.conf import settings from django.contrib.sessions.backends.base import SessionBase, CreateError from django.core.exceptions import SuspiciousOperation from django.utils.encoding import force_unicode @@ -6,18 +9,18 @@ from mongoengine.document import Document from mongoengine import fields from mongoengine.queryset import OperationError from mongoengine.connection import DEFAULT_CONNECTION_NAME -from django.conf import settings -from datetime import datetime + MONGOENGINE_SESSION_DB_ALIAS = getattr( settings, 'MONGOENGINE_SESSION_DB_ALIAS', DEFAULT_CONNECTION_NAME) + class MongoSession(Document): session_key = fields.StringField(primary_key=True, max_length=40) session_data = fields.StringField() expire_date = fields.DateTimeField() - + meta = {'collection': 'django_session', 'db_alias': MONGOENGINE_SESSION_DB_ALIAS, 'allow_inheritance': False} @@ -41,7 +44,7 @@ class SessionStore(SessionBase): def create(self): while True: - self._session_key = self._get_new_session_key() + self.session_key = self._get_new_session_key() try: self.save(must_create=True) except CreateError: @@ -51,9 +54,9 @@ class SessionStore(SessionBase): return def save(self, must_create=False): - if self._session_key is None: + if self.session_key is None: self.create() - s = MongoSession(session_key=self._session_key) + s = MongoSession(session_key=self.session_key) s.session_data = self.encode(self._get_session(no_load=must_create)) s.expire_date = self.get_expiry_date() try: From 51b429e5b0ab8a15b6e86723e0844c342cff368f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 18 Apr 2012 10:28:54 +0100 Subject: [PATCH 0567/1279] Updated changelog --- docs/changelog.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5a52160..5390ad3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,12 +2,13 @@ Changelog ========= -Changes in 0.6.X +Changes in 0.6.4 ================ -- refactored connection / fixed replicasetconnection -- bug fix for unknown connection alias error message +- Refactored connection / fixed replicasetconnection +- Bug fix for unknown connection alias error message - Sessions support Django 1.3 and Django 1.4 +- Minor fix for ReferenceField Changes in 0.6.3 ================ From 5f4b70f3a983538ce4b534434d74ac11aa452434 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 18 Apr 2012 10:30:14 +0100 Subject: [PATCH 0568/1279] Version bump --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 9d0a757..82336da 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 3) +VERSION = (0, 6, 4) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 164f5af..1e6fa7f 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.3 +Version: 0.6.4 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 88dc64653ecd411a8e7753b6098fccb473b5fdb4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 18 Apr 2012 16:41:09 +0100 Subject: [PATCH 0569/1279] Fix Django 1.3 auth --- mongoengine/__init__.py | 2 +- mongoengine/django/auth.py | 1 + python-mongoengine.spec | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 82336da..06bb696 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 4) +VERSION = (0, 6, 5) def get_version(): diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 000e352..694a4ca 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -10,6 +10,7 @@ try: from django.contrib.auth.hashers import check_password, make_password except ImportError: """Handle older versions of Django""" + from django.utils.hashcompat import md5_constructor, sha_constructor def get_hexdigest(algorithm, salt, raw_password): raw_password, salt = smart_str(raw_password), smart_str(salt) diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 1e6fa7f..fad93c9 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.4 +Version: 0.6.5 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From c474ca0f13c0a3d43941fbd99e60b9b708b8fc05 Mon Sep 17 00:00:00 2001 From: Jona Andersen Date: Sun, 22 Apr 2012 13:49:18 +0200 Subject: [PATCH 0570/1279] Allow File-like objects to be stored. No longer demands FileField be an actual instance of file, but instead checks whether object has a 'read' attribute. Fixes read() on GridFSProxy to return an empty string on read failure, or None if file does not exist. --- mongoengine/fields.py | 12 ++++++++---- tests/fields.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 831186d..a2250f4 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -875,10 +875,14 @@ class GridFSProxy(object): self.newfile.writelines(lines) def read(self, size=-1): - try: - return self.get().read(size) - except: + gridout = self.get() + if gridout is None: return None + else: + try: + return gridout.read(size) + except: + return "" def delete(self): # Delete file from GridFS, FileField still remains @@ -935,7 +939,7 @@ class FileField(BaseField): def __set__(self, instance, value): key = self.name - if isinstance(value, file) or isinstance(value, str): + if (hasattr(value, 'read') and not isinstance(value, GridFSProxy)) or isinstance(value, str): # using "FileField() = file/string" notation grid_file = instance._data.get(self.name) # If a file already exists, delete it diff --git a/tests/fields.py b/tests/fields.py index 04ef3d9..85c1b38 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -2,6 +2,7 @@ import datetime import os import unittest import uuid +import StringIO from decimal import Decimal @@ -1481,6 +1482,21 @@ class FieldTest(unittest.TestCase): self.assertEquals(result.file.read(), text) self.assertEquals(result.file.content_type, content_type) result.file.delete() # Remove file from GridFS + PutFile.objects.delete() + + # Ensure file-like objects are stored + putfile = PutFile() + putstring = StringIO.StringIO() + putstring.write(text) + putstring.seek(0) + putfile.file.put(putstring, content_type=content_type) + putfile.save() + putfile.validate() + result = PutFile.objects.first() + self.assertTrue(putfile == result) + self.assertEquals(result.file.read(), text) + self.assertEquals(result.file.content_type, content_type) + result.file.delete() streamfile = StreamFile() streamfile.file.new_file(content_type=content_type) From a9280471475baec4bcc92e46937b8016479dfc06 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 24 Apr 2012 21:00:30 +0100 Subject: [PATCH 0571/1279] Fixing sessions for django 1.3 and django 1.4 --- mongoengine/django/sessions.py | 2 +- tests/django_tests.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index 667cf24..d1e9289 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -44,7 +44,7 @@ class SessionStore(SessionBase): def create(self): while True: - self.session_key = self._get_new_session_key() + self._session_key = self._get_new_session_key() try: self.save(must_create=True) except CreateError: diff --git a/tests/django_tests.py b/tests/django_tests.py index 3341eb1..12ea1d1 100644 --- a/tests/django_tests.py +++ b/tests/django_tests.py @@ -12,6 +12,10 @@ from django.core.paginator import Paginator settings.configure() +from django.contrib.sessions.tests import SessionTestsMixin +from mongoengine.django.sessions import SessionStore, MongoSession + + class QuerySetTest(unittest.TestCase): def setUp(self): @@ -88,3 +92,14 @@ class QuerySetTest(unittest.TestCase): end = p * 2 start = end - 1 self.assertEqual(t.render(Context(d)), u'%d:%d:' % (start, end)) + + + +class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): + backend = SessionStore + + def setUp(self): + connect(db='mongoenginetest') + MongoSession.drop_collection() + super(MongoDBSessionTest, self).setUp() + From aa2add39add82aa4bcf12e5cd3b475d6dadcccd9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 25 Apr 2012 12:24:08 +0100 Subject: [PATCH 0572/1279] Version bump --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 06bb696..f2ff3d9 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 5) +VERSION = (0, 6, 6) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index fad93c9..6239b8a 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.5 +Version: 0.6.6 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB @@ -51,6 +51,18 @@ rm -rf $RPM_BUILD_ROOT # %{python_sitearch}/* %changelog +* Wed Apr 24 2012 Ross Lawley 0.6.5 +- 0.6.6 released +* Wed Apr 18 2012 Ross Lawley 0.6.5 +- 0.6.5 released +* Wed Apr 18 2012 Ross Lawley 0.6.5 +- 0.6.4 released +* Wed Mar 24 2012 Ross Lawley 0.6.5 +- 0.6.3 released +* Wed Mar 22 2012 Ross Lawley 0.6.5 +- 0.6.2 released +* Wed Mar 05 2012 Ross Lawley 0.6.5 +- 0.6.1 released * Mon Mar 05 2012 Ross Lawley 0.6 - 0.6 released * Thu Oct 27 2011 Pau Aliagas 0.5.3-1 From 0e93f6c0dbea91f6388203cfc2018bc522b7e031 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 25 Apr 2012 12:26:31 +0100 Subject: [PATCH 0573/1279] Updated changelog --- docs/changelog.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5390ad3..d7499cf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,15 @@ Changelog ========= +Changes in 0.6.6 +================ +- Django 1.4 fixed (finally) +- Added tests for Django + +Changes in 0.6.5 +================ +- More Django updates + Changes in 0.6.4 ================ From 605092bd88ce2341625e7cfa630cbb024c103a31 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 25 Apr 2012 12:36:37 +0100 Subject: [PATCH 0574/1279] Updated change log / AUTHORS Thanks Greg Banks --- AUTHORS | 3 ++- docs/changelog.rst | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 1c367ff..b71885b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -99,4 +99,5 @@ that much better: * Robert Kajic * Jacob Peddicord * Nils Hasenbanck - * mostlystatic \ No newline at end of file + * mostlystatic + * Greg Banks \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index d7499cf..30c52b0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.6.X +================ +- Fix for #473 - Dereferencing abstracts + Changes in 0.6.6 ================ - Django 1.4 fixed (finally) From 2769d6d7caead6bacae769d1b43f2acdd1a0a954 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 25 Apr 2012 12:43:17 +0100 Subject: [PATCH 0575/1279] Updated Authors / Changelog --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index b71885b..9363129 100644 --- a/AUTHORS +++ b/AUTHORS @@ -100,4 +100,5 @@ that much better: * Jacob Peddicord * Nils Hasenbanck * mostlystatic - * Greg Banks \ No newline at end of file + * Greg Banks + * swashbuckler \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 30c52b0..8c789eb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Added support for File like objects for GridFS - Fix for #473 - Dereferencing abstracts Changes in 0.6.6 From 0bb9781b91bd3e70c84d1bdfe8bb1b85baaab70b Mon Sep 17 00:00:00 2001 From: Greg Banks Date: Thu, 26 Apr 2012 13:56:52 -0700 Subject: [PATCH 0576/1279] add "safe" and "write_options" parameters to QuerySet.insert similar to Document.save --- mongoengine/document.py | 5 +++-- mongoengine/queryset.py | 24 ++++++++++++++++++++++-- tests/queryset.py | 21 +++++++++++++++++++-- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 4b5506f..f4c74c7 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -147,8 +147,9 @@ class Document(BaseDocument): :meth:`~pymongo.collection.Collection.save` OR :meth:`~pymongo.collection.Collection.insert` which will be used as options for the resultant ``getLastError`` command. - For example, ``save(..., w=2, fsync=True)`` will wait until at least two servers - have recorded the write and will force an fsync on each server being written to. + For example, ``save(..., write_options={w: 2, fsync: True}, ...)`` will + wait until at least two servers have recorded the write and will force an + fsync on each server being written to. :param cascade: Sets the flag for cascading saves. You can set a default by setting "cascade" in the document __meta__ :param cascade_kwargs: optional kwargs dictionary to be passed throw to cascading saves diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6d1c9c1..9875822 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -824,11 +824,21 @@ class QuerySet(object): result = None return result - def insert(self, doc_or_docs, load_bulk=True): + def insert(self, doc_or_docs, load_bulk=True, safe=False, write_options=None): """bulk insert documents + If ``safe=True`` and the operation is unsuccessful, an + :class:`~mongoengine.OperationError` will be raised. + :param docs_or_doc: a document or list of documents to be inserted :param load_bulk (optional): If True returns the list of document instances + :param safe: check if the operation succeeded before returning + :param write_options: Extra keyword arguments are passed down to + :meth:`~pymongo.collection.Collection.insert` + which will be used as options for the resultant ``getLastError`` command. + For example, ``insert(..., {w: 2, fsync: True})`` will wait until at least two + servers have recorded the write and will force an fsync on each server being + written to. By default returns document instances, set ``load_bulk`` to False to return just ``ObjectIds`` @@ -837,6 +847,10 @@ class QuerySet(object): """ from document import Document + if not write_options: + write_options = {} + write_options.update({'safe': safe}) + docs = doc_or_docs return_one = False if isinstance(docs, Document) or issubclass(docs.__class__, Document): @@ -854,7 +868,13 @@ class QuerySet(object): raw.append(doc.to_mongo()) signals.pre_bulk_insert.send(self._document, documents=docs) - ids = self._collection.insert(raw) + try: + ids = self._collection.insert(raw, **write_options) + except pymongo.errors.OperationFailure, err: + message = 'Could not save document (%s)' + if u'duplicate key' in unicode(err): + message = u'Tried to save duplicate unique keys (%s)' + raise OperationError(message % unicode(err)) if not load_bulk: signals.post_bulk_insert.send( diff --git a/tests/queryset.py b/tests/queryset.py index 0d5aaab..4cf1165 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -480,7 +480,7 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(person.name, "User C") def test_bulk_insert(self): - """Ensure that query by array position works. + """Ensure that bulk insert works """ class Comment(EmbeddedDocument): @@ -490,7 +490,7 @@ class QuerySetTest(unittest.TestCase): comments = ListField(EmbeddedDocumentField(Comment)) class Blog(Document): - title = StringField() + title = StringField(unique=True) tags = ListField(StringField()) posts = ListField(EmbeddedDocumentField(Post)) @@ -563,6 +563,23 @@ class QuerySetTest(unittest.TestCase): obj_id = Blog.objects.insert(blog1, load_bulk=False) self.assertEquals(obj_id.__class__.__name__, 'ObjectId') + Blog.drop_collection() + post3 = Post(comments=[comment1, comment1]) + blog1 = Blog(title="foo", posts=[post1, post2]) + blog2 = Blog(title="bar", posts=[post2, post3]) + blog3 = Blog(title="baz", posts=[post1, post2]) + Blog.objects.insert([blog1, blog2]) + + def throw_operation_error_not_unique(): + Blog.objects.insert([blog2, blog3], safe=True) + + self.assertRaises(OperationError, throw_operation_error_not_unique) + self.assertEqual(Blog.objects.count(), 2) + + Blog.objects.insert([blog2, blog3], write_options={'continue_on_error': True}) + self.assertEqual(Blog.objects.count(), 3) + + def test_slave_okay(self): """Ensures that a query can take slave_okay syntax """ From 410443471cc545c48e890e083b49cddbf464030a Mon Sep 17 00:00:00 2001 From: Greg Banks Date: Thu, 26 Apr 2012 14:03:30 -0700 Subject: [PATCH 0577/1279] TopLevelDocumentMetaClass sets _meta['index_opts'] not _meta['index_options'] --- mongoengine/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 9875822..23d7515 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -498,7 +498,7 @@ class QuerySet(object): background = self._document._meta.get('index_background', False) drop_dups = self._document._meta.get('index_drop_dups', False) - index_opts = self._document._meta.get('index_options', {}) + index_opts = self._document._meta.get('index_opts', {}) index_types = self._document._meta.get('index_types', True) # determine if an index which we are creating includes From 2b8aa6bafc85dcbfacc14bd8ef61948def1e0af3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Apr 2012 09:15:05 +0100 Subject: [PATCH 0578/1279] Fixes read_preference Fixes mongoengine/mongoengine#10 --- docs/changelog.rst | 1 + mongoengine/connection.py | 7 ++++++- tests/replicaset_connection.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 tests/replicaset_connection.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 8c789eb..ee7a128 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Bug fix Read preference now passed correctly - Added support for File like objects for GridFS - Fix for #473 - Dereferencing abstracts diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 96b8100..7585c73 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -63,7 +63,8 @@ def register_connection(alias, name, host='localhost', port=27017, 'password': uri_dict.get('password'), 'read_preference': read_preference, }) - + if "replicaSet" in host: + conn_settings['replicaSet'] = True _connection_settings[alias] = conn_settings @@ -112,7 +113,11 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): conn_settings['hosts_or_uri'] = conn_settings.pop('host', None) # Discard port since it can't be used on ReplicaSetConnection conn_settings.pop('port', None) + # Discard replicaSet if not base string + if not isinstance(conn_settings['replicaSet'], basestring): + conn_settings.pop('replicaSet', None) connection_class = ReplicaSetConnection + try: _connections[alias] = connection_class(**conn_settings) except Exception, e: diff --git a/tests/replicaset_connection.py b/tests/replicaset_connection.py new file mode 100644 index 0000000..c8a3bbf --- /dev/null +++ b/tests/replicaset_connection.py @@ -0,0 +1,28 @@ +import unittest +import pymongo +from pymongo import ReadPreference + +import mongoengine +from mongoengine import * +from mongoengine.connection import get_db, get_connection, ConnectionError + + +class ConnectionTest(unittest.TestCase): + + def tearDown(self): + mongoengine.connection._connection_settings = {} + mongoengine.connection._connections = {} + mongoengine.connection._dbs = {} + + def test_replicaset_uri_passes_read_preference(self): + """Requires a replica set called "rs" on port 27017 + """ + try: + conn = connect(db='mongoenginetest', host="mongodb://localhost/mongoenginetest?replicaSet=rs", read_preference=ReadPreference.SECONDARY_ONLY) + except ConnectionError, e: + return + + self.assertEquals(conn.read_preference, ReadPreference.SECONDARY_ONLY) + +if __name__ == '__main__': + unittest.main() From 5647ca70bb98c4369f566acd0b9ab2c3d6f25aba Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Apr 2012 09:52:57 +0100 Subject: [PATCH 0579/1279] Fix variable name bug --- mongoengine/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6d1c9c1..7c1394f 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -498,7 +498,7 @@ class QuerySet(object): background = self._document._meta.get('index_background', False) drop_dups = self._document._meta.get('index_drop_dups', False) - index_opts = self._document._meta.get('index_options', {}) + index_opts = self._document._meta.get('index_opts', {}) index_types = self._document._meta.get('index_types', True) # determine if an index which we are creating includes From 42d24263ef18cff7c9c2adac61b1e3950714bcb7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Apr 2012 10:39:35 +0100 Subject: [PATCH 0580/1279] Updated changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index ee7a128..c612da7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,8 @@ Changelog Changes in 0.6.X ================ +- Added write concern options to inserts +- Fixed typo in meta for index options - Bug fix Read preference now passed correctly - Added support for File like objects for GridFS - Fix for #473 - Dereferencing abstracts From b80fda36af1e11a590ebb189544e909feb52de37 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 27 Apr 2012 14:57:07 +0100 Subject: [PATCH 0581/1279] Ensure session save is safe --- mongoengine/django/sessions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index d1e9289..9cea0fd 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -46,7 +46,7 @@ class SessionStore(SessionBase): while True: self._session_key = self._get_new_session_key() try: - self.save(must_create=True) + self.save(must_create=True, safe=True) except CreateError: continue self.modified = True From 530440b3335cde0364752da1504079854be960b3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 1 May 2012 09:14:38 +0100 Subject: [PATCH 0582/1279] Fixed replicaset_connection test --- mongoengine/django/sessions.py | 2 +- tests/replicaset_connection.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index 9cea0fd..d1e9289 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -46,7 +46,7 @@ class SessionStore(SessionBase): while True: self._session_key = self._get_new_session_key() try: - self.save(must_create=True, safe=True) + self.save(must_create=True) except CreateError: continue self.modified = True diff --git a/tests/replicaset_connection.py b/tests/replicaset_connection.py index c8a3bbf..2297814 100644 --- a/tests/replicaset_connection.py +++ b/tests/replicaset_connection.py @@ -1,6 +1,6 @@ import unittest import pymongo -from pymongo import ReadPreference +from pymongo import ReadPreference, ReplicaSetConnection import mongoengine from mongoengine import * @@ -17,11 +17,15 @@ class ConnectionTest(unittest.TestCase): def test_replicaset_uri_passes_read_preference(self): """Requires a replica set called "rs" on port 27017 """ + try: conn = connect(db='mongoenginetest', host="mongodb://localhost/mongoenginetest?replicaSet=rs", read_preference=ReadPreference.SECONDARY_ONLY) except ConnectionError, e: return + if not isinstance(conn, ReplicaSetConnection): + return + self.assertEquals(conn.read_preference, ReadPreference.SECONDARY_ONLY) if __name__ == '__main__': From 0240a09056ec58918691d7da97dec2e0d1294a26 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 1 May 2012 10:14:16 +0100 Subject: [PATCH 0583/1279] Cleaned up ValidationError Refs #459 --- AUTHORS | 3 ++- mongoengine/base.py | 11 +++++---- tests/document.py | 57 +++++++++++++++++++++++++++++++++++++++++++++ tests/fields.py | 38 ------------------------------ 4 files changed, 65 insertions(+), 44 deletions(-) diff --git a/AUTHORS b/AUTHORS index 9363129..0dcc3ef 100644 --- a/AUTHORS +++ b/AUTHORS @@ -101,4 +101,5 @@ that much better: * Nils Hasenbanck * mostlystatic * Greg Banks - * swashbuckler \ No newline at end of file + * swashbuckler + * Adam Reeve \ No newline at end of file diff --git a/mongoengine/base.py b/mongoengine/base.py index 6007d3a..63d2a8f 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -53,9 +53,9 @@ class ValidationError(AssertionError): message = super(ValidationError, self).__getattribute__(name) if name == 'message': if self.field_name: - message += ' ("%s")' % self.field_name + message = '%s ("%s")' % (message, self.field_name) if self.errors: - message += ':\n' + self._format_errors() + message = '%s:\n%s' % (message, self._format_errors()) return message def _get_message(self): @@ -94,12 +94,13 @@ class ValidationError(AssertionError): """Returns a string listing all errors within a document""" def format_error(field, value, prefix=''): + prefix = "%s.%s" % (prefix, field) if prefix else "%s" % field if isinstance(value, dict): - new_prefix = (prefix + '.' if prefix else '') + str(field) + return '\n'.join( - [format_error(k, value[k], new_prefix) for k in value]) + [format_error(k, value[k], prefix) for k in value]) else: - return (prefix + ": " if prefix else '') + str(value) + return "%s: %s" % (prefix, value) return '\n'.join( [format_error(k, v) for k, v in self.to_dict().items()]) diff --git a/tests/document.py b/tests/document.py index f27bb61..54b0b97 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2861,5 +2861,62 @@ class DocumentTest(unittest.TestCase): } ) ]), "1,2") + +class ValidatorErrorTest(unittest.TestCase): + + def test_to_dict(self): + """Ensure a ValidationError handles error to_dict correctly. + """ + error = ValidationError('root') + self.assertEquals(error.to_dict(), {}) + + # 1st level error schema + error.errors = {'1st': ValidationError('bad 1st'), } + self.assertTrue('1st' in error.to_dict()) + self.assertEquals(error.to_dict()['1st'], 'bad 1st') + + # 2nd level error schema + error.errors = {'1st': ValidationError('bad 1st', errors={ + '2nd': ValidationError('bad 2nd'), + })} + self.assertTrue('1st' in error.to_dict()) + self.assertTrue(isinstance(error.to_dict()['1st'], dict)) + self.assertTrue('2nd' in error.to_dict()['1st']) + self.assertEquals(error.to_dict()['1st']['2nd'], 'bad 2nd') + + # moar levels + error.errors = {'1st': ValidationError('bad 1st', errors={ + '2nd': ValidationError('bad 2nd', errors={ + '3rd': ValidationError('bad 3rd', errors={ + '4th': ValidationError('Inception'), + }), + }), + })} + self.assertTrue('1st' in error.to_dict()) + self.assertTrue('2nd' in error.to_dict()['1st']) + self.assertTrue('3rd' in error.to_dict()['1st']['2nd']) + self.assertTrue('4th' in error.to_dict()['1st']['2nd']['3rd']) + self.assertEquals(error.to_dict()['1st']['2nd']['3rd']['4th'], + 'Inception') + + self.assertEquals(error.message, "root:\n1st.2nd.3rd.4th: Inception") + + def test_model_validation(self): + + class User(Document): + username = StringField(primary_key=True) + name = StringField(required=True) + + try: + User().validate() + except ValidationError, e: + expected_error_message = """Errors encountered validating document: +username: Field is required ("username") +name: Field is required ("name")""" + self.assertEquals(e.message, expected_error_message) + self.assertEquals(e.to_dict(), { + 'username': 'Field is required ("username")', + 'name': u'Field is required ("name")'}) + if __name__ == '__main__': unittest.main() diff --git a/tests/fields.py b/tests/fields.py index 85c1b38..81328f8 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1902,43 +1902,5 @@ class FieldTest(unittest.TestCase): post.validate() -class ValidatorErrorTest(unittest.TestCase): - - def test_to_dict(self): - """Ensure a ValidationError handles error to_dict correctly. - """ - error = ValidationError('root') - self.assertEquals(error.to_dict(), {}) - - # 1st level error schema - error.errors = {'1st': ValidationError('bad 1st'), } - self.assertTrue('1st' in error.to_dict()) - self.assertEquals(error.to_dict()['1st'], 'bad 1st') - - # 2nd level error schema - error.errors = {'1st': ValidationError('bad 1st', errors={ - '2nd': ValidationError('bad 2nd'), - })} - self.assertTrue('1st' in error.to_dict()) - self.assertTrue(isinstance(error.to_dict()['1st'], dict)) - self.assertTrue('2nd' in error.to_dict()['1st']) - self.assertEquals(error.to_dict()['1st']['2nd'], 'bad 2nd') - - # moar levels - error.errors = {'1st': ValidationError('bad 1st', errors={ - '2nd': ValidationError('bad 2nd', errors={ - '3rd': ValidationError('bad 3rd', errors={ - '4th': ValidationError('Inception'), - }), - }), - })} - self.assertTrue('1st' in error.to_dict()) - self.assertTrue('2nd' in error.to_dict()['1st']) - self.assertTrue('3rd' in error.to_dict()['1st']['2nd']) - self.assertTrue('4th' in error.to_dict()['1st']['2nd']['3rd']) - self.assertEquals(error.to_dict()['1st']['2nd']['3rd']['4th'], - 'Inception') - - if __name__ == '__main__': unittest.main() From d7765511ee12412d359e4a86632a1f7001881c38 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 1 May 2012 11:03:23 +0100 Subject: [PATCH 0584/1279] Invalid DB Data now raises an InvalidDocumentError fixes #2 --- docs/changelog.rst | 4 ++-- mongoengine/base.py | 17 ++++++++++++++--- tests/document.py | 16 ++++++++++++++++ 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5718fd2..a49ed06 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,8 @@ Changelog Changes in 0.6.X ================ +- Invalid data from the DB now raises a InvalidDocumentError +- Cleaned up the Validation Error - docs and code - Added meta `auto_create_index` so you can disable index creation - Added write concern options to inserts - Fixed typo in meta for index options @@ -281,5 +283,3 @@ Changes in v0.1.2 Changes in v0.1.1 -================= -- Documents may now use capped collections diff --git a/mongoengine/base.py b/mongoengine/base.py index 63d2a8f..b4530c9 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -979,8 +979,8 @@ class BaseDocument(object): """ # get the class name from the document, falling back to the given # class if unavailable - class_name = son.get(u'_cls', cls._class_name) - data = dict((str(key), value) for key, value in son.items()) + class_name = son.get('_cls', cls._class_name) + data = dict(("%s" % key, value) for key, value in son.items()) if '_types' in data: del data['_types'] @@ -993,11 +993,16 @@ class BaseDocument(object): cls = get_document(class_name) changed_fields = [] + errors_dict = {} + for field_name, field in cls._fields.items(): if field.db_field in data: value = data[field.db_field] - data[field_name] = (value if value is None + try: + data[field_name] = (value if value is None else field.to_python(value)) + except (AttributeError, ValueError), e: + errors_dict[field_name] = e elif field.default: default = field.default if callable(default): @@ -1005,7 +1010,13 @@ class BaseDocument(object): if isinstance(default, BaseDocument): changed_fields.append(field_name) + if errors_dict: + errors = "\n".join(["%s - %s" % (k, v) for k, v in errors_dict.items()]) + raise InvalidDocumentError(""" +Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, errors)) + obj = cls(**data) + obj._changed_fields = changed_fields obj._created = False return obj diff --git a/tests/document.py b/tests/document.py index 54b0b97..68fb3a6 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2398,6 +2398,22 @@ class DocumentTest(unittest.TestCase): self.assertRaises(InvalidDocumentError, throw_invalid_document_error) + def test_invalid_son(self): + """Raise an error if loading invalid data""" + class Occurrence(EmbeddedDocument): + number = IntField() + + class Word(Document): + stem = StringField() + count = IntField(default=1) + forms = ListField(StringField(), default=list) + occurs = ListField(EmbeddedDocumentField(Occurrence), default=list) + + def raise_invalid_document(): + Word._from_son({'stem': [1,2,3], 'forms': 1, 'count': 'one', 'occurs': {"hello": None}}) + + self.assertRaises(InvalidDocumentError, raise_invalid_document) + def test_reverse_delete_rule_cascade_and_nullify(self): """Ensure that a referenced document is also deleted upon deletion. """ From f80f0b416fc8f85a876000ebbd8ffa5e3759aed5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 1 May 2012 11:04:01 +0100 Subject: [PATCH 0585/1279] Updated changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a49ed06..b969b37 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -283,3 +283,5 @@ Changes in v0.1.2 Changes in v0.1.1 +================= +- Documents may now use capped collections From ca8b58d66d811fb429e53566272932dc7f9403d4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 1 May 2012 11:27:37 +0100 Subject: [PATCH 0586/1279] Fixed indexing on _id for covered indexes fixes #4 --- docs/changelog.rst | 1 + mongoengine/queryset.py | 118 ++++++++++++++++++++-------------------- setup.py | 2 +- tests/document.py | 20 +++++++ 4 files changed, 82 insertions(+), 59 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b969b37..c7278ba 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Fixed indexing on '_id' or 'pk' or 'id' - Invalid data from the DB now raises a InvalidDocumentError - Cleaned up the Validation Error - docs and code - Added meta `auto_create_index` so you can disable index creation diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b355a19..1e9a095 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -394,61 +394,6 @@ class QuerySet(object): unique=index_spec.get('unique', False)) return self - @classmethod - def _build_index_spec(cls, doc_cls, spec): - """Build a PyMongo index spec from a MongoEngine index spec. - """ - if isinstance(spec, basestring): - spec = {'fields': [spec]} - if isinstance(spec, (list, tuple)): - spec = {'fields': spec} - - index_list = [] - use_types = doc_cls._meta.get('allow_inheritance', True) - for key in spec['fields']: - # Get ASCENDING direction from +, DESCENDING from -, and GEO2D from * - direction = pymongo.ASCENDING - if key.startswith("-"): - direction = pymongo.DESCENDING - elif key.startswith("*"): - direction = pymongo.GEO2D - if key.startswith(("+", "-", "*")): - key = key[1:] - - # Use real field name, do it manually because we need field - # objects for the next part (list field checking) - parts = key.split('.') - fields = QuerySet._lookup_field(doc_cls, parts) - parts = [field.db_field for field in fields] - key = '.'.join(parts) - index_list.append((key, direction)) - - # Check if a list field is being used, don't use _types if it is - if use_types and not all(f._index_with_types for f in fields): - use_types = False - - # If _types is being used, prepend it to every specified index - index_types = doc_cls._meta.get('index_types', True) - allow_inheritance = doc_cls._meta.get('allow_inheritance') - if spec.get('types', index_types) and allow_inheritance and use_types and direction is not pymongo.GEO2D: - index_list.insert(0, ('_types', 1)) - - spec['fields'] = index_list - - if spec.get('sparse', False) and len(spec['fields']) > 1: - raise ValueError( - 'Sparse indexes can only have one field in them. ' - 'See https://jira.mongodb.org/browse/SERVER-2193') - - return spec - - @classmethod - def _reset_already_indexed(cls, document=None): - """Helper to reset already indexed, can be useful for testing purposes""" - if document: - cls.__already_indexed.discard(document) - cls.__already_indexed.clear() - def __call__(self, q_obj=None, class_check=True, slave_okay=False, **query): """Filter the selected documents by calling the :class:`~mongoengine.queryset.QuerySet` with a query. @@ -534,6 +479,62 @@ class QuerySet(object): self._collection.ensure_index(index_spec, background=background, **index_opts) + + @classmethod + def _build_index_spec(cls, doc_cls, spec): + """Build a PyMongo index spec from a MongoEngine index spec. + """ + if isinstance(spec, basestring): + spec = {'fields': [spec]} + if isinstance(spec, (list, tuple)): + spec = {'fields': spec} + + index_list = [] + use_types = doc_cls._meta.get('allow_inheritance', True) + for key in spec['fields']: + # Get ASCENDING direction from +, DESCENDING from -, and GEO2D from * + direction = pymongo.ASCENDING + if key.startswith("-"): + direction = pymongo.DESCENDING + elif key.startswith("*"): + direction = pymongo.GEO2D + if key.startswith(("+", "-", "*")): + key = key[1:] + + # Use real field name, do it manually because we need field + # objects for the next part (list field checking) + parts = key.split('.') + fields = QuerySet._lookup_field(doc_cls, parts) + parts = [field if field == '_id' else field.db_field for field in fields] + key = '.'.join(parts) + index_list.append((key, direction)) + + # Check if a list field is being used, don't use _types if it is + if use_types and not all(f._index_with_types for f in fields): + use_types = False + + # If _types is being used, prepend it to every specified index + index_types = doc_cls._meta.get('index_types', True) + allow_inheritance = doc_cls._meta.get('allow_inheritance') + if spec.get('types', index_types) and allow_inheritance and use_types and direction is not pymongo.GEO2D: + index_list.insert(0, ('_types', 1)) + + spec['fields'] = index_list + if spec.get('sparse', False) and len(spec['fields']) > 1: + raise ValueError( + 'Sparse indexes can only have one field in them. ' + 'See https://jira.mongodb.org/browse/SERVER-2193') + + return spec + + @classmethod + def _reset_already_indexed(cls, document=None): + """Helper to reset already indexed, can be useful for testing purposes""" + if document: + cls.__already_indexed.discard(document) + cls.__already_indexed.clear() + + @property def _collection(self): """Property that returns the collection object. This allows us to @@ -613,10 +614,11 @@ class QuerySet(object): continue if field is None: # Look up first field from the document - if field_name == 'pk': + if field_name in ('pk', 'id', '_id'): # Deal with "primary key" alias - field_name = document._meta['id_field'] - if field_name in document._fields: + field_name = document._meta['id_field'] or '_id' + field = "_id" + elif field_name in document._fields: field = document._fields[field_name] elif document._dynamic: from base import BaseDynamicField diff --git a/setup.py b/setup.py index d92c491..801ff90 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,6 @@ setup(name='mongoengine', platforms=['any'], classifiers=CLASSIFIERS, install_requires=['pymongo'], - test_suite='tests', + test_suite='tests.bugfix', tests_require=['blinker', 'django>=1.3', 'PIL'] ) diff --git a/tests/document.py b/tests/document.py index 68fb3a6..fc86d80 100644 --- a/tests/document.py +++ b/tests/document.py @@ -864,6 +864,26 @@ class DocumentTest(unittest.TestCase): query_plan = Test.objects(a=1).only('a').exclude('id').explain() self.assertTrue(query_plan['indexOnly']) + def test_index_on_id(self): + + class BlogPost(Document): + meta = { + 'indexes': [ + ['categories', 'id'] + ], + 'allow_inheritance': False + } + + title = StringField(required=True) + description = StringField(required=True) + categories = ListField() + + BlogPost.drop_collection() + + indexes = BlogPost.objects._collection.index_information() + self.assertEquals(indexes['categories_1__id_1']['key'], + [('categories', 1), ('_id', 1)]) + def test_hint(self): class BlogPost(Document): From dbe2f5f2b86fa14ee552e8826a2562222749ae86 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 1 May 2012 11:48:57 +0100 Subject: [PATCH 0587/1279] Updated setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 801ff90..d92c491 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,6 @@ setup(name='mongoengine', platforms=['any'], classifiers=CLASSIFIERS, install_requires=['pymongo'], - test_suite='tests.bugfix', + test_suite='tests', tests_require=['blinker', 'django>=1.3', 'PIL'] ) From 5bcbb4fdaac0e0fcf819708a4805a1d7321ee7cc Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 1 May 2012 12:02:45 +0100 Subject: [PATCH 0588/1279] Properly fixed indexing on _id for covered indexes refs #4 --- mongoengine/queryset.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 1e9a095..150cfc4 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -504,9 +504,12 @@ class QuerySet(object): # Use real field name, do it manually because we need field # objects for the next part (list field checking) parts = key.split('.') - fields = QuerySet._lookup_field(doc_cls, parts) - parts = [field if field == '_id' else field.db_field for field in fields] - key = '.'.join(parts) + if parts in (['pk'], ['id'], ['_id']): + key = '_id' + else: + fields = QuerySet._lookup_field(doc_cls, parts) + parts = [field if field == '_id' else field.db_field for field in fields] + key = '.'.join(parts) index_list.append((key, direction)) # Check if a list field is being used, don't use _types if it is @@ -614,11 +617,10 @@ class QuerySet(object): continue if field is None: # Look up first field from the document - if field_name in ('pk', 'id', '_id'): + if field_name == 'pk': # Deal with "primary key" alias - field_name = document._meta['id_field'] or '_id' - field = "_id" - elif field_name in document._fields: + field_name = document._meta['id_field'] + if field_name in document._fields: field = document._fields[field_name] elif document._dynamic: from base import BaseDynamicField From 233b13d670e25e13542978e591c51d47f04de93f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 1 May 2012 12:03:33 +0100 Subject: [PATCH 0589/1279] Version bump --- docs/changelog.rst | 2 +- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 24 ++---------------------- 3 files changed, 4 insertions(+), 24 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c7278ba..c84bbe8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,7 @@ Changelog ========= -Changes in 0.6.X +Changes in 0.6.7 ================ - Fixed indexing on '_id' or 'pk' or 'id' - Invalid data from the DB now raises a InvalidDocumentError diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index f2ff3d9..51b42e6 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 6) +VERSION = (0, 6, 7) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 6239b8a..47c4db5 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.6 +Version: 0.6.7 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB @@ -51,24 +51,4 @@ rm -rf $RPM_BUILD_ROOT # %{python_sitearch}/* %changelog -* Wed Apr 24 2012 Ross Lawley 0.6.5 -- 0.6.6 released -* Wed Apr 18 2012 Ross Lawley 0.6.5 -- 0.6.5 released -* Wed Apr 18 2012 Ross Lawley 0.6.5 -- 0.6.4 released -* Wed Mar 24 2012 Ross Lawley 0.6.5 -- 0.6.3 released -* Wed Mar 22 2012 Ross Lawley 0.6.5 -- 0.6.2 released -* Wed Mar 05 2012 Ross Lawley 0.6.5 -- 0.6.1 released -* Mon Mar 05 2012 Ross Lawley 0.6 -- 0.6 released -* Thu Oct 27 2011 Pau Aliagas 0.5.3-1 -- Update to latest dev version -- Add PIL dependency for ImageField -* Wed Oct 12 2011 Pau Aliagas 0.5.2-1 -- Update version -* Fri Sep 23 2011 Pau Aliagas 0.5.0-1 -- Initial version +* See: http://readthedocs.org/docs/mongoengine-odm/en/latest/changelog.html \ No newline at end of file From 3360b72531d4d702b09c205396a356bd78a414da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 4 May 2012 14:58:06 -0300 Subject: [PATCH 0590/1279] Small fixes for GenericReferenceField --- mongoengine/fields.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index a2250f4..f734448 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -734,6 +734,9 @@ class GenericReferenceField(BaseField): def to_mongo(self, document): if document is None: return None + + if isinstance(document, (dict, SON)): + return document id_field_name = document.__class__._meta['id_field'] id_field = document.__class__._fields[id_field_name] From e07ecc5cf8a845d7b76a336c943dd857e06eaacb Mon Sep 17 00:00:00 2001 From: David Ignacio Date: Sat, 5 May 2012 01:32:43 -0400 Subject: [PATCH 0591/1279] Cleanup referenced GridFS files when a document is deleted Note that drop_collection is not modified since there is no guarantee that a GridFS collection holds files for only one Document class. Otherwise you could drop files for other fields or documents accidentally. --- mongoengine/base.py | 5 +++++ mongoengine/document.py | 12 ++++++++++++ tests/fields.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index b4530c9..995f132 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -616,6 +616,10 @@ class DocumentMetaclass(type): raise InvalidDocumentError("Reverse delete rules are not supported for EmbeddedDocuments (field: %s)" % field.name) f.document_type.register_delete_rule(new_class, field.name, delete_rule) + proxy_class = getattr(field, 'proxy_class', None) + if proxy_class is not None: + new_class.register_proxy_field(field.name, proxy_class) + if field.name and hasattr(Document, field.name) and EmbeddedDocument not in new_class.mro(): raise InvalidDocumentError("%s is a document method and not a valid field name" % field.name) @@ -717,6 +721,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): 'index_opts': {}, 'queryset_class': QuerySet, 'delete_rules': {}, + 'proxy_fields': {}, 'allow_inheritance': True } diff --git a/mongoengine/document.py b/mongoengine/document.py index 1d89cff..fb9c4bc 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -277,6 +277,11 @@ class Document(BaseDocument): signals.pre_delete.send(self.__class__, document=self) try: + for field_name in self._meta['proxy_fields']: + proxy_class = self._meta['proxy_fields'][field_name] + if hasattr(proxy_class, 'delete'): + proxy = getattr(self, field_name) + proxy.delete() self.__class__.objects(pk=self.pk).delete(safe=safe) except pymongo.errors.OperationFailure, err: message = u'Could not delete document (%s)' % err.message @@ -341,6 +346,13 @@ class Document(BaseDocument): """ cls._meta['delete_rules'][(document_cls, field_name)] = rule + @classmethod + def register_proxy_field(cls, field_name, proxy_class): + """This method registers fields with proxy classes to delete them when + removing this object. + """ + cls._meta['proxy_fields'][field_name] = proxy_class + @classmethod def drop_collection(cls): """Drops the entire collection associated with this diff --git a/tests/fields.py b/tests/fields.py index 81328f8..31d3b58 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1546,6 +1546,39 @@ class FieldTest(unittest.TestCase): file = FileField() DemoFile.objects.create() + def test_file_delete_cleanup(self): + """Ensure that the gridfs file is deleted when a document + with a GridFSProxied Field is deleted""" + class TestFile(Document): + file = FileField() + + class TestImage(Document): + image = ImageField() + + TestFile.drop_collection() + + testfile = TestFile() + testfile.file.put('Hello, World!') + testfile.save() + + testfile_grid_id = testfile.file.grid_id + testfile_fs = testfile.file.fs + + testfile.delete() + self.assertFalse(testfile_fs.exists(testfile_grid_id)) + + TestImage.drop_collection() + + testimage = TestImage() + testimage.image.put(open(TEST_IMAGE_PATH, 'r')) + testimage.save() + + testimage_grid_id = testimage.image.grid_id + testimage_fs = testimage.image.fs + + testimage.delete() + self.assertFalse(testimage_fs.exists(testimage_grid_id)) + def test_file_uniqueness(self): """Ensure that each instance of a FileField is unique """ From 20e41b35237da7c34956fdad5e08ab038a7f58f4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 8 May 2012 18:23:51 +0100 Subject: [PATCH 0592/1279] Make the user model extendable --- mongoengine/django/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 694a4ca..a30fc57 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -66,6 +66,7 @@ class User(Document): verbose_name=_('date joined')) meta = { + 'allow_inheritance': True, 'indexes': [ {'fields': ['username'], 'unique': True} ] From be1c28fc452c8c09a888a309f46c9b6d300e7bae Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 8 May 2012 18:26:04 +0100 Subject: [PATCH 0593/1279] Updated changelog --- docs/changelog.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index c84bbe8..d832318 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.6.X +================ +- Django User document allows inheritance + Changes in 0.6.7 ================ - Fixed indexing on '_id' or 'pk' or 'id' From dd786d6fc4b4e1165d192870c1a09bf854038822 Mon Sep 17 00:00:00 2001 From: Anthony Nemitz Date: Wed, 9 May 2012 02:54:08 -0700 Subject: [PATCH 0594/1279] fix for #494 --- mongoengine/connection.py | 2 ++ tests/connection.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 7585c73..1ccbbe3 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -65,6 +65,8 @@ def register_connection(alias, name, host='localhost', port=27017, }) if "replicaSet" in host: conn_settings['replicaSet'] = True + + conn_settings.update(kwargs) _connection_settings[alias] = conn_settings diff --git a/tests/connection.py b/tests/connection.py index 9aad142..91bba55 100644 --- a/tests/connection.py +++ b/tests/connection.py @@ -65,6 +65,16 @@ class ConnectionTest(unittest.TestCase): self.assertTrue(isinstance(db, pymongo.database.Database)) self.assertEqual(db.name, 'mongoenginetest2') + def test_connection_kwargs(self): + """Ensure that connection kwargs get passed to pymongo. + """ + connect('mongoenginetest', alias='t1', tz_aware=True) + conn = get_connection('t1') + self.assertTrue(conn.tz_aware) + + connect('mongoenginetest2', alias='t2') + conn = get_connection('t2') + self.assertFalse(conn.tz_aware) if __name__ == '__main__': unittest.main() From 27734a7c267dae9cd21b56cc26fa31c73bb27bbb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 May 2012 11:04:05 +0100 Subject: [PATCH 0595/1279] Updated Author / changelog --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 0dcc3ef..fc28b26 100644 --- a/AUTHORS +++ b/AUTHORS @@ -102,4 +102,5 @@ that much better: * mostlystatic * Greg Banks * swashbuckler - * Adam Reeve \ No newline at end of file + * Adam Reeve + * Anthony Nemitz \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index d832318..a418646 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Fixed connection regression - Django User document allows inheritance Changes in 0.6.7 From 97c5b957dda048726cec4d86551532eacf7f271b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 May 2012 11:39:30 +0100 Subject: [PATCH 0596/1279] Updated docs regarding GridFS refs #492 --- AUTHORS | 3 ++- docs/changelog.rst | 2 ++ docs/guide/gridfs.rst | 7 +++---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/AUTHORS b/AUTHORS index fc28b26..2a91373 100644 --- a/AUTHORS +++ b/AUTHORS @@ -103,4 +103,5 @@ that much better: * Greg Banks * swashbuckler * Adam Reeve - * Anthony Nemitz \ No newline at end of file + * Anthony Nemitz + * deignacio \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index a418646..030266a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,8 @@ Changelog Changes in 0.6.X ================ +- FileField now automatically delete files on .delete() +- Fix for GenericReference to_mongo method - Fixed connection regression - Django User document allows inheritance diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst index 3abad77..695a0be 100644 --- a/docs/guide/gridfs.rst +++ b/docs/guide/gridfs.rst @@ -68,10 +68,9 @@ Deleting stored files is achieved with the :func:`delete` method:: .. note:: The FileField in a Document actually only stores the ID of a file in a - separate GridFS collection. This means that deleting a document - with a defined FileField does not actually delete the file. You must be - careful to delete any files in a Document as above before deleting the - Document itself. + separate GridFS collection. This means that `Animal.drop_collection()` will + not delete any files. Care should be taken to manually remove associated + files before dropping a collection. Replacing files From f48af8db3ba1d6ca8d4ad7328ed3805f2599492e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 May 2012 12:00:05 +0100 Subject: [PATCH 0597/1279] Django 1.4 first session save lost data fixes #477 --- docs/changelog.rst | 1 + mongoengine/django/sessions.py | 2 +- tests/django_tests.py | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 030266a..d93bf7b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Fixed Django 1.4 sessions first save data loss - FileField now automatically delete files on .delete() - Fix for GenericReference to_mongo method - Fixed connection regression diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index d1e9289..f178342 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -55,7 +55,7 @@ class SessionStore(SessionBase): def save(self, must_create=False): if self.session_key is None: - self.create() + self._session_key = self._get_new_session_key() s = MongoSession(session_key=self.session_key) s.session_data = self.encode(self._get_session(no_load=must_create)) s.expire_date = self.get_expiry_date() diff --git a/tests/django_tests.py b/tests/django_tests.py index 12ea1d1..f5e9624 100644 --- a/tests/django_tests.py +++ b/tests/django_tests.py @@ -103,3 +103,8 @@ class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): MongoSession.drop_collection() super(MongoDBSessionTest, self).setUp() + def test_first_save(self): + session = SessionStore() + session['test'] = True + session.save() + self.assertTrue('test' in session) From eb54037b66ef66429f9027d9a747d1bff12a3256 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 May 2012 12:08:00 +0100 Subject: [PATCH 0598/1279] Added note that get_or_create contains a race condition Refs #478 --- mongoengine/queryset.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 150cfc4..dc5bb80 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -793,6 +793,9 @@ class QuerySet(object): dictionary of default values for the new document may be provided as a keyword argument called :attr:`defaults`. + .. note:: This requires two separate operations and therefore a + potential race condition exists. + :param write_options: optional extra keyword arguments used if we have to create a new document. Passes any write_options onto :meth:`~mongoengine.Document.save` From 5c4b33e8e6a015904cac5fb4699c1affc528e15b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 May 2012 12:13:42 +0100 Subject: [PATCH 0599/1279] Made note stronger re: race condition Refs #478 --- mongoengine/queryset.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index dc5bb80..e3f83c7 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -794,7 +794,9 @@ class QuerySet(object): keyword argument called :attr:`defaults`. .. note:: This requires two separate operations and therefore a - potential race condition exists. + race condition exists. Because there are no transactions in mongoDB + other approaches should be investigated, to ensure you don't + accidently duplicate data when using this method. :param write_options: optional extra keyword arguments used if we have to create a new document. From debfcdf4983055a906173a9ae55491f53817643a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 May 2012 12:19:59 +0100 Subject: [PATCH 0600/1279] Updated docs re: required pymongo version refs #472 --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a8588c8..168b534 100644 --- a/README.rst +++ b/README.rst @@ -22,7 +22,7 @@ setup.py install``. Dependencies ============ -- pymongo 1.1+ +- pymongo 2.1.1+ - sphinx (optional - for documentation generation) Examples From aeebdfec51a112add0210c99844227555313c563 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 May 2012 12:58:45 +0100 Subject: [PATCH 0601/1279] Implemented Choices for GenericEmbeddedDocuments Refs mongoengine/mongoengine#13 --- docs/changelog.rst | 3 ++- mongoengine/base.py | 15 +++++++------ mongoengine/fields.py | 6 +++--- tests/fields.py | 50 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d93bf7b..628c1ee 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,11 +4,12 @@ Changelog Changes in 0.6.X ================ +- Added choices for GenericEmbeddedDocuments - Fixed Django 1.4 sessions first save data loss - FileField now automatically delete files on .delete() - Fix for GenericReference to_mongo method - Fixed connection regression -- Django User document allows inheritance +- Updated Django User document, now allows inheritance Changes in 0.6.7 ================ diff --git a/mongoengine/base.py b/mongoengine/base.py index 995f132..139c326 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -223,16 +223,19 @@ class BaseField(object): pass def _validate(self, value): + from mongoengine import EmbeddedDocument # check choices if self.choices: + is_cls = isinstance(value, EmbeddedDocument) + value_to_check = value.__class__ if is_cls else value + err_msg = 'an instance' if is_cls else 'one' if isinstance(self.choices[0], (list, tuple)): option_keys = [option_key for option_key, option_value in self.choices] - if value not in option_keys: - self.error('Value must be one of %s' % unicode(option_keys)) - else: - if value not in self.choices: - self.error('Value must be one of %s' % unicode(self.choices)) + if value_to_check not in option_keys: + self.error('Value must be %s of %s' % (err_msg, unicode(option_keys))) + elif value_to_check not in self.choices: + self.error('Value must be %s of %s' % (err_msg, unicode(self.choices))) # check validation argument if self.validation is not None: @@ -400,7 +403,7 @@ class ComplexBaseField(BaseField): sequence = enumerate(value) for k, v in sequence: try: - self.field.validate(v) + self.field._validate(v) except (ValidationError, AssertionError), error: if hasattr(error, 'errors'): errors[k] = error.errors diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f734448..2e614d2 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -659,7 +659,7 @@ class ReferenceField(BaseField): def to_mongo(self, document): if isinstance(document, DBRef): return document - + id_field_name = self.document_type._meta['id_field'] id_field = self.document_type._fields[id_field_name] @@ -734,9 +734,9 @@ class GenericReferenceField(BaseField): def to_mongo(self, document): if document is None: return None - + if isinstance(document, (dict, SON)): - return document + return document id_field_name = document.__class__._meta['id_field'] id_field = document.__class__._fields[id_field_name] diff --git a/tests/fields.py b/tests/fields.py index 31d3b58..9b9ea98 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1877,6 +1877,8 @@ class FieldTest(unittest.TestCase): name = StringField() like = GenericEmbeddedDocumentField() + Person.drop_collection() + person = Person(name='Test User') person.like = Car(name='Fiat') person.save() @@ -1890,6 +1892,54 @@ class FieldTest(unittest.TestCase): person = Person.objects.first() self.assertTrue(isinstance(person.like, Dish)) + def test_generic_embedded_document_choices(self): + class Car(EmbeddedDocument): + name = StringField() + + class Dish(EmbeddedDocument): + food = StringField(required=True) + number = IntField() + + class Person(Document): + name = StringField() + like = GenericEmbeddedDocumentField(choices=(Dish,)) + + Person.drop_collection() + + person = Person(name='Test User') + person.like = Car(name='Fiat') + self.assertRaises(ValidationError, person.validate) + + person.like = Dish(food="arroz", number=15) + person.save() + + person = Person.objects.first() + self.assertTrue(isinstance(person.like, Dish)) + + def test_generic_list_embedded_document_choices(self): + class Car(EmbeddedDocument): + name = StringField() + + class Dish(EmbeddedDocument): + food = StringField(required=True) + number = IntField() + + class Person(Document): + name = StringField() + likes = ListField(GenericEmbeddedDocumentField(choices=(Dish,))) + + Person.drop_collection() + + person = Person(name='Test User') + person.likes = [Car(name='Fiat')] + self.assertRaises(ValidationError, person.validate) + + person.likes = [Dish(food="arroz", number=15)] + person.save() + + person = Person.objects.first() + self.assertTrue(isinstance(person.likes[0], Dish)) + def test_recursive_validation(self): """Ensure that a validation result to_dict is available. """ From 2aa8b04c21c180546ee6de7643a9f035257fb25e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 May 2012 13:21:53 +0100 Subject: [PATCH 0602/1279] Implemented Choices for GenericReferenceFields Refs mongoengine/mongoengine#13 --- docs/changelog.rst | 3 +- mongoengine/base.py | 5 ++- mongoengine/fields.py | 5 +++ tests/fields.py | 73 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 628c1ee..c101937 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,8 @@ Changelog Changes in 0.6.X ================ -- Added choices for GenericEmbeddedDocuments +- Added support for choices with GenericReferenceFields +- Added support for choices with GenericEmbeddedDocumentFields - Fixed Django 1.4 sessions first save data loss - FileField now automatically delete files on .delete() - Fix for GenericReference to_mongo method diff --git a/mongoengine/base.py b/mongoengine/base.py index 139c326..347332b 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -223,11 +223,10 @@ class BaseField(object): pass def _validate(self, value): - from mongoengine import EmbeddedDocument - + from mongoengine import Document, EmbeddedDocument # check choices if self.choices: - is_cls = isinstance(value, EmbeddedDocument) + is_cls = isinstance(value, (Document, EmbeddedDocument)) value_to_check = value.__class__ if is_cls else value err_msg = 'an instance' if is_cls else 'one' if isinstance(self.choices[0], (list, tuple)): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 2e614d2..3e8f09a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -441,6 +441,9 @@ class GenericEmbeddedDocumentField(BaseField): :class:`~mongoengine.EmbeddedDocument` to be stored. Only valid values are subclasses of :class:`~mongoengine.EmbeddedDocument`. + + ..note :: You can use the choices param to limit the acceptable + EmbeddedDocument types """ def prepare_query_value(self, op, value): @@ -701,6 +704,8 @@ class GenericReferenceField(BaseField): ..note :: Any documents used as a generic reference must be registered in the document registry. Importing the model will automatically register it. + ..note :: You can use the choices param to limit the acceptable Document types + .. versionadded:: 0.3 """ diff --git a/tests/fields.py b/tests/fields.py index 9b9ea98..ea5262d 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1301,6 +1301,74 @@ class FieldTest(unittest.TestCase): self.assertEquals(repr(Person.objects(city=None)), "[]") + + def test_generic_reference_choices(self): + """Ensure that a GenericReferenceField can handle choices + """ + class Link(Document): + title = StringField() + + class Post(Document): + title = StringField() + + class Bookmark(Document): + bookmark_object = GenericReferenceField(choices=(Post,)) + + Link.drop_collection() + Post.drop_collection() + Bookmark.drop_collection() + + link_1 = Link(title="Pitchfork") + link_1.save() + + post_1 = Post(title="Behind the Scenes of the Pavement Reunion") + post_1.save() + + bm = Bookmark(bookmark_object=link_1) + self.assertRaises(ValidationError, bm.validate) + + bm = Bookmark(bookmark_object=post_1) + bm.save() + + bm = Bookmark.objects.first() + self.assertEqual(bm.bookmark_object, post_1) + + def test_generic_reference_list_choices(self): + """Ensure that a ListField properly dereferences generic references and + respects choices. + """ + class Link(Document): + title = StringField() + + class Post(Document): + title = StringField() + + class User(Document): + bookmarks = ListField(GenericReferenceField(choices=(Post,))) + + Link.drop_collection() + Post.drop_collection() + User.drop_collection() + + link_1 = Link(title="Pitchfork") + link_1.save() + + post_1 = Post(title="Behind the Scenes of the Pavement Reunion") + post_1.save() + + user = User(bookmarks=[link_1]) + self.assertRaises(ValidationError, user.validate) + + user = User(bookmarks=[post_1]) + user.save() + + user = User.objects.first() + self.assertEqual(user.bookmarks, [post_1]) + + Link.drop_collection() + Post.drop_collection() + User.drop_collection() + def test_binary_fields(self): """Ensure that binary fields can be stored and retrieved. """ @@ -1893,6 +1961,8 @@ class FieldTest(unittest.TestCase): self.assertTrue(isinstance(person.like, Dish)) def test_generic_embedded_document_choices(self): + """Ensure you can limit GenericEmbeddedDocument choices + """ class Car(EmbeddedDocument): name = StringField() @@ -1917,6 +1987,9 @@ class FieldTest(unittest.TestCase): self.assertTrue(isinstance(person.like, Dish)) def test_generic_list_embedded_document_choices(self): + """Ensure you can limit GenericEmbeddedDocument choices inside a list + field + """ class Car(EmbeddedDocument): name = StringField() From bbefd0fdf9921846d76419f03577fdf0d7b27958 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 May 2012 13:54:33 +0100 Subject: [PATCH 0603/1279] Added example of bi directional delete rules + test refs mongoengine/mongoengine#15 --- mongoengine/fields.py | 11 +++++++++++ tests/document.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 3e8f09a..d502aa0 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -616,6 +616,17 @@ class ReferenceField(BaseField): * CASCADE - Deletes the documents associated with the reference. * DENY - Prevent the deletion of the reference object. + Alternative syntax for registering delete rules (useful when implementing + bi-directional delete rules) + + .. code-block:: python + + class Bar(Document): + content = StringField() + foo = ReferenceField('Foo') + + Bar.register_delete_rule(Foo, 'bar', NULLIFY) + .. versionchanged:: 0.5 added `reverse_delete_rule` """ diff --git a/tests/document.py b/tests/document.py index fc86d80..2113f84 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2496,6 +2496,40 @@ class DocumentTest(unittest.TestCase): author.delete() self.assertEqual(len(BlogPost.objects), 0) + def test_two_way_reverse_delete_rule(self): + """Ensure that Bi-Directional relationships work with + reverse_delete_rule + """ + + class Bar(Document): + content = StringField() + foo = ReferenceField('Foo') + + class Foo(Document): + content = StringField() + bar = ReferenceField(Bar) + + Bar.register_delete_rule(Foo, 'bar', NULLIFY) + Foo.register_delete_rule(Bar, 'foo', NULLIFY) + + + Bar.drop_collection() + Foo.drop_collection() + + b = Bar(content="Hello") + b.save() + + f = Foo(content="world", bar=b) + f.save() + + b.foo = f + b.save() + + f.delete() + + self.assertEqual(len(Bar.objects), 1) # No effect on the BlogPost + self.assertEqual(Bar.objects.get().foo, None) + def test_invalid_reverse_delete_rules_raise_errors(self): def throw_invalid_document_error(): From a536097804d06355d75bb2b22e1ec66edd0b217b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 May 2012 14:38:53 +0100 Subject: [PATCH 0604/1279] Added support for pull operations on nested EmbeddedDocs fixes mongoengine/mongoengine#16 --- docs/changelog.rst | 1 + mongoengine/queryset.py | 13 +++++++++++-- tests/queryset.py | 31 ++++++++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c101937..3e93fc9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Added support for pull operations on nested EmbeddedDocuments - Added support for choices with GenericReferenceFields - Added support for choices with GenericEmbeddedDocumentFields - Fixed Django 1.4 sessions first save data loss diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index e3f83c7..6d8cc7a 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1380,9 +1380,18 @@ class QuerySet(object): if not op: raise InvalidQueryError("Updates must supply an operation eg: set__FIELD=value") - if op: + if 'pull' in op and '.' in key: + # Dot operators don't work on pull operations + # it uses nested dict syntax + if op == 'pullAll': + raise InvalidQueryError("pullAll operations only support a single field depth") + + parts.reverse() + for key in parts: + value = {key: value} + else: value = {key: value} - key = '$' + op + key = '$' + op if key not in mongo_update: mongo_update[key] = value diff --git a/tests/queryset.py b/tests/queryset.py index 4cf1165..3b66248 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -572,7 +572,7 @@ class QuerySetTest(unittest.TestCase): def throw_operation_error_not_unique(): Blog.objects.insert([blog2, blog3], safe=True) - + self.assertRaises(OperationError, throw_operation_error_not_unique) self.assertEqual(Blog.objects.count(), 2) @@ -1471,6 +1471,35 @@ class QuerySetTest(unittest.TestCase): post.reload() self.assertEqual(post.tags, ["code", "mongodb"]) + def test_pull_nested(self): + + class User(Document): + name = StringField() + + class Collaborator(EmbeddedDocument): + user = StringField() + + def __unicode__(self): + return '%s' % self.user + + class Site(Document): + name = StringField(max_length=75, unique=True, required=True) + collaborators = ListField(EmbeddedDocumentField(Collaborator)) + + + Site.drop_collection() + + c = Collaborator(user='Esteban') + s = Site(name="test", collaborators=[c]) + s.save() + + Site.objects(id=s.id).update_one(pull__collaborators__user='Esteban') + self.assertEqual(Site.objects.first().collaborators, []) + + def pull_all(): + Site.objects(id=s.id).update_one(pull_all__collaborators__user=['Ross']) + + self.assertRaises(InvalidQueryError, pull_all) def test_update_one_pop_generic_reference(self): From 0479bea40baf168945297773e18e87e14c41e4f8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 May 2012 15:35:28 +0100 Subject: [PATCH 0605/1279] Cleaned up GridFS refs hmarr/mongoengine#465 --- mongoengine/fields.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index d502aa0..96e11f5 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -946,12 +946,11 @@ class FileField(BaseField): # Check if a file already exists for this model grid_file = instance._data.get(self.name) - self.grid_file = grid_file - if isinstance(self.grid_file, self.proxy_class): - if not self.grid_file.key: - self.grid_file.key = self.name - self.grid_file.instance = instance - return self.grid_file + if isinstance(grid_file, self.proxy_class): + if not grid_file.key: + grid_file.key = self.name + grid_file.instance = instance + return grid_file return self.proxy_class(key=self.name, instance=instance, db_alias=self.db_alias, collection_name=self.collection_name) From ba298c3cfc97647081fc3554552aa40ae0029cf1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 May 2012 15:37:07 +0100 Subject: [PATCH 0606/1279] Save can be used in assignment --- docs/changelog.rst | 1 + mongoengine/document.py | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3e93fc9..bf4c8c1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Added assignment to save, can now do: b = MyDoc(**kwargs).save() - Added support for pull operations on nested EmbeddedDocuments - Added support for choices with GenericReferenceFields - Added support for choices with GenericEmbeddedDocumentFields diff --git a/mongoengine/document.py b/mongoengine/document.py index fb9c4bc..7f33812 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -233,6 +233,7 @@ class Document(BaseDocument): self._changed_fields = [] self._created = False signals.post_save.send(self.__class__, document=self, created=created) + return self def cascade_save(self, *args, **kwargs): """Recursively saves any references / generic references on an object""" From 0ff6531953e96b24877107860cad260f0afe461e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 May 2012 15:39:34 +0100 Subject: [PATCH 0607/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index bf4c8c1..ae680d5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Removed possible race condition from FileField (grid_file) - Added assignment to save, can now do: b = MyDoc(**kwargs).save() - Added support for pull operations on nested EmbeddedDocuments - Added support for choices with GenericReferenceFields From 45e015d71d9b3858f0db523efdfe7c598d138e74 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 May 2012 20:49:34 +0100 Subject: [PATCH 0608/1279] Added test for keys with spaces --- tests/document.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/document.py b/tests/document.py index 2113f84..055f980 100644 --- a/tests/document.py +++ b/tests/document.py @@ -2988,5 +2988,21 @@ name: Field is required ("name")""" 'username': 'Field is required ("username")', 'name': u'Field is required ("name")'}) + def test_spaces_in_keys(self): + + class Embedded(DynamicEmbeddedDocument): + pass + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + setattr(doc, 'hello world', 1) + doc.save() + + one = Doc.objects.filter(**{'hello world': 1}).count() + self.assertEqual(1, one) + if __name__ == '__main__': unittest.main() From 97114b59488dcdda8d72d33bc18b68252a12976d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 May 2012 20:50:11 +0100 Subject: [PATCH 0609/1279] Fix for FileField losing ref without default fixes hmarr/mongoengine#458 --- docs/changelog.rst | 1 + mongoengine/fields.py | 18 ++++++++-------- tests/fields.py | 48 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ae680d5..6fa9ea5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Fixed FileField losing reference when no default set - Removed possible race condition from FileField (grid_file) - Added assignment to save, can now do: b = MyDoc(**kwargs).save() - Added support for pull operations on nested EmbeddedDocuments diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 96e11f5..f88e5c1 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -946,14 +946,16 @@ class FileField(BaseField): # Check if a file already exists for this model grid_file = instance._data.get(self.name) - if isinstance(grid_file, self.proxy_class): - if not grid_file.key: - grid_file.key = self.name - grid_file.instance = instance - return grid_file - return self.proxy_class(key=self.name, instance=instance, - db_alias=self.db_alias, - collection_name=self.collection_name) + if not isinstance(grid_file, self.proxy_class): + grid_file = self.proxy_class(key=self.name, instance=instance, + db_alias=self.db_alias, + collection_name=self.collection_name) + instance._data[self.name] = grid_file + + if not grid_file.key: + grid_file.key = self.name + grid_file.instance = instance + return grid_file def __set__(self, instance, value): key = self.name diff --git a/tests/fields.py b/tests/fields.py index ea5262d..b75ef0a 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -3,6 +3,8 @@ import os import unittest import uuid import StringIO +import tempfile +import gridfs from decimal import Decimal @@ -19,6 +21,10 @@ class FieldTest(unittest.TestCase): connect(db='mongoenginetest') self.db = get_db() + def tearDown(self): + self.db.drop_collection('fs.files') + self.db.drop_collection('fs.chunks') + def test_default_values(self): """Ensure that default field values are used when creating a document. """ @@ -1647,6 +1653,48 @@ class FieldTest(unittest.TestCase): testimage.delete() self.assertFalse(testimage_fs.exists(testimage_grid_id)) + def test_file_field_no_default(self): + + class GridDocument(Document): + the_file = FileField() + + GridDocument.drop_collection() + + with tempfile.TemporaryFile() as f: + f.write("Hello World!") + f.flush() + + # Test without default + doc_a = GridDocument() + doc_a.save() + + + doc_b = GridDocument.objects.with_id(doc_a.id) + doc_b.the_file.replace(f, filename='doc_b') + doc_b.save() + self.assertNotEquals(doc_b.the_file.grid_id, None) + + # Test it matches + doc_c = GridDocument.objects.with_id(doc_b.id) + self.assertEquals(doc_b.the_file.grid_id, doc_c.the_file.grid_id) + + # Test with default + doc_d = GridDocument(the_file='') + doc_d.save() + + doc_e = GridDocument.objects.with_id(doc_d.id) + self.assertEquals(doc_d.the_file.grid_id, doc_e.the_file.grid_id) + + doc_e.the_file.replace(f, filename='doc_e') + doc_e.save() + + doc_f = GridDocument.objects.with_id(doc_e.id) + self.assertEquals(doc_e.the_file.grid_id, doc_f.the_file.grid_id) + + db = GridDocument._get_db() + grid_fs = gridfs.GridFS(db) + self.assertEquals(['doc_b', 'doc_e'], grid_fs.list()) + def test_file_uniqueness(self): """Ensure that each instance of a FileField is unique """ From bc7e874476bbaad16e42b80286343db61ba0a0b5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 May 2012 20:53:18 +0100 Subject: [PATCH 0610/1279] Version 0.6.8 --- docs/changelog.rst | 2 +- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6fa9ea5..3231084 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,7 @@ Changelog ========= -Changes in 0.6.X +Changes in 0.6.8 ================ - Fixed FileField losing reference when no default set - Removed possible race condition from FileField (grid_file) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 51b42e6..164aae8 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 7) +VERSION = (0, 6, 8) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 47c4db5..8ab4515 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.7 +Version: 0.6.8 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From bab186e1950de5902c914f8746f706802e31dfeb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 14 May 2012 12:02:07 +0100 Subject: [PATCH 0611/1279] Reverted document.delete auto gridfs delete --- docs/changelog.rst | 4 ++++ docs/guide/gridfs.rst | 9 +++++---- mongoengine/base.py | 5 ----- mongoengine/document.py | 12 ------------ tests/fields.py | 32 -------------------------------- 5 files changed, 9 insertions(+), 53 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3231084..44ae826 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.6.9 +================ +- Removed FileField auto deletion, needs more work maybe 0.7 + Changes in 0.6.8 ================ - Fixed FileField losing reference when no default set diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst index 695a0be..9c80a99 100644 --- a/docs/guide/gridfs.rst +++ b/docs/guide/gridfs.rst @@ -65,12 +65,13 @@ Deleting stored files is achieved with the :func:`delete` method:: marmot.photo.delete() -.. note:: +.. warning:: The FileField in a Document actually only stores the ID of a file in a - separate GridFS collection. This means that `Animal.drop_collection()` will - not delete any files. Care should be taken to manually remove associated - files before dropping a collection. + separate GridFS collection. This means that deleting a document + with a defined FileField does not actually delete the file. You must be + careful to delete any files in a Document as above before deleting the + Document itself. Replacing files diff --git a/mongoengine/base.py b/mongoengine/base.py index 347332b..ec7af45 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -618,10 +618,6 @@ class DocumentMetaclass(type): raise InvalidDocumentError("Reverse delete rules are not supported for EmbeddedDocuments (field: %s)" % field.name) f.document_type.register_delete_rule(new_class, field.name, delete_rule) - proxy_class = getattr(field, 'proxy_class', None) - if proxy_class is not None: - new_class.register_proxy_field(field.name, proxy_class) - if field.name and hasattr(Document, field.name) and EmbeddedDocument not in new_class.mro(): raise InvalidDocumentError("%s is a document method and not a valid field name" % field.name) @@ -723,7 +719,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): 'index_opts': {}, 'queryset_class': QuerySet, 'delete_rules': {}, - 'proxy_fields': {}, 'allow_inheritance': True } diff --git a/mongoengine/document.py b/mongoengine/document.py index 7f33812..9e281f5 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -278,11 +278,6 @@ class Document(BaseDocument): signals.pre_delete.send(self.__class__, document=self) try: - for field_name in self._meta['proxy_fields']: - proxy_class = self._meta['proxy_fields'][field_name] - if hasattr(proxy_class, 'delete'): - proxy = getattr(self, field_name) - proxy.delete() self.__class__.objects(pk=self.pk).delete(safe=safe) except pymongo.errors.OperationFailure, err: message = u'Could not delete document (%s)' % err.message @@ -347,13 +342,6 @@ class Document(BaseDocument): """ cls._meta['delete_rules'][(document_cls, field_name)] = rule - @classmethod - def register_proxy_field(cls, field_name, proxy_class): - """This method registers fields with proxy classes to delete them when - removing this object. - """ - cls._meta['proxy_fields'][field_name] = proxy_class - @classmethod def drop_collection(cls): """Drops the entire collection associated with this diff --git a/tests/fields.py b/tests/fields.py index b75ef0a..85f2911 100644 --- a/tests/fields.py +++ b/tests/fields.py @@ -1620,38 +1620,6 @@ class FieldTest(unittest.TestCase): file = FileField() DemoFile.objects.create() - def test_file_delete_cleanup(self): - """Ensure that the gridfs file is deleted when a document - with a GridFSProxied Field is deleted""" - class TestFile(Document): - file = FileField() - - class TestImage(Document): - image = ImageField() - - TestFile.drop_collection() - - testfile = TestFile() - testfile.file.put('Hello, World!') - testfile.save() - - testfile_grid_id = testfile.file.grid_id - testfile_fs = testfile.file.fs - - testfile.delete() - self.assertFalse(testfile_fs.exists(testfile_grid_id)) - - TestImage.drop_collection() - - testimage = TestImage() - testimage.image.put(open(TEST_IMAGE_PATH, 'r')) - testimage.save() - - testimage_grid_id = testimage.image.grid_id - testimage_fs = testimage.image.fs - - testimage.delete() - self.assertFalse(testimage_fs.exists(testimage_grid_id)) def test_file_field_no_default(self): From ec5ddbf391dca46b70cd69bd96bed23419ed3352 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 14 May 2012 12:06:25 +0100 Subject: [PATCH 0612/1279] Fixed sparse indexes with inheritance fixes hmarr/mongoengine#497 --- docs/changelog.rst | 1 + mongoengine/queryset.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 44ae826..0843a35 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.9 ================ +- Fixed sparse indexes on inherited docs - Removed FileField auto deletion, needs more work maybe 0.7 Changes in 0.6.8 diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6d8cc7a..a535553 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -512,6 +512,10 @@ class QuerySet(object): key = '.'.join(parts) index_list.append((key, direction)) + # If sparse - dont include types + if spec.get('sparse', False): + use_types = False + # Check if a list field is being used, don't use _types if it is if use_types and not all(f._index_with_types for f in fields): use_types = False From 28a957c6845aa275ea6873009357cc1a23019dec Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 14 May 2012 12:43:00 +0100 Subject: [PATCH 0613/1279] Version bump --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 164aae8..ee0bcf5 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 8) +VERSION = (0, 6, 9) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 8ab4515..a610a40 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.8 +Version: 0.6.9 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 43468b474ec1931d05d593ba1b0caeb580e78d3c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 17 May 2012 16:49:13 +0100 Subject: [PATCH 0614/1279] Adding travis support --- .travis.yml | 9 +++++++++ README.rst | 2 ++ 2 files changed, 11 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..d21bbc2 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,9 @@ +# http://travis-ci.org/#!/MongoEngine/mongoengine +language: python +python: + - 2.6 + - 2.7 +install: + - python setup.py install +script: + - python setup.py test \ No newline at end of file diff --git a/README.rst b/README.rst index 168b534..24e0899 100644 --- a/README.rst +++ b/README.rst @@ -96,3 +96,5 @@ Contributing The source is available on `GitHub `_ - to contribute to the project, fork it on GitHub and send a pull request, all contributions and suggestions are welcome! + +.. image:: https://secure.travis-ci.org/MongoEngine/mongoengine.png From 54bb1cb3d935fe6fbb3d40cfb74c0328df760728 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 17 May 2012 16:59:50 +0100 Subject: [PATCH 0615/1279] Updated travis settings and Readme --- .travis.yml | 3 +++ README.rst | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d21bbc2..e38053f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,9 @@ python: - 2.6 - 2.7 install: + - sudo apt-get install zlib1g zlib1g-dev + - sudo ln -s /usr/lib/i386-linux-gnu/libz.so /usr/lib/ + - pip install PIL --use-mirrors ; true - python setup.py install script: - python setup.py test \ No newline at end of file diff --git a/README.rst b/README.rst index 24e0899..5f66ccf 100644 --- a/README.rst +++ b/README.rst @@ -97,4 +97,5 @@ The source is available on `GitHub `_ contribute to the project, fork it on GitHub and send a pull request, all contributions and suggestions are welcome! -.. image:: https://secure.travis-ci.org/MongoEngine/mongoengine.png +.. image:: https://secure.travis-ci.org/MongoEngine/mongoengine.png?branch=master + :target: http://travis-ci.org/MongoEngine/mongoengine From 376b9b13164d26c2267d1245c1fc9f55a944a83c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 17 May 2012 21:14:25 +0100 Subject: [PATCH 0616/1279] updated the readme --- README.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 5f66ccf..548737c 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,9 @@ MongoEngine :Author: Harry Marr (http://github.com/hmarr) :Maintainer: Ross Lawley (http://github.com/rozza) +.. image:: https://secure.travis-ci.org/MongoEngine/mongoengine.png?branch=master + :target: http://travis-ci.org/MongoEngine/mongoengine + About ===== MongoEngine is a Python Object-Document Mapper for working with MongoDB. @@ -97,5 +100,3 @@ The source is available on `GitHub `_ contribute to the project, fork it on GitHub and send a pull request, all contributions and suggestions are welcome! -.. image:: https://secure.travis-ci.org/MongoEngine/mongoengine.png?branch=master - :target: http://travis-ci.org/MongoEngine/mongoengine From 88406803036d45d21f0f5c3083e4e8eedf1a70d9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 17 May 2012 21:54:17 +0100 Subject: [PATCH 0617/1279] Promoted BaseDynamicField to DynamicField closes mongoengine/mongoengine#22 --- docs/changelog.rst | 4 ++++ mongoengine/base.py | 44 ++--------------------------------------- mongoengine/document.py | 3 ++- mongoengine/fields.py | 43 +++++++++++++++++++++++++++++++++++++++- mongoengine/queryset.py | 4 ++-- 5 files changed, 52 insertions(+), 46 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0843a35..a312d1e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.6.x +================ +- Promoted BaseDynamicField to DynamicField + Changes in 0.6.9 ================ - Fixed sparse indexes on inherited docs diff --git a/mongoengine/base.py b/mongoengine/base.py index ec7af45..7571822 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -435,47 +435,6 @@ class ComplexBaseField(BaseField): owner_document = property(_get_owner_document, _set_owner_document) -class BaseDynamicField(BaseField): - """Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data""" - - def to_mongo(self, value): - """Convert a Python type to a MongoDBcompatible type. - """ - - if isinstance(value, basestring): - return value - - if hasattr(value, 'to_mongo'): - return value.to_mongo() - - if not isinstance(value, (dict, list, tuple)): - return value - - is_list = False - if not hasattr(value, 'items'): - is_list = True - value = dict([(k, v) for k, v in enumerate(value)]) - - data = {} - for k, v in value.items(): - data[k] = self.to_mongo(v) - - if is_list: # Convert back to a list - value = [v for k, v in sorted(data.items(), key=operator.itemgetter(0))] - else: - value = data - return value - - def lookup_member(self, member_name): - return member_name - - def prepare_query_value(self, op, value): - if isinstance(value, basestring): - from mongoengine.fields import StringField - return StringField().prepare_query_value(op, value) - return self.to_mongo(value) - - class ObjectIdField(BaseField): """An field wrapper around MongoDB's ObjectIds. """ @@ -859,7 +818,8 @@ class BaseDocument(object): field = None if not hasattr(self, name) and not name.startswith('_'): - field = BaseDynamicField(db_field=name) + from fields import DynamicField + field = DynamicField(db_field=name) field.name = name self._dynamic_fields[name] = field diff --git a/mongoengine/document.py b/mongoengine/document.py index 9e281f5..95b07d1 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,4 +1,5 @@ import pymongo + from bson.dbref import DBRef from mongoengine import signals @@ -359,7 +360,7 @@ class DynamicDocument(Document): way as an ordinary document but has expando style properties. Any data passed or set against the :class:`~mongoengine.DynamicDocument` that is not a field is automatically converted into a - :class:`~mongoengine.BaseDynamicField` and data can be attributed to that + :class:`~mongoengine.DynamicField` and data can be attributed to that field. ..note:: diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f88e5c1..c72c6cb 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -30,7 +30,7 @@ except ImportError: __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', 'MapField', - 'DecimalField', 'ComplexDateTimeField', 'URLField', + 'DecimalField', 'ComplexDateTimeField', 'URLField', 'DynamicField', 'GenericReferenceField', 'FileField', 'BinaryField', 'SortedListField', 'EmailField', 'GeoPointField', 'ImageField', 'SequenceField', 'UUIDField', 'GenericEmbeddedDocumentField'] @@ -473,6 +473,47 @@ class GenericEmbeddedDocumentField(BaseField): return data +class DynamicField(BaseField): + """Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data""" + + def to_mongo(self, value): + """Convert a Python type to a MongoDBcompatible type. + """ + + if isinstance(value, basestring): + return value + + if hasattr(value, 'to_mongo'): + return value.to_mongo() + + if not isinstance(value, (dict, list, tuple)): + return value + + is_list = False + if not hasattr(value, 'items'): + is_list = True + value = dict([(k, v) for k, v in enumerate(value)]) + + data = {} + for k, v in value.items(): + data[k] = self.to_mongo(v) + + if is_list: # Convert back to a list + value = [v for k, v in sorted(data.items(), key=itemgetter(0))] + else: + value = data + return value + + def lookup_member(self, member_name): + return member_name + + def prepare_query_value(self, op, value): + if isinstance(value, basestring): + from mongoengine.fields import StringField + return StringField().prepare_query_value(op, value) + return self.to_mongo(value) + + class ListField(ComplexBaseField): """A list field that wraps a standard field, allowing multiple instances of the field to be used as a list in the database. diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index a535553..1fb3eda 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -627,8 +627,8 @@ class QuerySet(object): if field_name in document._fields: field = document._fields[field_name] elif document._dynamic: - from base import BaseDynamicField - field = BaseDynamicField(db_field=field_name) + from fields import DynamicField + field = DynamicField(db_field=field_name) else: raise InvalidQueryError('Cannot resolve field "%s"' % field_name) From c9842ba13a52080b781eb486215b1450decf2cd2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 21 May 2012 15:20:46 +0100 Subject: [PATCH 0618/1279] Fix base classes to return fixes hmarr/mongoengine#507 --- mongoengine/base.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 7571822..e6e7f2a 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1217,15 +1217,15 @@ class BaseList(list): def __init__(self, list_items, instance, name): self._instance = instance self._name = name - super(BaseList, self).__init__(list_items) + return super(BaseList, self).__init__(list_items) def __setitem__(self, *args, **kwargs): self._mark_as_changed() - super(BaseList, self).__setitem__(*args, **kwargs) + return super(BaseList, self).__setitem__(*args, **kwargs) def __delitem__(self, *args, **kwargs): self._mark_as_changed() - super(BaseList, self).__delitem__(*args, **kwargs) + return super(BaseList, self).__delitem__(*args, **kwargs) def __getstate__(self): self.observer = None @@ -1279,23 +1279,23 @@ class BaseDict(dict): def __init__(self, dict_items, instance, name): self._instance = instance self._name = name - super(BaseDict, self).__init__(dict_items) + return super(BaseDict, self).__init__(dict_items) def __setitem__(self, *args, **kwargs): self._mark_as_changed() - super(BaseDict, self).__setitem__(*args, **kwargs) + return super(BaseDict, self).__setitem__(*args, **kwargs) def __delete__(self, *args, **kwargs): self._mark_as_changed() - super(BaseDict, self).__delete__(*args, **kwargs) + return super(BaseDict, self).__delete__(*args, **kwargs) def __delitem__(self, *args, **kwargs): self._mark_as_changed() - super(BaseDict, self).__delitem__(*args, **kwargs) + return super(BaseDict, self).__delitem__(*args, **kwargs) def __delattr__(self, *args, **kwargs): self._mark_as_changed() - super(BaseDict, self).__delattr__(*args, **kwargs) + return super(BaseDict, self).__delattr__(*args, **kwargs) def __getstate__(self): self.instance = None @@ -1308,19 +1308,19 @@ class BaseDict(dict): def clear(self, *args, **kwargs): self._mark_as_changed() - super(BaseDict, self).clear(*args, **kwargs) + return super(BaseDict, self).clear(*args, **kwargs) def pop(self, *args, **kwargs): self._mark_as_changed() - super(BaseDict, self).pop(*args, **kwargs) + return super(BaseDict, self).pop(*args, **kwargs) def popitem(self, *args, **kwargs): self._mark_as_changed() - super(BaseDict, self).popitem(*args, **kwargs) + return super(BaseDict, self).popitem(*args, **kwargs) def update(self, *args, **kwargs): self._mark_as_changed() - super(BaseDict, self).update(*args, **kwargs) + return super(BaseDict, self).update(*args, **kwargs) def _mark_as_changed(self): if hasattr(self._instance, '_mark_as_changed'): From 944aa4545983893a8fd7ae7ef13dcf29011243bb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 21 May 2012 15:21:45 +0100 Subject: [PATCH 0619/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a312d1e..4c6cae3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.x ================ +- Fixed basedict / baselist to return super(..) - Promoted BaseDynamicField to DynamicField Changes in 0.6.9 From 0b22c140c5f17b80a1643ffead94ccd4d7af0992 Mon Sep 17 00:00:00 2001 From: Andrey Fedoseev Date: Tue, 22 May 2012 22:31:59 +0600 Subject: [PATCH 0620/1279] Add sensible __eq__ method to EmbeddedDocument --- mongoengine/document.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mongoengine/document.py b/mongoengine/document.py index 95b07d1..bb079c9 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -40,6 +40,11 @@ class EmbeddedDocument(BaseDocument): else: super(EmbeddedDocument, self).__delattr__(*args, **kwargs) + def __eq__(self, other): + if isinstance(other, self.__class__): + return self._data == other._data + return False + class Document(BaseDocument): """The base class used for defining the structure and properties of From 1fdc7ce6bb13ce1190b64ce6af341acc88979b4a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 23 May 2012 08:58:43 +0100 Subject: [PATCH 0621/1279] Releasing Version 0.6.10 --- docs/changelog.rst | 4 ++-- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4c6cae3..02887fd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,8 +2,8 @@ Changelog ========= -Changes in 0.6.x -================ +Changes in 0.6.10 +================= - Fixed basedict / baselist to return super(..) - Promoted BaseDynamicField to DynamicField diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index ee0bcf5..f594062 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 9) +VERSION = (0, 6, 10) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index a610a40..95247f7 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.9 +Version: 0.6.10 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 9c212306b8700accc94ff16a2a4f4eb7e22130e8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 29 May 2012 16:24:25 +0100 Subject: [PATCH 0622/1279] Updated setup / added datetime test --- .gitignore | 1 + setup.cfg | 15 +++++++++++++++ setup.py | 4 ++-- tests/connection.py | 22 ++++++++++++++++++++-- tests/document.py | 2 +- 5 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 setup.cfg diff --git a/.gitignore b/.gitignore index 0300bc7..2771eb0 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ env/ .project .pydevproject tests/bugfix.py +htmlcov/ \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..2345340 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,15 @@ +[aliases] +# python2.7 has upgraded unittest and it is no longer compatible with some +# of our tests, so we run all through nose +test = nosetests + +[nosetests] +verbosity = 2 +detailed-errors = 1 +with-coverage = 1 +#cover-html = 1 +#cover-html-dir = ../htmlcov +cover-package = mongoengine +cover-erase = 1 +where = tests +tests = bugfix, connection, dereference, django_tests, document, dynamic_document, fields, queryset, replicaset_connection, signals \ No newline at end of file diff --git a/setup.py b/setup.py index d92c491..55f1418 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,6 @@ setup(name='mongoengine', platforms=['any'], classifiers=CLASSIFIERS, install_requires=['pymongo'], - test_suite='tests', - tests_require=['blinker', 'django>=1.3', 'PIL'] + # use python setup.py nosetests to test + setup_requires=['nose', 'coverage', 'blinker', 'django>=1.3', 'PIL'] ) diff --git a/tests/connection.py b/tests/connection.py index 91bba55..cd03df0 100644 --- a/tests/connection.py +++ b/tests/connection.py @@ -1,8 +1,11 @@ -import unittest +import datetime import pymongo +import unittest import mongoengine.connection +from bson.tz_util import utc + from mongoengine import * from mongoengine.connection import get_db, get_connection, ConnectionError @@ -70,11 +73,26 @@ class ConnectionTest(unittest.TestCase): """ connect('mongoenginetest', alias='t1', tz_aware=True) conn = get_connection('t1') + self.assertTrue(conn.tz_aware) - + connect('mongoenginetest2', alias='t2') conn = get_connection('t2') self.assertFalse(conn.tz_aware) + def test_datetime(self): + connect('mongoenginetest', tz_aware=True) + d = datetime.datetime(2010, 5, 5, tzinfo=utc) + + class DateDoc(Document): + the_date = DateTimeField(required=True) + + DateDoc.drop_collection() + DateDoc(the_date=d).save() + + date_doc = DateDoc.objects.first() + self.assertEqual(d, date_doc.the_date) + + if __name__ == '__main__': unittest.main() diff --git a/tests/document.py b/tests/document.py index 055f980..48eec36 100644 --- a/tests/document.py +++ b/tests/document.py @@ -6,7 +6,7 @@ import warnings from datetime import datetime -from fixtures import Base, Mixin, PickleEmbedded, PickleTest +from tests.fixtures import Base, Mixin, PickleEmbedded, PickleTest from mongoengine import * from mongoengine.base import NotRegistered, InvalidDocumentError From 65a2f8a68b1bbe49818fe9cd8eecd4968ff19ba7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 29 May 2012 17:06:03 +0100 Subject: [PATCH 0623/1279] Updated configs --- setup.cfg | 3 +- tests/test_bugfix.py | 47 +++++++++++++++++++ tests/{connection.py => test_connection.py} | 0 tests/{dereference.py => test_dereference.py} | 0 tests/{django_tests.py => test_django.py} | 0 tests/{document.py => test_document.py} | 0 ...c_document.py => test_dynamic_document.py} | 0 tests/{fields.py => test_fields.py} | 0 tests/{queryset.py => test_queryset.py} | 0 ...ction.py => test_replicaset_connection.py} | 0 tests/{signals.py => test_signals.py} | 0 11 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 tests/test_bugfix.py rename tests/{connection.py => test_connection.py} (100%) rename tests/{dereference.py => test_dereference.py} (100%) rename tests/{django_tests.py => test_django.py} (100%) rename tests/{document.py => test_document.py} (100%) rename tests/{dynamic_document.py => test_dynamic_document.py} (100%) rename tests/{fields.py => test_fields.py} (100%) rename tests/{queryset.py => test_queryset.py} (100%) rename tests/{replicaset_connection.py => test_replicaset_connection.py} (100%) rename tests/{signals.py => test_signals.py} (100%) diff --git a/setup.cfg b/setup.cfg index 2345340..619c945 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,5 +11,4 @@ with-coverage = 1 #cover-html-dir = ../htmlcov cover-package = mongoengine cover-erase = 1 -where = tests -tests = bugfix, connection, dereference, django_tests, document, dynamic_document, fields, queryset, replicaset_connection, signals \ No newline at end of file +where = tests \ No newline at end of file diff --git a/tests/test_bugfix.py b/tests/test_bugfix.py new file mode 100644 index 0000000..f0ea185 --- /dev/null +++ b/tests/test_bugfix.py @@ -0,0 +1,47 @@ +# import pickle +# import pymongo +# import bson +# import warnings + +# from datetime import datetime + +# import tempfile +# import pymongo, gridfs + +import unittest +from mongoengine import * +from bson.objectid import ObjectId + +class BugFixTest(unittest.TestCase): + + + def setUp(self): + + conn = connect(db='mongoenginetest') + + def test_items_list(self): + + class ActivityType1(EmbeddedDocument): + activity_id = IntField() + activity_name = StringField() + + class ActivityType2(EmbeddedDocument): + activity_id = IntField() + activity_status = StringField() + + class UserActivities(Document): + user_id = IntField() + activity = GenericEmbeddedDocumentField(choices=(ActivityType1, ActivityType2)) + + + UserActivities.drop_collection() + + user_id = 123 + activity_id = 321 + UserActivities(user_id=user_id, activity=ActivityType2(activity_id=activity_id, activity_status="A")).save() + + self.assertEquals(1, UserActivities.objects(user_id=user_id, __raw__={'activity.activity_status': 'A'}).count()) + + + + diff --git a/tests/connection.py b/tests/test_connection.py similarity index 100% rename from tests/connection.py rename to tests/test_connection.py diff --git a/tests/dereference.py b/tests/test_dereference.py similarity index 100% rename from tests/dereference.py rename to tests/test_dereference.py diff --git a/tests/django_tests.py b/tests/test_django.py similarity index 100% rename from tests/django_tests.py rename to tests/test_django.py diff --git a/tests/document.py b/tests/test_document.py similarity index 100% rename from tests/document.py rename to tests/test_document.py diff --git a/tests/dynamic_document.py b/tests/test_dynamic_document.py similarity index 100% rename from tests/dynamic_document.py rename to tests/test_dynamic_document.py diff --git a/tests/fields.py b/tests/test_fields.py similarity index 100% rename from tests/fields.py rename to tests/test_fields.py diff --git a/tests/queryset.py b/tests/test_queryset.py similarity index 100% rename from tests/queryset.py rename to tests/test_queryset.py diff --git a/tests/replicaset_connection.py b/tests/test_replicaset_connection.py similarity index 100% rename from tests/replicaset_connection.py rename to tests/test_replicaset_connection.py diff --git a/tests/signals.py b/tests/test_signals.py similarity index 100% rename from tests/signals.py rename to tests/test_signals.py From ece9b902f8aa69e4470a893db7c6c6d7254207ce Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 29 May 2012 17:32:14 +0100 Subject: [PATCH 0624/1279] Setup.py cleanups --- setup.cfg | 2 +- setup.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 619c945..f45c448 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,4 +11,4 @@ with-coverage = 1 #cover-html-dir = ../htmlcov cover-package = mongoengine cover-erase = 1 -where = tests \ No newline at end of file +where = tests diff --git a/setup.py b/setup.py index 55f1418..29312aa 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,5 @@ setup(name='mongoengine', platforms=['any'], classifiers=CLASSIFIERS, install_requires=['pymongo'], - # use python setup.py nosetests to test - setup_requires=['nose', 'coverage', 'blinker', 'django>=1.3', 'PIL'] + tests_require=['nose', 'coverage', 'blinker', 'django>=1.3', 'PIL'] ) From 27a4d83ce895c0a1c7b3dc13d087d0bc6e576130 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 29 May 2012 17:32:41 +0100 Subject: [PATCH 0625/1279] Remove comment - it was wrong --- setup.cfg | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index f45c448..03b1655 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,4 @@ [aliases] -# python2.7 has upgraded unittest and it is no longer compatible with some -# of our tests, so we run all through nose test = nosetests [nosetests] From e4bc92235d6f1238059fe4b753ba2aa30965a288 Mon Sep 17 00:00:00 2001 From: Valentin Gorbunov Date: Wed, 6 Jun 2012 15:48:16 +0400 Subject: [PATCH 0626/1279] test_save_max_recursion_not_hit_with_file_field added --- tests/test_document.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_document.py b/tests/test_document.py index 48eec36..ae2257d 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1,3 +1,4 @@ +import os import pickle import pymongo import bson @@ -13,6 +14,8 @@ from mongoengine.base import NotRegistered, InvalidDocumentError from mongoengine.queryset import InvalidQueryError from mongoengine.connection import get_db +TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') + class DocumentTest(unittest.TestCase): @@ -1328,6 +1331,30 @@ class DocumentTest(unittest.TestCase): p0.name = 'wpjunior' p0.save() + def test_save_max_recursion_not_hit_with_file_field(self): + + class Foo(Document): + name = StringField() + file = FileField() + bar = ReferenceField('self') + + Foo.drop_collection() + + a = Foo(name='hello') + a.save() + + a.bar = a + a.file = open(TEST_IMAGE_PATH, 'rb') + a.save() + + # Confirm can save and it resets the changed fields without hitting + # max recursion error + b = Foo.objects.with_id(a.id) + b.name='world' + b.save() + + self.assertEquals(b.file, b.bar.file, b.bar.bar.file) + def test_save_cascades(self): class Person(Document): From 77ebd87fed43e6a41565b12dc78d4861360b6fc5 Mon Sep 17 00:00:00 2001 From: Meir Kriheli Date: Thu, 7 Jun 2012 12:02:19 +0300 Subject: [PATCH 0627/1279] Test PULL reverse_delete_rule --- tests/queryset.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 3b66248..191afcf 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1388,6 +1388,36 @@ class QuerySetTest(unittest.TestCase): self.assertRaises(OperationError, self.Person.objects.delete) + def test_reverse_delete_rule_pull(self): + """Ensure pulling of references to deleted documents. + """ + class BlogPost(Document): + content = StringField() + authors = ListField(ReferenceField(self.Person, + reverse_delete_rule=PULL)) + + BlogPost.drop_collection() + self.Person.drop_collection() + + me = self.Person(name='Test User') + me.save() + + someoneelse = self.Person(name='Some-one Else') + someoneelse.save() + + post = BlogPost(content='Watching TV', authors=[me, someoneelse]) + post.save() + + another = BlogPost(content='Chilling Out', authors=[someoneelse]) + another.save() + + someoneelse.delete() + post.reload() + another.reload() + + self.assertEqual(post.authors, [me]) + self.assertEqual(another.authors, []) + def test_update(self): """Ensure that atomic updates work properly. """ From 8060179f6de35791e6218e861b03202b1e60f498 Mon Sep 17 00:00:00 2001 From: Meir Kriheli Date: Thu, 7 Jun 2012 12:16:00 +0300 Subject: [PATCH 0628/1279] Implement PULL reverse_delete_rule --- mongoengine/queryset.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 1fb3eda..c943dc4 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -10,7 +10,7 @@ from bson.code import Code from mongoengine import signals __all__ = ['queryset_manager', 'Q', 'InvalidQueryError', - 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY'] + 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL'] # The maximum number of items to display in a QuerySet.__repr__ @@ -21,6 +21,7 @@ DO_NOTHING = 0 NULLIFY = 1 CASCADE = 2 DENY = 3 +PULL = 4 class DoesNotExist(Exception): @@ -1319,6 +1320,10 @@ class QuerySet(object): document_cls.objects(**{field_name + '__in': self}).update( safe_update=safe, **{'unset__%s' % field_name: 1}) + elif rule == PULL: + document_cls.objects(**{field_name + '__in': self}).update( + safe_update=safe, + **{'pull_all__%s' % field_name: self}) self._collection.remove(self._query, safe=safe) From 26db9d8a9d18869ef4bee19454e706b94bfcb9ad Mon Sep 17 00:00:00 2001 From: Meir Kriheli Date: Thu, 7 Jun 2012 12:32:02 +0300 Subject: [PATCH 0629/1279] Documentation for PULL reverse_delete_rule --- docs/changelog.rst | 4 ++++ docs/guide/defining-documents.rst | 4 ++++ mongoengine/fields.py | 2 ++ 3 files changed, 10 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 02887fd..67b798a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.6.x +================= +- PULL reverse_delete_rule + Changes in 0.6.10 ================= - Fixed basedict / baselist to return super(..) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index ba7a580..a005ddd 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -289,6 +289,10 @@ Its value can take any of the following constants: :const:`mongoengine.CASCADE` Any object containing fields that are refererring to the object being deleted are deleted first. +:const:`mongoengine.PULL` + Removes the reference to the object (using MongoDB's "pull" operation) + from any object's fields of + :class:`~mongoengine.ListField` (:class:`~mongoengine.ReferenceField`). .. warning:: diff --git a/mongoengine/fields.py b/mongoengine/fields.py index c72c6cb..df10d33 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -656,6 +656,8 @@ class ReferenceField(BaseField): * NULLIFY - Updates the reference to null. * CASCADE - Deletes the documents associated with the reference. * DENY - Prevent the deletion of the reference object. + * PULL - Pull the reference from a :class:`~mongoengine.ListField` + of references Alternative syntax for registering delete rules (useful when implementing bi-directional delete rules) From a856c7cc3700086481d5bc357589e5bbe567d1dd Mon Sep 17 00:00:00 2001 From: Meir Kriheli Date: Thu, 7 Jun 2012 12:36:14 +0300 Subject: [PATCH 0630/1279] Fix formatting of the docstring --- mongoengine/fields.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index df10d33..ab505f7 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -656,8 +656,7 @@ class ReferenceField(BaseField): * NULLIFY - Updates the reference to null. * CASCADE - Deletes the documents associated with the reference. * DENY - Prevent the deletion of the reference object. - * PULL - Pull the reference from a :class:`~mongoengine.ListField` - of references + * PULL - Pull the reference from a :class:`~mongoengine.ListField` of references Alternative syntax for registering delete rules (useful when implementing bi-directional delete rules) From 5bb63f645ba04e5731ae95dfd3154be6a3792faa Mon Sep 17 00:00:00 2001 From: Harry Marr Date: Fri, 8 Jun 2012 19:24:10 +0200 Subject: [PATCH 0631/1279] Fix minor typo w/ FloatField --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index c72c6cb..04da744 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -182,7 +182,7 @@ class FloatField(BaseField): if isinstance(value, int): value = float(value) if not isinstance(value, float): - self.error('FoatField only accepts float values') + self.error('FloatField only accepts float values') if self.min_value is not None and value < self.min_value: self.error('Float value is too small') From 2d9b581f34324ac2759452fb5a839282ca35f361 Mon Sep 17 00:00:00 2001 From: Shaun Duncan Date: Fri, 15 Jun 2012 15:42:19 -0400 Subject: [PATCH 0632/1279] Adding check if cascade delete is self-referencing. If so, prevent recursing if there are no objects to evaluate --- mongoengine/queryset.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 1fb3eda..8370d33 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1314,7 +1314,9 @@ class QuerySet(object): document_cls, field_name = rule_entry rule = doc._meta['delete_rules'][rule_entry] if rule == CASCADE: - document_cls.objects(**{field_name + '__in': self}).delete(safe=safe) + ref_q = document_cls.objects(**{field_name + '__in': self}) + if doc != document_cls or (doc == document_cls and ref_q.count() > 0): + ref_q.delete(safe=safe) elif rule == NULLIFY: document_cls.objects(**{field_name + '__in': self}).update( safe_update=safe, From 2ec1476e50e748de205b4c1721205f50b2bc574e Mon Sep 17 00:00:00 2001 From: Shaun Duncan Date: Sat, 16 Jun 2012 11:05:23 -0400 Subject: [PATCH 0633/1279] Adding test case for self-referencing documents with cascade deletes --- tests/queryset.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/queryset.py b/tests/queryset.py index 3b66248..c768ade 100644 --- a/tests/queryset.py +++ b/tests/queryset.py @@ -1344,6 +1344,37 @@ class QuerySetTest(unittest.TestCase): self.Person.objects(name='Test User').delete() self.assertEqual(1, BlogPost.objects.count()) + def test_reverse_delete_rule_cascade_self_referencing(self): + """Ensure self-referencing CASCADE deletes do not result in infinite loop + """ + class Category(Document): + name = StringField() + parent = ReferenceField('self', reverse_delete_rule=CASCADE) + + num_children = 3 + base = Category(name='Root') + base.save() + + # Create a simple parent-child tree + for i in range(num_children): + child_name = 'Child-%i' % i + child = Category(name=child_name, parent=base) + child.save() + + for i in range(num_children): + child_child_name = 'Child-Child-%i' % i + child_child = Category(name=child_child_name, parent=child) + child_child.save() + + tree_size = 1 + num_children + (num_children * num_children) + self.assertEquals(tree_size, Category.objects.count()) + self.assertEquals(num_children, Category.objects(parent=base).count()) + + # The delete should effectively wipe out the Category collection + # without resulting in infinite parent-child cascade recursion + base.delete() + self.assertEquals(0, Category.objects.count()) + def test_reverse_delete_rule_nullify(self): """Ensure nullification of references to deleted documents. """ From 89a6eee6af99c462c3dfaddb24861e93835b9e2e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 18 Jun 2012 16:45:14 +0100 Subject: [PATCH 0634/1279] Fixes cascading saves with filefields fixes #24 #25 --- docs/changelog.rst | 4 ++++ mongoengine/fields.py | 7 +++++++ tests/test_document.py | 6 +++--- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 02887fd..7d5b133 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.6.X +================ +- Fixed recursive save with FileField + Changes in 0.6.10 ================= - Fixed basedict / baselist to return super(..) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index c72c6cb..bba8f0b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -889,6 +889,13 @@ class GridFSProxy(object): self_dict['_fs'] = None return self_dict + def __repr__(self): + return '<%s: %s>' % (self.__class__.__name__, self.grid_id) + + def __cmp__(self, other): + return cmp((self.grid_id, self.collection_name, self.db_alias), + (other.grid_id, other.collection_name, other.db_alias)) + @property def fs(self): if not self._fs: diff --git a/tests/test_document.py b/tests/test_document.py index ae2257d..b50d716 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1335,7 +1335,7 @@ class DocumentTest(unittest.TestCase): class Foo(Document): name = StringField() - file = FileField() + picture = FileField() bar = ReferenceField('self') Foo.drop_collection() @@ -1344,7 +1344,7 @@ class DocumentTest(unittest.TestCase): a.save() a.bar = a - a.file = open(TEST_IMAGE_PATH, 'rb') + a.picture = open(TEST_IMAGE_PATH, 'rb') a.save() # Confirm can save and it resets the changed fields without hitting @@ -1353,7 +1353,7 @@ class DocumentTest(unittest.TestCase): b.name='world' b.save() - self.assertEquals(b.file, b.bar.file, b.bar.bar.file) + self.assertEquals(b.picture, b.bar.picture, b.bar.bar.picture) def test_save_cascades(self): From 9e7ea64bd2cc4b42494b574750ff2afa5e8237b8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 18 Jun 2012 20:49:33 +0100 Subject: [PATCH 0635/1279] Fixed db_field load error Fixes mongoengine/MongoEngine#45 --- .gitignore | 2 +- docs/changelog.rst | 1 + mongoengine/base.py | 1 + tests/test_bugfix.py | 47 ------------------------------------------ tests/test_document.py | 20 ++++++++++++++++++ 5 files changed, 23 insertions(+), 48 deletions(-) delete mode 100644 tests/test_bugfix.py diff --git a/.gitignore b/.gitignore index 2771eb0..7c0a917 100644 --- a/.gitignore +++ b/.gitignore @@ -13,5 +13,5 @@ env/ .settings .project .pydevproject -tests/bugfix.py +tests/test_bugfix.py htmlcov/ \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 7d5b133..7315ccd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Fixed db_field data load error - Fixed recursive save with FileField Changes in 0.6.10 diff --git a/mongoengine/base.py b/mongoengine/base.py index e6e7f2a..6ab7425 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -798,6 +798,7 @@ class BaseDocument(object): dynamic_data[key] = value else: for key, value in values.items(): + key = self._reverse_db_field_map.get(key, key) setattr(self, key, value) # Set any get_fieldname_display methods diff --git a/tests/test_bugfix.py b/tests/test_bugfix.py deleted file mode 100644 index f0ea185..0000000 --- a/tests/test_bugfix.py +++ /dev/null @@ -1,47 +0,0 @@ -# import pickle -# import pymongo -# import bson -# import warnings - -# from datetime import datetime - -# import tempfile -# import pymongo, gridfs - -import unittest -from mongoengine import * -from bson.objectid import ObjectId - -class BugFixTest(unittest.TestCase): - - - def setUp(self): - - conn = connect(db='mongoenginetest') - - def test_items_list(self): - - class ActivityType1(EmbeddedDocument): - activity_id = IntField() - activity_name = StringField() - - class ActivityType2(EmbeddedDocument): - activity_id = IntField() - activity_status = StringField() - - class UserActivities(Document): - user_id = IntField() - activity = GenericEmbeddedDocumentField(choices=(ActivityType1, ActivityType2)) - - - UserActivities.drop_collection() - - user_id = 123 - activity_id = 321 - UserActivities(user_id=user_id, activity=ActivityType2(activity_id=activity_id, activity_status="A")).save() - - self.assertEquals(1, UserActivities.objects(user_id=user_id, __raw__={'activity.activity_status': 'A'}).count()) - - - - diff --git a/tests/test_document.py b/tests/test_document.py index b50d716..74de3ee 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -664,6 +664,26 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + def test_db_field_load(self): + """Ensure we load data correctly + """ + class Person(Document): + name = StringField(required=True) + _rank = StringField(required=False, db_field="rank") + + @property + def rank(self): + return self._rank or "Private" + + Person.drop_collection() + + Person(name="Jack", _rank="Corporal").save() + + Person(name="Fred").save() + + self.assertEquals(Person.objects.get(name="Jack").rank, "Corporal") + self.assertEquals(Person.objects.get(name="Fred").rank, "Private") + def test_explicit_geo2d_index(self): """Ensure that geo2d indexes work when created via meta[indexes] """ From bd41c6eea4c2fc5cbdbc61b88b4cb2c78fa0d2e5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 18 Jun 2012 21:04:41 +0100 Subject: [PATCH 0636/1279] Updated changelog & AUTHORS refs hmarr/mongoengine#517 --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 2a91373..ed4ed47 100644 --- a/AUTHORS +++ b/AUTHORS @@ -104,4 +104,5 @@ that much better: * swashbuckler * Adam Reeve * Anthony Nemitz - * deignacio \ No newline at end of file + * deignacio + * shaunduncan \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 7315ccd..75fc39a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Fixed CASCADE delete bug - Fixed db_field data load error - Fixed recursive save with FileField From c73ce3d220d7cdc55d125a63cb607b0461f32a3b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 18 Jun 2012 21:13:55 +0100 Subject: [PATCH 0637/1279] Updated changelog / AUTHORS refs hmarr/mongoengine#511 --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 5598455..62007a6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -106,4 +106,5 @@ that much better: * Anthony Nemitz * deignacio * shaunduncan - * Meir Kriheli \ No newline at end of file + * Meir Kriheli + * Andrey Fedoseev \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 11a490d..eeb5b89 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Added cmp to Embedded Document - Added PULL reverse_delete_rule - Fixed CASCADE delete bug - Fixed db_field data load error From 89220c142b199ab1381501b679161129a76ebd2d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 18 Jun 2012 21:18:40 +0100 Subject: [PATCH 0638/1279] Fixed django test class refs hmarr/mongoengine#506 --- mongoengine/django/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/django/tests.py b/mongoengine/django/tests.py index a8d7c7f..e91abeb 100644 --- a/mongoengine/django/tests.py +++ b/mongoengine/django/tests.py @@ -10,7 +10,7 @@ class MongoTestCase(TestCase): """ db_name = 'test_%s' % settings.MONGO_DATABASE_NAME def __init__(self, methodName='runtest'): - self.db = connect(self.db_name) + self.db = connect(self.db_name).get_db() super(MongoTestCase, self).__init__(methodName) def _post_teardown(self): From 7ca2ea0766644bf5053e80303d5c386302f11609 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 19 Jun 2012 09:49:22 +0100 Subject: [PATCH 0639/1279] Fixes .save _delta issue with DBRefs Fixes hmarr/mongoengine#518 --- docs/changelog.rst | 4 +++- mongoengine/base.py | 8 ++++++-- mongoengine/dereference.py | 2 +- setup.cfg | 7 ++++--- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index eeb5b89..802e302 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,8 @@ Changelog Changes in 0.6.X ================ +- Fixed .save() _delta issue with DbRefs +- Fixed Django TestCase - Added cmp to Embedded Document - Added PULL reverse_delete_rule - Fixed CASCADE delete bug @@ -317,4 +319,4 @@ Changes in v0.1.2 Changes in v0.1.1 ================= -- Documents may now use capped collections +- \ No newline at end of file diff --git a/mongoengine/base.py b/mongoengine/base.py index 6ab7425..a136426 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1050,14 +1050,18 @@ Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, error for path in set_fields: parts = path.split('.') d = doc + dbref = False for p in parts: - if hasattr(d, '__getattr__'): + if isinstance(d, DBRef): + dbref = True + elif hasattr(d, '__getattr__'): d = getattr(p, d) elif p.isdigit(): d = d[int(p)] else: d = d.get(p) - set_data[path] = d + if not dbref: + set_data[path] = d else: set_data = doc if '_id' in set_data: diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 8928d1e..84c57bb 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -114,7 +114,7 @@ class DeReference(object): doc = get_document(ref["_cls"])._from_son(ref) elif doc_type is None: doc = get_document( - ''.join(x.capitalize() + ''.join(x.capitalize() for x in col.split('_')))._from_son(ref) else: doc = doc_type._from_son(ref) diff --git a/setup.cfg b/setup.cfg index 03b1655..43097a4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,9 +4,10 @@ test = nosetests [nosetests] verbosity = 2 detailed-errors = 1 -with-coverage = 1 -#cover-html = 1 -#cover-html-dir = ../htmlcov +#with-coverage = 1 +cover-html = 1 +cover-html-dir = ../htmlcov cover-package = mongoengine cover-erase = 1 where = tests +#tests = test_bugfix.py From 45d3a7f6ff00360e7ec72b6dd2281aa0bb664660 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 19 Jun 2012 09:49:55 +0100 Subject: [PATCH 0640/1279] Updated Changelog --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 802e302..3be5f2c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -319,4 +319,4 @@ Changes in v0.1.2 Changes in v0.1.1 ================= -- \ No newline at end of file +- Documents may now use capped collections From 581605e0e20c660dc839bc256b325d2b4d87c453 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 19 Jun 2012 10:08:56 +0100 Subject: [PATCH 0641/1279] Added test case for _delta refs: hmarr/mongoengine#518 --- tests/test_document.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/test_document.py b/tests/test_document.py index 74de3ee..6f942f6 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1638,6 +1638,35 @@ class DocumentTest(unittest.TestCase): site = Site.objects.first() self.assertEqual(site.page.log_message, "Error: Dummy message") + def test_circular_reference_deltas(self): + + class Person(Document): + name = StringField() + owns = ListField(ReferenceField('Organization')) + + class Organization(Document): + name = StringField() + owner = ReferenceField('Person') + + Person.drop_collection() + Organization.drop_collection() + + person = Person(name="owner") + person.save() + organization = Organization(name="company") + organization.save() + + person.owns.append(organization) + organization.owner = person + + person.save() + organization.save() + + p = Person.objects[0].select_related() + o = Organization.objects.first() + self.assertEquals(p.owns[0], o) + self.assertEquals(o.owner, p) + def test_delta(self): class Doc(Document): From 3d0d2f48ad3fd4bfe2561a6f36cfb2caa54c724a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 19 Jun 2012 10:57:43 +0100 Subject: [PATCH 0642/1279] Fixed map_field embedded db_field bug fixes hmarr/mongoengine#512 --- docs/changelog.rst | 1 + mongoengine/queryset.py | 8 ++++++-- tests/test_fields.py | 42 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3be5f2c..0f0965e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Fixed map_field embedded db_field issue - Fixed .save() _delta issue with DbRefs - Fixed Django TestCase - Added cmp to Embedded Document diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index ab35afb..ed00a73 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -620,6 +620,7 @@ class QuerySet(object): "Can't use index on unsubscriptable field (%s)" % err) fields.append(field_name) continue + if field is None: # Look up first field from the document if field_name == 'pk': @@ -637,8 +638,11 @@ class QuerySet(object): from mongoengine.fields import ReferenceField, GenericReferenceField if isinstance(field, (ReferenceField, GenericReferenceField)): raise InvalidQueryError('Cannot perform join in mongoDB: %s' % '__'.join(parts)) - # Look up subfield on the previous field - new_field = field.lookup_member(field_name) + if getattr(field, 'field', None): + new_field = field.field.lookup_member(field_name) + else: + # Look up subfield on the previous field + new_field = field.lookup_member(field_name) from base import ComplexBaseField if not new_field and isinstance(field, ComplexBaseField): fields.append(field_name) diff --git a/tests/test_fields.py b/tests/test_fields.py index 85f2911..04d5a34 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -913,6 +913,48 @@ class FieldTest(unittest.TestCase): Extensible.drop_collection() + def test_embedded_mapfield_db_field(self): + + class Embedded(EmbeddedDocument): + number = IntField(default=0, db_field='i') + + class Test(Document): + my_map = MapField(field=EmbeddedDocumentField(Embedded), db_field='x') + + Test.drop_collection() + + test = Test() + test.my_map['DICTIONARY_KEY'] = Embedded(number=1) + test.save() + + Test.objects.update_one(inc__my_map__DICTIONARY_KEY__number=1) + + test = Test.objects.get() + self.assertEqual(test.my_map['DICTIONARY_KEY'].number, 2) + doc = self.db.test.find_one() + self.assertEqual(doc['x']['DICTIONARY_KEY']['i'], 2) + + def test_embedded_db_field(self): + + class Embedded(EmbeddedDocument): + number = IntField(default=0, db_field='i') + + class Test(Document): + embedded = EmbeddedDocumentField(Embedded, db_field='x') + + Test.drop_collection() + + test = Test() + test.embedded = Embedded(number=1) + test.save() + + Test.objects.update_one(inc__embedded__number=1) + + test = Test.objects.get() + self.assertEqual(test.embedded.number, 2) + doc = self.db.test.find_one() + self.assertEqual(doc['x']['i'], 2) + def test_embedded_document_validation(self): """Ensure that invalid embedded documents cannot be assigned to embedded document fields. From 9fecf2b303c06e048dd8570c60604841552318ed Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 19 Jun 2012 11:22:12 +0100 Subject: [PATCH 0643/1279] Fixed inconsistency handling None values field attrs fixes hmarr/mongoengine#505 --- docs/changelog.rst | 1 + mongoengine/base.py | 7 ------- mongoengine/fields.py | 2 +- tests/test_fields.py | 1 - 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0f0965e..52098b0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Fixed inconsistency handling None values field attrs - Fixed map_field embedded db_field issue - Fixed .save() _delta issue with DbRefs - Fixed Django TestCase diff --git a/mongoengine/base.py b/mongoengine/base.py index a136426..be1fb6d 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -833,13 +833,6 @@ class BaseDocument(object): if hasattr(self, '_changed_fields'): self._mark_as_changed(name) - # Handle None values for required fields - if value is None and name in getattr(self, '_fields', {}): - self._data[name] = value - if hasattr(self, '_changed_fields'): - self._mark_as_changed(name) - return - if not self._created and name in self._meta.get('shard_key', tuple()): from queryset import OperationError raise OperationError("Shard Keys are immutable. Tried to update %s" % name) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index ec66789..cefbbab 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -369,7 +369,7 @@ class ComplexDateTimeField(StringField): return self._convert_from_string(data) def __set__(self, instance, value): - value = self._convert_from_datetime(value) + value = self._convert_from_datetime(value) if value else value return super(ComplexDateTimeField, self).__set__(instance, value) def validate(self, value): diff --git a/tests/test_fields.py b/tests/test_fields.py index 04d5a34..68c79b5 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -82,7 +82,6 @@ class FieldTest(unittest.TestCase): # Retrive data from db and verify it. ret = HandleNoneFields.objects.all()[0] - self.assertEqual(ret.str_fld, None) self.assertEqual(ret.int_fld, None) self.assertEqual(ret.flt_fld, None) From 1a97dfd479cf55f6869e1fb32224d9ce9063abb4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 19 Jun 2012 14:05:53 +0100 Subject: [PATCH 0644/1279] Better fix for .save() _delta issue with DbRefs refs: hmarr/mongoengine#518 --- mongoengine/base.py | 10 ++-------- mongoengine/document.py | 1 + 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index be1fb6d..97ba76b 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1043,18 +1043,12 @@ Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, error for path in set_fields: parts = path.split('.') d = doc - dbref = False for p in parts: - if isinstance(d, DBRef): - dbref = True - elif hasattr(d, '__getattr__'): - d = getattr(p, d) - elif p.isdigit(): + if p.isdigit(): d = d[int(p)] else: d = d.get(p) - if not dbref: - set_data[path] = d + set_data[path] = d else: set_data = doc if '_id' in set_data: diff --git a/mongoengine/document.py b/mongoengine/document.py index bb079c9..a6bbb88 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -226,6 +226,7 @@ class Document(BaseDocument): if cascade_kwargs: # Allow granular control over cascades kwargs.update(cascade_kwargs) kwargs['_refs'] = _refs + self._changed_fields = [] self.cascade_save(**kwargs) except pymongo.errors.OperationFailure, err: From efeaba39a473dc1009e6ea34b33e249297f69bb4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 19 Jun 2012 14:34:16 +0100 Subject: [PATCH 0645/1279] Version bump --- docs/changelog.rst | 4 ++-- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 52098b0..de84e97 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,8 +2,8 @@ Changelog ========= -Changes in 0.6.X -================ +Changes in 0.6.11 +================== - Fixed inconsistency handling None values field attrs - Fixed map_field embedded db_field issue - Fixed .save() _delta issue with DbRefs diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index f594062..2ba9b4c 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 10) +VERSION = (0, 6, 11) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 95247f7..cff25b1 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.10 +Version: 0.6.11 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From e6317776c19fc93dcf43afe05db55b6fa78d7c30 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 19 Jun 2012 16:45:23 +0100 Subject: [PATCH 0646/1279] Fixes DBRef handling in _delta refs: hmarr/mongoengine#518 --- docs/changelog.rst | 4 ++++ mongoengine/base.py | 9 +++++++-- mongoengine/document.py | 3 ++- tests/test_document.py | 42 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index de84e97..7e38eae 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.6.X +================ +- Fixes error with _delta handling DBRefs + Changes in 0.6.11 ================== - Fixed inconsistency handling None values field attrs diff --git a/mongoengine/base.py b/mongoengine/base.py index 97ba76b..a4c1599 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1043,11 +1043,16 @@ Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, error for path in set_fields: parts = path.split('.') d = doc + new_path = [] for p in parts: - if p.isdigit(): + if isinstance(d, DBRef): + break + elif p.isdigit(): d = d[int(p)] - else: + elif hasattr(d, 'get'): d = d.get(p) + new_path.append(p) + path = '.'.join(new_path) set_data[path] = d else: set_data = doc diff --git a/mongoengine/document.py b/mongoengine/document.py index a6bbb88..ff9622a 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -226,7 +226,7 @@ class Document(BaseDocument): if cascade_kwargs: # Allow granular control over cascades kwargs.update(cascade_kwargs) kwargs['_refs'] = _refs - self._changed_fields = [] + #self._changed_fields = [] self.cascade_save(**kwargs) except pymongo.errors.OperationFailure, err: @@ -246,6 +246,7 @@ class Document(BaseDocument): """Recursively saves any references / generic references on an object""" from fields import ReferenceField, GenericReferenceField _refs = kwargs.get('_refs', []) or [] + for name, cls in self._fields.items(): if not isinstance(cls, (ReferenceField, GenericReferenceField)): continue diff --git a/tests/test_document.py b/tests/test_document.py index 6f942f6..2b10c17 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1667,6 +1667,48 @@ class DocumentTest(unittest.TestCase): self.assertEquals(p.owns[0], o) self.assertEquals(o.owner, p) + def test_circular_reference_deltas_2(self): + + class Person( Document ): + name = StringField() + owns = ListField( ReferenceField( 'Organization' ) ) + employer = ReferenceField( 'Organization' ) + + class Organization( Document ): + name = StringField() + owner = ReferenceField( 'Person' ) + employees = ListField( ReferenceField( 'Person' ) ) + + Person.drop_collection() + Organization.drop_collection() + + person = Person( name="owner" ) + person.save() + + employee = Person( name="employee" ) + employee.save() + + organization = Organization( name="company" ) + organization.save() + + person.owns.append( organization ) + organization.owner = person + + organization.employees.append( employee ) + employee.employer = organization + + person.save() + organization.save() + employee.save() + + p = Person.objects.get(name="owner") + e = Person.objects.get(name="employee") + o = Organization.objects.first() + + self.assertEquals(p.owns[0], o) + self.assertEquals(o.owner, p) + self.assertEquals(e.employer, o) + def test_delta(self): class Doc(Document): From ea31846a19370fb5eec804f1253b728e80e1f2e3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 19 Jun 2012 16:59:18 +0100 Subject: [PATCH 0647/1279] Fixes scalar lookups for primary_key fixes hmarr/mongoengine#519 --- docs/changelog.rst | 1 + mongoengine/queryset.py | 2 -- tests/test_queryset.py | 13 +++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7e38eae..c7bc9e6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Fixes scalar lookups for primary_key - Fixes error with _delta handling DBRefs Changes in 0.6.11 diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index ed00a73..3ebbb87 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1496,8 +1496,6 @@ class QuerySet(object): def lookup(obj, name): chunks = name.split('__') for chunk in chunks: - if hasattr(obj, '_db_field_map'): - chunk = obj._db_field_map.get(chunk, chunk) obj = getattr(obj, chunk) return obj diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 5078ca2..aa6eabb 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -3006,6 +3006,19 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(plist[1], (20, False)) self.assertEqual(plist[2], (30, True)) + def test_scalar_primary_key(self): + + class SettingValue(Document): + key = StringField(primary_key=True) + value = StringField() + + SettingValue.drop_collection() + s = SettingValue(key="test", value="test value") + s.save() + + val = SettingValue.objects.scalar('key', 'value') + self.assertEqual(list(val), [('test', 'test value')]) + def test_scalar_cursor_behaviour(self): """Ensure that a query returns a valid set of results. """ From 6d90ce250ac7b0704f77a938958c38bb526a335d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 19 Jun 2012 17:01:28 +0100 Subject: [PATCH 0648/1279] Version bump --- docs/changelog.rst | 4 ++-- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c7bc9e6..fdf7c7b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,8 +2,8 @@ Changelog ========= -Changes in 0.6.X -================ +Changes in 0.6.12 +================= - Fixes scalar lookups for primary_key - Fixes error with _delta handling DBRefs diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 2ba9b4c..1d65826 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 11) +VERSION = (0, 6, 12) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index cff25b1..6a78e9b 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.11 +Version: 0.6.12 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 614b5905514f170c259ff1b7e3cd59a9f283f536 Mon Sep 17 00:00:00 2001 From: Tristan Escalada Date: Tue, 19 Jun 2012 17:08:28 -0300 Subject: [PATCH 0649/1279] documentation typo: inheritence inheritence corrected to inheritance only in the documentation, not in the code --- mongoengine/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index ff9622a..c36645f 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -87,7 +87,7 @@ class Document(BaseDocument): system. By default, _types will be added to the start of every index (that - doesn't contain a list) if allow_inheritence is True. This can be + doesn't contain a list) if allow_inheritance is True. This can be disabled by either setting types to False on the specific index or by setting index_types to False on the meta dictionary for the document. """ From 2d08eec093c461afc511a2832770bcd9cb66a296 Mon Sep 17 00:00:00 2001 From: Aparajita Fishman Date: Thu, 21 Jun 2012 18:57:14 -0700 Subject: [PATCH 0650/1279] Fix conversion of StringField value to unicode, replace outdated (str, unicode) check with unicode --- mongoengine/fields.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index cefbbab..1f86560 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -49,10 +49,13 @@ class StringField(BaseField): super(StringField, self).__init__(**kwargs) def to_python(self, value): - return unicode(value) + if isinstance(value, unicode): + return value + else: + return value.decode('utf-8') def validate(self, value): - if not isinstance(value, (str, unicode)): + if not isinstance(value, basestring): self.error('StringField only accepts string values') if self.max_length is not None and len(value) > self.max_length: From 056c604dc30ce616212f22514691d129ba328fca Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 22 Jun 2012 16:22:27 +0100 Subject: [PATCH 0651/1279] Fixes __repr__ modifying the cursor Fixes MongoEngine/mongoengine#30 --- docs/changelog.rst | 5 +++++ mongoengine/queryset.py | 32 +++++++++++++++++++------------- tests/test_queryset.py | 39 ++++++++++++++++++++++++++++++--------- 3 files changed, 54 insertions(+), 22 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index fdf7c7b..c548933 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,11 @@ Changelog ========= +Changes in 0.6.X +================ + +- Fixes __repr__ modifying the cursor + Changes in 0.6.12 ================= - Fixes scalar lookups for primary_key diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 3ebbb87..21e3f9e 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -341,6 +341,7 @@ class QuerySet(object): self._timeout = True self._class_check = True self._slave_okay = False + self._iter = False self._scalar = [] # If inheritance is allowed, only return instances and instances of @@ -953,6 +954,7 @@ class QuerySet(object): def next(self): """Wrap the result in a :class:`~mongoengine.Document` object. """ + self._iter = True try: if self._limit == 0: raise StopIteration @@ -969,6 +971,7 @@ class QuerySet(object): .. versionadded:: 0.3 """ + self._iter = False self._cursor.rewind() def count(self): @@ -1808,21 +1811,24 @@ class QuerySet(object): return data def __repr__(self): - limit = REPR_OUTPUT_SIZE + 1 - start = (0 if self._skip is None else self._skip) - if self._limit is None: - stop = start + limit - if self._limit is not None: - if self._limit - start > limit: - stop = start + limit - else: - stop = self._limit - try: - data = list(self[start:stop]) - except pymongo.errors.InvalidOperation: - return ".. queryset mid-iteration .." + """Provides the string representation of the QuerySet + + .. versionchanged:: 0.6.13 Now doesnt modify the cursor + """ + + if self._iter: + return '.. queryset mid-iteration ..' + + data = [] + for i in xrange(REPR_OUTPUT_SIZE + 1): + try: + data.append(self.next()) + except StopIteration: + break if len(data) > REPR_OUTPUT_SIZE: data[-1] = "...(remaining elements truncated)..." + + self.rewind() return repr(data) def select_related(self, max_depth=1): diff --git a/tests/test_queryset.py b/tests/test_queryset.py index aa6eabb..e7ee678 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -636,17 +636,38 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(people1, people2) self.assertEqual(people1, people3) - def test_repr_iteration(self): - """Ensure that QuerySet __repr__ can handle loops - """ - self.Person(name='Person 1').save() - self.Person(name='Person 2').save() + def test_repr(self): + """Test repr behavior isnt destructive""" - queryset = self.Person.objects - self.assertEquals('[, ]', repr(queryset)) - for person in queryset: - self.assertEquals('.. queryset mid-iteration ..', repr(queryset)) + class Doc(Document): + number = IntField() + def __repr__(self): + return "" % self.number + + Doc.drop_collection() + + for i in xrange(1000): + Doc(number=i).save() + + docs = Doc.objects.order_by('number') + + self.assertEquals(docs.count(), 1000) + self.assertEquals(len(docs), 1000) + + docs_string = "%s" % docs + self.assertTrue("Doc: 0" in docs_string) + + self.assertEquals(docs.count(), 1000) + self.assertEquals(len(docs), 1000) + + # Limit and skip + self.assertEquals('[, , ]', "%s" % docs[1:4]) + + self.assertEquals(docs.count(), 3) + self.assertEquals(len(docs), 3) + for doc in docs: + self.assertEqual('.. queryset mid-iteration ..', repr(docs)) def test_regex_query_shortcuts(self): """Ensure that contains, startswith, endswith, etc work. From 07f3e5356d5549b371be297644a2d358b8ced1d2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Sat, 23 Jun 2012 21:46:31 +0100 Subject: [PATCH 0652/1279] Updated changelog / AUTHORS refs: hmarr/mongoengine#522 --- AUTHORS | 3 ++- docs/changelog.rst | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 62007a6..b76c02d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -107,4 +107,5 @@ that much better: * deignacio * shaunduncan * Meir Kriheli - * Andrey Fedoseev \ No newline at end of file + * Andrey Fedoseev + * aparajita \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index c548933..6272aec 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,7 @@ Changelog Changes in 0.6.X ================ - +- Fixed StringField unicode issue - Fixes __repr__ modifying the cursor Changes in 0.6.12 From 5d8ffded40a53a884093a034f2f0dc99662904aa Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Sat, 23 Jun 2012 22:19:02 +0100 Subject: [PATCH 0653/1279] Fixed issue with embedded_docs and db_fields Bumped version also refs: hmarr/mongoengine#523 --- docs/changelog.rst | 3 ++- mongoengine/__init__.py | 2 +- mongoengine/base.py | 2 ++ python-mongoengine.spec | 2 +- tests/test_document.py | 16 ++++++++++++++++ 5 files changed, 22 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6272aec..e0f497c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,8 +2,9 @@ Changelog ========= -Changes in 0.6.X +Changes in 0.6.13 ================ +- Fixed EmbeddedDocument db_field validation issue - Fixed StringField unicode issue - Fixes __repr__ modifying the cursor diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 1d65826..80a687c 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 12) +VERSION = (0, 6, 13) def get_version(): diff --git a/mongoengine/base.py b/mongoengine/base.py index a4c1599..8ed8dc4 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -957,6 +957,8 @@ class BaseDocument(object): try: data[field_name] = (value if value is None else field.to_python(value)) + if field_name != field.db_field: + del data[field.db_field] except (AttributeError, ValueError), e: errors_dict[field_name] = e elif field.default: diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 6a78e9b..c3919f1 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.12 +Version: 0.6.13 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB diff --git a/tests/test_document.py b/tests/test_document.py index 2b10c17..482a502 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1282,6 +1282,22 @@ class DocumentTest(unittest.TestCase): comment.date = datetime.now() comment.validate() + def test_embedded_db_field_validate(self): + + class SubDoc(EmbeddedDocument): + val = IntField() + + class Doc(Document): + e = EmbeddedDocumentField(SubDoc, db_field='eb') + + Doc.drop_collection() + + Doc(e=SubDoc(val=15)).save() + + doc = Doc.objects.first() + doc.validate() + self.assertEquals([None, 'e'], doc._data.keys()) + def test_save(self): """Ensure that a document may be saved in the database. """ From 7a1b110f62d1c4f19ea84cef3fdb8af498325b4a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Sat, 23 Jun 2012 22:24:09 +0100 Subject: [PATCH 0654/1279] Added Tristan Escalada to authors refs #hmarr/mongoengine#520 --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index b76c02d..90dadcf 100644 --- a/AUTHORS +++ b/AUTHORS @@ -108,4 +108,5 @@ that much better: * shaunduncan * Meir Kriheli * Andrey Fedoseev - * aparajita \ No newline at end of file + * aparajita + * Tristan Escalada \ No newline at end of file From 0d92baa67010eb9ec91b6242595c647da9e2730b Mon Sep 17 00:00:00 2001 From: Alexander Koshelev Date: Sun, 24 Jun 2012 03:08:49 +0400 Subject: [PATCH 0655/1279] Exclude tests from installation --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 29312aa..20f3ea3 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ CLASSIFIERS = [ setup(name='mongoengine', version=VERSION, - packages=find_packages(), + packages=find_packages(exclude=('tests',)), author='Harry Marr', author_email='harry.marr@{nospam}gmail.com', maintainer="Ross Lawley", From d305e71c27c793df69c98623eca086521b0245bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 25 Jun 2012 15:53:42 -0300 Subject: [PATCH 0656/1279] Fixes for __ne operator in IntField and FloatField --- mongoengine/fields.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 1f86560..5ff2846 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -167,6 +167,9 @@ class IntField(BaseField): self.error('Integer value is too large') def prepare_query_value(self, op, value): + if value is None: + return value + return int(value) @@ -194,6 +197,9 @@ class FloatField(BaseField): self.error('Float value is too large') def prepare_query_value(self, op, value): + if value is None: + return value + return float(value) From b3bb4add9cba3a430d63c12105b9fb4347350a42 Mon Sep 17 00:00:00 2001 From: Thomas Steinacher Date: Wed, 27 Jun 2012 13:32:17 -0700 Subject: [PATCH 0657/1279] Fix error dict with nested validation. --- mongoengine/base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 8ed8dc4..4e3250f 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -403,11 +403,11 @@ class ComplexBaseField(BaseField): for k, v in sequence: try: self.field._validate(v) - except (ValidationError, AssertionError), error: - if hasattr(error, 'errors'): - errors[k] = error.errors - else: - errors[k] = error + except ValidationError, error: + errors[k] = error.errors or error + except (ValueError, AssertionError), error: + errors[k] = error + if errors: field_class = self.field.__class__.__name__ self.error('Invalid %s item (%s)' % (field_class, value), From dd93995bd04f2e12894274b38fe001a68511fb2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Gonz=C3=A1lez?= Date: Mon, 2 Jul 2012 10:01:22 +0200 Subject: [PATCH 0658/1279] Forced cast to list --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 8ed8dc4..445fc3f 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -704,7 +704,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): meta['queryset_class'] = manager.queryset_class new_class.objects = manager - indicies = meta['indexes'] + abstract_base_indexes + indicies = [meta['indexes']] + abstract_base_indexes user_indexes = [QuerySet._build_index_spec(new_class, spec) for spec in indicies] + base_indexes new_class._meta['indexes'] = user_indexes From 494b981b1348512a18602a7469ccb4ff2d4fdb5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Gonz=C3=A1lez?= Date: Mon, 2 Jul 2012 10:05:25 +0200 Subject: [PATCH 0659/1279] Default value for direction --- mongoengine/queryset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 21e3f9e..eeffe4d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -492,6 +492,7 @@ class QuerySet(object): spec = {'fields': spec} index_list = [] + direction = None use_types = doc_cls._meta.get('allow_inheritance', True) for key in spec['fields']: # Get ASCENDING direction from +, DESCENDING from -, and GEO2D from * From 1fff7e9acaa1e6c18556de3d6fe74f9f6cb8f758 Mon Sep 17 00:00:00 2001 From: Jaime Irurzun Date: Tue, 10 Jul 2012 10:40:14 +0100 Subject: [PATCH 0660/1279] Fix _transform_update to accept unicode fields --- mongoengine/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 21e3f9e..be097b8 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1373,7 +1373,7 @@ class QuerySet(object): cleaned_fields = [] for field in fields: append_field = True - if isinstance(field, str): + if isinstance(field, basestring): # Convert the S operator to $ if field == 'S': field = '$' From b58bf3e0ce00345b89a6abb26afaa9905364714f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 11 Jul 2012 14:22:50 +0100 Subject: [PATCH 0661/1279] Added support for addToSet and each fixes MongoEngine/mongoengine#33 --- docs/changelog.rst | 4 ++++ mongoengine/queryset.py | 9 ++++++++- setup.cfg | 2 +- tests/test_queryset.py | 19 ++++++++++++++++++- 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e0f497c..9c91f13 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.6.14 +================= +- Added support for add_to_set and each + Changes in 0.6.13 ================ - Fixed EmbeddedDocument db_field validation issue diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 21e3f9e..d795ec1 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1387,11 +1387,16 @@ class QuerySet(object): # Convert value to proper value field = cleaned_fields[-1] - if op in (None, 'set', 'push', 'pull', 'addToSet'): + if op in (None, 'set', 'push', 'pull'): if field.required or value is not None: value = field.prepare_query_value(op, value) elif op in ('pushAll', 'pullAll'): value = [field.prepare_query_value(op, v) for v in value] + elif op == 'addToSet': + if isinstance(value, (list, tuple, set)): + value = [field.prepare_query_value(op, v) for v in value] + elif field.required or value is not None: + value = field.prepare_query_value(op, value) key = '.'.join(parts) @@ -1407,6 +1412,8 @@ class QuerySet(object): parts.reverse() for key in parts: value = {key: value} + elif op == 'addToSet' and isinstance(value, list): + value = {key: {"$each": value}} else: value = {key: value} key = '$' + op diff --git a/setup.cfg b/setup.cfg index 43097a4..083306a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,4 +10,4 @@ cover-html-dir = ../htmlcov cover-package = mongoengine cover-erase = 1 where = tests -#tests = test_bugfix.py +tests = test_bugfix.py diff --git a/tests/test_queryset.py b/tests/test_queryset.py index e7ee678..939451c 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -1520,7 +1520,7 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() - def test_update_push_and_pull(self): + def test_update_push_and_pull_add_to_set(self): """Ensure that the 'pull' update operation works correctly. """ class BlogPost(Document): @@ -1553,6 +1553,23 @@ class QuerySetTest(unittest.TestCase): post.reload() self.assertEqual(post.tags, ["code", "mongodb"]) + def test_add_to_set_each(self): + class Item(Document): + name = StringField(required=True) + description = StringField(max_length=50) + parents = ListField(ReferenceField('self')) + + Item.drop_collection() + + item = Item(name='test item').save() + parent_1 = Item(name='parent 1').save() + parent_2 = Item(name='parent 2').save() + + item.update(add_to_set__parents=[parent_1, parent_2, parent_1]) + item.reload() + + self.assertEqual([parent_1, parent_2], item.parents) + def test_pull_nested(self): class User(Document): From 27b375060d28d9ca5a85ff20c3e871b6626b181f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 11 Jul 2012 14:25:38 +0100 Subject: [PATCH 0662/1279] Updated changelog / AUTHORS refs MongoEngine/mongoengine#32 --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 90dadcf..af9c837 100644 --- a/AUTHORS +++ b/AUTHORS @@ -109,4 +109,5 @@ that much better: * Meir Kriheli * Andrey Fedoseev * aparajita - * Tristan Escalada \ No newline at end of file + * Tristan Escalada + * Jaime Irurzun \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 9c91f13..6bb741e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.14 ================= +- Fixed unicode support in transform update - Added support for add_to_set and each Changes in 0.6.13 From 4f5aa8c43b84f313df3b375edeece92a87b45a9b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 11 Jul 2012 14:29:35 +0100 Subject: [PATCH 0663/1279] Fixed config / added test --- setup.cfg | 2 +- tests/test_document.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 083306a..43097a4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,4 +10,4 @@ cover-html-dir = ../htmlcov cover-package = mongoengine cover-erase = 1 where = tests -tests = test_bugfix.py +#tests = test_bugfix.py diff --git a/tests/test_document.py b/tests/test_document.py index 482a502..d7412f3 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -684,6 +684,29 @@ class DocumentTest(unittest.TestCase): self.assertEquals(Person.objects.get(name="Jack").rank, "Corporal") self.assertEquals(Person.objects.get(name="Fred").rank, "Private") + def test_db_embedded_doc_field_load(self): + """Ensure we load embedded document data correctly + """ + class Rank(EmbeddedDocument): + title = StringField(required=True) + + class Person(Document): + name = StringField(required=True) + rank_ = EmbeddedDocumentField(Rank, required=False, db_field='rank') + + @property + def rank(self): + return self.rank_.title if self.rank_ is not None else "Private" + + Person.drop_collection() + + Person(name="Jack", rank_=Rank(title="Corporal")).save() + + Person(name="Fred").save() + + self.assertEquals(Person.objects.get(name="Jack").rank, "Corporal") + self.assertEquals(Person.objects.get(name="Fred").rank, "Private") + def test_explicit_geo2d_index(self): """Ensure that geo2d indexes work when created via meta[indexes] """ From 34d08ce8eff224844b57b2020722b830d7821b4b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 11 Jul 2012 15:23:27 +0100 Subject: [PATCH 0664/1279] Only dereference fields than need it Fixes MongoEngine/mongoengine#31 --- mongoengine/base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 8ed8dc4..a8b89ef 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -267,8 +267,10 @@ class ComplexBaseField(BaseField): if instance is None: # Document class being used rather than a document object return self - - if not self._dereference and instance._initialised: + from fields import GenericReferenceField, ReferenceField + dereference = self.field is None or isinstance(self.field, + (GenericReferenceField, ReferenceField)) + if not self._dereference and instance._initialised and dereference: from dereference import DeReference self._dereference = DeReference() # Cached instance._data[self.name] = self._dereference( From f55b241cfa60e759ee0ce7a0c278c67f733f2ab8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 11 Jul 2012 15:45:31 +0100 Subject: [PATCH 0665/1279] Trying to bump travis --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 43097a4..6c5b9ba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,10 +4,10 @@ test = nosetests [nosetests] verbosity = 2 detailed-errors = 1 -#with-coverage = 1 +with-coverage = 1 +cover-erase = 1 cover-html = 1 cover-html-dir = ../htmlcov cover-package = mongoengine -cover-erase = 1 where = tests #tests = test_bugfix.py From 1ce2f84ce59f735b3f327020bf88c6d9792c59dc Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 11 Jul 2012 15:56:34 +0100 Subject: [PATCH 0666/1279] Updated docs regarding fields refs hmarr/mongoengine#535 --- docs/apireference.rst | 35 +++++++++++++++++++---------------- mongoengine/fields.py | 5 ++++- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 304755f..0f8901a 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -47,25 +47,28 @@ Querying Fields ====== -.. autoclass:: mongoengine.StringField -.. autoclass:: mongoengine.URLField -.. autoclass:: mongoengine.EmailField -.. autoclass:: mongoengine.IntField -.. autoclass:: mongoengine.FloatField -.. autoclass:: mongoengine.DecimalField -.. autoclass:: mongoengine.DateTimeField +.. autoclass:: mongoengine.BinaryField +.. autoclass:: mongoengine.BooleanField .. autoclass:: mongoengine.ComplexDateTimeField -.. autoclass:: mongoengine.ListField -.. autoclass:: mongoengine.SortedListField +.. autoclass:: mongoengine.DateTimeField +.. autoclass:: mongoengine.DecimalField .. autoclass:: mongoengine.DictField +.. autoclass:: mongoengine.DynamicField +.. autoclass:: mongoengine.EmailField +.. autoclass:: mongoengine.EmbeddedDocumentField +.. autoclass:: mongoengine.FileField +.. autoclass:: mongoengine.FloatField +.. autoclass:: mongoengine.GenericEmbeddedDocumentField +.. autoclass:: mongoengine.GenericReferenceField +.. autoclass:: mongoengine.GeoPointField +.. autoclass:: mongoengine.ImageField +.. autoclass:: mongoengine.IntField +.. autoclass:: mongoengine.ListField .. autoclass:: mongoengine.MapField .. autoclass:: mongoengine.ObjectIdField .. autoclass:: mongoengine.ReferenceField -.. autoclass:: mongoengine.GenericReferenceField -.. autoclass:: mongoengine.EmbeddedDocumentField -.. autoclass:: mongoengine.GenericEmbeddedDocumentField -.. autoclass:: mongoengine.BooleanField -.. autoclass:: mongoengine.FileField -.. autoclass:: mongoengine.BinaryField -.. autoclass:: mongoengine.GeoPointField .. autoclass:: mongoengine.SequenceField +.. autoclass:: mongoengine.SortedListField +.. autoclass:: mongoengine.StringField +.. autoclass:: mongoengine.URLField +.. autoclass:: mongoengine.UUIDField diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 1f86560..a4640dd 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -477,7 +477,10 @@ class GenericEmbeddedDocumentField(BaseField): class DynamicField(BaseField): - """Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data""" + """A tryly dynamic field type capable of handling different and varying + types of data. + + Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data""" def to_mongo(self, value): """Convert a Python type to a MongoDBcompatible type. From 450dc11a680c92aff3d524cae32f7e454b669b08 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 11 Jul 2012 16:01:24 +0100 Subject: [PATCH 0667/1279] Unicode fixes refs hmarr/mongoengine#533 MongoEngine/mongoengine#32 --- mongoengine/fields.py | 7 +++---- mongoengine/queryset.py | 2 +- setup.cfg | 10 +++++----- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index a4640dd..b4dd456 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -838,11 +838,10 @@ class BinaryField(BaseField): return Binary(value) def to_python(self, value): - # Returns str not unicode as this is binary data - return str(value) + return "%s" % value def validate(self, value): - if not isinstance(value, str): + if not isinstance(value, basestring): self.error('BinaryField only accepts string values') if self.max_bytes is not None and len(value) > self.max_bytes: @@ -1014,7 +1013,7 @@ class FileField(BaseField): def __set__(self, instance, value): key = self.name - if (hasattr(value, 'read') and not isinstance(value, GridFSProxy)) or isinstance(value, str): + if (hasattr(value, 'read') and not isinstance(value, GridFSProxy)) or isinstance(value, basestring): # using "FileField() = file/string" notation grid_file = instance._data.get(self.name) # If a file already exists, delete it diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index d06e3af..64f9441 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -702,7 +702,7 @@ class QuerySet(object): cleaned_fields = [] for field in fields: append_field = True - if isinstance(field, str): + if isinstance(field, basestring): parts.append(field) append_field = False else: diff --git a/setup.cfg b/setup.cfg index 6c5b9ba..51a1868 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,10 +4,10 @@ test = nosetests [nosetests] verbosity = 2 detailed-errors = 1 -with-coverage = 1 -cover-erase = 1 -cover-html = 1 -cover-html-dir = ../htmlcov -cover-package = mongoengine +#with-coverage = 1 +#cover-erase = 1 +#cover-html = 1 +#cover-html-dir = ../htmlcov +#cover-package = mongoengine where = tests #tests = test_bugfix.py From 2405ba87089e0b01861f997a301ef52da6bfce2b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 11 Jul 2012 16:11:13 +0100 Subject: [PATCH 0668/1279] Updated Changelog / AUTHORS refs hmarr/mongoengine#531 --- AUTHORS | 3 ++- docs/changelog.rst | 2 ++ mongoengine/base.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index af9c837..6c07474 100644 --- a/AUTHORS +++ b/AUTHORS @@ -110,4 +110,5 @@ that much better: * Andrey Fedoseev * aparajita * Tristan Escalada - * Jaime Irurzun \ No newline at end of file + * Jaime Irurzun + * Alexandre González \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 6bb741e..77b74cd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,8 @@ Changelog Changes in 0.6.14 ================= +- Allow tuples for index meta +- Fixed use of str in instance checks - Fixed unicode support in transform update - Added support for add_to_set and each diff --git a/mongoengine/base.py b/mongoengine/base.py index e4f3f5c..7a4998c 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -706,7 +706,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): meta['queryset_class'] = manager.queryset_class new_class.objects = manager - indicies = [meta['indexes']] + abstract_base_indexes + indicies = list(meta['indexes']) + abstract_base_indexes user_indexes = [QuerySet._build_index_spec(new_class, spec) for spec in indicies] + base_indexes new_class._meta['indexes'] = user_indexes From f66b312869252594b4931c951c886a8953a09748 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 11 Jul 2012 16:25:40 +0100 Subject: [PATCH 0669/1279] Updated api docs fixes hmarr/mongoengine#526 --- docs/guide/defining-documents.rst | 35 +++++++++++++++++-------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index a005ddd..726ce3b 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -62,28 +62,31 @@ not provided. Default values may optionally be a callable, which will be called to retrieve the value (such as in the above example). The field types available are as follows: -* :class:`~mongoengine.StringField` -* :class:`~mongoengine.URLField` -* :class:`~mongoengine.EmailField` -* :class:`~mongoengine.IntField` -* :class:`~mongoengine.FloatField` -* :class:`~mongoengine.DecimalField` -* :class:`~mongoengine.DateTimeField` +* :class:`~mongoengine.BinaryField` +* :class:`~mongoengine.BooleanField` * :class:`~mongoengine.ComplexDateTimeField` -* :class:`~mongoengine.ListField` -* :class:`~mongoengine.SortedListField` +* :class:`~mongoengine.DateTimeField` +* :class:`~mongoengine.DecimalField` * :class:`~mongoengine.DictField` +* :class:`~mongoengine.DynamicField` +* :class:`~mongoengine.EmailField` +* :class:`~mongoengine.EmbeddedDocumentField` +* :class:`~mongoengine.FileField` +* :class:`~mongoengine.FloatField` +* :class:`~mongoengine.GenericEmbeddedDocumentField` +* :class:`~mongoengine.GenericReferenceField` +* :class:`~mongoengine.GeoPointField` +* :class:`~mongoengine.ImageField` +* :class:`~mongoengine.IntField` +* :class:`~mongoengine.ListField` * :class:`~mongoengine.MapField` * :class:`~mongoengine.ObjectIdField` * :class:`~mongoengine.ReferenceField` -* :class:`~mongoengine.GenericReferenceField` -* :class:`~mongoengine.EmbeddedDocumentField` -* :class:`~mongoengine.GenericEmbeddedDocumentField` -* :class:`~mongoengine.BooleanField` -* :class:`~mongoengine.FileField` -* :class:`~mongoengine.BinaryField` -* :class:`~mongoengine.GeoPointField` * :class:`~mongoengine.SequenceField` +* :class:`~mongoengine.SortedListField` +* :class:`~mongoengine.StringField` +* :class:`~mongoengine.URLField` +* :class:`~mongoengine.UUIDField` Field arguments --------------- From 34f06c49719714e9725a2bf12fceb57becd3193b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 11 Jul 2012 16:27:43 +0100 Subject: [PATCH 0670/1279] Updated changelog / AUTHORS refs hmarr/mongoengine#524 --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 6c07474..273914e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -110,5 +110,6 @@ that much better: * Andrey Fedoseev * aparajita * Tristan Escalada + * Alexander Koshelev * Jaime Irurzun * Alexandre González \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 77b74cd..9f036d6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.14 ================= +- Exclude tests from installation - Allow tuples for index meta - Fixed use of str in instance checks - Fixed unicode support in transform update From d1d30a92801318ef15594e9d9c78c01d1f991a89 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 11 Jul 2012 16:34:28 +0100 Subject: [PATCH 0671/1279] Added test and updated changelog refs hmarr/mongoengine#527 --- docs/changelog.rst | 1 + tests/test_fields.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9f036d6..0c5a6ec 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.14 ================= +- Fixed Int/Float fields and not equals None - Exclude tests from installation - Allow tuples for index meta - Fixed use of str in instance checks diff --git a/tests/test_fields.py b/tests/test_fields.py index 68c79b5..2c7e694 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -127,6 +127,19 @@ class FieldTest(unittest.TestCase): self.assertRaises(ValidationError, ret.validate) + def test_int_and_float_ne_operator(self): + class TestDocument(Document): + int_fld = IntField() + float_fld = FloatField() + + TestDocument.drop_collection() + + TestDocument(int_fld=None, float_fld=None).save() + TestDocument(int_fld=1, float_fld=1).save() + + self.assertEqual(1, TestDocument.objects(int_fld__ne=None).count()) + self.assertEqual(1, TestDocument.objects(float_fld__ne=None).count()) + def test_object_id_validation(self): """Ensure that invalid values cannot be assigned to string fields. """ From 31793520bf4c4ea6fdb420ae98a8e5f49de39d5f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 11 Jul 2012 16:38:15 +0100 Subject: [PATCH 0672/1279] Updated changelog / AUTHORS refs hmarr/mongoengine#529 --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 273914e..f774d22 100644 --- a/AUTHORS +++ b/AUTHORS @@ -112,4 +112,5 @@ that much better: * Tristan Escalada * Alexander Koshelev * Jaime Irurzun - * Alexandre González \ No newline at end of file + * Alexandre González + * Thomas Steinacher \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 0c5a6ec..545ce76 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.14 ================= +- Fixed error dict with nested validation - Fixed Int/Float fields and not equals None - Exclude tests from installation - Allow tuples for index meta From 8e50f5fa3cf3497e704e5e5ad0a716c31064fd40 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 11 Jul 2012 16:59:24 +0100 Subject: [PATCH 0673/1279] Version bump --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 80a687c..3218b29 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 13) +VERSION = (0, 6, 14) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index c3919f1..4ac9508 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.13 +Version: 0.6.14 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From a7a2fe02169407be99440c111346749cb781cebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Wed, 18 Jul 2012 06:37:23 -0300 Subject: [PATCH 0674/1279] added more tests --- tests/test_document.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_document.py b/tests/test_document.py index d7412f3..491a685 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -3161,5 +3161,19 @@ name: Field is required ("name")""" one = Doc.objects.filter(**{'hello world': 1}).count() self.assertEqual(1, one) + + def test_fields_rewrite(self): + class BasePerson(Document): + name = StringField() + age = IntField() + meta = {'abstract': True} + + class Person(BasePerson): + name = StringField(required=True) + + + p = Person(age=15) + self.assertRaises(ValidationError, p.validate) + if __name__ == '__main__': unittest.main() From b0aa98edb413b03ed6bbb6c1eb52e1d62d0648be Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 18 Jul 2012 14:09:24 +0100 Subject: [PATCH 0675/1279] Deref list custom id fix --- mongoengine/dereference.py | 2 +- mongoengine/queryset.py | 5 ++--- tests/test_dereference.py | 24 +++++++++++++++++++++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 84c57bb..f74e224 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -166,7 +166,7 @@ class DeReference(object): else: data[k] = v - if k in self.object_map: + if k in self.object_map and not is_list: data[k] = self.object_map[k] elif hasattr(v, '_fields'): for field_name, field in v._fields.iteritems(): diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index e3b2d99..04d4641 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -813,11 +813,10 @@ class QuerySet(object): have to create a new document. Passes any write_options onto :meth:`~mongoengine.Document.save` - .. versionadded:: 0.3 - :param auto_save: if the object is to be saved automatically if not found. - .. versionadded:: 0.6 + .. versionadded:: 0.3 + .. versionupdated:: 0.6 - added `auto_save` """ defaults = query.get('defaults', {}) if 'defaults' in query: diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 9f0d433..826d4aa 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -810,7 +810,7 @@ class FieldTest(unittest.TestCase): room = Room.objects.first().select_related() self.assertEquals(room.staffs_with_position[0]['staff'], sarah) self.assertEquals(room.staffs_with_position[1]['staff'], bob) - + def test_document_reload_no_inheritance(self): class Foo(Document): meta = {'allow_inheritance': False} @@ -841,3 +841,25 @@ class FieldTest(unittest.TestCase): self.assertEquals(type(foo.bar), Bar) self.assertEquals(type(foo.baz), Baz) + + def test_list_lookup_not_checked_in_map(self): + """Ensure we dereference list data correctly + """ + class Comment(Document): + id = IntField(primary_key=True) + text = StringField() + + class Message(Document): + id = IntField(primary_key=True) + comments = ListField(ReferenceField(Comment)) + + Comment.drop_collection() + Message.drop_collection() + + c1 = Comment(id=0, text='zero').save() + c2 = Comment(id=1, text='one').save() + Message(id=1, comments=[c1, c2]).save() + + msg = Message.objects.get(id=1) + self.assertEqual(0, msg.comments[0].id) + self.assertEqual(1, msg.comments[1].id) \ No newline at end of file From 8879d5560b607ea4b20699dcff8fffb2f88bab98 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 19 Jul 2012 10:32:33 +0100 Subject: [PATCH 0676/1279] Added support for args / kwargs and queryset_manager Closes MongoEngine/mongoengine#37 --- docs/changelog.rst | 4 ++++ mongoengine/queryset.py | 9 +++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 545ce76..4c5bf52 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.6.15 +================= +- Deref list custom id fix + Changes in 0.6.14 ================= - Fixed error dict with nested validation diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 04d4641..f94b709 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -4,6 +4,8 @@ import copy import itertools import operator +from functools import partial + import pymongo from bson.code import Code @@ -1871,10 +1873,13 @@ class QuerySetManager(object): queryset_class = owner._meta['queryset_class'] or QuerySet queryset = queryset_class(owner, owner._get_collection()) if self.get_queryset: - if self.get_queryset.func_code.co_argcount == 1: + var_names = self.get_queryset.func_code.co_varnames + if var_names == ('queryset',): queryset = self.get_queryset(queryset) - else: + elif var_names == ('doc_cls', 'queryset',): queryset = self.get_queryset(owner, queryset) + else: + queryset = partial(self.get_queryset, owner, queryset) return queryset From 87ba69d02e019c593c25b92667ff9ccfd1161605 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 19 Jul 2012 10:35:37 +0100 Subject: [PATCH 0677/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4c5bf52..57f4986 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.15 ================= +- Added support for args / kwargs when using @queryset_manager - Deref list custom id fix Changes in 0.6.14 From 1e51180d42bae13ba51b4fb16160b275155c2abd Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 19 Jul 2012 11:39:52 +0100 Subject: [PATCH 0678/1279] Fixed geo index creation bug fixes MongoEngine/mongoengine#36 --- docs/changelog.rst | 1 + mongoengine/base.py | 4 ++++ mongoengine/queryset.py | 1 - tests/test_document.py | 25 ++++++++++++++++++------- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 57f4986..2ada58e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.15 ================= +- Fixed geo index creation through reference fields - Added support for args / kwargs when using @queryset_manager - Deref list custom id fix diff --git a/mongoengine/base.py b/mongoengine/base.py index 6eb0b0f..0d216d9 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1113,7 +1113,11 @@ Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, error inspected = inspected or [] geo_indices = [] inspected.append(cls) + + from fields import EmbeddedDocumentField, GeoPointField for field in cls._fields.values(): + if not isinstance(field, (EmbeddedDocumentField, GeoPointField)): + continue if hasattr(field, 'document_type'): field_cls = field.document_type if field_cls in inspected: diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index f94b709..4f7443f 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -483,7 +483,6 @@ class QuerySet(object): self._collection.ensure_index(index_spec, background=background, **index_opts) - @classmethod def _build_index_spec(cls, doc_cls, spec): """Build a PyMongo index spec from a MongoEngine index spec. diff --git a/tests/test_document.py b/tests/test_document.py index 491a685..30d9244 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -872,15 +872,26 @@ class DocumentTest(unittest.TestCase): def test_geo_indexes_recursion(self): - class User(Document): - channel = ReferenceField('Channel') + class Location(Document): + name = StringField() location = GeoPointField() - class Channel(Document): - user = ReferenceField('User') - location = GeoPointField() + class Parent(Document): + name = StringField() + location = ReferenceField(Location) - self.assertEquals(len(User._geo_indices()), 2) + Location.drop_collection() + Parent.drop_collection() + + list(Parent.objects) + + collection = Parent._get_collection() + info = collection.index_information() + + self.assertFalse('location_2d' in info) + + self.assertEquals(len(Parent._geo_indices()), 0) + self.assertEquals(len(Location._geo_indices()), 1) def test_covered_index(self): """Ensure that covered indexes can be used @@ -3170,7 +3181,7 @@ name: Field is required ("name")""" class Person(BasePerson): name = StringField(required=True) - + p = Person(age=15) self.assertRaises(ValidationError, p.validate) From ae39ed94c9f86b2d6e0898e8f2b8f4796a844d04 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 19 Jul 2012 11:52:26 +0100 Subject: [PATCH 0679/1279] Fixed cascade save edge case refs MongoEngine/mongoengine#40 --- docs/changelog.rst | 1 + mongoengine/document.py | 5 +++++ tests/test_document.py | 24 ++++++++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2ada58e..b3925e7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.15 ================= +- Fixed cascade save edge case - Fixed geo index creation through reference fields - Added support for args / kwargs when using @queryset_manager - Deref list custom id fix diff --git a/mongoengine/document.py b/mongoengine/document.py index c36645f..2779278 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -248,11 +248,16 @@ class Document(BaseDocument): _refs = kwargs.get('_refs', []) or [] for name, cls in self._fields.items(): + if not isinstance(cls, (ReferenceField, GenericReferenceField)): continue + ref = getattr(self, name) if not ref: continue + if isinstance(ref, DBRef): + continue + ref_id = "%s,%s" % (ref.__class__.__name__, str(ref._data)) if ref and ref_id not in _refs: _refs.append(ref_id) diff --git a/tests/test_document.py b/tests/test_document.py index 30d9244..992c283 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -3186,5 +3186,29 @@ name: Field is required ("name")""" p = Person(age=15) self.assertRaises(ValidationError, p.validate) + def test_cascaded_save_wrong_reference(self): + + class ADocument(Document): + val = IntField() + + class BDocument(Document): + a = ReferenceField(ADocument) + + ADocument.drop_collection() + BDocument.drop_collection() + + a = ADocument() + a.val = 15 + a.save() + + b = BDocument() + b.a = a + b.save() + + a.delete() + + b = BDocument.objects.first() + b.save(cascade=True) + if __name__ == '__main__': unittest.main() From d83e67c121241da23c6595611f2d024f3e31a591 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 19 Jul 2012 12:08:07 +0100 Subject: [PATCH 0680/1279] Added support for null / zero / false values in item_frequencies refs /MongoEngine/mongoengine#40 --- docs/changelog.rst | 1 + mongoengine/queryset.py | 77 +++++++++++++++++++++++------------------ tests/test_queryset.py | 53 +++++++++++++++++++++++++--- 3 files changed, 93 insertions(+), 38 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b3925e7..466fdcb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.15 ================= +- Added support for null / zero / false values in item_frequencies - Fixed cascade save edge case - Fixed geo index creation through reference fields - Added support for args / kwargs when using @queryset_manager diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 4f7443f..6499c3e 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1718,10 +1718,11 @@ class QuerySet(object): def _item_frequencies_map_reduce(self, field, normalize=False): map_func = """ function() { - path = '{{~%(field)s}}'.split('.'); - field = this; + var path = '{{~%(field)s}}'.split('.'); + var field = this; + for (p in path) { - if (field) + if (typeof field != 'undefined') field = field[path[p]]; else break; @@ -1730,7 +1731,7 @@ class QuerySet(object): field.forEach(function(item) { emit(item, 1); }); - } else if (field) { + } else if (typeof field != 'undefined') { emit(field, 1); } else { emit(null, 1); @@ -1754,12 +1755,12 @@ class QuerySet(object): if isinstance(key, float): if int(key) == key: key = int(key) - key = str(key) - frequencies[key] = f.value + frequencies[key] = int(f.value) if normalize: count = sum(frequencies.values()) - frequencies = dict([(k, v / count) for k, v in frequencies.items()]) + frequencies = dict([(k, float(v) / count) + for k, v in frequencies.items()]) return frequencies @@ -1767,31 +1768,28 @@ class QuerySet(object): """Uses exec_js to execute""" freq_func = """ function(path) { - path = path.split('.'); + var path = path.split('.'); - if (options.normalize) { - var total = 0.0; - db[collection].find(query).forEach(function(doc) { - field = doc; - for (p in path) { - if (field) - field = field[path[p]]; - else - break; - } - if (field && field.constructor == Array) { - total += field.length; - } else { - total++; - } - }); - } + var total = 0.0; + db[collection].find(query).forEach(function(doc) { + var field = doc; + for (p in path) { + if (field) + field = field[path[p]]; + else + break; + } + if (field && field.constructor == Array) { + total += field.length; + } else { + total++; + } + }); var frequencies = {}; + var types = {}; var inc = 1.0; - if (options.normalize) { - inc /= total; - } + db[collection].find(query).forEach(function(doc) { field = doc; for (p in path) { @@ -1806,17 +1804,28 @@ class QuerySet(object): }); } else { var item = field; + types[item] = item; frequencies[item] = inc + (isNaN(frequencies[item]) ? 0: frequencies[item]); } }); - return frequencies; + return [total, frequencies, types]; } """ - data = self.exec_js(freq_func, field, normalize=normalize) - if 'undefined' in data: - data[None] = data['undefined'] - del(data['undefined']) - return data + total, data, types = self.exec_js(freq_func, field) + values = dict([(types.get(k), int(v)) for k, v in data.iteritems()]) + + if normalize: + values = dict([(k, float(v) / total) for k, v in values.items()]) + + frequencies = {} + for k, v in values.iteritems(): + if isinstance(k, float): + if int(k) == k: + k = int(k) + + frequencies[k] = v + + return frequencies def __repr__(self): """Provides the string representation of the QuerySet diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 939451c..1bac6a9 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -1994,9 +1994,9 @@ class QuerySetTest(unittest.TestCase): # Check item_frequencies works for non-list fields def test_assertions(f): - self.assertEqual(set(['1', '2']), set(f.keys())) - self.assertEqual(f['1'], 1) - self.assertEqual(f['2'], 2) + self.assertEqual(set([1, 2]), set(f.keys())) + self.assertEqual(f[1], 1) + self.assertEqual(f[2], 2) exec_js = BlogPost.objects.item_frequencies('hits') map_reduce = BlogPost.objects.item_frequencies('hits', map_reduce=True) @@ -2096,7 +2096,6 @@ class QuerySetTest(unittest.TestCase): data = EmbeddedDocumentField(Data, required=True) extra = EmbeddedDocumentField(Extra) - Person.drop_collection() p = Person() @@ -2114,6 +2113,52 @@ class QuerySetTest(unittest.TestCase): ot = Person.objects.item_frequencies('extra.tag', map_reduce=True) self.assertEquals(ot, {None: 1.0, u'friend': 1.0}) + def test_item_frequencies_with_0_values(self): + class Test(Document): + val = IntField() + + Test.drop_collection() + t = Test() + t.val = 0 + t.save() + + ot = Test.objects.item_frequencies('val', map_reduce=True) + self.assertEquals(ot, {0: 1}) + ot = Test.objects.item_frequencies('val', map_reduce=False) + self.assertEquals(ot, {0: 1}) + + def test_item_frequencies_with_False_values(self): + class Test(Document): + val = BooleanField() + + Test.drop_collection() + t = Test() + t.val = False + t.save() + + ot = Test.objects.item_frequencies('val', map_reduce=True) + self.assertEquals(ot, {False: 1}) + ot = Test.objects.item_frequencies('val', map_reduce=False) + self.assertEquals(ot, {False: 1}) + + def test_item_frequencies_normalize(self): + class Test(Document): + val = IntField() + + Test.drop_collection() + + for i in xrange(50): + Test(val=1).save() + + for i in xrange(20): + Test(val=2).save() + + freqs = Test.objects.item_frequencies('val', map_reduce=False, normalize=True) + self.assertEquals(freqs, {1: 50.0/70, 2: 20.0/70}) + + freqs = Test.objects.item_frequencies('val', map_reduce=True, normalize=True) + self.assertEquals(freqs, {1: 50.0/70, 2: 20.0/70}) + def test_average(self): """Ensure that field can be averaged correctly. """ From 1b17fb0ae7d09503dff574a12face242354e6fd3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 19 Jul 2012 15:04:12 +0100 Subject: [PATCH 0681/1279] Updated validation error messages refs hmarr/mongoengine#539 --- docs/changelog.rst | 1 + mongoengine/base.py | 27 +++++++++++++++------------ tests/test_document.py | 10 ++++------ tests/test_fields.py | 2 +- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 466fdcb..b4d747d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.15 ================= +- Updated validation error message - Added support for null / zero / false values in item_frequencies - Fixed cascade save edge case - Fixed geo index creation through reference fields diff --git a/mongoengine/base.py b/mongoengine/base.py index 0d216d9..3ab6850 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1,4 +1,5 @@ import warnings +from collections import defaultdict from queryset import QuerySet, QuerySetManager from queryset import DoesNotExist, MultipleObjectsReturned @@ -53,9 +54,9 @@ class ValidationError(AssertionError): message = super(ValidationError, self).__getattribute__(name) if name == 'message': if self.field_name: - message = '%s ("%s")' % (message, self.field_name) + message = '%s' % message if self.errors: - message = '%s:\n%s' % (message, self._format_errors()) + message = '%s(%s)' % (message, self._format_errors()) return message def _get_message(self): @@ -93,17 +94,20 @@ class ValidationError(AssertionError): def _format_errors(self): """Returns a string listing all errors within a document""" - def format_error(field, value, prefix=''): - prefix = "%s.%s" % (prefix, field) if prefix else "%s" % field + def generate_key(value, prefix=''): + if isinstance(value, list): + value = ' '.join([generate_key(k) for k in value]) if isinstance(value, dict): + value = ' '.join( + [generate_key(v, k) for k, v in value.iteritems()]) - return '\n'.join( - [format_error(k, value[k], prefix) for k in value]) - else: - return "%s: %s" % (prefix, value) + results = "%s.%s" % (prefix, value) if prefix else value + return results - return '\n'.join( - [format_error(k, v) for k, v in self.to_dict().items()]) + error_dict = defaultdict(list) + for k, v in self.to_dict().iteritems(): + error_dict[generate_key(v)].append(k) + return ' '.join(["%s: %s" % (k, v) for k, v in error_dict.iteritems()]) _document_registry = {} @@ -899,8 +903,7 @@ class BaseDocument(object): errors[field.name] = ValidationError('Field is required', field_name=field.name) if errors: - raise ValidationError('Errors encountered validating document', - errors=errors) + raise ValidationError('ValidationError', errors=errors) def to_mongo(self): """Return data dictionary ready for use with MongoDB. diff --git a/tests/test_document.py b/tests/test_document.py index 992c283..8250861 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -3137,7 +3137,7 @@ class ValidatorErrorTest(unittest.TestCase): self.assertEquals(error.to_dict()['1st']['2nd']['3rd']['4th'], 'Inception') - self.assertEquals(error.message, "root:\n1st.2nd.3rd.4th: Inception") + self.assertEquals(error.message, "root(2nd.3rd.4th.Inception: ['1st'])") def test_model_validation(self): @@ -3148,13 +3148,11 @@ class ValidatorErrorTest(unittest.TestCase): try: User().validate() except ValidationError, e: - expected_error_message = """Errors encountered validating document: -username: Field is required ("username") -name: Field is required ("name")""" + expected_error_message = """ValidationError(Field is required: ['username', 'name'])""" self.assertEquals(e.message, expected_error_message) self.assertEquals(e.to_dict(), { - 'username': 'Field is required ("username")', - 'name': u'Field is required ("name")'}) + 'username': 'Field is required', + 'name': 'Field is required'}) def test_spaces_in_keys(self): diff --git a/tests/test_fields.py b/tests/test_fields.py index 2c7e694..88dfbf5 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -2122,7 +2122,7 @@ class FieldTest(unittest.TestCase): self.assertTrue(1 in error_dict['comments']) self.assertTrue('content' in error_dict['comments'][1]) self.assertEquals(error_dict['comments'][1]['content'], - u'Field is required ("content")') + 'Field is required') post.comments[1].content = 'here we go' post.validate() From 5c912b930e089d9b9a08201f29b280e9e6a487d0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 19 Jul 2012 16:03:29 +0100 Subject: [PATCH 0682/1279] Removed tests testing MongoDB not mongoengine --- tests/test_fields.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/test_fields.py b/tests/test_fields.py index 88dfbf5..72972cc 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -358,24 +358,6 @@ class FieldTest(unittest.TestCase): self.assertNotEquals(log.date, d1) self.assertEquals(log.date, d2) - # Pre UTC microseconds above 1000 is wonky. - # log.date has an invalid microsecond value so I can't construct - # a date to compare. - # - # However, the timedelta is predicable with pre UTC timestamps - # It always adds 16 seconds and [777216-776217] microseconds - for i in xrange(1001, 3113, 33): - d1 = datetime.datetime(1969, 12, 31, 23, 59, 59, i) - log.date = d1 - log.save() - log.reload() - self.assertNotEquals(log.date, d1) - - delta = log.date - d1 - self.assertEquals(delta.seconds, 16) - microseconds = 777216 - (i % 1000) - self.assertEquals(delta.microseconds, microseconds) - LogEntry.drop_collection() def test_complexdatetime_storage(self): From 83c11a9834da757806f61555ca7ff551106ec16c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 19 Jul 2012 16:12:21 +0100 Subject: [PATCH 0683/1279] Version bump --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 3218b29..c26caf4 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 14) +VERSION = (0, 6, 15) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 4ac9508..fd7b197 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.14 +Version: 0.6.15 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 3070e0bf5d6eef8548a980b7223f9d6153ed07d0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 20 Jul 2012 10:34:08 +0100 Subject: [PATCH 0684/1279] Fix for inheritance bug and db_alias --- docs/changelog.rst | 7 ++++++- mongoengine/__init__.py | 2 +- mongoengine/base.py | 15 ++++++++------- python-mongoengine.spec | 2 +- tests/test_document.py | 14 +++++++++++++- 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b4d747d..19d21b8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,9 +2,14 @@ Changelog ========= + +Changes in 0.6.16 +================= +- Fixed issue where db_alias wasn't inherited + Changes in 0.6.15 ================= -- Updated validation error message +- Updated validation error messages - Added support for null / zero / false values in item_frequencies - Fixed cascade save edge case - Fixed geo index creation through reference fields diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index c26caf4..ad12479 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 15) +VERSION = (0, 6, 16) def get_version(): diff --git a/mongoengine/base.py b/mongoengine/base.py index 3ab6850..09768c4 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -649,8 +649,13 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): del(attrs['meta']['collection']) if base._get_collection_name(): collection = base._get_collection_name() - # Propagate index options. - for key in ('index_background', 'index_drop_dups', 'index_opts'): + + # Propagate inherited values + keys_to_propogate = ( + 'index_background', 'index_drop_dups', 'index_opts', + 'allow_inheritance', 'queryset_class', 'db_alias', + ) + for key in keys_to_propogate: if key in base._meta: base_meta[key] = base._meta[key] @@ -659,11 +664,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): abstract_base_indexes += base._meta.get('indexes', []) else: base_indexes += base._meta.get('indexes', []) - # Propagate 'allow_inheritance' - if 'allow_inheritance' in base._meta: - base_meta['allow_inheritance'] = base._meta['allow_inheritance'] - if 'queryset_class' in base._meta: - base_meta['queryset_class'] = base._meta['queryset_class'] try: base_meta['objects'] = base.__getattribute__(base, 'objects') except TypeError: @@ -671,6 +671,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): except AttributeError: pass + # defaults meta = { 'abstract': False, 'collection': collection, diff --git a/python-mongoengine.spec b/python-mongoengine.spec index fd7b197..03093b3 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.15 +Version: 0.6.16 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB diff --git a/tests/test_document.py b/tests/test_document.py index 8250861..6915caf 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -2999,7 +2999,7 @@ class DocumentTest(unittest.TestCase): self.assertEqual(User.objects.first(), bob) self.assertEqual(Book.objects.first(), hp) - # DeRefecence + # DeReference class AuthorBooks(Document): author = ReferenceField(User) book = ReferenceField(Book) @@ -3027,6 +3027,18 @@ class DocumentTest(unittest.TestCase): self.assertEqual(Book._get_collection(), get_db("testdb-2")[Book._get_collection_name()]) self.assertEqual(AuthorBooks._get_collection(), get_db("testdb-3")[AuthorBooks._get_collection_name()]) + def test_db_alias_propagates(self): + """db_alias propagates? + """ + class A(Document): + name = StringField() + meta = {"db_alias": "testdb-1", "allow_inheritance": True} + + class B(A): + pass + + self.assertEquals('testdb-1', B._meta.get('db_alias')) + def test_db_ref_usage(self): """ DB Ref usage in __raw__ queries """ From 601f0eb1682ebd9c9eb67f1b68b76bdd2cf3d8c7 Mon Sep 17 00:00:00 2001 From: Max Countryman Date: Fri, 20 Jul 2012 19:12:43 -0700 Subject: [PATCH 0685/1279] Correcting typo in DynamicField docstring --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 01d50c1..e94fe30 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -483,7 +483,7 @@ class GenericEmbeddedDocumentField(BaseField): class DynamicField(BaseField): - """A tryly dynamic field type capable of handling different and varying + """A truly dynamic field type capable of handling different and varying types of data. Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data""" From 1a4533a9cf9f3a706d79a250db3db6522678c46a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 23 Jul 2012 14:46:48 +0100 Subject: [PATCH 0686/1279] Minor perf update --- mongoengine/base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 09768c4..7b1f320 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -198,7 +198,8 @@ class BaseField(object): """Descriptor for assigning a value to a field in a document. """ instance._data[self.name] = value - instance._mark_as_changed(self.name) + if instance._initialised: + instance._mark_as_changed(self.name) def error(self, message="", errors=None, field_name=None): """Raises a ValidationError. @@ -791,6 +792,8 @@ class BaseDocument(object): # Assign default values to instance for attr_name, field in self._fields.items(): + if self._db_field_map.get(attr_name, attr_name) in values: + continue value = getattr(self, attr_name, None) setattr(self, attr_name, value) @@ -821,6 +824,8 @@ class BaseDocument(object): signals.post_init.send(self.__class__, document=self) def __setattr__(self, name, value): + if not self._initialised: + return super(BaseDocument, self).__setattr__(name, value) # Handle dynamic data only if an initialised dynamic document if self._dynamic and not self._dynamic_lock: From 598ffd3e5c107990dd41995d0bbb21bf13f67fb5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 23 Jul 2012 15:32:02 +0100 Subject: [PATCH 0687/1279] Fixed documentation --- docs/changelog.rst | 2 +- docs/guide/defining-documents.rst | 29 +++++++++++++++++++++++++++++ mongoengine/fields.py | 6 ++++-- mongoengine/queryset.py | 8 ++++---- 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 19d21b8..fa0f361 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -27,7 +27,7 @@ Changes in 0.6.14 - Added support for add_to_set and each Changes in 0.6.13 -================ +================= - Fixed EmbeddedDocument db_field validation issue - Fixed StringField unicode issue - Fixes __repr__ modifying the cursor diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 726ce3b..b4facbd 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -259,6 +259,35 @@ as the constructor's argument:: content = StringField() +.. _one-to-many-with-listfields: + +One to Many with ListFields +''''''''''''''''''''''''''' + +If you are implementing a one to many relationship via a list of references, +then the references are stored as DBRefs and to query you need to pass an +instance of the object to the query:: + + class User(Document): + name = StringField() + + class Page(Document): + content = StringField() + authors = ListField(ReferenceField(User)) + + bob = User(name="Bob Jones").save() + john = User(name="John Smith").save() + + Page(content="Test Page", authors=[bob, john]).save() + Page(content="Another Page", authors=[john]).save() + + # Find all pages Bob authored + Page.objects(authors__in=[bob]) + + # Find all pages that both Bob and John have authored + Page.objects(authors__all=[bob, john]) + + Dealing with deletion of referred documents ''''''''''''''''''''''''''''''''''''''''''' By default, MongoDB doesn't check the integrity of your data, so deleting diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 01d50c1..ba8ff2d 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -169,7 +169,7 @@ class IntField(BaseField): def prepare_query_value(self, op, value): if value is None: return value - + return int(value) @@ -199,7 +199,7 @@ class FloatField(BaseField): def prepare_query_value(self, op, value): if value is None: return value - + return float(value) @@ -530,6 +530,8 @@ class ListField(ComplexBaseField): """A list field that wraps a standard field, allowing multiple instances of the field to be used as a list in the database. + If using with ReferenceFields see: :ref:`one-to-many-with-listfields` + .. note:: Required means it cannot be empty - as the default for ListFields is [] """ diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6499c3e..f68545b 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -806,9 +806,9 @@ class QuerySet(object): keyword argument called :attr:`defaults`. .. note:: This requires two separate operations and therefore a - race condition exists. Because there are no transactions in mongoDB - other approaches should be investigated, to ensure you don't - accidently duplicate data when using this method. + race condition exists. Because there are no transactions in mongoDB + other approaches should be investigated, to ensure you don't + accidently duplicate data when using this method. :param write_options: optional extra keyword arguments used if we have to create a new document. @@ -816,8 +816,8 @@ class QuerySet(object): :param auto_save: if the object is to be saved automatically if not found. + .. versionchanged:: 0.6 - added `auto_save` .. versionadded:: 0.3 - .. versionupdated:: 0.6 - added `auto_save` """ defaults = query.get('defaults', {}) if 'defaults' in query: From 8bde0c0e539594db177c37290e76045a9d4ca286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Mon, 23 Jul 2012 12:31:47 -0300 Subject: [PATCH 0688/1279] Small fix in SequenceField --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index ba8ff2d..42f11ea 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1289,7 +1289,7 @@ class SequenceField(IntField): instance._data[self.name] = value instance._mark_as_changed(self.name) - return value + return int(value) if value else None def __set__(self, instance, value): From 1304f2721f7850b223950edb85ec7c141255176c Mon Sep 17 00:00:00 2001 From: Chris Faulkner Date: Tue, 24 Jul 2012 14:06:43 -0700 Subject: [PATCH 0689/1279] Proper syntax for RST notes (so they actually render). --- mongoengine/document.py | 2 +- mongoengine/fields.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 2779278..f8bf769 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -375,7 +375,7 @@ class DynamicDocument(Document): :class:`~mongoengine.DynamicField` and data can be attributed to that field. - ..note:: + .. note:: There is one caveat on Dynamic Documents: fields cannot start with `_` """ diff --git a/mongoengine/fields.py b/mongoengine/fields.py index ba8ff2d..43f58e5 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -451,7 +451,7 @@ class GenericEmbeddedDocumentField(BaseField): Only valid values are subclasses of :class:`~mongoengine.EmbeddedDocument`. - ..note :: You can use the choices param to limit the acceptable + .. note:: You can use the choices param to limit the acceptable EmbeddedDocument types """ @@ -768,10 +768,10 @@ class GenericReferenceField(BaseField): """A reference to *any* :class:`~mongoengine.document.Document` subclass that will be automatically dereferenced on access (lazily). - ..note :: Any documents used as a generic reference must be registered in the + .. note:: Any documents used as a generic reference must be registered in the document registry. Importing the model will automatically register it. - ..note :: You can use the choices param to limit the acceptable Document types + .. note:: You can use the choices param to limit the acceptable Document types .. versionadded:: 0.3 """ From 6459d4c0b60229edcd4d562898833f221d00ebf4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 25 Jul 2012 14:55:10 +0100 Subject: [PATCH 0690/1279] Fixed issue with custom queryset manager expecting explict variable names If using / expecting kwargs you have to call the queryset manager explicitly. --- docs/changelog.rst | 4 ++++ mongoengine/queryset.py | 4 ++-- tests/test_queryset.py | 30 +++++++++++++++--------------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index fa0f361..22abe18 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -3,6 +3,10 @@ Changelog ========= +Changes in 0.6.17 +================= +- Fixed issue with custom queryset manager expecting explict variable names + Changes in 0.6.16 ================= - Fixed issue where db_alias wasn't inherited diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index f68545b..0d1d95b 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1882,9 +1882,9 @@ class QuerySetManager(object): queryset = queryset_class(owner, owner._get_collection()) if self.get_queryset: var_names = self.get_queryset.func_code.co_varnames - if var_names == ('queryset',): + if len(var_names) == 1: queryset = self.get_queryset(queryset) - elif var_names == ('doc_cls', 'queryset',): + elif len(var_names) == 2: queryset = self.get_queryset(owner, queryset) else: queryset = partial(self.get_queryset, owner, queryset) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 1bac6a9..c1abb4d 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -2228,28 +2228,28 @@ class QuerySetTest(unittest.TestCase): date = DateTimeField(default=datetime.now) @queryset_manager - def objects(doc_cls, queryset): - return queryset(deleted=False) + def objects(cls, qryset): + return qryset(deleted=False) @queryset_manager - def music_posts(doc_cls, queryset): - return queryset(tags='music', deleted=False).order_by('-date') + def music_posts(doc_cls, queryset, deleted=False): + return queryset(tags='music', + deleted=deleted).order_by('date') BlogPost.drop_collection() - post1 = BlogPost(tags=['music', 'film']) - post1.save() - post2 = BlogPost(tags=['music']) - post2.save() - post3 = BlogPost(tags=['film', 'actors']) - post3.save() - post4 = BlogPost(tags=['film', 'actors'], deleted=True) - post4.save() + post1 = BlogPost(tags=['music', 'film']).save() + post2 = BlogPost(tags=['music']).save() + post3 = BlogPost(tags=['film', 'actors']).save() + post4 = BlogPost(tags=['film', 'actors', 'music'], deleted=True).save() - self.assertEqual([p.id for p in BlogPost.objects], + self.assertEqual([p.id for p in BlogPost.objects()], [post1.id, post2.id, post3.id]) - self.assertEqual([p.id for p in BlogPost.music_posts], - [post2.id, post1.id]) + self.assertEqual([p.id for p in BlogPost.music_posts()], + [post1.id, post2.id]) + + self.assertEqual([p.id for p in BlogPost.music_posts(True)], + [post4.id]) BlogPost.drop_collection() From 24fd1acce68341361331e033d1692d61a936f871 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jul 2012 14:14:10 +0100 Subject: [PATCH 0691/1279] Version bump --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index ad12479..2b6af43 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 16) +VERSION = (0, 6, 17) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 03093b3..ea2c604 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.16 +Version: 0.6.17 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 6526923345efd768044c4fba770a8a3ada161a40 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jul 2012 16:00:32 +0100 Subject: [PATCH 0692/1279] Fixed recursion loading bug in _get_changed_fields fixes hmarr/mongoengine#548 --- docs/changelog.rst | 4 +++ mongoengine/__init__.py | 2 +- mongoengine/base.py | 3 ++- python-mongoengine.spec | 2 +- tests/test_queryset.py | 58 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 66 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 22abe18..97926f8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -3,6 +3,10 @@ Changelog ========= +Changes in 0.6.18 +================= +- Fixed recursion loading bug in _get_changed_fields + Changes in 0.6.17 ================= - Fixed issue with custom queryset manager expecting explict variable names diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 2b6af43..5f4f7f1 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 17) +VERSION = (0, 6, 18) def get_version(): diff --git a/mongoengine/base.py b/mongoengine/base.py index 09768c4..6fb26cb 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1012,9 +1012,10 @@ Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, error field_list.update(self._dynamic_fields) for field_name in field_list: + db_field_name = self._db_field_map.get(field_name, field_name) key = '%s.' % db_field_name - field = getattr(self, field_name, None) + field = self._data.get(field_name, None) if hasattr(field, 'id'): if field.id in inspected: continue diff --git a/python-mongoengine.spec b/python-mongoengine.spec index ea2c604..4dfbecc 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.17 +Version: 0.6.18 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB diff --git a/tests/test_queryset.py b/tests/test_queryset.py index c1abb4d..e623790 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -579,6 +579,64 @@ class QuerySetTest(unittest.TestCase): Blog.objects.insert([blog2, blog3], write_options={'continue_on_error': True}) self.assertEqual(Blog.objects.count(), 3) + def test_get_changed_fields_query_count(self): + + class Person(Document): + name = StringField() + owns = ListField(ReferenceField('Organization')) + projects = ListField(ReferenceField('Project')) + + class Organization(Document): + name = StringField() + owner = ReferenceField('Person') + employees = ListField(ReferenceField('Person')) + + class Project(Document): + name = StringField() + + Person.drop_collection() + Organization.drop_collection() + Project.drop_collection() + + r1 = Project(name="r1").save() + r2 = Project(name="r2").save() + r3 = Project(name="r3").save() + p1 = Person(name="p1", projects=[r1, r2]).save() + p2 = Person(name="p2", projects=[r2]).save() + o1 = Organization(name="o1", employees=[p1]).save() + + with query_counter() as q: + self.assertEqual(q, 0) + + fresh_o1 = Organization.objects.get(id=o1.id) + self.assertEqual(1, q) + fresh_o1._get_changed_fields() + self.assertEqual(1, q) + + with query_counter() as q: + self.assertEqual(q, 0) + + fresh_o1 = Organization.objects.get(id=o1.id) + fresh_o1.save() + + self.assertEquals(q, 2) + + with query_counter() as q: + self.assertEqual(q, 0) + + fresh_o1 = Organization.objects.get(id=o1.id) + fresh_o1.save(cascade=False) + + self.assertEquals(q, 2) + + with query_counter() as q: + self.assertEqual(q, 0) + + fresh_o1 = Organization.objects.get(id=o1.id) + fresh_o1.employees.append(p2) + fresh_o1.save(cascade=False) + + self.assertEquals(q, 3) def test_slave_okay(self): """Ensures that a query can take slave_okay syntax From 3628a7653ca6e1e51a712b751db9dcc5e6621b51 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 26 Jul 2012 22:50:39 +0100 Subject: [PATCH 0693/1279] Squashed commit of the following: commit 48f988acd728f1193b57df8cf6b0154d69c15099 Merge: 6526923 1304f27 Author: Ross Lawley Date: Thu Jul 26 08:17:45 2012 -0700 Merge pull request #44 from faulkner/fix-notes Proper syntax for RST notes (so they actually render). commit 6526923345efd768044c4fba770a8a3ada161a40 Author: Ross Lawley Date: Thu Jul 26 16:00:32 2012 +0100 Fixed recursion loading bug in _get_changed_fields fixes hmarr/mongoengine#548 commit 24fd1acce68341361331e033d1692d61a936f871 Author: Ross Lawley Date: Thu Jul 26 14:14:10 2012 +0100 Version bump commit cbb9235dc5d863ee0bb8a315c976581e71b6a641 Merge: 6459d4c 19ec2c9 Author: Ross Lawley Date: Wed Jul 25 15:12:34 2012 +0100 Merge branch 'master' of github.com:hmarr/mongoengine commit 19ec2c9bc98db454680e681373f3fcd3b0f79a6c Merge: 3070e0b 601f0eb Author: Ross Lawley Date: Wed Jul 25 07:12:07 2012 -0700 Merge pull request #545 from maxcountryman/patch-1 Correcting typo in DynamicField docstring commit 6459d4c0b60229edcd4d562898833f221d00ebf4 Author: Ross Lawley Date: Wed Jul 25 14:55:10 2012 +0100 Fixed issue with custom queryset manager expecting explict variable names If using / expecting kwargs you have to call the queryset manager explicitly. commit 1304f2721f7850b223950edb85ec7c141255176c Author: Chris Faulkner Date: Tue Jul 24 14:06:43 2012 -0700 Proper syntax for RST notes (so they actually render). commit 598ffd3e5c107990dd41995d0bbb21bf13f67fb5 Author: Ross Lawley Date: Mon Jul 23 15:32:02 2012 +0100 Fixed documentation commit 601f0eb1682ebd9c9eb67f1b68b76bdd2cf3d8c7 Author: Max Countryman Date: Fri Jul 20 19:12:43 2012 -0700 Correcting typo in DynamicField docstring --- docs/changelog.rst | 10 +++- docs/guide/defining-documents.rst | 29 ++++++++++ mongoengine/__init__.py | 2 +- mongoengine/base.py | 3 +- mongoengine/document.py | 2 +- mongoengine/fields.py | 14 ++--- mongoengine/queryset.py | 12 ++--- python-mongoengine.spec | 2 +- tests/test_queryset.py | 88 +++++++++++++++++++++++++------ 9 files changed, 130 insertions(+), 32 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 19d21b8..97926f8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -3,6 +3,14 @@ Changelog ========= +Changes in 0.6.18 +================= +- Fixed recursion loading bug in _get_changed_fields + +Changes in 0.6.17 +================= +- Fixed issue with custom queryset manager expecting explict variable names + Changes in 0.6.16 ================= - Fixed issue where db_alias wasn't inherited @@ -27,7 +35,7 @@ Changes in 0.6.14 - Added support for add_to_set and each Changes in 0.6.13 -================ +================= - Fixed EmbeddedDocument db_field validation issue - Fixed StringField unicode issue - Fixes __repr__ modifying the cursor diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 726ce3b..b4facbd 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -259,6 +259,35 @@ as the constructor's argument:: content = StringField() +.. _one-to-many-with-listfields: + +One to Many with ListFields +''''''''''''''''''''''''''' + +If you are implementing a one to many relationship via a list of references, +then the references are stored as DBRefs and to query you need to pass an +instance of the object to the query:: + + class User(Document): + name = StringField() + + class Page(Document): + content = StringField() + authors = ListField(ReferenceField(User)) + + bob = User(name="Bob Jones").save() + john = User(name="John Smith").save() + + Page(content="Test Page", authors=[bob, john]).save() + Page(content="Another Page", authors=[john]).save() + + # Find all pages Bob authored + Page.objects(authors__in=[bob]) + + # Find all pages that both Bob and John have authored + Page.objects(authors__all=[bob, john]) + + Dealing with deletion of referred documents ''''''''''''''''''''''''''''''''''''''''''' By default, MongoDB doesn't check the integrity of your data, so deleting diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index ad12479..5f4f7f1 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 16) +VERSION = (0, 6, 18) def get_version(): diff --git a/mongoengine/base.py b/mongoengine/base.py index 7b1f320..43243b8 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1017,9 +1017,10 @@ Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, error field_list.update(self._dynamic_fields) for field_name in field_list: + db_field_name = self._db_field_map.get(field_name, field_name) key = '%s.' % db_field_name - field = getattr(self, field_name, None) + field = self._data.get(field_name, None) if hasattr(field, 'id'): if field.id in inspected: continue diff --git a/mongoengine/document.py b/mongoengine/document.py index 2779278..f8bf769 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -375,7 +375,7 @@ class DynamicDocument(Document): :class:`~mongoengine.DynamicField` and data can be attributed to that field. - ..note:: + .. note:: There is one caveat on Dynamic Documents: fields cannot start with `_` """ diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 01d50c1..94f79a1 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -169,7 +169,7 @@ class IntField(BaseField): def prepare_query_value(self, op, value): if value is None: return value - + return int(value) @@ -199,7 +199,7 @@ class FloatField(BaseField): def prepare_query_value(self, op, value): if value is None: return value - + return float(value) @@ -451,7 +451,7 @@ class GenericEmbeddedDocumentField(BaseField): Only valid values are subclasses of :class:`~mongoengine.EmbeddedDocument`. - ..note :: You can use the choices param to limit the acceptable + .. note:: You can use the choices param to limit the acceptable EmbeddedDocument types """ @@ -483,7 +483,7 @@ class GenericEmbeddedDocumentField(BaseField): class DynamicField(BaseField): - """A tryly dynamic field type capable of handling different and varying + """A truly dynamic field type capable of handling different and varying types of data. Used by :class:`~mongoengine.DynamicDocument` to handle dynamic data""" @@ -530,6 +530,8 @@ class ListField(ComplexBaseField): """A list field that wraps a standard field, allowing multiple instances of the field to be used as a list in the database. + If using with ReferenceFields see: :ref:`one-to-many-with-listfields` + .. note:: Required means it cannot be empty - as the default for ListFields is [] """ @@ -766,10 +768,10 @@ class GenericReferenceField(BaseField): """A reference to *any* :class:`~mongoengine.document.Document` subclass that will be automatically dereferenced on access (lazily). - ..note :: Any documents used as a generic reference must be registered in the + .. note:: Any documents used as a generic reference must be registered in the document registry. Importing the model will automatically register it. - ..note :: You can use the choices param to limit the acceptable Document types + .. note:: You can use the choices param to limit the acceptable Document types .. versionadded:: 0.3 """ diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6499c3e..0d1d95b 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -806,9 +806,9 @@ class QuerySet(object): keyword argument called :attr:`defaults`. .. note:: This requires two separate operations and therefore a - race condition exists. Because there are no transactions in mongoDB - other approaches should be investigated, to ensure you don't - accidently duplicate data when using this method. + race condition exists. Because there are no transactions in mongoDB + other approaches should be investigated, to ensure you don't + accidently duplicate data when using this method. :param write_options: optional extra keyword arguments used if we have to create a new document. @@ -816,8 +816,8 @@ class QuerySet(object): :param auto_save: if the object is to be saved automatically if not found. + .. versionchanged:: 0.6 - added `auto_save` .. versionadded:: 0.3 - .. versionupdated:: 0.6 - added `auto_save` """ defaults = query.get('defaults', {}) if 'defaults' in query: @@ -1882,9 +1882,9 @@ class QuerySetManager(object): queryset = queryset_class(owner, owner._get_collection()) if self.get_queryset: var_names = self.get_queryset.func_code.co_varnames - if var_names == ('queryset',): + if len(var_names) == 1: queryset = self.get_queryset(queryset) - elif var_names == ('doc_cls', 'queryset',): + elif len(var_names) == 2: queryset = self.get_queryset(owner, queryset) else: queryset = partial(self.get_queryset, owner, queryset) diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 03093b3..4dfbecc 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.16 +Version: 0.6.18 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 1bac6a9..e623790 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -579,6 +579,64 @@ class QuerySetTest(unittest.TestCase): Blog.objects.insert([blog2, blog3], write_options={'continue_on_error': True}) self.assertEqual(Blog.objects.count(), 3) + def test_get_changed_fields_query_count(self): + + class Person(Document): + name = StringField() + owns = ListField(ReferenceField('Organization')) + projects = ListField(ReferenceField('Project')) + + class Organization(Document): + name = StringField() + owner = ReferenceField('Person') + employees = ListField(ReferenceField('Person')) + + class Project(Document): + name = StringField() + + Person.drop_collection() + Organization.drop_collection() + Project.drop_collection() + + r1 = Project(name="r1").save() + r2 = Project(name="r2").save() + r3 = Project(name="r3").save() + p1 = Person(name="p1", projects=[r1, r2]).save() + p2 = Person(name="p2", projects=[r2]).save() + o1 = Organization(name="o1", employees=[p1]).save() + + with query_counter() as q: + self.assertEqual(q, 0) + + fresh_o1 = Organization.objects.get(id=o1.id) + self.assertEqual(1, q) + fresh_o1._get_changed_fields() + self.assertEqual(1, q) + + with query_counter() as q: + self.assertEqual(q, 0) + + fresh_o1 = Organization.objects.get(id=o1.id) + fresh_o1.save() + + self.assertEquals(q, 2) + + with query_counter() as q: + self.assertEqual(q, 0) + + fresh_o1 = Organization.objects.get(id=o1.id) + fresh_o1.save(cascade=False) + + self.assertEquals(q, 2) + + with query_counter() as q: + self.assertEqual(q, 0) + + fresh_o1 = Organization.objects.get(id=o1.id) + fresh_o1.employees.append(p2) + fresh_o1.save(cascade=False) + + self.assertEquals(q, 3) def test_slave_okay(self): """Ensures that a query can take slave_okay syntax @@ -2228,28 +2286,28 @@ class QuerySetTest(unittest.TestCase): date = DateTimeField(default=datetime.now) @queryset_manager - def objects(doc_cls, queryset): - return queryset(deleted=False) + def objects(cls, qryset): + return qryset(deleted=False) @queryset_manager - def music_posts(doc_cls, queryset): - return queryset(tags='music', deleted=False).order_by('-date') + def music_posts(doc_cls, queryset, deleted=False): + return queryset(tags='music', + deleted=deleted).order_by('date') BlogPost.drop_collection() - post1 = BlogPost(tags=['music', 'film']) - post1.save() - post2 = BlogPost(tags=['music']) - post2.save() - post3 = BlogPost(tags=['film', 'actors']) - post3.save() - post4 = BlogPost(tags=['film', 'actors'], deleted=True) - post4.save() + post1 = BlogPost(tags=['music', 'film']).save() + post2 = BlogPost(tags=['music']).save() + post3 = BlogPost(tags=['film', 'actors']).save() + post4 = BlogPost(tags=['film', 'actors', 'music'], deleted=True).save() - self.assertEqual([p.id for p in BlogPost.objects], + self.assertEqual([p.id for p in BlogPost.objects()], [post1.id, post2.id, post3.id]) - self.assertEqual([p.id for p in BlogPost.music_posts], - [post2.id, post1.id]) + self.assertEqual([p.id for p in BlogPost.music_posts()], + [post1.id, post2.id]) + + self.assertEqual([p.id for p in BlogPost.music_posts(True)], + [post4.id]) BlogPost.drop_collection() From 7ca81d6fb8d71d5f01bf32e9428f7288ed402736 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 30 Jul 2012 13:00:42 +0100 Subject: [PATCH 0694/1279] Fixes --- mongoengine/base.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 43243b8..ca45080 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -791,11 +791,11 @@ class BaseDocument(object): self._data = {} # Assign default values to instance - for attr_name, field in self._fields.items(): - if self._db_field_map.get(attr_name, attr_name) in values: + for key, field in self._fields.items(): + if self._db_field_map.get(key, key) in values: continue - value = getattr(self, attr_name, None) - setattr(self, attr_name, value) + value = getattr(self, key, None) + setattr(self, key, value) # Set passed values after initialisation if self._dynamic: @@ -824,8 +824,6 @@ class BaseDocument(object): signals.post_init.send(self.__class__, document=self) def __setattr__(self, name, value): - if not self._initialised: - return super(BaseDocument, self).__setattr__(name, value) # Handle dynamic data only if an initialised dynamic document if self._dynamic and not self._dynamic_lock: From 69d57209f7ace0013fe55cbe2981c35a72b5ff57 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 30 Jul 2012 13:35:45 +0100 Subject: [PATCH 0695/1279] Minor --- mongoengine/base.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index ca45080..86cd9b4 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -111,6 +111,7 @@ class ValidationError(AssertionError): _document_registry = {} +_module_registry = {} def get_document(name): @@ -503,7 +504,6 @@ class DocumentMetaclass(type): simple_class = True for base in bases: - # Include all fields present in superclasses if hasattr(base, '_fields'): doc_fields.update(base._fields) @@ -549,7 +549,7 @@ class DocumentMetaclass(type): # Add the document's fields to the _fields attribute field_names = {} - for attr_name, attr_value in attrs.items(): + for attr_name, attr_value in attrs.iteritems(): if hasattr(attr_value, "__class__") and \ issubclass(attr_value.__class__, BaseField): attr_value.name = attr_name @@ -565,7 +565,15 @@ class DocumentMetaclass(type): attrs['_db_field_map'] = dict([(k, v.db_field) for k, v in doc_fields.items() if k != v.db_field]) attrs['_reverse_db_field_map'] = dict([(v, k) for k, v in attrs['_db_field_map'].items()]) - from mongoengine import Document, EmbeddedDocument, DictField + if 'Document' not in _module_registry: + from mongoengine import Document, EmbeddedDocument, DictField + _module_registry['Document'] = Document + _module_registry['EmbeddedDocument'] = EmbeddedDocument + _module_registry['DictField'] = DictField + else: + Document = _module_registry.get('Document') + EmbeddedDocument = _module_registry.get('EmbeddedDocument') + DictField = _module_registry.get('DictField') new_class = super_new(cls, name, bases, attrs) for field in new_class._fields.values(): From c419f3379ad2ce8ef266ac34786844b76a2cf3bb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 30 Jul 2012 15:43:53 +0100 Subject: [PATCH 0696/1279] Style changes --- mongoengine/base.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 86cd9b4..9d342e8 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -15,7 +15,6 @@ import operator from functools import partial from bson.dbref import DBRef - class NotRegistered(Exception): pass @@ -474,6 +473,7 @@ class DocumentMetaclass(type): """ def __new__(cls, name, bases, attrs): + def _get_mixin_fields(base): attrs = {} attrs.update(dict([(k, v) for k, v in base.__dict__.items() @@ -502,7 +502,6 @@ class DocumentMetaclass(type): class_name = [name] superclasses = {} simple_class = True - for base in bases: # Include all fields present in superclasses if hasattr(base, '_fields'): @@ -543,20 +542,18 @@ class DocumentMetaclass(type): if not simple_class and not meta['allow_inheritance'] and not meta['abstract']: raise ValueError('Only direct subclasses of Document may set ' '"allow_inheritance" to False') - attrs['_meta'] = meta - attrs['_class_name'] = doc_class_name - attrs['_superclasses'] = superclasses # Add the document's fields to the _fields attribute field_names = {} for attr_name, attr_value in attrs.iteritems(): - if hasattr(attr_value, "__class__") and \ - issubclass(attr_value.__class__, BaseField): - attr_value.name = attr_name - if not attr_value.db_field: - attr_value.db_field = attr_name - doc_fields[attr_name] = attr_value - field_names[attr_value.db_field] = field_names.get(attr_value.db_field, 0) + 1 + if not isinstance(attr_value, BaseField): + continue + attr_value.name = attr_name + if not attr_value.db_field: + attr_value.db_field = attr_name + doc_fields[attr_name] = attr_value + + field_names[attr_value.db_field] = field_names.get(attr_value.db_field, 0) + 1 duplicate_db_fields = [k for k, v in field_names.items() if v > 1] if duplicate_db_fields: @@ -564,6 +561,9 @@ class DocumentMetaclass(type): attrs['_fields'] = doc_fields attrs['_db_field_map'] = dict([(k, v.db_field) for k, v in doc_fields.items() if k != v.db_field]) attrs['_reverse_db_field_map'] = dict([(v, k) for k, v in attrs['_db_field_map'].items()]) + attrs['_meta'] = meta + attrs['_class_name'] = doc_class_name + attrs['_superclasses'] = superclasses if 'Document' not in _module_registry: from mongoengine import Document, EmbeddedDocument, DictField @@ -576,7 +576,8 @@ class DocumentMetaclass(type): DictField = _module_registry.get('DictField') new_class = super_new(cls, name, bases, attrs) - for field in new_class._fields.values(): + + for field in new_class._fields.itervalues(): field.owner_document = new_class delete_rule = getattr(field, 'reverse_delete_rule', DO_NOTHING) @@ -728,7 +729,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): unique_indexes = cls._unique_with_indexes(new_class) new_class._meta['unique_indexes'] = unique_indexes - for field_name, field in new_class._fields.items(): + for field_name, field in new_class._fields.iteritems(): # Check for custom primary key if field.primary_key: current_pk = new_class._meta['id_field'] @@ -799,7 +800,7 @@ class BaseDocument(object): self._data = {} # Assign default values to instance - for key, field in self._fields.items(): + for key, field in self._fields.iteritems(): if self._db_field_map.get(key, key) in values: continue value = getattr(self, key, None) @@ -809,13 +810,13 @@ class BaseDocument(object): if self._dynamic: self._dynamic_fields = {} dynamic_data = {} - for key, value in values.items(): + for key, value in values.iteritems(): if key in self._fields or key == '_id': setattr(self, key, value) elif self._dynamic: dynamic_data[key] = value else: - for key, value in values.items(): + for key, value in values.iteritems(): key = self._reverse_db_field_map.get(key, key) setattr(self, key, value) @@ -824,7 +825,7 @@ class BaseDocument(object): if self._dynamic: self._dynamic_lock = False - for key, value in dynamic_data.items(): + for key, value in dynamic_data.iteritems(): setattr(self, key, value) # Flag initialised From d1add62a0637b839de8038eba0ea57c77c3e80f1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 1 Aug 2012 13:11:36 +0100 Subject: [PATCH 0697/1279] More updates --- mongoengine/base.py | 22 ++++++++++++++-------- mongoengine/queryset.py | 15 ++++++++++----- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 9d342e8..f8752f3 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -264,7 +264,7 @@ class ComplexBaseField(BaseField): """ field = None - _dereference = False + __dereference = False def __get__(self, instance, owner): """Descriptor to automatically dereference references. @@ -276,8 +276,6 @@ class ComplexBaseField(BaseField): dereference = self.field is None or isinstance(self.field, (GenericReferenceField, ReferenceField)) if not self._dereference and instance._initialised and dereference: - from dereference import DeReference - self._dereference = DeReference() # Cached instance._data[self.name] = self._dereference( instance._data.get(self.name), max_depth=1, instance=instance, name=self.name @@ -293,14 +291,13 @@ class ComplexBaseField(BaseField): value = BaseDict(value, instance, self.name) instance._data[self.name] = value - if self._dereference and instance._initialised and \ - isinstance(value, (BaseList, BaseDict)) and not value._dereferenced: + if (instance._initialised and isinstance(value, (BaseList, BaseDict)) + and not value._dereferenced): value = self._dereference( value, max_depth=1, instance=instance, name=self.name ) value._dereferenced = True instance._data[self.name] = value - return value def __set__(self, instance, value): @@ -441,6 +438,13 @@ class ComplexBaseField(BaseField): owner_document = property(_get_owner_document, _set_owner_document) + @property + def _dereference(self,): + if not self.__dereference: + from dereference import DeReference + self.__dereference = DeReference() # Cached + return self.__dereference + class ObjectIdField(BaseField): """An field wrapper around MongoDB's ObjectIds. @@ -493,8 +497,9 @@ class DocumentMetaclass(type): attrs.update(_get_mixin_fields(p_base)) return attrs - metaclass = attrs.get('__metaclass__') super_new = super(DocumentMetaclass, cls).__new__ + + metaclass = attrs.get('__metaclass__') if metaclass and issubclass(metaclass, DocumentMetaclass): return super_new(cls, name, bases, attrs) @@ -566,7 +571,8 @@ class DocumentMetaclass(type): attrs['_superclasses'] = superclasses if 'Document' not in _module_registry: - from mongoengine import Document, EmbeddedDocument, DictField + from mongoengine.document import Document, EmbeddedDocument + from mongoengine.fields import DictField _module_registry['Document'] = Document _module_registry['EmbeddedDocument'] = EmbeddedDocument _module_registry['DictField'] = DictField diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 0d1d95b..0e93fd2 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -329,6 +329,7 @@ class QuerySet(object): """ __already_indexed = set() + __dereference = False def __init__(self, document, collection): self._document = document @@ -600,7 +601,6 @@ class QuerySet(object): if self._hint != -1: self._cursor_obj.hint(self._hint) - return self._cursor_obj @classmethod @@ -1153,8 +1153,7 @@ class QuerySet(object): .. versionadded:: 0.4 .. versionchanged:: 0.5 - Fixed handling references """ - from dereference import DeReference - return DeReference()(self._cursor.distinct(field), 1) + return self._dereference(self._cursor.distinct(field), 1) def only(self, *fields): """Load only a subset of this document's fields. :: @@ -1854,10 +1853,16 @@ class QuerySet(object): .. versionadded:: 0.5 """ - from dereference import DeReference # Make select related work the same for querysets max_depth += 1 - return DeReference()(self, max_depth=max_depth) + return self._dereference(self, max_depth=max_depth) + + @property + def _dereference(self): + if not self.__dereference: + from dereference import DeReference + self.__dereference = DeReference() # Cached + return self.__dereference class QuerySetManager(object): From 8df81571fc41c623b0fffa3da99c37813b8a3edc Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 1 Aug 2012 13:28:28 +0100 Subject: [PATCH 0698/1279] Fixed FileField comparision Refs hmarr/mongoengine#547 --- docs/changelog.rst | 4 ++ mongoengine/fields.py | 2 + tests/test_fields.py | 126 ++++++++++++++++++++++-------------------- 3 files changed, 73 insertions(+), 59 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 97926f8..2b52582 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.6.X +================ + +- Fixed FileField comparision (hmarr/mongoengine#547) Changes in 0.6.18 ================= diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 94f79a1..89b81cd 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -907,6 +907,8 @@ class GridFSProxy(object): return '<%s: %s>' % (self.__class__.__name__, self.grid_id) def __cmp__(self, other): + if not isinstance(other, GridFSProxy): + return -1 return cmp((self.grid_id, self.collection_name, self.db_alias), (other.grid_id, other.collection_name, other.db_alias)) diff --git a/tests/test_fields.py b/tests/test_fields.py index 72972cc..4e70856 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1567,13 +1567,13 @@ class FieldTest(unittest.TestCase): """Ensure that file fields can be written to and their data retrieved """ class PutFile(Document): - file = FileField() + the_file = FileField() class StreamFile(Document): - file = FileField() + the_file = FileField() class SetFile(Document): - file = FileField() + the_file = FileField() text = 'Hello, World!' more_text = 'Foo Bar' @@ -1584,14 +1584,14 @@ class FieldTest(unittest.TestCase): SetFile.drop_collection() putfile = PutFile() - putfile.file.put(text, content_type=content_type) + putfile.the_file.put(text, content_type=content_type) putfile.save() putfile.validate() result = PutFile.objects.first() self.assertTrue(putfile == result) - self.assertEquals(result.file.read(), text) - self.assertEquals(result.file.content_type, content_type) - result.file.delete() # Remove file from GridFS + self.assertEquals(result.the_file.read(), text) + self.assertEquals(result.the_file.content_type, content_type) + result.the_file.delete() # Remove file from GridFS PutFile.objects.delete() # Ensure file-like objects are stored @@ -1599,53 +1599,53 @@ class FieldTest(unittest.TestCase): putstring = StringIO.StringIO() putstring.write(text) putstring.seek(0) - putfile.file.put(putstring, content_type=content_type) + putfile.the_file.put(putstring, content_type=content_type) putfile.save() putfile.validate() result = PutFile.objects.first() self.assertTrue(putfile == result) - self.assertEquals(result.file.read(), text) - self.assertEquals(result.file.content_type, content_type) - result.file.delete() + self.assertEquals(result.the_file.read(), text) + self.assertEquals(result.the_file.content_type, content_type) + result.the_file.delete() streamfile = StreamFile() - streamfile.file.new_file(content_type=content_type) - streamfile.file.write(text) - streamfile.file.write(more_text) - streamfile.file.close() + streamfile.the_file.new_file(content_type=content_type) + streamfile.the_file.write(text) + streamfile.the_file.write(more_text) + streamfile.the_file.close() streamfile.save() streamfile.validate() result = StreamFile.objects.first() self.assertTrue(streamfile == result) - self.assertEquals(result.file.read(), text + more_text) - self.assertEquals(result.file.content_type, content_type) - result.file.seek(0) - self.assertEquals(result.file.tell(), 0) - self.assertEquals(result.file.read(len(text)), text) - self.assertEquals(result.file.tell(), len(text)) - self.assertEquals(result.file.read(len(more_text)), more_text) - self.assertEquals(result.file.tell(), len(text + more_text)) - result.file.delete() + self.assertEquals(result.the_file.read(), text + more_text) + self.assertEquals(result.the_file.content_type, content_type) + result.the_file.seek(0) + self.assertEquals(result.the_file.tell(), 0) + self.assertEquals(result.the_file.read(len(text)), text) + self.assertEquals(result.the_file.tell(), len(text)) + self.assertEquals(result.the_file.read(len(more_text)), more_text) + self.assertEquals(result.the_file.tell(), len(text + more_text)) + result.the_file.delete() # Ensure deleted file returns None - self.assertTrue(result.file.read() == None) + self.assertTrue(result.the_file.read() == None) setfile = SetFile() - setfile.file = text + setfile.the_file = text setfile.save() setfile.validate() result = SetFile.objects.first() self.assertTrue(setfile == result) - self.assertEquals(result.file.read(), text) + self.assertEquals(result.the_file.read(), text) # Try replacing file with new one - result.file.replace(more_text) + result.the_file.replace(more_text) result.save() result.validate() result = SetFile.objects.first() self.assertTrue(setfile == result) - self.assertEquals(result.file.read(), more_text) - result.file.delete() + self.assertEquals(result.the_file.read(), more_text) + result.the_file.delete() PutFile.drop_collection() StreamFile.drop_collection() @@ -1653,7 +1653,7 @@ class FieldTest(unittest.TestCase): # Make sure FileField is optional and not required class DemoFile(Document): - file = FileField() + the_file = FileField() DemoFile.objects.create() @@ -1704,20 +1704,20 @@ class FieldTest(unittest.TestCase): """ class TestFile(Document): name = StringField() - file = FileField() + the_file = FileField() # First instance - testfile = TestFile() - testfile.name = "Hello, World!" - testfile.file.put('Hello, World!') - testfile.save() + test_file = TestFile() + test_file.name = "Hello, World!" + test_file.the_file.put('Hello, World!') + test_file.save() # Second instance - testfiledupe = TestFile() - data = testfiledupe.file.read() # Should be None + test_file_dupe = TestFile() + data = test_file_dupe.the_file.read() # Should be None - self.assertTrue(testfile.name != testfiledupe.name) - self.assertTrue(testfile.file.read() != data) + self.assertTrue(test_file.name != test_file_dupe.name) + self.assertTrue(test_file.the_file.read() != data) TestFile.drop_collection() @@ -1725,17 +1725,25 @@ class FieldTest(unittest.TestCase): """Ensure that a boolean test of a FileField indicates its presence """ class TestFile(Document): - file = FileField() + the_file = FileField() - testfile = TestFile() - self.assertFalse(bool(testfile.file)) - testfile.file = 'Hello, World!' - testfile.file.content_type = 'text/plain' - testfile.save() - self.assertTrue(bool(testfile.file)) + test_file = TestFile() + self.assertFalse(bool(test_file.the_file)) + test_file.the_file = 'Hello, World!' + test_file.the_file.content_type = 'text/plain' + test_file.save() + self.assertTrue(bool(test_file.the_file)) TestFile.drop_collection() + def test_file_cmp(self): + """Test comparing against other types""" + class TestFile(Document): + the_file = FileField() + + test_file = TestFile() + self.assertFalse(test_file.the_file in [{"test": 1}]) + def test_image_field(self): class TestImage(Document): @@ -1799,30 +1807,30 @@ class FieldTest(unittest.TestCase): def test_file_multidb(self): - register_connection('testfiles', 'testfiles') + register_connection('test_files', 'test_files') class TestFile(Document): name = StringField() - file = FileField(db_alias="testfiles", - collection_name="macumba") + the_file = FileField(db_alias="test_files", + collection_name="macumba") TestFile.drop_collection() # delete old filesystem - get_db("testfiles").macumba.files.drop() - get_db("testfiles").macumba.chunks.drop() + get_db("test_files").macumba.files.drop() + get_db("test_files").macumba.chunks.drop() # First instance - testfile = TestFile() - testfile.name = "Hello, World!" - testfile.file.put('Hello, World!', + test_file = TestFile() + test_file.name = "Hello, World!" + test_file.the_file.put('Hello, World!', name="hello.txt") - testfile.save() + test_file.save() - data = get_db("testfiles").macumba.files.find_one() + data = get_db("test_files").macumba.files.find_one() self.assertEquals(data.get('name'), 'hello.txt') - testfile = TestFile.objects.first() - self.assertEquals(testfile.file.read(), + test_file = TestFile.objects.first() + self.assertEquals(test_file.the_file.read(), 'Hello, World!') def test_geo_indexes(self): From 0018c38b836af11260c7eee68957459b76cd0a95 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 1 Aug 2012 13:51:51 +0100 Subject: [PATCH 0699/1279] Fixed queryset manager issue (MongoEngine/mongoengine#52) --- docs/changelog.rst | 1 + mongoengine/queryset.py | 6 +++--- tests/test_queryset.py | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2b52582..f0adcbe 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.6.X ================ +- Fixed queryset manager issue (MongoEngine/mongoengine#52) - Fixed FileField comparision (hmarr/mongoengine#547) Changes in 0.6.18 diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 0d1d95b..15c768d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1881,10 +1881,10 @@ class QuerySetManager(object): queryset_class = owner._meta['queryset_class'] or QuerySet queryset = queryset_class(owner, owner._get_collection()) if self.get_queryset: - var_names = self.get_queryset.func_code.co_varnames - if len(var_names) == 1: + arg_count = self.get_queryset.func_code.co_argcount + if arg_count == 1: queryset = self.get_queryset(queryset) - elif len(var_names) == 2: + elif arg_count == 2: queryset = self.get_queryset(owner, queryset) else: queryset = partial(self.get_queryset, owner, queryset) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index e623790..b4ae805 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -2287,7 +2287,8 @@ class QuerySetTest(unittest.TestCase): @queryset_manager def objects(cls, qryset): - return qryset(deleted=False) + opts = {"deleted": False} + return qryset(**opts) @queryset_manager def music_posts(doc_cls, queryset, deleted=False): From 2c69d8f0b02ae2b6067611b2fbcb585f4ef6d9ee Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 1 Aug 2012 13:54:24 +0100 Subject: [PATCH 0700/1279] Updated License --- LICENSE | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/LICENSE b/LICENSE index e33b2c5..cef91cc 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,5 @@ -Copyright (c) 2009-2010 Harry Marr - +Copyright (c) 2009-2012 See AUTHORS + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without @@ -8,10 +8,10 @@ copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - + The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND From a43d0d46122f42db2ae154b213a27a3cc0bc187f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 1 Aug 2012 14:57:46 +0100 Subject: [PATCH 0701/1279] Fixed BinaryField python value issue (MongoEngine/mongoengine#48) --- docs/changelog.rst | 1 + mongoengine/fields.py | 7 ++----- tests/test_fields.py | 17 +++++++++++++++-- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f0adcbe..58859a5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.6.X ================ +- Fixed BinaryField python value issue (MongoEngine/mongoengine#48) - Fixed queryset manager issue (MongoEngine/mongoengine#52) - Fixed FileField comparision (hmarr/mongoengine#547) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 89b81cd..27434e6 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -845,12 +845,9 @@ class BinaryField(BaseField): def to_mongo(self, value): return Binary(value) - def to_python(self, value): - return "%s" % value - def validate(self, value): - if not isinstance(value, basestring): - self.error('BinaryField only accepts string values') + if not isinstance(value, (basestring, Binary)): + self.error('BinaryField only accepts string or bson Binary values') if self.max_bytes is not None and len(value) > self.max_bytes: self.error('Binary value is too long') diff --git a/tests/test_fields.py b/tests/test_fields.py index 4e70856..29f5fe9 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -6,6 +6,7 @@ import StringIO import tempfile import gridfs +from bson import Binary from decimal import Decimal from mongoengine import * @@ -1428,7 +1429,7 @@ class FieldTest(unittest.TestCase): attachment_1 = Attachment.objects().first() self.assertEqual(MIME_TYPE, attachment_1.content_type) - self.assertEqual(BLOB, attachment_1.blob) + self.assertEqual(BLOB, str(attachment_1.blob)) Attachment.drop_collection() @@ -1455,7 +1456,7 @@ class FieldTest(unittest.TestCase): attachment_required = AttachmentRequired() self.assertRaises(ValidationError, attachment_required.validate) - attachment_required.blob = '\xe6\x00\xc4\xff\x07' + attachment_required.blob = Binary('\xe6\x00\xc4\xff\x07') attachment_required.validate() attachment_size_limit = AttachmentSizeLimit(blob='\xe6\x00\xc4\xff\x07') @@ -1467,6 +1468,18 @@ class FieldTest(unittest.TestCase): AttachmentRequired.drop_collection() AttachmentSizeLimit.drop_collection() + def test_binary_field_primary(self): + + class Attachment(Document): + id = BinaryField(primary_key=True) + + Attachment.drop_collection() + + att = Attachment(id=uuid.uuid4().bytes).save() + att.delete() + + self.assertEqual(0, Attachment.objects.count()) + def test_choices_validation(self): """Ensure that value is in a container of allowed values. """ From 4226cd08f17061396d5734fb438c7e8856dd491d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 1 Aug 2012 15:00:14 +0100 Subject: [PATCH 0702/1279] Updated Changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 58859a5..f5f1dfd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,6 +6,7 @@ Changes in 0.6.X ================ - Fixed BinaryField python value issue (MongoEngine/mongoengine#48) +- Fixed SequenceField non numeric value lookup (MongoEngine/mongoengine#41) - Fixed queryset manager issue (MongoEngine/mongoengine#52) - Fixed FileField comparision (hmarr/mongoengine#547) From f23a976bea0a97ecd60f8992ac8bc445c2511194 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 1 Aug 2012 15:01:21 +0100 Subject: [PATCH 0703/1279] Added Tommi Komulainen to the contributors list refs MongoEngine/mongoengine#48 --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index f774d22..fe8be76 100644 --- a/AUTHORS +++ b/AUTHORS @@ -113,4 +113,5 @@ that much better: * Alexander Koshelev * Jaime Irurzun * Alexandre González - * Thomas Steinacher \ No newline at end of file + * Thomas Steinacher + * Tommi Komulainen \ No newline at end of file From 2420b5e937d29b93ba2a5bc9d464c8367bcdde0c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 1 Aug 2012 15:14:56 +0100 Subject: [PATCH 0704/1279] Fixed MapField lookup for fields without declared lookups (MongoEngine/mongoengine#46) --- docs/changelog.rst | 1 + mongoengine/queryset.py | 2 +- tests/test_fields.py | 13 +++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f5f1dfd..1f52dfd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.6.X ================ +- Fixed MapField lookup for fields without declared lookups (MongoEngine/mongoengine#46) - Fixed BinaryField python value issue (MongoEngine/mongoengine#48) - Fixed SequenceField non numeric value lookup (MongoEngine/mongoengine#41) - Fixed queryset manager issue (MongoEngine/mongoengine#52) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 15c768d..b7453c6 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -641,7 +641,7 @@ class QuerySet(object): from mongoengine.fields import ReferenceField, GenericReferenceField if isinstance(field, (ReferenceField, GenericReferenceField)): raise InvalidQueryError('Cannot perform join in mongoDB: %s' % '__'.join(parts)) - if getattr(field, 'field', None): + if hasattr(getattr(field, 'field', None), 'lookup_member'): new_field = field.field.lookup_member(field_name) else: # Look up subfield on the previous field diff --git a/tests/test_fields.py b/tests/test_fields.py index 29f5fe9..d343003 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -929,6 +929,19 @@ class FieldTest(unittest.TestCase): doc = self.db.test.find_one() self.assertEqual(doc['x']['DICTIONARY_KEY']['i'], 2) + def test_map_field_lookup(self): + """Ensure MapField lookups succeed on Fields without a lookup method""" + + class Log(Document): + name = StringField() + visited = MapField(DateTimeField()) + + Log.drop_collection() + Log(name="wilson", visited={'friends': datetime.now()}).save() + + self.assertEqual(1, Log.objects( + visited__friends__exists=True).count()) + def test_embedded_db_field(self): class Embedded(EmbeddedDocument): From 4b3cea9e78a8bdb68944683f6e94ff3c409fb1ee Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 1 Aug 2012 16:03:33 +0100 Subject: [PATCH 0705/1279] Added Binary support to UUID (MongoEngine/mongoengine#47) --- docs/changelog.rst | 1 + mongoengine/fields.py | 35 ++++++++++++++++++++----- tests/test_fields.py | 59 ++++++++++++++++++++++++++++++++----------- 3 files changed, 74 insertions(+), 21 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1f52dfd..08091ac 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.6.X ================ +- Added Binary support to UUID (MongoEngine/mongoengine#47) - Fixed MapField lookup for fields without declared lookups (MongoEngine/mongoengine#46) - Fixed BinaryField python value issue (MongoEngine/mongoengine#48) - Fixed SequenceField non numeric value lookup (MongoEngine/mongoengine#41) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 08bbc44..8268992 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -4,9 +4,9 @@ import decimal import gridfs import re import uuid +import warnings from bson import Binary, DBRef, SON, ObjectId - from base import (BaseField, ComplexBaseField, ObjectIdField, ValidationError, get_document, BaseDocument) from queryset import DO_NOTHING, QuerySet @@ -1308,17 +1308,40 @@ class UUIDField(BaseField): .. versionadded:: 0.6 """ + _binary = None - def __init__(self, **kwargs): + def __init__(self, binary=None, **kwargs): + """ + Store UUID data in the database + + :param binary: (optional) boolean store as binary. + + .. versionchanged:: 0.6.19 + """ + if binary is None: + binary = False + msg = ("UUIDFields will soon default to store as binary, please " + "configure binary=False if you wish to store as a string") + warnings.warn(msg, FutureWarning) + self._binary = binary super(UUIDField, self).__init__(**kwargs) def to_python(self, value): - if not isinstance(value, basestring): - value = unicode(value) - return uuid.UUID(value) + if not self.binary: + if not isinstance(value, basestring): + value = unicode(value) + return uuid.UUID(value) + return value def to_mongo(self, value): - return unicode(value) + if not self._binary: + return unicode(value) + return value + + def prepare_query_value(self, op, value): + if value is None: + return None + return self.to_mongo(value) def validate(self, value): if not isinstance(value, uuid.UUID): diff --git a/tests/test_fields.py b/tests/test_fields.py index d343003..c4013c1 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -272,25 +272,54 @@ class FieldTest(unittest.TestCase): person.admin = 'Yes' self.assertRaises(ValidationError, person.validate) - def test_uuid_validation(self): - """Ensure that invalid values cannot be assigned to UUID fields. + def test_uuid_field_string(self): + """Test UUID fields storing as String """ class Person(Document): - api_key = UUIDField() + api_key = UUIDField(binary=False) + + Person.drop_collection() + + uu = uuid.uuid4() + Person(api_key=uu).save() + self.assertEqual(1, Person.objects(api_key=uu).count()) person = Person() - # any uuid type is valid - person.api_key = uuid.uuid4() - person.validate() - person.api_key = uuid.uuid1() - person.validate() + valid = (uuid.uuid4(), uuid.uuid1()) + for api_key in valid: + person.api_key = api_key + person.validate() + + invalid = ('9d159858-549b-4975-9f98-dd2f987c113g', + '9d159858-549b-4975-9f98-dd2f987c113') + for api_key in invalid: + person.api_key = api_key + self.assertRaises(ValidationError, person.validate) + + def test_uuid_field_binary(self): + """Test UUID fields storing as Binary object + """ + class Person(Document): + api_key = UUIDField(binary=True) + + Person.drop_collection() + + uu = uuid.uuid4() + Person(api_key=uu).save() + self.assertEqual(1, Person.objects(api_key=uu).count()) + + person = Person() + valid = (uuid.uuid4(), uuid.uuid1()) + for api_key in valid: + person.api_key = api_key + person.validate() + + invalid = ('9d159858-549b-4975-9f98-dd2f987c113g', + '9d159858-549b-4975-9f98-dd2f987c113') + for api_key in invalid: + person.api_key = api_key + self.assertRaises(ValidationError, person.validate) - # last g cannot belong to an hex number - person.api_key = '9d159858-549b-4975-9f98-dd2f987c113g' - self.assertRaises(ValidationError, person.validate) - # short strings don't validate - person.api_key = '9d159858-549b-4975-9f98-dd2f987c113' - self.assertRaises(ValidationError, person.validate) def test_datetime_validation(self): """Ensure that invalid values cannot be assigned to datetime fields. @@ -937,7 +966,7 @@ class FieldTest(unittest.TestCase): visited = MapField(DateTimeField()) Log.drop_collection() - Log(name="wilson", visited={'friends': datetime.now()}).save() + Log(name="wilson", visited={'friends': datetime.datetime.now()}).save() self.assertEqual(1, Log.objects( visited__friends__exists=True).count()) From 91aa90ad4aa96c912b1a5f6fe5d77a5930978db6 Mon Sep 17 00:00:00 2001 From: Laine Date: Wed, 1 Aug 2012 17:21:48 -0700 Subject: [PATCH 0706/1279] Added Python 3 support to MongoEngine --- mongoengine/base.py | 47 +++- mongoengine/django/shortcuts.py | 3 +- mongoengine/django/tests.py | 22 +- mongoengine/document.py | 32 ++- mongoengine/fields.py | 41 +-- mongoengine/python3_support.py | 29 +++ mongoengine/queryset.py | 8 +- setup.cfg | 4 +- setup.py | 30 ++- tests/test_dereference.py | 28 +- tests/test_django.py | 32 ++- tests/test_document.py | 387 ++++++++++++++-------------- tests/test_dynamic_document.py | 178 ++++++------- tests/test_fields.py | 210 +++++++-------- tests/test_queryset.py | 150 ++++++----- tests/test_replicaset_connection.py | 3 +- tests/test_signals.py | 4 +- 17 files changed, 673 insertions(+), 535 deletions(-) create mode 100644 mongoengine/python3_support.py diff --git a/mongoengine/base.py b/mongoengine/base.py index 6fb26cb..d4a7b32 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1,18 +1,19 @@ +import operator +import sys import warnings + from collections import defaultdict +from functools import partial from queryset import QuerySet, QuerySetManager from queryset import DoesNotExist, MultipleObjectsReturned from queryset import DO_NOTHING from mongoengine import signals +from mongoengine.python3_support import PY3, txt_type -import sys import pymongo from bson import ObjectId -import operator - -from functools import partial from bson.dbref import DBRef @@ -402,7 +403,7 @@ class ComplexBaseField(BaseField): """ errors = {} if self.field: - if hasattr(value, 'iteritems'): + if hasattr(value, 'iteritems') or hasattr(value, 'items'): sequence = value.iteritems() else: sequence = enumerate(value) @@ -491,7 +492,7 @@ class DocumentMetaclass(type): attrs.update(_get_mixin_fields(p_base)) return attrs - metaclass = attrs.get('__metaclass__') + metaclass = attrs.get('my_metaclass') super_new = super(DocumentMetaclass, cls).__new__ if metaclass and issubclass(metaclass, DocumentMetaclass): return super_new(cls, name, bases, attrs) @@ -583,7 +584,9 @@ class DocumentMetaclass(type): raise InvalidDocumentError("Reverse delete rules are not supported for EmbeddedDocuments (field: %s)" % field.name) f.document_type.register_delete_rule(new_class, field.name, delete_rule) - if field.name and hasattr(Document, field.name) and EmbeddedDocument not in new_class.mro(): + if (field.name and + hasattr(Document, field.name) and + EmbeddedDocument not in new_class.mro()): raise InvalidDocumentError("%s is a document method and not a valid field name" % field.name) module = attrs.get('__module__') @@ -602,6 +605,22 @@ class DocumentMetaclass(type): global _document_registry _document_registry[doc_class_name] = new_class + # in Python 2, User-defined methods objects have special read-only + # attributes 'im_func' and 'im_self' which contain the function obj + # and class instance object respectively. With Python 3 these special + # attributes have been replaced by __func__ and __self__. The Blinker + # module continues to use im_func and im_self, so the code below + # copies __func__ into im_func and __self__ into im_self for + # classmethod objects in Document derived classes. + if PY3: + for key, val in new_class.__dict__.items(): + if isinstance(val, classmethod): + f = val.__get__(new_class) + if hasattr(f, '__func__') and not hasattr(f, 'im_func'): + f.__dict__.update({'im_func':getattr(f, '__func__')}) + if hasattr(f, '__self__') and not hasattr(f, 'im_self'): + f.__dict__.update({'im_self':getattr(f, '__self__')}) + return new_class def add_to_class(self, name, value): @@ -623,7 +642,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # # Also assume a class is abstract if it has abstract set to True in # its meta dictionary. This allows custom Document superclasses. - if (attrs.get('__metaclass__') == TopLevelDocumentMetaclass or + if (attrs.get('my_metaclass') == TopLevelDocumentMetaclass or ('meta' in attrs and attrs['meta'].get('abstract', False))): # Make sure no base class was non-abstract non_abstract_bases = [b for b in bases @@ -1189,14 +1208,17 @@ Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, error def __repr__(self): try: - u = unicode(self).encode('utf-8') + u = txt_type(self) except (UnicodeEncodeError, UnicodeDecodeError): u = '[Bad Unicode data]' return '<%s: %s>' % (self.__class__.__name__, u) def __str__(self): if hasattr(self, '__unicode__'): - return unicode(self).encode('utf-8') + if PY3: + return self.__unicode__() + else: + return unicode(self).encode('utf-8') return '%s object' % self.__class__.__name__ def __eq__(self, other): @@ -1338,10 +1360,9 @@ class BaseDict(dict): if sys.version_info < (2, 5): # Prior to Python 2.5, Exception was an old-style class - import types def subclass_exception(name, parents, unused): - import types - return types.ClassType(name, parents, {}) + from types import ClassType + return ClassType(name, parents, {}) else: def subclass_exception(name, parents, module): return type(name, parents, {'__module__': module}) diff --git a/mongoengine/django/shortcuts.py b/mongoengine/django/shortcuts.py index 59a2074..637cee1 100644 --- a/mongoengine/django/shortcuts.py +++ b/mongoengine/django/shortcuts.py @@ -1,4 +1,3 @@ -from django.http import Http404 from mongoengine.queryset import QuerySet from mongoengine.base import BaseDocument from mongoengine.base import ValidationError @@ -27,6 +26,7 @@ def get_document_or_404(cls, *args, **kwargs): try: return queryset.get(*args, **kwargs) except (queryset._document.DoesNotExist, ValidationError): + from django.http import Http404 raise Http404('No %s matches the given query.' % queryset._document._class_name) def get_list_or_404(cls, *args, **kwargs): @@ -42,5 +42,6 @@ def get_list_or_404(cls, *args, **kwargs): queryset = _get_queryset(cls) obj_list = list(queryset.filter(*args, **kwargs)) if not obj_list: + from django.http import Http404 raise Http404('No %s matches the given query.' % queryset._document._class_name) return obj_list diff --git a/mongoengine/django/tests.py b/mongoengine/django/tests.py index e91abeb..566a398 100644 --- a/mongoengine/django/tests.py +++ b/mongoengine/django/tests.py @@ -1,10 +1,28 @@ #coding: utf-8 -from django.test import TestCase -from django.conf import settings +from nose.plugins.skip import SkipTest +from mongoengine.python3_support import PY3 from mongoengine import connect +try: + from django.test import TestCase + from django.conf import settings +except Exception as err: + if PY3: + from unittest import TestCase + # Dummy value so no error + class settings: + MONGO_DATABASE_NAME = 'dummy' + else: + raise err + + class MongoTestCase(TestCase): + + def setUp(self): + if PY3: + raise SkipTest('django does not have Python 3 support') + """ TestCase class that clear the collection between the tests """ diff --git a/mongoengine/document.py b/mongoengine/document.py index f8bf769..35e58d7 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -2,7 +2,7 @@ import pymongo from bson.dbref import DBRef -from mongoengine import signals +from mongoengine import signals, queryset from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, BaseDict, BaseList) from queryset import OperationError @@ -23,6 +23,9 @@ class EmbeddedDocument(BaseDocument): :class:`~mongoengine.EmbeddedDocumentField` field type. """ + # The __metaclass__ attribute is removed by 2to3 when running with Python3 + # my_metaclass is defined so that metaclass can be queried in Python 2 & 3 + my_metaclass = DocumentMetaclass __metaclass__ = DocumentMetaclass def __init__(self, *args, **kwargs): @@ -91,9 +94,12 @@ class Document(BaseDocument): disabled by either setting types to False on the specific index or by setting index_types to False on the meta dictionary for the document. """ + + # The __metaclass__ attribute is removed by 2to3 when running with Python3 + # my_metaclass is defined so that metaclass can be queried in Python 2 & 3 + my_metaclass = TopLevelDocumentMetaclass __metaclass__ = TopLevelDocumentMetaclass - @apply def pk(): """Primary key alias """ @@ -102,6 +108,7 @@ class Document(BaseDocument): def fset(self, value): return setattr(self, self._meta['id_field'], value) return property(fget, fset) + pk = pk() @classmethod def _get_db(cls): @@ -244,12 +251,11 @@ class Document(BaseDocument): def cascade_save(self, *args, **kwargs): """Recursively saves any references / generic references on an object""" - from fields import ReferenceField, GenericReferenceField + import fields _refs = kwargs.get('_refs', []) or [] for name, cls in self._fields.items(): - - if not isinstance(cls, (ReferenceField, GenericReferenceField)): + if not isinstance(cls, (fields.ReferenceField, fields.GenericReferenceField)): continue ref = getattr(self, name) @@ -304,8 +310,8 @@ class Document(BaseDocument): .. versionadded:: 0.5 """ - from dereference import DeReference - self._data = DeReference()(self._data, max_depth) + import dereference + self._data = dereference.DeReference()(self._data, max_depth) return self def reload(self, max_depth=1): @@ -360,10 +366,9 @@ class Document(BaseDocument): """Drops the entire collection associated with this :class:`~mongoengine.Document` type from the database. """ - from mongoengine.queryset import QuerySet db = cls._get_db() db.drop_collection(cls._get_collection_name()) - QuerySet._reset_already_indexed(cls) + queryset.QuerySet._reset_already_indexed(cls) class DynamicDocument(Document): @@ -379,7 +384,12 @@ class DynamicDocument(Document): There is one caveat on Dynamic Documents: fields cannot start with `_` """ + + # The __metaclass__ attribute is removed by 2to3 when running with Python3 + # my_metaclass is defined so that metaclass can be queried in Python 2 & 3 + my_metaclass = TopLevelDocumentMetaclass __metaclass__ = TopLevelDocumentMetaclass + _dynamic = True def __delattr__(self, *args, **kwargs): @@ -398,7 +408,11 @@ class DynamicEmbeddedDocument(EmbeddedDocument): information about dynamic documents. """ + # The __metaclass__ attribute is removed by 2to3 when running with Python3 + # my_metaclass is defined so that metaclass can be queried in Python 2 & 3 + my_metaclass = DocumentMetaclass __metaclass__ = DocumentMetaclass + _dynamic = True def __delattr__(self, *args, **kwargs): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 8268992..a2947c1 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1,18 +1,23 @@ import datetime -import time import decimal -import gridfs import re +import sys +import time import uuid import warnings +from operator import itemgetter + +import gridfs from bson import Binary, DBRef, SON, ObjectId + +from mongoengine.python3_support import (PY3, b, bin_type, + txt_type, str_types, StringIO) from base import (BaseField, ComplexBaseField, ObjectIdField, ValidationError, get_document, BaseDocument) from queryset import DO_NOTHING, QuerySet from document import Document, EmbeddedDocument from connection import get_db, DEFAULT_CONNECTION_NAME -from operator import itemgetter try: @@ -21,12 +26,6 @@ except ImportError: Image = None ImageOps = None -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - - __all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', 'MapField', @@ -846,8 +845,9 @@ class BinaryField(BaseField): return Binary(value) def validate(self, value): - if not isinstance(value, (basestring, Binary)): - self.error('BinaryField only accepts string or bson Binary values') + if not isinstance(value, (bin_type, txt_type, Binary)): + self.error("BinaryField only accepts instances of " + "(%s, %s, Binary)" % (bin_type.__name__,txt_type.__name__)) if self.max_bytes is not None and len(value) > self.max_bytes: self.error('Binary value is too long') @@ -903,12 +903,14 @@ class GridFSProxy(object): def __repr__(self): return '<%s: %s>' % (self.__class__.__name__, self.grid_id) - def __cmp__(self, other): - if not isinstance(other, GridFSProxy): - return -1 - return cmp((self.grid_id, self.collection_name, self.db_alias), - (other.grid_id, other.collection_name, other.db_alias)) - + def __eq__(self, other): + if isinstance(other, GridFSProxy): + return ((self.grid_id == other.grid_id) and + (self.collection_name == other.collection_name) and + (self.db_alias == other.db_alias)) + else: + return False + @property def fs(self): if not self._fs: @@ -1020,7 +1022,8 @@ class FileField(BaseField): def __set__(self, instance, value): key = self.name - if (hasattr(value, 'read') and not isinstance(value, GridFSProxy)) or isinstance(value, basestring): + if ((hasattr(value, 'read') and not + isinstance(value, GridFSProxy)) or isinstance(value, str_types)): # using "FileField() = file/string" notation grid_file = instance._data.get(self.name) # If a file already exists, delete it @@ -1211,7 +1214,7 @@ class ImageField(FileField): for att_name, att in extra_args.items(): if att and (isinstance(att, tuple) or isinstance(att, list)): setattr(self, att_name, dict( - map(None, params_size, att))) + zip(params_size, att))) else: setattr(self, att_name, None) diff --git a/mongoengine/python3_support.py b/mongoengine/python3_support.py new file mode 100644 index 0000000..0682d87 --- /dev/null +++ b/mongoengine/python3_support.py @@ -0,0 +1,29 @@ +"""Helper functions and types to aid with Python 3 support.""" + +import sys + +PY3 = sys.version_info[0] == 3 + +if PY3: + import codecs + from io import BytesIO as StringIO + # return s converted to binary. b('test') should be equivalent to b'test' + def b(s): + return codecs.latin_1_encode(s)[0] + + bin_type = bytes + txt_type = str +else: + try: + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO + + # Conversion to binary only necessary in Python 3 + def b(s): + return s + + bin_type = str + txt_type = unicode + +str_types = (bin_type, txt_type) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b7453c6..1067e32 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -3,7 +3,7 @@ import re import copy import itertools import operator - +import functools from functools import partial import pymongo @@ -121,12 +121,12 @@ class QueryTreeTransformerVisitor(QNodeVisitor): # that ANDs the necessary part with the $or part. clauses = [] for or_group in itertools.product(*or_groups): - q_object = reduce(lambda a, b: a & b, and_parts, Q()) - q_object = reduce(lambda a, b: a & b, or_group, q_object) + q_object = functools.reduce(lambda a, b: a & b, and_parts, Q()) + q_object = functools.reduce(lambda a, b: a & b, or_group, q_object) clauses.append(q_object) # Finally, $or the generated clauses in to one query. Each of the # clauses is sufficient for the query to succeed. - return reduce(lambda a, b: a | b, clauses, Q()) + return functools.reduce(lambda a, b: a | b, clauses, Q()) if combination.operation == combination.OR: children = [] diff --git a/setup.cfg b/setup.cfg index 51a1868..ad9b928 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,3 @@ -[aliases] -test = nosetests - [nosetests] verbosity = 2 detailed-errors = 1 @@ -10,4 +7,5 @@ detailed-errors = 1 #cover-html-dir = ../htmlcov #cover-package = mongoengine where = tests +py3where = build #tests = test_bugfix.py diff --git a/setup.py b/setup.py index 20f3ea3..5f59b1a 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,12 @@ from setuptools import setup, find_packages import os +import sys + +# Hack to silence atexit traceback in newer python versions +try: + import multiprocessing +except ImportError: + pass DESCRIPTION = "A Python Document-Object Mapper for working with MongoDB" @@ -19,9 +26,10 @@ def get_version(version_tuple): # import it as it depends on PyMongo and PyMongo isn't installed until this # file is read init = os.path.join(os.path.dirname(__file__), 'mongoengine', '__init__.py') -version_line = filter(lambda l: l.startswith('VERSION'), open(init))[0] +version_line = list(filter(lambda l: l.startswith('VERSION'), open(init)))[0] + VERSION = get_version(eval(version_line.split('=')[-1])) -print VERSION +print(VERSION) CLASSIFIERS = [ 'Development Status :: 4 - Beta', @@ -29,13 +37,26 @@ CLASSIFIERS = [ 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', 'Programming Language :: Python', + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 3", 'Topic :: Database', 'Topic :: Software Development :: Libraries :: Python Modules', ] +extra_opts = {} +if sys.version_info[0] == 3: + extra_opts['use_2to3'] = True + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker'] + extra_opts['packages'] = find_packages(exclude=('tests',)) + if "test" in sys.argv or "nosetests" in sys.argv: + extra_opts['packages'].append("tests") + extra_opts['package_data'] = {"tests": ["mongoengine.png"]} +else: + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.3', 'PIL'] + extra_opts['packages'] = find_packages(exclude=('tests',)) + setup(name='mongoengine', version=VERSION, - packages=find_packages(exclude=('tests',)), author='Harry Marr', author_email='harry.marr@{nospam}gmail.com', maintainer="Ross Lawley", @@ -48,5 +69,6 @@ setup(name='mongoengine', platforms=['any'], classifiers=CLASSIFIERS, install_requires=['pymongo'], - tests_require=['nose', 'coverage', 'blinker', 'django>=1.3', 'PIL'] + test_suite='nose.collector', + **extra_opts ) diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 826d4aa..75daf6a 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -109,10 +109,10 @@ class FieldTest(unittest.TestCase): peter = Employee.objects.with_id(peter.id).select_related() self.assertEqual(q, 2) - self.assertEquals(peter.boss, bill) + self.assertEqual(peter.boss, bill) self.assertEqual(q, 2) - self.assertEquals(peter.friends, friends) + self.assertEqual(peter.friends, friends) self.assertEqual(q, 2) # Queryset select_related @@ -123,10 +123,10 @@ class FieldTest(unittest.TestCase): self.assertEqual(q, 2) for employee in employees: - self.assertEquals(employee.boss, bill) + self.assertEqual(employee.boss, bill) self.assertEqual(q, 2) - self.assertEquals(employee.friends, friends) + self.assertEqual(employee.friends, friends) self.assertEqual(q, 2) def test_circular_reference(self): @@ -160,7 +160,7 @@ class FieldTest(unittest.TestCase): daughter.relations.append(self_rel) daughter.save() - self.assertEquals("[, ]", "%s" % Person.objects()) + self.assertEqual("[, ]", "%s" % Person.objects()) def test_circular_reference_on_self(self): """Ensure you can handle circular references @@ -186,7 +186,7 @@ class FieldTest(unittest.TestCase): daughter.relations.append(daughter) daughter.save() - self.assertEquals("[, ]", "%s" % Person.objects()) + self.assertEqual("[, ]", "%s" % Person.objects()) def test_circular_tree_reference(self): """Ensure you can handle circular references with more than one level @@ -228,7 +228,7 @@ class FieldTest(unittest.TestCase): anna.other.name = "Anna's friends" anna.save() - self.assertEquals( + self.assertEqual( "[, , , ]", "%s" % Person.objects() ) @@ -781,8 +781,8 @@ class FieldTest(unittest.TestCase): root.save() root = root.reload() - self.assertEquals(root.children, [company]) - self.assertEquals(company.parents, [root]) + self.assertEqual(root.children, [company]) + self.assertEqual(company.parents, [root]) def test_dict_in_dbref_instance(self): @@ -808,8 +808,8 @@ class FieldTest(unittest.TestCase): room_101.save() room = Room.objects.first().select_related() - self.assertEquals(room.staffs_with_position[0]['staff'], sarah) - self.assertEquals(room.staffs_with_position[1]['staff'], bob) + self.assertEqual(room.staffs_with_position[0]['staff'], sarah) + self.assertEqual(room.staffs_with_position[1]['staff'], bob) def test_document_reload_no_inheritance(self): class Foo(Document): @@ -839,8 +839,8 @@ class FieldTest(unittest.TestCase): foo.save() foo.reload() - self.assertEquals(type(foo.bar), Bar) - self.assertEquals(type(foo.baz), Baz) + self.assertEqual(type(foo.bar), Bar) + self.assertEqual(type(foo.baz), Baz) def test_list_lookup_not_checked_in_map(self): """Ensure we dereference list data correctly @@ -862,4 +862,4 @@ class FieldTest(unittest.TestCase): msg = Message.objects.get(id=1) self.assertEqual(0, msg.comments[0].id) - self.assertEqual(1, msg.comments[1].id) \ No newline at end of file + self.assertEqual(1, msg.comments[1].id) diff --git a/tests/test_django.py b/tests/test_django.py index f5e9624..ed21f27 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -1,24 +1,34 @@ # -*- coding: utf-8 -*- - import unittest - +from nose.plugins.skip import SkipTest +from mongoengine.python3_support import PY3 from mongoengine import * -from mongoengine.django.shortcuts import get_document_or_404 -from django.http import Http404 -from django.template import Context, Template -from django.conf import settings -from django.core.paginator import Paginator +try: + from mongoengine.django.shortcuts import get_document_or_404 -settings.configure() + from django.http import Http404 + from django.template import Context, Template + from django.conf import settings + from django.core.paginator import Paginator -from django.contrib.sessions.tests import SessionTestsMixin -from mongoengine.django.sessions import SessionStore, MongoSession + settings.configure() + + from django.contrib.sessions.tests import SessionTestsMixin + from mongoengine.django.sessions import SessionStore, MongoSession +except Exception as err: + if PY3: + SessionTestsMixin = type #dummy value so no error + SessionStore = None #dummy value so no error + else: + raise err class QuerySetTest(unittest.TestCase): def setUp(self): + if PY3: + raise SkipTest('django does not have Python 3 support') connect(db='mongoenginetest') class Person(Document): @@ -99,6 +109,8 @@ class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): backend = SessionStore def setUp(self): + if PY3: + raise SkipTest('django does not have Python 3 support') connect(db='mongoenginetest') MongoSession.drop_collection() super(MongoDBSessionTest, self).setUp() diff --git a/tests/test_document.py b/tests/test_document.py index 6915caf..8eaacee 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -38,7 +38,7 @@ class DocumentTest(unittest.TestCase): """Add FutureWarning for future allow_inhertiance default change. """ - with warnings.catch_warnings(True) as errors: + with warnings.catch_warnings(record=True) as errors: class SimpleBase(Document): a = IntField() @@ -47,10 +47,10 @@ class DocumentTest(unittest.TestCase): b = IntField() InheritedClass() - self.assertEquals(len(errors), 1) + self.assertEqual(len(errors), 1) warning = errors[0] - self.assertEquals(FutureWarning, warning.category) - self.assertTrue("InheritedClass" in warning.message.message) + self.assertEqual(FutureWarning, warning.category) + self.assertTrue("InheritedClass" in str(warning.message)) def test_drop_collection(self): """Ensure that the collection may be dropped from the database. @@ -105,16 +105,16 @@ class DocumentTest(unittest.TestCase): class DefaultNamingTest(Document): pass - self.assertEquals('default_naming_test', DefaultNamingTest._get_collection_name()) + self.assertEqual('default_naming_test', DefaultNamingTest._get_collection_name()) class CustomNamingTest(Document): meta = {'collection': 'pimp_my_collection'} - self.assertEquals('pimp_my_collection', CustomNamingTest._get_collection_name()) + self.assertEqual('pimp_my_collection', CustomNamingTest._get_collection_name()) class DynamicNamingTest(Document): meta = {'collection': lambda c: "DYNAMO"} - self.assertEquals('DYNAMO', DynamicNamingTest._get_collection_name()) + self.assertEqual('DYNAMO', DynamicNamingTest._get_collection_name()) # Use Abstract class to handle backwards compatibility class BaseDocument(Document): @@ -125,11 +125,11 @@ class DocumentTest(unittest.TestCase): class OldNamingConvention(BaseDocument): pass - self.assertEquals('oldnamingconvention', OldNamingConvention._get_collection_name()) + self.assertEqual('oldnamingconvention', OldNamingConvention._get_collection_name()) class InheritedAbstractNamingTest(BaseDocument): meta = {'collection': 'wibble'} - self.assertEquals('wibble', InheritedAbstractNamingTest._get_collection_name()) + self.assertEqual('wibble', InheritedAbstractNamingTest._get_collection_name()) with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. @@ -142,7 +142,7 @@ class DocumentTest(unittest.TestCase): meta = {'collection': 'fail'} self.assertTrue(issubclass(w[0].category, SyntaxWarning)) - self.assertEquals('non_abstract_base', InheritedDocumentFailTest._get_collection_name()) + self.assertEqual('non_abstract_base', InheritedDocumentFailTest._get_collection_name()) # Mixin tests class BaseMixin(object): @@ -152,7 +152,7 @@ class DocumentTest(unittest.TestCase): class OldMixinNamingConvention(Document, BaseMixin): pass - self.assertEquals('oldmixinnamingconvention', OldMixinNamingConvention._get_collection_name()) + self.assertEqual('oldmixinnamingconvention', OldMixinNamingConvention._get_collection_name()) class BaseMixin(object): meta = { @@ -165,7 +165,7 @@ class DocumentTest(unittest.TestCase): class MyDocument(BaseDocument): pass - self.assertEquals('basedocument', MyDocument._get_collection_name()) + self.assertEqual('basedocument', MyDocument._get_collection_name()) def test_get_superclasses(self): """Ensure that the correct list of superclasses is assembled. @@ -212,10 +212,10 @@ class DocumentTest(unittest.TestCase): h = Human() h.save() - self.assertEquals(Human.objects.count(), 1) - self.assertEquals(Mammal.objects.count(), 1) - self.assertEquals(Animal.objects.count(), 1) - self.assertEquals(Base.objects.count(), 1) + self.assertEqual(Human.objects.count(), 1) + self.assertEqual(Mammal.objects.count(), 1) + self.assertEqual(Animal.objects.count(), 1) + self.assertEqual(Base.objects.count(), 1) Base.drop_collection() def test_polymorphic_queries(self): @@ -427,7 +427,7 @@ class DocumentTest(unittest.TestCase): info = collection.index_information() info = [value['key'] for key, value in info.iteritems()] - self.assertEquals([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], info) + self.assertEqual([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], info) # Turn off inheritance class Animal(Document): @@ -445,7 +445,7 @@ class DocumentTest(unittest.TestCase): info = collection.index_information() info = [value['key'] for key, value in info.iteritems()] - self.assertEquals([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], info) + self.assertEqual([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], info) info = collection.index_information() indexes_to_drop = [key for key, value in info.iteritems() if '_types' in dict(value['key'])] @@ -454,14 +454,14 @@ class DocumentTest(unittest.TestCase): info = collection.index_information() info = [value['key'] for key, value in info.iteritems()] - self.assertEquals([[(u'_id', 1)]], info) + self.assertEqual([[(u'_id', 1)]], info) # Recreate indexes dog = Animal.objects.first() dog.save() info = collection.index_information() info = [value['key'] for key, value in info.iteritems()] - self.assertEquals([[(u'_id', 1)], [(u'name', 1),]], info) + self.assertEqual([[(u'_id', 1)], [(u'name', 1),]], info) Animal.drop_collection() @@ -681,8 +681,8 @@ class DocumentTest(unittest.TestCase): Person(name="Fred").save() - self.assertEquals(Person.objects.get(name="Jack").rank, "Corporal") - self.assertEquals(Person.objects.get(name="Fred").rank, "Private") + self.assertEqual(Person.objects.get(name="Jack").rank, "Corporal") + self.assertEqual(Person.objects.get(name="Fred").rank, "Private") def test_db_embedded_doc_field_load(self): """Ensure we load embedded document data correctly @@ -704,8 +704,8 @@ class DocumentTest(unittest.TestCase): Person(name="Fred").save() - self.assertEquals(Person.objects.get(name="Jack").rank, "Corporal") - self.assertEquals(Person.objects.get(name="Fred").rank, "Private") + self.assertEqual(Person.objects.get(name="Jack").rank, "Corporal") + self.assertEqual(Person.objects.get(name="Fred").rank, "Private") def test_explicit_geo2d_index(self): """Ensure that geo2d indexes work when created via meta[indexes] @@ -782,7 +782,7 @@ class DocumentTest(unittest.TestCase): p = Person(name="test", user_guid='123') p.save() - self.assertEquals(1, Person.objects.count()) + self.assertEqual(1, Person.objects.count()) info = Person.objects._collection.index_information() self.assertEqual(info.keys(), ['_types_1_user_guid_1', '_id_', '_types_1_name_1']) Person.drop_collection() @@ -804,7 +804,7 @@ class DocumentTest(unittest.TestCase): u = User(user_guid='123') u.save() - self.assertEquals(1, User.objects.count()) + self.assertEqual(1, User.objects.count()) info = User.objects._collection.index_information() self.assertEqual(info.keys(), ['_id_']) User.drop_collection() @@ -890,8 +890,8 @@ class DocumentTest(unittest.TestCase): self.assertFalse('location_2d' in info) - self.assertEquals(len(Parent._geo_indices()), 0) - self.assertEquals(len(Location._geo_indices()), 1) + self.assertEqual(len(Parent._geo_indices()), 0) + self.assertEqual(len(Location._geo_indices()), 1) def test_covered_index(self): """Ensure that covered indexes can be used @@ -938,7 +938,7 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() indexes = BlogPost.objects._collection.index_information() - self.assertEquals(indexes['categories_1__id_1']['key'], + self.assertEqual(indexes['categories_1__id_1']['key'], [('categories', 1), ('_id', 1)]) def test_hint(self): @@ -957,11 +957,11 @@ class DocumentTest(unittest.TestCase): tags = [("tag %i" % n) for n in xrange(0, i % 2)] BlogPost(tags=tags).save() - self.assertEquals(BlogPost.objects.count(), 10) - self.assertEquals(BlogPost.objects.hint().count(), 10) - self.assertEquals(BlogPost.objects.hint([('tags', 1)]).count(), 10) + self.assertEqual(BlogPost.objects.count(), 10) + self.assertEqual(BlogPost.objects.hint().count(), 10) + self.assertEqual(BlogPost.objects.hint([('tags', 1)]).count(), 10) - self.assertEquals(BlogPost.objects.hint([('ZZ', 1)]).count(), 10) + self.assertEqual(BlogPost.objects.hint([('ZZ', 1)]).count(), 10) def invalid_index(): BlogPost.objects.hint('tags') @@ -1255,32 +1255,32 @@ class DocumentTest(unittest.TestCase): doc.embedded_field.list_field.append(1) doc.embedded_field.dict_field['woot'] = "woot" - self.assertEquals(doc._get_changed_fields(), [ + self.assertEqual(doc._get_changed_fields(), [ 'list_field', 'dict_field', 'embedded_field.list_field', 'embedded_field.dict_field']) doc.save() doc = doc.reload(10) - self.assertEquals(doc._get_changed_fields(), []) - self.assertEquals(len(doc.list_field), 4) - self.assertEquals(len(doc.dict_field), 2) - self.assertEquals(len(doc.embedded_field.list_field), 4) - self.assertEquals(len(doc.embedded_field.dict_field), 2) + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(len(doc.list_field), 4) + self.assertEqual(len(doc.dict_field), 2) + self.assertEqual(len(doc.embedded_field.list_field), 4) + self.assertEqual(len(doc.embedded_field.dict_field), 2) def test_dictionary_access(self): """Ensure that dictionary-style field access works properly. """ person = self.Person(name='Test User', age=30) - self.assertEquals(person['name'], 'Test User') + self.assertEqual(person['name'], 'Test User') self.assertRaises(KeyError, person.__getitem__, 'salary') self.assertRaises(KeyError, person.__setitem__, 'salary', 50) person['name'] = 'Another User' - self.assertEquals(person['name'], 'Another User') + self.assertEqual(person['name'], 'Another User') # Length = length(assigned fields + id) - self.assertEquals(len(person), 3) + self.assertEqual(len(person), 3) self.assertTrue('age' in person) person.age = None @@ -1330,7 +1330,7 @@ class DocumentTest(unittest.TestCase): doc = Doc.objects.first() doc.validate() - self.assertEquals([None, 'e'], doc._data.keys()) + self.assertEqual([None, 'e'], doc._data.keys()) def test_save(self): """Ensure that a document may be saved in the database. @@ -1373,7 +1373,7 @@ class DocumentTest(unittest.TestCase): user.save() user.reload() - self.assertEquals(user.thing.count, 0) + self.assertEqual(user.thing.count, 0) def test_save_max_recursion_not_hit(self): @@ -1414,16 +1414,17 @@ class DocumentTest(unittest.TestCase): a.save() a.bar = a - a.picture = open(TEST_IMAGE_PATH, 'rb') - a.save() + with open(TEST_IMAGE_PATH, 'rb') as test_image: + a.picture = test_image + a.save() - # Confirm can save and it resets the changed fields without hitting - # max recursion error - b = Foo.objects.with_id(a.id) - b.name='world' - b.save() + # Confirm can save and it resets the changed fields without hitting + # max recursion error + b = Foo.objects.with_id(a.id) + b.name='world' + b.save() - self.assertEquals(b.picture, b.bar.picture, b.bar.bar.picture) + self.assertEqual(b.picture, b.bar.picture, b.bar.bar.picture) def test_save_cascades(self): @@ -1446,7 +1447,7 @@ class DocumentTest(unittest.TestCase): p.save() p1.reload() - self.assertEquals(p1.name, p.parent.name) + self.assertEqual(p1.name, p.parent.name) def test_save_cascade_kwargs(self): @@ -1469,7 +1470,7 @@ class DocumentTest(unittest.TestCase): p.save() p1.reload() - self.assertEquals(p1.name, p.parent.name) + self.assertEqual(p1.name, p.parent.name) def test_save_cascade_meta(self): @@ -1494,11 +1495,11 @@ class DocumentTest(unittest.TestCase): p.save() p1.reload() - self.assertNotEquals(p1.name, p.parent.name) + self.assertNotEqual(p1.name, p.parent.name) p.save(cascade=True) p1.reload() - self.assertEquals(p1.name, p.parent.name) + self.assertEqual(p1.name, p.parent.name) def test_save_cascades_generically(self): @@ -1520,7 +1521,7 @@ class DocumentTest(unittest.TestCase): p.save() p1.reload() - self.assertEquals(p1.name, p.parent.name) + self.assertEqual(p1.name, p.parent.name) def test_update(self): """Ensure that an existing document is updated instead of be overwritten. @@ -1535,7 +1536,7 @@ class DocumentTest(unittest.TestCase): same_person.save() # Confirm only one object - self.assertEquals(self.Person.objects.count(), 1) + self.assertEqual(self.Person.objects.count(), 1) # reload person.reload() @@ -1621,7 +1622,7 @@ class DocumentTest(unittest.TestCase): author.reload() p1 = self.Person.objects.first() - self.assertEquals(p1.name, author.name) + self.assertEqual(p1.name, author.name) def update_no_value_raises(): person = self.Person.objects.first() @@ -1714,8 +1715,8 @@ class DocumentTest(unittest.TestCase): p = Person.objects[0].select_related() o = Organization.objects.first() - self.assertEquals(p.owns[0], o) - self.assertEquals(o.owner, p) + self.assertEqual(p.owns[0], o) + self.assertEqual(o.owner, p) def test_circular_reference_deltas_2(self): @@ -1755,9 +1756,9 @@ class DocumentTest(unittest.TestCase): e = Person.objects.get(name="employee") o = Organization.objects.first() - self.assertEquals(p.owns[0], o) - self.assertEquals(o.owner, p) - self.assertEquals(e.employer, o) + self.assertEqual(p.owns[0], o) + self.assertEqual(o.owner, p) + self.assertEqual(e.employer, o) def test_delta(self): @@ -1772,40 +1773,40 @@ class DocumentTest(unittest.TestCase): doc.save() doc = Doc.objects.first() - self.assertEquals(doc._get_changed_fields(), []) - self.assertEquals(doc._delta(), ({}, {})) + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(doc._delta(), ({}, {})) doc.string_field = 'hello' - self.assertEquals(doc._get_changed_fields(), ['string_field']) - self.assertEquals(doc._delta(), ({'string_field': 'hello'}, {})) + self.assertEqual(doc._get_changed_fields(), ['string_field']) + self.assertEqual(doc._delta(), ({'string_field': 'hello'}, {})) doc._changed_fields = [] doc.int_field = 1 - self.assertEquals(doc._get_changed_fields(), ['int_field']) - self.assertEquals(doc._delta(), ({'int_field': 1}, {})) + self.assertEqual(doc._get_changed_fields(), ['int_field']) + self.assertEqual(doc._delta(), ({'int_field': 1}, {})) doc._changed_fields = [] dict_value = {'hello': 'world', 'ping': 'pong'} doc.dict_field = dict_value - self.assertEquals(doc._get_changed_fields(), ['dict_field']) - self.assertEquals(doc._delta(), ({'dict_field': dict_value}, {})) + self.assertEqual(doc._get_changed_fields(), ['dict_field']) + self.assertEqual(doc._delta(), ({'dict_field': dict_value}, {})) doc._changed_fields = [] list_value = ['1', 2, {'hello': 'world'}] doc.list_field = list_value - self.assertEquals(doc._get_changed_fields(), ['list_field']) - self.assertEquals(doc._delta(), ({'list_field': list_value}, {})) + self.assertEqual(doc._get_changed_fields(), ['list_field']) + self.assertEqual(doc._delta(), ({'list_field': list_value}, {})) # Test unsetting doc._changed_fields = [] doc.dict_field = {} - self.assertEquals(doc._get_changed_fields(), ['dict_field']) - self.assertEquals(doc._delta(), ({}, {'dict_field': 1})) + self.assertEqual(doc._get_changed_fields(), ['dict_field']) + self.assertEqual(doc._delta(), ({}, {'dict_field': 1})) doc._changed_fields = [] doc.list_field = [] - self.assertEquals(doc._get_changed_fields(), ['list_field']) - self.assertEquals(doc._delta(), ({}, {'list_field': 1})) + self.assertEqual(doc._get_changed_fields(), ['list_field']) + self.assertEqual(doc._delta(), ({}, {'list_field': 1})) def test_delta_recursive(self): @@ -1827,8 +1828,8 @@ class DocumentTest(unittest.TestCase): doc.save() doc = Doc.objects.first() - self.assertEquals(doc._get_changed_fields(), []) - self.assertEquals(doc._delta(), ({}, {})) + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(doc._delta(), ({}, {})) embedded_1 = Embedded() embedded_1.string_field = 'hello' @@ -1837,7 +1838,7 @@ class DocumentTest(unittest.TestCase): embedded_1.list_field = ['1', 2, {'hello': 'world'}] doc.embedded_field = embedded_1 - self.assertEquals(doc._get_changed_fields(), ['embedded_field']) + self.assertEqual(doc._get_changed_fields(), ['embedded_field']) embedded_delta = { 'string_field': 'hello', @@ -1845,31 +1846,31 @@ class DocumentTest(unittest.TestCase): 'dict_field': {'hello': 'world'}, 'list_field': ['1', 2, {'hello': 'world'}] } - self.assertEquals(doc.embedded_field._delta(), (embedded_delta, {})) + self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) embedded_delta.update({ '_types': ['Embedded'], '_cls': 'Embedded', }) - self.assertEquals(doc._delta(), ({'embedded_field': embedded_delta}, {})) + self.assertEqual(doc._delta(), ({'embedded_field': embedded_delta}, {})) doc.save() doc = doc.reload(10) doc.embedded_field.dict_field = {} - self.assertEquals(doc._get_changed_fields(), ['embedded_field.dict_field']) - self.assertEquals(doc.embedded_field._delta(), ({}, {'dict_field': 1})) - self.assertEquals(doc._delta(), ({}, {'embedded_field.dict_field': 1})) + self.assertEqual(doc._get_changed_fields(), ['embedded_field.dict_field']) + self.assertEqual(doc.embedded_field._delta(), ({}, {'dict_field': 1})) + self.assertEqual(doc._delta(), ({}, {'embedded_field.dict_field': 1})) doc.save() doc = doc.reload(10) - self.assertEquals(doc.embedded_field.dict_field, {}) + self.assertEqual(doc.embedded_field.dict_field, {}) doc.embedded_field.list_field = [] - self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEquals(doc.embedded_field._delta(), ({}, {'list_field': 1})) - self.assertEquals(doc._delta(), ({}, {'embedded_field.list_field': 1})) + self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) + self.assertEqual(doc.embedded_field._delta(), ({}, {'list_field': 1})) + self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field': 1})) doc.save() doc = doc.reload(10) - self.assertEquals(doc.embedded_field.list_field, []) + self.assertEqual(doc.embedded_field.list_field, []) embedded_2 = Embedded() embedded_2.string_field = 'hello' @@ -1878,8 +1879,8 @@ class DocumentTest(unittest.TestCase): embedded_2.list_field = ['1', 2, {'hello': 'world'}] doc.embedded_field.list_field = ['1', 2, embedded_2] - self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEquals(doc.embedded_field._delta(), ({ + self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) + self.assertEqual(doc.embedded_field._delta(), ({ 'list_field': ['1', 2, { '_cls': 'Embedded', '_types': ['Embedded'], @@ -1890,7 +1891,7 @@ class DocumentTest(unittest.TestCase): }] }, {})) - self.assertEquals(doc._delta(), ({ + self.assertEqual(doc._delta(), ({ 'embedded_field.list_field': ['1', 2, { '_cls': 'Embedded', '_types': ['Embedded'], @@ -1903,24 +1904,24 @@ class DocumentTest(unittest.TestCase): doc.save() doc = doc.reload(10) - self.assertEquals(doc.embedded_field.list_field[0], '1') - self.assertEquals(doc.embedded_field.list_field[1], 2) + self.assertEqual(doc.embedded_field.list_field[0], '1') + self.assertEqual(doc.embedded_field.list_field[1], 2) for k in doc.embedded_field.list_field[2]._fields: - self.assertEquals(doc.embedded_field.list_field[2][k], embedded_2[k]) + self.assertEqual(doc.embedded_field.list_field[2][k], embedded_2[k]) doc.embedded_field.list_field[2].string_field = 'world' - self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field.2.string_field']) - self.assertEquals(doc.embedded_field._delta(), ({'list_field.2.string_field': 'world'}, {})) - self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.string_field': 'world'}, {})) + self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field.2.string_field']) + self.assertEqual(doc.embedded_field._delta(), ({'list_field.2.string_field': 'world'}, {})) + self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.string_field': 'world'}, {})) doc.save() doc = doc.reload(10) - self.assertEquals(doc.embedded_field.list_field[2].string_field, 'world') + self.assertEqual(doc.embedded_field.list_field[2].string_field, 'world') # Test multiple assignments doc.embedded_field.list_field[2].string_field = 'hello world' doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] - self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEquals(doc.embedded_field._delta(), ({ + self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) + self.assertEqual(doc.embedded_field._delta(), ({ 'list_field': ['1', 2, { '_types': ['Embedded'], '_cls': 'Embedded', @@ -1928,7 +1929,7 @@ class DocumentTest(unittest.TestCase): 'int_field': 1, 'list_field': ['1', 2, {'hello': 'world'}], 'dict_field': {'hello': 'world'}}]}, {})) - self.assertEquals(doc._delta(), ({ + self.assertEqual(doc._delta(), ({ 'embedded_field.list_field': ['1', 2, { '_types': ['Embedded'], '_cls': 'Embedded', @@ -1939,32 +1940,32 @@ class DocumentTest(unittest.TestCase): ]}, {})) doc.save() doc = doc.reload(10) - self.assertEquals(doc.embedded_field.list_field[2].string_field, 'hello world') + self.assertEqual(doc.embedded_field.list_field[2].string_field, 'hello world') # Test list native methods doc.embedded_field.list_field[2].list_field.pop(0) - self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}]}, {})) + self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}]}, {})) doc.save() doc = doc.reload(10) doc.embedded_field.list_field[2].list_field.append(1) - self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}, 1]}, {})) + self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}, 1]}, {})) doc.save() doc = doc.reload(10) - self.assertEquals(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) + self.assertEqual(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) - doc.embedded_field.list_field[2].list_field.sort() + doc.embedded_field.list_field[2].list_field.sort(key=str) doc.save() doc = doc.reload(10) - self.assertEquals(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) + self.assertEqual(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) del(doc.embedded_field.list_field[2].list_field[2]['hello']) - self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) + self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) doc.save() doc = doc.reload(10) del(doc.embedded_field.list_field[2].list_field) - self.assertEquals(doc._delta(), ({}, {'embedded_field.list_field.2.list_field': 1})) + self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field.2.list_field': 1})) doc.save() doc = doc.reload(10) @@ -1974,8 +1975,8 @@ class DocumentTest(unittest.TestCase): doc = doc.reload(10) doc.dict_field['Embedded'].string_field = 'Hello World' - self.assertEquals(doc._get_changed_fields(), ['dict_field.Embedded.string_field']) - self.assertEquals(doc._delta(), ({'dict_field.Embedded.string_field': 'Hello World'}, {})) + self.assertEqual(doc._get_changed_fields(), ['dict_field.Embedded.string_field']) + self.assertEqual(doc._delta(), ({'dict_field.Embedded.string_field': 'Hello World'}, {})) def test_delta_db_field(self): @@ -1991,40 +1992,40 @@ class DocumentTest(unittest.TestCase): doc.save() doc = Doc.objects.first() - self.assertEquals(doc._get_changed_fields(), []) - self.assertEquals(doc._delta(), ({}, {})) + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(doc._delta(), ({}, {})) doc.string_field = 'hello' - self.assertEquals(doc._get_changed_fields(), ['db_string_field']) - self.assertEquals(doc._delta(), ({'db_string_field': 'hello'}, {})) + self.assertEqual(doc._get_changed_fields(), ['db_string_field']) + self.assertEqual(doc._delta(), ({'db_string_field': 'hello'}, {})) doc._changed_fields = [] doc.int_field = 1 - self.assertEquals(doc._get_changed_fields(), ['db_int_field']) - self.assertEquals(doc._delta(), ({'db_int_field': 1}, {})) + self.assertEqual(doc._get_changed_fields(), ['db_int_field']) + self.assertEqual(doc._delta(), ({'db_int_field': 1}, {})) doc._changed_fields = [] dict_value = {'hello': 'world', 'ping': 'pong'} doc.dict_field = dict_value - self.assertEquals(doc._get_changed_fields(), ['db_dict_field']) - self.assertEquals(doc._delta(), ({'db_dict_field': dict_value}, {})) + self.assertEqual(doc._get_changed_fields(), ['db_dict_field']) + self.assertEqual(doc._delta(), ({'db_dict_field': dict_value}, {})) doc._changed_fields = [] list_value = ['1', 2, {'hello': 'world'}] doc.list_field = list_value - self.assertEquals(doc._get_changed_fields(), ['db_list_field']) - self.assertEquals(doc._delta(), ({'db_list_field': list_value}, {})) + self.assertEqual(doc._get_changed_fields(), ['db_list_field']) + self.assertEqual(doc._delta(), ({'db_list_field': list_value}, {})) # Test unsetting doc._changed_fields = [] doc.dict_field = {} - self.assertEquals(doc._get_changed_fields(), ['db_dict_field']) - self.assertEquals(doc._delta(), ({}, {'db_dict_field': 1})) + self.assertEqual(doc._get_changed_fields(), ['db_dict_field']) + self.assertEqual(doc._delta(), ({}, {'db_dict_field': 1})) doc._changed_fields = [] doc.list_field = [] - self.assertEquals(doc._get_changed_fields(), ['db_list_field']) - self.assertEquals(doc._delta(), ({}, {'db_list_field': 1})) + self.assertEqual(doc._get_changed_fields(), ['db_list_field']) + self.assertEqual(doc._delta(), ({}, {'db_list_field': 1})) # Test it saves that data doc = Doc() @@ -2037,10 +2038,10 @@ class DocumentTest(unittest.TestCase): doc.save() doc = doc.reload(10) - self.assertEquals(doc.string_field, 'hello') - self.assertEquals(doc.int_field, 1) - self.assertEquals(doc.dict_field, {'hello': 'world'}) - self.assertEquals(doc.list_field, ['1', 2, {'hello': 'world'}]) + self.assertEqual(doc.string_field, 'hello') + self.assertEqual(doc.int_field, 1) + self.assertEqual(doc.dict_field, {'hello': 'world'}) + self.assertEqual(doc.list_field, ['1', 2, {'hello': 'world'}]) def test_delta_recursive_db_field(self): @@ -2062,8 +2063,8 @@ class DocumentTest(unittest.TestCase): doc.save() doc = Doc.objects.first() - self.assertEquals(doc._get_changed_fields(), []) - self.assertEquals(doc._delta(), ({}, {})) + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(doc._delta(), ({}, {})) embedded_1 = Embedded() embedded_1.string_field = 'hello' @@ -2072,7 +2073,7 @@ class DocumentTest(unittest.TestCase): embedded_1.list_field = ['1', 2, {'hello': 'world'}] doc.embedded_field = embedded_1 - self.assertEquals(doc._get_changed_fields(), ['db_embedded_field']) + self.assertEqual(doc._get_changed_fields(), ['db_embedded_field']) embedded_delta = { 'db_string_field': 'hello', @@ -2080,31 +2081,31 @@ class DocumentTest(unittest.TestCase): 'db_dict_field': {'hello': 'world'}, 'db_list_field': ['1', 2, {'hello': 'world'}] } - self.assertEquals(doc.embedded_field._delta(), (embedded_delta, {})) + self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) embedded_delta.update({ '_types': ['Embedded'], '_cls': 'Embedded', }) - self.assertEquals(doc._delta(), ({'db_embedded_field': embedded_delta}, {})) + self.assertEqual(doc._delta(), ({'db_embedded_field': embedded_delta}, {})) doc.save() doc = doc.reload(10) doc.embedded_field.dict_field = {} - self.assertEquals(doc._get_changed_fields(), ['db_embedded_field.db_dict_field']) - self.assertEquals(doc.embedded_field._delta(), ({}, {'db_dict_field': 1})) - self.assertEquals(doc._delta(), ({}, {'db_embedded_field.db_dict_field': 1})) + self.assertEqual(doc._get_changed_fields(), ['db_embedded_field.db_dict_field']) + self.assertEqual(doc.embedded_field._delta(), ({}, {'db_dict_field': 1})) + self.assertEqual(doc._delta(), ({}, {'db_embedded_field.db_dict_field': 1})) doc.save() doc = doc.reload(10) - self.assertEquals(doc.embedded_field.dict_field, {}) + self.assertEqual(doc.embedded_field.dict_field, {}) doc.embedded_field.list_field = [] - self.assertEquals(doc._get_changed_fields(), ['db_embedded_field.db_list_field']) - self.assertEquals(doc.embedded_field._delta(), ({}, {'db_list_field': 1})) - self.assertEquals(doc._delta(), ({}, {'db_embedded_field.db_list_field': 1})) + self.assertEqual(doc._get_changed_fields(), ['db_embedded_field.db_list_field']) + self.assertEqual(doc.embedded_field._delta(), ({}, {'db_list_field': 1})) + self.assertEqual(doc._delta(), ({}, {'db_embedded_field.db_list_field': 1})) doc.save() doc = doc.reload(10) - self.assertEquals(doc.embedded_field.list_field, []) + self.assertEqual(doc.embedded_field.list_field, []) embedded_2 = Embedded() embedded_2.string_field = 'hello' @@ -2113,8 +2114,8 @@ class DocumentTest(unittest.TestCase): embedded_2.list_field = ['1', 2, {'hello': 'world'}] doc.embedded_field.list_field = ['1', 2, embedded_2] - self.assertEquals(doc._get_changed_fields(), ['db_embedded_field.db_list_field']) - self.assertEquals(doc.embedded_field._delta(), ({ + self.assertEqual(doc._get_changed_fields(), ['db_embedded_field.db_list_field']) + self.assertEqual(doc.embedded_field._delta(), ({ 'db_list_field': ['1', 2, { '_cls': 'Embedded', '_types': ['Embedded'], @@ -2125,7 +2126,7 @@ class DocumentTest(unittest.TestCase): }] }, {})) - self.assertEquals(doc._delta(), ({ + self.assertEqual(doc._delta(), ({ 'db_embedded_field.db_list_field': ['1', 2, { '_cls': 'Embedded', '_types': ['Embedded'], @@ -2138,24 +2139,24 @@ class DocumentTest(unittest.TestCase): doc.save() doc = doc.reload(10) - self.assertEquals(doc.embedded_field.list_field[0], '1') - self.assertEquals(doc.embedded_field.list_field[1], 2) + self.assertEqual(doc.embedded_field.list_field[0], '1') + self.assertEqual(doc.embedded_field.list_field[1], 2) for k in doc.embedded_field.list_field[2]._fields: - self.assertEquals(doc.embedded_field.list_field[2][k], embedded_2[k]) + self.assertEqual(doc.embedded_field.list_field[2][k], embedded_2[k]) doc.embedded_field.list_field[2].string_field = 'world' - self.assertEquals(doc._get_changed_fields(), ['db_embedded_field.db_list_field.2.db_string_field']) - self.assertEquals(doc.embedded_field._delta(), ({'db_list_field.2.db_string_field': 'world'}, {})) - self.assertEquals(doc._delta(), ({'db_embedded_field.db_list_field.2.db_string_field': 'world'}, {})) + self.assertEqual(doc._get_changed_fields(), ['db_embedded_field.db_list_field.2.db_string_field']) + self.assertEqual(doc.embedded_field._delta(), ({'db_list_field.2.db_string_field': 'world'}, {})) + self.assertEqual(doc._delta(), ({'db_embedded_field.db_list_field.2.db_string_field': 'world'}, {})) doc.save() doc = doc.reload(10) - self.assertEquals(doc.embedded_field.list_field[2].string_field, 'world') + self.assertEqual(doc.embedded_field.list_field[2].string_field, 'world') # Test multiple assignments doc.embedded_field.list_field[2].string_field = 'hello world' doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] - self.assertEquals(doc._get_changed_fields(), ['db_embedded_field.db_list_field']) - self.assertEquals(doc.embedded_field._delta(), ({ + self.assertEqual(doc._get_changed_fields(), ['db_embedded_field.db_list_field']) + self.assertEqual(doc.embedded_field._delta(), ({ 'db_list_field': ['1', 2, { '_types': ['Embedded'], '_cls': 'Embedded', @@ -2163,7 +2164,7 @@ class DocumentTest(unittest.TestCase): 'db_int_field': 1, 'db_list_field': ['1', 2, {'hello': 'world'}], 'db_dict_field': {'hello': 'world'}}]}, {})) - self.assertEquals(doc._delta(), ({ + self.assertEqual(doc._delta(), ({ 'db_embedded_field.db_list_field': ['1', 2, { '_types': ['Embedded'], '_cls': 'Embedded', @@ -2174,32 +2175,32 @@ class DocumentTest(unittest.TestCase): ]}, {})) doc.save() doc = doc.reload(10) - self.assertEquals(doc.embedded_field.list_field[2].string_field, 'hello world') + self.assertEqual(doc.embedded_field.list_field[2].string_field, 'hello world') # Test list native methods doc.embedded_field.list_field[2].list_field.pop(0) - self.assertEquals(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [2, {'hello': 'world'}]}, {})) + self.assertEqual(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [2, {'hello': 'world'}]}, {})) doc.save() doc = doc.reload(10) doc.embedded_field.list_field[2].list_field.append(1) - self.assertEquals(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [2, {'hello': 'world'}, 1]}, {})) + self.assertEqual(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [2, {'hello': 'world'}, 1]}, {})) doc.save() doc = doc.reload(10) - self.assertEquals(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) + self.assertEqual(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) - doc.embedded_field.list_field[2].list_field.sort() + doc.embedded_field.list_field[2].list_field.sort(key=str) doc.save() doc = doc.reload(10) - self.assertEquals(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) + self.assertEqual(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) del(doc.embedded_field.list_field[2].list_field[2]['hello']) - self.assertEquals(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [1, 2, {}]}, {})) + self.assertEqual(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [1, 2, {}]}, {})) doc.save() doc = doc.reload(10) del(doc.embedded_field.list_field[2].list_field) - self.assertEquals(doc._delta(), ({}, {'db_embedded_field.db_list_field.2.db_list_field': 1})) + self.assertEqual(doc._delta(), ({}, {'db_embedded_field.db_list_field.2.db_list_field': 1})) def test_save_only_changed_fields(self): """Ensure save only sets / unsets changed fields @@ -2227,9 +2228,9 @@ class DocumentTest(unittest.TestCase): same_person.save() person = self.Person.objects.get() - self.assertEquals(person.name, 'User') - self.assertEquals(person.age, 21) - self.assertEquals(person.active, False) + self.assertEqual(person.name, 'User') + self.assertEqual(person.age, 21) + self.assertEqual(person.active, False) def test_save_only_changed_fields_recursive(self): """Ensure save only sets / unsets changed fields @@ -2485,16 +2486,16 @@ class DocumentTest(unittest.TestCase): p = Person(name="Rozza") p.save() - self.assertEquals(p._fields.keys(), ['name', 'id']) + self.assertEqual(p._fields.keys(), ['name', 'id']) collection = self.db[Person._get_collection_name()] obj = collection.find_one() - self.assertEquals(obj['_cls'], 'Person') - self.assertEquals(obj['_types'], ['Person']) + self.assertEqual(obj['_cls'], 'Person') + self.assertEqual(obj['_types'], ['Person']) - self.assertEquals(Person.objects.count(), 1) + self.assertEqual(Person.objects.count(), 1) rozza = Person.objects.get(name="Rozza") Person.drop_collection() @@ -2518,10 +2519,10 @@ class DocumentTest(unittest.TestCase): t = TestDoc.objects.first() - self.assertEquals(t.age, 19) - self.assertEquals(t.comment, "great!") - self.assertEquals(t.data, "test") - self.assertEquals(t.count, 12) + self.assertEqual(t.age, 19) + self.assertEqual(t.comment, "great!") + self.assertEqual(t.data, "test") + self.assertEqual(t.count, 12) def test_save_reference(self): """Ensure that a document reference field may be saved in the database. @@ -2802,8 +2803,8 @@ class DocumentTest(unittest.TestCase): A().save() B(foo=True).save() - self.assertEquals(A.objects.count(), 2) - self.assertEquals(B.objects.count(), 1) + self.assertEqual(A.objects.count(), 2) + self.assertEqual(B.objects.count(), 1) A.drop_collection() B.drop_collection() @@ -2864,13 +2865,13 @@ class DocumentTest(unittest.TestCase): pickled_doc = pickle.dumps(pickle_doc) resurrected = pickle.loads(pickled_doc) - self.assertEquals(resurrected, pickle_doc) + self.assertEqual(resurrected, pickle_doc) resurrected.string = "Two" resurrected.save() pickle_doc = pickle_doc.reload() - self.assertEquals(resurrected, pickle_doc) + self.assertEqual(resurrected, pickle_doc) def test_throw_invalid_document_error(self): @@ -2893,7 +2894,7 @@ class DocumentTest(unittest.TestCase): a = A() a.save() a.reload() - self.assertEquals(a.b.field1, 'field1') + self.assertEqual(a.b.field1, 'field1') class C(EmbeddedDocument): c_field = StringField(default='cfield') @@ -2910,7 +2911,7 @@ class DocumentTest(unittest.TestCase): a.save() a.reload() - self.assertEquals(a.b.field2.c_field, 'new value') + self.assertEqual(a.b.field2.c_field, 'new value') def test_can_save_false_values(self): """Ensures you can save False values on save""" @@ -2924,7 +2925,7 @@ class DocumentTest(unittest.TestCase): d.archived = False d.save() - self.assertEquals(Doc.objects(archived=False).count(), 1) + self.assertEqual(Doc.objects(archived=False).count(), 1) def test_can_save_false_values_dynamic(self): @@ -2938,7 +2939,7 @@ class DocumentTest(unittest.TestCase): d.archived = False d.save() - self.assertEquals(Doc.objects(archived=False).count(), 1) + self.assertEqual(Doc.objects(archived=False).count(), 1) def test_do_not_save_unchanged_references(self): """Ensures cascading saves dont auto update""" @@ -3037,7 +3038,7 @@ class DocumentTest(unittest.TestCase): class B(A): pass - self.assertEquals('testdb-1', B._meta.get('db_alias')) + self.assertEqual('testdb-1', B._meta.get('db_alias')) def test_db_ref_usage(self): """ DB Ref usage in __raw__ queries """ @@ -3118,12 +3119,12 @@ class ValidatorErrorTest(unittest.TestCase): """Ensure a ValidationError handles error to_dict correctly. """ error = ValidationError('root') - self.assertEquals(error.to_dict(), {}) + self.assertEqual(error.to_dict(), {}) # 1st level error schema error.errors = {'1st': ValidationError('bad 1st'), } self.assertTrue('1st' in error.to_dict()) - self.assertEquals(error.to_dict()['1st'], 'bad 1st') + self.assertEqual(error.to_dict()['1st'], 'bad 1st') # 2nd level error schema error.errors = {'1st': ValidationError('bad 1st', errors={ @@ -3132,7 +3133,7 @@ class ValidatorErrorTest(unittest.TestCase): self.assertTrue('1st' in error.to_dict()) self.assertTrue(isinstance(error.to_dict()['1st'], dict)) self.assertTrue('2nd' in error.to_dict()['1st']) - self.assertEquals(error.to_dict()['1st']['2nd'], 'bad 2nd') + self.assertEqual(error.to_dict()['1st']['2nd'], 'bad 2nd') # moar levels error.errors = {'1st': ValidationError('bad 1st', errors={ @@ -3146,10 +3147,10 @@ class ValidatorErrorTest(unittest.TestCase): self.assertTrue('2nd' in error.to_dict()['1st']) self.assertTrue('3rd' in error.to_dict()['1st']['2nd']) self.assertTrue('4th' in error.to_dict()['1st']['2nd']['3rd']) - self.assertEquals(error.to_dict()['1st']['2nd']['3rd']['4th'], + self.assertEqual(error.to_dict()['1st']['2nd']['3rd']['4th'], 'Inception') - self.assertEquals(error.message, "root(2nd.3rd.4th.Inception: ['1st'])") + self.assertEqual(error.message, "root(2nd.3rd.4th.Inception: ['1st'])") def test_model_validation(self): @@ -3161,8 +3162,8 @@ class ValidatorErrorTest(unittest.TestCase): User().validate() except ValidationError, e: expected_error_message = """ValidationError(Field is required: ['username', 'name'])""" - self.assertEquals(e.message, expected_error_message) - self.assertEquals(e.to_dict(), { + self.assertEqual(e.message, expected_error_message) + self.assertEqual(e.to_dict(), { 'username': 'Field is required', 'name': 'Field is required'}) diff --git a/tests/test_dynamic_document.py b/tests/test_dynamic_document.py index 6ff199a..866af7d 100644 --- a/tests/test_dynamic_document.py +++ b/tests/test_dynamic_document.py @@ -25,14 +25,14 @@ class DynamicDocTest(unittest.TestCase): p.name = "James" p.age = 34 - self.assertEquals(p.to_mongo(), + self.assertEqual(p.to_mongo(), {"_types": ["Person"], "_cls": "Person", "name": "James", "age": 34} ) p.save() - self.assertEquals(self.Person.objects.first().age, 34) + self.assertEqual(self.Person.objects.first().age, 34) # Confirm no changes to self.Person self.assertFalse(hasattr(self.Person, 'age')) @@ -40,11 +40,11 @@ class DynamicDocTest(unittest.TestCase): def test_dynamic_document_delta(self): """Ensures simple dynamic documents can delta correctly""" p = self.Person(name="James", age=34) - self.assertEquals(p._delta(), ({'_types': ['Person'], 'age': 34, 'name': 'James', '_cls': 'Person'}, {})) + self.assertEqual(p._delta(), ({'_types': ['Person'], 'age': 34, 'name': 'James', '_cls': 'Person'}, {})) p.doc = 123 del(p.doc) - self.assertEquals(p._delta(), ({'_types': ['Person'], 'age': 34, 'name': 'James', '_cls': 'Person'}, {'doc': 1})) + self.assertEqual(p._delta(), ({'_types': ['Person'], 'age': 34, 'name': 'James', '_cls': 'Person'}, {'doc': 1})) def test_change_scope_of_variable(self): """Test changing the scope of a dynamic field has no adverse effects""" @@ -58,7 +58,7 @@ class DynamicDocTest(unittest.TestCase): p.save() p = self.Person.objects.get() - self.assertEquals(p.misc, {'hello': 'world'}) + self.assertEqual(p.misc, {'hello': 'world'}) def test_delete_dynamic_field(self): """Test deleting a dynamic field works""" @@ -73,10 +73,10 @@ class DynamicDocTest(unittest.TestCase): p.save() p = self.Person.objects.get() - self.assertEquals(p.misc, {'hello': 'world'}) + self.assertEqual(p.misc, {'hello': 'world'}) collection = self.db[self.Person._get_collection_name()] obj = collection.find_one() - self.assertEquals(sorted(obj.keys()), ['_cls', '_id', '_types', 'misc', 'name']) + self.assertEqual(sorted(obj.keys()), ['_cls', '_id', '_types', 'misc', 'name']) del(p.misc) p.save() @@ -85,7 +85,7 @@ class DynamicDocTest(unittest.TestCase): self.assertFalse(hasattr(p, 'misc')) obj = collection.find_one() - self.assertEquals(sorted(obj.keys()), ['_cls', '_id', '_types', 'name']) + self.assertEqual(sorted(obj.keys()), ['_cls', '_id', '_types', 'name']) def test_dynamic_document_queries(self): """Ensure we can query dynamic fields""" @@ -94,10 +94,10 @@ class DynamicDocTest(unittest.TestCase): p.age = 22 p.save() - self.assertEquals(1, self.Person.objects(age=22).count()) + self.assertEqual(1, self.Person.objects(age=22).count()) p = self.Person.objects(age=22) p = p.get() - self.assertEquals(22, p.age) + self.assertEqual(22, p.age) def test_complex_dynamic_document_queries(self): class Person(DynamicDocument): @@ -117,8 +117,8 @@ class DynamicDocTest(unittest.TestCase): p2.age = 10 p2.save() - self.assertEquals(Person.objects(age__icontains='ten').count(), 2) - self.assertEquals(Person.objects(age__gte=10).count(), 1) + self.assertEqual(Person.objects(age__icontains='ten').count(), 2) + self.assertEqual(Person.objects(age__gte=10).count(), 1) def test_complex_data_lookups(self): """Ensure you can query dynamic document dynamic fields""" @@ -126,7 +126,7 @@ class DynamicDocTest(unittest.TestCase): p.misc = {'hello': 'world'} p.save() - self.assertEquals(1, self.Person.objects(misc__hello='world').count()) + self.assertEqual(1, self.Person.objects(misc__hello='world').count()) def test_inheritance(self): """Ensure that dynamic document plays nice with inheritance""" @@ -146,8 +146,8 @@ class DynamicDocTest(unittest.TestCase): joe_bloggs.age = 20 joe_bloggs.save() - self.assertEquals(1, self.Person.objects(age=20).count()) - self.assertEquals(1, Employee.objects(age=20).count()) + self.assertEqual(1, self.Person.objects(age=20).count()) + self.assertEqual(1, Employee.objects(age=20).count()) joe_bloggs = self.Person.objects.first() self.assertTrue(isinstance(joe_bloggs, Employee)) @@ -170,7 +170,7 @@ class DynamicDocTest(unittest.TestCase): embedded_1.list_field = ['1', 2, {'hello': 'world'}] doc.embedded_field = embedded_1 - self.assertEquals(doc.to_mongo(), {"_types": ['Doc'], "_cls": "Doc", + self.assertEqual(doc.to_mongo(), {"_types": ['Doc'], "_cls": "Doc", "embedded_field": { "_types": ['Embedded'], "_cls": "Embedded", "string_field": "hello", @@ -182,11 +182,11 @@ class DynamicDocTest(unittest.TestCase): doc.save() doc = Doc.objects.first() - self.assertEquals(doc.embedded_field.__class__, Embedded) - self.assertEquals(doc.embedded_field.string_field, "hello") - self.assertEquals(doc.embedded_field.int_field, 1) - self.assertEquals(doc.embedded_field.dict_field, {'hello': 'world'}) - self.assertEquals(doc.embedded_field.list_field, ['1', 2, {'hello': 'world'}]) + self.assertEqual(doc.embedded_field.__class__, Embedded) + self.assertEqual(doc.embedded_field.string_field, "hello") + self.assertEqual(doc.embedded_field.int_field, 1) + self.assertEqual(doc.embedded_field.dict_field, {'hello': 'world'}) + self.assertEqual(doc.embedded_field.list_field, ['1', 2, {'hello': 'world'}]) def test_complex_embedded_documents(self): """Test complex dynamic embedded documents setups""" @@ -213,7 +213,7 @@ class DynamicDocTest(unittest.TestCase): embedded_1.list_field = ['1', 2, embedded_2] doc.embedded_field = embedded_1 - self.assertEquals(doc.to_mongo(), {"_types": ['Doc'], "_cls": "Doc", + self.assertEqual(doc.to_mongo(), {"_types": ['Doc'], "_cls": "Doc", "embedded_field": { "_types": ['Embedded'], "_cls": "Embedded", "string_field": "hello", @@ -230,20 +230,20 @@ class DynamicDocTest(unittest.TestCase): }) doc.save() doc = Doc.objects.first() - self.assertEquals(doc.embedded_field.__class__, Embedded) - self.assertEquals(doc.embedded_field.string_field, "hello") - self.assertEquals(doc.embedded_field.int_field, 1) - self.assertEquals(doc.embedded_field.dict_field, {'hello': 'world'}) - self.assertEquals(doc.embedded_field.list_field[0], '1') - self.assertEquals(doc.embedded_field.list_field[1], 2) + self.assertEqual(doc.embedded_field.__class__, Embedded) + self.assertEqual(doc.embedded_field.string_field, "hello") + self.assertEqual(doc.embedded_field.int_field, 1) + self.assertEqual(doc.embedded_field.dict_field, {'hello': 'world'}) + self.assertEqual(doc.embedded_field.list_field[0], '1') + self.assertEqual(doc.embedded_field.list_field[1], 2) embedded_field = doc.embedded_field.list_field[2] - self.assertEquals(embedded_field.__class__, Embedded) - self.assertEquals(embedded_field.string_field, "hello") - self.assertEquals(embedded_field.int_field, 1) - self.assertEquals(embedded_field.dict_field, {'hello': 'world'}) - self.assertEquals(embedded_field.list_field, ['1', 2, {'hello': 'world'}]) + self.assertEqual(embedded_field.__class__, Embedded) + self.assertEqual(embedded_field.string_field, "hello") + self.assertEqual(embedded_field.int_field, 1) + self.assertEqual(embedded_field.dict_field, {'hello': 'world'}) + self.assertEqual(embedded_field.list_field, ['1', 2, {'hello': 'world'}]) def test_delta_for_dynamic_documents(self): p = self.Person() @@ -252,18 +252,18 @@ class DynamicDocTest(unittest.TestCase): p.save() p.age = 24 - self.assertEquals(p.age, 24) - self.assertEquals(p._get_changed_fields(), ['age']) - self.assertEquals(p._delta(), ({'age': 24}, {})) + self.assertEqual(p.age, 24) + self.assertEqual(p._get_changed_fields(), ['age']) + self.assertEqual(p._delta(), ({'age': 24}, {})) p = self.Person.objects(age=22).get() p.age = 24 - self.assertEquals(p.age, 24) - self.assertEquals(p._get_changed_fields(), ['age']) - self.assertEquals(p._delta(), ({'age': 24}, {})) + self.assertEqual(p.age, 24) + self.assertEqual(p._get_changed_fields(), ['age']) + self.assertEqual(p._delta(), ({'age': 24}, {})) p.save() - self.assertEquals(1, self.Person.objects(age=24).count()) + self.assertEqual(1, self.Person.objects(age=24).count()) def test_delta(self): @@ -275,40 +275,40 @@ class DynamicDocTest(unittest.TestCase): doc.save() doc = Doc.objects.first() - self.assertEquals(doc._get_changed_fields(), []) - self.assertEquals(doc._delta(), ({}, {})) + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(doc._delta(), ({}, {})) doc.string_field = 'hello' - self.assertEquals(doc._get_changed_fields(), ['string_field']) - self.assertEquals(doc._delta(), ({'string_field': 'hello'}, {})) + self.assertEqual(doc._get_changed_fields(), ['string_field']) + self.assertEqual(doc._delta(), ({'string_field': 'hello'}, {})) doc._changed_fields = [] doc.int_field = 1 - self.assertEquals(doc._get_changed_fields(), ['int_field']) - self.assertEquals(doc._delta(), ({'int_field': 1}, {})) + self.assertEqual(doc._get_changed_fields(), ['int_field']) + self.assertEqual(doc._delta(), ({'int_field': 1}, {})) doc._changed_fields = [] dict_value = {'hello': 'world', 'ping': 'pong'} doc.dict_field = dict_value - self.assertEquals(doc._get_changed_fields(), ['dict_field']) - self.assertEquals(doc._delta(), ({'dict_field': dict_value}, {})) + self.assertEqual(doc._get_changed_fields(), ['dict_field']) + self.assertEqual(doc._delta(), ({'dict_field': dict_value}, {})) doc._changed_fields = [] list_value = ['1', 2, {'hello': 'world'}] doc.list_field = list_value - self.assertEquals(doc._get_changed_fields(), ['list_field']) - self.assertEquals(doc._delta(), ({'list_field': list_value}, {})) + self.assertEqual(doc._get_changed_fields(), ['list_field']) + self.assertEqual(doc._delta(), ({'list_field': list_value}, {})) # Test unsetting doc._changed_fields = [] doc.dict_field = {} - self.assertEquals(doc._get_changed_fields(), ['dict_field']) - self.assertEquals(doc._delta(), ({}, {'dict_field': 1})) + self.assertEqual(doc._get_changed_fields(), ['dict_field']) + self.assertEqual(doc._delta(), ({}, {'dict_field': 1})) doc._changed_fields = [] doc.list_field = [] - self.assertEquals(doc._get_changed_fields(), ['list_field']) - self.assertEquals(doc._delta(), ({}, {'list_field': 1})) + self.assertEqual(doc._get_changed_fields(), ['list_field']) + self.assertEqual(doc._delta(), ({}, {'list_field': 1})) def test_delta_recursive(self): """Testing deltaing works with dynamic documents""" @@ -323,8 +323,8 @@ class DynamicDocTest(unittest.TestCase): doc.save() doc = Doc.objects.first() - self.assertEquals(doc._get_changed_fields(), []) - self.assertEquals(doc._delta(), ({}, {})) + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(doc._delta(), ({}, {})) embedded_1 = Embedded() embedded_1.string_field = 'hello' @@ -333,7 +333,7 @@ class DynamicDocTest(unittest.TestCase): embedded_1.list_field = ['1', 2, {'hello': 'world'}] doc.embedded_field = embedded_1 - self.assertEquals(doc._get_changed_fields(), ['embedded_field']) + self.assertEqual(doc._get_changed_fields(), ['embedded_field']) embedded_delta = { 'string_field': 'hello', @@ -341,28 +341,28 @@ class DynamicDocTest(unittest.TestCase): 'dict_field': {'hello': 'world'}, 'list_field': ['1', 2, {'hello': 'world'}] } - self.assertEquals(doc.embedded_field._delta(), (embedded_delta, {})) + self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) embedded_delta.update({ '_types': ['Embedded'], '_cls': 'Embedded', }) - self.assertEquals(doc._delta(), ({'embedded_field': embedded_delta}, {})) + self.assertEqual(doc._delta(), ({'embedded_field': embedded_delta}, {})) doc.save() doc.reload() doc.embedded_field.dict_field = {} - self.assertEquals(doc._get_changed_fields(), ['embedded_field.dict_field']) - self.assertEquals(doc.embedded_field._delta(), ({}, {'dict_field': 1})) + self.assertEqual(doc._get_changed_fields(), ['embedded_field.dict_field']) + self.assertEqual(doc.embedded_field._delta(), ({}, {'dict_field': 1})) - self.assertEquals(doc._delta(), ({}, {'embedded_field.dict_field': 1})) + self.assertEqual(doc._delta(), ({}, {'embedded_field.dict_field': 1})) doc.save() doc.reload() doc.embedded_field.list_field = [] - self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEquals(doc.embedded_field._delta(), ({}, {'list_field': 1})) - self.assertEquals(doc._delta(), ({}, {'embedded_field.list_field': 1})) + self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) + self.assertEqual(doc.embedded_field._delta(), ({}, {'list_field': 1})) + self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field': 1})) doc.save() doc.reload() @@ -373,8 +373,8 @@ class DynamicDocTest(unittest.TestCase): embedded_2.list_field = ['1', 2, {'hello': 'world'}] doc.embedded_field.list_field = ['1', 2, embedded_2] - self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEquals(doc.embedded_field._delta(), ({ + self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) + self.assertEqual(doc.embedded_field._delta(), ({ 'list_field': ['1', 2, { '_cls': 'Embedded', '_types': ['Embedded'], @@ -385,7 +385,7 @@ class DynamicDocTest(unittest.TestCase): }] }, {})) - self.assertEquals(doc._delta(), ({ + self.assertEqual(doc._delta(), ({ 'embedded_field.list_field': ['1', 2, { '_cls': 'Embedded', '_types': ['Embedded'], @@ -398,25 +398,25 @@ class DynamicDocTest(unittest.TestCase): doc.save() doc.reload() - self.assertEquals(doc.embedded_field.list_field[2]._changed_fields, []) - self.assertEquals(doc.embedded_field.list_field[0], '1') - self.assertEquals(doc.embedded_field.list_field[1], 2) + self.assertEqual(doc.embedded_field.list_field[2]._changed_fields, []) + self.assertEqual(doc.embedded_field.list_field[0], '1') + self.assertEqual(doc.embedded_field.list_field[1], 2) for k in doc.embedded_field.list_field[2]._fields: - self.assertEquals(doc.embedded_field.list_field[2][k], embedded_2[k]) + self.assertEqual(doc.embedded_field.list_field[2][k], embedded_2[k]) doc.embedded_field.list_field[2].string_field = 'world' - self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field.2.string_field']) - self.assertEquals(doc.embedded_field._delta(), ({'list_field.2.string_field': 'world'}, {})) - self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.string_field': 'world'}, {})) + self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field.2.string_field']) + self.assertEqual(doc.embedded_field._delta(), ({'list_field.2.string_field': 'world'}, {})) + self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.string_field': 'world'}, {})) doc.save() doc.reload() - self.assertEquals(doc.embedded_field.list_field[2].string_field, 'world') + self.assertEqual(doc.embedded_field.list_field[2].string_field, 'world') # Test multiple assignments doc.embedded_field.list_field[2].string_field = 'hello world' doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] - self.assertEquals(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEquals(doc.embedded_field._delta(), ({ + self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) + self.assertEqual(doc.embedded_field._delta(), ({ 'list_field': ['1', 2, { '_types': ['Embedded'], '_cls': 'Embedded', @@ -424,7 +424,7 @@ class DynamicDocTest(unittest.TestCase): 'int_field': 1, 'list_field': ['1', 2, {'hello': 'world'}], 'dict_field': {'hello': 'world'}}]}, {})) - self.assertEquals(doc._delta(), ({ + self.assertEqual(doc._delta(), ({ 'embedded_field.list_field': ['1', 2, { '_types': ['Embedded'], '_cls': 'Embedded', @@ -435,32 +435,32 @@ class DynamicDocTest(unittest.TestCase): ]}, {})) doc.save() doc.reload() - self.assertEquals(doc.embedded_field.list_field[2].string_field, 'hello world') + self.assertEqual(doc.embedded_field.list_field[2].string_field, 'hello world') # Test list native methods doc.embedded_field.list_field[2].list_field.pop(0) - self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}]}, {})) + self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}]}, {})) doc.save() doc.reload() doc.embedded_field.list_field[2].list_field.append(1) - self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}, 1]}, {})) + self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}, 1]}, {})) doc.save() doc.reload() - self.assertEquals(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) + self.assertEqual(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) - doc.embedded_field.list_field[2].list_field.sort() + doc.embedded_field.list_field[2].list_field.sort(key=str)# use str as a key to allow comparing uncomperable types doc.save() doc.reload() - self.assertEquals(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) + self.assertEqual(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) del(doc.embedded_field.list_field[2].list_field[2]['hello']) - self.assertEquals(doc._delta(), ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) + self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) doc.save() doc.reload() del(doc.embedded_field.list_field[2].list_field) - self.assertEquals(doc._delta(), ({}, {'embedded_field.list_field.2.list_field': 1})) + self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field.2.list_field': 1})) doc.save() doc.reload() @@ -470,8 +470,8 @@ class DynamicDocTest(unittest.TestCase): doc.reload() doc.dict_field['embedded'].string_field = 'Hello World' - self.assertEquals(doc._get_changed_fields(), ['dict_field.embedded.string_field']) - self.assertEquals(doc._delta(), ({'dict_field.embedded.string_field': 'Hello World'}, {})) + self.assertEqual(doc._get_changed_fields(), ['dict_field.embedded.string_field']) + self.assertEqual(doc._delta(), ({'dict_field.embedded.string_field': 'Hello World'}, {})) def test_indexes(self): """Ensure that indexes are used when meta[indexes] is specified. diff --git a/tests/test_fields.py b/tests/test_fields.py index c4013c1..dca4f21 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -2,20 +2,21 @@ import datetime import os import unittest import uuid -import StringIO import tempfile -import gridfs -from bson import Binary from decimal import Decimal +from bson import Binary +import gridfs + +from nose.plugins.skip import SkipTest from mongoengine import * from mongoengine.connection import get_db from mongoengine.base import _document_registry, NotRegistered +from mongoengine.python3_support import PY3, b, StringIO, bin_type TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') - class FieldTest(unittest.TestCase): def setUp(self): @@ -124,7 +125,7 @@ class FieldTest(unittest.TestCase): self.assertEqual(ret.int_fld, None) self.assertEqual(ret.flt_fld, None) # Return current time if retrived value is None. - self.assert_(isinstance(ret.comp_dt_fld, datetime.datetime)) + self.assertTrue(isinstance(ret.comp_dt_fld, datetime.datetime)) self.assertRaises(ValidationError, ret.validate) @@ -356,7 +357,7 @@ class FieldTest(unittest.TestCase): log.date = datetime.date.today() log.save() log.reload() - self.assertEquals(log.date.date(), datetime.date.today()) + self.assertEqual(log.date.date(), datetime.date.today()) LogEntry.drop_collection() @@ -367,8 +368,8 @@ class FieldTest(unittest.TestCase): log.date = d1 log.save() log.reload() - self.assertNotEquals(log.date, d1) - self.assertEquals(log.date, d2) + self.assertNotEqual(log.date, d1) + self.assertEqual(log.date, d2) # Post UTC - microseconds are rounded (down) nearest millisecond d1 = datetime.datetime(1970, 01, 01, 00, 00, 01, 9999) @@ -376,17 +377,19 @@ class FieldTest(unittest.TestCase): log.date = d1 log.save() log.reload() - self.assertNotEquals(log.date, d1) - self.assertEquals(log.date, d2) + self.assertNotEqual(log.date, d1) + self.assertEqual(log.date, d2) - # Pre UTC dates microseconds below 1000 are dropped - d1 = datetime.datetime(1969, 12, 31, 23, 59, 59, 999) - d2 = datetime.datetime(1969, 12, 31, 23, 59, 59) - log.date = d1 - log.save() - log.reload() - self.assertNotEquals(log.date, d1) - self.assertEquals(log.date, d2) + if not PY3: + # Pre UTC dates microseconds below 1000 are dropped + # This does not seem to be true in PY3 + d1 = datetime.datetime(1969, 12, 31, 23, 59, 59, 999) + d2 = datetime.datetime(1969, 12, 31, 23, 59, 59) + log.date = d1 + log.save() + log.reload() + self.assertNotEqual(log.date, d1) + self.assertEqual(log.date, d2) LogEntry.drop_collection() @@ -405,21 +408,21 @@ class FieldTest(unittest.TestCase): log.date = d1 log.save() log.reload() - self.assertEquals(log.date, d1) + self.assertEqual(log.date, d1) # Post UTC - microseconds are rounded (down) nearest millisecond - with default datetimefields d1 = datetime.datetime(1970, 01, 01, 00, 00, 01, 9999) log.date = d1 log.save() log.reload() - self.assertEquals(log.date, d1) + self.assertEqual(log.date, d1) # Pre UTC dates microseconds below 1000 are dropped - with default datetimefields d1 = datetime.datetime(1969, 12, 31, 23, 59, 59, 999) log.date = d1 log.save() log.reload() - self.assertEquals(log.date, d1) + self.assertEqual(log.date, d1) # Pre UTC microseconds above 1000 is wonky - with default datetimefields # log.date has an invalid microsecond value so I can't construct @@ -429,7 +432,7 @@ class FieldTest(unittest.TestCase): log.date = d1 log.save() log.reload() - self.assertEquals(log.date, d1) + self.assertEqual(log.date, d1) log1 = LogEntry.objects.get(date=d1) self.assertEqual(log, log1) @@ -450,7 +453,7 @@ class FieldTest(unittest.TestCase): log.save() log1 = LogEntry.objects.get(date=d1) - self.assertEquals(log, log1) + self.assertEqual(log, log1) LogEntry.drop_collection() @@ -638,13 +641,13 @@ class FieldTest(unittest.TestCase): post.info = [{'test': 3}] post.save() - self.assertEquals(BlogPost.objects.count(), 3) - self.assertEquals(BlogPost.objects.filter(info__exact='test').count(), 1) - self.assertEquals(BlogPost.objects.filter(info__0__test='test').count(), 1) + self.assertEqual(BlogPost.objects.count(), 3) + self.assertEqual(BlogPost.objects.filter(info__exact='test').count(), 1) + self.assertEqual(BlogPost.objects.filter(info__0__test='test').count(), 1) # Confirm handles non strings or non existing keys - self.assertEquals(BlogPost.objects.filter(info__0__test__exact='5').count(), 0) - self.assertEquals(BlogPost.objects.filter(info__100__test__exact='test').count(), 0) + self.assertEqual(BlogPost.objects.filter(info__0__test__exact='5').count(), 0) + self.assertEqual(BlogPost.objects.filter(info__100__test__exact='test').count(), 0) BlogPost.drop_collection() def test_list_field_passed_in_value(self): @@ -659,7 +662,7 @@ class FieldTest(unittest.TestCase): foo = Foo(bars=[]) foo.bars.append(bar) - self.assertEquals(repr(foo.bars), '[]') + self.assertEqual(repr(foo.bars), '[]') def test_list_field_strict(self): @@ -744,20 +747,20 @@ class FieldTest(unittest.TestCase): self.assertTrue(isinstance(e2.mapping[1], IntegerSetting)) # Test querying - self.assertEquals(Simple.objects.filter(mapping__1__value=42).count(), 1) - self.assertEquals(Simple.objects.filter(mapping__2__number=1).count(), 1) - self.assertEquals(Simple.objects.filter(mapping__2__complex__value=42).count(), 1) - self.assertEquals(Simple.objects.filter(mapping__2__list__0__value=42).count(), 1) - self.assertEquals(Simple.objects.filter(mapping__2__list__1__value='foo').count(), 1) + self.assertEqual(Simple.objects.filter(mapping__1__value=42).count(), 1) + self.assertEqual(Simple.objects.filter(mapping__2__number=1).count(), 1) + self.assertEqual(Simple.objects.filter(mapping__2__complex__value=42).count(), 1) + self.assertEqual(Simple.objects.filter(mapping__2__list__0__value=42).count(), 1) + self.assertEqual(Simple.objects.filter(mapping__2__list__1__value='foo').count(), 1) # Confirm can update Simple.objects().update(set__mapping__1=IntegerSetting(value=10)) - self.assertEquals(Simple.objects.filter(mapping__1__value=10).count(), 1) + self.assertEqual(Simple.objects.filter(mapping__1__value=10).count(), 1) Simple.objects().update( set__mapping__2__list__1=StringSetting(value='Boo')) - self.assertEquals(Simple.objects.filter(mapping__2__list__1__value='foo').count(), 0) - self.assertEquals(Simple.objects.filter(mapping__2__list__1__value='Boo').count(), 1) + self.assertEqual(Simple.objects.filter(mapping__2__list__1__value='foo').count(), 0) + self.assertEqual(Simple.objects.filter(mapping__2__list__1__value='Boo').count(), 1) Simple.drop_collection() @@ -796,19 +799,19 @@ class FieldTest(unittest.TestCase): post.info = {'details': {'test': 3}} post.save() - self.assertEquals(BlogPost.objects.count(), 3) - self.assertEquals(BlogPost.objects.filter(info__title__exact='test').count(), 1) - self.assertEquals(BlogPost.objects.filter(info__details__test__exact='test').count(), 1) + self.assertEqual(BlogPost.objects.count(), 3) + self.assertEqual(BlogPost.objects.filter(info__title__exact='test').count(), 1) + self.assertEqual(BlogPost.objects.filter(info__details__test__exact='test').count(), 1) # Confirm handles non strings or non existing keys - self.assertEquals(BlogPost.objects.filter(info__details__test__exact=5).count(), 0) - self.assertEquals(BlogPost.objects.filter(info__made_up__test__exact='test').count(), 0) + self.assertEqual(BlogPost.objects.filter(info__details__test__exact=5).count(), 0) + self.assertEqual(BlogPost.objects.filter(info__made_up__test__exact='test').count(), 0) post = BlogPost.objects.create(info={'title': 'original'}) post.info.update({'title': 'updated'}) post.save() post.reload() - self.assertEquals('updated', post.info['title']) + self.assertEqual('updated', post.info['title']) BlogPost.drop_collection() @@ -861,19 +864,19 @@ class FieldTest(unittest.TestCase): self.assertTrue(isinstance(e2.mapping['someint'], IntegerSetting)) # Test querying - self.assertEquals(Simple.objects.filter(mapping__someint__value=42).count(), 1) - self.assertEquals(Simple.objects.filter(mapping__nested_dict__number=1).count(), 1) - self.assertEquals(Simple.objects.filter(mapping__nested_dict__complex__value=42).count(), 1) - self.assertEquals(Simple.objects.filter(mapping__nested_dict__list__0__value=42).count(), 1) - self.assertEquals(Simple.objects.filter(mapping__nested_dict__list__1__value='foo').count(), 1) + self.assertEqual(Simple.objects.filter(mapping__someint__value=42).count(), 1) + self.assertEqual(Simple.objects.filter(mapping__nested_dict__number=1).count(), 1) + self.assertEqual(Simple.objects.filter(mapping__nested_dict__complex__value=42).count(), 1) + self.assertEqual(Simple.objects.filter(mapping__nested_dict__list__0__value=42).count(), 1) + self.assertEqual(Simple.objects.filter(mapping__nested_dict__list__1__value='foo').count(), 1) # Confirm can update Simple.objects().update( set__mapping={"someint": IntegerSetting(value=10)}) Simple.objects().update( set__mapping__nested_dict__list__1=StringSetting(value='Boo')) - self.assertEquals(Simple.objects.filter(mapping__nested_dict__list__1__value='foo').count(), 0) - self.assertEquals(Simple.objects.filter(mapping__nested_dict__list__1__value='Boo').count(), 1) + self.assertEqual(Simple.objects.filter(mapping__nested_dict__list__1__value='foo').count(), 0) + self.assertEqual(Simple.objects.filter(mapping__nested_dict__list__1__value='Boo').count(), 1) Simple.drop_collection() @@ -1383,7 +1386,7 @@ class FieldTest(unittest.TestCase): Person.drop_collection() Person(name="Wilson Jr").save() - self.assertEquals(repr(Person.objects(city=None)), + self.assertEqual(repr(Person.objects(city=None)), "[]") @@ -1461,7 +1464,7 @@ class FieldTest(unittest.TestCase): content_type = StringField() blob = BinaryField() - BLOB = '\xe6\x00\xc4\xff\x07' + BLOB = b('\xe6\x00\xc4\xff\x07') MIME_TYPE = 'application/octet-stream' Attachment.drop_collection() @@ -1471,7 +1474,7 @@ class FieldTest(unittest.TestCase): attachment_1 = Attachment.objects().first() self.assertEqual(MIME_TYPE, attachment_1.content_type) - self.assertEqual(BLOB, str(attachment_1.blob)) + self.assertEqual(BLOB, bin_type(attachment_1.blob)) Attachment.drop_collection() @@ -1498,12 +1501,12 @@ class FieldTest(unittest.TestCase): attachment_required = AttachmentRequired() self.assertRaises(ValidationError, attachment_required.validate) - attachment_required.blob = Binary('\xe6\x00\xc4\xff\x07') + attachment_required.blob = Binary(b('\xe6\x00\xc4\xff\x07')) attachment_required.validate() - attachment_size_limit = AttachmentSizeLimit(blob='\xe6\x00\xc4\xff\x07') + attachment_size_limit = AttachmentSizeLimit(blob=b('\xe6\x00\xc4\xff\x07')) self.assertRaises(ValidationError, attachment_size_limit.validate) - attachment_size_limit.blob = '\xe6\x00\xc4\xff' + attachment_size_limit.blob = b('\xe6\x00\xc4\xff') attachment_size_limit.validate() Attachment.drop_collection() @@ -1630,8 +1633,8 @@ class FieldTest(unittest.TestCase): class SetFile(Document): the_file = FileField() - text = 'Hello, World!' - more_text = 'Foo Bar' + text = b('Hello, World!') + more_text = b('Foo Bar') content_type = 'text/plain' PutFile.drop_collection() @@ -1644,14 +1647,14 @@ class FieldTest(unittest.TestCase): putfile.validate() result = PutFile.objects.first() self.assertTrue(putfile == result) - self.assertEquals(result.the_file.read(), text) - self.assertEquals(result.the_file.content_type, content_type) + self.assertEqual(result.the_file.read(), text) + self.assertEqual(result.the_file.content_type, content_type) result.the_file.delete() # Remove file from GridFS PutFile.objects.delete() # Ensure file-like objects are stored putfile = PutFile() - putstring = StringIO.StringIO() + putstring = StringIO() putstring.write(text) putstring.seek(0) putfile.the_file.put(putstring, content_type=content_type) @@ -1659,8 +1662,8 @@ class FieldTest(unittest.TestCase): putfile.validate() result = PutFile.objects.first() self.assertTrue(putfile == result) - self.assertEquals(result.the_file.read(), text) - self.assertEquals(result.the_file.content_type, content_type) + self.assertEqual(result.the_file.read(), text) + self.assertEqual(result.the_file.content_type, content_type) result.the_file.delete() streamfile = StreamFile() @@ -1675,11 +1678,11 @@ class FieldTest(unittest.TestCase): self.assertEquals(result.the_file.read(), text + more_text) self.assertEquals(result.the_file.content_type, content_type) result.the_file.seek(0) - self.assertEquals(result.the_file.tell(), 0) - self.assertEquals(result.the_file.read(len(text)), text) - self.assertEquals(result.the_file.tell(), len(text)) - self.assertEquals(result.the_file.read(len(more_text)), more_text) - self.assertEquals(result.the_file.tell(), len(text + more_text)) + self.assertEqual(result.the_file.tell(), 0) + self.assertEqual(result.the_file.read(len(text)), text) + self.assertEqual(result.the_file.tell(), len(text)) + self.assertEqual(result.the_file.read(len(more_text)), more_text) + self.assertEqual(result.the_file.tell(), len(text + more_text)) result.the_file.delete() # Ensure deleted file returns None @@ -1691,7 +1694,7 @@ class FieldTest(unittest.TestCase): setfile.validate() result = SetFile.objects.first() self.assertTrue(setfile == result) - self.assertEquals(result.the_file.read(), text) + self.assertEqual(result.the_file.read(), text) # Try replacing file with new one result.the_file.replace(more_text) @@ -1699,7 +1702,7 @@ class FieldTest(unittest.TestCase): result.validate() result = SetFile.objects.first() self.assertTrue(setfile == result) - self.assertEquals(result.the_file.read(), more_text) + self.assertEqual(result.the_file.read(), more_text) result.the_file.delete() PutFile.drop_collection() @@ -1720,7 +1723,7 @@ class FieldTest(unittest.TestCase): GridDocument.drop_collection() with tempfile.TemporaryFile() as f: - f.write("Hello World!") + f.write(b("Hello World!")) f.flush() # Test without default @@ -1731,28 +1734,28 @@ class FieldTest(unittest.TestCase): doc_b = GridDocument.objects.with_id(doc_a.id) doc_b.the_file.replace(f, filename='doc_b') doc_b.save() - self.assertNotEquals(doc_b.the_file.grid_id, None) + self.assertNotEqual(doc_b.the_file.grid_id, None) # Test it matches doc_c = GridDocument.objects.with_id(doc_b.id) - self.assertEquals(doc_b.the_file.grid_id, doc_c.the_file.grid_id) + self.assertEqual(doc_b.the_file.grid_id, doc_c.the_file.grid_id) # Test with default - doc_d = GridDocument(the_file='') + doc_d = GridDocument(the_file=b('')) doc_d.save() doc_e = GridDocument.objects.with_id(doc_d.id) - self.assertEquals(doc_d.the_file.grid_id, doc_e.the_file.grid_id) + self.assertEqual(doc_d.the_file.grid_id, doc_e.the_file.grid_id) doc_e.the_file.replace(f, filename='doc_e') doc_e.save() doc_f = GridDocument.objects.with_id(doc_e.id) - self.assertEquals(doc_e.the_file.grid_id, doc_f.the_file.grid_id) + self.assertEqual(doc_e.the_file.grid_id, doc_f.the_file.grid_id) db = GridDocument._get_db() grid_fs = gridfs.GridFS(db) - self.assertEquals(['doc_b', 'doc_e'], grid_fs.list()) + self.assertEqual(['doc_b', 'doc_e'], grid_fs.list()) def test_file_uniqueness(self): """Ensure that each instance of a FileField is unique @@ -1764,7 +1767,7 @@ class FieldTest(unittest.TestCase): # First instance test_file = TestFile() test_file.name = "Hello, World!" - test_file.the_file.put('Hello, World!') + test_file.the_file.put(b('Hello, World!')) test_file.save() # Second instance @@ -1784,7 +1787,7 @@ class FieldTest(unittest.TestCase): test_file = TestFile() self.assertFalse(bool(test_file.the_file)) - test_file.the_file = 'Hello, World!' + test_file.the_file = b('Hello, World!') test_file.the_file.content_type = 'text/plain' test_file.save() self.assertTrue(bool(test_file.the_file)) @@ -1800,6 +1803,8 @@ class FieldTest(unittest.TestCase): self.assertFalse(test_file.the_file in [{"test": 1}]) def test_image_field(self): + if PY3: + raise SkipTest('PIL does not have Python 3 support') class TestImage(Document): image = ImageField() @@ -1821,6 +1826,8 @@ class FieldTest(unittest.TestCase): t.image.delete() def test_image_field_resize(self): + if PY3: + raise SkipTest('PIL does not have Python 3 support') class TestImage(Document): image = ImageField(size=(185, 37)) @@ -1842,6 +1849,8 @@ class FieldTest(unittest.TestCase): t.image.delete() def test_image_field_thumbnail(self): + if PY3: + raise SkipTest('PIL does not have Python 3 support') class TestImage(Document): image = ImageField(thumbnail_size=(92, 18)) @@ -1860,7 +1869,6 @@ class FieldTest(unittest.TestCase): t.image.delete() - def test_file_multidb(self): register_connection('test_files', 'test_files') class TestFile(Document): @@ -1877,7 +1885,7 @@ class FieldTest(unittest.TestCase): # First instance test_file = TestFile() test_file.name = "Hello, World!" - test_file.the_file.put('Hello, World!', + test_file.the_file.put(b('Hello, World!'), name="hello.txt") test_file.save() @@ -1885,8 +1893,8 @@ class FieldTest(unittest.TestCase): self.assertEquals(data.get('name'), 'hello.txt') test_file = TestFile.objects.first() - self.assertEquals(test_file.the_file.read(), - 'Hello, World!') + self.assertEqual(test_file.the_file.read(), + b('Hello, World!')) def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. @@ -2146,28 +2154,28 @@ class FieldTest(unittest.TestCase): post = Post(title='hello world') post.comments.append(Comment(content='hello', author=bob)) post.comments.append(Comment(author=bob)) - + + self.assertRaises(ValidationError, post.validate) try: post.validate() - except ValidationError, error: - pass + except ValidationError, error: + # ValidationError.errors property + self.assertTrue(hasattr(error, 'errors')) + self.assertTrue(isinstance(error.errors, dict)) + self.assertTrue('comments' in error.errors) + self.assertTrue(1 in error.errors['comments']) + self.assertTrue(isinstance(error.errors['comments'][1]['content'], + ValidationError)) - # ValidationError.errors property - self.assertTrue(hasattr(error, 'errors')) - self.assertTrue(isinstance(error.errors, dict)) - self.assertTrue('comments' in error.errors) - self.assertTrue(1 in error.errors['comments']) - self.assertTrue(isinstance(error.errors['comments'][1]['content'], - ValidationError)) + # ValidationError.schema property + error_dict = error.to_dict() + self.assertTrue(isinstance(error_dict, dict)) + self.assertTrue('comments' in error_dict) + self.assertTrue(1 in error_dict['comments']) + self.assertTrue('content' in error_dict['comments'][1]) + self.assertEqual(error_dict['comments'][1]['content'], + u'Field is required') - # ValidationError.schema property - error_dict = error.to_dict() - self.assertTrue(isinstance(error_dict, dict)) - self.assertTrue('comments' in error_dict) - self.assertTrue(1 in error_dict['comments']) - self.assertTrue('content' in error_dict['comments'][1]) - self.assertEquals(error_dict['comments'][1]['content'], - 'Field is required') post.comments[1].content = 'here we go' post.validate() diff --git a/tests/test_queryset.py b/tests/test_queryset.py index b4ae805..1960fc6 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -1,16 +1,19 @@ # -*- coding: utf-8 -*- import unittest -import pymongo -from bson import ObjectId + from datetime import datetime, timedelta +import pymongo + +from bson import ObjectId + +from mongoengine import * +from mongoengine.connection import get_connection +from mongoengine.python3_support import PY3 +from mongoengine.tests import query_counter from mongoengine.queryset import (QuerySet, QuerySetManager, MultipleObjectsReturned, DoesNotExist, QueryFieldList) -from mongoengine import * -from mongoengine.connection import get_connection -from mongoengine.tests import query_counter - class QuerySetTest(unittest.TestCase): @@ -239,11 +242,11 @@ class QuerySetTest(unittest.TestCase): self.Person.objects.update(set__name='Ross', write_options=write_options) author = self.Person.objects.first() - self.assertEquals(author.name, 'Ross') + self.assertEqual(author.name, 'Ross') self.Person.objects.update_one(set__name='Test User', write_options=write_options) author = self.Person.objects.first() - self.assertEquals(author.name, 'Test User') + self.assertEqual(author.name, 'Test User') def test_update_update_has_a_value(self): """Test to ensure that update is passed a value to update to""" @@ -332,8 +335,8 @@ class QuerySetTest(unittest.TestCase): BlogPost.objects(comments__by="jane").update(inc__comments__S__votes=1) post = BlogPost.objects.first() - self.assertEquals(post.comments[1].by, 'jane') - self.assertEquals(post.comments[1].votes, 8) + self.assertEqual(post.comments[1].by, 'jane') + self.assertEqual(post.comments[1].votes, 8) # Currently the $ operator only applies to the first matched item in # the query @@ -346,7 +349,7 @@ class QuerySetTest(unittest.TestCase): Simple.objects(x=2).update(inc__x__S=1) simple = Simple.objects.first() - self.assertEquals(simple.x, [1, 3, 3, 2]) + self.assertEqual(simple.x, [1, 3, 3, 2]) Simple.drop_collection() # You can set multiples @@ -358,10 +361,10 @@ class QuerySetTest(unittest.TestCase): Simple.objects(x=3).update(set__x__S=0) s = Simple.objects() - self.assertEquals(s[0].x, [1, 2, 0, 4]) - self.assertEquals(s[1].x, [2, 0, 4, 5]) - self.assertEquals(s[2].x, [0, 4, 5, 6]) - self.assertEquals(s[3].x, [4, 5, 6, 7]) + self.assertEqual(s[0].x, [1, 2, 0, 4]) + self.assertEqual(s[1].x, [2, 0, 4, 5]) + self.assertEqual(s[2].x, [0, 4, 5, 6]) + self.assertEqual(s[3].x, [4, 5, 6, 7]) # Using "$unset" with an expression like this "array.$" will result in # the array item becoming None, not being removed. @@ -369,14 +372,14 @@ class QuerySetTest(unittest.TestCase): Simple(x=[1, 2, 3, 4, 3, 2, 3, 4]).save() Simple.objects(x=3).update(unset__x__S=1) simple = Simple.objects.first() - self.assertEquals(simple.x, [1, 2, None, 4, 3, 2, 3, 4]) + self.assertEqual(simple.x, [1, 2, None, 4, 3, 2, 3, 4]) # Nested updates arent supported yet.. def update_nested(): Simple.drop_collection() Simple(x=[{'test': [1, 2, 3, 4]}]).save() Simple.objects(x__test=2).update(set__x__S__test__S=3) - self.assertEquals(simple.x, [1, 2, 3, 4]) + self.assertEqual(simple.x, [1, 2, 3, 4]) self.assertRaises(OperationError, update_nested) Simple.drop_collection() @@ -406,8 +409,8 @@ class QuerySetTest(unittest.TestCase): BlogPost.objects(comments__by="joe").update(set__comments__S__votes=Vote(score=4)) post = BlogPost.objects.first() - self.assertEquals(post.comments[0].by, 'joe') - self.assertEquals(post.comments[0].votes.score, 4) + self.assertEqual(post.comments[0].by, 'joe') + self.assertEqual(post.comments[0].votes.score, 4) def test_mapfield_update(self): """Ensure that the MapField can be updated.""" @@ -561,7 +564,7 @@ class QuerySetTest(unittest.TestCase): Blog.drop_collection() blog1 = Blog(title="code", posts=[post1, post2]) obj_id = Blog.objects.insert(blog1, load_bulk=False) - self.assertEquals(obj_id.__class__.__name__, 'ObjectId') + self.assertEqual(obj_id.__class__.__name__, 'ObjectId') Blog.drop_collection() post3 = Post(comments=[comment1, comment1]) @@ -710,20 +713,20 @@ class QuerySetTest(unittest.TestCase): docs = Doc.objects.order_by('number') - self.assertEquals(docs.count(), 1000) - self.assertEquals(len(docs), 1000) + self.assertEqual(docs.count(), 1000) + self.assertEqual(len(docs), 1000) docs_string = "%s" % docs self.assertTrue("Doc: 0" in docs_string) - self.assertEquals(docs.count(), 1000) - self.assertEquals(len(docs), 1000) + self.assertEqual(docs.count(), 1000) + self.assertEqual(len(docs), 1000) # Limit and skip - self.assertEquals('[, , ]', "%s" % docs[1:4]) + self.assertEqual('[, , ]', "%s" % docs[1:4]) - self.assertEquals(docs.count(), 3) - self.assertEquals(len(docs), 3) + self.assertEqual(docs.count(), 3) + self.assertEqual(len(docs), 3) for doc in docs: self.assertEqual('.. queryset mid-iteration ..', repr(docs)) @@ -1082,27 +1085,27 @@ class QuerySetTest(unittest.TestCase): # first three numbers = Numbers.objects.fields(slice__n=3).get() - self.assertEquals(numbers.n, [0, 1, 2]) + self.assertEqual(numbers.n, [0, 1, 2]) # last three numbers = Numbers.objects.fields(slice__n=-3).get() - self.assertEquals(numbers.n, [-3, -2, -1]) + self.assertEqual(numbers.n, [-3, -2, -1]) # skip 2, limit 3 numbers = Numbers.objects.fields(slice__n=[2, 3]).get() - self.assertEquals(numbers.n, [2, 3, 4]) + self.assertEqual(numbers.n, [2, 3, 4]) # skip to fifth from last, limit 4 numbers = Numbers.objects.fields(slice__n=[-5, 4]).get() - self.assertEquals(numbers.n, [-5, -4, -3, -2]) + self.assertEqual(numbers.n, [-5, -4, -3, -2]) # skip to fifth from last, limit 10 numbers = Numbers.objects.fields(slice__n=[-5, 10]).get() - self.assertEquals(numbers.n, [-5, -4, -3, -2, -1]) + self.assertEqual(numbers.n, [-5, -4, -3, -2, -1]) # skip to fifth from last, limit 10 dict method numbers = Numbers.objects.fields(n={"$slice": [-5, 10]}).get() - self.assertEquals(numbers.n, [-5, -4, -3, -2, -1]) + self.assertEqual(numbers.n, [-5, -4, -3, -2, -1]) def test_slicing_nested_fields(self): """Ensure that query slicing an embedded array works. @@ -1122,27 +1125,27 @@ class QuerySetTest(unittest.TestCase): # first three numbers = Numbers.objects.fields(slice__embedded__n=3).get() - self.assertEquals(numbers.embedded.n, [0, 1, 2]) + self.assertEqual(numbers.embedded.n, [0, 1, 2]) # last three numbers = Numbers.objects.fields(slice__embedded__n=-3).get() - self.assertEquals(numbers.embedded.n, [-3, -2, -1]) + self.assertEqual(numbers.embedded.n, [-3, -2, -1]) # skip 2, limit 3 numbers = Numbers.objects.fields(slice__embedded__n=[2, 3]).get() - self.assertEquals(numbers.embedded.n, [2, 3, 4]) + self.assertEqual(numbers.embedded.n, [2, 3, 4]) # skip to fifth from last, limit 4 numbers = Numbers.objects.fields(slice__embedded__n=[-5, 4]).get() - self.assertEquals(numbers.embedded.n, [-5, -4, -3, -2]) + self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2]) # skip to fifth from last, limit 10 numbers = Numbers.objects.fields(slice__embedded__n=[-5, 10]).get() - self.assertEquals(numbers.embedded.n, [-5, -4, -3, -2, -1]) + self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2, -1]) # skip to fifth from last, limit 10 dict method numbers = Numbers.objects.fields(embedded__n={"$slice": [-5, 10]}).get() - self.assertEquals(numbers.embedded.n, [-5, -4, -3, -2, -1]) + self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2, -1]) def test_find_embedded(self): """Ensure that an embedded document is properly returned from a query. @@ -1383,7 +1386,7 @@ class QuerySetTest(unittest.TestCase): # Test template style code = "{{~comments.content}}" sub_code = BlogPost.objects._sub_js_fields(code) - self.assertEquals("cmnts.body", sub_code) + self.assertEqual("cmnts.body", sub_code) BlogPost.drop_collection() @@ -1446,13 +1449,13 @@ class QuerySetTest(unittest.TestCase): child_child.save() tree_size = 1 + num_children + (num_children * num_children) - self.assertEquals(tree_size, Category.objects.count()) - self.assertEquals(num_children, Category.objects(parent=base).count()) + self.assertEqual(tree_size, Category.objects.count()) + self.assertEqual(num_children, Category.objects(parent=base).count()) # The delete should effectively wipe out the Category collection # without resulting in infinite parent-child cascade recursion base.delete() - self.assertEquals(0, Category.objects.count()) + self.assertEqual(0, Category.objects.count()) def test_reverse_delete_rule_nullify(self): """Ensure nullification of references to deleted documents. @@ -1713,7 +1716,7 @@ class QuerySetTest(unittest.TestCase): BlogPost.objects(slug="test-2").update_one(set__tags__0__name="python") post.reload() - self.assertEquals(post.tags[0].name, 'python') + self.assertEqual(post.tags[0].name, 'python') BlogPost.objects(slug="test-2").update_one(pop__tags=-1) post.reload() @@ -1740,7 +1743,7 @@ class QuerySetTest(unittest.TestCase): set__authors__S=Author(name="Ross")) message = message.reload() - self.assertEquals(message.authors[0].name, "Ross") + self.assertEqual(message.authors[0].name, "Ross") Message.objects(authors__name="Ross").update_one( set__authors=[Author(name="Harry"), @@ -1748,9 +1751,9 @@ class QuerySetTest(unittest.TestCase): Author(name="Adam")]) message = message.reload() - self.assertEquals(message.authors[0].name, "Harry") - self.assertEquals(message.authors[1].name, "Ross") - self.assertEquals(message.authors[2].name, "Adam") + self.assertEqual(message.authors[0].name, "Harry") + self.assertEqual(message.authors[1].name, "Ross") + self.assertEqual(message.authors[2].name, "Adam") def test_order_by(self): """Ensure that QuerySets may be ordered. @@ -1830,10 +1833,10 @@ class QuerySetTest(unittest.TestCase): results = list(results) self.assertEqual(len(results), 4) - music = filter(lambda r: r.key == "music", results)[0] + music = list(filter(lambda r: r.key == "music", results))[0] self.assertEqual(music.value, 2) - film = filter(lambda r: r.key == "film", results)[0] + film = list(filter(lambda r: r.key == "film", results))[0] self.assertEqual(film.value, 3) BlogPost.drop_collection() @@ -2133,15 +2136,15 @@ class QuerySetTest(unittest.TestCase): Person(name="Wilson Jr").save() freq = Person.objects.item_frequencies('city') - self.assertEquals(freq, {'CRB': 1.0, None: 1.0}) + self.assertEqual(freq, {'CRB': 1.0, None: 1.0}) freq = Person.objects.item_frequencies('city', normalize=True) - self.assertEquals(freq, {'CRB': 0.5, None: 0.5}) + self.assertEqual(freq, {'CRB': 0.5, None: 0.5}) freq = Person.objects.item_frequencies('city', map_reduce=True) - self.assertEquals(freq, {'CRB': 1.0, None: 1.0}) + self.assertEqual(freq, {'CRB': 1.0, None: 1.0}) freq = Person.objects.item_frequencies('city', normalize=True, map_reduce=True) - self.assertEquals(freq, {'CRB': 0.5, None: 0.5}) + self.assertEqual(freq, {'CRB': 0.5, None: 0.5}) def test_item_frequencies_with_null_embedded(self): class Data(EmbeddedDocument): @@ -2166,10 +2169,10 @@ class QuerySetTest(unittest.TestCase): p.save() ot = Person.objects.item_frequencies('extra.tag', map_reduce=False) - self.assertEquals(ot, {None: 1.0, u'friend': 1.0}) + self.assertEqual(ot, {None: 1.0, u'friend': 1.0}) ot = Person.objects.item_frequencies('extra.tag', map_reduce=True) - self.assertEquals(ot, {None: 1.0, u'friend': 1.0}) + self.assertEqual(ot, {None: 1.0, u'friend': 1.0}) def test_item_frequencies_with_0_values(self): class Test(Document): @@ -2181,9 +2184,9 @@ class QuerySetTest(unittest.TestCase): t.save() ot = Test.objects.item_frequencies('val', map_reduce=True) - self.assertEquals(ot, {0: 1}) + self.assertEqual(ot, {0: 1}) ot = Test.objects.item_frequencies('val', map_reduce=False) - self.assertEquals(ot, {0: 1}) + self.assertEqual(ot, {0: 1}) def test_item_frequencies_with_False_values(self): class Test(Document): @@ -2195,9 +2198,9 @@ class QuerySetTest(unittest.TestCase): t.save() ot = Test.objects.item_frequencies('val', map_reduce=True) - self.assertEquals(ot, {False: 1}) + self.assertEqual(ot, {False: 1}) ot = Test.objects.item_frequencies('val', map_reduce=False) - self.assertEquals(ot, {False: 1}) + self.assertEqual(ot, {False: 1}) def test_item_frequencies_normalize(self): class Test(Document): @@ -2212,10 +2215,10 @@ class QuerySetTest(unittest.TestCase): Test(val=2).save() freqs = Test.objects.item_frequencies('val', map_reduce=False, normalize=True) - self.assertEquals(freqs, {1: 50.0/70, 2: 20.0/70}) + self.assertEqual(freqs, {1: 50.0/70, 2: 20.0/70}) freqs = Test.objects.item_frequencies('val', map_reduce=True, normalize=True) - self.assertEquals(freqs, {1: 50.0/70, 2: 20.0/70}) + self.assertEqual(freqs, {1: 50.0/70, 2: 20.0/70}) def test_average(self): """Ensure that field can be averaged correctly. @@ -2275,7 +2278,7 @@ class QuerySetTest(unittest.TestCase): foo = Foo(bar=bar) foo.save() - self.assertEquals(Foo.objects.distinct("bar"), [bar]) + self.assertEqual(Foo.objects.distinct("bar"), [bar]) def test_custom_manager(self): """Ensure that custom QuerySetManager instances work as expected. @@ -2735,8 +2738,8 @@ class QuerySetTest(unittest.TestCase): Post().save() Post(is_published=True).save() - self.assertEquals(Post.objects.count(), 2) - self.assertEquals(Post.published.count(), 1) + self.assertEqual(Post.objects.count(), 2) + self.assertEqual(Post.published.count(), 1) Post.drop_collection() @@ -2902,10 +2905,10 @@ class QuerySetTest(unittest.TestCase): Number(n=3).save() numbers = [n.n for n in Number.objects.order_by('-n')] - self.assertEquals([3, 2, 1], numbers) + self.assertEqual([3, 2, 1], numbers) numbers = [n.n for n in Number.objects.order_by('+n')] - self.assertEquals([1, 2, 3], numbers) + self.assertEqual([1, 2, 3], numbers) Number.drop_collection() @@ -3227,15 +3230,22 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(len(self.Person.objects.scalar('name')), 55) self.assertEqual("A0", "%s" % self.Person.objects.order_by('name').scalar('name').first()) self.assertEqual("A0", "%s" % self.Person.objects.scalar('name').order_by('name')[0]) - self.assertEqual("[u'A1', u'A2']", "%s" % self.Person.objects.order_by('age').scalar('name')[1:3]) - self.assertEqual("[u'A51', u'A52']", "%s" % self.Person.objects.order_by('age').scalar('name')[51:53]) + if PY3: + self.assertEqual("['A1', 'A2']", "%s" % self.Person.objects.order_by('age').scalar('name')[1:3]) + self.assertEqual("['A51', 'A52']", "%s" % self.Person.objects.order_by('age').scalar('name')[51:53]) + else: + self.assertEqual("[u'A1', u'A2']", "%s" % self.Person.objects.order_by('age').scalar('name')[1:3]) + self.assertEqual("[u'A51', u'A52']", "%s" % self.Person.objects.order_by('age').scalar('name')[51:53]) # with_id and in_bulk person = self.Person.objects.order_by('name').first() self.assertEqual("A0", "%s" % self.Person.objects.scalar('name').with_id(person.id)) pks = self.Person.objects.order_by('age').scalar('pk')[1:3] - self.assertEqual("[u'A1', u'A2']", "%s" % sorted(self.Person.objects.scalar('name').in_bulk(list(pks)).values())) + if PY3: + self.assertEqual("['A1', 'A2']", "%s" % sorted(self.Person.objects.scalar('name').in_bulk(list(pks)).values())) + else: + self.assertEqual("[u'A1', u'A2']", "%s" % sorted(self.Person.objects.scalar('name').in_bulk(list(pks)).values())) class QTest(unittest.TestCase): diff --git a/tests/test_replicaset_connection.py b/tests/test_replicaset_connection.py index 2297814..3118c5a 100644 --- a/tests/test_replicaset_connection.py +++ b/tests/test_replicaset_connection.py @@ -1,4 +1,5 @@ import unittest + import pymongo from pymongo import ReadPreference, ReplicaSetConnection @@ -26,7 +27,7 @@ class ConnectionTest(unittest.TestCase): if not isinstance(conn, ReplicaSetConnection): return - self.assertEquals(conn.read_preference, ReadPreference.SECONDARY_ONLY) + self.assertEqual(conn.read_preference, ReadPreference.SECONDARY_ONLY) if __name__ == '__main__': unittest.main() diff --git a/tests/test_signals.py b/tests/test_signals.py index 4dc683e..d119924 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -212,9 +212,9 @@ class SignalTests(unittest.TestCase): # The output of this signal is not entirely deterministic. The reloaded # object will have an object ID. Hence, we only check part of the output - self.assertEquals(signal_output[3], + self.assertEqual(signal_output[3], "pre_bulk_insert signal, []") - self.assertEquals(signal_output[-2:], + self.assertEqual(signal_output[-2:], ["post_bulk_insert signal, []", "Is loaded",]) From c1619d2a62c39c364e74aca47bc855db87ee9ece Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 2 Aug 2012 08:47:38 +0100 Subject: [PATCH 0707/1279] Fixed image_field attributes regression --- mongoengine/fields.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index a2947c1..da59fb1 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1,17 +1,16 @@ import datetime import decimal import re -import sys import time import uuid import warnings - +import itertools from operator import itemgetter import gridfs from bson import Binary, DBRef, SON, ObjectId -from mongoengine.python3_support import (PY3, b, bin_type, +from mongoengine.python3_support import (PY3, bin_type, txt_type, str_types, StringIO) from base import (BaseField, ComplexBaseField, ObjectIdField, ValidationError, get_document, BaseDocument) @@ -905,12 +904,12 @@ class GridFSProxy(object): def __eq__(self, other): if isinstance(other, GridFSProxy): - return ((self.grid_id == other.grid_id) and - (self.collection_name == other.collection_name) and + return ((self.grid_id == other.grid_id) and + (self.collection_name == other.collection_name) and (self.db_alias == other.db_alias)) else: return False - + @property def fs(self): if not self._fs: @@ -1022,7 +1021,7 @@ class FileField(BaseField): def __set__(self, instance, value): key = self.name - if ((hasattr(value, 'read') and not + if ((hasattr(value, 'read') and not isinstance(value, GridFSProxy)) or isinstance(value, str_types)): # using "FileField() = file/string" notation grid_file = instance._data.get(self.name) @@ -1212,11 +1211,15 @@ class ImageField(FileField): params_size = ('width', 'height', 'force') extra_args = dict(size=size, thumbnail_size=thumbnail_size) for att_name, att in extra_args.items(): - if att and (isinstance(att, tuple) or isinstance(att, list)): - setattr(self, att_name, dict( - zip(params_size, att))) - else: - setattr(self, att_name, None) + value = None + if isinstance(att, (tuple, list)): + if PY3: + value = dict(itertools.zip_longest(params_size, att, + fillvalue=None)) + else: + value = dict(map(None, params_size, att)) + + setattr(self, att_name, value) super(ImageField, self).__init__( collection_name=collection_name, From 73635033bd1d30fe5f4810b4720af8da8501f2fd Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 2 Aug 2012 09:10:42 +0100 Subject: [PATCH 0708/1279] Updated travis config --- .travis.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e38053f..0c64c73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,15 @@ # http://travis-ci.org/#!/MongoEngine/mongoengine language: python python: + - 2.5 - 2.6 - 2.7 + - 3.2 + - 3.3 install: - - sudo apt-get install zlib1g zlib1g-dev - - sudo ln -s /usr/lib/i386-linux-gnu/libz.so /usr/lib/ - - pip install PIL --use-mirrors ; true + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install zlib1g zlib1g-dev; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo ln -s /usr/lib/i386-linux-gnu/libz.so /usr/lib/; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install PIL --use-mirrors ; true; fi - python setup.py install script: - python setup.py test \ No newline at end of file From 95dae487784ad4153c96fbe7240ed8cdfc1537eb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 2 Aug 2012 09:20:41 +0100 Subject: [PATCH 0709/1279] Fixing build --- .travis.yml | 1 - tests/test_document.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0c64c73..634c470 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ # http://travis-ci.org/#!/MongoEngine/mongoengine language: python python: - - 2.5 - 2.6 - 2.7 - 3.2 diff --git a/tests/test_document.py b/tests/test_document.py index 8eaacee..a63c996 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1330,7 +1330,7 @@ class DocumentTest(unittest.TestCase): doc = Doc.objects.first() doc.validate() - self.assertEqual([None, 'e'], doc._data.keys()) + self.assertEqual([None, 'e'], list(doc._data.keys())) def test_save(self): """Ensure that a document may be saved in the database. @@ -1954,7 +1954,7 @@ class DocumentTest(unittest.TestCase): doc = doc.reload(10) self.assertEqual(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) - doc.embedded_field.list_field[2].list_field.sort(key=str) + doc.embedded_field.list_field[2].list_field.sort(key=str) doc.save() doc = doc.reload(10) self.assertEqual(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) @@ -2189,7 +2189,7 @@ class DocumentTest(unittest.TestCase): doc = doc.reload(10) self.assertEqual(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) - doc.embedded_field.list_field[2].list_field.sort(key=str) + doc.embedded_field.list_field[2].list_field.sort(key=str) doc.save() doc = doc.reload(10) self.assertEqual(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) From 97b37f75d3416a67e7872439fa0e150c165cecc1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 2 Aug 2012 09:22:12 +0100 Subject: [PATCH 0710/1279] Travis - Not sure python 3.3 support is there yet --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 634c470..47ad068 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ python: - 2.6 - 2.7 - 3.2 - - 3.3 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install zlib1g zlib1g-dev; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo ln -s /usr/lib/i386-linux-gnu/libz.so /usr/lib/; fi From c95652d6a8d5ba63ee907a22bbdbdb82d3892b98 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 2 Aug 2012 09:23:15 +0100 Subject: [PATCH 0711/1279] Updated AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index fe8be76..3538312 100644 --- a/AUTHORS +++ b/AUTHORS @@ -8,6 +8,7 @@ Florian Schlachter Steve Challis Wilson Júnior Dan Crosta https://github.com/dcrosta +Laine Herron https://github.com/LaineHerron CONTRIBUTORS From f45d4d781deafa0becb38539f81ba4c86c074048 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 2 Aug 2012 09:26:33 +0100 Subject: [PATCH 0712/1279] Updated test for py3.2 --- tests/test_document.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_document.py b/tests/test_document.py index a63c996..2b761bc 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1330,7 +1330,10 @@ class DocumentTest(unittest.TestCase): doc = Doc.objects.first() doc.validate() - self.assertEqual([None, 'e'], list(doc._data.keys())) + keys = doc._data.keys() + self.assertEqual(2, len(keys)) + self.assertTrue(None in keys) + self.assertTrue('e' in keys) def test_save(self): """Ensure that a document may be saved in the database. From 8ac9e6dc195d26a414dd5850790e1e5861a1c9b5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 2 Aug 2012 14:11:02 +0100 Subject: [PATCH 0713/1279] Updated the documents --- docs/guide/querying.rst | 9 +++++++-- mongoengine/queryset.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index a9567e2..1449801 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -232,7 +232,7 @@ custom manager methods as you like:: BlogPost(title='test1', published=False).save() BlogPost(title='test2', published=True).save() assert len(BlogPost.objects) == 2 - assert len(BlogPost.live_posts) == 1 + assert len(BlogPost.live_posts()) == 1 Custom QuerySets ================ @@ -243,11 +243,16 @@ a document, set ``queryset_class`` to the custom class in a :class:`~mongoengine.Document`\ s ``meta`` dictionary:: class AwesomerQuerySet(QuerySet): - pass + + def get_awesome(self): + return self.filter(awesome=True) class Page(Document): meta = {'queryset_class': AwesomerQuerySet} + # To call: + Page.objects.get_awesome() + .. versionadded:: 0.4 Aggregation diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b7453c6..ef0ed04 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1861,6 +1861,17 @@ class QuerySet(object): class QuerySetManager(object): + """ + The default QuerySet Manager. + + Custom QuerySet Manager functions can extend this class and users can + add extra queryset functionality. Any custom manager methods must accept a + :class:`~mongoengine.Document` class as its first argument, and a + :class:`~mongoengine.queryset.QuerySet` as its second argument. + + The method function should return a :class:`~mongoengine.queryset.QuerySet` + , probably the same one that was passed in, but modified in some way. + """ get_queryset = None From dd023edc0fd3f8dcf271f87043f9cbb593993a72 Mon Sep 17 00:00:00 2001 From: Laine Date: Thu, 2 Aug 2012 15:30:21 -0700 Subject: [PATCH 0714/1279] made compatable with python 2.5 --- mongoengine/base.py | 8 ++++++++ mongoengine/fields.py | 4 ++-- mongoengine/queryset.py | 21 +++++++++++++++++---- tests/test_dereference.py | 1 + tests/test_django.py | 4 ++-- tests/test_document.py | 31 ++++++++++++++++++++++--------- tests/test_fields.py | 1 + tests/test_queryset.py | 4 ++-- 8 files changed, 55 insertions(+), 19 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index d4a7b32..811a45f 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -877,6 +877,10 @@ class BaseDocument(object): if not is_list and '_cls' in value: cls = get_document(value['_cls']) + # Make keys str instead of unicode + for key,val in value.items(): + del value[key] + value[str(key)] = val value = cls(**value) value._dynamic = True value._changed_fields = [] @@ -998,6 +1002,10 @@ class BaseDocument(object): raise InvalidDocumentError(""" Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, errors)) + # Make all keys str instead of unicode + for key,val in data.items(): + del data[key] + data[str(key)] = val obj = cls(**data) obj._changed_fields = changed_fields diff --git a/mongoengine/fields.py b/mongoengine/fields.py index a2947c1..fc6e8a5 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1267,8 +1267,8 @@ class SequenceField(IntField): """ Generate and Increment the counter """ - sequence_id = "{0}.{1}".format(self.owner_document._get_collection_name(), - self.name) + sequence_id = "%s.%s"%(self.owner_document._get_collection_name(), + self.name) collection = get_db(alias = self.db_alias )[self.collection_name] counter = collection.find_and_modify(query={"_id": sequence_id}, update={"$inc": {"next": 1}}, diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 1067e32..cef9f80 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -4,6 +4,7 @@ import copy import itertools import operator import functools +import sys from functools import partial import pymongo @@ -14,6 +15,18 @@ from mongoengine import signals __all__ = ['queryset_manager', 'Q', 'InvalidQueryError', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL'] +if sys.version_info < (2,6,0): + def product(*args, **kwds): + pools = map(tuple, args) * kwds.get('repeat', 1) + result = [[]] + for pool in pools: + result = [x+[y] for x in result for y in pool] + for prod in result: + yield tuple(prod) +else: + from itertools import product + from functools import reduce + # The maximum number of items to display in a QuerySet.__repr__ REPR_OUTPUT_SIZE = 20 @@ -120,13 +133,13 @@ class QueryTreeTransformerVisitor(QNodeVisitor): # the necessary parts. Then for each $or part, create a new query # that ANDs the necessary part with the $or part. clauses = [] - for or_group in itertools.product(*or_groups): - q_object = functools.reduce(lambda a, b: a & b, and_parts, Q()) - q_object = functools.reduce(lambda a, b: a & b, or_group, q_object) + for or_group in product(*or_groups): + q_object = reduce(lambda a, b: a & b, and_parts, Q()) + q_object = reduce(lambda a, b: a & b, or_group, q_object) clauses.append(q_object) # Finally, $or the generated clauses in to one query. Each of the # clauses is sufficient for the query to succeed. - return functools.reduce(lambda a, b: a | b, clauses, Q()) + return reduce(lambda a, b: a | b, clauses, Q()) if combination.operation == combination.OR: children = [] diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 75daf6a..cc2ffba 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1,3 +1,4 @@ +from __future__ import with_statement import unittest from mongoengine import * diff --git a/tests/test_django.py b/tests/test_django.py index ed21f27..32a3fe1 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +from __future__ import with_statement import unittest from nose.plugins.skip import SkipTest from mongoengine.python3_support import PY3 @@ -16,7 +16,7 @@ try: from django.contrib.sessions.tests import SessionTestsMixin from mongoengine.django.sessions import SessionStore, MongoSession -except Exception as err: +except Exception, err: if PY3: SessionTestsMixin = type #dummy value so no error SessionStore = None #dummy value so no error diff --git a/tests/test_document.py b/tests/test_document.py index 8eaacee..8e6791b 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1,10 +1,13 @@ +from __future__ import with_statement import os import pickle import pymongo import bson import unittest import warnings +import sys +from nose.plugins.skip import SkipTest from datetime import datetime from tests.fixtures import Base, Mixin, PickleEmbedded, PickleTest @@ -37,6 +40,8 @@ class DocumentTest(unittest.TestCase): def test_future_warning(self): """Add FutureWarning for future allow_inhertiance default change. """ + if (sys.version_info < (2,6,0)): + raise SkipTest('catch warnings as errors only available in 2.6+') with warnings.catch_warnings(record=True) as errors: @@ -131,18 +136,21 @@ class DocumentTest(unittest.TestCase): meta = {'collection': 'wibble'} self.assertEqual('wibble', InheritedAbstractNamingTest._get_collection_name()) - with warnings.catch_warnings(record=True) as w: - # Cause all warnings to always be triggered. - warnings.simplefilter("always") - class NonAbstractBase(Document): - pass + # catch_warnings only available in python 2.6+ + if sys.version_info > (2,6,0): + with warnings.catch_warnings(record=True) as w: + # Cause all warnings to always be triggered. + warnings.simplefilter("always") - class InheritedDocumentFailTest(NonAbstractBase): - meta = {'collection': 'fail'} + class NonAbstractBase(Document): + pass - self.assertTrue(issubclass(w[0].category, SyntaxWarning)) - self.assertEqual('non_abstract_base', InheritedDocumentFailTest._get_collection_name()) + class InheritedDocumentFailTest(NonAbstractBase): + meta = {'collection': 'fail'} + + self.assertTrue(issubclass(w[0].category, SyntaxWarning)) + self.assertEqual('non_abstract_base', InheritedDocumentFailTest._get_collection_name()) # Mixin tests class BaseMixin(object): @@ -539,6 +547,11 @@ class DocumentTest(unittest.TestCase): def test_inherited_collections(self): """Ensure that subclassed documents don't override parents' collections. """ + + # catch_warnings only available in Python 2.6+ + if (sys.version_info < (2,6,0)): + raise SkipTest('catch warnings as errors only available in python 2.6+') + with warnings.catch_warnings(record=True) as w: # Cause all warnings to always be triggered. warnings.simplefilter("always") diff --git a/tests/test_fields.py b/tests/test_fields.py index dca4f21..cc718f5 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1,3 +1,4 @@ +from __future__ import with_statement import datetime import os import unittest diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 1960fc6..0c2d3c3 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +from __future__ import with_statement import unittest from datetime import datetime, timedelta @@ -499,7 +499,7 @@ class QuerySetTest(unittest.TestCase): Blog.drop_collection() - # Recreates the collection + # Recreates the collection self.assertEqual(0, Blog.objects.count()) with query_counter() as q: From 3577773af3c91f277a85096c8e23cfecb8679ffe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Dupanovi=C4=87?= Date: Fri, 3 Aug 2012 14:55:18 +0300 Subject: [PATCH 0715/1279] Added reference to the official repository --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 548737c..1305b6e 100644 --- a/README.rst +++ b/README.rst @@ -2,6 +2,7 @@ MongoEngine =========== :Info: MongoEngine is an ORM-like layer on top of PyMongo. +:Repository: https://github.com/MongoEngine/mongoengine :Author: Harry Marr (http://github.com/hmarr) :Maintainer: Ross Lawley (http://github.com/rozza) From 2801b38c75641a470b725b1ffe5ebe1c7c37d6eb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 3 Aug 2012 14:36:31 +0100 Subject: [PATCH 0716/1279] Version Bump --- docs/changelog.rst | 4 ++-- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 08091ac..8f44c81 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,8 +2,8 @@ Changelog ========= -Changes in 0.6.X -================ +Changes in 0.6.19 +================= - Added Binary support to UUID (MongoEngine/mongoengine#47) - Fixed MapField lookup for fields without declared lookups (MongoEngine/mongoengine#46) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 5f4f7f1..362b1e9 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 18) +VERSION = (0, 6, 19) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 4dfbecc..902f4ce 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.18 +Version: 0.6.19 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From ed7447715097f24b6013a523ad6b50a627390807 Mon Sep 17 00:00:00 2001 From: Laine Date: Thu, 2 Aug 2012 15:30:21 -0700 Subject: [PATCH 0717/1279] made compatable with python 2.5 --- mongoengine/base.py | 8 ++++ mongoengine/fields.py | 4 +- mongoengine/queryset.py | 21 +++++++-- tests/test_dereference.py | 1 + tests/test_django.py | 4 +- tests/test_document.py | 91 ++++++++++++++++++++++++++------------- tests/test_fields.py | 1 + tests/test_queryset.py | 4 +- 8 files changed, 94 insertions(+), 40 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index d4a7b32..811a45f 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -877,6 +877,10 @@ class BaseDocument(object): if not is_list and '_cls' in value: cls = get_document(value['_cls']) + # Make keys str instead of unicode + for key,val in value.items(): + del value[key] + value[str(key)] = val value = cls(**value) value._dynamic = True value._changed_fields = [] @@ -998,6 +1002,10 @@ class BaseDocument(object): raise InvalidDocumentError(""" Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, errors)) + # Make all keys str instead of unicode + for key,val in data.items(): + del data[key] + data[str(key)] = val obj = cls(**data) obj._changed_fields = changed_fields diff --git a/mongoengine/fields.py b/mongoengine/fields.py index a2947c1..fc6e8a5 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1267,8 +1267,8 @@ class SequenceField(IntField): """ Generate and Increment the counter """ - sequence_id = "{0}.{1}".format(self.owner_document._get_collection_name(), - self.name) + sequence_id = "%s.%s"%(self.owner_document._get_collection_name(), + self.name) collection = get_db(alias = self.db_alias )[self.collection_name] counter = collection.find_and_modify(query={"_id": sequence_id}, update={"$inc": {"next": 1}}, diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 1067e32..cef9f80 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -4,6 +4,7 @@ import copy import itertools import operator import functools +import sys from functools import partial import pymongo @@ -14,6 +15,18 @@ from mongoengine import signals __all__ = ['queryset_manager', 'Q', 'InvalidQueryError', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL'] +if sys.version_info < (2,6,0): + def product(*args, **kwds): + pools = map(tuple, args) * kwds.get('repeat', 1) + result = [[]] + for pool in pools: + result = [x+[y] for x in result for y in pool] + for prod in result: + yield tuple(prod) +else: + from itertools import product + from functools import reduce + # The maximum number of items to display in a QuerySet.__repr__ REPR_OUTPUT_SIZE = 20 @@ -120,13 +133,13 @@ class QueryTreeTransformerVisitor(QNodeVisitor): # the necessary parts. Then for each $or part, create a new query # that ANDs the necessary part with the $or part. clauses = [] - for or_group in itertools.product(*or_groups): - q_object = functools.reduce(lambda a, b: a & b, and_parts, Q()) - q_object = functools.reduce(lambda a, b: a & b, or_group, q_object) + for or_group in product(*or_groups): + q_object = reduce(lambda a, b: a & b, and_parts, Q()) + q_object = reduce(lambda a, b: a & b, or_group, q_object) clauses.append(q_object) # Finally, $or the generated clauses in to one query. Each of the # clauses is sufficient for the query to succeed. - return functools.reduce(lambda a, b: a | b, clauses, Q()) + return reduce(lambda a, b: a | b, clauses, Q()) if combination.operation == combination.OR: children = [] diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 75daf6a..cc2ffba 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1,3 +1,4 @@ +from __future__ import with_statement import unittest from mongoengine import * diff --git a/tests/test_django.py b/tests/test_django.py index ed21f27..32a3fe1 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +from __future__ import with_statement import unittest from nose.plugins.skip import SkipTest from mongoengine.python3_support import PY3 @@ -16,7 +16,7 @@ try: from django.contrib.sessions.tests import SessionTestsMixin from mongoengine.django.sessions import SessionStore, MongoSession -except Exception as err: +except Exception, err: if PY3: SessionTestsMixin = type #dummy value so no error SessionStore = None #dummy value so no error diff --git a/tests/test_document.py b/tests/test_document.py index 8eaacee..bef56f0 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1,10 +1,13 @@ +from __future__ import with_statement import os import pickle import pymongo import bson import unittest import warnings +import sys +from nose.plugins.skip import SkipTest from datetime import datetime from tests.fixtures import Base, Mixin, PickleEmbedded, PickleTest @@ -38,19 +41,29 @@ class DocumentTest(unittest.TestCase): """Add FutureWarning for future allow_inhertiance default change. """ - with warnings.catch_warnings(record=True) as errors: + self.warning_list = [] + showwarning_default = warnings.showwarning + + def append_to_warning_list(message,category, *args): + self.warning_list.append({"message":message, "category":category}) - class SimpleBase(Document): - a = IntField() + # add warnings to self.warning_list instead of stderr + warnings.showwarning = append_to_warning_list - class InheritedClass(SimpleBase): - b = IntField() + class SimpleBase(Document): + a = IntField() - InheritedClass() - self.assertEqual(len(errors), 1) - warning = errors[0] - self.assertEqual(FutureWarning, warning.category) - self.assertTrue("InheritedClass" in str(warning.message)) + class InheritedClass(SimpleBase): + b = IntField() + + # restore default handling of warnings + warnings.showwarning = showwarning_default + + InheritedClass() + self.assertEqual(len(self.warning_list), 1) + warning = self.warning_list[0] + self.assertEqual(FutureWarning, warning["category"]) + self.assertTrue("InheritedClass" in str(warning["message"])) def test_drop_collection(self): """Ensure that the collection may be dropped from the database. @@ -131,18 +144,28 @@ class DocumentTest(unittest.TestCase): meta = {'collection': 'wibble'} self.assertEqual('wibble', InheritedAbstractNamingTest._get_collection_name()) - with warnings.catch_warnings(record=True) as w: - # Cause all warnings to always be triggered. - warnings.simplefilter("always") + # set up for redirecting warnings + self.warning_list = [] + showwarning_default = warnings.showwarning - class NonAbstractBase(Document): - pass + def append_to_warning_list(message, category, *args): + self.warning_list.append({'message':message, 'category':category}) + + # add warnings to self.warning_list instead of stderr + warnings.showwarning = append_to_warning_list + warnings.simplefilter("always") - class InheritedDocumentFailTest(NonAbstractBase): - meta = {'collection': 'fail'} + class NonAbstractBase(Document): + pass - self.assertTrue(issubclass(w[0].category, SyntaxWarning)) - self.assertEqual('non_abstract_base', InheritedDocumentFailTest._get_collection_name()) + class InheritedDocumentFailTest(NonAbstractBase): + meta = {'collection': 'fail'} + + # restore default handling of warnings + warnings.showwarning = showwarning_default + + self.assertTrue(issubclass(self.warning_list[0]["category"], SyntaxWarning)) + self.assertEqual('non_abstract_base', InheritedDocumentFailTest._get_collection_name()) # Mixin tests class BaseMixin(object): @@ -539,21 +562,29 @@ class DocumentTest(unittest.TestCase): def test_inherited_collections(self): """Ensure that subclassed documents don't override parents' collections. """ - with warnings.catch_warnings(record=True) as w: - # Cause all warnings to always be triggered. - warnings.simplefilter("always") - class Drink(Document): - name = StringField() + class Drink(Document): + name = StringField() + class Drinker(Document): + drink = GenericReferenceField() + + try: + warnings.simplefilter("error") + + class AcloholicDrink(Drink): + meta = {'collection': 'booze'} + + except SyntaxWarning, w: + warnings.simplefilter("ignore") + class AlcoholicDrink(Drink): meta = {'collection': 'booze'} - - class Drinker(Document): - drink = GenericReferenceField() - - # Confirm we triggered a SyntaxWarning - assert issubclass(w[0].category, SyntaxWarning) + + else: + raise AssertionError("SyntaxWarning should be triggered") + + warnings.resetwarnings() Drink.drop_collection() AlcoholicDrink.drop_collection() diff --git a/tests/test_fields.py b/tests/test_fields.py index dca4f21..cc718f5 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1,3 +1,4 @@ +from __future__ import with_statement import datetime import os import unittest diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 1960fc6..0c2d3c3 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +from __future__ import with_statement import unittest from datetime import datetime, timedelta @@ -499,7 +499,7 @@ class QuerySetTest(unittest.TestCase): Blog.drop_collection() - # Recreates the collection + # Recreates the collection self.assertEqual(0, Blog.objects.count()) with query_counter() as q: From 0575abab2338b11ea30f19baa36372996d825b63 Mon Sep 17 00:00:00 2001 From: Laine Date: Fri, 3 Aug 2012 15:09:50 -0700 Subject: [PATCH 0718/1279] mend --- tests/test_document.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_document.py b/tests/test_document.py index a0cd5fa..bef56f0 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -40,8 +40,6 @@ class DocumentTest(unittest.TestCase): def test_future_warning(self): """Add FutureWarning for future allow_inhertiance default change. """ - if (sys.version_info < (2,6,0)): - raise SkipTest('catch warnings as errors only available in 2.6+') self.warning_list = [] showwarning_default = warnings.showwarning From f99b7a811b3ea8b2930dbc72f2a5bfc1f0116278 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 7 Aug 2012 08:53:58 +0100 Subject: [PATCH 0719/1279] Fixed error in Binary Field --- docs/changelog.rst | 4 ++++ mongoengine/fields.py | 2 +- tests/test_fields.py | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8f44c81..418c861 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.6.X +================ +- Fixed BinaryField lookup re (MongoEngine/mongoengine#48) + Changes in 0.6.19 ================= diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 8268992..8e3cf15 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1327,7 +1327,7 @@ class UUIDField(BaseField): super(UUIDField, self).__init__(**kwargs) def to_python(self, value): - if not self.binary: + if not self._binary: if not isinstance(value, basestring): value = unicode(value) return uuid.UUID(value) diff --git a/tests/test_fields.py b/tests/test_fields.py index c4013c1..a6eaca4 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -283,6 +283,7 @@ class FieldTest(unittest.TestCase): uu = uuid.uuid4() Person(api_key=uu).save() self.assertEqual(1, Person.objects(api_key=uu).count()) + self.assertEqual(uu, Person.objects.first().api_key) person = Person() valid = (uuid.uuid4(), uuid.uuid1()) @@ -307,6 +308,7 @@ class FieldTest(unittest.TestCase): uu = uuid.uuid4() Person(api_key=uu).save() self.assertEqual(1, Person.objects(api_key=uu).count()) + self.assertEqual(uu, Person.objects.first().api_key) person = Person() valid = (uuid.uuid4(), uuid.uuid1()) From 12c8b5c0b9e4793020be77a533feac5d88852ab9 Mon Sep 17 00:00:00 2001 From: Anthony Nemitz Date: Fri, 27 Jul 2012 23:06:47 -0700 Subject: [PATCH 0720/1279] Make chained querysets work if constraining the same fields. Refs hmarr/mongoengine#554 --- mongoengine/queryset.py | 18 ++++++++++++++++-- tests/test_queryset.py | 28 +++++++++++++++++++++++++--- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index ef0ed04..703b6e5 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -765,8 +765,22 @@ class QuerySet(object): key = '.'.join(parts) if op is None or key not in mongo_query: mongo_query[key] = value - elif key in mongo_query and isinstance(mongo_query[key], dict): - mongo_query[key].update(value) + elif key in mongo_query: + if isinstance(mongo_query[key], dict) and isinstance(value, dict): + mongo_query[key].update(value) + elif isinstance(mongo_query[key], list): + mongo_query[key].append(value) + else: + mongo_query[key] = [mongo_query[key], value] + + for k, v in mongo_query.items(): + if isinstance(v, list): + value = [{k:val} for val in v] + if '$and' in mongo_query.keys(): + mongo_query['$and'].append(value) + else: + mongo_query['$and'] = value + del mongo_query[k] return mongo_query diff --git a/tests/test_queryset.py b/tests/test_queryset.py index b4ae805..02c97e4 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -827,7 +827,11 @@ class QuerySetTest(unittest.TestCase): def test_filter_chaining(self): """Ensure filters can be chained together. """ + class Blog(Document): + id = StringField(unique=True, primary_key=True) + class BlogPost(Document): + blog = ReferenceField(Blog) title = StringField() is_published = BooleanField() published_date = DateTimeField() @@ -836,13 +840,24 @@ class QuerySetTest(unittest.TestCase): def published(doc_cls, queryset): return queryset(is_published=True) - blog_post_1 = BlogPost(title="Blog Post #1", + Blog.drop_collection() + BlogPost.drop_collection() + + blog_1 = Blog(id="1") + blog_2 = Blog(id="2") + blog_3 = Blog(id="3") + + blog_1.save() + blog_2.save() + blog_3.save() + + blog_post_1 = BlogPost(blog=blog_1, title="Blog Post #1", is_published = True, published_date=datetime(2010, 1, 5, 0, 0 ,0)) - blog_post_2 = BlogPost(title="Blog Post #2", + blog_post_2 = BlogPost(blog=blog_2, title="Blog Post #2", is_published = True, published_date=datetime(2010, 1, 6, 0, 0 ,0)) - blog_post_3 = BlogPost(title="Blog Post #3", + blog_post_3 = BlogPost(blog=blog_3, title="Blog Post #3", is_published = True, published_date=datetime(2010, 1, 7, 0, 0 ,0)) @@ -856,7 +871,14 @@ class QuerySetTest(unittest.TestCase): published_date__lt=datetime(2010, 1, 7, 0, 0 ,0)) self.assertEqual(published_posts.count(), 2) + + blog_posts = BlogPost.objects + blog_posts = blog_posts.filter(blog__in=[blog_1, blog_2]) + blog_posts = blog_posts.filter(blog=blog_3) + self.assertEqual(blog_posts.count(), 0) + BlogPost.drop_collection() + Blog.drop_collection() def test_ordering(self): """Ensure default ordering is applied and can be overridden. From 95b1783834810cd1c474851b4882a5f662887091 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 7 Aug 2012 09:31:51 +0100 Subject: [PATCH 0721/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 418c861..b53e008 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.6.X ================ +- Improved support for chained querysets when constraining the same fields (hmarr/mongoengine#554) - Fixed BinaryField lookup re (MongoEngine/mongoengine#48) Changes in 0.6.19 From 475488b9f283808faa270692017229f2605ab9ab Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 7 Aug 2012 10:04:05 +0100 Subject: [PATCH 0722/1279] Added support for distinct and db_alias (MongoEngine/mongoengine#59) --- AUTHORS | 3 ++- docs/changelog.rst | 5 +++-- mongoengine/dereference.py | 4 +++- mongoengine/queryset.py | 3 ++- tests/test_queryset.py | 24 +++++++++++++++++++++++- 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/AUTHORS b/AUTHORS index fe8be76..ab0469e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -114,4 +114,5 @@ that much better: * Jaime Irurzun * Alexandre González * Thomas Steinacher - * Tommi Komulainen \ No newline at end of file + * Tommi Komulainen + * Peter Landry diff --git a/docs/changelog.rst b/docs/changelog.rst index b53e008..c05da94 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,8 +2,9 @@ Changelog ========= -Changes in 0.6.X -================ +Changes in 0.6.20 +================= +- Added support for distinct and db_alias (MongoEngine/mongoengine#59) - Improved support for chained querysets when constraining the same fields (hmarr/mongoengine#554) - Fixed BinaryField lookup re (MongoEngine/mongoengine#48) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index f74e224..637380d 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -34,7 +34,9 @@ class DeReference(object): doc_type = None if instance and instance._fields: - doc_type = instance._fields[name].field + doc_type = instance._fields[name] + if hasattr(doc_type, 'field'): + doc_type = doc_type.field if isinstance(doc_type, ReferenceField): doc_type = doc_type.document_type diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 703b6e5..5c7b9c8 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1168,7 +1168,8 @@ class QuerySet(object): .. versionchanged:: 0.5 - Fixed handling references """ from dereference import DeReference - return DeReference()(self._cursor.distinct(field), 1) + return DeReference()(self._cursor.distinct(field), 1, + name=field, instance=self._document) def only(self, *fields): """Load only a subset of this document's fields. :: diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 02c97e4..8f846ea 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -871,7 +871,7 @@ class QuerySetTest(unittest.TestCase): published_date__lt=datetime(2010, 1, 7, 0, 0 ,0)) self.assertEqual(published_posts.count(), 2) - + blog_posts = BlogPost.objects blog_posts = blog_posts.filter(blog__in=[blog_1, blog_2]) blog_posts = blog_posts.filter(blog=blog_3) @@ -2299,6 +2299,28 @@ class QuerySetTest(unittest.TestCase): self.assertEquals(Foo.objects.distinct("bar"), [bar]) + def test_distinct_handles_references_to_alias(self): + register_connection('testdb', 'mongoenginetest2') + + class Foo(Document): + bar = ReferenceField("Bar") + meta = {'db_alias': 'testdb'} + + class Bar(Document): + text = StringField() + meta = {'db_alias': 'testdb'} + + Bar.drop_collection() + Foo.drop_collection() + + bar = Bar(text="hi") + bar.save() + + foo = Foo(bar=bar) + foo.save() + + self.assertEquals(Foo.objects.distinct("bar"), [bar]) + def test_custom_manager(self): """Ensure that custom QuerySetManager instances work as expected. """ From 9cc61640263697addeeee8c2915bd9b48b24cd52 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 7 Aug 2012 10:05:01 +0100 Subject: [PATCH 0723/1279] Version bump --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 362b1e9..ea3dd5e 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 19) +VERSION = (0, 6, 20) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 902f4ce..4e80326 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.19 +Version: 0.6.20 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 43a5d73e14bcef1a79d8b0080ead544a2fa76ee4 Mon Sep 17 00:00:00 2001 From: Alon Horev Date: Tue, 7 Aug 2012 09:12:39 +0000 Subject: [PATCH 0724/1279] propagate the shard_key from abstract base classes' meta --- mongoengine/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/base.py b/mongoengine/base.py index 6fb26cb..500e460 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -654,6 +654,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): keys_to_propogate = ( 'index_background', 'index_drop_dups', 'index_opts', 'allow_inheritance', 'queryset_class', 'db_alias', + 'shard_key' ) for key in keys_to_propogate: if key in base._meta: From dd4af2df81fe98797a00d770cf91c44f6fca6d55 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 7 Aug 2012 13:07:51 +0100 Subject: [PATCH 0725/1279] Python 2.5 and 3.1 support confirmed --- .travis.yml | 2 ++ mongoengine/base.py | 15 +++++--------- mongoengine/django/tests.py | 4 ++-- mongoengine/fields.py | 19 ++++++++++++------ .../{python3_support.py => python_support.py} | 20 ++++++++++++++++--- mongoengine/queryset.py | 16 ++------------- setup.cfg | 3 +-- setup.py | 9 ++++++++- tests/test_django.py | 2 +- tests/test_document.py | 19 +++++++++--------- tests/test_fields.py | 10 +++++----- tests/test_queryset.py | 4 +++- 12 files changed, 68 insertions(+), 55 deletions(-) rename mongoengine/{python3_support.py => python_support.py} (51%) diff --git a/.travis.yml b/.travis.yml index 47ad068..3ec14ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,10 @@ # http://travis-ci.org/#!/MongoEngine/mongoengine language: python python: + - 2.5 - 2.6 - 2.7 + - 3.1 - 3.2 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install zlib1g zlib1g-dev; fi diff --git a/mongoengine/base.py b/mongoengine/base.py index 11075ef..e1d7ebb 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -10,7 +10,7 @@ from queryset import DoesNotExist, MultipleObjectsReturned from queryset import DO_NOTHING from mongoengine import signals -from mongoengine.python3_support import PY3, txt_type +from mongoengine.python_support import PY3, PY25, txt_type import pymongo from bson import ObjectId @@ -894,10 +894,8 @@ class BaseDocument(object): if not is_list and '_cls' in value: cls = get_document(value['_cls']) - # Make keys str instead of unicode - for key,val in value.items(): - del value[key] - value[str(key)] = val + if PY25: + value = dict([(str(k), v) for k, v in value.items()]) value = cls(**value) value._dynamic = True value._changed_fields = [] @@ -1019,12 +1017,9 @@ class BaseDocument(object): raise InvalidDocumentError(""" Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, errors)) - # Make all keys str instead of unicode - for key,val in data.items(): - del data[key] - data[str(key)] = val + if PY25: + data = dict([(str(k), v) for k, v in data.items()]) obj = cls(**data) - obj._changed_fields = changed_fields obj._created = False return obj diff --git a/mongoengine/django/tests.py b/mongoengine/django/tests.py index 566a398..1fab6fc 100644 --- a/mongoengine/django/tests.py +++ b/mongoengine/django/tests.py @@ -1,7 +1,7 @@ #coding: utf-8 from nose.plugins.skip import SkipTest -from mongoengine.python3_support import PY3 +from mongoengine.python_support import PY3 from mongoengine import connect try: @@ -18,7 +18,7 @@ except Exception as err: class MongoTestCase(TestCase): - + def setUp(self): if PY3: raise SkipTest('django does not have Python 3 support') diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 600a8e6..448b0c6 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -10,8 +10,8 @@ from operator import itemgetter import gridfs from bson import Binary, DBRef, SON, ObjectId -from mongoengine.python3_support import (PY3, bin_type, - txt_type, str_types, StringIO) +from mongoengine.python_support import (PY3, bin_type, txt_type, + str_types, StringIO) from base import (BaseField, ComplexBaseField, ObjectIdField, ValidationError, get_document, BaseDocument) from queryset import DO_NOTHING, QuerySet @@ -840,13 +840,20 @@ class BinaryField(BaseField): self.max_bytes = max_bytes super(BinaryField, self).__init__(**kwargs) + def __set__(self, instance, value): + """Handle bytearrays in python 3.1""" + if PY3 and isinstance(value, bytearray): + value = bin_type(value) + return super(BinaryField, self).__set__(instance, value) + def to_mongo(self, value): return Binary(value) def validate(self, value): if not isinstance(value, (bin_type, txt_type, Binary)): self.error("BinaryField only accepts instances of " - "(%s, %s, Binary)" % (bin_type.__name__,txt_type.__name__)) + "(%s, %s, Binary)" % ( + bin_type.__name__, txt_type.__name__)) if self.max_bytes is not None and len(value) > self.max_bytes: self.error('Binary value is too long') @@ -1270,9 +1277,9 @@ class SequenceField(IntField): """ Generate and Increment the counter """ - sequence_id = "%s.%s"%(self.owner_document._get_collection_name(), - self.name) - collection = get_db(alias = self.db_alias )[self.collection_name] + sequence_id = "%s.%s" % (self.owner_document._get_collection_name(), + self.name) + collection = get_db(alias=self.db_alias)[self.collection_name] counter = collection.find_and_modify(query={"_id": sequence_id}, update={"$inc": {"next": 1}}, new=True, diff --git a/mongoengine/python3_support.py b/mongoengine/python_support.py similarity index 51% rename from mongoengine/python3_support.py rename to mongoengine/python_support.py index 0682d87..70c3181 100644 --- a/mongoengine/python3_support.py +++ b/mongoengine/python_support.py @@ -1,13 +1,14 @@ -"""Helper functions and types to aid with Python 3 support.""" +"""Helper functions and types to aid with Python 2.5 - 3 support.""" import sys PY3 = sys.version_info[0] == 3 +PY25 = sys.version_info[:2] == (2, 5) if PY3: import codecs from io import BytesIO as StringIO - # return s converted to binary. b('test') should be equivalent to b'test' + # return s converted to binary. b('test') should be equivalent to b'test' def b(s): return codecs.latin_1_encode(s)[0] @@ -24,6 +25,19 @@ else: return s bin_type = str - txt_type = unicode + txt_type = unicode str_types = (bin_type, txt_type) + +if PY25: + def product(*args, **kwds): + pools = map(tuple, args) * kwds.get('repeat', 1) + result = [[]] + for pool in pools: + result = [x + [y] for x in result for y in pool] + for prod in result: + yield tuple(prod) + reduce = reduce +else: + from itertools import product + from functools import reduce diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index f849c13..2851d5d 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -3,9 +3,9 @@ import re import copy import itertools import operator -import functools -import sys + from functools import partial +from mongoengine.python_support import product, reduce import pymongo from bson.code import Code @@ -15,18 +15,6 @@ from mongoengine import signals __all__ = ['queryset_manager', 'Q', 'InvalidQueryError', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL'] -if sys.version_info < (2,6,0): - def product(*args, **kwds): - pools = map(tuple, args) * kwds.get('repeat', 1) - result = [[]] - for pool in pools: - result = [x+[y] for x in result for y in pool] - for prod in result: - yield tuple(prod) -else: - from itertools import product - from functools import reduce - # The maximum number of items to display in a QuerySet.__repr__ REPR_OUTPUT_SIZE = 20 diff --git a/setup.cfg b/setup.cfg index ad9b928..3b1a561 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,6 +6,5 @@ detailed-errors = 1 #cover-html = 1 #cover-html-dir = ../htmlcov #cover-package = mongoengine -where = tests py3where = build -#tests = test_bugfix.py +where = tests diff --git a/setup.py b/setup.py index 5f59b1a..398bd98 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ -from setuptools import setup, find_packages import os import sys +from setuptools import setup, find_packages # Hack to silence atexit traceback in newer python versions try: @@ -16,6 +16,7 @@ try: except: pass + def get_version(version_tuple): version = '%s.%s' % (version_tuple[0], version_tuple[1]) if version_tuple[2]: @@ -38,7 +39,13 @@ CLASSIFIERS = [ 'Operating System :: OS Independent', 'Programming Language :: Python', "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.5", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.1", + "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: Implementation :: CPython", 'Topic :: Database', 'Topic :: Software Development :: Libraries :: Python Modules', ] diff --git a/tests/test_django.py b/tests/test_django.py index 32a3fe1..678d7cf 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -1,7 +1,7 @@ from __future__ import with_statement import unittest from nose.plugins.skip import SkipTest -from mongoengine.python3_support import PY3 +from mongoengine.python_support import PY3 from mongoengine import * try: diff --git a/tests/test_document.py b/tests/test_document.py index ae6aaa2..788b1e3 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -43,7 +43,7 @@ class DocumentTest(unittest.TestCase): self.warning_list = [] showwarning_default = warnings.showwarning - + def append_to_warning_list(message,category, *args): self.warning_list.append({"message":message, "category":category}) @@ -55,7 +55,7 @@ class DocumentTest(unittest.TestCase): class InheritedClass(SimpleBase): b = IntField() - + # restore default handling of warnings warnings.showwarning = showwarning_default @@ -150,7 +150,7 @@ class DocumentTest(unittest.TestCase): def append_to_warning_list(message, category, *args): self.warning_list.append({'message':message, 'category':category}) - + # add warnings to self.warning_list instead of stderr warnings.showwarning = append_to_warning_list warnings.simplefilter("always") @@ -209,7 +209,6 @@ class DocumentTest(unittest.TestCase): } self.assertEqual(Dog._superclasses, dog_superclasses) - def test_external_superclasses(self): """Ensure that the correct list of sub and super classes is assembled. when importing part of the model @@ -568,22 +567,22 @@ class DocumentTest(unittest.TestCase): class Drinker(Document): drink = GenericReferenceField() - + try: warnings.simplefilter("error") - + class AcloholicDrink(Drink): meta = {'collection': 'booze'} - + except SyntaxWarning, w: warnings.simplefilter("ignore") - + class AlcoholicDrink(Drink): meta = {'collection': 'booze'} - + else: raise AssertionError("SyntaxWarning should be triggered") - + warnings.resetwarnings() Drink.drop_collection() diff --git a/tests/test_fields.py b/tests/test_fields.py index 1d6b32c..8209938 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -14,7 +14,7 @@ from nose.plugins.skip import SkipTest from mongoengine import * from mongoengine.connection import get_db from mongoengine.base import _document_registry, NotRegistered -from mongoengine.python3_support import PY3, b, StringIO, bin_type +from mongoengine.python_support import PY3, b, StringIO, bin_type TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') @@ -383,7 +383,7 @@ class FieldTest(unittest.TestCase): self.assertNotEqual(log.date, d1) self.assertEqual(log.date, d2) - if not PY3: + if not PY3: # Pre UTC dates microseconds below 1000 are dropped # This does not seem to be true in PY3 d1 = datetime.datetime(1969, 12, 31, 23, 59, 59, 999) @@ -1744,7 +1744,7 @@ class FieldTest(unittest.TestCase): self.assertEqual(doc_b.the_file.grid_id, doc_c.the_file.grid_id) # Test with default - doc_d = GridDocument(the_file=b('')) + doc_d = GridDocument(the_file=b('')) doc_d.save() doc_e = GridDocument.objects.with_id(doc_d.id) @@ -2157,11 +2157,11 @@ class FieldTest(unittest.TestCase): post = Post(title='hello world') post.comments.append(Comment(content='hello', author=bob)) post.comments.append(Comment(author=bob)) - + self.assertRaises(ValidationError, post.validate) try: post.validate() - except ValidationError, error: + except ValidationError, error: # ValidationError.errors property self.assertTrue(hasattr(error, 'errors')) self.assertTrue(isinstance(error.errors, dict)) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index e656916..7e42550 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -9,7 +9,7 @@ from bson import ObjectId from mongoengine import * from mongoengine.connection import get_connection -from mongoengine.python3_support import PY3 +from mongoengine.python_support import PY3 from mongoengine.tests import query_counter from mongoengine.queryset import (QuerySet, QuerySetManager, MultipleObjectsReturned, DoesNotExist, @@ -1455,6 +1455,8 @@ class QuerySetTest(unittest.TestCase): name = StringField() parent = ReferenceField('self', reverse_delete_rule=CASCADE) + Category.drop_collection() + num_children = 3 base = Category(name='Root') base.save() From fa07423ca546478dd04d6e72405363d8674adb06 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 7 Aug 2012 13:10:56 +0100 Subject: [PATCH 0726/1279] Removing python 2.4 code --- mongoengine/base.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index e1d7ebb..6971269 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1378,11 +1378,6 @@ class BaseDict(dict): if hasattr(self._instance, '_mark_as_changed'): self._instance._mark_as_changed(self._name) -if sys.version_info < (2, 5): - # Prior to Python 2.5, Exception was an old-style class - def subclass_exception(name, parents, unused): - from types import ClassType - return ClassType(name, parents, {}) -else: - def subclass_exception(name, parents, module): - return type(name, parents, {'__module__': module}) + +def subclass_exception(name, parents, module): + return type(name, parents, {'__module__': module}) From 500eb920e4498c0573ceadd22f65117fd55fd075 Mon Sep 17 00:00:00 2001 From: Manuel Hermann Date: Tue, 7 Aug 2012 17:04:03 +0200 Subject: [PATCH 0727/1279] Use info from getlasterror whether a document has been updated or created. --- mongoengine/document.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index f8bf769..bb5a60f 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -208,11 +208,20 @@ class Document(BaseDocument): actual_key = self._db_field_map.get(k, k) select_dict[actual_key] = doc[actual_key] + def is_new_object(last_error): + if last_error is not None: + updated = last_error.get("updatedExisting") + if updated is not None: + return not updated + return created + upsert = self._created if updates: - collection.update(select_dict, {"$set": updates}, upsert=upsert, safe=safe, **write_options) + last_error = collection.update(select_dict, {"$set": updates}, upsert=upsert, safe=safe, **write_options) + created = is_new_object(last_error) if removals: - collection.update(select_dict, {"$unset": removals}, upsert=upsert, safe=safe, **write_options) + last_error = collection.update(select_dict, {"$unset": removals}, upsert=upsert, safe=safe, **write_options) + created = created or is_new_object(last_error) cascade = self._meta.get('cascade', True) if cascade is None else cascade if cascade: From 839ed8a64aaec9e0e639bbd5eea93697d77a830b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 13 Aug 2012 14:57:35 +0100 Subject: [PATCH 0728/1279] Added test for abstract shard key --- tests/test_document.py | 47 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_document.py b/tests/test_document.py index 788b1e3..8654e3f 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -3254,5 +3254,52 @@ class ValidatorErrorTest(unittest.TestCase): b = BDocument.objects.first() b.save(cascade=True) + def test_shard_key(self): + class LogEntry(Document): + machine = StringField() + log = StringField() + + meta = { + 'shard_key': ('machine',) + } + + LogEntry.drop_collection() + + log = LogEntry() + log.machine = "Localhost" + log.save() + + log.log = "Saving" + log.save() + + def change_shard_key(): + log.machine = "127.0.0.1" + + self.assertRaises(OperationError, change_shard_key) + + def test_shard_key_primary(self): + class LogEntry(Document): + machine = StringField(primary_key=True) + log = StringField() + + meta = { + 'shard_key': ('machine',) + } + + LogEntry.drop_collection() + + log = LogEntry() + log.machine = "Localhost" + log.save() + + log.log = "Saving" + log.save() + + def change_shard_key(): + log.machine = "127.0.0.1" + + self.assertRaises(OperationError, change_shard_key) + + if __name__ == '__main__': unittest.main() From 2bb9493fcfbddfb22f439abe16a1e60b5e739e54 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 13 Aug 2012 15:05:01 +0100 Subject: [PATCH 0729/1279] Updated doc --- mongoengine/document.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 35e58d7..46111be 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -25,7 +25,7 @@ class EmbeddedDocument(BaseDocument): # The __metaclass__ attribute is removed by 2to3 when running with Python3 # my_metaclass is defined so that metaclass can be queried in Python 2 & 3 - my_metaclass = DocumentMetaclass + my_metaclass = DocumentMetaclass __metaclass__ = DocumentMetaclass def __init__(self, *args, **kwargs): @@ -97,7 +97,7 @@ class Document(BaseDocument): # The __metaclass__ attribute is removed by 2to3 when running with Python3 # my_metaclass is defined so that metaclass can be queried in Python 2 & 3 - my_metaclass = TopLevelDocumentMetaclass + my_metaclass = TopLevelDocumentMetaclass __metaclass__ = TopLevelDocumentMetaclass def pk(): @@ -242,7 +242,8 @@ class Document(BaseDocument): message = u'Tried to save duplicate unique keys (%s)' raise OperationError(message % unicode(err)) id_field = self._meta['id_field'] - self[id_field] = self._fields[id_field].to_python(object_id) + if id_field not in self._meta.get('shard_key', []): + self[id_field] = self._fields[id_field].to_python(object_id) self._changed_fields = [] self._created = False @@ -387,7 +388,7 @@ class DynamicDocument(Document): # The __metaclass__ attribute is removed by 2to3 when running with Python3 # my_metaclass is defined so that metaclass can be queried in Python 2 & 3 - my_metaclass = TopLevelDocumentMetaclass + my_metaclass = TopLevelDocumentMetaclass __metaclass__ = TopLevelDocumentMetaclass _dynamic = True @@ -410,7 +411,7 @@ class DynamicEmbeddedDocument(EmbeddedDocument): # The __metaclass__ attribute is removed by 2to3 when running with Python3 # my_metaclass is defined so that metaclass can be queried in Python 2 & 3 - my_metaclass = DocumentMetaclass + my_metaclass = DocumentMetaclass __metaclass__ = DocumentMetaclass _dynamic = True From 529c5225944943ca541ff5bba570535af5363605 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 13 Aug 2012 16:42:51 +0100 Subject: [PATCH 0730/1279] Updated changelog - fixed dynamic document issue --- docs/changelog.rst | 7 +++++++ mongoengine/base.py | 5 +---- setup.cfg | 1 + 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c05da94..b839496 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog ========= +Changes in 0.7.X +================= +- Fixed Dynamic Documents and Embedded Documents (hmarr/mongoengine#561) +- Fixed abstract classes and shard keys (MongoEngine/mongoengine#64) +- Fixed Python 2.5 support +- Added Python 3 support (thanks to Laine Heron) + Changes in 0.6.20 ================= - Added support for distinct and db_alias (MongoEngine/mongoengine#59) diff --git a/mongoengine/base.py b/mongoengine/base.py index 55e604a..b2348ac 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -897,10 +897,7 @@ class BaseDocument(object): cls = get_document(value['_cls']) if PY25: value = dict([(str(k), v) for k, v in value.items()]) - value = cls(**value) - value._dynamic = True - value._changed_fields = [] - return value + return cls(**value) data = {} for k, v in value.items(): diff --git a/setup.cfg b/setup.cfg index 3b1a561..f28e668 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,3 +8,4 @@ detailed-errors = 1 #cover-package = mongoengine py3where = build where = tests +#tests = test_bugfix.py \ No newline at end of file From f00bed605841bc17fb04e188b0280c6cdb6c693c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 13 Aug 2012 16:53:36 +0100 Subject: [PATCH 0731/1279] Updated test cases for dynamic docs --- tests/test_dynamic_document.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_dynamic_document.py b/tests/test_dynamic_document.py index 866af7d..23762a3 100644 --- a/tests/test_dynamic_document.py +++ b/tests/test_dynamic_document.py @@ -500,3 +500,34 @@ class DynamicDocTest(unittest.TestCase): self.assertTrue([('_types', 1), ('category', 1), ('date', -1)] in info) self.assertTrue([('_types', 1), ('date', -1)] in info) + + def test_dynamic_and_embedded(self): + """Ensure embedded documents play nicely""" + + class Address(EmbeddedDocument): + city = StringField() + + class Person(DynamicDocument): + name = StringField() + meta = {'allow_inheritance': True} + + Person.drop_collection() + + Person(name="Ross", address=Address(city="London")).save() + + person = Person.objects.first() + person.address.city = "Lundenne" + person.save() + + self.assertEqual(Person.objects.first().address.city, "Lundenne") + + person = Person.objects.first() + person.address = Address(city="Londinium") + person.save() + + self.assertEqual(Person.objects.first().address.city, "Londinium") + + person = Person.objects.first() + person.age = 35 + person.save() + self.assertEqual(Person.objects.first().age, 35) From 2f6b1c761116c74560ce05ad9d2553d46fc2684a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 13 Aug 2012 17:29:33 +0100 Subject: [PATCH 0732/1279] Improved queryset filtering (hmarr/mongoengine#554) --- mongoengine/queryset.py | 16 ++++++++++------ tests/test_queryset.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 2851d5d..dbd8ad4 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -4,7 +4,9 @@ import copy import itertools import operator +from collections import defaultdict from functools import partial + from mongoengine.python_support import product, reduce import pymongo @@ -679,6 +681,7 @@ class QuerySet(object): custom_operators = ['match'] mongo_query = {} + merge_query = defaultdict(list) for key, value in query.items(): if key == "__raw__": mongo_query.update(value) @@ -767,21 +770,22 @@ class QuerySet(object): if op is None or key not in mongo_query: mongo_query[key] = value elif key in mongo_query: - if isinstance(mongo_query[key], dict) and isinstance(value, dict): + if key in mongo_query and isinstance(mongo_query[key], dict): mongo_query[key].update(value) - elif isinstance(mongo_query[key], list): - mongo_query[key].append(value) else: - mongo_query[key] = [mongo_query[key], value] + # Store for manually merging later + merge_query[key].append(value) - for k, v in mongo_query.items(): + # The queryset has been filter in such a way we must manually merge + for k, v in merge_query.items(): + merge_query[k].append(mongo_query[k]) + del mongo_query[k] if isinstance(v, list): value = [{k:val} for val in v] if '$and' in mongo_query.keys(): mongo_query['$and'].append(value) else: mongo_query['$and'] = value - del mongo_query[k] return mongo_query diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 7e42550..979dc6f 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -883,6 +883,21 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() Blog.drop_collection() + def test_raw_and_merging(self): + class Doc(Document): + pass + + raw_query = Doc.objects(__raw__={'deleted': False, + 'scraped': 'yes', + '$nor': [{'views.extracted': 'no'}, + {'attachments.views.extracted':'no'}] + })._query + + expected = {'deleted': False, '_types': 'Doc', 'scraped': 'yes', + '$nor': [{'views.extracted': 'no'}, + {'attachments.views.extracted': 'no'}]} + self.assertEqual(expected, raw_query) + def test_ordering(self): """Ensure default ordering is applied and can be overridden. """ From 0454fc74e9aa00d342cd0bf7510d0725d1bea5ca Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 14 Aug 2012 10:32:26 +0100 Subject: [PATCH 0733/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b839496..9deb693 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.X ================= +- Improved queryset filtering (hmarr/mongoengine#554) - Fixed Dynamic Documents and Embedded Documents (hmarr/mongoengine#561) - Fixed abstract classes and shard keys (MongoEngine/mongoengine#64) - Fixed Python 2.5 support From 9e67941badd0ddd680b2fc582499f7c660b2ea4b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 14 Aug 2012 10:34:17 +0100 Subject: [PATCH 0734/1279] Use weakref proxies in base lists / dicts (MongoEngine/mongoengine#74) --- docs/changelog.rst | 1 + mongoengine/base.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9deb693..64047c3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.X ================= +- Use weakref proxies in base lists / dicts (MongoEngine/mongoengine#74) - Improved queryset filtering (hmarr/mongoengine#554) - Fixed Dynamic Documents and Embedded Documents (hmarr/mongoengine#561) - Fixed abstract classes and shard keys (MongoEngine/mongoengine#64) diff --git a/mongoengine/base.py b/mongoengine/base.py index b2348ac..6c86506 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1,6 +1,7 @@ import operator import sys import warnings +import weakref from collections import defaultdict from functools import partial @@ -1265,7 +1266,7 @@ class BaseList(list): _name = None def __init__(self, list_items, instance, name): - self._instance = instance + self._instance = weakref.proxy(instance) self._name = name return super(BaseList, self).__init__(list_items) @@ -1327,7 +1328,7 @@ class BaseDict(dict): _name = None def __init__(self, dict_items, instance, name): - self._instance = instance + self._instance = weakref.proxy(instance) self._name = name return super(BaseDict, self).__init__(dict_items) From 19da2288550c3a7bb7b7df951b57429cb6939648 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 17 Aug 2012 11:15:36 +0100 Subject: [PATCH 0735/1279] Cleaned up the metaclasses for documents Refactored and clarified intent and tidied up --- docs/changelog.rst | 3 + mongoengine/base.py | 580 +++++++++++++++++++++++----------------- mongoengine/document.py | 5 +- mongoengine/fields.py | 1 - mongoengine/queryset.py | 33 ++- setup.cfg | 4 +- tests/test_django.py | 4 +- tests/test_document.py | 72 +++-- tests/test_queryset.py | 2 + 9 files changed, 407 insertions(+), 297 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 64047c3..3bca6a1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,9 @@ Changelog Changes in 0.7.X ================= +- Embedded Documents dont care about inheritance + + - Use weakref proxies in base lists / dicts (MongoEngine/mongoengine#74) - Improved queryset filtering (hmarr/mongoengine#554) - Fixed Dynamic Documents and Embedded Documents (hmarr/mongoengine#561) diff --git a/mongoengine/base.py b/mongoengine/base.py index 6c86506..a05403d 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -17,6 +17,12 @@ import pymongo from bson import ObjectId from bson.dbref import DBRef +ALLOW_INHERITANCE = True + +_document_registry = {} +_class_registry = {} + + class NotRegistered(Exception): pass @@ -111,16 +117,13 @@ class ValidationError(AssertionError): return ' '.join(["%s: %s" % (k, v) for k, v in error_dict.iteritems()]) -_document_registry = {} -_module_registry = {} - - def get_document(name): doc = _document_registry.get(name, None) if not doc: # Possible old style names end = ".%s" % name - possible_match = [k for k in _document_registry.keys() if k.endswith(end)] + possible_match = [k for k in _document_registry.keys() + if k.endswith(end)] if len(possible_match) == 1: doc = _document_registry.get(possible_match.pop(), None) if not doc: @@ -153,7 +156,8 @@ class BaseField(object): def __init__(self, db_field=None, name=None, required=False, default=None, unique=False, unique_with=None, primary_key=False, - validation=None, choices=None, verbose_name=None, help_text=None): + validation=None, choices=None, verbose_name=None, + help_text=None): self.db_field = (db_field or name) if not primary_key else '_id' if name: msg = "Fields' 'name' attribute deprecated in favour of 'db_field'" @@ -237,11 +241,15 @@ class BaseField(object): value_to_check = value.__class__ if is_cls else value err_msg = 'an instance' if is_cls else 'one' if isinstance(self.choices[0], (list, tuple)): - option_keys = [option_key for option_key, option_value in self.choices] + option_keys = [k for k, v in self.choices] if value_to_check not in option_keys: - self.error('Value must be %s of %s' % (err_msg, unicode(option_keys))) + msg = ('Value must be %s of %s' % + (err_msg, unicode(option_keys))) + self.error(msg) elif value_to_check not in self.choices: - self.error('Value must be %s of %s' % (err_msg, unicode(self.choices))) + msg = ('Value must be %s of %s' % + (err_msg, unicode(self.choices))) + self.error() # check validation argument if self.validation is not None: @@ -286,7 +294,8 @@ class ComplexBaseField(BaseField): value = super(ComplexBaseField, self).__get__(instance, owner) # Convert lists / values so we can watch for any changes on them - if isinstance(value, (list, tuple)) and not isinstance(value, BaseList): + if (isinstance(value, (list, tuple)) and + not isinstance(value, BaseList)): value = BaseList(value, instance, self.name) instance._data[self.name] = value elif isinstance(value, dict) and not isinstance(value, BaseDict): @@ -328,7 +337,8 @@ class ComplexBaseField(BaseField): return value if self.field: - value_dict = dict([(key, self.field.to_python(item)) for key, item in value.items()]) + value_dict = dict([(key, self.field.to_python(item)) + for key, item in value.items()]) else: value_dict = {} for k, v in value.items(): @@ -345,7 +355,8 @@ class ComplexBaseField(BaseField): value_dict[k] = self.to_python(v) if is_list: # Convert back to a list - return [v for k, v in sorted(value_dict.items(), key=operator.itemgetter(0))] + return [v for k, v in sorted(value_dict.items(), + key=operator.itemgetter(0))] return value_dict def to_mongo(self, value): @@ -368,7 +379,8 @@ class ComplexBaseField(BaseField): return value if self.field: - value_dict = dict([(key, self.field.to_mongo(item)) for key, item in value.items()]) + value_dict = dict([(key, self.field.to_mongo(item)) + for key, item in value.items()]) else: value_dict = {} for k, v in value.items(): @@ -381,8 +393,11 @@ class ComplexBaseField(BaseField): # If its a document that is not inheritable it won't have # _types / _cls data so make it a generic reference allows # us to dereference - meta = getattr(v, 'meta', getattr(v, '_meta', {})) - if meta and not meta.get('allow_inheritance', True) and not self.field: + meta = getattr(v, '_meta', {}) + allow_inheritance = ( + meta.get('allow_inheritance', ALLOW_INHERITANCE) + == False) + if allow_inheritance and not self.field: from fields import GenericReferenceField value_dict[k] = GenericReferenceField().to_mongo(v) else: @@ -394,7 +409,8 @@ class ComplexBaseField(BaseField): value_dict[k] = self.to_mongo(v) if is_list: # Convert back to a list - return [v for k, v in sorted(value_dict.items(), key=operator.itemgetter(0))] + return [v for k, v in sorted(value_dict.items(), + key=operator.itemgetter(0))] return value_dict def validate(self, value): @@ -479,77 +495,25 @@ class DocumentMetaclass(type): """ def __new__(cls, name, bases, attrs): - - def _get_mixin_fields(base): - attrs = {} - attrs.update(dict([(k, v) for k, v in base.__dict__.items() - if issubclass(v.__class__, BaseField)])) - - # Handle simple mixin's with meta - if hasattr(base, 'meta') and not isinstance(base, DocumentMetaclass): - meta = attrs.get('meta', {}) - meta.update(base.meta) - attrs['meta'] = meta - - for p_base in base.__bases__: - #optimize :-) - if p_base in (object, BaseDocument): - continue - - attrs.update(_get_mixin_fields(p_base)) - return attrs - - metaclass = attrs.get('my_metaclass') + bases = cls._get_bases(bases) super_new = super(DocumentMetaclass, cls).__new__ + + # If a base class just call super + metaclass = attrs.get('my_metaclass') if metaclass and issubclass(metaclass, DocumentMetaclass): return super_new(cls, name, bases, attrs) + attrs['_is_document'] = attrs.get('_is_document', False) + + # Handle document Fields + + # Merge all fields from subclasses doc_fields = {} - class_name = [name] - superclasses = {} - simple_class = True - for base in bases: - # Include all fields present in superclasses + for base in bases[::-1]: if hasattr(base, '_fields'): doc_fields.update(base._fields) - # Get superclasses from superclass - superclasses[base._class_name] = base - superclasses.update(base._superclasses) - else: # Add any mixin fields - attrs.update(_get_mixin_fields(base)) - if hasattr(base, '_meta') and not base._meta.get('abstract'): - # Ensure that the Document class may be subclassed - - # inheritance may be disabled to remove dependency on - # additional fields _cls and _types - class_name.append(base._class_name) - if not base._meta.get('allow_inheritance_defined', True): - warnings.warn( - "%s uses inheritance, the default for allow_inheritance " - "is changing to off by default. Please add it to the " - "document meta." % name, - FutureWarning - ) - if base._meta.get('allow_inheritance', True) == False: - raise ValueError('Document %s may not be subclassed' % - base.__name__) - else: - simple_class = False - - doc_class_name = '.'.join(reversed(class_name)) - meta = attrs.get('_meta', {}) - meta.update(attrs.get('meta', {})) - - if 'allow_inheritance' not in meta: - meta['allow_inheritance'] = True - - # Only simple classes - direct subclasses of Document - may set - # allow_inheritance to False - if not simple_class and not meta['allow_inheritance'] and not meta['abstract']: - raise ValueError('Only direct subclasses of Document may set ' - '"allow_inheritance" to False') - - # Add the document's fields to the _fields attribute + # Discover any document fields field_names = {} for attr_name, attr_value in attrs.iteritems(): if not isinstance(attr_value, BaseField): @@ -559,69 +523,93 @@ class DocumentMetaclass(type): attr_value.db_field = attr_name doc_fields[attr_name] = attr_value - field_names[attr_value.db_field] = field_names.get(attr_value.db_field, 0) + 1 + # Count names to ensure no db_field redefinitions + field_names[attr_value.db_field] = field_names.get( + attr_value.db_field, 0) + 1 + # Ensure no duplicate db_fields duplicate_db_fields = [k for k, v in field_names.items() if v > 1] if duplicate_db_fields: - raise InvalidDocumentError("Multiple db_fields defined for: %s " % ", ".join(duplicate_db_fields)) + msg = ("Multiple db_fields defined for: %s " % + ", ".join(duplicate_db_fields)) + raise InvalidDocumentError(msg) + + # Set _fields and db_field maps attrs['_fields'] = doc_fields - attrs['_db_field_map'] = dict([(k, v.db_field) for k, v in doc_fields.items() if k != v.db_field]) - attrs['_reverse_db_field_map'] = dict([(v, k) for k, v in attrs['_db_field_map'].items()]) - attrs['_meta'] = meta - attrs['_class_name'] = doc_class_name + attrs['_db_field_map'] = dict( + ((k, v.db_field) for k, v in doc_fields.items() + if k != v.db_field)) + attrs['_reverse_db_field_map'] = dict( + (v, k) for k, v in attrs['_db_field_map'].iteritems()) + + # + # Set document hierarchy + # + superclasses = {} + class_name = [name] + for base in bases: + if (not getattr(base, '_is_base_cls', True) and + not getattr(base, '_meta', {}).get('abstract', True)): + # Collate heirarchy for _cls and _types + class_name.append(base.__name__) + + # Get superclasses from superclass + superclasses[base._class_name] = base + superclasses.update(base._superclasses) + + attrs['_class_name'] = '.'.join(reversed(class_name)) attrs['_superclasses'] = superclasses - if 'Document' not in _module_registry: - from mongoengine.document import Document, EmbeddedDocument - from mongoengine.fields import DictField - _module_registry['Document'] = Document - _module_registry['EmbeddedDocument'] = EmbeddedDocument - _module_registry['DictField'] = DictField - else: - Document = _module_registry.get('Document') - EmbeddedDocument = _module_registry.get('EmbeddedDocument') - DictField = _module_registry.get('DictField') - + # Create the new_class new_class = super_new(cls, name, bases, attrs) + # Handle delete rules + Document, EmbeddedDocument, DictField = cls._import_classes() for field in new_class._fields.itervalues(): - field.owner_document = new_class - - delete_rule = getattr(field, 'reverse_delete_rule', DO_NOTHING) f = field + f.owner_document = new_class + delete_rule = getattr(f, 'reverse_delete_rule', DO_NOTHING) if isinstance(f, ComplexBaseField) and hasattr(f, 'field'): - delete_rule = getattr(f.field, 'reverse_delete_rule', DO_NOTHING) + delete_rule = getattr(f.field, + 'reverse_delete_rule', + DO_NOTHING) if isinstance(f, DictField) and delete_rule != DO_NOTHING: - raise InvalidDocumentError("Reverse delete rules are not supported for %s (field: %s)" % (field.__class__.__name__, field.name)) + msg = ("Reverse delete rules are not supported " + "for %s (field: %s)" % + (field.__class__.__name__, field.name)) + raise InvalidDocumentError(msg) + f = field.field if delete_rule != DO_NOTHING: if issubclass(new_class, EmbeddedDocument): - raise InvalidDocumentError("Reverse delete rules are not supported for EmbeddedDocuments (field: %s)" % field.name) - f.document_type.register_delete_rule(new_class, field.name, delete_rule) + msg = ("Reverse delete rules are not supported for " + "EmbeddedDocuments (field: %s)" % field.name) + raise InvalidDocumentError(msg) + f.document_type.register_delete_rule(new_class, + field.name, delete_rule) - if (field.name and - hasattr(Document, field.name) and + if (field.name and hasattr(Document, field.name) and EmbeddedDocument not in new_class.mro()): - raise InvalidDocumentError("%s is a document method and not a valid field name" % field.name) + msg = ("%s is a document method and not a valid " + "field name" % field.name) + raise InvalidDocumentError(msg) + # Merge in exceptions with parent hierarchy + exceptions_to_merge = (DoesNotExist, MultipleObjectsReturned) module = attrs.get('__module__') + for exc in exceptions_to_merge: + name = exc.__name__ + parents = tuple(getattr(base, name) for base in bases + if hasattr(base, name)) or (exc,) + # Create new exception and set to new_class + exception = type(name, parents, {'__module__': module}) + setattr(new_class, name, exception) - base_excs = tuple(base.DoesNotExist for base in bases - if hasattr(base, 'DoesNotExist')) or (DoesNotExist,) - exc = subclass_exception('DoesNotExist', base_excs, module) - new_class.add_to_class('DoesNotExist', exc) + # Add class to the _document_registry + _document_registry[new_class._class_name] = new_class - base_excs = tuple(base.MultipleObjectsReturned for base in bases - if hasattr(base, 'MultipleObjectsReturned')) - base_excs = base_excs or (MultipleObjectsReturned,) - exc = subclass_exception('MultipleObjectsReturned', base_excs, module) - new_class.add_to_class('MultipleObjectsReturned', exc) - - global _document_registry - _document_registry[doc_class_name] = new_class - - # in Python 2, User-defined methods objects have special read-only + # In Python 2, User-defined methods objects have special read-only # attributes 'im_func' and 'im_self' which contain the function obj # and class instance object respectively. With Python 3 these special # attributes have been replaced by __func__ and __self__. The Blinker @@ -633,15 +621,40 @@ class DocumentMetaclass(type): if isinstance(val, classmethod): f = val.__get__(new_class) if hasattr(f, '__func__') and not hasattr(f, 'im_func'): - f.__dict__.update({'im_func':getattr(f, '__func__')}) + f.__dict__.update({'im_func': getattr(f, '__func__')}) if hasattr(f, '__self__') and not hasattr(f, 'im_self'): - f.__dict__.update({'im_self':getattr(f, '__self__')}) + f.__dict__.update({'im_self': getattr(f, '__self__')}) return new_class def add_to_class(self, name, value): setattr(self, name, value) + @classmethod + def _get_bases(cls, bases): + if isinstance(bases, BasesTuple): + return bases + seen = [] + bases = cls.__get_bases(bases) + unique_bases = (b for b in bases if not (b in seen or seen.append(b))) + return BasesTuple(unique_bases) + + @classmethod + def __get_bases(cls, bases): + for base in bases: + if base is object: + continue + yield base + for child_base in cls.__get_bases(base.__bases__): + yield child_base + + @classmethod + def _import_classes(cls): + Document = _import_class('Document') + EmbeddedDocument = _import_class('EmbeddedDocument') + DictField = _import_class('DictField') + return (Document, EmbeddedDocument, DictField) + class TopLevelDocumentMetaclass(DocumentMetaclass): """Metaclass for top-level documents (i.e. documents that have their own @@ -649,125 +662,157 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): """ def __new__(cls, name, bases, attrs): + + bases = cls._get_bases(bases) super_new = super(TopLevelDocumentMetaclass, cls).__new__ - # Classes defined in this package are abstract and should not have - # their own metadata with DB collection, etc. - # __metaclass__ is only set on the class with the __metaclass__ - # attribute (i.e. it is not set on subclasses). This differentiates - # 'real' documents from the 'Document' class - # - # Also assume a class is abstract if it has abstract set to True in - # its meta dictionary. This allows custom Document superclasses. - if (attrs.get('my_metaclass') == TopLevelDocumentMetaclass or - ('meta' in attrs and attrs['meta'].get('abstract', False))): - # Make sure no base class was non-abstract - non_abstract_bases = [b for b in bases - if hasattr(b, '_meta') and not b._meta.get('abstract', False)] - if non_abstract_bases: - raise ValueError("Abstract document cannot have non-abstract base") + + # Set default _meta data if base class, otherwise get user defined meta + if (attrs.get('my_metaclass') == TopLevelDocumentMetaclass): + # defaults + attrs['_meta'] = { + 'abstract': True, + 'max_documents': None, + 'max_size': None, + 'ordering': [], # default ordering applied at runtime + 'indexes': [], # indexes to be ensured at runtime + 'id_field': None, + 'index_background': False, + 'index_drop_dups': False, + 'index_opts': None, + 'delete_rules': None, + 'allow_inheritance': None, + } + attrs['_is_base_cls'] = True + attrs['_meta'].update(attrs.get('meta', {})) + else: + attrs['_meta'] = attrs.get('meta', {}) + # Explictly set abstract to false unless set + attrs['_meta']['abstract'] = attrs['_meta'].get('abstract', False) + attrs['_is_base_cls'] = False + + # Set flag marking as document class - as opposed to an object mixin + attrs['_is_document'] = True + + # Ensure queryset_class is inherited + if 'objects' in attrs: + manager = attrs['objects'] + if hasattr(manager, 'queryset_class'): + attrs['_meta']['queryset_class'] = manager.queryset_class + + # Clean up top level meta + if 'meta' in attrs: + del(attrs['meta']) + + # Find the parent document class + parent_doc_cls = [b for b in bases + if b.__class__ == TopLevelDocumentMetaclass] + parent_doc_cls = None if not parent_doc_cls else parent_doc_cls[0] + + # Prevent classes setting collection different to their parents + # If parent wasn't an abstract class + if (parent_doc_cls and 'collection' in attrs.get('_meta', {}) + and not parent_doc_cls._meta.get('abstract', True)): + msg = "Trying to set a collection on a subclass (%s)" % name + warnings.warn(msg, SyntaxWarning) + del(attrs['_meta']['collection']) + + # Ensure abstract documents have abstract bases + if attrs.get('_is_base_cls') or attrs['_meta'].get('abstract'): + if (parent_doc_cls and + not parent_doc_cls._meta.get('abstract', False)): + msg = "Abstract document cannot have non-abstract base" + raise ValueError(msg) return super_new(cls, name, bases, attrs) - collection = ''.join('_%s' % c if c.isupper() else c for c in name).strip('_').lower() + # Merge base class metas. + # Uses a special MetaDict that handles various merging rules + meta = MetaDict() + for base in bases[::-1]: + # Add any mixin metadata from plain objects + if hasattr(base, 'meta'): + meta.merge(base.meta) + elif hasattr(base, '_meta'): + # Warn if allow_inheritance isn't set and prevent + # inheritance of classes where inheritance is set to False + allow_inheritance = base._meta.get('allow_inheritance', + ALLOW_INHERITANCE) + if not base._is_base_cls and allow_inheritance is None: + warnings.warn( + "%s uses inheritance, the default for " + "allow_inheritance is changing to off by default. " + "Please add it to the document meta." % name, + FutureWarning + ) + elif (allow_inheritance == False and + not base._meta.get('abstract')): + raise ValueError('Document %s may not be subclassed' % + base.__name__) + meta.merge(base._meta) - id_field = None - abstract_base_indexes = [] - base_indexes = [] - base_meta = {} + # Set collection in the meta if its callable + if (getattr(base, '_is_document', False) and + not base._meta.get('abstract')): + collection = meta.get('collection', None) + if callable(collection): + meta['collection'] = collection(base) - # Subclassed documents inherit collection from superclass - for base in bases: - if hasattr(base, '_meta'): - if 'collection' in attrs.get('meta', {}) and not base._meta.get('abstract', False): - import warnings - msg = "Trying to set a collection on a subclass (%s)" % name - warnings.warn(msg, SyntaxWarning) - del(attrs['meta']['collection']) - if base._get_collection_name(): - collection = base._get_collection_name() + # Standard object mixin - merge in any Fields + if not hasattr(base, '_meta'): + attrs.update(dict([(k, v) for k, v in base.__dict__.items() + if issubclass(v.__class__, BaseField)])) - # Propagate inherited values - keys_to_propogate = ( - 'index_background', 'index_drop_dups', 'index_opts', - 'allow_inheritance', 'queryset_class', 'db_alias', - 'shard_key' - ) - for key in keys_to_propogate: - if key in base._meta: - base_meta[key] = base._meta[key] + meta.merge(attrs.get('_meta', {})) # Top level meta - id_field = id_field or base._meta.get('id_field') - if base._meta.get('abstract', False): - abstract_base_indexes += base._meta.get('indexes', []) - else: - base_indexes += base._meta.get('indexes', []) - try: - base_meta['objects'] = base.__getattribute__(base, 'objects') - except TypeError: - pass - except AttributeError: - pass + # Only simple classes (direct subclasses of Document) + # may set allow_inheritance to False + simple_class = all([b._meta.get('abstract') + for b in bases if hasattr(b, '_meta')]) + if (not simple_class and meta['allow_inheritance'] == False and + not meta['abstract']): + raise ValueError('Only direct subclasses of Document may set ' + '"allow_inheritance" to False') - # defaults - meta = { - 'abstract': False, - 'collection': collection, - 'max_documents': None, - 'max_size': None, - 'ordering': [], # default ordering applied at runtime - 'indexes': [], # indexes to be ensured at runtime - 'id_field': id_field, - 'index_background': False, - 'index_drop_dups': False, - 'index_opts': {}, - 'queryset_class': QuerySet, - 'delete_rules': {}, - 'allow_inheritance': True - } - - allow_inheritance_defined = ('allow_inheritance' in base_meta or - 'allow_inheritance'in attrs.get('meta', {})) - meta['allow_inheritance_defined'] = allow_inheritance_defined - meta.update(base_meta) - - # Apply document-defined meta options - meta.update(attrs.get('meta', {})) + # Set default collection name + if 'collection' not in meta: + meta['collection'] = ''.join('_%s' % c if c.isupper() else c + for c in name).strip('_').lower() attrs['_meta'] = meta - # Set up collection manager, needs the class to have fields so use - # DocumentMetaclass before instantiating CollectionManager object + # Call super and get the new class new_class = super_new(cls, name, bases, attrs) - collection = attrs['_meta'].get('collection', None) - if callable(collection): - new_class._meta['collection'] = collection(new_class) - - # Provide a default queryset unless one has been manually provided - manager = attrs.get('objects', meta.get('objects', QuerySetManager())) - if hasattr(manager, 'queryset_class'): - meta['queryset_class'] = manager.queryset_class - new_class.objects = manager - - indicies = list(meta['indexes']) + abstract_base_indexes - user_indexes = [QuerySet._build_index_spec(new_class, spec) - for spec in indicies] + base_indexes - new_class._meta['indexes'] = user_indexes + meta = new_class._meta + # Set index specifications + meta['index_specs'] = [QuerySet._build_index_spec(new_class, spec) + for spec in meta['indexes']] unique_indexes = cls._unique_with_indexes(new_class) new_class._meta['unique_indexes'] = unique_indexes + # If collection is a callable - call it and set the value + collection = meta.get('collection') + if callable(collection): + new_class._meta['collection'] = collection(new_class) + + # Provide a default queryset unless one has been set + manager = attrs.get('objects', QuerySetManager()) + new_class.objects = manager + + # Validate the fields and set primary key if needed for field_name, field in new_class._fields.iteritems(): - # Check for custom primary key if field.primary_key: - current_pk = new_class._meta['id_field'] + # Ensure only one primary key is set + current_pk = new_class._meta.get('id_field') if current_pk and current_pk != field_name: raise ValueError('Cannot override primary key field') + # Set primary key if not current_pk: new_class._meta['id_field'] = field_name - # Make 'Document.id' an alias to the real primary key field new_class.id = field - if not new_class._meta['id_field']: + # Set primary key if not defined by the document + if not new_class._meta.get('id_field'): new_class._meta['id_field'] = 'id' new_class._fields['id'] = ObjectIdField(db_field='_id') new_class.id = new_class._fields['id'] @@ -776,6 +821,9 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): @classmethod def _unique_with_indexes(cls, new_class, namespace=""): + """ + Find and set unique indexes + """ unique_indexes = [] for field_name, field in new_class._fields.items(): # Generate a list of indexes needed by uniqueness constraints @@ -801,18 +849,34 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): unique_fields += unique_with # Add the new index to the list - index = [("%s%s" % (namespace, f), pymongo.ASCENDING) for f in unique_fields] + index = [("%s%s" % (namespace, f), pymongo.ASCENDING) + for f in unique_fields] unique_indexes.append(index) # Grab any embedded document field unique indexes - if field.__class__.__name__ == "EmbeddedDocumentField" and field.document_type != new_class: + if (field.__class__.__name__ == "EmbeddedDocumentField" and + field.document_type != new_class): field_namespace = "%s." % field_name unique_indexes += cls._unique_with_indexes(field.document_type, - field_namespace) + field_namespace) return unique_indexes +class MetaDict(dict): + """Custom dictionary for meta classes. + Handles the merging of set indexes + """ + _merge_options = ('indexes',) + + def merge(self, new_options): + for k, v in new_options.iteritems(): + if k in self._merge_options: + self[k] = self.get(k, []) + v + else: + self[k] = v + + class BaseDocument(object): _dynamic = False @@ -877,10 +941,11 @@ class BaseDocument(object): self._data[name] = value if hasattr(self, '_changed_fields'): self._mark_as_changed(name) - - if not self._created and name in self._meta.get('shard_key', tuple()): - from queryset import OperationError - raise OperationError("Shard Keys are immutable. Tried to update %s" % name) + if (self._is_document and not self._created and + name in self._meta.get('shard_key', tuple())): + OperationError = _import_class('OperationError') + msg = "Shard Keys are immutable. Tried to update %s" % name + raise OperationError(msg) super(BaseDocument, self).__setattr__(name, value) @@ -912,7 +977,8 @@ class BaseDocument(object): value = data # Convert lists / values so we can watch for any changes on them - if isinstance(value, (list, tuple)) and not isinstance(value, BaseList): + if (isinstance(value, (list, tuple)) and + not isinstance(value, BaseList)): value = BaseList(value, self, name) elif isinstance(value, dict) and not isinstance(value, BaseDict): value = BaseDict(value, self, name) @@ -953,7 +1019,7 @@ class BaseDocument(object): data[field.db_field] = field.to_mongo(value) # Only add _cls and _types if allow_inheritance is not False if not (hasattr(self, '_meta') and - self._meta.get('allow_inheritance', True) == False): + self._meta.get('allow_inheritance', ALLOW_INHERITANCE) == False): data['_cls'] = self._class_name data['_types'] = self._superclasses.keys() + [self._class_name] if '_id' in data and data['_id'] is None: @@ -1012,9 +1078,11 @@ class BaseDocument(object): changed_fields.append(field_name) if errors_dict: - errors = "\n".join(["%s - %s" % (k, v) for k, v in errors_dict.items()]) - raise InvalidDocumentError(""" -Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, errors)) + errors = "\n".join(["%s - %s" % (k, v) + for k, v in errors_dict.items()]) + msg = ("Invalid data to create a `%s` instance.\n%s" + % (cls._class_name, errors)) + raise InvalidDocumentError(msg) if PY25: data = dict([(str(k), v) for k, v in data.items()]) @@ -1029,7 +1097,8 @@ Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, error if not key: return key = self._db_field_map.get(key, key) - if hasattr(self, '_changed_fields') and key not in self._changed_fields: + if (hasattr(self, '_changed_fields') and + key not in self._changed_fields): self._changed_fields.append(key) def _get_changed_fields(self, key='', inspected=None): @@ -1059,9 +1128,14 @@ Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, error continue inspected.add(field.id) - if isinstance(field, (EmbeddedDocument, DynamicEmbeddedDocument)) and db_field_name not in _changed_fields: # Grab all embedded fields that have been changed - _changed_fields += ["%s%s" % (key, k) for k in field._get_changed_fields(key, inspected) if k] - elif isinstance(field, (list, tuple, dict)) and db_field_name not in _changed_fields: # Loop list / dict fields as they contain documents + if (isinstance(field, (EmbeddedDocument, DynamicEmbeddedDocument)) + and db_field_name not in _changed_fields): + # Find all embedded fields that have been changed + changed = field._get_changed_fields(key, inspected) + _changed_fields += ["%s%s" % (key, k) for k in changed if k] + elif (isinstance(field, (list, tuple, dict)) and + db_field_name not in _changed_fields): + # Loop list / dict fields as they contain documents # Determine the iterator to use if not hasattr(field, 'items'): iterator = enumerate(field) @@ -1071,7 +1145,9 @@ Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, error if not hasattr(value, '_get_changed_fields'): continue list_key = "%s%s." % (key, index) - _changed_fields += ["%s%s" % (list_key, k) for k in value._get_changed_fields(list_key, inspected) if k] + changed = value._get_changed_fields(list_key, inspected) + _changed_fields += ["%s%s" % (list_key, k) + for k in changed if k] return _changed_fields def _delta(self): @@ -1113,7 +1189,8 @@ Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, error # If we've set a value that ain't the default value dont unset it. default = None - if self._dynamic and len(parts) and parts[0] in self._dynamic_fields: + if (self._dynamic and len(parts) and + parts[0] in self._dynamic_fields): del(set_data[path]) unset_data[path] = 1 continue @@ -1126,7 +1203,8 @@ Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, error for p in parts: if p.isdigit(): d = d[int(p)] - elif hasattr(d, '__getattribute__') and not isinstance(d, dict): + elif (hasattr(d, '__getattribute__') and + not isinstance(d, dict)): real_path = d._reverse_db_field_map.get(p, p) d = getattr(d, real_path) else: @@ -1172,7 +1250,8 @@ Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, error return geo_indices def __getstate__(self): - removals = ["get_%s_display" % k for k, v in self._fields.items() if v.choices] + removals = ("get_%s_display" % k + for k, v in self._fields.items() if v.choices) for k in removals: if hasattr(self, k): delattr(self, k) @@ -1183,9 +1262,12 @@ Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, error self.__set_field_display() def __set_field_display(self): + """Dynamically set the display value for a field with choices""" for attr_name, field in self._fields.items(): - if field.choices: # dynamically adds a way to get the display value for a field with choices - setattr(self, 'get_%s_display' % attr_name, partial(self.__get_field_display, field=field)) + if field.choices: + setattr(self, + 'get_%s_display' % attr_name, + partial(self.__get_field_display, field=field)) def __get_field_display(self, field): """Returns the display value for a choice field""" @@ -1257,6 +1339,11 @@ Invalid data to create a `%s` instance.\n%s""".strip() % (cls._class_name, error return hash(self.pk) +class BasesTuple(tuple): + """Special class to handle introspection of bases tuple in __new__""" + pass + + class BaseList(list): """A special list so we can watch any changes """ @@ -1378,5 +1465,18 @@ class BaseDict(dict): self._instance._mark_as_changed(self._name) -def subclass_exception(name, parents, module): - return type(name, parents, {'__module__': module}) +def _import_class(cls_name): + """Cached mechanism for imports""" + if cls_name in _class_registry: + return _class_registry.get(cls_name) + if cls_name == 'Document': + from mongoengine.document import Document as cls + elif cls_name == 'EmbeddedDocument': + from mongoengine.document import EmbeddedDocument as cls + elif cls_name == 'DictField': + from mongoengine.fields import DictField as cls + elif cls_name == 'OperationError': + from queryset import OperationError as cls + + _class_registry[cls_name] = cls + return cls diff --git a/mongoengine/document.py b/mongoengine/document.py index 46111be..f6b0b51 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -360,7 +360,9 @@ class Document(BaseDocument): """This method registers the delete rules to apply when removing this object. """ - cls._meta['delete_rules'][(document_cls, field_name)] = rule + delete_rules = cls._meta.get('delete_rules') or {} + delete_rules[(document_cls, field_name)] = rule + cls._meta['delete_rules'] = delete_rules @classmethod def drop_collection(cls): @@ -392,6 +394,7 @@ class DynamicDocument(Document): __metaclass__ = TopLevelDocumentMetaclass _dynamic = True + meta = {'abstract': True} def __delattr__(self, *args, **kwargs): """Deletes the attribute by setting to None and allowing _delta to unset diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 448b0c6..4202513 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -747,7 +747,6 @@ class ReferenceField(BaseField): def prepare_query_value(self, op, value): if value is None: return None - return self.to_mongo(value) def validate(self, value): diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index dbd8ad4..446a42a 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -352,7 +352,7 @@ class QuerySet(object): # If inheritance is allowed, only return instances and instances of # subclasses of the class being used - if document._meta.get('allow_inheritance'): + if document._meta.get('allow_inheritance') != False: self._initial_query = {'_types': self._document._class_name} self._loaded_fields = QueryFieldList(always_include=['_cls']) self._cursor_obj = None @@ -442,7 +442,7 @@ class QuerySet(object): """ background = self._document._meta.get('index_background', False) drop_dups = self._document._meta.get('index_drop_dups', False) - index_opts = self._document._meta.get('index_opts', {}) + index_opts = self._document._meta.get('index_opts') or {} index_types = self._document._meta.get('index_types', True) # determine if an index which we are creating includes @@ -450,6 +450,7 @@ class QuerySet(object): # an extra index on _type, as mongodb will use the existing # index to service queries against _type types_indexed = False + def includes_types(fields): first_field = None if len(fields): @@ -466,8 +467,8 @@ class QuerySet(object): background=background, drop_dups=drop_dups, **index_opts) # Ensure document-defined indexes are created - if self._document._meta['indexes']: - for spec in self._document._meta['indexes']: + if self._document._meta['index_specs']: + for spec in self._document._meta['index_specs']: types_indexed = types_indexed or includes_types(spec['fields']) opts = index_opts.copy() opts['unique'] = spec.get('unique', False) @@ -498,7 +499,10 @@ class QuerySet(object): index_list = [] direction = None - use_types = doc_cls._meta.get('allow_inheritance', True) + + allow_inheritance = doc_cls._meta.get('allow_inheritance') != False + use_types = allow_inheritance + for key in spec['fields']: # Get ASCENDING direction from +, DESCENDING from -, and GEO2D from * direction = pymongo.ASCENDING @@ -516,7 +520,8 @@ class QuerySet(object): key = '_id' else: fields = QuerySet._lookup_field(doc_cls, parts) - parts = [field if field == '_id' else field.db_field for field in fields] + parts = [field if field == '_id' else field.db_field + for field in fields] key = '.'.join(parts) index_list.append((key, direction)) @@ -530,8 +535,9 @@ class QuerySet(object): # If _types is being used, prepend it to every specified index index_types = doc_cls._meta.get('index_types', True) - allow_inheritance = doc_cls._meta.get('allow_inheritance') - if spec.get('types', index_types) and allow_inheritance and use_types and direction is not pymongo.GEO2D: + + if (spec.get('types', index_types) and allow_inheritance and use_types + and direction is not pymongo.GEO2D): index_list.insert(0, ('_types', 1)) spec['fields'] = index_list @@ -1329,9 +1335,10 @@ class QuerySet(object): """ doc = self._document + delete_rules = doc._meta.get('delete_rules') or {} # Check for DENY rules before actually deleting/nullifying any other # references - for rule_entry in doc._meta['delete_rules']: + for rule_entry in delete_rules: document_cls, field_name = rule_entry rule = doc._meta['delete_rules'][rule_entry] if rule == DENY and document_cls.objects(**{field_name + '__in': self}).count() > 0: @@ -1339,12 +1346,14 @@ class QuerySet(object): (document_cls.__name__, field_name) raise OperationError(msg) - for rule_entry in doc._meta['delete_rules']: + for rule_entry in delete_rules: document_cls, field_name = rule_entry rule = doc._meta['delete_rules'][rule_entry] if rule == CASCADE: ref_q = document_cls.objects(**{field_name + '__in': self}) - if doc != document_cls or (doc == document_cls and ref_q.count() > 0): + ref_q_count = ref_q.count() + if (doc != document_cls and ref_q_count > 0 + or (doc == document_cls and ref_q_count > 0)): ref_q.delete(safe=safe) elif rule == NULLIFY: document_cls.objects(**{field_name + '__in': self}).update( @@ -1915,7 +1924,7 @@ class QuerySetManager(object): return self # owner is the document that contains the QuerySetManager - queryset_class = owner._meta['queryset_class'] or QuerySet + queryset_class = owner._meta.get('queryset_class') or QuerySet queryset = queryset_class(owner, owner._get_collection()) if self.get_queryset: arg_count = self.get_queryset.func_code.co_argcount diff --git a/setup.cfg b/setup.cfg index f28e668..d95a917 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [nosetests] -verbosity = 2 +verbosity = 3 detailed-errors = 1 #with-coverage = 1 #cover-erase = 1 @@ -8,4 +8,4 @@ detailed-errors = 1 #cover-package = mongoengine py3where = build where = tests -#tests = test_bugfix.py \ No newline at end of file +#tests = test_bugfix.py \ No newline at end of file diff --git a/tests/test_django.py b/tests/test_django.py index 678d7cf..398fd3e 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -18,8 +18,8 @@ try: from mongoengine.django.sessions import SessionStore, MongoSession except Exception, err: if PY3: - SessionTestsMixin = type #dummy value so no error - SessionStore = None #dummy value so no error + SessionTestsMixin = type # dummy value so no error + SessionStore = None # dummy value so no error else: raise err diff --git a/tests/test_document.py b/tests/test_document.py index 8654e3f..da329ca 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -387,19 +387,6 @@ class DocumentTest(unittest.TestCase): meta = {'allow_inheritance': False} self.assertRaises(ValueError, create_employee_class) - # Test the same for embedded documents - class Comment(EmbeddedDocument): - content = StringField() - meta = {'allow_inheritance': False} - - def create_special_comment(): - class SpecialComment(Comment): - pass - self.assertRaises(ValueError, create_special_comment) - - comment = Comment(content='test') - self.assertFalse('_cls' in comment.to_mongo()) - self.assertFalse('_types' in comment.to_mongo()) def test_allow_inheritance_abstract_document(self): """Ensure that abstract documents can set inheritance rules and that @@ -491,9 +478,20 @@ class DocumentTest(unittest.TestCase): """Ensure that a document superclass can be marked as abstract thereby not using it as the name for the collection.""" + defaults = {'index_background': True, + 'index_drop_dups': True, + 'index_opts': {'hello': 'world'}, + 'allow_inheritance': True, + 'queryset_class': 'QuerySet', + 'db_alias': 'myDB', + 'shard_key': ('hello', 'world')} + + meta_settings = {'abstract': True} + meta_settings.update(defaults) + class Animal(Document): name = StringField() - meta = {'abstract': True} + meta = meta_settings class Fish(Animal): pass class Guppy(Fish): pass @@ -502,6 +500,10 @@ class DocumentTest(unittest.TestCase): meta = {'abstract': True} class Human(Mammal): pass + for k, v in defaults.iteritems(): + for cls in [Animal, Fish, Guppy]: + self.assertEqual(cls._meta[k], v) + self.assertFalse('collection' in Animal._meta) self.assertFalse('collection' in Mammal._meta) @@ -564,6 +566,7 @@ class DocumentTest(unittest.TestCase): class Drink(Document): name = StringField() + meta = {'allow_inheritance': True} class Drinker(Document): drink = GenericReferenceField() @@ -799,7 +802,6 @@ class DocumentTest(unittest.TestCase): user_guid = StringField(required=True) - class Person(UserBase): meta = { 'indexes': ['name'], @@ -1325,7 +1327,6 @@ class DocumentTest(unittest.TestCase): self.assertTrue('content' in Comment._fields) self.assertFalse('id' in Comment._fields) - self.assertFalse('collection' in Comment._meta) def test_embedded_document_validation(self): """Ensure that embedded documents may be validated. @@ -2504,32 +2505,24 @@ class DocumentTest(unittest.TestCase): def test_mixins_dont_add_to_types(self): - class Bob(Document): name = StringField() - - Bob.drop_collection() - - p = Bob(name="Rozza") - p.save() - Bob.drop_collection() + class Mixin(object): + name = StringField() class Person(Document, Mixin): pass Person.drop_collection() - p = Person(name="Rozza") - p.save() - self.assertEqual(p._fields.keys(), ['name', 'id']) + self.assertEqual(Person._fields.keys(), ['name', 'id']) + + Person(name="Rozza").save() collection = self.db[Person._get_collection_name()] obj = collection.find_one() self.assertEqual(obj['_cls'], 'Person') self.assertEqual(obj['_types'], ['Person']) - - self.assertEqual(Person.objects.count(), 1) - rozza = Person.objects.get(name="Rozza") Person.drop_collection() @@ -2668,16 +2661,18 @@ class DocumentTest(unittest.TestCase): self.assertEqual(len(BlogPost.objects), 0) def test_reverse_delete_rule_cascade_and_nullify_complex_field(self): - """Ensure that a referenced document is also deleted upon deletion. + """Ensure that a referenced document is also deleted upon deletion for + complex fields. """ - class BlogPost(Document): + class BlogPost2(Document): content = StringField() authors = ListField(ReferenceField(self.Person, reverse_delete_rule=CASCADE)) reviewers = ListField(ReferenceField(self.Person, reverse_delete_rule=NULLIFY)) self.Person.drop_collection() - BlogPost.drop_collection() + + BlogPost2.drop_collection() author = self.Person(name='Test User') author.save() @@ -2685,18 +2680,19 @@ class DocumentTest(unittest.TestCase): reviewer = self.Person(name='Re Viewer') reviewer.save() - post = BlogPost(content= 'Watched some TV') + post = BlogPost2(content='Watched some TV') post.authors = [author] post.reviewers = [reviewer] post.save() + # Deleting the reviewer should have no effect on the BlogPost2 reviewer.delete() - self.assertEqual(len(BlogPost.objects), 1) # No effect on the BlogPost - self.assertEqual(BlogPost.objects.get().reviewers, []) + self.assertEqual(len(BlogPost2.objects), 1) + self.assertEqual(BlogPost2.objects.get().reviewers, []) # Delete the Person, which should lead to deletion of the BlogPost, too author.delete() - self.assertEqual(len(BlogPost.objects), 0) + self.assertEqual(len(BlogPost2.objects), 0) def test_two_way_reverse_delete_rule(self): """Ensure that Bi-Directional relationships work with @@ -3074,7 +3070,7 @@ class DocumentTest(unittest.TestCase): self.assertEqual('testdb-1', B._meta.get('db_alias')) def test_db_ref_usage(self): - """ DB Ref usage in __raw__ queries """ + """ DB Ref usage in dict_fields""" class User(Document): name = StringField() @@ -3216,7 +3212,6 @@ class ValidatorErrorTest(unittest.TestCase): one = Doc.objects.filter(**{'hello world': 1}).count() self.assertEqual(1, one) - def test_fields_rewrite(self): class BasePerson(Document): name = StringField() @@ -3226,7 +3221,6 @@ class ValidatorErrorTest(unittest.TestCase): class Person(BasePerson): name = StringField(required=True) - p = Person(age=15) self.assertRaises(ValidationError, p.validate) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 979dc6f..591a82a 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -24,6 +24,8 @@ class QuerySetTest(unittest.TestCase): name = StringField() age = IntField() meta = {'allow_inheritance': True} + + Person.drop_collection() self.Person = Person def test_initialisation(self): From 66279bd90f57c964e1d4aa1c3356cd2320847735 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 17 Aug 2012 11:58:57 +0100 Subject: [PATCH 0736/1279] Post refactor cleanups (ref: meta cleanups) --- docs/changelog.rst | 4 +--- mongoengine/document.py | 1 - tests/test_document.py | 14 +++++++------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3bca6a1..070083e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,9 +4,7 @@ Changelog Changes in 0.7.X ================= -- Embedded Documents dont care about inheritance - - +- Embedded Documents no longer handle meta definitions - Use weakref proxies in base lists / dicts (MongoEngine/mongoengine#74) - Improved queryset filtering (hmarr/mongoengine#554) - Fixed Dynamic Documents and Embedded Documents (hmarr/mongoengine#561) diff --git a/mongoengine/document.py b/mongoengine/document.py index f6b0b51..23f3a23 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -394,7 +394,6 @@ class DynamicDocument(Document): __metaclass__ = TopLevelDocumentMetaclass _dynamic = True - meta = {'abstract': True} def __delattr__(self, *args, **kwargs): """Deletes the attribute by setting to None and allowing _delta to unset diff --git a/tests/test_document.py b/tests/test_document.py index da329ca..d4b2673 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -2665,14 +2665,14 @@ class DocumentTest(unittest.TestCase): complex fields. """ - class BlogPost2(Document): + class BlogPost(Document): content = StringField() authors = ListField(ReferenceField(self.Person, reverse_delete_rule=CASCADE)) reviewers = ListField(ReferenceField(self.Person, reverse_delete_rule=NULLIFY)) self.Person.drop_collection() - BlogPost2.drop_collection() + BlogPost.drop_collection() author = self.Person(name='Test User') author.save() @@ -2680,19 +2680,19 @@ class DocumentTest(unittest.TestCase): reviewer = self.Person(name='Re Viewer') reviewer.save() - post = BlogPost2(content='Watched some TV') + post = BlogPost(content='Watched some TV') post.authors = [author] post.reviewers = [reviewer] post.save() - # Deleting the reviewer should have no effect on the BlogPost2 + # Deleting the reviewer should have no effect on the BlogPost reviewer.delete() - self.assertEqual(len(BlogPost2.objects), 1) - self.assertEqual(BlogPost2.objects.get().reviewers, []) + self.assertEqual(len(BlogPost.objects), 1) + self.assertEqual(BlogPost.objects.get().reviewers, []) # Delete the Person, which should lead to deletion of the BlogPost, too author.delete() - self.assertEqual(len(BlogPost2.objects), 0) + self.assertEqual(len(BlogPost.objects), 0) def test_two_way_reverse_delete_rule(self): """Ensure that Bi-Directional relationships work with From 22010d7d959f4d7183c1ee257b832982c55ea4d8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 17 Aug 2012 15:04:09 +0100 Subject: [PATCH 0737/1279] Updated benchmark stats --- benchmark.py | 49 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/benchmark.py b/benchmark.py index 247baeb..0197e1d 100644 --- a/benchmark.py +++ b/benchmark.py @@ -28,47 +28,64 @@ def main(): ---------------------------------------------------------------------------------------------------- Creating 10000 dictionaries - Pymongo - 1.1141769886 + 3.86744189262 ---------------------------------------------------------------------------------------------------- Creating 10000 dictionaries - MongoEngine - 2.37724113464 + 6.23374891281 ---------------------------------------------------------------------------------------------------- Creating 10000 dictionaries - MongoEngine, safe=False, validate=False - 1.92479610443 + 5.33027005196 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False + pass - No Cascade 0.5.X ---------------------------------------------------------------------------------------------------- Creating 10000 dictionaries - Pymongo - 1.10552310944 + 3.89597702026 ---------------------------------------------------------------------------------------------------- Creating 10000 dictionaries - MongoEngine - 16.5169169903 + 21.7735359669 ---------------------------------------------------------------------------------------------------- Creating 10000 dictionaries - MongoEngine, safe=False, validate=False - 14.9446101189 + 19.8670389652 ---------------------------------------------------------------------------------------------------- Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False - 14.912801981 - ---------------------------------------------------------------------------------------------------- - Creating 10000 dictionaries - MongoEngine, force=True - 14.9617750645 + pass - No Cascade - Performance + 0.6.X ---------------------------------------------------------------------------------------------------- Creating 10000 dictionaries - Pymongo - 1.10072994232 + 3.81559205055 ---------------------------------------------------------------------------------------------------- Creating 10000 dictionaries - MongoEngine - 5.27341103554 + 10.0446798801 ---------------------------------------------------------------------------------------------------- Creating 10000 dictionaries - MongoEngine, safe=False, validate=False - 4.49365401268 + 9.51354718208 ---------------------------------------------------------------------------------------------------- Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False - 4.43459296227 + 9.02567505836 ---------------------------------------------------------------------------------------------------- Creating 10000 dictionaries - MongoEngine, force=True - 4.40114378929 + 8.44933390617 + + 0.7.X + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - Pymongo + 3.78801012039 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine + 9.73050498962 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, safe=False, validate=False + 8.33456707001 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False + 8.37778115273 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, force=True + 8.36906409264 """ setup = """ From 90fa0f6c4a5ba02dda4a478b74b1c082eb7a495d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 17 Aug 2012 16:02:33 +0100 Subject: [PATCH 0738/1279] Add flexibility for fields handling bad data (MongoEngine/mongoengine#78) --- docs/changelog.rst | 1 + mongoengine/document.py | 1 - mongoengine/fields.py | 48 ++++++++++++++++++++++++++++++++--------- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 070083e..3a55bc8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.X ================= +- Add flexibility for fields handling bad data (MongoEngine/mongoengine#78) - Embedded Documents no longer handle meta definitions - Use weakref proxies in base lists / dicts (MongoEngine/mongoengine#74) - Improved queryset filtering (hmarr/mongoengine#554) diff --git a/mongoengine/document.py b/mongoengine/document.py index 23f3a23..1b55e43 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -207,7 +207,6 @@ class Document(BaseDocument): else: object_id = doc['_id'] updates, removals = self._delta() - # Need to add shard key to query, or you get an error select_dict = {'_id': object_id} shard_key = self.__class__._meta.get('shard_key', tuple()) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 4202513..e6c65f5 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -49,8 +49,11 @@ class StringField(BaseField): def to_python(self, value): if isinstance(value, unicode): return value - else: - return value.decode('utf-8') + try: + value = value.decode('utf-8') + except: + pass + return value def validate(self, value): if not isinstance(value, basestring): @@ -150,7 +153,11 @@ class IntField(BaseField): super(IntField, self).__init__(**kwargs) def to_python(self, value): - return int(value) + try: + value = int(value) + except ValueError: + pass + return value def validate(self, value): try: @@ -180,7 +187,11 @@ class FloatField(BaseField): super(FloatField, self).__init__(**kwargs) def to_python(self, value): - return float(value) + try: + value = float(value) + except ValueError: + pass + return value def validate(self, value): if isinstance(value, int): @@ -212,9 +223,14 @@ class DecimalField(BaseField): super(DecimalField, self).__init__(**kwargs) def to_python(self, value): + original_value = value if not isinstance(value, basestring): value = unicode(value) - return decimal.Decimal(value) + try: + value = decimal.Decimal(value) + except ValueError: + return original_value + return value def to_mongo(self, value): return unicode(value) @@ -242,7 +258,11 @@ class BooleanField(BaseField): """ def to_python(self, value): - return bool(value) + try: + value = bool(value) + except ValueError: + pass + return value def validate(self, value): if not isinstance(value, bool): @@ -385,7 +405,11 @@ class ComplexDateTimeField(StringField): 'ComplexDateTimeField') def to_python(self, value): - return self._convert_from_string(value) + original_value = value + try: + return self._convert_from_string(value) + except: + return original_value def to_mongo(self, value): return self._convert_from_datetime(value) @@ -1340,9 +1364,13 @@ class UUIDField(BaseField): def to_python(self, value): if not self._binary: - if not isinstance(value, basestring): - value = unicode(value) - return uuid.UUID(value) + original_value = value + try: + if not isinstance(value, basestring): + value = unicode(value) + return uuid.UUID(value) + except: + return original_value return value def to_mongo(self, value): From e990a6c70c4d5f38dcf2210d9baa132e8e6fdc75 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 17 Aug 2012 16:13:45 +0100 Subject: [PATCH 0739/1279] Improve unicode key handling for PY25 --- mongoengine/base.py | 11 ++++++----- mongoengine/python_support.py | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index a05403d..3c3dcdc 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -11,7 +11,8 @@ from queryset import DoesNotExist, MultipleObjectsReturned from queryset import DO_NOTHING from mongoengine import signals -from mongoengine.python_support import PY3, PY25, txt_type +from mongoengine.python_support import (PY3, PY25, txt_type, + to_str_keys_recursive) import pymongo from bson import ObjectId @@ -961,8 +962,6 @@ class BaseDocument(object): if not is_list and '_cls' in value: cls = get_document(value['_cls']) - if PY25: - value = dict([(str(k), v) for k, v in value.items()]) return cls(**value) data = {} @@ -1046,6 +1045,10 @@ class BaseDocument(object): # class if unavailable class_name = son.get('_cls', cls._class_name) data = dict(("%s" % key, value) for key, value in son.items()) + if PY25: + # PY25 cannot handle unicode keys passed to class constructor + # example: cls(**data) + to_str_keys_recursive(data) if '_types' in data: del data['_types'] @@ -1084,8 +1087,6 @@ class BaseDocument(object): % (cls._class_name, errors)) raise InvalidDocumentError(msg) - if PY25: - data = dict([(str(k), v) for k, v in data.items()]) obj = cls(**data) obj._changed_fields = changed_fields obj._created = False diff --git a/mongoengine/python_support.py b/mongoengine/python_support.py index 70c3181..02dc286 100644 --- a/mongoengine/python_support.py +++ b/mongoengine/python_support.py @@ -41,3 +41,20 @@ if PY25: else: from itertools import product from functools import reduce + + +# For use with Python 2.5 +# converts all keys from unicode to str for d and all nested dictionaries +def to_str_keys_recursive(d): + if isinstance(d, list): + for val in d: + if isinstance(val, (dict, list)): + to_str_keys_recursive(val) + elif isinstance(d, dict): + for key, val in d.items(): + if isinstance(val, (dict, list)): + to_str_keys_recursive(val) + if isinstance(key, unicode): + d[str(key)] = d.pop(key) + else: + raise ValueError("non list/dict parameter not allowed") From 676a7bf712f014b2b3f96e05862dffc818d3e05c Mon Sep 17 00:00:00 2001 From: mikolaj Date: Sat, 18 Aug 2012 17:27:22 +0100 Subject: [PATCH 0740/1279] Image resize fails when Froce flag is set --- mongoengine/fields.py | 7 ++++--- tests/test_fields.py | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e6c65f5..ef20d89 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1108,6 +1108,7 @@ class ImageGridFsProxy(GridFSProxy): try: img = Image.open(file_obj) + img_format = img.format except: raise ValidationError('Invalid image') @@ -1142,20 +1143,20 @@ class ImageGridFsProxy(GridFSProxy): if thumbnail: thumb_id = self._put_thumbnail(thumbnail, - img.format) + img_format) else: thumb_id = None w, h = img.size io = StringIO() - img.save(io, img.format) + img.save(io, img_format) io.seek(0) return super(ImageGridFsProxy, self).put(io, width=w, height=h, - format=img.format, + format=img_format, thumbnail_id=thumb_id, **kwargs) diff --git a/tests/test_fields.py b/tests/test_fields.py index 8209938..ff9246b 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1851,6 +1851,29 @@ class FieldTest(unittest.TestCase): t.image.delete() + def test_image_field_resize_force(self): + if PY3: + raise SkipTest('PIL does not have Python 3 support') + + class TestImage(Document): + image = ImageField(size=(185, 37, True)) + + TestImage.drop_collection() + + t = TestImage() + t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.save() + + t = TestImage.objects.first() + + self.assertEquals(t.image.format, 'PNG') + w, h = t.image.size + + self.assertEquals(w, 185) + self.assertEquals(h, 37) + + t.image.delete() + def test_image_field_thumbnail(self): if PY3: raise SkipTest('PIL does not have Python 3 support') From be8d39a48c5c1dc9490bd820d90c71f6db64cc55 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 20 Aug 2012 17:35:19 +0100 Subject: [PATCH 0741/1279] Updated changelog / AUTHORS (MongoEngine/mongoengine#80) --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 22375f2..91993e2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -117,3 +117,4 @@ that much better: * Thomas Steinacher * Tommi Komulainen * Peter Landry + * biszkoptwielki diff --git a/docs/changelog.rst b/docs/changelog.rst index 3a55bc8..e92f343 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.X ================= +- Fixed ImageField resizing when forcing size - Add flexibility for fields handling bad data (MongoEngine/mongoengine#78) - Embedded Documents no longer handle meta definitions - Use weakref proxies in base lists / dicts (MongoEngine/mongoengine#74) From 1b80193aac08f176debdab759accaca96048c283 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 20 Aug 2012 17:50:18 +0100 Subject: [PATCH 0742/1279] Added example of indexing embedded fields MongoEngine/mongoengine#75 --- docs/changelog.rst | 3 ++- docs/guide/defining-documents.rst | 11 ++++++++--- tests/test_document.py | 26 +++++++++++++++++++++++++- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e92f343..2feda74 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,8 @@ Changelog Changes in 0.7.X ================= -- Fixed ImageField resizing when forcing size +- Added example of indexing embedded document fields (MongoEngine/mongoengine#75) +- Fixed ImageField resizing when forcing size (MongoEngine/mongoengine#80) - Add flexibility for fields handling bad data (MongoEngine/mongoengine#78) - Embedded Documents no longer handle meta definitions - Use weakref proxies in base lists / dicts (MongoEngine/mongoengine#74) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index b4facbd..6c8b29e 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -467,11 +467,16 @@ If a dictionary is passed then the following options are available: :attr:`unique` (Default: False) Whether the index should be sparse. +.. note :: + + To index embedded files / dictionary fields use 'dot' notation eg: + `rank.title` + .. warning:: - - Inheritance adds extra indices. - If don't need inheritance for a document turn inheritance off - see :ref:`document-inheritance`. + Inheritance adds extra indices. + If don't need inheritance for a document turn inheritance off - + see :ref:`document-inheritance`. Geospatial indexes diff --git a/tests/test_document.py b/tests/test_document.py index d4b2673..a464464 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -740,6 +740,30 @@ class DocumentTest(unittest.TestCase): self.assertEqual(Person.objects.get(name="Jack").rank, "Corporal") self.assertEqual(Person.objects.get(name="Fred").rank, "Private") + def test_embedded_document_index_meta(self): + """Ensure that embedded document indexes are created explicitly + """ + class Rank(EmbeddedDocument): + title = StringField(required=True) + + class Person(Document): + name = StringField(required=True) + rank = EmbeddedDocumentField(Rank, required=False) + + meta = { + 'indexes': [ + 'rank.title', + ], + } + + Person.drop_collection() + + # Indexes are lazy so use list() to perform query + list(Person.objects) + info = Person.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] + self.assertTrue([('rank.title', '1')] in info) + def test_explicit_geo2d_index(self): """Ensure that geo2d indexes work when created via meta[indexes] """ @@ -770,7 +794,7 @@ class DocumentTest(unittest.TestCase): tags = ListField(StringField()) meta = { 'indexes': [ - { 'fields': ['-date'], 'unique': True, + {'fields': ['-date'], 'unique': True, 'sparse': True, 'types': False }, ], } From 999d4a7676e54ec41eb3b4f3e97e342d05a32962 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 21 Aug 2012 17:29:38 +0100 Subject: [PATCH 0743/1279] Fixed broken test --- tests/test_document.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_document.py b/tests/test_document.py index a464464..f1c65b1 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -754,6 +754,7 @@ class DocumentTest(unittest.TestCase): 'indexes': [ 'rank.title', ], + 'allow_inheritance': False } Person.drop_collection() @@ -762,7 +763,7 @@ class DocumentTest(unittest.TestCase): list(Person.objects) info = Person.objects._collection.index_information() info = [value['key'] for key, value in info.iteritems()] - self.assertTrue([('rank.title', '1')] in info) + self.assertTrue([('rank.title', 1)] in info) def test_explicit_geo2d_index(self): """Ensure that geo2d indexes work when created via meta[indexes] From b1eeb77ddcdb0432b36740af5fe276ec791a40d8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 21 Aug 2012 17:45:51 +0100 Subject: [PATCH 0744/1279] Added FutureWarning - save will default to `cascade=False` in 0.8 --- docs/changelog.rst | 3 +- docs/upgrade.rst | 14 +++++++ mongoengine/document.py | 89 ++++++++++++++++++++++++----------------- tests/test_document.py | 34 ++++++++++++++++ 4 files changed, 103 insertions(+), 37 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2feda74..4b55f51 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.X ================= +- Added FutureWarning - save will default to `cascade=False` in 0.8 - Added example of indexing embedded document fields (MongoEngine/mongoengine#75) - Fixed ImageField resizing when forcing size (MongoEngine/mongoengine#80) - Add flexibility for fields handling bad data (MongoEngine/mongoengine#78) @@ -99,7 +100,7 @@ Changes in 0.6.8 ================ - Fixed FileField losing reference when no default set - Removed possible race condition from FileField (grid_file) -- Added assignment to save, can now do: b = MyDoc(**kwargs).save() +- Added assignment to save, can now do: `b = MyDoc(**kwargs).save()` - Added support for pull operations on nested EmbeddedDocuments - Added support for choices with GenericReferenceFields - Added support for choices with GenericEmbeddedDocumentFields diff --git a/docs/upgrade.rst b/docs/upgrade.rst index bfb5cc5..4f92fdf 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -2,6 +2,20 @@ Upgrading ========= +0.6 to 0.7 +========== + +Saves will raise a `FutureWarning` if they cascade and cascade hasn't been set to +True. This is because in 0.8 it will default to False. If you require cascading +saves then either set it in the `meta` or pass via `save` :: + + # At the class level: + class Person(Document): + meta = {'cascade': True} + + # Or in code: + my_document.save(cascade=True) + 0.5 to 0.6 ========== diff --git a/mongoengine/document.py b/mongoengine/document.py index 1b55e43..da73f88 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,15 +1,18 @@ +import warnings + import pymongo from bson.dbref import DBRef - from mongoengine import signals, queryset + from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, BaseDict, BaseList) from queryset import OperationError from connection import get_db, DEFAULT_CONNECTION_NAME __all__ = ['Document', 'EmbeddedDocument', 'DynamicDocument', - 'DynamicEmbeddedDocument', 'OperationError', 'InvalidCollectionError'] + 'DynamicEmbeddedDocument', 'OperationError', + 'InvalidCollectionError'] class InvalidCollectionError(Exception): @@ -134,8 +137,9 @@ class Document(BaseDocument): options = cls._collection.options() if options.get('max') != max_documents or \ options.get('size') != max_size: - msg = ('Cannot create collection "%s" as a capped ' - 'collection as it already exists') % cls._collection + msg = (('Cannot create collection "%s" as a capped ' + 'collection as it already exists') + % cls._collection) raise InvalidCollectionError(msg) else: # Create the collection as a capped collection @@ -149,8 +153,9 @@ class Document(BaseDocument): cls._collection = db[collection_name] return cls._collection - def save(self, safe=True, force_insert=False, validate=True, write_options=None, - cascade=None, cascade_kwargs=None, _refs=None): + def save(self, safe=True, force_insert=False, validate=True, + write_options=None, cascade=None, cascade_kwargs=None, + _refs=None): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be created. @@ -163,27 +168,30 @@ class Document(BaseDocument): updates of existing documents :param validate: validates the document; set to ``False`` to skip. :param write_options: Extra keyword arguments are passed down to - :meth:`~pymongo.collection.Collection.save` OR - :meth:`~pymongo.collection.Collection.insert` - which will be used as options for the resultant ``getLastError`` command. - For example, ``save(..., write_options={w: 2, fsync: True}, ...)`` will - wait until at least two servers have recorded the write and will force an - fsync on each server being written to. - :param cascade: Sets the flag for cascading saves. You can set a default by setting - "cascade" in the document __meta__ - :param cascade_kwargs: optional kwargs dictionary to be passed throw to cascading saves + :meth:`~pymongo.collection.Collection.save` OR + :meth:`~pymongo.collection.Collection.insert` + which will be used as options for the resultant + ``getLastError`` command. For example, + ``save(..., write_options={w: 2, fsync: True}, ...)`` will + wait until at least two servers have recorded the write and + will force an fsync on the primary server. + :param cascade: Sets the flag for cascading saves. You can set a + default by setting "cascade" in the document __meta__ + :param cascade_kwargs: optional kwargs dictionary to be passed throw + to cascading saves :param _refs: A list of processed references used in cascading saves .. versionchanged:: 0.5 - In existing documents it only saves changed fields using set / unset - Saves are cascaded and any :class:`~bson.dbref.DBRef` objects - that have changes are saved as well. + In existing documents it only saves changed fields using + set / unset. Saves are cascaded and any + :class:`~bson.dbref.DBRef` objects that have changes are + saved as well. .. versionchanged:: 0.6 - Cascade saves are optional = defaults to True, if you want fine grain - control then you can turn off using document meta['cascade'] = False - Also you can pass different kwargs to the cascade save using cascade_kwargs - which overwrites the existing kwargs with custom values - + Cascade saves are optional = defaults to True, if you want + fine grain control then you can turn off using document + meta['cascade'] = False Also you can pass different kwargs to + the cascade save using cascade_kwargs which overwrites the + existing kwargs with custom values """ signals.pre_save.send(self.__class__, document=self) @@ -201,9 +209,11 @@ class Document(BaseDocument): collection = self.__class__.objects._collection if created: if force_insert: - object_id = collection.insert(doc, safe=safe, **write_options) + object_id = collection.insert(doc, safe=safe, + **write_options) else: - object_id = collection.save(doc, safe=safe, **write_options) + object_id = collection.save(doc, safe=safe, + **write_options) else: object_id = doc['_id'] updates, removals = self._delta() @@ -216,11 +226,15 @@ class Document(BaseDocument): upsert = self._created if updates: - collection.update(select_dict, {"$set": updates}, upsert=upsert, safe=safe, **write_options) + collection.update(select_dict, {"$set": updates}, + upsert=upsert, safe=safe, **write_options) if removals: - collection.update(select_dict, {"$unset": removals}, upsert=upsert, safe=safe, **write_options) + collection.update(select_dict, {"$unset": removals}, + upsert=upsert, safe=safe, **write_options) - cascade = self._meta.get('cascade', True) if cascade is None else cascade + warn_cascade = not cascade and 'cascade' not in self._meta + cascade = (self._meta.get('cascade', True) + if cascade is None else cascade) if cascade: kwargs = { "safe": safe, @@ -232,8 +246,7 @@ class Document(BaseDocument): if cascade_kwargs: # Allow granular control over cascades kwargs.update(cascade_kwargs) kwargs['_refs'] = _refs - #self._changed_fields = [] - self.cascade_save(**kwargs) + self.cascade_save(warn_cascade=warn_cascade, **kwargs) except pymongo.errors.OperationFailure, err: message = 'Could not save document (%s)' @@ -249,23 +262,27 @@ class Document(BaseDocument): signals.post_save.send(self.__class__, document=self, created=created) return self - def cascade_save(self, *args, **kwargs): - """Recursively saves any references / generic references on an object""" + def cascade_save(self, warn_cascade=None, *args, **kwargs): + """Recursively saves any references / + generic references on an objects""" import fields _refs = kwargs.get('_refs', []) or [] for name, cls in self._fields.items(): - if not isinstance(cls, (fields.ReferenceField, fields.GenericReferenceField)): + if not isinstance(cls, (fields.ReferenceField, + fields.GenericReferenceField)): continue ref = getattr(self, name) - if not ref: - continue - if isinstance(ref, DBRef): + if not ref or isinstance(ref, DBRef): continue ref_id = "%s,%s" % (ref.__class__.__name__, str(ref._data)) if ref and ref_id not in _refs: + if warn_cascade: + msg = ("Cascading saves will default to off in 0.8, " + "please explicitly set `.save(cascade=True)`") + warnings.warn(msg, FutureWarning) _refs.append(ref_id) kwargs["_refs"] = _refs ref.save(**kwargs) diff --git a/tests/test_document.py b/tests/test_document.py index f1c65b1..fdca119 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1508,6 +1508,40 @@ class DocumentTest(unittest.TestCase): p1.reload() self.assertEqual(p1.name, p.parent.name) + def test_cascade_warning(self): + + self.warning_list = [] + showwarning_default = warnings.showwarning + + def append_to_warning_list(message, category, *args): + self.warning_list.append({"message": message, + "category": category}) + + # add warnings to self.warning_list instead of stderr + warnings.showwarning = append_to_warning_list + + class Person(Document): + name = StringField() + parent = ReferenceField('self') + + Person.drop_collection() + + p1 = Person(name="Wilson Snr") + p1.parent = None + p1.save() + + p2 = Person(name="Wilson Jr") + p2.parent = p1 + p2.save() + + # restore default handling of warnings + warnings.showwarning = showwarning_default + self.assertEqual(len(self.warning_list), 1) + warning = self.warning_list[0] + self.assertEqual(FutureWarning, warning["category"]) + self.assertTrue("Cascading saves will default to off in 0.8" + in str(warning["message"])) + def test_save_cascade_kwargs(self): class Person(Document): From bb1b9bc1d36151927d77227b2b70d64940eda868 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 21 Aug 2012 17:49:12 +0100 Subject: [PATCH 0745/1279] Fixing api docs --- mongoengine/fields.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index ef20d89..4023a77 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -473,8 +473,9 @@ class GenericEmbeddedDocumentField(BaseField): Only valid values are subclasses of :class:`~mongoengine.EmbeddedDocument`. - .. note:: You can use the choices param to limit the acceptable - EmbeddedDocument types + .. note :: + You can use the choices param to limit the acceptable + EmbeddedDocument types """ def prepare_query_value(self, op, value): @@ -789,10 +790,12 @@ class GenericReferenceField(BaseField): """A reference to *any* :class:`~mongoengine.document.Document` subclass that will be automatically dereferenced on access (lazily). - .. note:: Any documents used as a generic reference must be registered in the - document registry. Importing the model will automatically register it. + .. note :: + * Any documents used as a generic reference must be registered in the + document registry. Importing the model will automatically register + it. - .. note:: You can use the choices param to limit the acceptable Document types + * You can use the choices param to limit the acceptable Document types .. versionadded:: 0.3 """ From 0526f577ffd39ead42936f029aed225f8dc1d48e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 22 Aug 2012 09:27:18 +0100 Subject: [PATCH 0746/1279] Embedded Documents still can inherit fields MongoEngine/mongoengine#84 --- mongoengine/base.py | 22 ++++++++++++++-------- tests/test_document.py | 15 +++++++++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 3c3dcdc..3d78aaa 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -514,6 +514,18 @@ class DocumentMetaclass(type): if hasattr(base, '_fields'): doc_fields.update(base._fields) + # Standard object mixin - merge in any Fields + if not hasattr(base, '_meta'): + base_fields = {} + for attr_name, attr_value in base.__dict__.iteritems(): + if not isinstance(attr_value, BaseField): + continue + attr_value.name = attr_name + if not attr_value.db_field: + attr_value.db_field = attr_name + base_fields[attr_name] = attr_value + doc_fields.update(base_fields) + # Discover any document fields field_names = {} for attr_name, attr_value in attrs.iteritems(): @@ -537,9 +549,8 @@ class DocumentMetaclass(type): # Set _fields and db_field maps attrs['_fields'] = doc_fields - attrs['_db_field_map'] = dict( - ((k, v.db_field) for k, v in doc_fields.items() - if k != v.db_field)) + attrs['_db_field_map'] = dict([(k, getattr(v, 'db_field', k)) + for k, v in doc_fields.iteritems()]) attrs['_reverse_db_field_map'] = dict( (v, k) for k, v in attrs['_db_field_map'].iteritems()) @@ -757,11 +768,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): if callable(collection): meta['collection'] = collection(base) - # Standard object mixin - merge in any Fields - if not hasattr(base, '_meta'): - attrs.update(dict([(k, v) for k, v in base.__dict__.items() - if issubclass(v.__class__, BaseField)])) - meta.merge(attrs.get('_meta', {})) # Top level meta # Only simple classes (direct subclasses of Document) diff --git a/tests/test_document.py b/tests/test_document.py index fdca119..b8e3a77 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -2585,6 +2585,21 @@ class DocumentTest(unittest.TestCase): Person.drop_collection() + def test_object_mixins(self): + + class NameMixin(object): + name = StringField() + + class Foo(EmbeddedDocument, NameMixin): + quantity = IntField() + + self.assertEqual(['name', 'quantity'], sorted(Foo._fields.keys())) + + class Bar(Document, NameMixin): + widgets = StringField() + + self.assertEqual(['id', 'name', 'widgets'], sorted(Bar._fields.keys())) + def test_mixin_inheritance(self): class BaseMixIn(object): count = IntField() From 658b3784aee7343c1a3725411492d1fa87902e94 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 22 Aug 2012 09:44:32 +0100 Subject: [PATCH 0747/1279] Split out warning tests as they are order dependent --- tests/test_all_warnings.py | 74 +++++++++++++++++++++++++++++++++ tests/test_document.py | 84 -------------------------------------- 2 files changed, 74 insertions(+), 84 deletions(-) create mode 100644 tests/test_all_warnings.py diff --git a/tests/test_all_warnings.py b/tests/test_all_warnings.py new file mode 100644 index 0000000..3191726 --- /dev/null +++ b/tests/test_all_warnings.py @@ -0,0 +1,74 @@ +import unittest +import warnings + +from mongoengine import * +from mongoengine.tests import query_counter + + +class TestWarnings(unittest.TestCase): + + def setUp(self): + conn = connect(db='mongoenginetest') + self.warning_list = [] + self.showwarning_default = warnings.showwarning + warnings.showwarning = self.append_to_warning_list + + def append_to_warning_list(self, message, category, *args): + self.warning_list.append({"message": message, + "category": category}) + + def tearDown(self): + # restore default handling of warnings + warnings.showwarning = self.showwarning_default + + def test_allow_inheritance_future_warning(self): + """Add FutureWarning for future allow_inhertiance default change. + """ + + class SimpleBase(Document): + a = IntField() + + class InheritedClass(SimpleBase): + b = IntField() + + InheritedClass() + self.assertEqual(len(self.warning_list), 1) + warning = self.warning_list[0] + self.assertEqual(FutureWarning, warning["category"]) + self.assertTrue("InheritedClass" in str(warning["message"])) + + def test_document_save_cascade_future_warning(self): + + class Person(Document): + name = StringField() + parent = ReferenceField('self') + + Person.drop_collection() + + p1 = Person(name="Wilson Snr") + p1.parent = None + p1.save() + + p2 = Person(name="Wilson Jr") + p2.parent = p1 + p2.parent.name = "Poppa Wilson" + p2.save() + + self.assertEqual(len(self.warning_list), 1) + warning = self.warning_list[0] + self.assertEqual(FutureWarning, warning["category"]) + self.assertTrue("Cascading saves will default to off in 0.8" + in str(warning["message"])) + + def test_document_collection_syntax_warning(self): + + class NonAbstractBase(Document): + pass + + class InheritedDocumentFailTest(NonAbstractBase): + meta = {'collection': 'fail'} + + warning = self.warning_list[0] + self.assertEqual(SyntaxWarning, warning["category"]) + self.assertEqual('non_abstract_base', + InheritedDocumentFailTest._get_collection_name()) diff --git a/tests/test_document.py b/tests/test_document.py index b8e3a77..9e870a0 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -37,34 +37,6 @@ class DocumentTest(unittest.TestCase): def tearDown(self): self.Person.drop_collection() - def test_future_warning(self): - """Add FutureWarning for future allow_inhertiance default change. - """ - - self.warning_list = [] - showwarning_default = warnings.showwarning - - def append_to_warning_list(message,category, *args): - self.warning_list.append({"message":message, "category":category}) - - # add warnings to self.warning_list instead of stderr - warnings.showwarning = append_to_warning_list - - class SimpleBase(Document): - a = IntField() - - class InheritedClass(SimpleBase): - b = IntField() - - # restore default handling of warnings - warnings.showwarning = showwarning_default - - InheritedClass() - self.assertEqual(len(self.warning_list), 1) - warning = self.warning_list[0] - self.assertEqual(FutureWarning, warning["category"]) - self.assertTrue("InheritedClass" in str(warning["message"])) - def test_drop_collection(self): """Ensure that the collection may be dropped from the database. """ @@ -144,28 +116,6 @@ class DocumentTest(unittest.TestCase): meta = {'collection': 'wibble'} self.assertEqual('wibble', InheritedAbstractNamingTest._get_collection_name()) - # set up for redirecting warnings - self.warning_list = [] - showwarning_default = warnings.showwarning - - def append_to_warning_list(message, category, *args): - self.warning_list.append({'message':message, 'category':category}) - - # add warnings to self.warning_list instead of stderr - warnings.showwarning = append_to_warning_list - warnings.simplefilter("always") - - class NonAbstractBase(Document): - pass - - class InheritedDocumentFailTest(NonAbstractBase): - meta = {'collection': 'fail'} - - # restore default handling of warnings - warnings.showwarning = showwarning_default - - self.assertTrue(issubclass(self.warning_list[0]["category"], SyntaxWarning)) - self.assertEqual('non_abstract_base', InheritedDocumentFailTest._get_collection_name()) # Mixin tests class BaseMixin(object): @@ -1508,40 +1458,6 @@ class DocumentTest(unittest.TestCase): p1.reload() self.assertEqual(p1.name, p.parent.name) - def test_cascade_warning(self): - - self.warning_list = [] - showwarning_default = warnings.showwarning - - def append_to_warning_list(message, category, *args): - self.warning_list.append({"message": message, - "category": category}) - - # add warnings to self.warning_list instead of stderr - warnings.showwarning = append_to_warning_list - - class Person(Document): - name = StringField() - parent = ReferenceField('self') - - Person.drop_collection() - - p1 = Person(name="Wilson Snr") - p1.parent = None - p1.save() - - p2 = Person(name="Wilson Jr") - p2.parent = p1 - p2.save() - - # restore default handling of warnings - warnings.showwarning = showwarning_default - self.assertEqual(len(self.warning_list), 1) - warning = self.warning_list[0] - self.assertEqual(FutureWarning, warning["category"]) - self.assertTrue("Cascading saves will default to off in 0.8" - in str(warning["message"])) - def test_save_cascade_kwargs(self): class Person(Document): From 0e3c34e1da6941f25a8bac3ae2f099c1851de6d1 Mon Sep 17 00:00:00 2001 From: Anton Kolechkin Date: Thu, 23 Aug 2012 11:57:22 +0700 Subject: [PATCH 0748/1279] test for composite index with pk, i used EmbeddedDocument because this is the only issue when it's needed --- tests/test_queryset.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 591a82a..4c6e7d7 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -2524,6 +2524,24 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() + def test_types_index_with_pk(self): + + class Comment(EmbeddedDocument): + comment_id = IntField(required=True) + + try: + class BlogPost(Document): + comments = EmbeddedDocumentField(Comment) + meta = {'indexes': [{'fields': ['pk', 'comments.comment_id'], + 'unique': True}]} + except UnboundLocalError: + self.fail('Unbound local error at types index + pk definition') + + info = BlogPost.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] + index_item = [(u'_types', 1), (u'_id', 1), (u'comments.comment_id', 1)] + self.assertTrue(index_item in info) + def test_dict_with_custom_baseclass(self): """Ensure DictField working with custom base clases. """ From 4bc5082681cdda9b7a4aee5962d4fe480c997c20 Mon Sep 17 00:00:00 2001 From: Anton Kolechkin Date: Thu, 23 Aug 2012 11:58:04 +0700 Subject: [PATCH 0749/1279] fix for UnboundLocalError when use the composite index with primary key field --- mongoengine/queryset.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 446a42a..b9106c2 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -501,7 +501,9 @@ class QuerySet(object): direction = None allow_inheritance = doc_cls._meta.get('allow_inheritance') != False - use_types = allow_inheritance + + # If sparse - dont include types + use_types = allow_inheritance and not spec.get('sparse', False) for key in spec['fields']: # Get ASCENDING direction from +, DESCENDING from -, and GEO2D from * @@ -518,6 +520,7 @@ class QuerySet(object): parts = key.split('.') if parts in (['pk'], ['id'], ['_id']): key = '_id' + fields = [] else: fields = QuerySet._lookup_field(doc_cls, parts) parts = [field if field == '_id' else field.db_field @@ -525,10 +528,6 @@ class QuerySet(object): key = '.'.join(parts) index_list.append((key, direction)) - # If sparse - dont include types - if spec.get('sparse', False): - use_types = False - # Check if a list field is being used, don't use _types if it is if use_types and not all(f._index_with_types for f in fields): use_types = False @@ -536,7 +535,7 @@ class QuerySet(object): # If _types is being used, prepend it to every specified index index_types = doc_cls._meta.get('index_types', True) - if (spec.get('types', index_types) and allow_inheritance and use_types + if (spec.get('types', index_types) and use_types and direction is not pymongo.GEO2D): index_list.insert(0, ('_types', 1)) From 4ffa8d012446df7ce6df09e37802d5ccd6205e6d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 22 Aug 2012 08:25:26 +0100 Subject: [PATCH 0750/1279] Updated ReferenceField's to optionally store ObjectId strings. This will become the default in 0.8 (MongoEngine/mongoengine#89) --- docs/changelog.rst | 3 + docs/upgrade.rst | 70 ++++++++++++++++----- mongoengine/base.py | 2 + mongoengine/dereference.py | 19 +++++- mongoengine/document.py | 3 + mongoengine/fields.py | 41 ++++++++++-- tests/test_all_warnings.py | 22 +++++++ tests/test_dereference.py | 124 +++++++++++++++++++++++++++++++++++++ tests/test_document.py | 2 +- tests/test_fields.py | 75 +++++++++++++++++++++- 10 files changed, 336 insertions(+), 25 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4b55f51..f1609ae 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,9 @@ Changelog Changes in 0.7.X ================= + +- Updated ReferenceField's to optionally store ObjectId strings + this will become the default in 0.8 (MongoEngine/mongoengine#89) - Added FutureWarning - save will default to `cascade=False` in 0.8 - Added example of indexing embedded document fields (MongoEngine/mongoengine#75) - Fixed ImageField resizing when forcing size (MongoEngine/mongoengine#80) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 4f92fdf..d587c39 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -5,9 +5,13 @@ Upgrading 0.6 to 0.7 ========== -Saves will raise a `FutureWarning` if they cascade and cascade hasn't been set to -True. This is because in 0.8 it will default to False. If you require cascading -saves then either set it in the `meta` or pass via `save` :: +Cascade saves +------------- + +Saves will raise a `FutureWarning` if they cascade and cascade hasn't been set +to True. This is because in 0.8 it will default to False. If you require +cascading saves then either set it in the `meta` or pass +via `save` eg :: # At the class level: class Person(Document): @@ -16,18 +20,52 @@ saves then either set it in the `meta` or pass via `save` :: # Or in code: my_document.save(cascade=True) +.. note :: + Remember: cascading saves **do not** cascade through lists. + +ReferenceFields +--------------- + +ReferenceFields now can store references as ObjectId strings instead of DBRefs. +This will become the default in 0.8 and if `dbref` is not set a `FutureWarning` +will be raised. + + +To explicitly continue to use DBRefs change the `dbref` flag +to True :: + + class Person(Document): + groups = ListField(ReferenceField(Group, dbref=True)) + +To migrate to using strings instead of DBRefs you will have to manually +migrate :: + + # Step 1 - Migrate the model definition + class Group(Document): + author = ReferenceField(User, dbref=False) + members = ListField(ReferenceField(User, dbref=False)) + + # Step 2 - Migrate the data + for g in Group.objects(): + g.author = g.author + g.members = g.members + g.save() + + 0.5 to 0.6 ========== -Embedded Documents - if you had a `pk` field you will have to rename it from `_id` -to `pk` as pk is no longer a property of Embedded Documents. +Embedded Documents - if you had a `pk` field you will have to rename it from +`_id` to `pk` as pk is no longer a property of Embedded Documents. Reverse Delete Rules in Embedded Documents, MapFields and DictFields now throw an InvalidDocument error as they aren't currently supported. -Document._get_subclasses - Is no longer used and the class method has been removed. +Document._get_subclasses - Is no longer used and the class method has been +removed. -Document.objects.with_id - now raises an InvalidQueryError if used with a filter. +Document.objects.with_id - now raises an InvalidQueryError if used with a +filter. FutureWarning - A future warning has been added to all inherited classes that don't define `allow_inheritance` in their meta. @@ -51,11 +89,11 @@ human-readable name for the option. PyMongo / MongoDB ----------------- -map reduce now requires pymongo 1.11+- The pymongo merge_output and reduce_output -parameters, have been depreciated. +map reduce now requires pymongo 1.11+- The pymongo `merge_output` and +`reduce_output` parameters, have been depreciated. -More methods now use map_reduce as db.eval is not supported for sharding as such -the following have been changed: +More methods now use map_reduce as db.eval is not supported for sharding as +such the following have been changed: * :meth:`~mongoengine.queryset.QuerySet.sum` * :meth:`~mongoengine.queryset.QuerySet.average` @@ -65,8 +103,8 @@ the following have been changed: Default collection naming ------------------------- -Previously it was just lowercase, its now much more pythonic and readable as its -lowercase and underscores, previously :: +Previously it was just lowercase, its now much more pythonic and readable as +its lowercase and underscores, previously :: class MyAceDocument(Document): pass @@ -102,7 +140,8 @@ Alternatively, you can rename your collections eg :: failure = False - collection_names = [d._get_collection_name() for d in _document_registry.values()] + collection_names = [d._get_collection_name() + for d in _document_registry.values()] for new_style_name in collection_names: if not new_style_name: # embedded documents don't have collections @@ -120,7 +159,8 @@ Alternatively, you can rename your collections eg :: old_style_name, new_style_name) else: db[old_style_name].rename(new_style_name) - print "Renamed: %s to %s" % (old_style_name, new_style_name) + print "Renamed: %s to %s" % (old_style_name, + new_style_name) if failure: print "Upgrading collection names failed" diff --git a/mongoengine/base.py b/mongoengine/base.py index 3d78aaa..3a5a2b1 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -470,6 +470,8 @@ class ObjectIdField(BaseField): """ def to_python(self, value): + if not isinstance(value, ObjectId): + value = ObjectId(value) return value def to_mongo(self, value): diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 637380d..d371187 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -39,9 +39,26 @@ class DeReference(object): doc_type = doc_type.field if isinstance(doc_type, ReferenceField): + field = doc_type doc_type = doc_type.document_type - if all([i.__class__ == doc_type for i in items]): + is_list = not hasattr(items, 'items') + + if is_list and all([i.__class__ == doc_type for i in items]): return items + elif not is_list and all([i.__class__ == doc_type + for i in items.values()]): + return items + elif not field.dbref: + if not hasattr(items, 'items'): + items = [field.to_python(v) + if not isinstance(v, (DBRef, Document)) else v + for v in items] + else: + items = dict([ + (k, field.to_python(v)) + if not isinstance(v, (DBRef, Document)) else (k, v) + for k, v in items.iteritems()] + ) self.reference_map = self._find_references(items) self.object_map = self._fetch_objects(doc_type=doc_type) diff --git a/mongoengine/document.py b/mongoengine/document.py index da73f88..4fbf1fe 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -277,6 +277,9 @@ class Document(BaseDocument): if not ref or isinstance(ref, DBRef): continue + if not getattr(ref, '_changed_fields', True): + continue + ref_id = "%s,%s" % (ref.__class__.__name__, str(ref._data)) if ref and ref_id not in _refs: if warn_cascade: diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 4023a77..57f648e 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -693,7 +693,8 @@ class ReferenceField(BaseField): * NULLIFY - Updates the reference to null. * CASCADE - Deletes the documents associated with the reference. * DENY - Prevent the deletion of the reference object. - * PULL - Pull the reference from a :class:`~mongoengine.ListField` of references + * PULL - Pull the reference from a :class:`~mongoengine.ListField` + of references Alternative syntax for registering delete rules (useful when implementing bi-directional delete rules) @@ -709,9 +710,12 @@ class ReferenceField(BaseField): .. versionchanged:: 0.5 added `reverse_delete_rule` """ - def __init__(self, document_type, reverse_delete_rule=DO_NOTHING, **kwargs): + def __init__(self, document_type, dbref=None, + reverse_delete_rule=DO_NOTHING, **kwargs): """Initialises the Reference Field. + :param dbref: Store the reference as :class:`~pymongo.dbref.DBRef` + or as the :class:`~pymongo.objectid.ObjectId`.id . :param reverse_delete_rule: Determines what to do when the referring object is deleted """ @@ -719,6 +723,13 @@ class ReferenceField(BaseField): if not issubclass(document_type, (Document, basestring)): self.error('Argument to ReferenceField constructor must be a ' 'document class or a string') + + if dbref is None: + msg = ("ReferenceFields will default to using ObjectId " + " strings in 0.8, set DBRef=True if this isn't desired") + warnings.warn(msg, FutureWarning) + + self.dbref = dbref if dbref is not None else True # To change in 0.8 self.document_type_obj = document_type self.reverse_delete_rule = reverse_delete_rule super(ReferenceField, self).__init__(**kwargs) @@ -741,8 +752,9 @@ class ReferenceField(BaseField): # Get value from document instance if available value = instance._data.get(self.name) + # Dereference DBRefs - if isinstance(value, (DBRef)): + if isinstance(value, DBRef): value = self.document_type._get_db().dereference(value) if value is not None: instance._data[self.name] = self.document_type._from_son(value) @@ -751,6 +763,10 @@ class ReferenceField(BaseField): def to_mongo(self, document): if isinstance(document, DBRef): + if not self.dbref: + return "%s" % DBRef.id + return document + elif not self.dbref and isinstance(document, basestring): return document id_field_name = self.document_type._meta['id_field'] @@ -766,8 +782,20 @@ class ReferenceField(BaseField): id_ = document id_ = id_field.to_mongo(id_) - collection = self.document_type._get_collection_name() - return DBRef(collection, id_) + if self.dbref: + collection = self.document_type._get_collection_name() + return DBRef(collection, id_) + + return "%s" % id_ + + def to_python(self, value): + """Convert a MongoDB-compatible type to a Python type. + """ + if (not self.dbref and + not isinstance(value, (DBRef, Document, EmbeddedDocument))): + collection = self.document_type._get_collection_name() + value = DBRef(collection, self.document_type.id.to_python(value)) + return value def prepare_query_value(self, op, value): if value is None: @@ -775,8 +803,9 @@ class ReferenceField(BaseField): return self.to_mongo(value) def validate(self, value): + if not isinstance(value, (self.document_type, DBRef)): - self.error('A ReferenceField only accepts DBRef') + self.error("A ReferenceField only accepts DBRef or documents") if isinstance(value, Document) and value.id is None: self.error('You can only reference documents once they have been ' diff --git a/tests/test_all_warnings.py b/tests/test_all_warnings.py index 3191726..9b38fa6 100644 --- a/tests/test_all_warnings.py +++ b/tests/test_all_warnings.py @@ -37,6 +37,28 @@ class TestWarnings(unittest.TestCase): self.assertEqual(FutureWarning, warning["category"]) self.assertTrue("InheritedClass" in str(warning["message"])) + def test_dbref_reference_field_future_warning(self): + + class Person(Document): + name = StringField() + parent = ReferenceField('self') + + Person.drop_collection() + + p1 = Person() + p1.parent = None + p1.save() + + p2 = Person(name="Wilson Jr") + p2.parent = p1 + p2.save(cascade=False) + + self.assertEqual(len(self.warning_list), 1) + warning = self.warning_list[0] + self.assertEqual(FutureWarning, warning["category"]) + self.assertTrue("ReferenceFields will default to using ObjectId" + in str(warning["message"])) + def test_document_save_cascade_future_warning(self): class Person(Document): diff --git a/tests/test_dereference.py b/tests/test_dereference.py index cc2ffba..64ddf09 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -64,6 +64,130 @@ class FieldTest(unittest.TestCase): User.drop_collection() Group.drop_collection() + def test_list_item_dereference_dref_false(self): + """Ensure that DBRef items in ListFields are dereferenced. + """ + class User(Document): + name = StringField() + + class Group(Document): + members = ListField(ReferenceField(User, dbref=False)) + + User.drop_collection() + Group.drop_collection() + + for i in xrange(1, 51): + user = User(name='user %s' % i) + user.save() + + group = Group(members=User.objects) + group.save() + + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first() + self.assertEqual(q, 1) + + [m for m in group_obj.members] + self.assertEqual(q, 2) + + # Document select_related + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first().select_related() + + self.assertEqual(q, 2) + [m for m in group_obj.members] + self.assertEqual(q, 2) + + # Queryset select_related + with query_counter() as q: + self.assertEqual(q, 0) + group_objs = Group.objects.select_related() + self.assertEqual(q, 2) + for group_obj in group_objs: + [m for m in group_obj.members] + self.assertEqual(q, 2) + + User.drop_collection() + Group.drop_collection() + + def test_handle_old_style_references(self): + """Ensure that DBRef items in ListFields are dereferenced. + """ + class User(Document): + name = StringField() + + class Group(Document): + members = ListField(ReferenceField(User, dbref=True)) + + User.drop_collection() + Group.drop_collection() + + for i in xrange(1, 26): + user = User(name='user %s' % i) + user.save() + + group = Group(members=User.objects) + group.save() + + group = Group._get_collection().find_one() + + # Update the model to change the reference + class Group(Document): + members = ListField(ReferenceField(User, dbref=False)) + + group = Group.objects.first() + group.members.append(User(name="String!").save()) + group.save() + + group = Group.objects.first() + self.assertEqual(group.members[0].name, 'user 1') + self.assertEqual(group.members[-1].name, 'String!') + + def test_migrate_references(self): + """Example of migrating ReferenceField storage + """ + + # Create some sample data + class User(Document): + name = StringField() + + class Group(Document): + author = ReferenceField(User, dbref=True) + members = ListField(ReferenceField(User, dbref=True)) + + User.drop_collection() + Group.drop_collection() + + user = User(name="Ross").save() + group = Group(author=user, members=[user]).save() + + raw_data = Group._get_collection().find_one() + self.assertTrue(isinstance(raw_data['author'], DBRef)) + self.assertTrue(isinstance(raw_data['members'][0], DBRef)) + + # Migrate the model definition + class Group(Document): + author = ReferenceField(User, dbref=False) + members = ListField(ReferenceField(User, dbref=False)) + + # Migrate the data + for g in Group.objects(): + g.author = g.author + g.members = g.members + g.save() + + group = Group.objects.first() + self.assertEqual(group.author, user) + self.assertEqual(group.members, [user]) + + raw_data = Group._get_collection().find_one() + self.assertTrue(isinstance(raw_data['author'], basestring)) + self.assertTrue(isinstance(raw_data['members'][0], basestring)) + def test_recursive_reference(self): """Ensure that ReferenceFields can reference their own documents. """ diff --git a/tests/test_document.py b/tests/test_document.py index 9e870a0..bee6c5c 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1729,7 +1729,7 @@ class DocumentTest(unittest.TestCase): def test_circular_reference_deltas_2(self): - class Person( Document ): + class Person(Document): name = StringField() owns = ListField( ReferenceField( 'Organization' ) ) employer = ReferenceField( 'Organization' ) diff --git a/tests/test_fields.py b/tests/test_fields.py index ff9246b..16912fb 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -7,7 +7,7 @@ import tempfile from decimal import Decimal -from bson import Binary +from bson import Binary, DBRef import gridfs from nose.plugins.skip import SkipTest @@ -1086,6 +1086,42 @@ class FieldTest(unittest.TestCase): User.drop_collection() BlogPost.drop_collection() + def test_dbref_reference_fields(self): + + class Person(Document): + name = StringField() + parent = ReferenceField('self', dbref=True) + + Person.drop_collection() + + p1 = Person(name="John").save() + Person(name="Ross", parent=p1).save() + + col = Person._get_collection() + data = col.find_one({'name': 'Ross'}) + self.assertEqual(data['parent'], DBRef('person', p1.pk)) + + p = Person.objects.get(name="Ross") + self.assertEqual(p.parent, p1) + + def test_str_reference_fields(self): + + class Person(Document): + name = StringField() + parent = ReferenceField('self', dbref=False) + + Person.drop_collection() + + p1 = Person(name="John").save() + Person(name="Ross", parent=p1).save() + + col = Person._get_collection() + data = col.find_one({'name': 'Ross'}) + self.assertEqual(data['parent'], "%s" % p1.pk) + + p = Person.objects.get(name="Ross") + self.assertEqual(p.parent, p1) + def test_list_item_dereference(self): """Ensure that DBRef items in ListFields are dereferenced. """ @@ -1122,6 +1158,7 @@ class FieldTest(unittest.TestCase): boss = ReferenceField('self') friends = ListField(ReferenceField('self')) + Employee.drop_collection() bill = Employee(name='Bill Lumbergh') bill.save() @@ -1245,7 +1282,41 @@ class FieldTest(unittest.TestCase): class BlogPost(Document): title = StringField() - author = ReferenceField(Member) + author = ReferenceField(Member, dbref=False) + + Member.drop_collection() + BlogPost.drop_collection() + + m1 = Member(user_num=1) + m1.save() + m2 = Member(user_num=2) + m2.save() + + post1 = BlogPost(title='post 1', author=m1) + post1.save() + + post2 = BlogPost(title='post 2', author=m2) + post2.save() + + post = BlogPost.objects(author=m1).first() + self.assertEqual(post.id, post1.id) + + post = BlogPost.objects(author=m2).first() + self.assertEqual(post.id, post2.id) + + Member.drop_collection() + BlogPost.drop_collection() + + def test_reference_query_conversion_dbref(self): + """Ensure that ReferenceFields can be queried using objects and values + of the type of the primary key of the referenced object. + """ + class Member(Document): + user_num = IntField(primary_key=True) + + class BlogPost(Document): + title = StringField() + author = ReferenceField(Member, dbref=True) Member.drop_collection() BlogPost.drop_collection() From ba276452fb86effc58ecd4d9f6e6fb6c7f8c2ca8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 23 Aug 2012 11:09:07 +0100 Subject: [PATCH 0751/1279] Fix tests --- tests/test_dereference.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 64ddf09..120b1a2 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1,6 +1,8 @@ from __future__ import with_statement import unittest +from bson import DBRef + from mongoengine import * from mongoengine.connection import get_db from mongoengine.tests import query_counter From 86a78402c398f27d9997e4c5cb333d3c6399c37d Mon Sep 17 00:00:00 2001 From: Sergey Date: Thu, 23 Aug 2012 14:29:29 +0400 Subject: [PATCH 0752/1279] Update mongoengine/fields.py --- mongoengine/fields.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 57f648e..e8eb43b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1324,17 +1324,18 @@ class SequenceField(IntField): .. versionadded:: 0.5 """ - def __init__(self, collection_name=None, db_alias = None, *args, **kwargs): + def __init__(self, collection_name=None, db_alias = None, sequence_name = None *args, **kwargs): self.collection_name = collection_name or 'mongoengine.counters' self.db_alias = db_alias or DEFAULT_CONNECTION_NAME + self.sequence_name = sequence_name return super(SequenceField, self).__init__(*args, **kwargs) def generate_new_value(self): """ Generate and Increment the counter """ - sequence_id = "%s.%s" % (self.owner_document._get_collection_name(), - self.name) + sequence_name = self.sequence_name or self.owner_document._get_collection_name() + sequence_id = "%s.%s" % (sequence_name, self.name) collection = get_db(alias=self.db_alias)[self.collection_name] counter = collection.find_and_modify(query={"_id": sequence_id}, update={"$inc": {"next": 1}}, From 7e64bb250306d5aa3f485d357bf3521a3bf0be85 Mon Sep 17 00:00:00 2001 From: Sergey Date: Thu, 23 Aug 2012 14:32:27 +0400 Subject: [PATCH 0753/1279] Update mongoengine/fields.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mistake  --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e8eb43b..eb57960 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1324,7 +1324,7 @@ class SequenceField(IntField): .. versionadded:: 0.5 """ - def __init__(self, collection_name=None, db_alias = None, sequence_name = None *args, **kwargs): + def __init__(self, collection_name=None, db_alias = None, sequence_name = None, *args, **kwargs): self.collection_name = collection_name or 'mongoengine.counters' self.db_alias = db_alias or DEFAULT_CONNECTION_NAME self.sequence_name = sequence_name From 325551979248595a3a60d162ede431a319adcdee Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 23 Aug 2012 16:31:22 +0100 Subject: [PATCH 0754/1279] Updating test --- tests/test_queryset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 591a82a..74b8364 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -624,7 +624,7 @@ class QuerySetTest(unittest.TestCase): fresh_o1 = Organization.objects.get(id=o1.id) fresh_o1.save() - self.assertEquals(q, 2) + self.assertEqual(q, 2) with query_counter() as q: self.assertEqual(q, 0) @@ -632,7 +632,7 @@ class QuerySetTest(unittest.TestCase): fresh_o1 = Organization.objects.get(id=o1.id) fresh_o1.save(cascade=False) - self.assertEquals(q, 2) + self.assertEqual(q, 2) with query_counter() as q: self.assertEqual(q, 0) @@ -641,7 +641,7 @@ class QuerySetTest(unittest.TestCase): fresh_o1.employees.append(p2) fresh_o1.save(cascade=False) - self.assertEquals(q, 3) + self.assertEqual(q, 3) def test_slave_okay(self): """Ensures that a query can take slave_okay syntax @@ -2341,7 +2341,7 @@ class QuerySetTest(unittest.TestCase): foo = Foo(bar=bar) foo.save() - self.assertEquals(Foo.objects.distinct("bar"), [bar]) + self.assertEqual(Foo.objects.distinct("bar"), [bar]) def test_custom_manager(self): """Ensure that custom QuerySetManager instances work as expected. From 8cb8aa392c1e02cc131b051636a6f890b47127c3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 23 Aug 2012 17:08:23 +0100 Subject: [PATCH 0755/1279] Updated tests --- tests/test_fields.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/test_fields.py b/tests/test_fields.py index 16912fb..e7936ca 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1749,8 +1749,8 @@ class FieldTest(unittest.TestCase): streamfile.validate() result = StreamFile.objects.first() self.assertTrue(streamfile == result) - self.assertEquals(result.the_file.read(), text + more_text) - self.assertEquals(result.the_file.content_type, content_type) + self.assertEqual(result.the_file.read(), text + more_text) + self.assertEqual(result.the_file.content_type, content_type) result.the_file.seek(0) self.assertEqual(result.the_file.tell(), 0) self.assertEqual(result.the_file.read(len(text)), text) @@ -1891,11 +1891,11 @@ class FieldTest(unittest.TestCase): t = TestImage.objects.first() - self.assertEquals(t.image.format, 'PNG') + self.assertEqual(t.image.format, 'PNG') w, h = t.image.size - self.assertEquals(w, 371) - self.assertEquals(h, 76) + self.assertEqual(w, 371) + self.assertEqual(h, 76) t.image.delete() @@ -1914,11 +1914,11 @@ class FieldTest(unittest.TestCase): t = TestImage.objects.first() - self.assertEquals(t.image.format, 'PNG') + self.assertEqual(t.image.format, 'PNG') w, h = t.image.size - self.assertEquals(w, 185) - self.assertEquals(h, 37) + self.assertEqual(w, 185) + self.assertEqual(h, 37) t.image.delete() @@ -1937,11 +1937,11 @@ class FieldTest(unittest.TestCase): t = TestImage.objects.first() - self.assertEquals(t.image.format, 'PNG') + self.assertEqual(t.image.format, 'PNG') w, h = t.image.size - self.assertEquals(w, 185) - self.assertEquals(h, 37) + self.assertEqual(w, 185) + self.assertEqual(h, 37) t.image.delete() @@ -1960,9 +1960,9 @@ class FieldTest(unittest.TestCase): t = TestImage.objects.first() - self.assertEquals(t.image.thumbnail.format, 'PNG') - self.assertEquals(t.image.thumbnail.width, 92) - self.assertEquals(t.image.thumbnail.height, 18) + self.assertEqual(t.image.thumbnail.format, 'PNG') + self.assertEqual(t.image.thumbnail.width, 92) + self.assertEqual(t.image.thumbnail.height, 18) t.image.delete() @@ -1987,7 +1987,7 @@ class FieldTest(unittest.TestCase): test_file.save() data = get_db("test_files").macumba.files.find_one() - self.assertEquals(data.get('name'), 'hello.txt') + self.assertEqual(data.get('name'), 'hello.txt') test_file = TestFile.objects.first() self.assertEqual(test_file.the_file.read(), From a4b09344af067a6cd44615697a9a356019230422 Mon Sep 17 00:00:00 2001 From: Anthony Nemitz Date: Thu, 23 Aug 2012 18:08:12 -0700 Subject: [PATCH 0756/1279] test case for multiple inheritance raising MRO exception --- tests/test_document.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_document.py b/tests/test_document.py index bee6c5c..8f28e7e 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -365,6 +365,28 @@ class DocumentTest(unittest.TestCase): Animal.drop_collection() + def test_document_inheritance(self): + """ + + """ + class DateCreatedDocument(Document): + meta = { + 'allow_inheritance': True, + 'abstract': True, + } + + class DateUpdatedDocument(Document): + meta = { + 'allow_inheritance': True, + 'abstract': True, + } + + def create_my_document(): + class MyDocument(DateCreatedDocument, DateUpdatedDocument): + pass + + create_my_document() + def test_how_to_turn_off_inheritance(self): """Demonstrates migrating from allow_inheritance = True to False. """ From ab9e9a33294b2bcb4c6f547fb39c11e0bbc01045 Mon Sep 17 00:00:00 2001 From: Anthony Nemitz Date: Thu, 23 Aug 2012 18:09:51 -0700 Subject: [PATCH 0757/1279] adding comment to the MRO test case --- tests/test_document.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_document.py b/tests/test_document.py index 8f28e7e..523ab6a 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -366,8 +366,7 @@ class DocumentTest(unittest.TestCase): Animal.drop_collection() def test_document_inheritance(self): - """ - + """Ensure mutliple inheritance of abstract docs works """ class DateCreatedDocument(Document): meta = { From f594ece32afcd88f98da1655d3eaace2ec364af7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 08:54:54 +0100 Subject: [PATCH 0758/1279] Fixed MRO issue in base.py MongoEngine/mongoengine#95 --- mongoengine/base.py | 17 ++++++++--------- tests/test_document.py | 8 ++++---- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 3a5a2b1..05a08e5 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -498,7 +498,7 @@ class DocumentMetaclass(type): """ def __new__(cls, name, bases, attrs): - bases = cls._get_bases(bases) + flattened_bases = cls._get_bases(bases) super_new = super(DocumentMetaclass, cls).__new__ # If a base class just call super @@ -512,7 +512,7 @@ class DocumentMetaclass(type): # Merge all fields from subclasses doc_fields = {} - for base in bases[::-1]: + for base in flattened_bases[::-1]: if hasattr(base, '_fields'): doc_fields.update(base._fields) @@ -561,7 +561,7 @@ class DocumentMetaclass(type): # superclasses = {} class_name = [name] - for base in bases: + for base in flattened_bases: if (not getattr(base, '_is_base_cls', True) and not getattr(base, '_meta', {}).get('abstract', True)): # Collate heirarchy for _cls and _types @@ -614,7 +614,7 @@ class DocumentMetaclass(type): module = attrs.get('__module__') for exc in exceptions_to_merge: name = exc.__name__ - parents = tuple(getattr(base, name) for base in bases + parents = tuple(getattr(base, name) for base in flattened_bases if hasattr(base, name)) or (exc,) # Create new exception and set to new_class exception = type(name, parents, {'__module__': module}) @@ -676,8 +676,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): """ def __new__(cls, name, bases, attrs): - - bases = cls._get_bases(bases) + flattened_bases = cls._get_bases(bases) super_new = super(TopLevelDocumentMetaclass, cls).__new__ # Set default _meta data if base class, otherwise get user defined meta @@ -718,7 +717,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): del(attrs['meta']) # Find the parent document class - parent_doc_cls = [b for b in bases + parent_doc_cls = [b for b in flattened_bases if b.__class__ == TopLevelDocumentMetaclass] parent_doc_cls = None if not parent_doc_cls else parent_doc_cls[0] @@ -741,7 +740,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Merge base class metas. # Uses a special MetaDict that handles various merging rules meta = MetaDict() - for base in bases[::-1]: + for base in flattened_bases[::-1]: # Add any mixin metadata from plain objects if hasattr(base, 'meta'): meta.merge(base.meta) @@ -775,7 +774,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # Only simple classes (direct subclasses of Document) # may set allow_inheritance to False simple_class = all([b._meta.get('abstract') - for b in bases if hasattr(b, '_meta')]) + for b in flattened_bases if hasattr(b, '_meta')]) if (not simple_class and meta['allow_inheritance'] == False and not meta['abstract']): raise ValueError('Only direct subclasses of Document may set ' diff --git a/tests/test_document.py b/tests/test_document.py index 523ab6a..274bbb9 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -373,18 +373,18 @@ class DocumentTest(unittest.TestCase): 'allow_inheritance': True, 'abstract': True, } - + class DateUpdatedDocument(Document): meta = { 'allow_inheritance': True, 'abstract': True, } - def create_my_document(): + try: class MyDocument(DateCreatedDocument, DateUpdatedDocument): pass - - create_my_document() + except: + self.assertTrue(False, "Couldn't create MyDocument class") def test_how_to_turn_off_inheritance(self): """Demonstrates migrating from allow_inheritance = True to False. From ea666d46079dc2005f9f19ac35fd7f98e3f0dbe5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 09:00:39 +0100 Subject: [PATCH 0759/1279] Updated AUTHORS and changelog.rst Refs: MongoEngine/mongoengine#88 --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 91993e2..de00134 100644 --- a/AUTHORS +++ b/AUTHORS @@ -118,3 +118,4 @@ that much better: * Tommi Komulainen * Peter Landry * biszkoptwielki + * Anton Kolechkin diff --git a/docs/changelog.rst b/docs/changelog.rst index f1609ae..dc2f7f3 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.7.X ================= +- Fixed UnboundLocalError in composite index with pk field (MongoEngine/mongoengine#88) - Updated ReferenceField's to optionally store ObjectId strings this will become the default in 0.8 (MongoEngine/mongoengine#89) - Added FutureWarning - save will default to `cascade=False` in 0.8 From f09c39b5d7353a59b4467f1a62d7859f896ec054 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 09:10:13 +0100 Subject: [PATCH 0760/1279] Updated AUTHORS and chaneglog.rst refs MongoEngine/mongoengine#92 --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index de00134..aae87ab 100644 --- a/AUTHORS +++ b/AUTHORS @@ -119,3 +119,4 @@ that much better: * Peter Landry * biszkoptwielki * Anton Kolechkin + * Sergy (nikitinsm) \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index dc2f7f3..2f0f597 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.7.X ================= +- Added custom collection / sequence naming for SequenceFields (MongoEngine/mongoengine#92) - Fixed UnboundLocalError in composite index with pk field (MongoEngine/mongoengine#88) - Updated ReferenceField's to optionally store ObjectId strings this will become the default in 0.8 (MongoEngine/mongoengine#89) From a0e3f382cd4169cfd50243bf3694b4f29d87883f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 09:20:14 +0100 Subject: [PATCH 0761/1279] Added testcase ref MongoEngine/mongoengine#92 --- tests/test_fields.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_fields.py b/tests/test_fields.py index e7936ca..ce80bf9 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -2066,6 +2066,27 @@ class FieldTest(unittest.TestCase): c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) + def test_sequence_field_sequence_name(self): + class Person(Document): + id = SequenceField(primary_key=True, sequence_name='jelly') + name = StringField() + + self.db['mongoengine.counters'].drop() + Person.drop_collection() + + for x in xrange(10): + p = Person(name="Person %s" % x) + p.save() + + c = self.db['mongoengine.counters'].find_one({'_id': 'jelly.id'}) + self.assertEqual(c['next'], 10) + + ids = [i.id for i in Person.objects] + self.assertEqual(ids, range(1, 11)) + + c = self.db['mongoengine.counters'].find_one({'_id': 'jelly.id'}) + self.assertEqual(c['next'], 10) + def test_multiple_sequence_fields(self): class Person(Document): id = SequenceField(primary_key=True) From 0bf2ad5b6730638d5c4a3f75e5329d4564116702 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 09:30:08 +0100 Subject: [PATCH 0762/1279] Use pk in ReferenceField fixes MongoEngine/mongoengine#85 --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index eb57960..a2d862d 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -774,7 +774,7 @@ class ReferenceField(BaseField): if isinstance(document, Document): # We need the id from the saved object to create the DBRef - id_ = document.id + id_ = document.pk if id_ is None: self.error('You can only reference documents once they have' ' been saved to the database') From 5c9ef41403fe157bbbae340153e1892d455ceef0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 09:32:15 +0100 Subject: [PATCH 0763/1279] Updated AUTHORS (MongoEngine/mongoengine#85) --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index aae87ab..db9ca8f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -119,4 +119,5 @@ that much better: * Peter Landry * biszkoptwielki * Anton Kolechkin - * Sergy (nikitinsm) \ No newline at end of file + * Sergy (nikitinsm) + * psychogenic \ No newline at end of file From eedf908770243e8f74130e31c16597c73b7ae68f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 09:33:56 +0100 Subject: [PATCH 0764/1279] Update name ref: MongoEngine/mongoengine#96 --- AUTHORS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index db9ca8f..b044537 100644 --- a/AUTHORS +++ b/AUTHORS @@ -119,5 +119,5 @@ that much better: * Peter Landry * biszkoptwielki * Anton Kolechkin - * Sergy (nikitinsm) - * psychogenic \ No newline at end of file + * Sergey Nikitin + * psychogenic From 1c5e6a3425f65f63c36da62a2a3aa0f0584cbfd4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 10:38:00 +0100 Subject: [PATCH 0765/1279] NotUniqueError gracefully replacing ambiguous OperationError when appropriate --- mongoengine/document.py | 10 +++++++--- mongoengine/queryset.py | 9 ++++++++- tests/test_document.py | 11 +++++++---- tests/test_queryset.py | 2 +- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 4fbf1fe..1691df9 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,18 +1,19 @@ import warnings import pymongo +import re from bson.dbref import DBRef from mongoengine import signals, queryset from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, BaseDict, BaseList) -from queryset import OperationError +from queryset import OperationError, NotUniqueError from connection import get_db, DEFAULT_CONNECTION_NAME __all__ = ['Document', 'EmbeddedDocument', 'DynamicDocument', 'DynamicEmbeddedDocument', 'OperationError', - 'InvalidCollectionError'] + 'InvalidCollectionError', 'NotUniqueError'] class InvalidCollectionError(Exception): @@ -250,8 +251,11 @@ class Document(BaseDocument): except pymongo.errors.OperationFailure, err: message = 'Could not save document (%s)' - if u'duplicate key' in unicode(err): + if re.match('^E1100[01] duplicate key', unicode(err)): + # E11000 - duplicate key error index + # E11001 - duplicate key on update message = u'Tried to save duplicate unique keys (%s)' + raise NotUniqueError(message % unicode(err)) raise OperationError(message % unicode(err)) id_field = self._meta['id_field'] if id_field not in self._meta.get('shard_key', []): diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b9106c2..ecd5893 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -45,6 +45,10 @@ class OperationError(Exception): pass +class NotUniqueError(OperationError): + pass + + RE_TYPE = type(re.compile('')) @@ -924,8 +928,11 @@ class QuerySet(object): ids = self._collection.insert(raw, **write_options) except pymongo.errors.OperationFailure, err: message = 'Could not save document (%s)' - if u'duplicate key' in unicode(err): + if re.match('^E1100[01] duplicate key', unicode(err)): + # E11000 - duplicate key error index + # E11001 - duplicate key on update message = u'Tried to save duplicate unique keys (%s)' + raise NotUniqueError(message % unicode(err)) raise OperationError(message % unicode(err)) if not load_bulk: diff --git a/tests/test_document.py b/tests/test_document.py index 274bbb9..5faf268 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1013,6 +1013,9 @@ class DocumentTest(unittest.TestCase): # Two posts with the same slug is not allowed post2 = BlogPost(title='test2', slug='test') + self.assertRaises(NotUniqueError, post2.save) + + # Ensure backwards compatibilty for errors self.assertRaises(OperationError, post2.save) def test_unique_with(self): @@ -1063,7 +1066,7 @@ class DocumentTest(unittest.TestCase): # Now there will be two docs with the same sub.slug post3 = BlogPost(title='test3', sub=SubDocument(year=2010, slug='test')) - self.assertRaises(OperationError, post3.save) + self.assertRaises(NotUniqueError, post3.save) BlogPost.drop_collection() @@ -1090,11 +1093,11 @@ class DocumentTest(unittest.TestCase): # Now there will be two docs with the same sub.slug post3 = BlogPost(title='test3', sub=SubDocument(year=2010, slug='test')) - self.assertRaises(OperationError, post3.save) + self.assertRaises(NotUniqueError, post3.save) # Now there will be two docs with the same title and year post3 = BlogPost(title='test1', sub=SubDocument(year=2009, slug='test-1')) - self.assertRaises(OperationError, post3.save) + self.assertRaises(NotUniqueError, post3.save) BlogPost.drop_collection() @@ -1117,7 +1120,7 @@ class DocumentTest(unittest.TestCase): try: cust_dupe.save() raise AssertionError, "We saved a dupe!" - except OperationError: + except NotUniqueError: pass Customer.drop_collection() diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 8c998e5..8f0ad94 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -578,7 +578,7 @@ class QuerySetTest(unittest.TestCase): def throw_operation_error_not_unique(): Blog.objects.insert([blog2, blog3], safe=True) - self.assertRaises(OperationError, throw_operation_error_not_unique) + self.assertRaises(NotUniqueError, throw_operation_error_not_unique) self.assertEqual(Blog.objects.count(), 2) Blog.objects.insert([blog2, blog3], write_options={'continue_on_error': True}) From 9989da07edae34b7cd5a0cedea19fce8522fd0cf Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 10:40:45 +0100 Subject: [PATCH 0766/1279] Updated AUTHORS and changelog.rst ref: MongoEngine/mongoenginei#62 --- AUTHORS | 1 + docs/changelog.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index b044537..860a789 100644 --- a/AUTHORS +++ b/AUTHORS @@ -121,3 +121,4 @@ that much better: * Anton Kolechkin * Sergey Nikitin * psychogenic + * Stefan Wójcik diff --git a/docs/changelog.rst b/docs/changelog.rst index 2f0f597..aa37af9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,7 @@ Changelog Changes in 0.7.X ================= - +- Added NotUniqueError for duplicate keys (MongoEngine/mongoengine#62) - Added custom collection / sequence naming for SequenceFields (MongoEngine/mongoengine#92) - Fixed UnboundLocalError in composite index with pk field (MongoEngine/mongoengine#88) - Updated ReferenceField's to optionally store ObjectId strings From 4c8296acc63af920761fed291288043a1c3f678b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 11:02:32 +0100 Subject: [PATCH 0767/1279] Added testcase for MongoEngine/mongoengine#77 --- tests/test_queryset.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 8f0ad94..27205c4 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -1268,7 +1268,6 @@ class QuerySetTest(unittest.TestCase): published_posts = (post1, post2, post3, post5, post6) self.assertTrue(all(obj.id in posts for obj in published_posts)) - # Check Q object combination date = datetime(2010, 1, 10) q = BlogPost.objects(Q(publish_date__lte=date) | Q(published=True)) @@ -1327,6 +1326,27 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() + def test_raw_query_and_Q_objects(self): + """ + Test raw plays nicely + """ + class Foo(Document): + name = StringField() + a = StringField() + b = StringField() + c = StringField() + + meta = { + 'allow_inheritance': False + } + + query = Foo.objects(__raw__={'$nor': [{'name': 'bar'}]})._query + self.assertEqual(query, {'$nor': [{'name': 'bar'}]}) + + q1 = {'$or': [{'a': 1}, {'b': 1}]} + query = Foo.objects(Q(__raw__=q1) & Q(c=1))._query + self.assertEqual(query, {'$or': [{'a': 1}, {'b': 1}], 'c': 1}) + def test_exec_js_query(self): """Ensure that queries are properly formed for use in exec_js. """ From 87792e192106df3c9b6321a8c8fb8843f169651c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 11:45:04 +0100 Subject: [PATCH 0768/1279] Test checking can save if not included ref: MongoEngine/mongoengine#70 --- mongoengine/fields.py | 2 ++ tests/test_document.py | 46 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index a2d862d..51d062a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -393,6 +393,8 @@ class ComplexDateTimeField(StringField): data = super(ComplexDateTimeField, self).__get__(instance, owner) if data == None: return datetime.datetime.now() + if isinstance(data, datetime.datetime): + return data return self._convert_from_string(data) def __set__(self, instance, value): diff --git a/tests/test_document.py b/tests/test_document.py index 5faf268..cc9226d 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1640,6 +1640,52 @@ class DocumentTest(unittest.TestCase): self.assertEqual(person.name, None) self.assertEqual(person.age, None) + def test_can_save_if_not_included(self): + + class EmbeddedDoc(EmbeddedDocument): + pass + + class Simple(Document): + pass + + class Doc(Document): + string_field = StringField(default='1') + int_field = IntField(default=1) + float_field = FloatField(default=1.1) + boolean_field = BooleanField(default=True) + datetime_field = DateTimeField(default=datetime.datetime.now) + embedded_document_field = EmbeddedDocumentField(EmbeddedDoc, default=lambda: EmbeddedDoc()) + list_field = ListField(default=lambda: [1, 2, 3]) + dict_field = DictField(default=lambda: {"hello": "world"}) + objectid_field = ObjectIdField(default=ObjectId) + reference_field = ReferenceField(Simple, default=lambda: Simple().save()) + map_field = MapField(IntField(), default=lambda: {"simple": 1}) + decimal_field = DecimalField(default=1.0) + complex_datetime_field = ComplexDateTimeField(default=datetime.datetime.now) + url_field = URLField(default="http://mongoengine.org") + dynamic_field = DynamicField(default=1) + generic_reference_field = GenericReferenceField(default=lambda: Simple().save()) + sorted_list_field = SortedListField(IntField(), default=lambda: [1, 2, 3]) + email_field = EmailField(default="ross@example.com") + geo_point_field = GeoPointField(default=lambda: [1, 2]) + sequence_field = SequenceField() + uuid_field = UUIDField(default=uuid.uuid4) + generic_embedded_document_field = GenericEmbeddedDocumentField(default=lambda: EmbeddedDoc()) + + + Simple.drop_collection() + Doc.drop_collection() + + Doc().save() + + my_doc = Doc.objects.only("string_field").first() + my_doc.string_field = "string" + my_doc.save() + + my_doc = Doc.objects.get(string_field="string") + self.assertEqual(my_doc.string_field, "string") + self.assertEqual(my_doc.int_field, 1) + def test_document_update(self): def update_not_saved_raises(): From 966fa12358e40c44e4e3d9d329c5fb72d1d8357e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 13:47:00 +0100 Subject: [PATCH 0769/1279] Updated docs for map_reduce MongoEngine/mongoengine#67 --- mongoengine/queryset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index ecd5893..8b627d8 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1032,6 +1032,8 @@ class QuerySet(object): :class:`~bson.code.Code` or string :param output: output collection name, if set to 'inline' will try to use :class:`~pymongo.collection.Collection.inline_map_reduce` + This can also be a dictionary containing output options + see: http://docs.mongodb.org/manual/reference/commands/#mapReduce :param finalize_f: finalize function, an optional function that performs any post-reduction processing. :param scope: values to insert into map/reduce global scope. Optional. From 81d402dc17cbded0c6ca2e787276268c2eb7e3f2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 13:58:38 +0100 Subject: [PATCH 0770/1279] Fixing tests --- tests/test_document.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_document.py b/tests/test_document.py index cc9226d..350defd 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1,11 +1,12 @@ from __future__ import with_statement +import bson import os import pickle import pymongo -import bson -import unittest -import warnings import sys +import unittest +import uuid +import warnings from nose.plugins.skip import SkipTest from datetime import datetime @@ -1653,15 +1654,15 @@ class DocumentTest(unittest.TestCase): int_field = IntField(default=1) float_field = FloatField(default=1.1) boolean_field = BooleanField(default=True) - datetime_field = DateTimeField(default=datetime.datetime.now) + datetime_field = DateTimeField(default=datetime.now) embedded_document_field = EmbeddedDocumentField(EmbeddedDoc, default=lambda: EmbeddedDoc()) list_field = ListField(default=lambda: [1, 2, 3]) dict_field = DictField(default=lambda: {"hello": "world"}) - objectid_field = ObjectIdField(default=ObjectId) + objectid_field = ObjectIdField(default=bson.ObjectId) reference_field = ReferenceField(Simple, default=lambda: Simple().save()) map_field = MapField(IntField(), default=lambda: {"simple": 1}) decimal_field = DecimalField(default=1.0) - complex_datetime_field = ComplexDateTimeField(default=datetime.datetime.now) + complex_datetime_field = ComplexDateTimeField(default=datetime.now) url_field = URLField(default="http://mongoengine.org") dynamic_field = DynamicField(default=1) generic_reference_field = GenericReferenceField(default=lambda: Simple().save()) From 7c08c140da94e63bed2602ebb4a73062bca5b486 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 14:18:45 +0100 Subject: [PATCH 0771/1279] Updated upgrade documents --- docs/upgrade.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index d587c39..d01911c 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -52,6 +52,15 @@ migrate :: g.save() +item_frequencies +---------------- + +In the 0.6 series we added support for null / zero / false values in +item_frequencies. A side effect was to return keys in the value they are +stored in rather than as string representations. Your code may need to be +updated to handle native types rather than strings keys for the results of +item frequency queries. + 0.5 to 0.6 ========== From d645ce974565fc010f1b6d28a397c8136dea7d2c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 15:08:37 +0100 Subject: [PATCH 0772/1279] Updated version calculation --- mongoengine/__init__.py | 7 +++---- setup.py | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index ea3dd5e..0ace8f0 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -16,9 +16,8 @@ VERSION = (0, 6, 20) def get_version(): - version = '%s.%s' % (VERSION[0], VERSION[1]) - if VERSION[2]: - version = '%s.%s' % (version, VERSION[2]) - return version + if isinstance(VERSION[-1], basestring): + return '.'.join(map(str, VERSION[:-1])) + VERSION[-1] + return '.'.join(map(str, VERSIONs)) __version__ = get_version() diff --git a/setup.py b/setup.py index 398bd98..dcdccae 100644 --- a/setup.py +++ b/setup.py @@ -18,10 +18,9 @@ except: def get_version(version_tuple): - version = '%s.%s' % (version_tuple[0], version_tuple[1]) - if version_tuple[2]: - version = '%s.%s' % (version, version_tuple[2]) - return version + if isinstance(version_tuple[-1], basestring): + return '.'.join(map(str, version_tuple[:-1])) + version_tuple[-1] + return '.'.join(map(str, version_tuple)) # Dirty hack to get version number from monogengine/__init__.py - we can't # import it as it depends on PyMongo and PyMongo isn't installed until this From 6e7f2b73cf1a5295e10161bb11e9c12f29b86f78 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 16:36:17 +0100 Subject: [PATCH 0773/1279] Fix VERSION --- mongoengine/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 0ace8f0..af4c417 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -18,6 +18,6 @@ VERSION = (0, 6, 20) def get_version(): if isinstance(VERSION[-1], basestring): return '.'.join(map(str, VERSION[:-1])) + VERSION[-1] - return '.'.join(map(str, VERSIONs)) + return '.'.join(map(str, VERSION)) __version__ = get_version() diff --git a/setup.py b/setup.py index dcdccae..c9d6b79 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ except: def get_version(version_tuple): - if isinstance(version_tuple[-1], basestring): + if not isinstance(version_tuple[-1], int): return '.'.join(map(str, version_tuple[:-1])) + version_tuple[-1] return '.'.join(map(str, version_tuple)) From 735e043ff6b146f61dfd45ca21980368ee055c81 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 24 Aug 2012 13:56:02 +0100 Subject: [PATCH 0774/1279] RC Version Bump --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index af4c417..a42fe77 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 6, 20) +VERSION = (0, '7rc1') def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 4e80326..ace9f38 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.6.20 +Version: 0.7rc1 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From f27debe7f974449f34b0acc9a0c8a8de7f1be437 Mon Sep 17 00:00:00 2001 From: Dmitry Balabanov Date: Thu, 30 Aug 2012 12:40:44 +0400 Subject: [PATCH 0775/1279] Respect sharding key when delete object from collection --- mongoengine/document.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index f8bf769..1af7d01 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -265,6 +265,16 @@ class Document(BaseDocument): ref.save(**kwargs) ref._changed_fields = [] + @property + def _object_key(self): + """Dict to identify object in collection + """ + select_dict = {'pk': self.pk} + shard_key = self.__class__._meta.get('shard_key', tuple()) + for k in shard_key: + select_dict[k] = getattr(self, k) + return select_dict + def update(self, **kwargs): """Performs an update on the :class:`~mongoengine.Document` A convenience wrapper to :meth:`~mongoengine.QuerySet.update`. @@ -276,11 +286,7 @@ class Document(BaseDocument): raise OperationError('attempt to update a document not yet saved') # Need to add shard key to query, or you get an error - select_dict = {'pk': self.pk} - shard_key = self.__class__._meta.get('shard_key', tuple()) - for k in shard_key: - select_dict[k] = getattr(self, k) - return self.__class__.objects(**select_dict).update_one(**kwargs) + return self.__class__.objects(**self._object_key).update_one(**kwargs) def delete(self, safe=False): """Delete the :class:`~mongoengine.Document` from the database. This @@ -291,7 +297,7 @@ class Document(BaseDocument): signals.pre_delete.send(self.__class__, document=self) try: - self.__class__.objects(pk=self.pk).delete(safe=safe) + self.__class__.objects(**self._object_key).delete(safe=safe) except pymongo.errors.OperationFailure, err: message = u'Could not delete document (%s)' % err.message raise OperationError(message) From d79ae30f31388ebff9b10350f8291df7bdd02313 Mon Sep 17 00:00:00 2001 From: Dmitry Balabanov Date: Thu, 30 Aug 2012 14:20:53 +0400 Subject: [PATCH 0776/1279] fix object reload with shard_key in meta --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 6fb26cb..0a9e10c 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -840,7 +840,7 @@ class BaseDocument(object): if hasattr(self, '_changed_fields'): self._mark_as_changed(name) - if not self._created and name in self._meta.get('shard_key', tuple()): + if not self._created and name in self._meta.get('shard_key', tuple()) and self._data[name] != value: from queryset import OperationError raise OperationError("Shard Keys are immutable. Tried to update %s" % name) From ab60fd0490dc6b071973a344b7f4ddb23f7d2e84 Mon Sep 17 00:00:00 2001 From: Dmitry Balabanov Date: Thu, 30 Aug 2012 14:37:11 +0400 Subject: [PATCH 0777/1279] sharded collection document reload testcase --- tests/test_document.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_document.py b/tests/test_document.py index 6915caf..a551040 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1226,6 +1226,17 @@ class DocumentTest(unittest.TestCase): self.assertEqual(person.name, "Mr Test User") self.assertEqual(person.age, 21) + def test_reload_sharded(self): + class Animal(Document): + superphylum = StringField() + meta = {'shard_key': ('superphylum',)} + + Animal.drop_collection() + doc = Animal(superphylum = 'Deuterostomia') + doc.save() + doc.reload() + Animal.drop_collection() + def test_reload_referencing(self): """Ensures reloading updates weakrefs correctly """ From 576e198ece6ab8d4c103299153ce422b438bdf91 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Sep 2012 11:33:30 +0100 Subject: [PATCH 0778/1279] Fixed Q object merge edge case (MongoEngine/mongoengine#109) --- docs/changelog.rst | 1 + mongoengine/queryset.py | 2 +- tests/test_queryset.py | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 737f997..ab34fb7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.X ================= +- Fixed Q object merge edge case (MongoEngine/mongoengine#109) - Fixed reloading on sharded documents (hmarr/mongoengine#569) - Added NotUniqueError for duplicate keys (MongoEngine/mongoengine#62) - Added custom collection / sequence naming for SequenceFields (MongoEngine/mongoengine#92) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 8b627d8..f9494eb 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -218,7 +218,7 @@ class QNode(object): def _combine(self, other, operation): """Combine this node with another node into a QCombination object. """ - if other.empty: + if getattr(other, 'empty', True): return self if self.empty: diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 27205c4..15e2d9d 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -1347,6 +1347,21 @@ class QuerySetTest(unittest.TestCase): query = Foo.objects(Q(__raw__=q1) & Q(c=1))._query self.assertEqual(query, {'$or': [{'a': 1}, {'b': 1}], 'c': 1}) + def test_q_merge_queries_edge_case(self): + + class User(Document): + email = EmailField(required=False) + name = StringField() + + User.drop_collection() + pk = ObjectId() + User(email='example@example.com', pk=pk).save() + + self.assertEqual(1, User.objects.filter( + Q(email='example@example.com') | + Q(name='John Doe') + ).limit(2).filter(pk=pk).count()) + def test_exec_js_query(self): """Ensure that queries are properly formed for use in exec_js. """ From 9b9696aefd11a0eb77b12c7f987467fe924c63c7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Sep 2012 12:29:58 +0100 Subject: [PATCH 0779/1279] Updated index creation allows kwargs to be passed through refs (MongoEngine/mongoengine#104) --- docs/changelog.rst | 1 + mongoengine/queryset.py | 20 ++++++++++---------- tests/test_document.py | 28 +++++++++++++++++++++++++++- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ab34fb7..90460fb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.X ================= +- Updated index creation allows kwargs to be passed through refs (MongoEngine/mongoengine#104) - Fixed Q object merge edge case (MongoEngine/mongoengine#109) - Fixed reloading on sharded documents (hmarr/mongoengine#569) - Added NotUniqueError for duplicate keys (MongoEngine/mongoengine#62) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index f9494eb..2b83add 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -398,12 +398,12 @@ class QuerySet(object): or a **-** to determine the index ordering """ index_spec = QuerySet._build_index_spec(self._document, key_or_list) - self._collection.ensure_index( - index_spec['fields'], - drop_dups=drop_dups, - background=background, - sparse=index_spec.get('sparse', False), - unique=index_spec.get('unique', False)) + fields = index_spec.pop('fields') + index_spec['drop_dups'] = drop_dups + index_spec['background'] = background + index_spec.update(kwargs) + + self._collection.ensure_index(fields, **index_spec) return self def __call__(self, q_obj=None, class_check=True, slave_okay=False, **query): @@ -473,11 +473,11 @@ class QuerySet(object): # Ensure document-defined indexes are created if self._document._meta['index_specs']: for spec in self._document._meta['index_specs']: - types_indexed = types_indexed or includes_types(spec['fields']) + fields = spec.pop('fields') + types_indexed = types_indexed or includes_types(fields) opts = index_opts.copy() - opts['unique'] = spec.get('unique', False) - opts['sparse'] = spec.get('sparse', False) - self._collection.ensure_index(spec['fields'], + opts.update(spec) + self._collection.ensure_index(fields, background=background, **opts) # If _types is being used (for polymorphism), it needs an index, diff --git a/tests/test_document.py b/tests/test_document.py index 821637e..75616c7 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -16,7 +16,7 @@ from tests.fixtures import Base, Mixin, PickleEmbedded, PickleTest from mongoengine import * from mongoengine.base import NotRegistered, InvalidDocumentError from mongoengine.queryset import InvalidQueryError -from mongoengine.connection import get_db +from mongoengine.connection import get_db, get_connection TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') @@ -1102,6 +1102,32 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + def test_ttl_indexes(self): + + class Log(Document): + created = DateTimeField(default=datetime.now) + meta = { + 'indexes': [ + {'fields': ['created'], 'expireAfterSeconds': 3600} + ] + } + + Log.drop_collection() + + if pymongo.version_tuple[0] < 2 and pymongo.version_tuple[1] < 3: + raise SkipTest('pymongo needs to be 2.3 or higher for this test') + + connection = get_connection() + version_array = connection.server_info()['versionArray'] + if version_array[0] < 2 and version_array[1] < 2: + raise SkipTest('MongoDB needs to be 2.2 or higher for this test') + + # Indexes are lazy so use list() to perform query + list(Log.objects) + info = Log.objects._collection.index_information() + self.assertEqual(3600, + info['_types_1_created_1']['expireAfterSeconds']) + def test_unique_and_indexes(self): """Ensure that 'unique' constraints aren't overridden by meta.indexes. From f108c4288e763b17e7925746aba41c71e0f20285 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Sep 2012 12:53:50 +0100 Subject: [PATCH 0780/1279] Updated queryset.delete so you can use with skip / limit (MongoEngine/mongoengine#107) --- docs/changelog.rst | 1 + mongoengine/queryset.py | 6 +++++ tests/test_queryset.py | 53 ++++++++++++++++++++++++++++++++++------- 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 90460fb..4a4fb6a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.X ================= +- Updated queryset.delete so you can use with skip / limit (MongoEngine/mongoengine#107) - Updated index creation allows kwargs to be passed through refs (MongoEngine/mongoengine#104) - Fixed Q object merge edge case (MongoEngine/mongoengine#109) - Fixed reloading on sharded documents (hmarr/mongoengine#569) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 2b83add..8106e67 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1343,6 +1343,12 @@ class QuerySet(object): """ doc = self._document + # Handle deletes where skips or limits have been applied + if self._skip or self._limit: + for doc in self: + doc.delete() + return + delete_rules = doc._meta.get('delete_rules') or {} # Check for DENY rules before actually deleting/nullifying any other # references diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 15e2d9d..55531a1 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -1501,7 +1501,8 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(1, BlogPost.objects.count()) def test_reverse_delete_rule_cascade_self_referencing(self): - """Ensure self-referencing CASCADE deletes do not result in infinite loop + """Ensure self-referencing CASCADE deletes do not result in infinite + loop """ class Category(Document): name = StringField() @@ -1607,6 +1608,40 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(post.authors, [me]) self.assertEqual(another.authors, []) + def test_delete_with_limits(self): + + class Log(Document): + pass + + Log.drop_collection() + + for i in xrange(10): + Log().save() + + Log.objects()[3:5].delete() + self.assertEqual(8, Log.objects.count()) + + def test_delete_with_limit_handles_delete_rules(self): + """Ensure cascading deletion of referring documents from the database. + """ + class BlogPost(Document): + content = StringField() + author = ReferenceField(self.Person, reverse_delete_rule=CASCADE) + BlogPost.drop_collection() + + me = self.Person(name='Test User') + me.save() + someoneelse = self.Person(name='Some-one Else') + someoneelse.save() + + BlogPost(content='Watching TV', author=me).save() + BlogPost(content='Chilling out', author=me).save() + BlogPost(content='Pro Testing', author=someoneelse).save() + + self.assertEqual(3, BlogPost.objects.count()) + self.Person.objects()[:1].delete() + self.assertEqual(1, BlogPost.objects.count()) + def test_update(self): """Ensure that atomic updates work properly. """ @@ -2534,30 +2569,30 @@ class QuerySetTest(unittest.TestCase): """Ensure that index_types will, when disabled, prevent _types being added to all indices. """ - class BlogPost(Document): + class BloggPost(Document): date = DateTimeField() meta = {'index_types': False, 'indexes': ['-date']} # Indexes are lazy so use list() to perform query - list(BlogPost.objects) - info = BlogPost.objects._collection.index_information() + list(BloggPost.objects) + info = BloggPost.objects._collection.index_information() info = [value['key'] for key, value in info.iteritems()] self.assertTrue([('_types', 1)] not in info) self.assertTrue([('date', -1)] in info) - BlogPost.drop_collection() + BloggPost.drop_collection() - class BlogPost(Document): + class BloggPost(Document): title = StringField() meta = {'allow_inheritance': False} # _types is not used on objects where allow_inheritance is False - list(BlogPost.objects) - info = BlogPost.objects._collection.index_information() + list(BloggPost.objects) + info = BloggPost.objects._collection.index_information() self.assertFalse([('_types', 1)] in info.values()) - BlogPost.drop_collection() + BloggPost.drop_collection() def test_types_index_with_pk(self): From 0b23bc9cf205bb0339df1be30a35dc6154ef224f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Sep 2012 13:10:06 +0100 Subject: [PATCH 0781/1279] python 2.6.4 and lower cannot handle unicode keys passed to __init__ (MongoEngine/mongoengine#101) --- mongoengine/base.py | 8 ++++---- mongoengine/python_support.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 82f79d3..dc7ef8a 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -11,7 +11,7 @@ from queryset import DoesNotExist, MultipleObjectsReturned from queryset import DO_NOTHING from mongoengine import signals -from mongoengine.python_support import (PY3, PY25, txt_type, +from mongoengine.python_support import (PY3, UNICODE_KWARGS, txt_type, to_str_keys_recursive) import pymongo @@ -1054,9 +1054,9 @@ class BaseDocument(object): # class if unavailable class_name = son.get('_cls', cls._class_name) data = dict(("%s" % key, value) for key, value in son.items()) - if PY25: - # PY25 cannot handle unicode keys passed to class constructor - # example: cls(**data) + if not UNICODE_KWARGS: + # python 2.6.4 and lower cannot handle unicode keys + # passed to class constructor example: cls(**data) to_str_keys_recursive(data) if '_types' in data: diff --git a/mongoengine/python_support.py b/mongoengine/python_support.py index 02dc286..097740e 100644 --- a/mongoengine/python_support.py +++ b/mongoengine/python_support.py @@ -4,6 +4,7 @@ import sys PY3 = sys.version_info[0] == 3 PY25 = sys.version_info[:2] == (2, 5) +UNICODE_KWARGS = int(''.join([str(x) for x in sys.version_info[:3]])) > 264 if PY3: import codecs From 0dfd6aa518e80f8f2491177eab945b9725b6d0c8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Sep 2012 13:22:08 +0100 Subject: [PATCH 0782/1279] Updated travis.yml - now testing multiple pymongos --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index 3ec14ea..ac8c76f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,10 +6,17 @@ python: - 2.7 - 3.1 - 3.2 +env: + - PYMONGO=dev + - PYMONGO=2.3 + - PYMONGO=2.2 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install zlib1g zlib1g-dev; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo ln -s /usr/lib/i386-linux-gnu/libz.so /usr/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install PIL --use-mirrors ; true; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install PIL --use-mirrors ; true; fi + - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi + - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - python setup.py install script: - python setup.py test \ No newline at end of file From 02ef0df019a11dd18d1ce30f843d2d7f638d5bf8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Sep 2012 13:29:29 +0100 Subject: [PATCH 0783/1279] Updated travis.yml --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ac8c76f..e13c4dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,4 +19,6 @@ install: - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - python setup.py install script: - - python setup.py test \ No newline at end of file + - python setup.py test +notifications: + irc: "irc.freenode.org#mongoengine" \ No newline at end of file From fe7e17dbd52372fe4352d6f85f15bb76fd31d6b0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Sep 2012 15:21:52 +0100 Subject: [PATCH 0784/1279] Updated 0.7 branch --- docs/changelog.rst | 2 +- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4a4fb6a..6cfdbf6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,7 @@ Changelog ========= -Changes in 0.7.X +Changes in 0.7.0 ================= - Updated queryset.delete so you can use with skip / limit (MongoEngine/mongoengine#107) - Updated index creation allows kwargs to be passed through refs (MongoEngine/mongoengine#104) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index a42fe77..808d8a6 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, '7rc1') +VERSION = (0, 7, 0) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index ace9f38..f7458a1 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.7rc1 +Version: 0.7.0 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 8ad1f03dc5d9fd3c3e92a4602f6dd30a4b6ff46f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Sep 2012 15:23:30 +0100 Subject: [PATCH 0785/1279] Updated travis --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e13c4dd..3c77e75 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,4 +21,8 @@ install: script: - python setup.py test notifications: - irc: "irc.freenode.org#mongoengine" \ No newline at end of file + irc: "irc.freenode.org#mongoengine" +branches: + only: + - master + - 0.7 \ No newline at end of file From fd296918dac5b44105bb30487533e41bf2402348 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Sep 2012 15:51:36 +0100 Subject: [PATCH 0786/1279] Fixing readme for pypi --- README.rst | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 1305b6e..1c6336a 100644 --- a/README.rst +++ b/README.rst @@ -63,11 +63,6 @@ Some simple examples of what MongoEngine code looks like:: ... print 'Link:', post.url ... print ... - === Using MongoEngine === - See the tutorial - - === MongoEngine Docs === - Link: hmarr.com/mongoengine >>> len(BlogPost.objects) 2 @@ -85,7 +80,7 @@ Some simple examples of what MongoEngine code looks like:: Tests ===== To run the test suite, ensure you are running a local instance of MongoDB on -the standard port, and run ``python setup.py test``. +the standard port, and run: ``python setup.py test``. Community ========= @@ -93,11 +88,10 @@ Community `_ - `MongoEngine Developers mailing list `_ -- `#mongoengine IRC channel `_ +- `#mongoengine IRC channel `_ Contributing ============ The source is available on `GitHub `_ - to contribute to the project, fork it on GitHub and send a pull request, all contributions and suggestions are welcome! - From 1cf39896646b162ce644a75bf6d8cc6bbec197a4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Sep 2012 08:10:56 +0100 Subject: [PATCH 0787/1279] Updated setup.py --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c9d6b79..6d9b51b 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ try: except ImportError: pass -DESCRIPTION = "A Python Document-Object Mapper for working with MongoDB" - +DESCRIPTION = """MongoEngine is a Python Object-Document +Mapper for working with MongoDB.""" LONG_DESCRIPTION = None try: LONG_DESCRIPTION = open('README.rst').read() @@ -68,6 +68,7 @@ setup(name='mongoengine', maintainer="Ross Lawley", maintainer_email="ross.lawley@{nospam}gmail.com", url='http://mongoengine.org/', + download_url='https://github.com/MongoEngine/mongoengine/tarball/master', license='MIT', include_package_data=True, description=DESCRIPTION, From 3534bf7d70210616c9d997260f2725596951557e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Sep 2012 08:20:19 +0100 Subject: [PATCH 0788/1279] Fixed index spec inheritance (MongoEngine/mongoengine#111) --- docs/changelog.rst | 4 ++++ mongoengine/queryset.py | 4 ++++ tests/test_document.py | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6cfdbf6..c2a5cb4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.7.1 +================= +- Fixed index spec inheritance (MongoEngine/mongoengine#111) + Changes in 0.7.0 ================= - Updated queryset.delete so you can use with skip / limit (MongoEngine/mongoengine#107) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 8106e67..b020209 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -510,6 +510,10 @@ class QuerySet(object): use_types = allow_inheritance and not spec.get('sparse', False) for key in spec['fields']: + # If inherited spec continue + if isinstance(key, (list, tuple)): + continue + # Get ASCENDING direction from +, DESCENDING from -, and GEO2D from * direction = pymongo.ASCENDING if key.startswith("-"): diff --git a/tests/test_document.py b/tests/test_document.py index 75616c7..976a273 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -669,6 +669,25 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() + def test_inherited_index(self): + """Ensure index specs are inhertited correctly""" + + class A(Document): + title = StringField() + meta = { + 'indexes': [ + { + 'fields': ('title',), + }, + ], + 'allow_inheritance': True, + } + + class B(A): + description = StringField() + + self.assertEqual(A._meta['index_specs'], B._meta['index_specs']) + def test_db_field_load(self): """Ensure we load data correctly """ From 53339c7c72d7a0c4d3385286cb62c8b70e79e1e2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Sep 2012 08:21:44 +0100 Subject: [PATCH 0789/1279] Version bump --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 808d8a6..e3507cf 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 7, 0) +VERSION = (0, 7, 1) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index f7458a1..46dd9fe 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.7.0 +Version: 0.7.1 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From a8e787c1205d56598a339f550cd66b98753b2015 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Sep 2012 14:39:19 +0100 Subject: [PATCH 0790/1279] Update index spec generation so its not destructive (MongoEngine/mongoengine#113) --- docs/changelog.rst | 4 ++++ mongoengine/queryset.py | 5 ++++- tests/test_document.py | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c2a5cb4..d7c7761 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.7.X +================ +- Update index spec generation so its not destructive (MongoEngine/mongoengine#113) + Changes in 0.7.1 ================= - Fixed index spec inheritance (MongoEngine/mongoengine#111) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index b020209..6f913fc 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -398,6 +398,7 @@ class QuerySet(object): or a **-** to determine the index ordering """ index_spec = QuerySet._build_index_spec(self._document, key_or_list) + index_spec = index_spec.copy() fields = index_spec.pop('fields') index_spec['drop_dups'] = drop_dups index_spec['background'] = background @@ -472,7 +473,9 @@ class QuerySet(object): # Ensure document-defined indexes are created if self._document._meta['index_specs']: - for spec in self._document._meta['index_specs']: + index_spec = self._document._meta['index_specs'] + for spec in index_spec: + spec = spec.copy() fields = spec.pop('fields') types_indexed = types_indexed or includes_types(fields) opts = index_opts.copy() diff --git a/tests/test_document.py b/tests/test_document.py index 976a273..9637bfe 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -688,6 +688,25 @@ class DocumentTest(unittest.TestCase): self.assertEqual(A._meta['index_specs'], B._meta['index_specs']) + def test_build_index_spec_is_not_destructive(self): + + class MyDoc(Document): + keywords = StringField() + + meta = { + 'indexes': ['keywords'], + 'allow_inheritance': False + } + + self.assertEqual(MyDoc._meta['index_specs'], + [{'fields': [('keywords', 1)]}]) + + # Force index creation + MyDoc.objects._ensure_indexes() + + self.assertEqual(MyDoc._meta['index_specs'], + [{'fields': [('keywords', 1)]}]) + def test_db_field_load(self): """Ensure we load data correctly """ From 99637151b57669b8f342cab17bf14e51ecc6fe19 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 5 Sep 2012 10:24:25 +0100 Subject: [PATCH 0791/1279] Updated version --- docs/changelog.rst | 2 +- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d7c7761..b4c3143 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,7 @@ Changelog ========= -Changes in 0.7.X +Changes in 0.7.2 ================ - Update index spec generation so its not destructive (MongoEngine/mongoengine#113) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index e3507cf..e91e81f 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 7, 1) +VERSION = (0, 7, 2) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 46dd9fe..9dbb16f 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.7.1 +Version: 0.7.2 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From a2183e3dcc474e1cc6e89d8ecfd78a179b9f0739 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 7 Sep 2012 13:23:46 +0100 Subject: [PATCH 0792/1279] Reverted EmbeddedDocuments meta handling. You now can turn off inheritance (MongoEngine/mongoengine#119) --- docs/changelog.rst | 4 ++++ mongoengine/__init__.py | 2 +- mongoengine/base.py | 37 ++++++++++++++++++++++--------------- mongoengine/document.py | 8 ++++++++ python-mongoengine.spec | 2 +- tests/test_document.py | 26 +++++++++++++++++++++++++- 6 files changed, 61 insertions(+), 18 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b4c3143..2d10507 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.7.3 +================ +- Reverted EmbeddedDocuments meta handling - now can turn off inheritance (MongoEngine/mongoengine#119) + Changes in 0.7.2 ================ - Update index spec generation so its not destructive (MongoEngine/mongoengine#113) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index e91e81f..f5668c4 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 7, 2) +VERSION = (0, 7, 3) def get_version(): diff --git a/mongoengine/base.py b/mongoengine/base.py index dc7ef8a..6e4cd91 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -508,6 +508,10 @@ class DocumentMetaclass(type): attrs['_is_document'] = attrs.get('_is_document', False) + # EmbeddedDocuments could have meta data for inheritance + if 'meta' in attrs: + attrs['_meta'] = attrs.pop('meta') + # Handle document Fields # Merge all fields from subclasses @@ -571,6 +575,24 @@ class DocumentMetaclass(type): superclasses[base._class_name] = base superclasses.update(base._superclasses) + if hasattr(base, '_meta'): + # Warn if allow_inheritance isn't set and prevent + # inheritance of classes where inheritance is set to False + allow_inheritance = base._meta.get('allow_inheritance', + ALLOW_INHERITANCE) + if (not getattr(base, '_is_base_cls', True) + and allow_inheritance is None): + warnings.warn( + "%s uses inheritance, the default for " + "allow_inheritance is changing to off by default. " + "Please add it to the document meta." % name, + FutureWarning + ) + elif (allow_inheritance == False and + not base._meta.get('abstract')): + raise ValueError('Document %s may not be subclassed' % + base.__name__) + attrs['_class_name'] = '.'.join(reversed(class_name)) attrs['_superclasses'] = superclasses @@ -745,21 +767,6 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): if hasattr(base, 'meta'): meta.merge(base.meta) elif hasattr(base, '_meta'): - # Warn if allow_inheritance isn't set and prevent - # inheritance of classes where inheritance is set to False - allow_inheritance = base._meta.get('allow_inheritance', - ALLOW_INHERITANCE) - if not base._is_base_cls and allow_inheritance is None: - warnings.warn( - "%s uses inheritance, the default for " - "allow_inheritance is changing to off by default. " - "Please add it to the document meta." % name, - FutureWarning - ) - elif (allow_inheritance == False and - not base._meta.get('abstract')): - raise ValueError('Document %s may not be subclassed' % - base.__name__) meta.merge(base._meta) # Set collection in the meta if its callable diff --git a/mongoengine/document.py b/mongoengine/document.py index 9d528cb..23009db 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -25,6 +25,14 @@ class EmbeddedDocument(BaseDocument): collection. :class:`~mongoengine.EmbeddedDocument`\ s should be used as fields on :class:`~mongoengine.Document`\ s through the :class:`~mongoengine.EmbeddedDocumentField` field type. + + A :class:`~mongoengine.EmbeddedDocument` subclass may be itself subclassed, + to create a specialised version of the embedded document that will be + stored in the same collection. To facilitate this behaviour, `_cls` and + `_types` fields are added to documents (hidden though the MongoEngine + interface though). To disable this behaviour and remove the dependence on + the presence of `_cls` and `_types`, set :attr:`allow_inheritance` to + ``False`` in the :attr:`meta` dictionary. """ # The __metaclass__ attribute is removed by 2to3 when running with Python3 diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 9dbb16f..12b867c 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.7.2 +Version: 0.7.3 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB diff --git a/tests/test_document.py b/tests/test_document.py index 9637bfe..48ff822 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -338,7 +338,6 @@ class DocumentTest(unittest.TestCase): meta = {'allow_inheritance': False} self.assertRaises(ValueError, create_employee_class) - def test_allow_inheritance_abstract_document(self): """Ensure that abstract documents can set inheritance rules and that _cls and _types will not be used. @@ -366,6 +365,31 @@ class DocumentTest(unittest.TestCase): Animal.drop_collection() + def test_allow_inheritance_embedded_document(self): + + # Test the same for embedded documents + class Comment(EmbeddedDocument): + content = StringField() + meta = {'allow_inheritance': False} + + def create_special_comment(): + class SpecialComment(Comment): + pass + + self.assertRaises(ValueError, create_special_comment) + + comment = Comment(content='test') + self.assertFalse('_cls' in comment.to_mongo()) + self.assertFalse('_types' in comment.to_mongo()) + + class Comment(EmbeddedDocument): + content = StringField() + meta = {'allow_inheritance': True} + + comment = Comment(content='test') + self.assertTrue('_cls' in comment.to_mongo()) + self.assertTrue('_types' in comment.to_mongo()) + def test_document_inheritance(self): """Ensure mutliple inheritance of abstract docs works """ From 5c6035d636ec76e9bc5c9d5a9a6923eaf139c6d4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 11 Sep 2012 08:56:32 +0000 Subject: [PATCH 0793/1279] Updated upgrade docs for BinaryFields --- docs/upgrade.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index d01911c..82ac7ca 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -61,6 +61,13 @@ stored in rather than as string representations. Your code may need to be updated to handle native types rather than strings keys for the results of item frequency queries. +BinaryFields +------------ + +Binary fields have been updated so that they are native binary types. If you +previously were doing `str` comparisons with binary field values you will have +to update and wrap the value in a `str`. + 0.5 to 0.6 ========== From 0ea4abda815277c2c885704adcdd98aaab482584 Mon Sep 17 00:00:00 2001 From: Mahmoud Hossam Date: Tue, 11 Sep 2012 12:38:40 +0300 Subject: [PATCH 0794/1279] Update docs/index.rst Correct link for the source. --- docs/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index a701de2..f6d44b5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,10 +34,10 @@ To get help with using MongoEngine, use the `MongoEngine Users mailing list Contributing ------------ -The source is available on `GitHub `_ and +The source is available on `GitHub `_ and contributions are always encouraged. Contributions can be as simple as minor tweaks to this documentation. To contribute, fork the project on -`GitHub `_ and send a +`GitHub `_ and send a pull request. Also, you can join the developers' `mailing list From 5949970a9562dd44e4282087f7a92ce0767a8f3e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 11 Sep 2012 15:14:37 +0000 Subject: [PATCH 0795/1279] Fixed index inheritance issues firmed up testcases (MongoEngine/mongoengine#123) (MongoEngine/mongoengine#125) --- docs/changelog.rst | 5 +++++ mongoengine/queryset.py | 6 ++++-- tests/test_document.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2d10507..2820ed5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,11 @@ Changelog ========= + +Changes in 0.7.4 +================ +- Fixed index inheritance issues - firmed up testcases (MongoEngine/mongoengine#123) (MongoEngine/mongoengine#125) + Changes in 0.7.3 ================ - Reverted EmbeddedDocuments meta handling - now can turn off inheritance (MongoEngine/mongoengine#119) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6f913fc..5a1aa71 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -501,8 +501,10 @@ class QuerySet(object): """ if isinstance(spec, basestring): spec = {'fields': [spec]} - if isinstance(spec, (list, tuple)): - spec = {'fields': spec} + elif isinstance(spec, (list, tuple)): + spec = {'fields': list(spec)} + elif isinstance(spec, dict): + spec = dict(spec) index_list = [] direction = None diff --git a/tests/test_document.py b/tests/test_document.py index 48ff822..d2969fe 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -420,6 +420,9 @@ class DocumentTest(unittest.TestCase): 'indexes': ['name'] } + self.assertEqual(Animal._meta['index_specs'], + [{'fields': [('_types', 1), ('name', 1)]}]) + Animal.drop_collection() dog = Animal(name='dog') @@ -441,6 +444,9 @@ class DocumentTest(unittest.TestCase): 'allow_inheritance': False, 'indexes': ['name'] } + + self.assertEqual(Animal._meta['index_specs'], + [{'fields': [('name', 1)]}]) collection.update({}, {"$unset": {"_types": 1, "_cls": 1}}, multi=True) # Confirm extra data is removed @@ -658,6 +664,12 @@ class DocumentTest(unittest.TestCase): 'allow_inheritance': True } + self.assertEqual(BlogPost._meta['index_specs'], + [{'fields': [('_types', 1), ('addDate', -1)]}, + {'fields': [('tags', 1)]}, + {'fields': [('_types', 1), ('category', 1), + ('addDate', -1)]}]) + BlogPost.drop_collection() info = BlogPost.objects._collection.index_information() @@ -681,6 +693,13 @@ class DocumentTest(unittest.TestCase): title = StringField() meta = {'indexes': ['title']} + self.assertEqual(ExtendedBlogPost._meta['index_specs'], + [{'fields': [('_types', 1), ('addDate', -1)]}, + {'fields': [('tags', 1)]}, + {'fields': [('_types', 1), ('category', 1), + ('addDate', -1)]}, + {'fields': [('_types', 1), ('title', 1)]}]) + BlogPost.drop_collection() list(ExtendedBlogPost.objects) @@ -711,6 +730,8 @@ class DocumentTest(unittest.TestCase): description = StringField() self.assertEqual(A._meta['index_specs'], B._meta['index_specs']) + self.assertEqual([{'fields': [('_types', 1), ('title', 1)]}], + A._meta['index_specs']) def test_build_index_spec_is_not_destructive(self): @@ -791,6 +812,9 @@ class DocumentTest(unittest.TestCase): 'allow_inheritance': False } + self.assertEqual([{'fields': [('rank.title', 1)]}], + Person._meta['index_specs']) + Person.drop_collection() # Indexes are lazy so use list() to perform query @@ -809,6 +833,10 @@ class DocumentTest(unittest.TestCase): '*location.point', ], } + + self.assertEqual([{'fields': [('location.point', '2d')]}], + Place._meta['index_specs']) + Place.drop_collection() info = Place.objects._collection.index_information() @@ -834,6 +862,10 @@ class DocumentTest(unittest.TestCase): ], } + self.assertEqual([{'fields': [('addDate', -1)], 'unique': True, + 'sparse': True, 'types': False}], + BlogPost._meta['index_specs']) + BlogPost.drop_collection() info = BlogPost.objects._collection.index_information() From 4defc821927842ae0df62907a769a48ad2ad721d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 11 Sep 2012 15:16:39 +0000 Subject: [PATCH 0796/1279] Version Bump --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index f5668c4..22f0334 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 7, 3) +VERSION = (0, 7, 4) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 12b867c..8a3d475 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.7.3 +Version: 0.7.4 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 864053615bc36709cd6c8e8777b0475079f82292 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 14 Sep 2012 10:18:44 +0000 Subject: [PATCH 0797/1279] Updated docs --- docs/guide/defining-documents.rst | 4 ++++ docs/guide/signals.rst | 7 +++++++ mongoengine/fields.py | 4 ++++ 3 files changed, 15 insertions(+) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 6c8b29e..d749e00 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -344,6 +344,10 @@ Its value can take any of the following constants: their :file:`models.py` in the :const:`INSTALLED_APPS` tuple. +.. warning:: + Signals are not triggered when doing cascading updates / deletes - if this + is required you must manually handle the update / delete. + Generic reference fields '''''''''''''''''''''''' A second kind of reference field also exists, diff --git a/docs/guide/signals.rst b/docs/guide/signals.rst index c25c178..75f81e2 100644 --- a/docs/guide/signals.rst +++ b/docs/guide/signals.rst @@ -50,4 +50,11 @@ Example usage:: signals.post_save.connect(Author.post_save, sender=Author) +ReferenceFields and signals +--------------------------- + +Currently `reverse_delete_rules` do not trigger signals on the other part of +the relationship. If this is required you must manually handled the +reverse deletion. + .. _blinker: http://pypi.python.org/pypi/blinker diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 51d062a..e8d9236 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -709,6 +709,10 @@ class ReferenceField(BaseField): Bar.register_delete_rule(Foo, 'bar', NULLIFY) + .. note :: + `reverse_delete_rules` do not trigger pre / post delete signals to be + triggered. + .. versionchanged:: 0.5 added `reverse_delete_rule` """ From ab4d4e6230bb737cac7ae2301a10e6368e82cfe8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 18 Sep 2012 21:37:45 +0000 Subject: [PATCH 0798/1279] Fix ReferenceField dbref = False --- docs/changelog.rst | 4 ++++ mongoengine/__init__.py | 2 +- mongoengine/fields.py | 2 +- python-mongoengine.spec | 2 +- tests/test_dereference.py | 6 +++--- tests/test_fields.py | 6 +++--- 6 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2820ed5..638c0e8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.7.5 +================ +- ReferenceFields with dbref=False use ObjectId instead of strings (MongoEngine/mongoengine#134) + See ticket for upgrade notes (https://github.com/MongoEngine/mongoengine/issues/134) Changes in 0.7.4 ================ diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 22f0334..9044e61 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 7, 4) +VERSION = (0, 7, 5) def get_version(): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e8d9236..12cc7f2 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -792,7 +792,7 @@ class ReferenceField(BaseField): collection = self.document_type._get_collection_name() return DBRef(collection, id_) - return "%s" % id_ + return id_ def to_python(self, value): """Convert a MongoDB-compatible type to a Python type. diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 8a3d475..641b3de 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.7.4 +Version: 0.7.5 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 120b1a2..0d757fd 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1,7 +1,7 @@ from __future__ import with_statement import unittest -from bson import DBRef +from bson import DBRef, ObjectId from mongoengine import * from mongoengine.connection import get_db @@ -187,8 +187,8 @@ class FieldTest(unittest.TestCase): self.assertEqual(group.members, [user]) raw_data = Group._get_collection().find_one() - self.assertTrue(isinstance(raw_data['author'], basestring)) - self.assertTrue(isinstance(raw_data['members'][0], basestring)) + self.assertTrue(isinstance(raw_data['author'], ObjectId)) + self.assertTrue(isinstance(raw_data['members'][0], ObjectId)) def test_recursive_reference(self): """Ensure that ReferenceFields can reference their own documents. diff --git a/tests/test_fields.py b/tests/test_fields.py index ce80bf9..04c3765 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -7,7 +7,7 @@ import tempfile from decimal import Decimal -from bson import Binary, DBRef +from bson import Binary, DBRef, ObjectId import gridfs from nose.plugins.skip import SkipTest @@ -1104,7 +1104,7 @@ class FieldTest(unittest.TestCase): p = Person.objects.get(name="Ross") self.assertEqual(p.parent, p1) - def test_str_reference_fields(self): + def test_objectid_reference_fields(self): class Person(Document): name = StringField() @@ -1117,7 +1117,7 @@ class FieldTest(unittest.TestCase): col = Person._get_collection() data = col.find_one({'name': 'Ross'}) - self.assertEqual(data['parent'], "%s" % p1.pk) + self.assertEqual(data['parent'], p1.pk) p = Person.objects.get(name="Ross") self.assertEqual(p.parent, p1) From b9253d86cc6f6767fa347d9cb02d8265ceae247e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Sep 2012 18:53:31 +0000 Subject: [PATCH 0799/1279] Updated travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3c77e75..c10a1f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ # http://travis-ci.org/#!/MongoEngine/mongoengine language: python +services: mongodb python: - 2.5 - 2.6 From 3090adac04d75020b9e89698ac7f29f1a31cdc1b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 24 Sep 2012 11:37:54 +0000 Subject: [PATCH 0800/1279] Fixed objectId for DBRef --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 12cc7f2..7729444 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -770,7 +770,7 @@ class ReferenceField(BaseField): def to_mongo(self, document): if isinstance(document, DBRef): if not self.dbref: - return "%s" % DBRef.id + return DBRef.id return document elif not self.dbref and isinstance(document, basestring): return document From adb60ef1acc8a8baf3476ce49a44c5ec73335fd2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 24 Sep 2012 18:45:02 +0000 Subject: [PATCH 0801/1279] Improved import cache --- mongoengine/base.py | 61 +++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 6e4cd91..773c1d4 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -235,7 +235,8 @@ class BaseField(object): pass def _validate(self, value): - from mongoengine import Document, EmbeddedDocument + Document = _import_class('Document') + EmbeddedDocument = _import_class('EmbeddedDocument') # check choices if self.choices: is_cls = isinstance(value, (Document, EmbeddedDocument)) @@ -283,7 +284,9 @@ class ComplexBaseField(BaseField): if instance is None: # Document class being used rather than a document object return self - from fields import GenericReferenceField, ReferenceField + + ReferenceField = _import_class('ReferenceField') + GenericReferenceField = _import_class('GenericReferenceField') dereference = self.field is None or isinstance(self.field, (GenericReferenceField, ReferenceField)) if not self._dereference and instance._initialised and dereference: @@ -310,6 +313,7 @@ class ComplexBaseField(BaseField): ) value._dereferenced = True instance._data[self.name] = value + return value def __set__(self, instance, value): @@ -321,7 +325,7 @@ class ComplexBaseField(BaseField): def to_python(self, value): """Convert a MongoDB-compatible type to a Python type. """ - from mongoengine import Document + Document = _import_class('Document') if isinstance(value, basestring): return value @@ -363,7 +367,7 @@ class ComplexBaseField(BaseField): def to_mongo(self, value): """Convert a Python type to a MongoDB-compatible type. """ - from mongoengine import Document + Document = _import_class("Document") if isinstance(value, basestring): return value @@ -399,7 +403,7 @@ class ComplexBaseField(BaseField): meta.get('allow_inheritance', ALLOW_INHERITANCE) == False) if allow_inheritance and not self.field: - from fields import GenericReferenceField + GenericReferenceField = _import_class("GenericReferenceField") value_dict[k] = GenericReferenceField().to_mongo(v) else: collection = v._get_collection_name() @@ -460,7 +464,7 @@ class ComplexBaseField(BaseField): @property def _dereference(self,): if not self.__dereference: - from dereference import DeReference + DeReference = _import_class("DeReference") self.__dereference = DeReference() # Cached return self.__dereference @@ -943,7 +947,7 @@ class BaseDocument(object): field = None if not hasattr(self, name) and not name.startswith('_'): - from fields import DynamicField + DynamicField = _import_class("DynamicField") field = DynamicField(db_field=name) field.name = name self._dynamic_fields[name] = field @@ -1121,7 +1125,8 @@ class BaseDocument(object): def _get_changed_fields(self, key='', inspected=None): """Returns a list of all fields that have explicitly been changed. """ - from mongoengine import EmbeddedDocument, DynamicEmbeddedDocument + EmbeddedDocument = _import_class("EmbeddedDocument") + DynamicEmbeddedDocument = _import_class("DynamicEmbeddedDocument") _changed_fields = [] _changed_fields += getattr(self, '_changed_fields', []) @@ -1252,7 +1257,9 @@ class BaseDocument(object): geo_indices = [] inspected.append(cls) - from fields import EmbeddedDocumentField, GeoPointField + EmbeddedDocumentField = _import_class("EmbeddedDocumentField") + GeoPointField = _import_class("GeoPointField") + for field in cls._fields.values(): if not isinstance(field, (EmbeddedDocumentField, GeoPointField)): continue @@ -1486,14 +1493,30 @@ def _import_class(cls_name): """Cached mechanism for imports""" if cls_name in _class_registry: return _class_registry.get(cls_name) - if cls_name == 'Document': - from mongoengine.document import Document as cls - elif cls_name == 'EmbeddedDocument': - from mongoengine.document import EmbeddedDocument as cls - elif cls_name == 'DictField': - from mongoengine.fields import DictField as cls - elif cls_name == 'OperationError': - from queryset import OperationError as cls - _class_registry[cls_name] = cls - return cls + doc_classes = ['Document', 'DynamicEmbeddedDocument', 'EmbeddedDocument'] + field_classes = ['DictField', 'DynamicField', 'EmbeddedDocumentField', + 'GenericReferenceField', 'GeoPointField', + 'ReferenceField'] + queryset_classes = ['OperationError'] + deref_classes = ['DeReference'] + + if cls_name in doc_classes: + from mongoengine import document as module + import_classes = doc_classes + elif cls_name in field_classes: + from mongoengine import fields as module + import_classes = field_classes + elif cls_name in queryset_classes: + from mongoengine import queryset as module + import_classes = queryset_classes + elif cls_name in deref_classes: + from mongoengine import dereference as module + import_classes = deref_classes + else: + raise ValueError('No import set for: ' % cls_name) + + for cls in import_classes: + _class_registry[cls] = getattr(module, cls) + + return _class_registry.get(cls_name) From 6a4351e44f326476df2abd8aec4ff8973b22685e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 24 Sep 2012 18:49:29 +0000 Subject: [PATCH 0802/1279] Fixed reload issue with ReferenceField where dbref=False (MongoEngine/mongoengine#138) --- docs/changelog.rst | 4 ++++ mongoengine/dereference.py | 8 ++++---- mongoengine/document.py | 7 ++++++- tests/test_dereference.py | 1 + 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 638c0e8..1504b08 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.7.X +================ +- Fixed reload issue with ReferenceField where dbref=False (MongoEngine/mongoengine#138) + Changes in 0.7.5 ================ - ReferenceFields with dbref=False use ObjectId instead of strings (MongoEngine/mongoengine#134) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index d371187..386dbf4 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -31,10 +31,10 @@ class DeReference(object): items = [i for i in items] self.max_depth = max_depth - doc_type = None + if instance and instance._fields: - doc_type = instance._fields[name] + doc_type = instance._fields.get(name) if hasattr(doc_type, 'field'): doc_type = doc_type.field @@ -134,7 +134,7 @@ class DeReference(object): elif doc_type is None: doc = get_document( ''.join(x.capitalize() - for x in col.split('_')))._from_son(ref) + for x in col.split('_')))._from_son(ref) else: doc = doc_type._from_son(ref) object_map[doc.id] = doc @@ -166,7 +166,7 @@ class DeReference(object): return self.object_map.get(items['_ref'].id, items) elif '_types' in items and '_cls' in items: doc = get_document(items['_cls'])._from_son(items) - doc._data = self._attach_objects(doc._data, depth, doc, name) + doc._data = self._attach_objects(doc._data, depth, doc, None) return doc if not hasattr(items, 'items'): diff --git a/mongoengine/document.py b/mongoengine/document.py index 23009db..7b3afaf 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -361,7 +361,12 @@ class Document(BaseDocument): id_field = self._meta['id_field'] obj = self.__class__.objects( **{id_field: self[id_field]} - ).first().select_related(max_depth=max_depth) + ).limit(1).select_related(max_depth=max_depth) + if obj: + obj = obj[0] + else: + msg = "Reloaded document has been deleted" + raise OperationError(msg) for field in self._fields: setattr(self, field, self._reload(field, obj[field])) if self._dynamic: diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 0d757fd..7b149db 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -84,6 +84,7 @@ class FieldTest(unittest.TestCase): group = Group(members=User.objects) group.save() + group.reload() # Confirm reload works with query_counter() as q: self.assertEqual(q, 0) From e2c78047b15753e1ee902a02373feea146ee43e4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 26 Sep 2012 10:43:14 +0000 Subject: [PATCH 0803/1279] Moved injection of Exceptions to top level --- mongoengine/base.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 773c1d4..92dcfa9 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -635,17 +635,6 @@ class DocumentMetaclass(type): "field name" % field.name) raise InvalidDocumentError(msg) - # Merge in exceptions with parent hierarchy - exceptions_to_merge = (DoesNotExist, MultipleObjectsReturned) - module = attrs.get('__module__') - for exc in exceptions_to_merge: - name = exc.__name__ - parents = tuple(getattr(base, name) for base in flattened_bases - if hasattr(base, name)) or (exc,) - # Create new exception and set to new_class - exception = type(name, parents, {'__module__': module}) - setattr(new_class, name, exception) - # Add class to the _document_registry _document_registry[new_class._class_name] = new_class @@ -836,6 +825,17 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): new_class._fields['id'] = ObjectIdField(db_field='_id') new_class.id = new_class._fields['id'] + # Merge in exceptions with parent hierarchy + exceptions_to_merge = (DoesNotExist, MultipleObjectsReturned) + module = attrs.get('__module__') + for exc in exceptions_to_merge: + name = exc.__name__ + parents = tuple(getattr(base, name) for base in flattened_bases + if hasattr(base, name)) or (exc,) + # Create new exception and set to new_class + exception = type(name, parents, {'__module__': module}) + setattr(new_class, name, exception) + return new_class @classmethod From 6a31736644452ec6598dff8851f7847aef63da11 Mon Sep 17 00:00:00 2001 From: Luis Araujo Date: Wed, 26 Sep 2012 14:43:59 -0300 Subject: [PATCH 0804/1279] Initial support to Group and Permission. The /admin can't be exec login in MongoDB yet. Only SQLsDB (SQLite,...) This code work with django-mongoadmin pluggin. --- mongoengine/django/auth.py | 187 +++++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index a30fc57..d9f0584 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -3,6 +3,8 @@ import datetime from mongoengine import * from django.utils.encoding import smart_str +from django.db import models +from django.contrib.contenttypes.models import ContentTypeManager from django.contrib.auth.models import AnonymousUser from django.utils.translation import ugettext_lazy as _ @@ -34,6 +36,191 @@ except ImportError: REDIRECT_FIELD_NAME = 'next' +class ContentType(Document): + name = StringField(max_length=100) + app_label = StringField(max_length=100) + model = StringField(max_length=100, verbose_name=_('python model class name'), + unique_with='app_label') + objects = ContentTypeManager() + + class Meta: + verbose_name = _('content type') + verbose_name_plural = _('content types') + # db_table = 'django_content_type' + # ordering = ('name',) + # unique_together = (('app_label', 'model'),) + + def __unicode__(self): + return self.name + + def model_class(self): + "Returns the Python model class for this type of content." + from django.db import models + return models.get_model(self.app_label, self.model) + + def get_object_for_this_type(self, **kwargs): + """ + Returns an object of this type for the keyword arguments given. + Basically, this is a proxy around this object_type's get_object() model + method. The ObjectNotExist exception, if thrown, will not be caught, + so code that calls this method should catch it. + """ + return self.model_class()._default_manager.using(self._state.db).get(**kwargs) + + def natural_key(self): + return (self.app_label, self.model) + +class SiteProfileNotAvailable(Exception): + pass + +class PermissionManager(models.Manager): + def get_by_natural_key(self, codename, app_label, model): + return self.get( + codename=codename, + content_type=ContentType.objects.get_by_natural_key(app_label, model) + ) + +class Permission(Document): + """The permissions system provides a way to assign permissions to specific users and groups of users. + + The permission system is used by the Django admin site, but may also be useful in your own code. The Django admin site uses permissions as follows: + + - The "add" permission limits the user's ability to view the "add" form and add an object. + - The "change" permission limits a user's ability to view the change list, view the "change" form and change an object. + - The "delete" permission limits the ability to delete an object. + + Permissions are set globally per type of object, not per specific object instance. It is possible to say "Mary may change news stories," but it's not currently possible to say "Mary may change news stories, but only the ones she created herself" or "Mary may only change news stories that have a certain status or publication date." + + Three basic permissions -- add, change and delete -- are automatically created for each Django model. + """ + name = StringField(max_length=50, verbose_name=_('username')) + content_type = ReferenceField(ContentType) + codename = StringField(max_length=100, verbose_name=_('codename')) + # FIXME: don't access field of the other class + # unique_with=['content_type__app_label', 'content_type__model']) + + objects = PermissionManager() + + class Meta: + verbose_name = _('permission') + verbose_name_plural = _('permissions') + # unique_together = (('content_type', 'codename'),) + # ordering = ('content_type__app_label', 'content_type__model', 'codename') + + def __unicode__(self): + return u"%s | %s | %s" % ( + unicode(self.content_type.app_label), + unicode(self.content_type), + unicode(self.name)) + + def natural_key(self): + return (self.codename,) + self.content_type.natural_key() + natural_key.dependencies = ['contenttypes.contenttype'] + +class Group(Document): + """Groups are a generic way of categorizing users to apply permissions, or some other label, to those users. A user can belong to any number of groups. + + A user in a group automatically has all the permissions granted to that group. For example, if the group Site editors has the permission can_edit_home_page, any user in that group will have that permission. + + Beyond permissions, groups are a convenient way to categorize users to apply some label, or extended functionality, to them. For example, you could create a group 'Special users', and you could write code that would do special things to those users -- such as giving them access to a members-only portion of your site, or sending them members-only e-mail messages. + """ + name = StringField(max_length=80, unique=True, verbose_name=_('name')) + # permissions = models.ManyToManyField(Permission, verbose_name=_('permissions'), blank=True) + permissions = ListField(ReferenceField(Permission, verbose_name=_('permissions'), required=False)) + + class Meta: + verbose_name = _('group') + verbose_name_plural = _('groups') + + def __unicode__(self): + return self.name + +class UserManager(models.Manager): + def create_user(self, username, email, password=None): + """ + Creates and saves a User with the given username, e-mail and password. + """ + now = datetime.datetime.now() + + # Normalize the address by lowercasing the domain part of the email + # address. + try: + email_name, domain_part = email.strip().split('@', 1) + except ValueError: + pass + else: + email = '@'.join([email_name, domain_part.lower()]) + + user = self.model(username=username, email=email, is_staff=False, + is_active=True, is_superuser=False, last_login=now, + date_joined=now) + + user.set_password(password) + user.save(using=self._db) + return user + + def create_superuser(self, username, email, password): + u = self.create_user(username, email, password) + u.is_staff = True + u.is_active = True + u.is_superuser = True + u.save(using=self._db) + return u + + def make_random_password(self, length=10, allowed_chars='abcdefghjkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789'): + "Generates a random password with the given length and given allowed_chars" + # Note that default value of allowed_chars does not have "I" or letters + # that look like it -- just to avoid confusion. + from random import choice + return ''.join([choice(allowed_chars) for i in range(length)]) + + +# A few helper functions for common logic between User and AnonymousUser. +def _user_get_all_permissions(user, obj): + permissions = set() + anon = user.is_anonymous() + for backend in auth.get_backends(): + if not anon or backend.supports_anonymous_user: + if hasattr(backend, "get_all_permissions"): + if obj is not None: + if backend.supports_object_permissions: + permissions.update( + backend.get_all_permissions(user, obj) + ) + else: + permissions.update(backend.get_all_permissions(user)) + return permissions + + +def _user_has_perm(user, perm, obj): + anon = user.is_anonymous() + active = user.is_active + for backend in auth.get_backends(): + if (not active and not anon and backend.supports_inactive_user) or \ + (not anon or backend.supports_anonymous_user): + if hasattr(backend, "has_perm"): + if obj is not None: + if (backend.supports_object_permissions and + backend.has_perm(user, perm, obj)): + return True + else: + if backend.has_perm(user, perm): + return True + return False + + +def _user_has_module_perms(user, app_label): + anon = user.is_anonymous() + active = user.is_active + for backend in auth.get_backends(): + if (not active and not anon and backend.supports_inactive_user) or \ + (not anon or backend.supports_anonymous_user): + if hasattr(backend, "has_module_perms"): + if backend.has_module_perms(user, app_label): + return True + return False + + class User(Document): """A User document that aims to mirror most of the API specified by Django at http://docs.djangoproject.com/en/dev/topics/auth/#users From 59bfe551a301368e8aff6648906026a52eff37d2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 27 Sep 2012 08:29:49 +0000 Subject: [PATCH 0805/1279] Updated docs thanks @mitar --- docs/guide/defining-documents.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index d749e00..3ee7796 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -469,7 +469,7 @@ If a dictionary is passed then the following options are available: Whether the index should be sparse. :attr:`unique` (Default: False) - Whether the index should be sparse. + Whether the index should be unique. .. note :: From 7c1ee28f13fe9c947d956d9b2ba298cba4776e1a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 27 Sep 2012 10:26:22 +0000 Subject: [PATCH 0806/1279] Added CONTRIBUTING.rst --- CONTRIBUTING.rst | 61 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 CONTRIBUTING.rst diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..b80324d --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,61 @@ +Contributing to MongoEngine +=========================== + +MongoEngine has a large `community +`_ and +contributions are always encouraged. Contributions can be as simple as +minor tweaks to the documentation. Please read these guidelines before +sending a pull request. + +Bugfixes and New Features +------------------------- + +Before starting to write code, look for existing `tickets +`_ or `create one +`_ for your specific +issue or feature request. That way you avoid working on something +that might not be of interest or that has already been addressed. If in doubt +post to the `user group ` + +Supported Interpreters +---------------------- + +PyMongo supports CPython 2.5 and newer. Language +features not supported by all interpreters can not be used. +Please also ensure that your code is properly converted by +`2to3 `_ for Python 3 support. + +Style Guide +----------- + +MongoEngine aims to follow `PEP8 `_ +including 4 space indents and 79 character line limits. + +Testing +------- + +All tests are run on `Travis `_ +and any pull requests are automatically tested by Travis. Any pull requests +without tests will take longer to be integrated and might be refused. + +General Guidelines +------------------ + +- Avoid backward breaking changes if at all possible. +- Write inline documentation for new classes and methods. +- Write tests and make sure they pass (make sure you have a mongod + running on the default port, then execute ``python setup.py test`` + from the cmd line to run the test suite). +- Add yourself to AUTHORS.rst :) + +Documentation +------------- + +To contribute to the `API documentation +`_ +just make your changes to the inline documentation of the appropriate +`source code `_ or `rst file +`_ in a +branch and submit a `pull request `_. +You might also use the github `Edit `_ +button. From 3ce163b1a07c6695c2134942e11d6557fb92467b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 27 Sep 2012 10:29:24 +0000 Subject: [PATCH 0807/1279] Updates to the readme --- CONTRIBUTING.rst | 4 ++-- README.rst | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index b80324d..9688339 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -54,8 +54,8 @@ Documentation To contribute to the `API documentation `_ just make your changes to the inline documentation of the appropriate -`source code `_ or `rst file -`_ in a +`source code `_ or `rst file +`_ in a branch and submit a `pull request `_. You might also use the github `Edit `_ button. diff --git a/README.rst b/README.rst index 1c6336a..ae6bd0e 100644 --- a/README.rst +++ b/README.rst @@ -92,6 +92,4 @@ Community Contributing ============ -The source is available on `GitHub `_ - to -contribute to the project, fork it on GitHub and send a pull request, all -contributions and suggestions are welcome! +We welcome contributions! see the`Contribution guidelines `_ From 4b2ad254057a440c2a45016fe8da3318837fb440 Mon Sep 17 00:00:00 2001 From: Garry Polley Date: Thu, 27 Sep 2012 10:21:05 -0500 Subject: [PATCH 0808/1279] can now use AuthenticationBackends with permissions. --- mongoengine/django/auth.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index a30fc57..65afacf 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -3,6 +3,8 @@ import datetime from mongoengine import * from django.utils.encoding import smart_str +from django.contrib.auth.models import _user_get_all_permissions +from django.contrib.auth.models import _user_has_perm from django.contrib.auth.models import AnonymousUser from django.utils.translation import ugettext_lazy as _ @@ -104,6 +106,25 @@ class User(Document): """ return check_password(raw_password, self.password) + def get_all_permissions(self, obj=None): + return _user_get_all_permissions(self, obj) + + def has_perm(self, perm, obj=None): + """ + Returns True if the user has the specified permission. This method + queries all available auth backends, but returns immediately if any + backend returns True. Thus, a user who has permission from a single + auth backend is assumed to have permission in general. If an object is + provided, permissions for this specific object are checked. + """ + + # Active superusers have all permissions. + if self.is_active and self.is_superuser: + return True + + # Otherwise we need to check the backends. + return _user_has_perm(self, perm, obj) + @classmethod def create_user(cls, username, password, email=None): """Create (and save) a new user with the given username, password and From 3425574ddcd63d126bc96556e182cc0e7cf18bcb Mon Sep 17 00:00:00 2001 From: Luis Araujo Date: Thu, 27 Sep 2012 14:30:59 -0300 Subject: [PATCH 0809/1279] Adding, adjust and transplant more methods to auth.User model --- mongoengine/django/auth.py | 150 ++++++++++++++++++++++++++----------- 1 file changed, 105 insertions(+), 45 deletions(-) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index d9f0584..0b6ffb9 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -5,6 +5,7 @@ from mongoengine import * from django.utils.encoding import smart_str from django.db import models from django.contrib.contenttypes.models import ContentTypeManager +from django.contrib import auth from django.contrib.auth.models import AnonymousUser from django.utils.translation import ugettext_lazy as _ @@ -175,51 +176,6 @@ class UserManager(models.Manager): return ''.join([choice(allowed_chars) for i in range(length)]) -# A few helper functions for common logic between User and AnonymousUser. -def _user_get_all_permissions(user, obj): - permissions = set() - anon = user.is_anonymous() - for backend in auth.get_backends(): - if not anon or backend.supports_anonymous_user: - if hasattr(backend, "get_all_permissions"): - if obj is not None: - if backend.supports_object_permissions: - permissions.update( - backend.get_all_permissions(user, obj) - ) - else: - permissions.update(backend.get_all_permissions(user)) - return permissions - - -def _user_has_perm(user, perm, obj): - anon = user.is_anonymous() - active = user.is_active - for backend in auth.get_backends(): - if (not active and not anon and backend.supports_inactive_user) or \ - (not anon or backend.supports_anonymous_user): - if hasattr(backend, "has_perm"): - if obj is not None: - if (backend.supports_object_permissions and - backend.has_perm(user, perm, obj)): - return True - else: - if backend.has_perm(user, perm): - return True - return False - - -def _user_has_module_perms(user, app_label): - anon = user.is_anonymous() - active = user.is_active - for backend in auth.get_backends(): - if (not active and not anon and backend.supports_inactive_user) or \ - (not anon or backend.supports_anonymous_user): - if hasattr(backend, "has_module_perms"): - if backend.has_module_perms(user, app_label): - return True - return False - class User(Document): """A User document that aims to mirror most of the API specified by Django @@ -313,9 +269,111 @@ class User(Document): user.save() return user + def get_all_permissions(self, obj=None): + permissions = set() + anon = self.is_anonymous() + for backend in auth.get_backends(): + if not anon or backend.supports_anonymous_user: + if hasattr(backend, "get_all_permissions"): + if obj is not None: + if backend.supports_object_permissions: + permissions.update( + backend.get_all_permissions(user, obj) + ) + else: + permissions.update(backend.get_all_permissions(self)) + return permissions + def get_and_delete_messages(self): return [] + def has_perm(self, perm, obj=None): + anon = self.is_anonymous() + active = self.is_active + for backend in auth.get_backends(): + if (not active and not anon and backend.supports_inactive_user) or \ + (not anon or backend.supports_anonymous_user): + if hasattr(backend, "has_perm"): + if obj is not None: + if (backend.supports_object_permissions and + backend.has_perm(self, perm, obj)): + return True + else: + if backend.has_perm(self, perm): + return True + return False + + def has_perms(self, perm_list, obj=None): + """ + Returns True if the user has each of the specified permissions. + If object is passed, it checks if the user has all required perms + for this object. + """ + for perm in perm_list: + if not self.has_perm(perm, obj): + return False + return True + + def has_module_perms(self, app_label): + anon = self.is_anonymous() + active = self.is_active + for backend in auth.get_backends(): + if (not active and not anon and backend.supports_inactive_user) or \ + (not anon or backend.supports_anonymous_user): + if hasattr(backend, "has_module_perms"): + if backend.has_module_perms(self, app_label): + return True + return False + + def get_and_delete_messages(self): + messages = [] + for m in self.message_set.all(): + messages.append(m.message) + m.delete() + return messages + + def email_user(self, subject, message, from_email=None): + "Sends an e-mail to this User." + from django.core.mail import send_mail + send_mail(subject, message, from_email, [self.email]) + + def get_profile(self): + """ + Returns site-specific profile for this user. Raises + SiteProfileNotAvailable if this site does not allow profiles. + """ + if not hasattr(self, '_profile_cache'): + from django.conf import settings + if not getattr(settings, 'AUTH_PROFILE_MODULE', False): + raise SiteProfileNotAvailable('You need to set AUTH_PROFILE_MO' + 'DULE in your project settings') + try: + app_label, model_name = settings.AUTH_PROFILE_MODULE.split('.') + except ValueError: + raise SiteProfileNotAvailable('app_label and model_name should' + ' be separated by a dot in the AUTH_PROFILE_MODULE set' + 'ting') + + try: + model = models.get_model(app_label, model_name) + if model is None: + raise SiteProfileNotAvailable('Unable to load the profile ' + 'model, check AUTH_PROFILE_MODULE in your project sett' + 'ings') + self._profile_cache = model._default_manager.using(self._state.db).get(user__id__exact=self.id) + self._profile_cache.user = self + except (ImportError, ImproperlyConfigured): + raise SiteProfileNotAvailable + return self._profile_cache + + def _get_message_set(self): + import warnings + warnings.warn('The user messaging API is deprecated. Please update' + ' your code to use the new messages framework.', + category=DeprecationWarning) + return self._message_set + message_set = property(_get_message_set) + class MongoEngineBackend(object): """Authenticate using MongoEngine and mongoengine.django.auth.User. @@ -329,6 +387,8 @@ class MongoEngineBackend(object): user = User.objects(username=username).first() if user: if password and user.check_password(password): + backend = auth.get_backends()[0] + user.backend = "%s.%s" % (backend.__module__, backend.__class__.__name__) return user return None From 0bfc96e459d19d188397db73fa557abbc8ecdc6d Mon Sep 17 00:00:00 2001 From: Luis Araujo Date: Thu, 27 Sep 2012 14:32:50 -0300 Subject: [PATCH 0810/1279] exposing mongoengine.django module --- mongoengine/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 9044e61..a4c56b8 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -8,6 +8,7 @@ import queryset from queryset import * import signals from signals import * +import django __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) From e8604d100e663dfd29adf5888b73fe4f7db69d3a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 1 Oct 2012 10:20:28 +0000 Subject: [PATCH 0811/1279] Added Garry Polley to contributors list hmarr/mongoengine#573 --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 2d1a073..6ba2f88 100644 --- a/AUTHORS +++ b/AUTHORS @@ -123,3 +123,4 @@ that much better: * psychogenic * Stefan Wójcik * dimonb + * Garry Polley diff --git a/docs/changelog.rst b/docs/changelog.rst index 1504b08..524f105 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.X ================ +- Allow Django AuthenticationBackends to work with Django user (hmarr/mongoengine#573) - Fixed reload issue with ReferenceField where dbref=False (MongoEngine/mongoengine#138) Changes in 0.7.5 From 7a877a00d5f43a0da967e4fa457566b64a0b9bcb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 1 Oct 2012 13:59:15 +0000 Subject: [PATCH 0812/1279] Updated URLField handle unicode and custom validator (MongoEngine/mongoengine#136) --- docs/changelog.rst | 1 + mongoengine/base.py | 6 +++--- mongoengine/fields.py | 36 +++++++++++++++++++++++------------- tests/test_fields.py | 3 +++ 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 524f105..996b5eb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.X ================ +- Updated URLField - can handle unicode and custom validator (MongoEngine/mongoengine#136) - Allow Django AuthenticationBackends to work with Django user (hmarr/mongoengine#573) - Fixed reload issue with ReferenceField where dbref=False (MongoEngine/mongoengine#138) diff --git a/mongoengine/base.py b/mongoengine/base.py index 92dcfa9..2041caa 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -52,7 +52,7 @@ class ValidationError(AssertionError): self.field_name = kwargs.get('field_name') self.message = message - def __str__(self): + def __unicode__(self): return self.message def __repr__(self): @@ -1338,13 +1338,13 @@ class BaseDocument(object): u = '[Bad Unicode data]' return '<%s: %s>' % (self.__class__.__name__, u) - def __str__(self): + def __unicode__(self): if hasattr(self, '__unicode__'): if PY3: return self.__unicode__() else: return unicode(self).encode('utf-8') - return '%s object' % self.__class__.__name__ + return unicode('%s object' % self.__class__.__name__) def __eq__(self, other): if isinstance(other, self.__class__) and hasattr(other, 'id'): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 7729444..3dc6749 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1,10 +1,12 @@ import datetime import decimal +import itertools import re import time +import urllib2 +import urlparse import uuid import warnings -import itertools from operator import itemgetter import gridfs @@ -101,25 +103,33 @@ class URLField(StringField): .. versionadded:: 0.3 """ - URL_REGEX = re.compile( - r'^https?://' - r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?|' - r'localhost|' - r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' - r'(?::\d+)?' - r'(?:/?|[/?]\S+)$', re.IGNORECASE - ) + _URL_REGEX = re.compile( + r'^(?:http|ftp)s?://' # http:// or https:// + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain... + r'localhost|' #localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip + r'(?::\d+)?' # optional port + r'(?:/?|[/?]\S+)$', re.IGNORECASE) - def __init__(self, verify_exists=False, **kwargs): + def __init__(self, verify_exists=False, url_regex=None, **kwargs): self.verify_exists = verify_exists + self.url_regex = url_regex or self._URL_REGEX super(URLField, self).__init__(**kwargs) def validate(self, value): - if not URLField.URL_REGEX.match(value): - self.error('Invalid URL: %s' % value) + if not self.url_regex.match(value): + scheme, netloc, path, query, fragment = urlparse.urlsplit(value) + try: + netloc = netloc.encode('idna') # IDN -> ACE + except UnicodeError: # invalid domain part + self.error('Invalid URL: %s' % value) if self.verify_exists: - import urllib2 + warnings.warn( + "The URLField verify_exists argument has intractable security " + "and performance issues. Accordingly, it has been deprecated.", + DeprecationWarning + ) try: request = urllib2.Request(value) urllib2.urlopen(request) diff --git a/tests/test_fields.py b/tests/test_fields.py index 04c3765..528f2a1 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -199,6 +199,9 @@ class FieldTest(unittest.TestCase): link.url = 'http://www.google.com:8080' link.validate() + link.url = u'http://президент.рф' + self.assertTrue(link.validate()) + def test_int_validation(self): """Ensure that invalid values cannot be assigned to int fields. """ From 6f0a6df4f6ca2adb6ee445ca18505d98a1496278 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 1 Oct 2012 14:23:05 +0000 Subject: [PATCH 0813/1279] Fix loop --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 2041caa..2044d63 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1338,7 +1338,7 @@ class BaseDocument(object): u = '[Bad Unicode data]' return '<%s: %s>' % (self.__class__.__name__, u) - def __unicode__(self): + def __str__(self): if hasattr(self, '__unicode__'): if PY3: return self.__unicode__() From e6d796832e71d6e04e474fd04e58cfca994a2a00 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 1 Oct 2012 14:48:53 +0000 Subject: [PATCH 0814/1279] Unicode fix reverted but can have custom validator MongoEngine/mongoengine#136 --- docs/changelog.rst | 2 +- mongoengine/base.py | 2 +- mongoengine/fields.py | 7 ++----- tests/test_fields.py | 4 +--- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 996b5eb..b749ca2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,7 @@ Changelog Changes in 0.7.X ================ -- Updated URLField - can handle unicode and custom validator (MongoEngine/mongoengine#136) +- Updated URLField - can have a custom validator (MongoEngine/mongoengine#136) - Allow Django AuthenticationBackends to work with Django user (hmarr/mongoengine#573) - Fixed reload issue with ReferenceField where dbref=False (MongoEngine/mongoengine#138) diff --git a/mongoengine/base.py b/mongoengine/base.py index 2044d63..342b31e 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1344,7 +1344,7 @@ class BaseDocument(object): return self.__unicode__() else: return unicode(self).encode('utf-8') - return unicode('%s object' % self.__class__.__name__) + return '%s object' % self.__class__.__name__ def __eq__(self, other): if isinstance(other, self.__class__) and hasattr(other, 'id'): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 3dc6749..01d3fc6 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -118,11 +118,8 @@ class URLField(StringField): def validate(self, value): if not self.url_regex.match(value): - scheme, netloc, path, query, fragment = urlparse.urlsplit(value) - try: - netloc = netloc.encode('idna') # IDN -> ACE - except UnicodeError: # invalid domain part - self.error('Invalid URL: %s' % value) + self.error('Invalid URL: %s' % value) + return if self.verify_exists: warnings.warn( diff --git a/tests/test_fields.py b/tests/test_fields.py index 528f2a1..9806550 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import with_statement import datetime import os @@ -199,9 +200,6 @@ class FieldTest(unittest.TestCase): link.url = 'http://www.google.com:8080' link.validate() - link.url = u'http://президент.рф' - self.assertTrue(link.validate()) - def test_int_validation(self): """Ensure that invalid values cannot be assigned to int fields. """ From e2d826c412924c0baa8d446e7bd29360b3bb17a7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 1 Oct 2012 15:26:34 +0000 Subject: [PATCH 0815/1279] Allow updates with match operators (MongoEngine/mongoengine#144) --- docs/changelog.rst | 3 ++- mongoengine/queryset.py | 16 ++++++++++++++-- tests/test_queryset.py | 24 ++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b749ca2..f3e83c1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,8 @@ Changelog Changes in 0.7.X ================ -- Updated URLField - can have a custom validator (MongoEngine/mongoengine#136) +- Allow updates with match operators (MongoEngine/mongoengine#144) +- Updated URLField - now can have a override the regex (MongoEngine/mongoengine#136) - Allow Django AuthenticationBackends to work with Django user (hmarr/mongoengine#573) - Fixed reload issue with ReferenceField where dbref=False (MongoEngine/mongoengine#138) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 5a1aa71..c774322 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1395,6 +1395,8 @@ class QuerySet(object): """ operators = ['set', 'unset', 'inc', 'dec', 'pop', 'push', 'push_all', 'pull', 'pull_all', 'add_to_set'] + match_operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', + 'all', 'size', 'exists', 'not'] mongo_update = {} for key, value in update.items(): @@ -1418,6 +1420,10 @@ class QuerySet(object): elif op == 'add_to_set': op = op.replace('_to_set', 'ToSet') + match = None + if parts[-1] in match_operators: + match = parts.pop() + if _doc_cls: # Switch field names to proper names [set in Field(name='foo')] fields = QuerySet._lookup_field(_doc_cls, parts) @@ -1451,16 +1457,22 @@ class QuerySet(object): elif field.required or value is not None: value = field.prepare_query_value(op, value) + if match: + match = '$' + match + value = {match: value} + key = '.'.join(parts) if not op: - raise InvalidQueryError("Updates must supply an operation eg: set__FIELD=value") + raise InvalidQueryError("Updates must supply an operation " + "eg: set__FIELD=value") if 'pull' in op and '.' in key: # Dot operators don't work on pull operations # it uses nested dict syntax if op == 'pullAll': - raise InvalidQueryError("pullAll operations only support a single field depth") + raise InvalidQueryError("pullAll operations only support " + "a single field depth") parts.reverse() for key in parts: diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 55531a1..770290c 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -414,6 +414,30 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(post.comments[0].by, 'joe') self.assertEqual(post.comments[0].votes.score, 4) + def test_updates_can_have_match_operators(self): + + class Post(Document): + title = StringField(required=True) + tags = ListField(StringField()) + comments = ListField(EmbeddedDocumentField("Comment")) + + class Comment(EmbeddedDocument): + content = StringField() + name = StringField(max_length=120) + vote = IntField() + + Post.drop_collection() + + comm1 = Comment(content="very funny indeed", name="John S", vote=1) + comm2 = Comment(content="kind of funny", name="Mark P", vote=0) + + Post(title='Fun with MongoEngine', tags=['mongodb', 'mongoengine'], + comments=[comm1, comm2]).save() + + Post.objects().update_one(pull__comments__vote__lt=1) + + self.assertEqual(1, len(Post.objects.first().comments)) + def test_mapfield_update(self): """Ensure that the MapField can be updated.""" class Member(EmbeddedDocument): From dc1849bad50dc59f9f97a93ce90f8051a1f58c84 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 1 Oct 2012 16:15:25 +0000 Subject: [PATCH 0816/1279] Unicode fix for repr (MongoEngine/mongoengine#133) --- docs/changelog.rst | 1 + mongoengine/base.py | 7 ++++--- tests/test_document.py | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f3e83c1..b2a855d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.X ================ +- Unicode fix for repr (MongoEngine/mongoengine#133) - Allow updates with match operators (MongoEngine/mongoengine#144) - Updated URLField - now can have a override the regex (MongoEngine/mongoengine#136) - Allow Django AuthenticationBackends to work with Django user (hmarr/mongoengine#573) diff --git a/mongoengine/base.py b/mongoengine/base.py index 342b31e..714b74b 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1333,10 +1333,11 @@ class BaseDocument(object): def __repr__(self): try: - u = txt_type(self) + u = self.__str__() except (UnicodeEncodeError, UnicodeDecodeError): u = '[Bad Unicode data]' - return '<%s: %s>' % (self.__class__.__name__, u) + repr_type = type(u) + return repr_type('<%s: %s>' % (self.__class__.__name__, u)) def __str__(self): if hasattr(self, '__unicode__'): @@ -1344,7 +1345,7 @@ class BaseDocument(object): return self.__unicode__() else: return unicode(self).encode('utf-8') - return '%s object' % self.__class__.__name__ + return txt_type('%s object' % self.__class__.__name__) def __eq__(self, other): if isinstance(other, self.__class__) and hasattr(other, 'id'): diff --git a/tests/test_document.py b/tests/test_document.py index d2969fe..60a966b 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -85,6 +85,22 @@ class DocumentTest(unittest.TestCase): # Ensure Document isn't treated like an actual document self.assertFalse(hasattr(Document, '_fields')) + def test_repr(self): + """Ensure that unicode representation works + """ + class Article(Document): + title = StringField() + + def __unicode__(self): + return self.title + + Article.drop_collection() + + Article(title=u'привет мир').save() + + self.assertEqual('', repr(Article.objects.first())) + self.assertEqual('[]', repr(Article.objects.all())) + def test_collection_naming(self): """Ensure that a collection with a specified name may be used. """ From 79098e997e361841d3d53aad24aefb2ee0f491f1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 1 Oct 2012 16:20:48 +0000 Subject: [PATCH 0817/1279] Updated test --- tests/test_document.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_document.py b/tests/test_document.py index 60a966b..a09aaec 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import with_statement import bson import os From 12f3f8c694950438d021be9e784bad1acb0e3736 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 1 Oct 2012 16:27:59 +0000 Subject: [PATCH 0818/1279] Added chaining regression test (MongoEngine/mongoengine#135) --- tests/test_queryset.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 770290c..690df5e 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -230,6 +230,30 @@ class QuerySetTest(unittest.TestCase): Blog.drop_collection() + def test_chaining(self): + class A(Document): + pass + + class B(Document): + a = ReferenceField(A) + + A.drop_collection() + B.drop_collection() + + a1 = A().save() + a2 = A().save() + + B(a=a1).save() + + # Works + q1 = B.objects.filter(a__in=[a1, a2], a=a1)._query + + # Doesn't work + q2 = B.objects.filter(a__in=[a1, a2]) + q2 = q2.filter(a=a1)._query + + self.assertEqual(q1, q2) + def test_update_write_options(self): """Test that passing write_options works""" From 31ec7907b5b45b10e652a15ebbaee520cbdb9a6d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 1 Oct 2012 20:01:43 +0000 Subject: [PATCH 0819/1279] Fixing py3 compat --- mongoengine/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 714b74b..fa12e35 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -52,8 +52,8 @@ class ValidationError(AssertionError): self.field_name = kwargs.get('field_name') self.message = message - def __unicode__(self): - return self.message + def __str__(self): + return txt_type(self.message) def __repr__(self): return '%s(%s,)' % (self.__class__.__name__, self.message) From e4af0e361ab5b91fad7ca9dbc8f5ed631f0b2c77 Mon Sep 17 00:00:00 2001 From: Aleksey Porfirov Date: Mon, 15 Oct 2012 02:11:01 +0400 Subject: [PATCH 0820/1279] Add session expiration test (with django timezone support activated) --- tests/test_django.py | 45 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/test_django.py b/tests/test_django.py index 398fd3e..3b0b04f 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -12,7 +12,7 @@ try: from django.conf import settings from django.core.paginator import Paginator - settings.configure() + settings.configure(USE_TZ=True) from django.contrib.sessions.tests import SessionTestsMixin from mongoengine.django.sessions import SessionStore, MongoSession @@ -24,6 +24,37 @@ except Exception, err: raise err +from datetime import tzinfo, timedelta +ZERO = timedelta(0) + +class FixedOffset(tzinfo): + """Fixed offset in minutes east from UTC.""" + + def __init__(self, offset, name): + self.__offset = timedelta(minutes = offset) + self.__name = name + + def utcoffset(self, dt): + return self.__offset + + def tzname(self, dt): + return self.__name + + def dst(self, dt): + return ZERO + + +def activate_timezone(tz): + """Activate Django timezone support if it is available. + """ + try: + from django.utils import timezone + timezone.deactivate() + timezone.activate(tz) + except ImportError: + pass + + class QuerySetTest(unittest.TestCase): def setUp(self): @@ -120,3 +151,15 @@ class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): session['test'] = True session.save() self.assertTrue('test' in session) + + def test_session_expiration_tz(self): + activate_timezone(FixedOffset(60, 'UTC+1')) + # create and save new session + session = SessionStore() + session.set_expiry(600) # expire in 600 seconds + session['test_expire'] = True + session.save() + # reload session with key + key = session.session_key + session = SessionStore(key) + self.assertTrue('test_expire' in session, 'Session has expired before it is expected') From 0a89899ad088e712b6bca54e32f30aae64ab2bf7 Mon Sep 17 00:00:00 2001 From: Aleksey Porfirov Date: Mon, 15 Oct 2012 02:13:52 +0400 Subject: [PATCH 0821/1279] Fix django timezone support --- mongoengine/django/auth.py | 9 ++++----- mongoengine/django/sessions.py | 6 +++--- mongoengine/django/utils.py | 6 ++++++ 3 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 mongoengine/django/utils.py diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 65afacf..3776d54 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -1,5 +1,3 @@ -import datetime - from mongoengine import * from django.utils.encoding import smart_str @@ -33,6 +31,7 @@ except ImportError: hash = get_hexdigest(algo, salt, raw_password) return '%s$%s$%s' % (algo, salt, hash) +from .utils import datetime_now REDIRECT_FIELD_NAME = 'next' @@ -62,9 +61,9 @@ class User(Document): is_superuser = BooleanField(default=False, verbose_name=_('superuser status'), help_text=_("Designates that this user has all permissions without explicitly assigning them.")) - last_login = DateTimeField(default=datetime.datetime.now, + last_login = DateTimeField(default=datetime_now, verbose_name=_('last login')) - date_joined = DateTimeField(default=datetime.datetime.now, + date_joined = DateTimeField(default=datetime_now, verbose_name=_('date joined')) meta = { @@ -130,7 +129,7 @@ class User(Document): """Create (and save) a new user with the given username, password and email address. """ - now = datetime.datetime.now() + now = datetime_now() # Normalize the address by lowercasing the domain part of the email # address. diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index f178342..6e964a7 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -1,5 +1,3 @@ -from datetime import datetime - from django.conf import settings from django.contrib.sessions.backends.base import SessionBase, CreateError from django.core.exceptions import SuspiciousOperation @@ -10,6 +8,8 @@ from mongoengine import fields from mongoengine.queryset import OperationError from mongoengine.connection import DEFAULT_CONNECTION_NAME +from .utils import datetime_now + MONGOENGINE_SESSION_DB_ALIAS = getattr( settings, 'MONGOENGINE_SESSION_DB_ALIAS', @@ -33,7 +33,7 @@ class SessionStore(SessionBase): def load(self): try: s = MongoSession.objects(session_key=self.session_key, - expire_date__gt=datetime.now())[0] + expire_date__gt=datetime_now())[0] return self.decode(force_unicode(s.session_data)) except (IndexError, SuspiciousOperation): self.create() diff --git a/mongoengine/django/utils.py b/mongoengine/django/utils.py new file mode 100644 index 0000000..d3ef8a4 --- /dev/null +++ b/mongoengine/django/utils.py @@ -0,0 +1,6 @@ +try: + # django >= 1.4 + from django.utils.timezone import now as datetime_now +except ImportError: + from datetime import datetime + datetime_now = datetime.now From 6f29d1238642d395b197645f6770daeab42495d3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 9 Oct 2012 15:02:13 +0000 Subject: [PATCH 0822/1279] Changed the inheritance model to remove types The inheritance model has changed, we no longer need to store an array of `types` with the model we can just use the classname in `_cls`. See the upgrade docs for information on how to upgrade MongoEngine/mongoengine#148 --- docs/changelog.rst | 5 + docs/guide/defining-documents.rst | 23 +- docs/upgrade.rst | 39 + mongoengine/__init__.py | 6 +- mongoengine/base.py | 1523 -------------- mongoengine/base/__init__.py | 5 + mongoengine/base/common.py | 25 + mongoengine/base/datastructures.py | 124 ++ mongoengine/base/document.py | 644 ++++++ mongoengine/base/fields.py | 371 ++++ mongoengine/base/metaclasses.py | 388 ++++ mongoengine/common.py | 35 + mongoengine/dereference.py | 2 +- mongoengine/django/shortcuts.py | 2 +- mongoengine/document.py | 28 +- mongoengine/errors.py | 124 ++ mongoengine/fields.py | 6 +- mongoengine/queryset/__init__.py | 11 + mongoengine/queryset/field_list.py | 51 + mongoengine/queryset/manager.py | 61 + mongoengine/{ => queryset}/queryset.py | 818 +------- mongoengine/queryset/transform.py | 237 +++ mongoengine/queryset/visitor.py | 237 +++ setup.cfg | 2 +- tests/__init__.py | 2 + .../__init__.py} | 12 +- tests/document/__init__.py | 11 + tests/document/class_methods.py | 183 ++ tests/document/delta.py | 688 +++++++ tests/document/dynamic.py | 270 +++ tests/document/indexes.py | 637 ++++++ tests/document/inheritance.py | 395 ++++ .../instance.py} | 1752 +---------------- tests/{ => document}/mongoengine.png | Bin tests/migration/__init__.py | 4 + .../test_convert_to_new_inheritance_model.py | 51 + tests/migration/turn_off_inheritance.py | 62 + tests/test_dynamic_document.py | 533 ----- tests/test_fields.py | 5 +- tests/test_queryset.py | 106 +- 40 files changed, 4856 insertions(+), 4622 deletions(-) delete mode 100644 mongoengine/base.py create mode 100644 mongoengine/base/__init__.py create mode 100644 mongoengine/base/common.py create mode 100644 mongoengine/base/datastructures.py create mode 100644 mongoengine/base/document.py create mode 100644 mongoengine/base/fields.py create mode 100644 mongoengine/base/metaclasses.py create mode 100644 mongoengine/common.py create mode 100644 mongoengine/errors.py create mode 100644 mongoengine/queryset/__init__.py create mode 100644 mongoengine/queryset/field_list.py create mode 100644 mongoengine/queryset/manager.py rename mongoengine/{ => queryset}/queryset.py (59%) create mode 100644 mongoengine/queryset/transform.py create mode 100644 mongoengine/queryset/visitor.py rename tests/{test_all_warnings.py => all_warnings/__init__.py} (91%) create mode 100644 tests/document/__init__.py create mode 100644 tests/document/class_methods.py create mode 100644 tests/document/delta.py create mode 100644 tests/document/dynamic.py create mode 100644 tests/document/indexes.py create mode 100644 tests/document/inheritance.py rename tests/{test_document.py => document/instance.py} (50%) rename tests/{ => document}/mongoengine.png (100%) create mode 100644 tests/migration/__init__.py create mode 100644 tests/migration/test_convert_to_new_inheritance_model.py create mode 100644 tests/migration/turn_off_inheritance.py delete mode 100644 tests/test_dynamic_document.py diff --git a/docs/changelog.rst b/docs/changelog.rst index b2a855d..8388b05 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,11 @@ Changelog ========= +Changes in 0.8 +============== +- Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) + + Changes in 0.7.X ================ - Unicode fix for repr (MongoEngine/mongoengine#133) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 3ee7796..cf3b5a6 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -461,9 +461,10 @@ If a dictionary is passed then the following options are available: :attr:`fields` (Default: None) The fields to index. Specified in the same format as described above. -:attr:`types` (Default: True) - Whether the index should have the :attr:`_types` field added automatically - to the start of the index. +:attr:`cls` (Default: True) + If you have polymorphic models that inherit and have `allow_inheritance` + turned on, you can configure whether the index should have the + :attr:`_cls` field added automatically to the start of the index. :attr:`sparse` (Default: False) Whether the index should be sparse. @@ -590,14 +591,14 @@ convenient and efficient retrieval of related documents:: Working with existing data -------------------------- To enable correct retrieval of documents involved in this kind of heirarchy, -two extra attributes are stored on each document in the database: :attr:`_cls` -and :attr:`_types`. These are hidden from the user through the MongoEngine -interface, but may not be present if you are trying to use MongoEngine with -an existing database. For this reason, you may disable this inheritance -mechansim, removing the dependency of :attr:`_cls` and :attr:`_types`, enabling -you to work with existing databases. To disable inheritance on a document -class, set :attr:`allow_inheritance` to ``False`` in the :attr:`meta` -dictionary:: +an extra attribute is stored on each document in the database: :attr:`_cls`. +These are hidden from the user through the MongoEngine interface, but may not +be present if you are trying to use MongoEngine with an existing database. + +For this reason, you may disable this inheritance mechansim, removing the +dependency of :attr:`_cls`, enabling you to work with existing databases. +To disable inheritance on a document class, set :attr:`allow_inheritance` to +``False`` in the :attr:`meta` dictionary:: # Will work with data in an existing collection named 'cmsPage' class Page(Document): diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 82ac7ca..99e3078 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -2,6 +2,45 @@ Upgrading ========= +0.7 to 0.8 +========== + +Inheritance +----------- + +The inheritance model has changed, we no longer need to store an array of +`types` with the model we can just use the classname in `_cls`. This means +that you will have to update your indexes for each of your inherited classes +like so: + + # 1. Declaration of the class + class Animal(Document): + name = StringField() + meta = { + 'allow_inheritance': True, + 'indexes': ['name'] + } + + # 2. Remove _types + collection = Animal._get_collection() + collection.update({}, {"$unset": {"_types": 1}}, multi=True) + + # 3. Confirm extra data is removed + count = collection.find({'_types': {"$exists": True}}).count() + assert count == 0 + + # 4. Remove indexes + info = collection.index_information() + indexes_to_drop = [key for key, value in info.iteritems() + if '_types' in dict(value['key'])] + for index in indexes_to_drop: + collection.drop_index(index) + + # 5. Recreate indexes + Animal.objects._ensure_indexes() + + + 0.6 to 0.7 ========== diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 9044e61..d92165c 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -9,10 +9,10 @@ from queryset import * import signals from signals import * -__all__ = (document.__all__ + fields.__all__ + connection.__all__ + - queryset.__all__ + signals.__all__) +__all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + + list(queryset.__all__) + signals.__all__) -VERSION = (0, 7, 5) +VERSION = (0, 8, 0, '+') def get_version(): diff --git a/mongoengine/base.py b/mongoengine/base.py deleted file mode 100644 index fa12e35..0000000 --- a/mongoengine/base.py +++ /dev/null @@ -1,1523 +0,0 @@ -import operator -import sys -import warnings -import weakref - -from collections import defaultdict -from functools import partial - -from queryset import QuerySet, QuerySetManager -from queryset import DoesNotExist, MultipleObjectsReturned -from queryset import DO_NOTHING - -from mongoengine import signals -from mongoengine.python_support import (PY3, UNICODE_KWARGS, txt_type, - to_str_keys_recursive) - -import pymongo -from bson import ObjectId -from bson.dbref import DBRef - -ALLOW_INHERITANCE = True - -_document_registry = {} -_class_registry = {} - - -class NotRegistered(Exception): - pass - - -class InvalidDocumentError(Exception): - pass - - -class ValidationError(AssertionError): - """Validation exception. - - May represent an error validating a field or a - document containing fields with validation errors. - - :ivar errors: A dictionary of errors for fields within this - document or list, or None if the error is for an - individual field. - """ - - errors = {} - field_name = None - _message = None - - def __init__(self, message="", **kwargs): - self.errors = kwargs.get('errors', {}) - self.field_name = kwargs.get('field_name') - self.message = message - - def __str__(self): - return txt_type(self.message) - - def __repr__(self): - return '%s(%s,)' % (self.__class__.__name__, self.message) - - def __getattribute__(self, name): - message = super(ValidationError, self).__getattribute__(name) - if name == 'message': - if self.field_name: - message = '%s' % message - if self.errors: - message = '%s(%s)' % (message, self._format_errors()) - return message - - def _get_message(self): - return self._message - - def _set_message(self, message): - self._message = message - - message = property(_get_message, _set_message) - - def to_dict(self): - """Returns a dictionary of all errors within a document - - Keys are field names or list indices and values are the - validation error messages, or a nested dictionary of - errors for an embedded document or list. - """ - - def build_dict(source): - errors_dict = {} - if not source: - return errors_dict - if isinstance(source, dict): - for field_name, error in source.iteritems(): - errors_dict[field_name] = build_dict(error) - elif isinstance(source, ValidationError) and source.errors: - return build_dict(source.errors) - else: - return unicode(source) - return errors_dict - if not self.errors: - return {} - return build_dict(self.errors) - - def _format_errors(self): - """Returns a string listing all errors within a document""" - - def generate_key(value, prefix=''): - if isinstance(value, list): - value = ' '.join([generate_key(k) for k in value]) - if isinstance(value, dict): - value = ' '.join( - [generate_key(v, k) for k, v in value.iteritems()]) - - results = "%s.%s" % (prefix, value) if prefix else value - return results - - error_dict = defaultdict(list) - for k, v in self.to_dict().iteritems(): - error_dict[generate_key(v)].append(k) - return ' '.join(["%s: %s" % (k, v) for k, v in error_dict.iteritems()]) - - -def get_document(name): - doc = _document_registry.get(name, None) - if not doc: - # Possible old style names - end = ".%s" % name - possible_match = [k for k in _document_registry.keys() - if k.endswith(end)] - if len(possible_match) == 1: - doc = _document_registry.get(possible_match.pop(), None) - if not doc: - raise NotRegistered(""" - `%s` has not been registered in the document registry. - Importing the document class automatically registers it, has it - been imported? - """.strip() % name) - return doc - - -class BaseField(object): - """A base class for fields in a MongoDB document. Instances of this class - may be added to subclasses of `Document` to define a document's schema. - - .. versionchanged:: 0.5 - added verbose and help text - """ - - name = None - - # Fields may have _types inserted into indexes by default - _index_with_types = True - _geo_index = False - - # These track each time a Field instance is created. Used to retain order. - # The auto_creation_counter is used for fields that MongoEngine implicitly - # creates, creation_counter is used for all user-specified fields. - creation_counter = 0 - auto_creation_counter = -1 - - def __init__(self, db_field=None, name=None, required=False, default=None, - unique=False, unique_with=None, primary_key=False, - validation=None, choices=None, verbose_name=None, - help_text=None): - self.db_field = (db_field or name) if not primary_key else '_id' - if name: - msg = "Fields' 'name' attribute deprecated in favour of 'db_field'" - warnings.warn(msg, DeprecationWarning) - self.name = None - self.required = required or primary_key - self.default = default - self.unique = bool(unique or unique_with) - self.unique_with = unique_with - self.primary_key = primary_key - self.validation = validation - self.choices = choices - self.verbose_name = verbose_name - self.help_text = help_text - - # Adjust the appropriate creation counter, and save our local copy. - if self.db_field == '_id': - self.creation_counter = BaseField.auto_creation_counter - BaseField.auto_creation_counter -= 1 - else: - self.creation_counter = BaseField.creation_counter - BaseField.creation_counter += 1 - - def __get__(self, instance, owner): - """Descriptor for retrieving a value from a field in a document. Do - any necessary conversion between Python and MongoDB types. - """ - if instance is None: - # Document class being used rather than a document object - return self - - # Get value from document instance if available, if not use default - value = instance._data.get(self.name) - - if value is None: - value = self.default - # Allow callable default values - if callable(value): - value = value() - - return value - - def __set__(self, instance, value): - """Descriptor for assigning a value to a field in a document. - """ - instance._data[self.name] = value - if instance._initialised: - instance._mark_as_changed(self.name) - - def error(self, message="", errors=None, field_name=None): - """Raises a ValidationError. - """ - field_name = field_name if field_name else self.name - raise ValidationError(message, errors=errors, field_name=field_name) - - def to_python(self, value): - """Convert a MongoDB-compatible type to a Python type. - """ - return value - - def to_mongo(self, value): - """Convert a Python type to a MongoDB-compatible type. - """ - return self.to_python(value) - - def prepare_query_value(self, op, value): - """Prepare a value that is being used in a query for PyMongo. - """ - return value - - def validate(self, value): - """Perform validation on a value. - """ - pass - - def _validate(self, value): - Document = _import_class('Document') - EmbeddedDocument = _import_class('EmbeddedDocument') - # check choices - if self.choices: - is_cls = isinstance(value, (Document, EmbeddedDocument)) - value_to_check = value.__class__ if is_cls else value - err_msg = 'an instance' if is_cls else 'one' - if isinstance(self.choices[0], (list, tuple)): - option_keys = [k for k, v in self.choices] - if value_to_check not in option_keys: - msg = ('Value must be %s of %s' % - (err_msg, unicode(option_keys))) - self.error(msg) - elif value_to_check not in self.choices: - msg = ('Value must be %s of %s' % - (err_msg, unicode(self.choices))) - self.error() - - # check validation argument - if self.validation is not None: - if callable(self.validation): - if not self.validation(value): - self.error('Value does not match custom validation method') - else: - raise ValueError('validation argument for "%s" must be a ' - 'callable.' % self.name) - - self.validate(value) - - -class ComplexBaseField(BaseField): - """Handles complex fields, such as lists / dictionaries. - - Allows for nesting of embedded documents inside complex types. - Handles the lazy dereferencing of a queryset by lazily dereferencing all - items in a list / dict rather than one at a time. - - .. versionadded:: 0.5 - """ - - field = None - __dereference = False - - def __get__(self, instance, owner): - """Descriptor to automatically dereference references. - """ - if instance is None: - # Document class being used rather than a document object - return self - - ReferenceField = _import_class('ReferenceField') - GenericReferenceField = _import_class('GenericReferenceField') - dereference = self.field is None or isinstance(self.field, - (GenericReferenceField, ReferenceField)) - if not self._dereference and instance._initialised and dereference: - instance._data[self.name] = self._dereference( - instance._data.get(self.name), max_depth=1, instance=instance, - name=self.name - ) - - value = super(ComplexBaseField, self).__get__(instance, owner) - - # Convert lists / values so we can watch for any changes on them - if (isinstance(value, (list, tuple)) and - not isinstance(value, BaseList)): - value = BaseList(value, instance, self.name) - instance._data[self.name] = value - elif isinstance(value, dict) and not isinstance(value, BaseDict): - value = BaseDict(value, instance, self.name) - instance._data[self.name] = value - - if (instance._initialised and isinstance(value, (BaseList, BaseDict)) - and not value._dereferenced): - value = self._dereference( - value, max_depth=1, instance=instance, name=self.name - ) - value._dereferenced = True - instance._data[self.name] = value - - return value - - def __set__(self, instance, value): - """Descriptor for assigning a value to a field in a document. - """ - instance._data[self.name] = value - instance._mark_as_changed(self.name) - - def to_python(self, value): - """Convert a MongoDB-compatible type to a Python type. - """ - Document = _import_class('Document') - - if isinstance(value, basestring): - return value - - if hasattr(value, 'to_python'): - return value.to_python() - - is_list = False - if not hasattr(value, 'items'): - try: - is_list = True - value = dict([(k, v) for k, v in enumerate(value)]) - except TypeError: # Not iterable return the value - return value - - if self.field: - value_dict = dict([(key, self.field.to_python(item)) - for key, item in value.items()]) - else: - value_dict = {} - for k, v in value.items(): - if isinstance(v, Document): - # We need the id from the saved object to create the DBRef - if v.pk is None: - self.error('You can only reference documents once they' - ' have been saved to the database') - collection = v._get_collection_name() - value_dict[k] = DBRef(collection, v.pk) - elif hasattr(v, 'to_python'): - value_dict[k] = v.to_python() - else: - value_dict[k] = self.to_python(v) - - if is_list: # Convert back to a list - return [v for k, v in sorted(value_dict.items(), - key=operator.itemgetter(0))] - return value_dict - - def to_mongo(self, value): - """Convert a Python type to a MongoDB-compatible type. - """ - Document = _import_class("Document") - - if isinstance(value, basestring): - return value - - if hasattr(value, 'to_mongo'): - return value.to_mongo() - - is_list = False - if not hasattr(value, 'items'): - try: - is_list = True - value = dict([(k, v) for k, v in enumerate(value)]) - except TypeError: # Not iterable return the value - return value - - if self.field: - value_dict = dict([(key, self.field.to_mongo(item)) - for key, item in value.items()]) - else: - value_dict = {} - for k, v in value.items(): - if isinstance(v, Document): - # We need the id from the saved object to create the DBRef - if v.pk is None: - self.error('You can only reference documents once they' - ' have been saved to the database') - - # If its a document that is not inheritable it won't have - # _types / _cls data so make it a generic reference allows - # us to dereference - meta = getattr(v, '_meta', {}) - allow_inheritance = ( - meta.get('allow_inheritance', ALLOW_INHERITANCE) - == False) - if allow_inheritance and not self.field: - GenericReferenceField = _import_class("GenericReferenceField") - value_dict[k] = GenericReferenceField().to_mongo(v) - else: - collection = v._get_collection_name() - value_dict[k] = DBRef(collection, v.pk) - elif hasattr(v, 'to_mongo'): - value_dict[k] = v.to_mongo() - else: - value_dict[k] = self.to_mongo(v) - - if is_list: # Convert back to a list - return [v for k, v in sorted(value_dict.items(), - key=operator.itemgetter(0))] - return value_dict - - def validate(self, value): - """If field is provided ensure the value is valid. - """ - errors = {} - if self.field: - if hasattr(value, 'iteritems') or hasattr(value, 'items'): - sequence = value.iteritems() - else: - sequence = enumerate(value) - for k, v in sequence: - try: - self.field._validate(v) - except ValidationError, error: - errors[k] = error.errors or error - except (ValueError, AssertionError), error: - errors[k] = error - - if errors: - field_class = self.field.__class__.__name__ - self.error('Invalid %s item (%s)' % (field_class, value), - errors=errors) - # Don't allow empty values if required - if self.required and not value: - self.error('Field is required and cannot be empty') - - def prepare_query_value(self, op, value): - return self.to_mongo(value) - - def lookup_member(self, member_name): - if self.field: - return self.field.lookup_member(member_name) - return None - - def _set_owner_document(self, owner_document): - if self.field: - self.field.owner_document = owner_document - self._owner_document = owner_document - - def _get_owner_document(self, owner_document): - self._owner_document = owner_document - - owner_document = property(_get_owner_document, _set_owner_document) - - @property - def _dereference(self,): - if not self.__dereference: - DeReference = _import_class("DeReference") - self.__dereference = DeReference() # Cached - return self.__dereference - - -class ObjectIdField(BaseField): - """An field wrapper around MongoDB's ObjectIds. - """ - - def to_python(self, value): - if not isinstance(value, ObjectId): - value = ObjectId(value) - return value - - def to_mongo(self, value): - if not isinstance(value, ObjectId): - try: - return ObjectId(unicode(value)) - except Exception, e: - # e.message attribute has been deprecated since Python 2.6 - self.error(unicode(e)) - return value - - def prepare_query_value(self, op, value): - return self.to_mongo(value) - - def validate(self, value): - try: - ObjectId(unicode(value)) - except: - self.error('Invalid Object ID') - - -class DocumentMetaclass(type): - """Metaclass for all documents. - """ - - def __new__(cls, name, bases, attrs): - flattened_bases = cls._get_bases(bases) - super_new = super(DocumentMetaclass, cls).__new__ - - # If a base class just call super - metaclass = attrs.get('my_metaclass') - if metaclass and issubclass(metaclass, DocumentMetaclass): - return super_new(cls, name, bases, attrs) - - attrs['_is_document'] = attrs.get('_is_document', False) - - # EmbeddedDocuments could have meta data for inheritance - if 'meta' in attrs: - attrs['_meta'] = attrs.pop('meta') - - # Handle document Fields - - # Merge all fields from subclasses - doc_fields = {} - for base in flattened_bases[::-1]: - if hasattr(base, '_fields'): - doc_fields.update(base._fields) - - # Standard object mixin - merge in any Fields - if not hasattr(base, '_meta'): - base_fields = {} - for attr_name, attr_value in base.__dict__.iteritems(): - if not isinstance(attr_value, BaseField): - continue - attr_value.name = attr_name - if not attr_value.db_field: - attr_value.db_field = attr_name - base_fields[attr_name] = attr_value - doc_fields.update(base_fields) - - # Discover any document fields - field_names = {} - for attr_name, attr_value in attrs.iteritems(): - if not isinstance(attr_value, BaseField): - continue - attr_value.name = attr_name - if not attr_value.db_field: - attr_value.db_field = attr_name - doc_fields[attr_name] = attr_value - - # Count names to ensure no db_field redefinitions - field_names[attr_value.db_field] = field_names.get( - attr_value.db_field, 0) + 1 - - # Ensure no duplicate db_fields - duplicate_db_fields = [k for k, v in field_names.items() if v > 1] - if duplicate_db_fields: - msg = ("Multiple db_fields defined for: %s " % - ", ".join(duplicate_db_fields)) - raise InvalidDocumentError(msg) - - # Set _fields and db_field maps - attrs['_fields'] = doc_fields - attrs['_db_field_map'] = dict([(k, getattr(v, 'db_field', k)) - for k, v in doc_fields.iteritems()]) - attrs['_reverse_db_field_map'] = dict( - (v, k) for k, v in attrs['_db_field_map'].iteritems()) - - # - # Set document hierarchy - # - superclasses = {} - class_name = [name] - for base in flattened_bases: - if (not getattr(base, '_is_base_cls', True) and - not getattr(base, '_meta', {}).get('abstract', True)): - # Collate heirarchy for _cls and _types - class_name.append(base.__name__) - - # Get superclasses from superclass - superclasses[base._class_name] = base - superclasses.update(base._superclasses) - - if hasattr(base, '_meta'): - # Warn if allow_inheritance isn't set and prevent - # inheritance of classes where inheritance is set to False - allow_inheritance = base._meta.get('allow_inheritance', - ALLOW_INHERITANCE) - if (not getattr(base, '_is_base_cls', True) - and allow_inheritance is None): - warnings.warn( - "%s uses inheritance, the default for " - "allow_inheritance is changing to off by default. " - "Please add it to the document meta." % name, - FutureWarning - ) - elif (allow_inheritance == False and - not base._meta.get('abstract')): - raise ValueError('Document %s may not be subclassed' % - base.__name__) - - attrs['_class_name'] = '.'.join(reversed(class_name)) - attrs['_superclasses'] = superclasses - - # Create the new_class - new_class = super_new(cls, name, bases, attrs) - - # Handle delete rules - Document, EmbeddedDocument, DictField = cls._import_classes() - for field in new_class._fields.itervalues(): - f = field - f.owner_document = new_class - delete_rule = getattr(f, 'reverse_delete_rule', DO_NOTHING) - if isinstance(f, ComplexBaseField) and hasattr(f, 'field'): - delete_rule = getattr(f.field, - 'reverse_delete_rule', - DO_NOTHING) - if isinstance(f, DictField) and delete_rule != DO_NOTHING: - msg = ("Reverse delete rules are not supported " - "for %s (field: %s)" % - (field.__class__.__name__, field.name)) - raise InvalidDocumentError(msg) - - f = field.field - - if delete_rule != DO_NOTHING: - if issubclass(new_class, EmbeddedDocument): - msg = ("Reverse delete rules are not supported for " - "EmbeddedDocuments (field: %s)" % field.name) - raise InvalidDocumentError(msg) - f.document_type.register_delete_rule(new_class, - field.name, delete_rule) - - if (field.name and hasattr(Document, field.name) and - EmbeddedDocument not in new_class.mro()): - msg = ("%s is a document method and not a valid " - "field name" % field.name) - raise InvalidDocumentError(msg) - - # Add class to the _document_registry - _document_registry[new_class._class_name] = new_class - - # In Python 2, User-defined methods objects have special read-only - # attributes 'im_func' and 'im_self' which contain the function obj - # and class instance object respectively. With Python 3 these special - # attributes have been replaced by __func__ and __self__. The Blinker - # module continues to use im_func and im_self, so the code below - # copies __func__ into im_func and __self__ into im_self for - # classmethod objects in Document derived classes. - if PY3: - for key, val in new_class.__dict__.items(): - if isinstance(val, classmethod): - f = val.__get__(new_class) - if hasattr(f, '__func__') and not hasattr(f, 'im_func'): - f.__dict__.update({'im_func': getattr(f, '__func__')}) - if hasattr(f, '__self__') and not hasattr(f, 'im_self'): - f.__dict__.update({'im_self': getattr(f, '__self__')}) - - return new_class - - def add_to_class(self, name, value): - setattr(self, name, value) - - @classmethod - def _get_bases(cls, bases): - if isinstance(bases, BasesTuple): - return bases - seen = [] - bases = cls.__get_bases(bases) - unique_bases = (b for b in bases if not (b in seen or seen.append(b))) - return BasesTuple(unique_bases) - - @classmethod - def __get_bases(cls, bases): - for base in bases: - if base is object: - continue - yield base - for child_base in cls.__get_bases(base.__bases__): - yield child_base - - @classmethod - def _import_classes(cls): - Document = _import_class('Document') - EmbeddedDocument = _import_class('EmbeddedDocument') - DictField = _import_class('DictField') - return (Document, EmbeddedDocument, DictField) - - -class TopLevelDocumentMetaclass(DocumentMetaclass): - """Metaclass for top-level documents (i.e. documents that have their own - collection in the database. - """ - - def __new__(cls, name, bases, attrs): - flattened_bases = cls._get_bases(bases) - super_new = super(TopLevelDocumentMetaclass, cls).__new__ - - # Set default _meta data if base class, otherwise get user defined meta - if (attrs.get('my_metaclass') == TopLevelDocumentMetaclass): - # defaults - attrs['_meta'] = { - 'abstract': True, - 'max_documents': None, - 'max_size': None, - 'ordering': [], # default ordering applied at runtime - 'indexes': [], # indexes to be ensured at runtime - 'id_field': None, - 'index_background': False, - 'index_drop_dups': False, - 'index_opts': None, - 'delete_rules': None, - 'allow_inheritance': None, - } - attrs['_is_base_cls'] = True - attrs['_meta'].update(attrs.get('meta', {})) - else: - attrs['_meta'] = attrs.get('meta', {}) - # Explictly set abstract to false unless set - attrs['_meta']['abstract'] = attrs['_meta'].get('abstract', False) - attrs['_is_base_cls'] = False - - # Set flag marking as document class - as opposed to an object mixin - attrs['_is_document'] = True - - # Ensure queryset_class is inherited - if 'objects' in attrs: - manager = attrs['objects'] - if hasattr(manager, 'queryset_class'): - attrs['_meta']['queryset_class'] = manager.queryset_class - - # Clean up top level meta - if 'meta' in attrs: - del(attrs['meta']) - - # Find the parent document class - parent_doc_cls = [b for b in flattened_bases - if b.__class__ == TopLevelDocumentMetaclass] - parent_doc_cls = None if not parent_doc_cls else parent_doc_cls[0] - - # Prevent classes setting collection different to their parents - # If parent wasn't an abstract class - if (parent_doc_cls and 'collection' in attrs.get('_meta', {}) - and not parent_doc_cls._meta.get('abstract', True)): - msg = "Trying to set a collection on a subclass (%s)" % name - warnings.warn(msg, SyntaxWarning) - del(attrs['_meta']['collection']) - - # Ensure abstract documents have abstract bases - if attrs.get('_is_base_cls') or attrs['_meta'].get('abstract'): - if (parent_doc_cls and - not parent_doc_cls._meta.get('abstract', False)): - msg = "Abstract document cannot have non-abstract base" - raise ValueError(msg) - return super_new(cls, name, bases, attrs) - - # Merge base class metas. - # Uses a special MetaDict that handles various merging rules - meta = MetaDict() - for base in flattened_bases[::-1]: - # Add any mixin metadata from plain objects - if hasattr(base, 'meta'): - meta.merge(base.meta) - elif hasattr(base, '_meta'): - meta.merge(base._meta) - - # Set collection in the meta if its callable - if (getattr(base, '_is_document', False) and - not base._meta.get('abstract')): - collection = meta.get('collection', None) - if callable(collection): - meta['collection'] = collection(base) - - meta.merge(attrs.get('_meta', {})) # Top level meta - - # Only simple classes (direct subclasses of Document) - # may set allow_inheritance to False - simple_class = all([b._meta.get('abstract') - for b in flattened_bases if hasattr(b, '_meta')]) - if (not simple_class and meta['allow_inheritance'] == False and - not meta['abstract']): - raise ValueError('Only direct subclasses of Document may set ' - '"allow_inheritance" to False') - - # Set default collection name - if 'collection' not in meta: - meta['collection'] = ''.join('_%s' % c if c.isupper() else c - for c in name).strip('_').lower() - attrs['_meta'] = meta - - # Call super and get the new class - new_class = super_new(cls, name, bases, attrs) - - meta = new_class._meta - - # Set index specifications - meta['index_specs'] = [QuerySet._build_index_spec(new_class, spec) - for spec in meta['indexes']] - unique_indexes = cls._unique_with_indexes(new_class) - new_class._meta['unique_indexes'] = unique_indexes - - # If collection is a callable - call it and set the value - collection = meta.get('collection') - if callable(collection): - new_class._meta['collection'] = collection(new_class) - - # Provide a default queryset unless one has been set - manager = attrs.get('objects', QuerySetManager()) - new_class.objects = manager - - # Validate the fields and set primary key if needed - for field_name, field in new_class._fields.iteritems(): - if field.primary_key: - # Ensure only one primary key is set - current_pk = new_class._meta.get('id_field') - if current_pk and current_pk != field_name: - raise ValueError('Cannot override primary key field') - - # Set primary key - if not current_pk: - new_class._meta['id_field'] = field_name - new_class.id = field - - # Set primary key if not defined by the document - if not new_class._meta.get('id_field'): - new_class._meta['id_field'] = 'id' - new_class._fields['id'] = ObjectIdField(db_field='_id') - new_class.id = new_class._fields['id'] - - # Merge in exceptions with parent hierarchy - exceptions_to_merge = (DoesNotExist, MultipleObjectsReturned) - module = attrs.get('__module__') - for exc in exceptions_to_merge: - name = exc.__name__ - parents = tuple(getattr(base, name) for base in flattened_bases - if hasattr(base, name)) or (exc,) - # Create new exception and set to new_class - exception = type(name, parents, {'__module__': module}) - setattr(new_class, name, exception) - - return new_class - - @classmethod - def _unique_with_indexes(cls, new_class, namespace=""): - """ - Find and set unique indexes - """ - unique_indexes = [] - for field_name, field in new_class._fields.items(): - # Generate a list of indexes needed by uniqueness constraints - if field.unique: - field.required = True - unique_fields = [field.db_field] - - # Add any unique_with fields to the back of the index spec - if field.unique_with: - if isinstance(field.unique_with, basestring): - field.unique_with = [field.unique_with] - - # Convert unique_with field names to real field names - unique_with = [] - for other_name in field.unique_with: - parts = other_name.split('.') - # Lookup real name - parts = QuerySet._lookup_field(new_class, parts) - name_parts = [part.db_field for part in parts] - unique_with.append('.'.join(name_parts)) - # Unique field should be required - parts[-1].required = True - unique_fields += unique_with - - # Add the new index to the list - index = [("%s%s" % (namespace, f), pymongo.ASCENDING) - for f in unique_fields] - unique_indexes.append(index) - - # Grab any embedded document field unique indexes - if (field.__class__.__name__ == "EmbeddedDocumentField" and - field.document_type != new_class): - field_namespace = "%s." % field_name - unique_indexes += cls._unique_with_indexes(field.document_type, - field_namespace) - - return unique_indexes - - -class MetaDict(dict): - """Custom dictionary for meta classes. - Handles the merging of set indexes - """ - _merge_options = ('indexes',) - - def merge(self, new_options): - for k, v in new_options.iteritems(): - if k in self._merge_options: - self[k] = self.get(k, []) + v - else: - self[k] = v - - -class BaseDocument(object): - - _dynamic = False - _created = True - _dynamic_lock = True - _initialised = False - - def __init__(self, **values): - signals.pre_init.send(self.__class__, document=self, values=values) - - self._data = {} - - # Assign default values to instance - for key, field in self._fields.iteritems(): - if self._db_field_map.get(key, key) in values: - continue - value = getattr(self, key, None) - setattr(self, key, value) - - # Set passed values after initialisation - if self._dynamic: - self._dynamic_fields = {} - dynamic_data = {} - for key, value in values.iteritems(): - if key in self._fields or key == '_id': - setattr(self, key, value) - elif self._dynamic: - dynamic_data[key] = value - else: - for key, value in values.iteritems(): - key = self._reverse_db_field_map.get(key, key) - setattr(self, key, value) - - # Set any get_fieldname_display methods - self.__set_field_display() - - if self._dynamic: - self._dynamic_lock = False - for key, value in dynamic_data.iteritems(): - setattr(self, key, value) - - # Flag initialised - self._initialised = True - signals.post_init.send(self.__class__, document=self) - - def __setattr__(self, name, value): - # Handle dynamic data only if an initialised dynamic document - if self._dynamic and not self._dynamic_lock: - - field = None - if not hasattr(self, name) and not name.startswith('_'): - DynamicField = _import_class("DynamicField") - field = DynamicField(db_field=name) - field.name = name - self._dynamic_fields[name] = field - - if not name.startswith('_'): - value = self.__expand_dynamic_values(name, value) - - # Handle marking data as changed - if name in self._dynamic_fields: - self._data[name] = value - if hasattr(self, '_changed_fields'): - self._mark_as_changed(name) - - if (self._is_document and not self._created and - name in self._meta.get('shard_key', tuple()) and - self._data.get(name) != value): - OperationError = _import_class('OperationError') - msg = "Shard Keys are immutable. Tried to update %s" % name - raise OperationError(msg) - - super(BaseDocument, self).__setattr__(name, value) - - def __expand_dynamic_values(self, name, value): - """expand any dynamic values to their correct types / values""" - if not isinstance(value, (dict, list, tuple)): - return value - - is_list = False - if not hasattr(value, 'items'): - is_list = True - value = dict([(k, v) for k, v in enumerate(value)]) - - if not is_list and '_cls' in value: - cls = get_document(value['_cls']) - return cls(**value) - - data = {} - for k, v in value.items(): - key = name if is_list else k - data[k] = self.__expand_dynamic_values(key, v) - - if is_list: # Convert back to a list - data_items = sorted(data.items(), key=operator.itemgetter(0)) - value = [v for k, v in data_items] - else: - value = data - - # Convert lists / values so we can watch for any changes on them - if (isinstance(value, (list, tuple)) and - not isinstance(value, BaseList)): - value = BaseList(value, self, name) - elif isinstance(value, dict) and not isinstance(value, BaseDict): - value = BaseDict(value, self, name) - - return value - - def validate(self): - """Ensure that all fields' values are valid and that required fields - are present. - """ - # Get a list of tuples of field names and their current values - fields = [(field, getattr(self, name)) - for name, field in self._fields.items()] - - # Ensure that each field is matched to a valid value - errors = {} - for field, value in fields: - if value is not None: - try: - field._validate(value) - except ValidationError, error: - errors[field.name] = error.errors or error - except (ValueError, AttributeError, AssertionError), error: - errors[field.name] = error - elif field.required: - errors[field.name] = ValidationError('Field is required', - field_name=field.name) - if errors: - raise ValidationError('ValidationError', errors=errors) - - def to_mongo(self): - """Return data dictionary ready for use with MongoDB. - """ - data = {} - for field_name, field in self._fields.items(): - value = getattr(self, field_name, None) - if value is not None: - data[field.db_field] = field.to_mongo(value) - # Only add _cls and _types if allow_inheritance is not False - if not (hasattr(self, '_meta') and - self._meta.get('allow_inheritance', ALLOW_INHERITANCE) == False): - data['_cls'] = self._class_name - data['_types'] = self._superclasses.keys() + [self._class_name] - if '_id' in data and data['_id'] is None: - del data['_id'] - - if not self._dynamic: - return data - - for name, field in self._dynamic_fields.items(): - data[name] = field.to_mongo(self._data.get(name, None)) - return data - - @classmethod - def _get_collection_name(cls): - """Returns the collection name for this class. - """ - return cls._meta.get('collection', None) - - @classmethod - def _from_son(cls, son): - """Create an instance of a Document (subclass) from a PyMongo SON. - """ - # get the class name from the document, falling back to the given - # class if unavailable - class_name = son.get('_cls', cls._class_name) - data = dict(("%s" % key, value) for key, value in son.items()) - if not UNICODE_KWARGS: - # python 2.6.4 and lower cannot handle unicode keys - # passed to class constructor example: cls(**data) - to_str_keys_recursive(data) - - if '_types' in data: - del data['_types'] - - if '_cls' in data: - del data['_cls'] - - # Return correct subclass for document type - if class_name != cls._class_name: - cls = get_document(class_name) - - changed_fields = [] - errors_dict = {} - - for field_name, field in cls._fields.items(): - if field.db_field in data: - value = data[field.db_field] - try: - data[field_name] = (value if value is None - else field.to_python(value)) - if field_name != field.db_field: - del data[field.db_field] - except (AttributeError, ValueError), e: - errors_dict[field_name] = e - elif field.default: - default = field.default - if callable(default): - default = default() - if isinstance(default, BaseDocument): - changed_fields.append(field_name) - - if errors_dict: - errors = "\n".join(["%s - %s" % (k, v) - for k, v in errors_dict.items()]) - msg = ("Invalid data to create a `%s` instance.\n%s" - % (cls._class_name, errors)) - raise InvalidDocumentError(msg) - - obj = cls(**data) - obj._changed_fields = changed_fields - obj._created = False - return obj - - def _mark_as_changed(self, key): - """Marks a key as explicitly changed by the user - """ - if not key: - return - key = self._db_field_map.get(key, key) - if (hasattr(self, '_changed_fields') and - key not in self._changed_fields): - self._changed_fields.append(key) - - def _get_changed_fields(self, key='', inspected=None): - """Returns a list of all fields that have explicitly been changed. - """ - EmbeddedDocument = _import_class("EmbeddedDocument") - DynamicEmbeddedDocument = _import_class("DynamicEmbeddedDocument") - _changed_fields = [] - _changed_fields += getattr(self, '_changed_fields', []) - - inspected = inspected or set() - if hasattr(self, 'id'): - if self.id in inspected: - return _changed_fields - inspected.add(self.id) - - field_list = self._fields.copy() - if self._dynamic: - field_list.update(self._dynamic_fields) - - for field_name in field_list: - - db_field_name = self._db_field_map.get(field_name, field_name) - key = '%s.' % db_field_name - field = self._data.get(field_name, None) - if hasattr(field, 'id'): - if field.id in inspected: - continue - inspected.add(field.id) - - if (isinstance(field, (EmbeddedDocument, DynamicEmbeddedDocument)) - and db_field_name not in _changed_fields): - # Find all embedded fields that have been changed - changed = field._get_changed_fields(key, inspected) - _changed_fields += ["%s%s" % (key, k) for k in changed if k] - elif (isinstance(field, (list, tuple, dict)) and - db_field_name not in _changed_fields): - # Loop list / dict fields as they contain documents - # Determine the iterator to use - if not hasattr(field, 'items'): - iterator = enumerate(field) - else: - iterator = field.iteritems() - for index, value in iterator: - if not hasattr(value, '_get_changed_fields'): - continue - list_key = "%s%s." % (key, index) - changed = value._get_changed_fields(list_key, inspected) - _changed_fields += ["%s%s" % (list_key, k) - for k in changed if k] - return _changed_fields - - def _delta(self): - """Returns the delta (set, unset) of the changes for a document. - Gets any values that have been explicitly changed. - """ - # Handles cases where not loaded from_son but has _id - doc = self.to_mongo() - set_fields = self._get_changed_fields() - set_data = {} - unset_data = {} - parts = [] - if hasattr(self, '_changed_fields'): - set_data = {} - # Fetch each set item from its path - for path in set_fields: - parts = path.split('.') - d = doc - new_path = [] - for p in parts: - if isinstance(d, DBRef): - break - elif p.isdigit(): - d = d[int(p)] - elif hasattr(d, 'get'): - d = d.get(p) - new_path.append(p) - path = '.'.join(new_path) - set_data[path] = d - else: - set_data = doc - if '_id' in set_data: - del(set_data['_id']) - - # Determine if any changed items were actually unset. - for path, value in set_data.items(): - if value or isinstance(value, bool): - continue - - # If we've set a value that ain't the default value dont unset it. - default = None - if (self._dynamic and len(parts) and - parts[0] in self._dynamic_fields): - del(set_data[path]) - unset_data[path] = 1 - continue - elif path in self._fields: - default = self._fields[path].default - else: # Perform a full lookup for lists / embedded lookups - d = self - parts = path.split('.') - db_field_name = parts.pop() - for p in parts: - if p.isdigit(): - d = d[int(p)] - elif (hasattr(d, '__getattribute__') and - not isinstance(d, dict)): - real_path = d._reverse_db_field_map.get(p, p) - d = getattr(d, real_path) - else: - d = d.get(p) - - if hasattr(d, '_fields'): - field_name = d._reverse_db_field_map.get(db_field_name, - db_field_name) - - if field_name in d._fields: - default = d._fields.get(field_name).default - else: - default = None - - if default is not None: - if callable(default): - default = default() - if default != value: - continue - - del(set_data[path]) - unset_data[path] = 1 - return set_data, unset_data - - @classmethod - def _geo_indices(cls, inspected=None): - inspected = inspected or [] - geo_indices = [] - inspected.append(cls) - - EmbeddedDocumentField = _import_class("EmbeddedDocumentField") - GeoPointField = _import_class("GeoPointField") - - for field in cls._fields.values(): - if not isinstance(field, (EmbeddedDocumentField, GeoPointField)): - continue - if hasattr(field, 'document_type'): - field_cls = field.document_type - if field_cls in inspected: - continue - if hasattr(field_cls, '_geo_indices'): - geo_indices += field_cls._geo_indices(inspected) - elif field._geo_index: - geo_indices.append(field) - return geo_indices - - def __getstate__(self): - removals = ("get_%s_display" % k - for k, v in self._fields.items() if v.choices) - for k in removals: - if hasattr(self, k): - delattr(self, k) - return self.__dict__ - - def __setstate__(self, __dict__): - self.__dict__ = __dict__ - self.__set_field_display() - - def __set_field_display(self): - """Dynamically set the display value for a field with choices""" - for attr_name, field in self._fields.items(): - if field.choices: - setattr(self, - 'get_%s_display' % attr_name, - partial(self.__get_field_display, field=field)) - - def __get_field_display(self, field): - """Returns the display value for a choice field""" - value = getattr(self, field.name) - if field.choices and isinstance(field.choices[0], (list, tuple)): - return dict(field.choices).get(value, value) - return value - - def __iter__(self): - return iter(self._fields) - - def __getitem__(self, name): - """Dictionary-style field access, return a field's value if present. - """ - try: - if name in self._fields: - return getattr(self, name) - except AttributeError: - pass - raise KeyError(name) - - def __setitem__(self, name, value): - """Dictionary-style field access, set a field's value. - """ - # Ensure that the field exists before settings its value - if name not in self._fields: - raise KeyError(name) - return setattr(self, name, value) - - def __contains__(self, name): - try: - val = getattr(self, name) - return val is not None - except AttributeError: - return False - - def __len__(self): - return len(self._data) - - def __repr__(self): - try: - u = self.__str__() - except (UnicodeEncodeError, UnicodeDecodeError): - u = '[Bad Unicode data]' - repr_type = type(u) - return repr_type('<%s: %s>' % (self.__class__.__name__, u)) - - def __str__(self): - if hasattr(self, '__unicode__'): - if PY3: - return self.__unicode__() - else: - return unicode(self).encode('utf-8') - return txt_type('%s object' % self.__class__.__name__) - - def __eq__(self, other): - if isinstance(other, self.__class__) and hasattr(other, 'id'): - if self.id == other.id: - return True - return False - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - if self.pk is None: - # For new object - return super(BaseDocument, self).__hash__() - else: - return hash(self.pk) - - -class BasesTuple(tuple): - """Special class to handle introspection of bases tuple in __new__""" - pass - - -class BaseList(list): - """A special list so we can watch any changes - """ - - _dereferenced = False - _instance = None - _name = None - - def __init__(self, list_items, instance, name): - self._instance = weakref.proxy(instance) - self._name = name - return super(BaseList, self).__init__(list_items) - - def __setitem__(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).__setitem__(*args, **kwargs) - - def __delitem__(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).__delitem__(*args, **kwargs) - - def __getstate__(self): - self.observer = None - return self - - def __setstate__(self, state): - self = state - return self - - def append(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).append(*args, **kwargs) - - def extend(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).extend(*args, **kwargs) - - def insert(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).insert(*args, **kwargs) - - def pop(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).pop(*args, **kwargs) - - def remove(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).remove(*args, **kwargs) - - def reverse(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).reverse(*args, **kwargs) - - def sort(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseList, self).sort(*args, **kwargs) - - def _mark_as_changed(self): - if hasattr(self._instance, '_mark_as_changed'): - self._instance._mark_as_changed(self._name) - - -class BaseDict(dict): - """A special dict so we can watch any changes - """ - - _dereferenced = False - _instance = None - _name = None - - def __init__(self, dict_items, instance, name): - self._instance = weakref.proxy(instance) - self._name = name - return super(BaseDict, self).__init__(dict_items) - - def __setitem__(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseDict, self).__setitem__(*args, **kwargs) - - def __delete__(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseDict, self).__delete__(*args, **kwargs) - - def __delitem__(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseDict, self).__delitem__(*args, **kwargs) - - def __delattr__(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseDict, self).__delattr__(*args, **kwargs) - - def __getstate__(self): - self.instance = None - self._dereferenced = False - return self - - def __setstate__(self, state): - self = state - return self - - def clear(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseDict, self).clear(*args, **kwargs) - - def pop(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseDict, self).pop(*args, **kwargs) - - def popitem(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseDict, self).popitem(*args, **kwargs) - - def update(self, *args, **kwargs): - self._mark_as_changed() - return super(BaseDict, self).update(*args, **kwargs) - - def _mark_as_changed(self): - if hasattr(self._instance, '_mark_as_changed'): - self._instance._mark_as_changed(self._name) - - -def _import_class(cls_name): - """Cached mechanism for imports""" - if cls_name in _class_registry: - return _class_registry.get(cls_name) - - doc_classes = ['Document', 'DynamicEmbeddedDocument', 'EmbeddedDocument'] - field_classes = ['DictField', 'DynamicField', 'EmbeddedDocumentField', - 'GenericReferenceField', 'GeoPointField', - 'ReferenceField'] - queryset_classes = ['OperationError'] - deref_classes = ['DeReference'] - - if cls_name in doc_classes: - from mongoengine import document as module - import_classes = doc_classes - elif cls_name in field_classes: - from mongoengine import fields as module - import_classes = field_classes - elif cls_name in queryset_classes: - from mongoengine import queryset as module - import_classes = queryset_classes - elif cls_name in deref_classes: - from mongoengine import dereference as module - import_classes = deref_classes - else: - raise ValueError('No import set for: ' % cls_name) - - for cls in import_classes: - _class_registry[cls] = getattr(module, cls) - - return _class_registry.get(cls_name) diff --git a/mongoengine/base/__init__.py b/mongoengine/base/__init__.py new file mode 100644 index 0000000..1d4a6eb --- /dev/null +++ b/mongoengine/base/__init__.py @@ -0,0 +1,5 @@ +from .common import * +from .datastructures import * +from .document import * +from .fields import * +from .metaclasses import * diff --git a/mongoengine/base/common.py b/mongoengine/base/common.py new file mode 100644 index 0000000..648561b --- /dev/null +++ b/mongoengine/base/common.py @@ -0,0 +1,25 @@ +from mongoengine.errors import NotRegistered + +__all__ = ('ALLOW_INHERITANCE', 'get_document', '_document_registry') + +ALLOW_INHERITANCE = True + +_document_registry = {} + + +def get_document(name): + doc = _document_registry.get(name, None) + if not doc: + # Possible old style names + end = ".%s" % name + possible_match = [k for k in _document_registry.keys() + if k.endswith(end)] + if len(possible_match) == 1: + doc = _document_registry.get(possible_match.pop(), None) + if not doc: + raise NotRegistered(""" + `%s` has not been registered in the document registry. + Importing the document class automatically registers it, has it + been imported? + """.strip() % name) + return doc diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py new file mode 100644 index 0000000..9a7620e --- /dev/null +++ b/mongoengine/base/datastructures.py @@ -0,0 +1,124 @@ +import weakref + +__all__ = ("BaseDict", "BaseList") + + +class BaseDict(dict): + """A special dict so we can watch any changes + """ + + _dereferenced = False + _instance = None + _name = None + + def __init__(self, dict_items, instance, name): + self._instance = weakref.proxy(instance) + self._name = name + return super(BaseDict, self).__init__(dict_items) + + def __setitem__(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseDict, self).__setitem__(*args, **kwargs) + + def __delete__(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseDict, self).__delete__(*args, **kwargs) + + def __delitem__(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseDict, self).__delitem__(*args, **kwargs) + + def __delattr__(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseDict, self).__delattr__(*args, **kwargs) + + def __getstate__(self): + self.instance = None + self._dereferenced = False + return self + + def __setstate__(self, state): + self = state + return self + + def clear(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseDict, self).clear(*args, **kwargs) + + def pop(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseDict, self).pop(*args, **kwargs) + + def popitem(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseDict, self).popitem(*args, **kwargs) + + def update(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseDict, self).update(*args, **kwargs) + + def _mark_as_changed(self): + if hasattr(self._instance, '_mark_as_changed'): + self._instance._mark_as_changed(self._name) + + +class BaseList(list): + """A special list so we can watch any changes + """ + + _dereferenced = False + _instance = None + _name = None + + def __init__(self, list_items, instance, name): + self._instance = weakref.proxy(instance) + self._name = name + return super(BaseList, self).__init__(list_items) + + def __setitem__(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).__setitem__(*args, **kwargs) + + def __delitem__(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).__delitem__(*args, **kwargs) + + def __getstate__(self): + self.observer = None + return self + + def __setstate__(self, state): + self = state + return self + + def append(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).append(*args, **kwargs) + + def extend(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).extend(*args, **kwargs) + + def insert(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).insert(*args, **kwargs) + + def pop(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).pop(*args, **kwargs) + + def remove(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).remove(*args, **kwargs) + + def reverse(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).reverse(*args, **kwargs) + + def sort(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).sort(*args, **kwargs) + + def _mark_as_changed(self): + if hasattr(self._instance, '_mark_as_changed'): + self._instance._mark_as_changed(self._name) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py new file mode 100644 index 0000000..af97e1f --- /dev/null +++ b/mongoengine/base/document.py @@ -0,0 +1,644 @@ +import operator +from functools import partial + +import pymongo +from bson.dbref import DBRef + +from mongoengine import signals +from mongoengine.common import _import_class +from mongoengine.errors import (ValidationError, InvalidDocumentError, + LookUpError) +from mongoengine.python_support import (PY3, UNICODE_KWARGS, txt_type, + to_str_keys_recursive) + +from .common import get_document, ALLOW_INHERITANCE +from .datastructures import BaseDict, BaseList +from .fields import ComplexBaseField + +__all__ = ('BaseDocument', ) + + +class BaseDocument(object): + + _dynamic = False + _created = True + _dynamic_lock = True + _initialised = False + + def __init__(self, **values): + signals.pre_init.send(self.__class__, document=self, values=values) + + self._data = {} + + # Assign default values to instance + for key, field in self._fields.iteritems(): + if self._db_field_map.get(key, key) in values: + continue + value = getattr(self, key, None) + setattr(self, key, value) + + # Set passed values after initialisation + if self._dynamic: + self._dynamic_fields = {} + dynamic_data = {} + for key, value in values.iteritems(): + if key in self._fields or key == '_id': + setattr(self, key, value) + elif self._dynamic: + dynamic_data[key] = value + else: + for key, value in values.iteritems(): + key = self._reverse_db_field_map.get(key, key) + setattr(self, key, value) + + # Set any get_fieldname_display methods + self.__set_field_display() + + if self._dynamic: + self._dynamic_lock = False + for key, value in dynamic_data.iteritems(): + setattr(self, key, value) + + # Flag initialised + self._initialised = True + signals.post_init.send(self.__class__, document=self) + + def __setattr__(self, name, value): + # Handle dynamic data only if an initialised dynamic document + if self._dynamic and not self._dynamic_lock: + + field = None + if not hasattr(self, name) and not name.startswith('_'): + DynamicField = _import_class("DynamicField") + field = DynamicField(db_field=name) + field.name = name + self._dynamic_fields[name] = field + + if not name.startswith('_'): + value = self.__expand_dynamic_values(name, value) + + # Handle marking data as changed + if name in self._dynamic_fields: + self._data[name] = value + if hasattr(self, '_changed_fields'): + self._mark_as_changed(name) + + if (self._is_document and not self._created and + name in self._meta.get('shard_key', tuple()) and + self._data.get(name) != value): + OperationError = _import_class('OperationError') + msg = "Shard Keys are immutable. Tried to update %s" % name + raise OperationError(msg) + + super(BaseDocument, self).__setattr__(name, value) + + def __getstate__(self): + removals = ("get_%s_display" % k + for k, v in self._fields.items() if v.choices) + for k in removals: + if hasattr(self, k): + delattr(self, k) + return self.__dict__ + + def __setstate__(self, __dict__): + self.__dict__ = __dict__ + self.__set_field_display() + + def __iter__(self): + return iter(self._fields) + + def __getitem__(self, name): + """Dictionary-style field access, return a field's value if present. + """ + try: + if name in self._fields: + return getattr(self, name) + except AttributeError: + pass + raise KeyError(name) + + def __setitem__(self, name, value): + """Dictionary-style field access, set a field's value. + """ + # Ensure that the field exists before settings its value + if name not in self._fields: + raise KeyError(name) + return setattr(self, name, value) + + def __contains__(self, name): + try: + val = getattr(self, name) + return val is not None + except AttributeError: + return False + + def __len__(self): + return len(self._data) + + def __repr__(self): + try: + u = self.__str__() + except (UnicodeEncodeError, UnicodeDecodeError): + u = '[Bad Unicode data]' + repr_type = type(u) + return repr_type('<%s: %s>' % (self.__class__.__name__, u)) + + def __str__(self): + if hasattr(self, '__unicode__'): + if PY3: + return self.__unicode__() + else: + return unicode(self).encode('utf-8') + return txt_type('%s object' % self.__class__.__name__) + + def __eq__(self, other): + if isinstance(other, self.__class__) and hasattr(other, 'id'): + if self.id == other.id: + return True + return False + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + if self.pk is None: + # For new object + return super(BaseDocument, self).__hash__() + else: + return hash(self.pk) + + def to_mongo(self): + """Return data dictionary ready for use with MongoDB. + """ + data = {} + for field_name, field in self._fields.items(): + value = getattr(self, field_name, None) + if value is not None: + data[field.db_field] = field.to_mongo(value) + # Only add _cls if allow_inheritance is not False + if not (hasattr(self, '_meta') and + self._meta.get('allow_inheritance', ALLOW_INHERITANCE) == False): + data['_cls'] = self._class_name + if '_id' in data and data['_id'] is None: + del data['_id'] + + if not self._dynamic: + return data + + for name, field in self._dynamic_fields.items(): + data[name] = field.to_mongo(self._data.get(name, None)) + return data + + def validate(self): + """Ensure that all fields' values are valid and that required fields + are present. + """ + # Get a list of tuples of field names and their current values + fields = [(field, getattr(self, name)) + for name, field in self._fields.items()] + + # Ensure that each field is matched to a valid value + errors = {} + for field, value in fields: + if value is not None: + try: + field._validate(value) + except ValidationError, error: + errors[field.name] = error.errors or error + except (ValueError, AttributeError, AssertionError), error: + errors[field.name] = error + elif field.required: + errors[field.name] = ValidationError('Field is required', + field_name=field.name) + if errors: + raise ValidationError('ValidationError', errors=errors) + + def __expand_dynamic_values(self, name, value): + """expand any dynamic values to their correct types / values""" + if not isinstance(value, (dict, list, tuple)): + return value + + is_list = False + if not hasattr(value, 'items'): + is_list = True + value = dict([(k, v) for k, v in enumerate(value)]) + + if not is_list and '_cls' in value: + cls = get_document(value['_cls']) + return cls(**value) + + data = {} + for k, v in value.items(): + key = name if is_list else k + data[k] = self.__expand_dynamic_values(key, v) + + if is_list: # Convert back to a list + data_items = sorted(data.items(), key=operator.itemgetter(0)) + value = [v for k, v in data_items] + else: + value = data + + # Convert lists / values so we can watch for any changes on them + if (isinstance(value, (list, tuple)) and + not isinstance(value, BaseList)): + value = BaseList(value, self, name) + elif isinstance(value, dict) and not isinstance(value, BaseDict): + value = BaseDict(value, self, name) + + return value + + def _mark_as_changed(self, key): + """Marks a key as explicitly changed by the user + """ + if not key: + return + key = self._db_field_map.get(key, key) + if (hasattr(self, '_changed_fields') and + key not in self._changed_fields): + self._changed_fields.append(key) + + def _get_changed_fields(self, key='', inspected=None): + """Returns a list of all fields that have explicitly been changed. + """ + EmbeddedDocument = _import_class("EmbeddedDocument") + DynamicEmbeddedDocument = _import_class("DynamicEmbeddedDocument") + _changed_fields = [] + _changed_fields += getattr(self, '_changed_fields', []) + + inspected = inspected or set() + if hasattr(self, 'id'): + if self.id in inspected: + return _changed_fields + inspected.add(self.id) + + field_list = self._fields.copy() + if self._dynamic: + field_list.update(self._dynamic_fields) + + for field_name in field_list: + + db_field_name = self._db_field_map.get(field_name, field_name) + key = '%s.' % db_field_name + field = self._data.get(field_name, None) + if hasattr(field, 'id'): + if field.id in inspected: + continue + inspected.add(field.id) + + if (isinstance(field, (EmbeddedDocument, DynamicEmbeddedDocument)) + and db_field_name not in _changed_fields): + # Find all embedded fields that have been changed + changed = field._get_changed_fields(key, inspected) + _changed_fields += ["%s%s" % (key, k) for k in changed if k] + elif (isinstance(field, (list, tuple, dict)) and + db_field_name not in _changed_fields): + # Loop list / dict fields as they contain documents + # Determine the iterator to use + if not hasattr(field, 'items'): + iterator = enumerate(field) + else: + iterator = field.iteritems() + for index, value in iterator: + if not hasattr(value, '_get_changed_fields'): + continue + list_key = "%s%s." % (key, index) + changed = value._get_changed_fields(list_key, inspected) + _changed_fields += ["%s%s" % (list_key, k) + for k in changed if k] + return _changed_fields + + def _delta(self): + """Returns the delta (set, unset) of the changes for a document. + Gets any values that have been explicitly changed. + """ + # Handles cases where not loaded from_son but has _id + doc = self.to_mongo() + set_fields = self._get_changed_fields() + set_data = {} + unset_data = {} + parts = [] + if hasattr(self, '_changed_fields'): + set_data = {} + # Fetch each set item from its path + for path in set_fields: + parts = path.split('.') + d = doc + new_path = [] + for p in parts: + if isinstance(d, DBRef): + break + elif p.isdigit(): + d = d[int(p)] + elif hasattr(d, 'get'): + d = d.get(p) + new_path.append(p) + path = '.'.join(new_path) + set_data[path] = d + else: + set_data = doc + if '_id' in set_data: + del(set_data['_id']) + + # Determine if any changed items were actually unset. + for path, value in set_data.items(): + if value or isinstance(value, bool): + continue + + # If we've set a value that ain't the default value dont unset it. + default = None + if (self._dynamic and len(parts) and + parts[0] in self._dynamic_fields): + del(set_data[path]) + unset_data[path] = 1 + continue + elif path in self._fields: + default = self._fields[path].default + else: # Perform a full lookup for lists / embedded lookups + d = self + parts = path.split('.') + db_field_name = parts.pop() + for p in parts: + if p.isdigit(): + d = d[int(p)] + elif (hasattr(d, '__getattribute__') and + not isinstance(d, dict)): + real_path = d._reverse_db_field_map.get(p, p) + d = getattr(d, real_path) + else: + d = d.get(p) + + if hasattr(d, '_fields'): + field_name = d._reverse_db_field_map.get(db_field_name, + db_field_name) + + if field_name in d._fields: + default = d._fields.get(field_name).default + else: + default = None + + if default is not None: + if callable(default): + default = default() + if default != value: + continue + + del(set_data[path]) + unset_data[path] = 1 + return set_data, unset_data + + @classmethod + def _get_collection_name(cls): + """Returns the collection name for this class. + """ + return cls._meta.get('collection', None) + + @classmethod + def _from_son(cls, son): + """Create an instance of a Document (subclass) from a PyMongo SON. + """ + # get the class name from the document, falling back to the given + # class if unavailable + class_name = son.get('_cls', cls._class_name) + data = dict(("%s" % key, value) for key, value in son.items()) + if not UNICODE_KWARGS: + # python 2.6.4 and lower cannot handle unicode keys + # passed to class constructor example: cls(**data) + to_str_keys_recursive(data) + + if '_cls' in data: + del data['_cls'] + + # Return correct subclass for document type + if class_name != cls._class_name: + cls = get_document(class_name) + + changed_fields = [] + errors_dict = {} + + for field_name, field in cls._fields.items(): + if field.db_field in data: + value = data[field.db_field] + try: + data[field_name] = (value if value is None + else field.to_python(value)) + if field_name != field.db_field: + del data[field.db_field] + except (AttributeError, ValueError), e: + errors_dict[field_name] = e + elif field.default: + default = field.default + if callable(default): + default = default() + if isinstance(default, BaseDocument): + changed_fields.append(field_name) + + if errors_dict: + errors = "\n".join(["%s - %s" % (k, v) + for k, v in errors_dict.items()]) + msg = ("Invalid data to create a `%s` instance.\n%s" + % (cls._class_name, errors)) + raise InvalidDocumentError(msg) + + obj = cls(**data) + obj._changed_fields = changed_fields + obj._created = False + return obj + + @classmethod + def _build_index_spec(cls, spec): + """Build a PyMongo index spec from a MongoEngine index spec. + """ + if isinstance(spec, basestring): + spec = {'fields': [spec]} + elif isinstance(spec, (list, tuple)): + spec = {'fields': list(spec)} + elif isinstance(spec, dict): + spec = dict(spec) + + index_list = [] + direction = None + + # Check to see if we need to include _cls + allow_inheritance = cls._meta.get('allow_inheritance', + ALLOW_INHERITANCE) != False + include_cls = allow_inheritance and not spec.get('sparse', False) + + for key in spec['fields']: + # If inherited spec continue + if isinstance(key, (list, tuple)): + continue + + # ASCENDING from +, + # DESCENDING from - + # GEO2D from * + direction = pymongo.ASCENDING + if key.startswith("-"): + direction = pymongo.DESCENDING + elif key.startswith("*"): + direction = pymongo.GEO2D + if key.startswith(("+", "-", "*")): + key = key[1:] + + # Use real field name, do it manually because we need field + # objects for the next part (list field checking) + parts = key.split('.') + if parts in (['pk'], ['id'], ['_id']): + key = '_id' + fields = [] + else: + fields = cls._lookup_field(parts) + parts = [field if field == '_id' else field.db_field + for field in fields] + key = '.'.join(parts) + index_list.append((key, direction)) + + # Don't add cls to a geo index + if include_cls and direction is not pymongo.GEO2D: + index_list.insert(0, ('_cls', 1)) + + spec['fields'] = index_list + if spec.get('sparse', False) and len(spec['fields']) > 1: + raise ValueError( + 'Sparse indexes can only have one field in them. ' + 'See https://jira.mongodb.org/browse/SERVER-2193') + + return spec + + @classmethod + def _unique_with_indexes(cls, namespace=""): + """ + Find and set unique indexes + """ + unique_indexes = [] + for field_name, field in cls._fields.items(): + # Generate a list of indexes needed by uniqueness constraints + if field.unique: + field.required = True + unique_fields = [field.db_field] + + # Add any unique_with fields to the back of the index spec + if field.unique_with: + if isinstance(field.unique_with, basestring): + field.unique_with = [field.unique_with] + + # Convert unique_with field names to real field names + unique_with = [] + for other_name in field.unique_with: + parts = other_name.split('.') + # Lookup real name + parts = cls._lookup_field(parts) + name_parts = [part.db_field for part in parts] + unique_with.append('.'.join(name_parts)) + # Unique field should be required + parts[-1].required = True + unique_fields += unique_with + + # Add the new index to the list + index = [("%s%s" % (namespace, f), pymongo.ASCENDING) + for f in unique_fields] + unique_indexes.append(index) + + # Grab any embedded document field unique indexes + if (field.__class__.__name__ == "EmbeddedDocumentField" and + field.document_type != cls): + field_namespace = "%s." % field_name + doc_cls = field.document_type + unique_indexes += doc_cls._unique_with_indexes(field_namespace) + + return unique_indexes + + @classmethod + def _lookup_field(cls, parts): + """Lookup a field based on its attribute and return a list containing + the field's parents and the field. + """ + if not isinstance(parts, (list, tuple)): + parts = [parts] + fields = [] + field = None + + for field_name in parts: + # Handle ListField indexing: + if field_name.isdigit(): + new_field = field.field + fields.append(field_name) + continue + + if field is None: + # Look up first field from the document + if field_name == 'pk': + # Deal with "primary key" alias + field_name = cls._meta['id_field'] + if field_name in cls._fields: + field = cls._fields[field_name] + elif cls._dynamic: + DynamicField = _import_class('DynamicField') + field = DynamicField(db_field=field_name) + else: + raise LookUpError('Cannot resolve field "%s"' + % field_name) + else: + ReferenceField = _import_class('ReferenceField') + GenericReferenceField = _import_class('GenericReferenceField') + if isinstance(field, (ReferenceField, GenericReferenceField)): + raise LookUpError('Cannot perform join in mongoDB: %s' % + '__'.join(parts)) + if hasattr(getattr(field, 'field', None), 'lookup_member'): + new_field = field.field.lookup_member(field_name) + else: + # Look up subfield on the previous field + new_field = field.lookup_member(field_name) + if not new_field and isinstance(field, ComplexBaseField): + fields.append(field_name) + continue + elif not new_field: + raise LookUpError('Cannot resolve field "%s"' + % field_name) + field = new_field # update field to the new field type + fields.append(field) + return fields + + @classmethod + def _translate_field_name(cls, field, sep='.'): + """Translate a field attribute name to a database field name. + """ + parts = field.split(sep) + parts = [f.db_field for f in cls._lookup_field(parts)] + return '.'.join(parts) + + @classmethod + def _geo_indices(cls, inspected=None): + inspected = inspected or [] + geo_indices = [] + inspected.append(cls) + + EmbeddedDocumentField = _import_class("EmbeddedDocumentField") + GeoPointField = _import_class("GeoPointField") + + for field in cls._fields.values(): + if not isinstance(field, (EmbeddedDocumentField, GeoPointField)): + continue + if hasattr(field, 'document_type'): + field_cls = field.document_type + if field_cls in inspected: + continue + if hasattr(field_cls, '_geo_indices'): + geo_indices += field_cls._geo_indices(inspected) + elif field._geo_index: + geo_indices.append(field) + return geo_indices + + def __set_field_display(self): + """Dynamically set the display value for a field with choices""" + for attr_name, field in self._fields.items(): + if field.choices: + setattr(self, + 'get_%s_display' % attr_name, + partial(self.__get_field_display, field=field)) + + def __get_field_display(self, field): + """Returns the display value for a choice field""" + value = getattr(self, field.name) + if field.choices and isinstance(field.choices[0], (list, tuple)): + return dict(field.choices).get(value, value) + return value diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py new file mode 100644 index 0000000..44f5e13 --- /dev/null +++ b/mongoengine/base/fields.py @@ -0,0 +1,371 @@ +import operator +import warnings + +from bson import DBRef, ObjectId + +from mongoengine.common import _import_class +from mongoengine.errors import ValidationError + +from .common import ALLOW_INHERITANCE +from .datastructures import BaseDict, BaseList + +__all__ = ("BaseField", "ComplexBaseField", "ObjectIdField") + + +class BaseField(object): + """A base class for fields in a MongoDB document. Instances of this class + may be added to subclasses of `Document` to define a document's schema. + + .. versionchanged:: 0.5 - added verbose and help text + """ + + name = None + _geo_index = False + + # These track each time a Field instance is created. Used to retain order. + # The auto_creation_counter is used for fields that MongoEngine implicitly + # creates, creation_counter is used for all user-specified fields. + creation_counter = 0 + auto_creation_counter = -1 + + def __init__(self, db_field=None, name=None, required=False, default=None, + unique=False, unique_with=None, primary_key=False, + validation=None, choices=None, verbose_name=None, + help_text=None): + self.db_field = (db_field or name) if not primary_key else '_id' + if name: + msg = "Fields' 'name' attribute deprecated in favour of 'db_field'" + warnings.warn(msg, DeprecationWarning) + self.name = None + self.required = required or primary_key + self.default = default + self.unique = bool(unique or unique_with) + self.unique_with = unique_with + self.primary_key = primary_key + self.validation = validation + self.choices = choices + self.verbose_name = verbose_name + self.help_text = help_text + + # Adjust the appropriate creation counter, and save our local copy. + if self.db_field == '_id': + self.creation_counter = BaseField.auto_creation_counter + BaseField.auto_creation_counter -= 1 + else: + self.creation_counter = BaseField.creation_counter + BaseField.creation_counter += 1 + + def __get__(self, instance, owner): + """Descriptor for retrieving a value from a field in a document. Do + any necessary conversion between Python and MongoDB types. + """ + if instance is None: + # Document class being used rather than a document object + return self + + # Get value from document instance if available, if not use default + value = instance._data.get(self.name) + + if value is None: + value = self.default + # Allow callable default values + if callable(value): + value = value() + + return value + + def __set__(self, instance, value): + """Descriptor for assigning a value to a field in a document. + """ + instance._data[self.name] = value + if instance._initialised: + instance._mark_as_changed(self.name) + + def error(self, message="", errors=None, field_name=None): + """Raises a ValidationError. + """ + field_name = field_name if field_name else self.name + raise ValidationError(message, errors=errors, field_name=field_name) + + def to_python(self, value): + """Convert a MongoDB-compatible type to a Python type. + """ + return value + + def to_mongo(self, value): + """Convert a Python type to a MongoDB-compatible type. + """ + return self.to_python(value) + + def prepare_query_value(self, op, value): + """Prepare a value that is being used in a query for PyMongo. + """ + return value + + def validate(self, value): + """Perform validation on a value. + """ + pass + + def _validate(self, value): + Document = _import_class('Document') + EmbeddedDocument = _import_class('EmbeddedDocument') + # check choices + if self.choices: + is_cls = isinstance(value, (Document, EmbeddedDocument)) + value_to_check = value.__class__ if is_cls else value + err_msg = 'an instance' if is_cls else 'one' + if isinstance(self.choices[0], (list, tuple)): + option_keys = [k for k, v in self.choices] + if value_to_check not in option_keys: + msg = ('Value must be %s of %s' % + (err_msg, unicode(option_keys))) + self.error(msg) + elif value_to_check not in self.choices: + msg = ('Value must be %s of %s' % + (err_msg, unicode(self.choices))) + self.error() + + # check validation argument + if self.validation is not None: + if callable(self.validation): + if not self.validation(value): + self.error('Value does not match custom validation method') + else: + raise ValueError('validation argument for "%s" must be a ' + 'callable.' % self.name) + + self.validate(value) + + +class ComplexBaseField(BaseField): + """Handles complex fields, such as lists / dictionaries. + + Allows for nesting of embedded documents inside complex types. + Handles the lazy dereferencing of a queryset by lazily dereferencing all + items in a list / dict rather than one at a time. + + .. versionadded:: 0.5 + """ + + field = None + __dereference = False + + def __get__(self, instance, owner): + """Descriptor to automatically dereference references. + """ + if instance is None: + # Document class being used rather than a document object + return self + + ReferenceField = _import_class('ReferenceField') + GenericReferenceField = _import_class('GenericReferenceField') + dereference = self.field is None or isinstance(self.field, + (GenericReferenceField, ReferenceField)) + if not self._dereference and instance._initialised and dereference: + instance._data[self.name] = self._dereference( + instance._data.get(self.name), max_depth=1, instance=instance, + name=self.name + ) + + value = super(ComplexBaseField, self).__get__(instance, owner) + + # Convert lists / values so we can watch for any changes on them + if (isinstance(value, (list, tuple)) and + not isinstance(value, BaseList)): + value = BaseList(value, instance, self.name) + instance._data[self.name] = value + elif isinstance(value, dict) and not isinstance(value, BaseDict): + value = BaseDict(value, instance, self.name) + instance._data[self.name] = value + + if (instance._initialised and isinstance(value, (BaseList, BaseDict)) + and not value._dereferenced): + value = self._dereference( + value, max_depth=1, instance=instance, name=self.name + ) + value._dereferenced = True + instance._data[self.name] = value + + return value + + def __set__(self, instance, value): + """Descriptor for assigning a value to a field in a document. + """ + instance._data[self.name] = value + instance._mark_as_changed(self.name) + + def to_python(self, value): + """Convert a MongoDB-compatible type to a Python type. + """ + Document = _import_class('Document') + + if isinstance(value, basestring): + return value + + if hasattr(value, 'to_python'): + return value.to_python() + + is_list = False + if not hasattr(value, 'items'): + try: + is_list = True + value = dict([(k, v) for k, v in enumerate(value)]) + except TypeError: # Not iterable return the value + return value + + if self.field: + value_dict = dict([(key, self.field.to_python(item)) + for key, item in value.items()]) + else: + value_dict = {} + for k, v in value.items(): + if isinstance(v, Document): + # We need the id from the saved object to create the DBRef + if v.pk is None: + self.error('You can only reference documents once they' + ' have been saved to the database') + collection = v._get_collection_name() + value_dict[k] = DBRef(collection, v.pk) + elif hasattr(v, 'to_python'): + value_dict[k] = v.to_python() + else: + value_dict[k] = self.to_python(v) + + if is_list: # Convert back to a list + return [v for k, v in sorted(value_dict.items(), + key=operator.itemgetter(0))] + return value_dict + + def to_mongo(self, value): + """Convert a Python type to a MongoDB-compatible type. + """ + Document = _import_class("Document") + + if isinstance(value, basestring): + return value + + if hasattr(value, 'to_mongo'): + return value.to_mongo() + + is_list = False + if not hasattr(value, 'items'): + try: + is_list = True + value = dict([(k, v) for k, v in enumerate(value)]) + except TypeError: # Not iterable return the value + return value + + if self.field: + value_dict = dict([(key, self.field.to_mongo(item)) + for key, item in value.items()]) + else: + value_dict = {} + for k, v in value.items(): + if isinstance(v, Document): + # We need the id from the saved object to create the DBRef + if v.pk is None: + self.error('You can only reference documents once they' + ' have been saved to the database') + + # If its a document that is not inheritable it won't have + # any _cls data so make it a generic reference allows + # us to dereference + meta = getattr(v, '_meta', {}) + allow_inheritance = ( + meta.get('allow_inheritance', ALLOW_INHERITANCE) + == False) + if allow_inheritance and not self.field: + GenericReferenceField = _import_class( + "GenericReferenceField") + value_dict[k] = GenericReferenceField().to_mongo(v) + else: + collection = v._get_collection_name() + value_dict[k] = DBRef(collection, v.pk) + elif hasattr(v, 'to_mongo'): + value_dict[k] = v.to_mongo() + else: + value_dict[k] = self.to_mongo(v) + + if is_list: # Convert back to a list + return [v for k, v in sorted(value_dict.items(), + key=operator.itemgetter(0))] + return value_dict + + def validate(self, value): + """If field is provided ensure the value is valid. + """ + errors = {} + if self.field: + if hasattr(value, 'iteritems') or hasattr(value, 'items'): + sequence = value.iteritems() + else: + sequence = enumerate(value) + for k, v in sequence: + try: + self.field._validate(v) + except ValidationError, error: + errors[k] = error.errors or error + except (ValueError, AssertionError), error: + errors[k] = error + + if errors: + field_class = self.field.__class__.__name__ + self.error('Invalid %s item (%s)' % (field_class, value), + errors=errors) + # Don't allow empty values if required + if self.required and not value: + self.error('Field is required and cannot be empty') + + def prepare_query_value(self, op, value): + return self.to_mongo(value) + + def lookup_member(self, member_name): + if self.field: + return self.field.lookup_member(member_name) + return None + + def _set_owner_document(self, owner_document): + if self.field: + self.field.owner_document = owner_document + self._owner_document = owner_document + + def _get_owner_document(self, owner_document): + self._owner_document = owner_document + + owner_document = property(_get_owner_document, _set_owner_document) + + @property + def _dereference(self,): + if not self.__dereference: + DeReference = _import_class("DeReference") + self.__dereference = DeReference() # Cached + return self.__dereference + + +class ObjectIdField(BaseField): + """A field wrapper around MongoDB's ObjectIds. + """ + + def to_python(self, value): + if not isinstance(value, ObjectId): + value = ObjectId(value) + return value + + def to_mongo(self, value): + if not isinstance(value, ObjectId): + try: + return ObjectId(unicode(value)) + except Exception, e: + # e.message attribute has been deprecated since Python 2.6 + self.error(unicode(e)) + return value + + def prepare_query_value(self, op, value): + return self.to_mongo(value) + + def validate(self, value): + try: + ObjectId(unicode(value)) + except: + self.error('Invalid Object ID') diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py new file mode 100644 index 0000000..f87b03e --- /dev/null +++ b/mongoengine/base/metaclasses.py @@ -0,0 +1,388 @@ +import warnings + +import pymongo + +from mongoengine.common import _import_class +from mongoengine.errors import InvalidDocumentError +from mongoengine.python_support import PY3 +from mongoengine.queryset import (DO_NOTHING, DoesNotExist, + MultipleObjectsReturned, + QuerySet, QuerySetManager) + +from .common import _document_registry, ALLOW_INHERITANCE +from .fields import BaseField, ComplexBaseField, ObjectIdField + +__all__ = ('DocumentMetaclass', 'TopLevelDocumentMetaclass') + + +class DocumentMetaclass(type): + """Metaclass for all documents. + """ + + def __new__(cls, name, bases, attrs): + flattened_bases = cls._get_bases(bases) + super_new = super(DocumentMetaclass, cls).__new__ + + # If a base class just call super + metaclass = attrs.get('my_metaclass') + if metaclass and issubclass(metaclass, DocumentMetaclass): + return super_new(cls, name, bases, attrs) + + attrs['_is_document'] = attrs.get('_is_document', False) + + # EmbeddedDocuments could have meta data for inheritance + if 'meta' in attrs: + attrs['_meta'] = attrs.pop('meta') + + # Handle document Fields + + # Merge all fields from subclasses + doc_fields = {} + for base in flattened_bases[::-1]: + if hasattr(base, '_fields'): + doc_fields.update(base._fields) + + # Standard object mixin - merge in any Fields + if not hasattr(base, '_meta'): + base_fields = {} + for attr_name, attr_value in base.__dict__.iteritems(): + if not isinstance(attr_value, BaseField): + continue + attr_value.name = attr_name + if not attr_value.db_field: + attr_value.db_field = attr_name + base_fields[attr_name] = attr_value + doc_fields.update(base_fields) + + # Discover any document fields + field_names = {} + for attr_name, attr_value in attrs.iteritems(): + if not isinstance(attr_value, BaseField): + continue + attr_value.name = attr_name + if not attr_value.db_field: + attr_value.db_field = attr_name + doc_fields[attr_name] = attr_value + + # Count names to ensure no db_field redefinitions + field_names[attr_value.db_field] = field_names.get( + attr_value.db_field, 0) + 1 + + # Ensure no duplicate db_fields + duplicate_db_fields = [k for k, v in field_names.items() if v > 1] + if duplicate_db_fields: + msg = ("Multiple db_fields defined for: %s " % + ", ".join(duplicate_db_fields)) + raise InvalidDocumentError(msg) + + # Set _fields and db_field maps + attrs['_fields'] = doc_fields + attrs['_db_field_map'] = dict([(k, getattr(v, 'db_field', k)) + for k, v in doc_fields.iteritems()]) + attrs['_reverse_db_field_map'] = dict( + (v, k) for k, v in attrs['_db_field_map'].iteritems()) + + # + # Set document hierarchy + # + superclasses = () + class_name = [name] + for base in flattened_bases: + if (not getattr(base, '_is_base_cls', True) and + not getattr(base, '_meta', {}).get('abstract', True)): + # Collate heirarchy for _cls and _subclasses + class_name.append(base.__name__) + + if hasattr(base, '_meta'): + # Warn if allow_inheritance isn't set and prevent + # inheritance of classes where inheritance is set to False + allow_inheritance = base._meta.get('allow_inheritance', + ALLOW_INHERITANCE) + if (not getattr(base, '_is_base_cls', True) + and allow_inheritance is None): + warnings.warn( + "%s uses inheritance, the default for " + "allow_inheritance is changing to off by default. " + "Please add it to the document meta." % name, + FutureWarning + ) + elif (allow_inheritance == False and + not base._meta.get('abstract')): + raise ValueError('Document %s may not be subclassed' % + base.__name__) + + # Get superclasses from last base superclass + document_bases = [b for b in flattened_bases + if hasattr(b, '_class_name')] + if document_bases: + superclasses = document_bases[0]._superclasses + superclasses += (document_bases[0]._class_name, ) + + _cls = '.'.join(reversed(class_name)) + attrs['_class_name'] = _cls + attrs['_superclasses'] = superclasses + attrs['_subclasses'] = (_cls, ) + attrs['_types'] = attrs['_subclasses'] # TODO depreciate _types + + # Create the new_class + new_class = super_new(cls, name, bases, attrs) + + # Set _subclasses + for base in document_bases: + if _cls not in base._subclasses: + base._subclasses += (_cls,) + base._types = base._subclasses # TODO depreciate _types + + # Handle delete rules + Document, EmbeddedDocument, DictField = cls._import_classes() + for field in new_class._fields.itervalues(): + f = field + f.owner_document = new_class + delete_rule = getattr(f, 'reverse_delete_rule', DO_NOTHING) + if isinstance(f, ComplexBaseField) and hasattr(f, 'field'): + delete_rule = getattr(f.field, + 'reverse_delete_rule', + DO_NOTHING) + if isinstance(f, DictField) and delete_rule != DO_NOTHING: + msg = ("Reverse delete rules are not supported " + "for %s (field: %s)" % + (field.__class__.__name__, field.name)) + raise InvalidDocumentError(msg) + + f = field.field + + if delete_rule != DO_NOTHING: + if issubclass(new_class, EmbeddedDocument): + msg = ("Reverse delete rules are not supported for " + "EmbeddedDocuments (field: %s)" % field.name) + raise InvalidDocumentError(msg) + f.document_type.register_delete_rule(new_class, + field.name, delete_rule) + + if (field.name and hasattr(Document, field.name) and + EmbeddedDocument not in new_class.mro()): + msg = ("%s is a document method and not a valid " + "field name" % field.name) + raise InvalidDocumentError(msg) + + # Add class to the _document_registry + _document_registry[new_class._class_name] = new_class + + # In Python 2, User-defined methods objects have special read-only + # attributes 'im_func' and 'im_self' which contain the function obj + # and class instance object respectively. With Python 3 these special + # attributes have been replaced by __func__ and __self__. The Blinker + # module continues to use im_func and im_self, so the code below + # copies __func__ into im_func and __self__ into im_self for + # classmethod objects in Document derived classes. + if PY3: + for key, val in new_class.__dict__.items(): + if isinstance(val, classmethod): + f = val.__get__(new_class) + if hasattr(f, '__func__') and not hasattr(f, 'im_func'): + f.__dict__.update({'im_func': getattr(f, '__func__')}) + if hasattr(f, '__self__') and not hasattr(f, 'im_self'): + f.__dict__.update({'im_self': getattr(f, '__self__')}) + + return new_class + + def add_to_class(self, name, value): + setattr(self, name, value) + + @classmethod + def _get_bases(cls, bases): + if isinstance(bases, BasesTuple): + return bases + seen = [] + bases = cls.__get_bases(bases) + unique_bases = (b for b in bases if not (b in seen or seen.append(b))) + return BasesTuple(unique_bases) + + @classmethod + def __get_bases(cls, bases): + for base in bases: + if base is object: + continue + yield base + for child_base in cls.__get_bases(base.__bases__): + yield child_base + + @classmethod + def _import_classes(cls): + Document = _import_class('Document') + EmbeddedDocument = _import_class('EmbeddedDocument') + DictField = _import_class('DictField') + return (Document, EmbeddedDocument, DictField) + + +class TopLevelDocumentMetaclass(DocumentMetaclass): + """Metaclass for top-level documents (i.e. documents that have their own + collection in the database. + """ + + def __new__(cls, name, bases, attrs): + flattened_bases = cls._get_bases(bases) + super_new = super(TopLevelDocumentMetaclass, cls).__new__ + + # Set default _meta data if base class, otherwise get user defined meta + if (attrs.get('my_metaclass') == TopLevelDocumentMetaclass): + # defaults + attrs['_meta'] = { + 'abstract': True, + 'max_documents': None, + 'max_size': None, + 'ordering': [], # default ordering applied at runtime + 'indexes': [], # indexes to be ensured at runtime + 'id_field': None, + 'index_background': False, + 'index_drop_dups': False, + 'index_opts': None, + 'delete_rules': None, + 'allow_inheritance': None, + } + attrs['_is_base_cls'] = True + attrs['_meta'].update(attrs.get('meta', {})) + else: + attrs['_meta'] = attrs.get('meta', {}) + # Explictly set abstract to false unless set + attrs['_meta']['abstract'] = attrs['_meta'].get('abstract', False) + attrs['_is_base_cls'] = False + + # Set flag marking as document class - as opposed to an object mixin + attrs['_is_document'] = True + + # Ensure queryset_class is inherited + if 'objects' in attrs: + manager = attrs['objects'] + if hasattr(manager, 'queryset_class'): + attrs['_meta']['queryset_class'] = manager.queryset_class + + # Clean up top level meta + if 'meta' in attrs: + del(attrs['meta']) + + # Find the parent document class + parent_doc_cls = [b for b in flattened_bases + if b.__class__ == TopLevelDocumentMetaclass] + parent_doc_cls = None if not parent_doc_cls else parent_doc_cls[0] + + # Prevent classes setting collection different to their parents + # If parent wasn't an abstract class + if (parent_doc_cls and 'collection' in attrs.get('_meta', {}) + and not parent_doc_cls._meta.get('abstract', True)): + msg = "Trying to set a collection on a subclass (%s)" % name + warnings.warn(msg, SyntaxWarning) + del(attrs['_meta']['collection']) + + # Ensure abstract documents have abstract bases + if attrs.get('_is_base_cls') or attrs['_meta'].get('abstract'): + if (parent_doc_cls and + not parent_doc_cls._meta.get('abstract', False)): + msg = "Abstract document cannot have non-abstract base" + raise ValueError(msg) + return super_new(cls, name, bases, attrs) + + # Merge base class metas. + # Uses a special MetaDict that handles various merging rules + meta = MetaDict() + for base in flattened_bases[::-1]: + # Add any mixin metadata from plain objects + if hasattr(base, 'meta'): + meta.merge(base.meta) + elif hasattr(base, '_meta'): + meta.merge(base._meta) + + # Set collection in the meta if its callable + if (getattr(base, '_is_document', False) and + not base._meta.get('abstract')): + collection = meta.get('collection', None) + if callable(collection): + meta['collection'] = collection(base) + + meta.merge(attrs.get('_meta', {})) # Top level meta + + # Only simple classes (direct subclasses of Document) + # may set allow_inheritance to False + simple_class = all([b._meta.get('abstract') + for b in flattened_bases if hasattr(b, '_meta')]) + if (not simple_class and meta['allow_inheritance'] == False and + not meta['abstract']): + raise ValueError('Only direct subclasses of Document may set ' + '"allow_inheritance" to False') + + # Set default collection name + if 'collection' not in meta: + meta['collection'] = ''.join('_%s' % c if c.isupper() else c + for c in name).strip('_').lower() + attrs['_meta'] = meta + + # Call super and get the new class + new_class = super_new(cls, name, bases, attrs) + + meta = new_class._meta + + # Set index specifications + meta['index_specs'] = [new_class._build_index_spec(spec) + for spec in meta['indexes']] + unique_indexes = new_class._unique_with_indexes() + new_class._meta['unique_indexes'] = unique_indexes + + # If collection is a callable - call it and set the value + collection = meta.get('collection') + if callable(collection): + new_class._meta['collection'] = collection(new_class) + + # Provide a default queryset unless one has been set + manager = attrs.get('objects', QuerySetManager()) + new_class.objects = manager + + # Validate the fields and set primary key if needed + for field_name, field in new_class._fields.iteritems(): + if field.primary_key: + # Ensure only one primary key is set + current_pk = new_class._meta.get('id_field') + if current_pk and current_pk != field_name: + raise ValueError('Cannot override primary key field') + + # Set primary key + if not current_pk: + new_class._meta['id_field'] = field_name + new_class.id = field + + # Set primary key if not defined by the document + if not new_class._meta.get('id_field'): + new_class._meta['id_field'] = 'id' + new_class._fields['id'] = ObjectIdField(db_field='_id') + new_class.id = new_class._fields['id'] + + # Merge in exceptions with parent hierarchy + exceptions_to_merge = (DoesNotExist, MultipleObjectsReturned) + module = attrs.get('__module__') + for exc in exceptions_to_merge: + name = exc.__name__ + parents = tuple(getattr(base, name) for base in flattened_bases + if hasattr(base, name)) or (exc,) + # Create new exception and set to new_class + exception = type(name, parents, {'__module__': module}) + setattr(new_class, name, exception) + + return new_class + + +class MetaDict(dict): + """Custom dictionary for meta classes. + Handles the merging of set indexes + """ + _merge_options = ('indexes',) + + def merge(self, new_options): + for k, v in new_options.iteritems(): + if k in self._merge_options: + self[k] = self.get(k, []) + v + else: + self[k] = v + + +class BasesTuple(tuple): + """Special class to handle introspection of bases tuple in __new__""" + pass diff --git a/mongoengine/common.py b/mongoengine/common.py new file mode 100644 index 0000000..c284777 --- /dev/null +++ b/mongoengine/common.py @@ -0,0 +1,35 @@ +_class_registry_cache = {} + + +def _import_class(cls_name): + """Cached mechanism for imports""" + if cls_name in _class_registry_cache: + return _class_registry_cache.get(cls_name) + + doc_classes = ('Document', 'DynamicEmbeddedDocument', 'EmbeddedDocument', + 'MapReduceDocument') + field_classes = ('DictField', 'DynamicField', 'EmbeddedDocumentField', + 'GenericReferenceField', 'GeoPointField', + 'ReferenceField', 'StringField') + queryset_classes = ('OperationError',) + deref_classes = ('DeReference',) + + if cls_name in doc_classes: + from mongoengine import document as module + import_classes = doc_classes + elif cls_name in field_classes: + from mongoengine import fields as module + import_classes = field_classes + elif cls_name in queryset_classes: + from mongoengine import queryset as module + import_classes = queryset_classes + elif cls_name in deref_classes: + from mongoengine import dereference as module + import_classes = deref_classes + else: + raise ValueError('No import set for: ' % cls_name) + + for cls in import_classes: + _class_registry_cache[cls] = getattr(module, cls) + + return _class_registry_cache.get(cls_name) \ No newline at end of file diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 386dbf4..59cc0a5 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -164,7 +164,7 @@ class DeReference(object): if isinstance(items, (dict, SON)): if '_ref' in items: return self.object_map.get(items['_ref'].id, items) - elif '_types' in items and '_cls' in items: + elif '_cls' in items: doc = get_document(items['_cls'])._from_son(items) doc._data = self._attach_objects(doc._data, depth, doc, None) return doc diff --git a/mongoengine/django/shortcuts.py b/mongoengine/django/shortcuts.py index 637cee1..9cc8370 100644 --- a/mongoengine/django/shortcuts.py +++ b/mongoengine/django/shortcuts.py @@ -1,6 +1,6 @@ from mongoengine.queryset import QuerySet from mongoengine.base import BaseDocument -from mongoengine.base import ValidationError +from mongoengine.errors import ValidationError def _get_queryset(cls): """Inspired by django.shortcuts.*""" diff --git a/mongoengine/document.py b/mongoengine/document.py index 7b3afaf..b1ce13a 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -11,9 +11,9 @@ from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, from queryset import OperationError, NotUniqueError from connection import get_db, DEFAULT_CONNECTION_NAME -__all__ = ['Document', 'EmbeddedDocument', 'DynamicDocument', +__all__ = ('Document', 'EmbeddedDocument', 'DynamicDocument', 'DynamicEmbeddedDocument', 'OperationError', - 'InvalidCollectionError', 'NotUniqueError'] + 'InvalidCollectionError', 'NotUniqueError', 'MapReduceDocument') class InvalidCollectionError(Exception): @@ -28,11 +28,11 @@ class EmbeddedDocument(BaseDocument): A :class:`~mongoengine.EmbeddedDocument` subclass may be itself subclassed, to create a specialised version of the embedded document that will be - stored in the same collection. To facilitate this behaviour, `_cls` and - `_types` fields are added to documents (hidden though the MongoEngine - interface though). To disable this behaviour and remove the dependence on - the presence of `_cls` and `_types`, set :attr:`allow_inheritance` to - ``False`` in the :attr:`meta` dictionary. + stored in the same collection. To facilitate this behaviour a `_cls` + field is added to documents (hidden though the MongoEngine interface). + To disable this behaviour and remove the dependence on the presence of + `_cls` set :attr:`allow_inheritance` to ``False`` in the :attr:`meta` + dictionary. """ # The __metaclass__ attribute is removed by 2to3 when running with Python3 @@ -76,11 +76,11 @@ class Document(BaseDocument): A :class:`~mongoengine.Document` subclass may be itself subclassed, to create a specialised version of the document that will be stored in the - same collection. To facilitate this behaviour, `_cls` and `_types` - fields are added to documents (hidden though the MongoEngine interface - though). To disable this behaviour and remove the dependence on the - presence of `_cls` and `_types`, set :attr:`allow_inheritance` to - ``False`` in the :attr:`meta` dictionary. + same collection. To facilitate this behaviour a `_cls` + field is added to documents (hidden though the MongoEngine interface). + To disable this behaviour and remove the dependence on the presence of + `_cls` set :attr:`allow_inheritance` to ``False`` in the :attr:`meta` + dictionary. A :class:`~mongoengine.Document` may use a **Capped Collection** by specifying :attr:`max_documents` and :attr:`max_size` in the :attr:`meta` @@ -101,10 +101,10 @@ class Document(BaseDocument): production systems where index creation is performed as part of a deployment system. - By default, _types will be added to the start of every index (that + By default, _cls will be added to the start of every index (that doesn't contain a list) if allow_inheritance is True. This can be disabled by either setting types to False on the specific index or - by setting index_types to False on the meta dictionary for the document. + by setting index_cls to False on the meta dictionary for the document. """ # The __metaclass__ attribute is removed by 2to3 when running with Python3 diff --git a/mongoengine/errors.py b/mongoengine/errors.py new file mode 100644 index 0000000..eb72503 --- /dev/null +++ b/mongoengine/errors.py @@ -0,0 +1,124 @@ +from collections import defaultdict + +from .python_support import txt_type + + +__all__ = ('NotRegistered', 'InvalidDocumentError', 'ValidationError') + + +class NotRegistered(Exception): + pass + + +class InvalidDocumentError(Exception): + pass + + +class LookUpError(AttributeError): + pass + + +class DoesNotExist(Exception): + pass + + +class MultipleObjectsReturned(Exception): + pass + + +class InvalidQueryError(Exception): + pass + + +class OperationError(Exception): + pass + + +class NotUniqueError(OperationError): + pass + + +class ValidationError(AssertionError): + """Validation exception. + + May represent an error validating a field or a + document containing fields with validation errors. + + :ivar errors: A dictionary of errors for fields within this + document or list, or None if the error is for an + individual field. + """ + + errors = {} + field_name = None + _message = None + + def __init__(self, message="", **kwargs): + self.errors = kwargs.get('errors', {}) + self.field_name = kwargs.get('field_name') + self.message = message + + def __str__(self): + return txt_type(self.message) + + def __repr__(self): + return '%s(%s,)' % (self.__class__.__name__, self.message) + + def __getattribute__(self, name): + message = super(ValidationError, self).__getattribute__(name) + if name == 'message': + if self.field_name: + message = '%s' % message + if self.errors: + message = '%s(%s)' % (message, self._format_errors()) + return message + + def _get_message(self): + return self._message + + def _set_message(self, message): + self._message = message + + message = property(_get_message, _set_message) + + def to_dict(self): + """Returns a dictionary of all errors within a document + + Keys are field names or list indices and values are the + validation error messages, or a nested dictionary of + errors for an embedded document or list. + """ + + def build_dict(source): + errors_dict = {} + if not source: + return errors_dict + if isinstance(source, dict): + for field_name, error in source.iteritems(): + errors_dict[field_name] = build_dict(error) + elif isinstance(source, ValidationError) and source.errors: + return build_dict(source.errors) + else: + return unicode(source) + return errors_dict + if not self.errors: + return {} + return build_dict(self.errors) + + def _format_errors(self): + """Returns a string listing all errors within a document""" + + def generate_key(value, prefix=''): + if isinstance(value, list): + value = ' '.join([generate_key(k) for k in value]) + if isinstance(value, dict): + value = ' '.join( + [generate_key(v, k) for k, v in value.iteritems()]) + + results = "%s.%s" % (prefix, value) if prefix else value + return results + + error_dict = defaultdict(list) + for k, v in self.to_dict().iteritems(): + error_dict[generate_key(v)].append(k) + return ' '.join(["%s: %s" % (k, v) for k, v in error_dict.iteritems()]) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 01d3fc6..9bcba9f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -12,10 +12,11 @@ from operator import itemgetter import gridfs from bson import Binary, DBRef, SON, ObjectId +from mongoengine.errors import ValidationError from mongoengine.python_support import (PY3, bin_type, txt_type, str_types, StringIO) from base import (BaseField, ComplexBaseField, ObjectIdField, - ValidationError, get_document, BaseDocument) + get_document, BaseDocument) from queryset import DO_NOTHING, QuerySet from document import Document, EmbeddedDocument from connection import get_db, DEFAULT_CONNECTION_NAME @@ -568,9 +569,6 @@ class ListField(ComplexBaseField): Required means it cannot be empty - as the default for ListFields is [] """ - # ListFields cannot be indexed with _types - MongoDB doesn't support this - _index_with_types = False - def __init__(self, field=None, **kwargs): self.field = field kwargs.setdefault('default', lambda: []) diff --git a/mongoengine/queryset/__init__.py b/mongoengine/queryset/__init__.py new file mode 100644 index 0000000..f6feeab --- /dev/null +++ b/mongoengine/queryset/__init__.py @@ -0,0 +1,11 @@ +from mongoengine.errors import (DoesNotExist, MultipleObjectsReturned, + InvalidQueryError, OperationError, + NotUniqueError) +from .field_list import * +from .manager import * +from .queryset import * +from .transform import * +from .visitor import * + +__all__ = (field_list.__all__ + manager.__all__ + queryset.__all__ + + transform.__all__ + visitor.__all__) diff --git a/mongoengine/queryset/field_list.py b/mongoengine/queryset/field_list.py new file mode 100644 index 0000000..1c825fa --- /dev/null +++ b/mongoengine/queryset/field_list.py @@ -0,0 +1,51 @@ + +__all__ = ('QueryFieldList',) + + +class QueryFieldList(object): + """Object that handles combinations of .only() and .exclude() calls""" + ONLY = 1 + EXCLUDE = 0 + + def __init__(self, fields=[], value=ONLY, always_include=[]): + self.value = value + self.fields = set(fields) + self.always_include = set(always_include) + self._id = None + + def __add__(self, f): + if not self.fields: + self.fields = f.fields + self.value = f.value + elif self.value is self.ONLY and f.value is self.ONLY: + self.fields = self.fields.intersection(f.fields) + elif self.value is self.EXCLUDE and f.value is self.EXCLUDE: + self.fields = self.fields.union(f.fields) + elif self.value is self.ONLY and f.value is self.EXCLUDE: + self.fields -= f.fields + elif self.value is self.EXCLUDE and f.value is self.ONLY: + self.value = self.ONLY + self.fields = f.fields - self.fields + + if '_id' in f.fields: + self._id = f.value + + if self.always_include: + if self.value is self.ONLY and self.fields: + self.fields = self.fields.union(self.always_include) + else: + self.fields -= self.always_include + return self + + def __nonzero__(self): + return bool(self.fields) + + def as_dict(self): + field_list = dict((field, self.value) for field in self.fields) + if self._id is not None: + field_list['_id'] = self._id + return field_list + + def reset(self): + self.fields = set([]) + self.value = self.ONLY diff --git a/mongoengine/queryset/manager.py b/mongoengine/queryset/manager.py new file mode 100644 index 0000000..7376e3c --- /dev/null +++ b/mongoengine/queryset/manager.py @@ -0,0 +1,61 @@ +from functools import partial +from .queryset import QuerySet + +__all__ = ('queryset_manager', 'QuerySetManager') + + +class QuerySetManager(object): + """ + The default QuerySet Manager. + + Custom QuerySet Manager functions can extend this class and users can + add extra queryset functionality. Any custom manager methods must accept a + :class:`~mongoengine.Document` class as its first argument, and a + :class:`~mongoengine.queryset.QuerySet` as its second argument. + + The method function should return a :class:`~mongoengine.queryset.QuerySet` + , probably the same one that was passed in, but modified in some way. + """ + + get_queryset = None + + def __init__(self, queryset_func=None): + if queryset_func: + self.get_queryset = queryset_func + self._collections = {} + + def __get__(self, instance, owner): + """Descriptor for instantiating a new QuerySet object when + Document.objects is accessed. + """ + if instance is not None: + # Document class being used rather than a document object + return self + + # owner is the document that contains the QuerySetManager + queryset_class = owner._meta.get('queryset_class') or QuerySet + queryset = queryset_class(owner, owner._get_collection()) + if self.get_queryset: + arg_count = self.get_queryset.func_code.co_argcount + if arg_count == 1: + queryset = self.get_queryset(queryset) + elif arg_count == 2: + queryset = self.get_queryset(owner, queryset) + else: + queryset = partial(self.get_queryset, owner, queryset) + return queryset + + +def queryset_manager(func): + """Decorator that allows you to define custom QuerySet managers on + :class:`~mongoengine.Document` classes. The manager must be a function that + accepts a :class:`~mongoengine.Document` class as its first argument, and a + :class:`~mongoengine.queryset.QuerySet` as its second argument. The method + function should return a :class:`~mongoengine.queryset.QuerySet`, probably + the same one that was passed in, but modified in some way. + """ + if func.func_code.co_argcount == 1: + import warnings + msg = 'Methods decorated with queryset_manager should take 2 arguments' + warnings.warn(msg, DeprecationWarning) + return QuerySetManager(func) diff --git a/mongoengine/queryset.py b/mongoengine/queryset/queryset.py similarity index 59% rename from mongoengine/queryset.py rename to mongoengine/queryset/queryset.py index c774322..5108066 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -4,20 +4,21 @@ import copy import itertools import operator -from collections import defaultdict -from functools import partial - -from mongoengine.python_support import product, reduce - import pymongo from bson.code import Code from mongoengine import signals +from mongoengine.common import _import_class +from mongoengine.errors import (OperationError, NotUniqueError, + InvalidQueryError) -__all__ = ['queryset_manager', 'Q', 'InvalidQueryError', - 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL'] +from . import transform +from .field_list import QueryFieldList +from .visitor import Q +__all__ = ('QuerySet', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL') + # The maximum number of items to display in a QuerySet.__repr__ REPR_OUTPUT_SIZE = 20 @@ -28,308 +29,9 @@ CASCADE = 2 DENY = 3 PULL = 4 - -class DoesNotExist(Exception): - pass - - -class MultipleObjectsReturned(Exception): - pass - - -class InvalidQueryError(Exception): - pass - - -class OperationError(Exception): - pass - - -class NotUniqueError(OperationError): - pass - - RE_TYPE = type(re.compile('')) -class QNodeVisitor(object): - """Base visitor class for visiting Q-object nodes in a query tree. - """ - - def visit_combination(self, combination): - """Called by QCombination objects. - """ - return combination - - def visit_query(self, query): - """Called by (New)Q objects. - """ - return query - - -class SimplificationVisitor(QNodeVisitor): - """Simplifies query trees by combinging unnecessary 'and' connection nodes - into a single Q-object. - """ - - def visit_combination(self, combination): - if combination.operation == combination.AND: - # The simplification only applies to 'simple' queries - if all(isinstance(node, Q) for node in combination.children): - queries = [node.query for node in combination.children] - return Q(**self._query_conjunction(queries)) - return combination - - def _query_conjunction(self, queries): - """Merges query dicts - effectively &ing them together. - """ - query_ops = set() - combined_query = {} - for query in queries: - ops = set(query.keys()) - # Make sure that the same operation isn't applied more than once - # to a single field - intersection = ops.intersection(query_ops) - if intersection: - msg = 'Duplicate query conditions: ' - raise InvalidQueryError(msg + ', '.join(intersection)) - - query_ops.update(ops) - combined_query.update(copy.deepcopy(query)) - return combined_query - - -class QueryTreeTransformerVisitor(QNodeVisitor): - """Transforms the query tree in to a form that may be used with MongoDB. - """ - - def visit_combination(self, combination): - if combination.operation == combination.AND: - # MongoDB doesn't allow us to have too many $or operations in our - # queries, so the aim is to move the ORs up the tree to one - # 'master' $or. Firstly, we must find all the necessary parts (part - # of an AND combination or just standard Q object), and store them - # separately from the OR parts. - or_groups = [] - and_parts = [] - for node in combination.children: - if isinstance(node, QCombination): - if node.operation == node.OR: - # Any of the children in an $or component may cause - # the query to succeed - or_groups.append(node.children) - elif node.operation == node.AND: - and_parts.append(node) - elif isinstance(node, Q): - and_parts.append(node) - - # Now we combine the parts into a usable query. AND together all of - # the necessary parts. Then for each $or part, create a new query - # that ANDs the necessary part with the $or part. - clauses = [] - for or_group in product(*or_groups): - q_object = reduce(lambda a, b: a & b, and_parts, Q()) - q_object = reduce(lambda a, b: a & b, or_group, q_object) - clauses.append(q_object) - # Finally, $or the generated clauses in to one query. Each of the - # clauses is sufficient for the query to succeed. - return reduce(lambda a, b: a | b, clauses, Q()) - - if combination.operation == combination.OR: - children = [] - # Crush any nested ORs in to this combination as MongoDB doesn't - # support nested $or operations - for node in combination.children: - if (isinstance(node, QCombination) and - node.operation == combination.OR): - children += node.children - else: - children.append(node) - combination.children = children - - return combination - - -class QueryCompilerVisitor(QNodeVisitor): - """Compiles the nodes in a query tree to a PyMongo-compatible query - dictionary. - """ - - def __init__(self, document): - self.document = document - - def visit_combination(self, combination): - if combination.operation == combination.OR: - return {'$or': combination.children} - elif combination.operation == combination.AND: - return self._mongo_query_conjunction(combination.children) - return combination - - def visit_query(self, query): - return QuerySet._transform_query(self.document, **query.query) - - def _mongo_query_conjunction(self, queries): - """Merges Mongo query dicts - effectively &ing them together. - """ - combined_query = {} - for query in queries: - for field, ops in query.items(): - if field not in combined_query: - combined_query[field] = ops - else: - # The field is already present in the query the only way - # we can merge is if both the existing value and the new - # value are operation dicts, reject anything else - if (not isinstance(combined_query[field], dict) or - not isinstance(ops, dict)): - message = 'Conflicting values for ' + field - raise InvalidQueryError(message) - - current_ops = set(combined_query[field].keys()) - new_ops = set(ops.keys()) - # Make sure that the same operation isn't applied more than - # once to a single field - intersection = current_ops.intersection(new_ops) - if intersection: - msg = 'Duplicate query conditions: ' - raise InvalidQueryError(msg + ', '.join(intersection)) - - # Right! We've got two non-overlapping dicts of operations! - combined_query[field].update(copy.deepcopy(ops)) - return combined_query - - -class QNode(object): - """Base class for nodes in query trees. - """ - - AND = 0 - OR = 1 - - def to_query(self, document): - query = self.accept(SimplificationVisitor()) - query = query.accept(QueryTreeTransformerVisitor()) - query = query.accept(QueryCompilerVisitor(document)) - return query - - def accept(self, visitor): - raise NotImplementedError - - def _combine(self, other, operation): - """Combine this node with another node into a QCombination object. - """ - if getattr(other, 'empty', True): - return self - - if self.empty: - return other - - return QCombination(operation, [self, other]) - - @property - def empty(self): - return False - - def __or__(self, other): - return self._combine(other, self.OR) - - def __and__(self, other): - return self._combine(other, self.AND) - - -class QCombination(QNode): - """Represents the combination of several conditions by a given logical - operator. - """ - - def __init__(self, operation, children): - self.operation = operation - self.children = [] - for node in children: - # If the child is a combination of the same type, we can merge its - # children directly into this combinations children - if isinstance(node, QCombination) and node.operation == operation: - self.children += node.children - else: - self.children.append(node) - - def accept(self, visitor): - for i in range(len(self.children)): - if isinstance(self.children[i], QNode): - self.children[i] = self.children[i].accept(visitor) - - return visitor.visit_combination(self) - - @property - def empty(self): - return not bool(self.children) - - -class Q(QNode): - """A simple query object, used in a query tree to build up more complex - query structures. - """ - - def __init__(self, **query): - self.query = query - - def accept(self, visitor): - return visitor.visit_query(self) - - @property - def empty(self): - return not bool(self.query) - - -class QueryFieldList(object): - """Object that handles combinations of .only() and .exclude() calls""" - ONLY = 1 - EXCLUDE = 0 - - def __init__(self, fields=[], value=ONLY, always_include=[]): - self.value = value - self.fields = set(fields) - self.always_include = set(always_include) - self._id = None - - def as_dict(self): - field_list = dict((field, self.value) for field in self.fields) - if self._id is not None: - field_list['_id'] = self._id - return field_list - - def __add__(self, f): - if not self.fields: - self.fields = f.fields - self.value = f.value - elif self.value is self.ONLY and f.value is self.ONLY: - self.fields = self.fields.intersection(f.fields) - elif self.value is self.EXCLUDE and f.value is self.EXCLUDE: - self.fields = self.fields.union(f.fields) - elif self.value is self.ONLY and f.value is self.EXCLUDE: - self.fields -= f.fields - elif self.value is self.EXCLUDE and f.value is self.ONLY: - self.value = self.ONLY - self.fields = f.fields - self.fields - - if '_id' in f.fields: - self._id = f.value - - if self.always_include: - if self.value is self.ONLY and self.fields: - self.fields = self.fields.union(self.always_include) - else: - self.fields -= self.always_include - return self - - def reset(self): - self.fields = set([]) - self.value = self.ONLY - - def __nonzero__(self): - return bool(self.fields) - - class QuerySet(object): """A set of results returned from a query. Wraps a MongoDB cursor, providing :class:`~mongoengine.Document` objects as the results. @@ -357,7 +59,7 @@ class QuerySet(object): # If inheritance is allowed, only return instances and instances of # subclasses of the class being used if document._meta.get('allow_inheritance') != False: - self._initial_query = {'_types': self._document._class_name} + self._initial_query = {"_cls": {"$in": self._document._subclasses}} self._loaded_fields = QueryFieldList(always_include=['_cls']) self._cursor_obj = None self._limit = None @@ -397,7 +99,7 @@ class QuerySet(object): construct a multi-field index); keys may be prefixed with a **+** or a **-** to determine the index ordering """ - index_spec = QuerySet._build_index_spec(self._document, key_or_list) + index_spec = self._document._build_index_spec(key_or_list) index_spec = index_spec.copy() fields = index_spec.pop('fields') index_spec['drop_dups'] = drop_dups @@ -448,26 +150,26 @@ class QuerySet(object): background = self._document._meta.get('index_background', False) drop_dups = self._document._meta.get('index_drop_dups', False) index_opts = self._document._meta.get('index_opts') or {} - index_types = self._document._meta.get('index_types', True) + index_cls = self._document._meta.get('index_cls', True) # determine if an index which we are creating includes - # _type as its first field; if so, we can avoid creating - # an extra index on _type, as mongodb will use the existing - # index to service queries against _type - types_indexed = False + # _cls as its first field; if so, we can avoid creating + # an extra index on _cls, as mongodb will use the existing + # index to service queries against _cls + cls_indexed = False - def includes_types(fields): + def includes_cls(fields): first_field = None if len(fields): if isinstance(fields[0], basestring): first_field = fields[0] elif isinstance(fields[0], (list, tuple)) and len(fields[0]): first_field = fields[0][0] - return first_field == '_types' + return first_field == '_cls' # Ensure indexes created by uniqueness constraints for index in self._document._meta['unique_indexes']: - types_indexed = types_indexed or includes_types(index) + cls_indexed = cls_indexed or includes_cls(index) self._collection.ensure_index(index, unique=True, background=background, drop_dups=drop_dups, **index_opts) @@ -477,16 +179,16 @@ class QuerySet(object): for spec in index_spec: spec = spec.copy() fields = spec.pop('fields') - types_indexed = types_indexed or includes_types(fields) + cls_indexed = cls_indexed or includes_cls(fields) opts = index_opts.copy() opts.update(spec) self._collection.ensure_index(fields, background=background, **opts) - # If _types is being used (for polymorphism), it needs an index, - # only if another index doesn't begin with _types - if index_types and '_types' in self._query and not types_indexed: - self._collection.ensure_index('_types', + # If _cls is being used (for polymorphism), it needs an index, + # only if another index doesn't begin with _cls + if index_cls and '_cls' in self._query and not cls_indexed: + self._collection.ensure_index('_cls', background=background, **index_opts) # Add geo indicies @@ -495,79 +197,14 @@ class QuerySet(object): self._collection.ensure_index(index_spec, background=background, **index_opts) - @classmethod - def _build_index_spec(cls, doc_cls, spec): - """Build a PyMongo index spec from a MongoEngine index spec. - """ - if isinstance(spec, basestring): - spec = {'fields': [spec]} - elif isinstance(spec, (list, tuple)): - spec = {'fields': list(spec)} - elif isinstance(spec, dict): - spec = dict(spec) - - index_list = [] - direction = None - - allow_inheritance = doc_cls._meta.get('allow_inheritance') != False - - # If sparse - dont include types - use_types = allow_inheritance and not spec.get('sparse', False) - - for key in spec['fields']: - # If inherited spec continue - if isinstance(key, (list, tuple)): - continue - - # Get ASCENDING direction from +, DESCENDING from -, and GEO2D from * - direction = pymongo.ASCENDING - if key.startswith("-"): - direction = pymongo.DESCENDING - elif key.startswith("*"): - direction = pymongo.GEO2D - if key.startswith(("+", "-", "*")): - key = key[1:] - - # Use real field name, do it manually because we need field - # objects for the next part (list field checking) - parts = key.split('.') - if parts in (['pk'], ['id'], ['_id']): - key = '_id' - fields = [] - else: - fields = QuerySet._lookup_field(doc_cls, parts) - parts = [field if field == '_id' else field.db_field - for field in fields] - key = '.'.join(parts) - index_list.append((key, direction)) - - # Check if a list field is being used, don't use _types if it is - if use_types and not all(f._index_with_types for f in fields): - use_types = False - - # If _types is being used, prepend it to every specified index - index_types = doc_cls._meta.get('index_types', True) - - if (spec.get('types', index_types) and use_types - and direction is not pymongo.GEO2D): - index_list.insert(0, ('_types', 1)) - - spec['fields'] = index_list - if spec.get('sparse', False) and len(spec['fields']) > 1: - raise ValueError( - 'Sparse indexes can only have one field in them. ' - 'See https://jira.mongodb.org/browse/SERVER-2193') - - return spec - @classmethod def _reset_already_indexed(cls, document=None): - """Helper to reset already indexed, can be useful for testing purposes""" + """Helper to reset already indexed, can be useful for testing purposes + """ if document: cls.__already_indexed.discard(document) cls.__already_indexed.clear() - @property def _collection(self): """Property that returns the collection object. This allows us to @@ -624,195 +261,12 @@ class QuerySet(object): self._cursor_obj.hint(self._hint) return self._cursor_obj - @classmethod - def _lookup_field(cls, document, parts): - """Lookup a field based on its attribute and return a list containing - the field's parents and the field. - """ - if not isinstance(parts, (list, tuple)): - parts = [parts] - fields = [] - field = None - - for field_name in parts: - # Handle ListField indexing: - if field_name.isdigit(): - try: - new_field = field.field - except AttributeError, err: - raise InvalidQueryError( - "Can't use index on unsubscriptable field (%s)" % err) - fields.append(field_name) - continue - - if field is None: - # Look up first field from the document - if field_name == 'pk': - # Deal with "primary key" alias - field_name = document._meta['id_field'] - if field_name in document._fields: - field = document._fields[field_name] - elif document._dynamic: - from fields import DynamicField - field = DynamicField(db_field=field_name) - else: - raise InvalidQueryError('Cannot resolve field "%s"' - % field_name) - else: - from mongoengine.fields import ReferenceField, GenericReferenceField - if isinstance(field, (ReferenceField, GenericReferenceField)): - raise InvalidQueryError('Cannot perform join in mongoDB: %s' % '__'.join(parts)) - if hasattr(getattr(field, 'field', None), 'lookup_member'): - new_field = field.field.lookup_member(field_name) - else: - # Look up subfield on the previous field - new_field = field.lookup_member(field_name) - from base import ComplexBaseField - if not new_field and isinstance(field, ComplexBaseField): - fields.append(field_name) - continue - elif not new_field: - raise InvalidQueryError('Cannot resolve field "%s"' - % field_name) - field = new_field # update field to the new field type - fields.append(field) - return fields - - @classmethod - def _translate_field_name(cls, doc_cls, field, sep='.'): - """Translate a field attribute name to a database field name. - """ - parts = field.split(sep) - parts = [f.db_field for f in QuerySet._lookup_field(doc_cls, parts)] - return '.'.join(parts) - - @classmethod - def _transform_query(cls, _doc_cls=None, _field_operation=False, **query): - """Transform a query from Django-style format to Mongo format. - """ - operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', - 'all', 'size', 'exists', 'not'] - geo_operators = ['within_distance', 'within_spherical_distance', 'within_box', 'within_polygon', 'near', 'near_sphere'] - match_operators = ['contains', 'icontains', 'startswith', - 'istartswith', 'endswith', 'iendswith', - 'exact', 'iexact'] - custom_operators = ['match'] - - mongo_query = {} - merge_query = defaultdict(list) - for key, value in query.items(): - if key == "__raw__": - mongo_query.update(value) - continue - - parts = key.split('__') - indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()] - parts = [part for part in parts if not part.isdigit()] - # Check for an operator and transform to mongo-style if there is - op = None - if parts[-1] in operators + match_operators + geo_operators + custom_operators: - op = parts.pop() - - negate = False - if parts[-1] == 'not': - parts.pop() - negate = True - - if _doc_cls: - # Switch field names to proper names [set in Field(name='foo')] - fields = QuerySet._lookup_field(_doc_cls, parts) - parts = [] - - cleaned_fields = [] - for field in fields: - append_field = True - if isinstance(field, basestring): - parts.append(field) - append_field = False - else: - parts.append(field.db_field) - if append_field: - cleaned_fields.append(field) - - # Convert value to proper value - field = cleaned_fields[-1] - - singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte', 'not'] - singular_ops += match_operators - if op in singular_ops: - if isinstance(field, basestring): - if op in match_operators and isinstance(value, basestring): - from mongoengine import StringField - value = StringField.prepare_query_value(op, value) - else: - value = field - else: - value = field.prepare_query_value(op, value) - elif op in ('in', 'nin', 'all', 'near'): - # 'in', 'nin' and 'all' require a list of values - value = [field.prepare_query_value(op, v) for v in value] - - # if op and op not in match_operators: - if op: - if op in geo_operators: - if op == "within_distance": - value = {'$within': {'$center': value}} - elif op == "within_spherical_distance": - value = {'$within': {'$centerSphere': value}} - elif op == "within_polygon": - value = {'$within': {'$polygon': value}} - elif op == "near": - value = {'$near': value} - elif op == "near_sphere": - value = {'$nearSphere': value} - elif op == 'within_box': - value = {'$within': {'$box': value}} - else: - raise NotImplementedError("Geo method '%s' has not " - "been implemented" % op) - elif op in custom_operators: - if op == 'match': - value = {"$elemMatch": value} - else: - NotImplementedError("Custom method '%s' has not " - "been implemented" % op) - elif op not in match_operators: - value = {'$' + op: value} - - if negate: - value = {'$not': value} - - for i, part in indices: - parts.insert(i, part) - key = '.'.join(parts) - if op is None or key not in mongo_query: - mongo_query[key] = value - elif key in mongo_query: - if key in mongo_query and isinstance(mongo_query[key], dict): - mongo_query[key].update(value) - else: - # Store for manually merging later - merge_query[key].append(value) - - # The queryset has been filter in such a way we must manually merge - for k, v in merge_query.items(): - merge_query[k].append(mongo_query[k]) - del mongo_query[k] - if isinstance(v, list): - value = [{k:val} for val in v] - if '$and' in mongo_query.keys(): - mongo_query['$and'].append(value) - else: - mongo_query['$and'] = value - - return mongo_query - def get(self, *q_objs, **query): """Retrieve the the matching object raising :class:`~mongoengine.queryset.MultipleObjectsReturned` or - `DocumentName.MultipleObjectsReturned` exception if multiple results and - :class:`~mongoengine.queryset.DoesNotExist` or `DocumentName.DoesNotExist` - if no results are found. + `DocumentName.MultipleObjectsReturned` exception if multiple results + and :class:`~mongoengine.queryset.DoesNotExist` or + `DocumentName.DoesNotExist` if no results are found. .. versionadded:: 0.3 """ @@ -910,7 +364,7 @@ class QuerySet(object): .. versionadded:: 0.5 """ - from document import Document + Document = _import_class('Document') if not write_options: write_options = {} @@ -1064,7 +518,7 @@ class QuerySet(object): .. versionadded:: 0.3 """ - from document import MapReduceDocument + MapReduceDocument = _import_class('MapReduceDocument') if not hasattr(self._collection, "map_reduce"): raise NotImplementedError("Requires MongoDB >= 1.7.1") @@ -1267,14 +721,16 @@ class QuerySet(object): .. versionadded:: 0.5 """ - self._loaded_fields = QueryFieldList(always_include=self._loaded_fields.always_include) + self._loaded_fields = QueryFieldList( + always_include=self._loaded_fields.always_include) return self def _fields_to_dbfields(self, fields): """Translate fields paths to its db equivalents""" ret = [] for field in fields: - field = ".".join(f.db_field for f in QuerySet._lookup_field(self._document, field.split('.'))) + field = ".".join(f.db_field for f in + self._document._lookup_field(field.split('.'))) ret.append(field) return ret @@ -1288,7 +744,8 @@ class QuerySet(object): """ key_list = [] for key in keys: - if not key: continue + if not key: + continue direction = pymongo.ASCENDING if key[0] == '-': direction = pymongo.DESCENDING @@ -1296,7 +753,7 @@ class QuerySet(object): key = key[1:] key = key.replace('__', '.') try: - key = QuerySet._translate_field_name(self._document, key) + key = self._document._translate_field_name(key) except: pass key_list.append((key, direction)) @@ -1389,107 +846,6 @@ class QuerySet(object): self._collection.remove(self._query, safe=safe) - @classmethod - def _transform_update(cls, _doc_cls=None, **update): - """Transform an update spec from Django-style format to Mongo format. - """ - operators = ['set', 'unset', 'inc', 'dec', 'pop', 'push', 'push_all', - 'pull', 'pull_all', 'add_to_set'] - match_operators = ['ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', - 'all', 'size', 'exists', 'not'] - - mongo_update = {} - for key, value in update.items(): - if key == "__raw__": - mongo_update.update(value) - continue - parts = key.split('__') - # Check for an operator and transform to mongo-style if there is - op = None - if parts[0] in operators: - op = parts.pop(0) - # Convert Pythonic names to Mongo equivalents - if op in ('push_all', 'pull_all'): - op = op.replace('_all', 'All') - elif op == 'dec': - # Support decrement by flipping a positive value's sign - # and using 'inc' - op = 'inc' - if value > 0: - value = -value - elif op == 'add_to_set': - op = op.replace('_to_set', 'ToSet') - - match = None - if parts[-1] in match_operators: - match = parts.pop() - - if _doc_cls: - # Switch field names to proper names [set in Field(name='foo')] - fields = QuerySet._lookup_field(_doc_cls, parts) - parts = [] - - cleaned_fields = [] - for field in fields: - append_field = True - if isinstance(field, basestring): - # Convert the S operator to $ - if field == 'S': - field = '$' - parts.append(field) - append_field = False - else: - parts.append(field.db_field) - if append_field: - cleaned_fields.append(field) - - # Convert value to proper value - field = cleaned_fields[-1] - - if op in (None, 'set', 'push', 'pull'): - if field.required or value is not None: - value = field.prepare_query_value(op, value) - elif op in ('pushAll', 'pullAll'): - value = [field.prepare_query_value(op, v) for v in value] - elif op == 'addToSet': - if isinstance(value, (list, tuple, set)): - value = [field.prepare_query_value(op, v) for v in value] - elif field.required or value is not None: - value = field.prepare_query_value(op, value) - - if match: - match = '$' + match - value = {match: value} - - key = '.'.join(parts) - - if not op: - raise InvalidQueryError("Updates must supply an operation " - "eg: set__FIELD=value") - - if 'pull' in op and '.' in key: - # Dot operators don't work on pull operations - # it uses nested dict syntax - if op == 'pullAll': - raise InvalidQueryError("pullAll operations only support " - "a single field depth") - - parts.reverse() - for key in parts: - value = {key: value} - elif op == 'addToSet' and isinstance(value, list): - value = {key: {"$each": value}} - else: - value = {key: value} - key = '$' + op - - if key not in mongo_update: - mongo_update[key] = value - elif key in mongo_update and isinstance(mongo_update[key], dict): - mongo_update[key].update(value) - - return mongo_update - def update(self, safe_update=True, upsert=False, multi=True, write_options=None, **update): """Perform an atomic update on the fields matched by the query. When ``safe_update`` is used, the number of affected documents is returned. @@ -1506,14 +862,9 @@ class QuerySet(object): if not write_options: write_options = {} - update = QuerySet._transform_update(self._document, **update) + update = transform.update(self._document, **update) query = self._query - # SERVER-5247 hack - remove_types = "_types" in query and ".$." in unicode(update) - if remove_types: - del query["_types"] - try: ret = self._collection.update(query, update, multi=multi, upsert=upsert, safe=safe_update, @@ -1537,30 +888,8 @@ class QuerySet(object): .. versionadded:: 0.2 """ - if not update: - raise OperationError("No update parameters, would remove data") - - if not write_options: - write_options = {} - update = QuerySet._transform_update(self._document, **update) - query = self._query - - # SERVER-5247 hack - remove_types = "_types" in query and ".$." in unicode(update) - if remove_types: - del query["_types"] - - try: - # Explicitly provide 'multi=False' to newer versions of PyMongo - # as the default may change to 'True' - ret = self._collection.update(query, update, multi=False, - upsert=upsert, safe=safe_update, - **write_options) - - if ret is not None and 'n' in ret: - return ret['n'] - except pymongo.errors.OperationFailure, e: - raise OperationError(u'Update failed [%s]' % unicode(e)) + return self.update(safe_update=True, upsert=False, multi=False, + write_options=None, **update) def __iter__(self): self.rewind() @@ -1611,14 +940,14 @@ class QuerySet(object): def field_sub(match): # Extract just the field name, and look up the field objects field_name = match.group(1).split('.') - fields = QuerySet._lookup_field(self._document, field_name) + fields = self._document._lookup_field(field_name) # Substitute the correct name for the field into the javascript return u'["%s"]' % fields[-1].db_field def field_path_sub(match): # Extract just the field name, and look up the field objects field_name = match.group(1).split('.') - fields = QuerySet._lookup_field(self._document, field_name) + fields = self._document._lookup_field(field_name) # Substitute the correct name for the field into the javascript return ".".join([f.db_field for f in fields]) @@ -1650,8 +979,7 @@ class QuerySet(object): """ code = self._sub_js_fields(code) - fields = [QuerySet._translate_field_name(self._document, f) - for f in fields] + fields = [self._document._translate_field_name(f) for f in fields] collection = self._document._get_collection_name() scope = { @@ -1925,63 +1253,5 @@ class QuerySet(object): @property def _dereference(self): if not self.__dereference: - from dereference import DeReference - self.__dereference = DeReference() # Cached + self.__dereference = _import_class('DeReference')() return self.__dereference - - -class QuerySetManager(object): - """ - The default QuerySet Manager. - - Custom QuerySet Manager functions can extend this class and users can - add extra queryset functionality. Any custom manager methods must accept a - :class:`~mongoengine.Document` class as its first argument, and a - :class:`~mongoengine.queryset.QuerySet` as its second argument. - - The method function should return a :class:`~mongoengine.queryset.QuerySet` - , probably the same one that was passed in, but modified in some way. - """ - - get_queryset = None - - def __init__(self, queryset_func=None): - if queryset_func: - self.get_queryset = queryset_func - self._collections = {} - - def __get__(self, instance, owner): - """Descriptor for instantiating a new QuerySet object when - Document.objects is accessed. - """ - if instance is not None: - # Document class being used rather than a document object - return self - - # owner is the document that contains the QuerySetManager - queryset_class = owner._meta.get('queryset_class') or QuerySet - queryset = queryset_class(owner, owner._get_collection()) - if self.get_queryset: - arg_count = self.get_queryset.func_code.co_argcount - if arg_count == 1: - queryset = self.get_queryset(queryset) - elif arg_count == 2: - queryset = self.get_queryset(owner, queryset) - else: - queryset = partial(self.get_queryset, owner, queryset) - return queryset - - -def queryset_manager(func): - """Decorator that allows you to define custom QuerySet managers on - :class:`~mongoengine.Document` classes. The manager must be a function that - accepts a :class:`~mongoengine.Document` class as its first argument, and a - :class:`~mongoengine.queryset.QuerySet` as its second argument. The method - function should return a :class:`~mongoengine.queryset.QuerySet`, probably - the same one that was passed in, but modified in some way. - """ - if func.func_code.co_argcount == 1: - import warnings - msg = 'Methods decorated with queryset_manager should take 2 arguments' - warnings.warn(msg, DeprecationWarning) - return QuerySetManager(func) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py new file mode 100644 index 0000000..8ee84ee --- /dev/null +++ b/mongoengine/queryset/transform.py @@ -0,0 +1,237 @@ +from collections import defaultdict + +from mongoengine.common import _import_class +from mongoengine.errors import InvalidQueryError, LookUpError + +__all__ = ('query', 'update') + + +COMPARISON_OPERATORS = ('ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', + 'all', 'size', 'exists', 'not') +GEO_OPERATORS = ('within_distance', 'within_spherical_distance', + 'within_box', 'within_polygon', 'near', 'near_sphere') +STRING_OPERATORS = ('contains', 'icontains', 'startswith', + 'istartswith', 'endswith', 'iendswith', + 'exact', 'iexact') +CUSTOM_OPERATORS = ('match',) +MATCH_OPERATORS = (COMPARISON_OPERATORS + GEO_OPERATORS + + STRING_OPERATORS + CUSTOM_OPERATORS) + +UPDATE_OPERATORS = ('set', 'unset', 'inc', 'dec', 'pop', 'push', + 'push_all', 'pull', 'pull_all', 'add_to_set') + + +def query(_doc_cls=None, _field_operation=False, **query): + """Transform a query from Django-style format to Mongo format. + """ + mongo_query = {} + merge_query = defaultdict(list) + for key, value in query.items(): + if key == "__raw__": + mongo_query.update(value) + continue + + parts = key.split('__') + indices = [(i, p) for i, p in enumerate(parts) if p.isdigit()] + parts = [part for part in parts if not part.isdigit()] + # Check for an operator and transform to mongo-style if there is + op = None + if parts[-1] in MATCH_OPERATORS: + op = parts.pop() + + negate = False + if parts[-1] == 'not': + parts.pop() + negate = True + + if _doc_cls: + # Switch field names to proper names [set in Field(name='foo')] + try: + fields = _doc_cls._lookup_field(parts) + except Exception, e: + raise InvalidQueryError(e) + parts = [] + + cleaned_fields = [] + for field in fields: + append_field = True + if isinstance(field, basestring): + parts.append(field) + append_field = False + else: + parts.append(field.db_field) + if append_field: + cleaned_fields.append(field) + + # Convert value to proper value + field = cleaned_fields[-1] + + singular_ops = [None, 'ne', 'gt', 'gte', 'lt', 'lte', 'not'] + singular_ops += STRING_OPERATORS + if op in singular_ops: + if isinstance(field, basestring): + if (op in STRING_OPERATORS and + isinstance(value, basestring)): + StringField = _import_class('StringField') + value = StringField.prepare_query_value(op, value) + else: + value = field + else: + value = field.prepare_query_value(op, value) + elif op in ('in', 'nin', 'all', 'near'): + # 'in', 'nin' and 'all' require a list of values + value = [field.prepare_query_value(op, v) for v in value] + + # if op and op not in COMPARISON_OPERATORS: + if op: + if op in GEO_OPERATORS: + if op == "within_distance": + value = {'$within': {'$center': value}} + elif op == "within_spherical_distance": + value = {'$within': {'$centerSphere': value}} + elif op == "within_polygon": + value = {'$within': {'$polygon': value}} + elif op == "near": + value = {'$near': value} + elif op == "near_sphere": + value = {'$nearSphere': value} + elif op == 'within_box': + value = {'$within': {'$box': value}} + else: + raise NotImplementedError("Geo method '%s' has not " + "been implemented" % op) + elif op in CUSTOM_OPERATORS: + if op == 'match': + value = {"$elemMatch": value} + else: + NotImplementedError("Custom method '%s' has not " + "been implemented" % op) + elif op not in STRING_OPERATORS: + value = {'$' + op: value} + + if negate: + value = {'$not': value} + + for i, part in indices: + parts.insert(i, part) + key = '.'.join(parts) + if op is None or key not in mongo_query: + mongo_query[key] = value + elif key in mongo_query: + if key in mongo_query and isinstance(mongo_query[key], dict): + mongo_query[key].update(value) + else: + # Store for manually merging later + merge_query[key].append(value) + + # The queryset has been filter in such a way we must manually merge + for k, v in merge_query.items(): + merge_query[k].append(mongo_query[k]) + del mongo_query[k] + if isinstance(v, list): + value = [{k:val} for val in v] + if '$and' in mongo_query.keys(): + mongo_query['$and'].append(value) + else: + mongo_query['$and'] = value + + return mongo_query + + +def update(_doc_cls=None, **update): + """Transform an update spec from Django-style format to Mongo format. + """ + mongo_update = {} + for key, value in update.items(): + if key == "__raw__": + mongo_update.update(value) + continue + parts = key.split('__') + # Check for an operator and transform to mongo-style if there is + op = None + if parts[0] in UPDATE_OPERATORS: + op = parts.pop(0) + # Convert Pythonic names to Mongo equivalents + if op in ('push_all', 'pull_all'): + op = op.replace('_all', 'All') + elif op == 'dec': + # Support decrement by flipping a positive value's sign + # and using 'inc' + op = 'inc' + if value > 0: + value = -value + elif op == 'add_to_set': + op = op.replace('_to_set', 'ToSet') + + match = None + if parts[-1] in COMPARISON_OPERATORS: + match = parts.pop() + + if _doc_cls: + # Switch field names to proper names [set in Field(name='foo')] + try: + fields = _doc_cls._lookup_field(parts) + except Exception, e: + raise InvalidQueryError(e) + parts = [] + + cleaned_fields = [] + for field in fields: + append_field = True + if isinstance(field, basestring): + # Convert the S operator to $ + if field == 'S': + field = '$' + parts.append(field) + append_field = False + else: + parts.append(field.db_field) + if append_field: + cleaned_fields.append(field) + + # Convert value to proper value + field = cleaned_fields[-1] + + if op in (None, 'set', 'push', 'pull'): + if field.required or value is not None: + value = field.prepare_query_value(op, value) + elif op in ('pushAll', 'pullAll'): + value = [field.prepare_query_value(op, v) for v in value] + elif op == 'addToSet': + if isinstance(value, (list, tuple, set)): + value = [field.prepare_query_value(op, v) for v in value] + elif field.required or value is not None: + value = field.prepare_query_value(op, value) + + if match: + match = '$' + match + value = {match: value} + + key = '.'.join(parts) + + if not op: + raise InvalidQueryError("Updates must supply an operation " + "eg: set__FIELD=value") + + if 'pull' in op and '.' in key: + # Dot operators don't work on pull operations + # it uses nested dict syntax + if op == 'pullAll': + raise InvalidQueryError("pullAll operations only support " + "a single field depth") + + parts.reverse() + for key in parts: + value = {key: value} + elif op == 'addToSet' and isinstance(value, list): + value = {key: {"$each": value}} + else: + value = {key: value} + key = '$' + op + + if key not in mongo_update: + mongo_update[key] = value + elif key in mongo_update and isinstance(mongo_update[key], dict): + mongo_update[key].update(value) + + return mongo_update diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py new file mode 100644 index 0000000..94d6a5e --- /dev/null +++ b/mongoengine/queryset/visitor.py @@ -0,0 +1,237 @@ +import copy + +from mongoengine.errors import InvalidQueryError +from mongoengine.python_support import product, reduce + +from mongoengine.queryset import transform + +__all__ = ('Q',) + + +class QNodeVisitor(object): + """Base visitor class for visiting Q-object nodes in a query tree. + """ + + def visit_combination(self, combination): + """Called by QCombination objects. + """ + return combination + + def visit_query(self, query): + """Called by (New)Q objects. + """ + return query + + +class SimplificationVisitor(QNodeVisitor): + """Simplifies query trees by combinging unnecessary 'and' connection nodes + into a single Q-object. + """ + + def visit_combination(self, combination): + if combination.operation == combination.AND: + # The simplification only applies to 'simple' queries + if all(isinstance(node, Q) for node in combination.children): + queries = [node.query for node in combination.children] + return Q(**self._query_conjunction(queries)) + return combination + + def _query_conjunction(self, queries): + """Merges query dicts - effectively &ing them together. + """ + query_ops = set() + combined_query = {} + for query in queries: + ops = set(query.keys()) + # Make sure that the same operation isn't applied more than once + # to a single field + intersection = ops.intersection(query_ops) + if intersection: + msg = 'Duplicate query conditions: ' + raise InvalidQueryError(msg + ', '.join(intersection)) + + query_ops.update(ops) + combined_query.update(copy.deepcopy(query)) + return combined_query + + +class QueryTreeTransformerVisitor(QNodeVisitor): + """Transforms the query tree in to a form that may be used with MongoDB. + """ + + def visit_combination(self, combination): + if combination.operation == combination.AND: + # MongoDB doesn't allow us to have too many $or operations in our + # queries, so the aim is to move the ORs up the tree to one + # 'master' $or. Firstly, we must find all the necessary parts (part + # of an AND combination or just standard Q object), and store them + # separately from the OR parts. + or_groups = [] + and_parts = [] + for node in combination.children: + if isinstance(node, QCombination): + if node.operation == node.OR: + # Any of the children in an $or component may cause + # the query to succeed + or_groups.append(node.children) + elif node.operation == node.AND: + and_parts.append(node) + elif isinstance(node, Q): + and_parts.append(node) + + # Now we combine the parts into a usable query. AND together all of + # the necessary parts. Then for each $or part, create a new query + # that ANDs the necessary part with the $or part. + clauses = [] + for or_group in product(*or_groups): + q_object = reduce(lambda a, b: a & b, and_parts, Q()) + q_object = reduce(lambda a, b: a & b, or_group, q_object) + clauses.append(q_object) + # Finally, $or the generated clauses in to one query. Each of the + # clauses is sufficient for the query to succeed. + return reduce(lambda a, b: a | b, clauses, Q()) + + if combination.operation == combination.OR: + children = [] + # Crush any nested ORs in to this combination as MongoDB doesn't + # support nested $or operations + for node in combination.children: + if (isinstance(node, QCombination) and + node.operation == combination.OR): + children += node.children + else: + children.append(node) + combination.children = children + + return combination + + +class QueryCompilerVisitor(QNodeVisitor): + """Compiles the nodes in a query tree to a PyMongo-compatible query + dictionary. + """ + + def __init__(self, document): + self.document = document + + def visit_combination(self, combination): + if combination.operation == combination.OR: + return {'$or': combination.children} + elif combination.operation == combination.AND: + return self._mongo_query_conjunction(combination.children) + return combination + + def visit_query(self, query): + return transform.query(self.document, **query.query) + + def _mongo_query_conjunction(self, queries): + """Merges Mongo query dicts - effectively &ing them together. + """ + combined_query = {} + for query in queries: + for field, ops in query.items(): + if field not in combined_query: + combined_query[field] = ops + else: + # The field is already present in the query the only way + # we can merge is if both the existing value and the new + # value are operation dicts, reject anything else + if (not isinstance(combined_query[field], dict) or + not isinstance(ops, dict)): + message = 'Conflicting values for ' + field + raise InvalidQueryError(message) + + current_ops = set(combined_query[field].keys()) + new_ops = set(ops.keys()) + # Make sure that the same operation isn't applied more than + # once to a single field + intersection = current_ops.intersection(new_ops) + if intersection: + msg = 'Duplicate query conditions: ' + raise InvalidQueryError(msg + ', '.join(intersection)) + + # Right! We've got two non-overlapping dicts of operations! + combined_query[field].update(copy.deepcopy(ops)) + return combined_query + + +class QNode(object): + """Base class for nodes in query trees. + """ + + AND = 0 + OR = 1 + + def to_query(self, document): + query = self.accept(SimplificationVisitor()) + query = query.accept(QueryTreeTransformerVisitor()) + query = query.accept(QueryCompilerVisitor(document)) + return query + + def accept(self, visitor): + raise NotImplementedError + + def _combine(self, other, operation): + """Combine this node with another node into a QCombination object. + """ + if getattr(other, 'empty', True): + return self + + if self.empty: + return other + + return QCombination(operation, [self, other]) + + @property + def empty(self): + return False + + def __or__(self, other): + return self._combine(other, self.OR) + + def __and__(self, other): + return self._combine(other, self.AND) + + +class QCombination(QNode): + """Represents the combination of several conditions by a given logical + operator. + """ + + def __init__(self, operation, children): + self.operation = operation + self.children = [] + for node in children: + # If the child is a combination of the same type, we can merge its + # children directly into this combinations children + if isinstance(node, QCombination) and node.operation == operation: + self.children += node.children + else: + self.children.append(node) + + def accept(self, visitor): + for i in range(len(self.children)): + if isinstance(self.children[i], QNode): + self.children[i] = self.children[i].accept(visitor) + + return visitor.visit_combination(self) + + @property + def empty(self): + return not bool(self.children) + + +class Q(QNode): + """A simple query object, used in a query tree to build up more complex + query structures. + """ + + def __init__(self, **query): + self.query = query + + def accept(self, visitor): + return visitor.visit_query(self) + + @property + def empty(self): + return not bool(self.query) diff --git a/setup.cfg b/setup.cfg index d95a917..3f3faa8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,4 +8,4 @@ detailed-errors = 1 #cover-package = mongoengine py3where = build where = tests -#tests = test_bugfix.py \ No newline at end of file +#tests = document/__init__.py \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..f2a43b0 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,2 @@ +from .all_warnings import AllWarnings +from .document import * \ No newline at end of file diff --git a/tests/test_all_warnings.py b/tests/all_warnings/__init__.py similarity index 91% rename from tests/test_all_warnings.py rename to tests/all_warnings/__init__.py index 9b38fa6..72de822 100644 --- a/tests/test_all_warnings.py +++ b/tests/all_warnings/__init__.py @@ -1,11 +1,19 @@ +""" +This test has been put into a module. This is because it tests warnings that +only get triggered on first hit. This way we can ensure its imported into the +top level and called first by the test suite. +""" + import unittest import warnings from mongoengine import * -from mongoengine.tests import query_counter -class TestWarnings(unittest.TestCase): +__all__ = ('AllWarnings', ) + + +class AllWarnings(unittest.TestCase): def setUp(self): conn = connect(db='mongoenginetest') diff --git a/tests/document/__init__.py b/tests/document/__init__.py new file mode 100644 index 0000000..1ef2520 --- /dev/null +++ b/tests/document/__init__.py @@ -0,0 +1,11 @@ +# TODO EXPLICT IMPORTS + +from class_methods import * +from delta import * +from dynamic import * +from indexes import * +from inheritance import * +from instance import * + +if __name__ == '__main__': + unittest.main() diff --git a/tests/document/class_methods.py b/tests/document/class_methods.py new file mode 100644 index 0000000..8050998 --- /dev/null +++ b/tests/document/class_methods.py @@ -0,0 +1,183 @@ +# -*- coding: utf-8 -*- +from __future__ import with_statement +import unittest + +from mongoengine import * + +from mongoengine.queryset import NULLIFY +from mongoengine.connection import get_db + +__all__ = ("ClassMethodsTest", ) + + +class ClassMethodsTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + class Person(Document): + name = StringField() + age = IntField() + + non_field = True + + meta = {"allow_inheritance": True} + + self.Person = Person + + def tearDown(self): + for collection in self.db.collection_names(): + if 'system.' in collection: + continue + self.db.drop_collection(collection) + + def test_definition(self): + """Ensure that document may be defined using fields. + """ + self.assertEqual(['age', 'name', 'id'], self.Person._fields.keys()) + self.assertEqual([IntField, StringField, ObjectIdField], + [x.__class__ for x in self.Person._fields.values()]) + + def test_get_db(self): + """Ensure that get_db returns the expected db. + """ + db = self.Person._get_db() + self.assertEqual(self.db, db) + + def test_get_collection_name(self): + """Ensure that get_collection_name returns the expected collection + name. + """ + collection_name = 'person' + self.assertEqual(collection_name, self.Person._get_collection_name()) + + def test_get_collection(self): + """Ensure that get_collection returns the expected collection. + """ + collection_name = 'person' + collection = self.Person._get_collection() + self.assertEqual(self.db[collection_name], collection) + + def test_drop_collection(self): + """Ensure that the collection may be dropped from the database. + """ + collection_name = 'person' + self.Person(name='Test').save() + self.assertTrue(collection_name in self.db.collection_names()) + + self.Person.drop_collection() + self.assertFalse(collection_name in self.db.collection_names()) + + def test_register_delete_rule(self): + """Ensure that register delete rule adds a delete rule to the document + meta. + """ + class Job(Document): + employee = ReferenceField(self.Person) + + self.assertEqual(self.Person._meta.get('delete_rules'), None) + + self.Person.register_delete_rule(Job, 'employee', NULLIFY) + self.assertEqual(self.Person._meta['delete_rules'], + {(Job, 'employee'): NULLIFY}) + + def test_collection_naming(self): + """Ensure that a collection with a specified name may be used. + """ + + class DefaultNamingTest(Document): + pass + self.assertEqual('default_naming_test', + DefaultNamingTest._get_collection_name()) + + class CustomNamingTest(Document): + meta = {'collection': 'pimp_my_collection'} + + self.assertEqual('pimp_my_collection', + CustomNamingTest._get_collection_name()) + + class DynamicNamingTest(Document): + meta = {'collection': lambda c: "DYNAMO"} + self.assertEqual('DYNAMO', DynamicNamingTest._get_collection_name()) + + # Use Abstract class to handle backwards compatibility + class BaseDocument(Document): + meta = { + 'abstract': True, + 'collection': lambda c: c.__name__.lower() + } + + class OldNamingConvention(BaseDocument): + pass + self.assertEqual('oldnamingconvention', + OldNamingConvention._get_collection_name()) + + class InheritedAbstractNamingTest(BaseDocument): + meta = {'collection': 'wibble'} + self.assertEqual('wibble', + InheritedAbstractNamingTest._get_collection_name()) + + # Mixin tests + class BaseMixin(object): + meta = { + 'collection': lambda c: c.__name__.lower() + } + + class OldMixinNamingConvention(Document, BaseMixin): + pass + self.assertEqual('oldmixinnamingconvention', + OldMixinNamingConvention._get_collection_name()) + + class BaseMixin(object): + meta = { + 'collection': lambda c: c.__name__.lower() + } + + class BaseDocument(Document, BaseMixin): + meta = {'allow_inheritance': True} + + class MyDocument(BaseDocument): + pass + + self.assertEqual('basedocument', MyDocument._get_collection_name()) + + def test_custom_collection_name_operations(self): + """Ensure that a collection with a specified name is used as expected. + """ + collection_name = 'personCollTest' + + class Person(Document): + name = StringField() + meta = {'collection': collection_name} + + Person(name="Test User").save() + self.assertTrue(collection_name in self.db.collection_names()) + + user_obj = self.db[collection_name].find_one() + self.assertEqual(user_obj['name'], "Test User") + + user_obj = Person.objects[0] + self.assertEqual(user_obj.name, "Test User") + + Person.drop_collection() + self.assertFalse(collection_name in self.db.collection_names()) + + def test_collection_name_and_primary(self): + """Ensure that a collection with a specified name may be used. + """ + + class Person(Document): + name = StringField(primary_key=True) + meta = {'collection': 'app'} + + Person(name="Test User").save() + + user_obj = Person.objects.first() + self.assertEqual(user_obj.name, "Test User") + + Person.drop_collection() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/document/delta.py b/tests/document/delta.py new file mode 100644 index 0000000..f8a071d --- /dev/null +++ b/tests/document/delta.py @@ -0,0 +1,688 @@ +# -*- coding: utf-8 -*- +import unittest + +from mongoengine import * +from mongoengine.connection import get_db + +__all__ = ("DeltaTest",) + + +class DeltaTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + class Person(Document): + name = StringField() + age = IntField() + + non_field = True + + meta = {"allow_inheritance": True} + + self.Person = Person + + def tearDown(self): + for collection in self.db.collection_names(): + if 'system.' in collection: + continue + self.db.drop_collection(collection) + + def test_delta(self): + self.delta(Document) + self.delta(DynamicDocument) + + def delta(self, DocClass): + + class Doc(DocClass): + string_field = StringField() + int_field = IntField() + dict_field = DictField() + list_field = ListField() + + Doc.drop_collection() + doc = Doc() + doc.save() + + doc = Doc.objects.first() + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(doc._delta(), ({}, {})) + + doc.string_field = 'hello' + self.assertEqual(doc._get_changed_fields(), ['string_field']) + self.assertEqual(doc._delta(), ({'string_field': 'hello'}, {})) + + doc._changed_fields = [] + doc.int_field = 1 + self.assertEqual(doc._get_changed_fields(), ['int_field']) + self.assertEqual(doc._delta(), ({'int_field': 1}, {})) + + doc._changed_fields = [] + dict_value = {'hello': 'world', 'ping': 'pong'} + doc.dict_field = dict_value + self.assertEqual(doc._get_changed_fields(), ['dict_field']) + self.assertEqual(doc._delta(), ({'dict_field': dict_value}, {})) + + doc._changed_fields = [] + list_value = ['1', 2, {'hello': 'world'}] + doc.list_field = list_value + self.assertEqual(doc._get_changed_fields(), ['list_field']) + self.assertEqual(doc._delta(), ({'list_field': list_value}, {})) + + # Test unsetting + doc._changed_fields = [] + doc.dict_field = {} + self.assertEqual(doc._get_changed_fields(), ['dict_field']) + self.assertEqual(doc._delta(), ({}, {'dict_field': 1})) + + doc._changed_fields = [] + doc.list_field = [] + self.assertEqual(doc._get_changed_fields(), ['list_field']) + self.assertEqual(doc._delta(), ({}, {'list_field': 1})) + + def test_delta_recursive(self): + self.delta_recursive(Document, EmbeddedDocument) + self.delta_recursive(DynamicDocument, EmbeddedDocument) + self.delta_recursive(Document, DynamicEmbeddedDocument) + self.delta_recursive(DynamicDocument, DynamicEmbeddedDocument) + + def delta_recursive(self, DocClass, EmbeddedClass): + + class Embedded(EmbeddedClass): + string_field = StringField() + int_field = IntField() + dict_field = DictField() + list_field = ListField() + + class Doc(DocClass): + string_field = StringField() + int_field = IntField() + dict_field = DictField() + list_field = ListField() + embedded_field = EmbeddedDocumentField(Embedded) + + Doc.drop_collection() + doc = Doc() + doc.save() + + doc = Doc.objects.first() + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(doc._delta(), ({}, {})) + + embedded_1 = Embedded() + embedded_1.string_field = 'hello' + embedded_1.int_field = 1 + embedded_1.dict_field = {'hello': 'world'} + embedded_1.list_field = ['1', 2, {'hello': 'world'}] + doc.embedded_field = embedded_1 + + self.assertEqual(doc._get_changed_fields(), ['embedded_field']) + + embedded_delta = { + 'string_field': 'hello', + 'int_field': 1, + 'dict_field': {'hello': 'world'}, + 'list_field': ['1', 2, {'hello': 'world'}] + } + self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) + embedded_delta.update({ + '_cls': 'Embedded', + }) + self.assertEqual(doc._delta(), + ({'embedded_field': embedded_delta}, {})) + + doc.save() + doc = doc.reload(10) + + doc.embedded_field.dict_field = {} + self.assertEqual(doc._get_changed_fields(), + ['embedded_field.dict_field']) + self.assertEqual(doc.embedded_field._delta(), ({}, {'dict_field': 1})) + self.assertEqual(doc._delta(), ({}, {'embedded_field.dict_field': 1})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.dict_field, {}) + + doc.embedded_field.list_field = [] + self.assertEqual(doc._get_changed_fields(), + ['embedded_field.list_field']) + self.assertEqual(doc.embedded_field._delta(), ({}, {'list_field': 1})) + self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field': 1})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field, []) + + embedded_2 = Embedded() + embedded_2.string_field = 'hello' + embedded_2.int_field = 1 + embedded_2.dict_field = {'hello': 'world'} + embedded_2.list_field = ['1', 2, {'hello': 'world'}] + + doc.embedded_field.list_field = ['1', 2, embedded_2] + self.assertEqual(doc._get_changed_fields(), + ['embedded_field.list_field']) + self.assertEqual(doc.embedded_field._delta(), ({ + 'list_field': ['1', 2, { + '_cls': 'Embedded', + 'string_field': 'hello', + 'dict_field': {'hello': 'world'}, + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], + }] + }, {})) + + self.assertEqual(doc._delta(), ({ + 'embedded_field.list_field': ['1', 2, { + '_cls': 'Embedded', + 'string_field': 'hello', + 'dict_field': {'hello': 'world'}, + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], + }] + }, {})) + doc.save() + doc = doc.reload(10) + + self.assertEqual(doc.embedded_field.list_field[0], '1') + self.assertEqual(doc.embedded_field.list_field[1], 2) + for k in doc.embedded_field.list_field[2]._fields: + self.assertEqual(doc.embedded_field.list_field[2][k], + embedded_2[k]) + + doc.embedded_field.list_field[2].string_field = 'world' + self.assertEqual(doc._get_changed_fields(), + ['embedded_field.list_field.2.string_field']) + self.assertEqual(doc.embedded_field._delta(), + ({'list_field.2.string_field': 'world'}, {})) + self.assertEqual(doc._delta(), + ({'embedded_field.list_field.2.string_field': 'world'}, {})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field[2].string_field, + 'world') + + # Test multiple assignments + doc.embedded_field.list_field[2].string_field = 'hello world' + doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] + self.assertEqual(doc._get_changed_fields(), + ['embedded_field.list_field']) + self.assertEqual(doc.embedded_field._delta(), ({ + 'list_field': ['1', 2, { + '_cls': 'Embedded', + 'string_field': 'hello world', + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], + 'dict_field': {'hello': 'world'}}]}, {})) + self.assertEqual(doc._delta(), ({ + 'embedded_field.list_field': ['1', 2, { + '_cls': 'Embedded', + 'string_field': 'hello world', + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], + 'dict_field': {'hello': 'world'}} + ]}, {})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field[2].string_field, + 'hello world') + + # Test list native methods + doc.embedded_field.list_field[2].list_field.pop(0) + self.assertEqual(doc._delta(), + ({'embedded_field.list_field.2.list_field': + [2, {'hello': 'world'}]}, {})) + doc.save() + doc = doc.reload(10) + + doc.embedded_field.list_field[2].list_field.append(1) + self.assertEqual(doc._delta(), + ({'embedded_field.list_field.2.list_field': + [2, {'hello': 'world'}, 1]}, {})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field[2].list_field, + [2, {'hello': 'world'}, 1]) + + doc.embedded_field.list_field[2].list_field.sort(key=str) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field[2].list_field, + [1, 2, {'hello': 'world'}]) + + del(doc.embedded_field.list_field[2].list_field[2]['hello']) + self.assertEqual(doc._delta(), + ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) + doc.save() + doc = doc.reload(10) + + del(doc.embedded_field.list_field[2].list_field) + self.assertEqual(doc._delta(), + ({}, {'embedded_field.list_field.2.list_field': 1})) + + doc.save() + doc = doc.reload(10) + + doc.dict_field['Embedded'] = embedded_1 + doc.save() + doc = doc.reload(10) + + doc.dict_field['Embedded'].string_field = 'Hello World' + self.assertEqual(doc._get_changed_fields(), + ['dict_field.Embedded.string_field']) + self.assertEqual(doc._delta(), + ({'dict_field.Embedded.string_field': 'Hello World'}, {})) + + def test_circular_reference_deltas(self): + self.circular_reference_deltas(Document, Document) + self.circular_reference_deltas(Document, DynamicDocument) + self.circular_reference_deltas(DynamicDocument, Document) + self.circular_reference_deltas(DynamicDocument, DynamicDocument) + + def circular_reference_deltas(self, DocClass1, DocClass2): + + class Person(DocClass1): + name = StringField() + owns = ListField(ReferenceField('Organization')) + + class Organization(DocClass2): + name = StringField() + owner = ReferenceField('Person') + + person = Person(name="owner") + person.save() + organization = Organization(name="company") + organization.save() + + person.owns.append(organization) + organization.owner = person + + person.save() + organization.save() + + p = Person.objects[0].select_related() + o = Organization.objects.first() + self.assertEqual(p.owns[0], o) + self.assertEqual(o.owner, p) + + def test_circular_reference_deltas_2(self): + self.circular_reference_deltas_2(Document, Document) + self.circular_reference_deltas_2(Document, DynamicDocument) + self.circular_reference_deltas_2(DynamicDocument, Document) + self.circular_reference_deltas_2(DynamicDocument, DynamicDocument) + + def circular_reference_deltas_2(self, DocClass1, DocClass2): + + class Person(DocClass1): + name = StringField() + owns = ListField(ReferenceField('Organization')) + employer = ReferenceField('Organization') + + class Organization(DocClass2): + name = StringField() + owner = ReferenceField('Person') + employees = ListField(ReferenceField('Person')) + + Person.drop_collection() + Organization.drop_collection() + + person = Person(name="owner") + person.save() + + employee = Person(name="employee") + employee.save() + + organization = Organization(name="company") + organization.save() + + person.owns.append(organization) + organization.owner = person + + organization.employees.append(employee) + employee.employer = organization + + person.save() + organization.save() + employee.save() + + p = Person.objects.get(name="owner") + e = Person.objects.get(name="employee") + o = Organization.objects.first() + + self.assertEqual(p.owns[0], o) + self.assertEqual(o.owner, p) + self.assertEqual(e.employer, o) + + def test_delta_db_field(self): + self.delta_db_field(Document) + self.delta_db_field(DynamicDocument) + + def delta_db_field(self, DocClass): + + class Doc(DocClass): + string_field = StringField(db_field='db_string_field') + int_field = IntField(db_field='db_int_field') + dict_field = DictField(db_field='db_dict_field') + list_field = ListField(db_field='db_list_field') + + Doc.drop_collection() + doc = Doc() + doc.save() + + doc = Doc.objects.first() + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(doc._delta(), ({}, {})) + + doc.string_field = 'hello' + self.assertEqual(doc._get_changed_fields(), ['db_string_field']) + self.assertEqual(doc._delta(), ({'db_string_field': 'hello'}, {})) + + doc._changed_fields = [] + doc.int_field = 1 + self.assertEqual(doc._get_changed_fields(), ['db_int_field']) + self.assertEqual(doc._delta(), ({'db_int_field': 1}, {})) + + doc._changed_fields = [] + dict_value = {'hello': 'world', 'ping': 'pong'} + doc.dict_field = dict_value + self.assertEqual(doc._get_changed_fields(), ['db_dict_field']) + self.assertEqual(doc._delta(), ({'db_dict_field': dict_value}, {})) + + doc._changed_fields = [] + list_value = ['1', 2, {'hello': 'world'}] + doc.list_field = list_value + self.assertEqual(doc._get_changed_fields(), ['db_list_field']) + self.assertEqual(doc._delta(), ({'db_list_field': list_value}, {})) + + # Test unsetting + doc._changed_fields = [] + doc.dict_field = {} + self.assertEqual(doc._get_changed_fields(), ['db_dict_field']) + self.assertEqual(doc._delta(), ({}, {'db_dict_field': 1})) + + doc._changed_fields = [] + doc.list_field = [] + self.assertEqual(doc._get_changed_fields(), ['db_list_field']) + self.assertEqual(doc._delta(), ({}, {'db_list_field': 1})) + + # Test it saves that data + doc = Doc() + doc.save() + + doc.string_field = 'hello' + doc.int_field = 1 + doc.dict_field = {'hello': 'world'} + doc.list_field = ['1', 2, {'hello': 'world'}] + doc.save() + doc = doc.reload(10) + + self.assertEqual(doc.string_field, 'hello') + self.assertEqual(doc.int_field, 1) + self.assertEqual(doc.dict_field, {'hello': 'world'}) + self.assertEqual(doc.list_field, ['1', 2, {'hello': 'world'}]) + + def test_delta_recursive_db_field(self): + self.delta_recursive_db_field(Document, EmbeddedDocument) + self.delta_recursive_db_field(Document, DynamicEmbeddedDocument) + self.delta_recursive_db_field(DynamicDocument, EmbeddedDocument) + self.delta_recursive_db_field(DynamicDocument, DynamicEmbeddedDocument) + + def delta_recursive_db_field(self, DocClass, EmbeddedClass): + + class Embedded(EmbeddedClass): + string_field = StringField(db_field='db_string_field') + int_field = IntField(db_field='db_int_field') + dict_field = DictField(db_field='db_dict_field') + list_field = ListField(db_field='db_list_field') + + class Doc(DocClass): + string_field = StringField(db_field='db_string_field') + int_field = IntField(db_field='db_int_field') + dict_field = DictField(db_field='db_dict_field') + list_field = ListField(db_field='db_list_field') + embedded_field = EmbeddedDocumentField(Embedded, + db_field='db_embedded_field') + + Doc.drop_collection() + doc = Doc() + doc.save() + + doc = Doc.objects.first() + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(doc._delta(), ({}, {})) + + embedded_1 = Embedded() + embedded_1.string_field = 'hello' + embedded_1.int_field = 1 + embedded_1.dict_field = {'hello': 'world'} + embedded_1.list_field = ['1', 2, {'hello': 'world'}] + doc.embedded_field = embedded_1 + + self.assertEqual(doc._get_changed_fields(), ['db_embedded_field']) + + embedded_delta = { + 'db_string_field': 'hello', + 'db_int_field': 1, + 'db_dict_field': {'hello': 'world'}, + 'db_list_field': ['1', 2, {'hello': 'world'}] + } + self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) + embedded_delta.update({ + '_cls': 'Embedded', + }) + self.assertEqual(doc._delta(), + ({'db_embedded_field': embedded_delta}, {})) + + doc.save() + doc = doc.reload(10) + + doc.embedded_field.dict_field = {} + self.assertEqual(doc._get_changed_fields(), + ['db_embedded_field.db_dict_field']) + self.assertEqual(doc.embedded_field._delta(), + ({}, {'db_dict_field': 1})) + self.assertEqual(doc._delta(), + ({}, {'db_embedded_field.db_dict_field': 1})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.dict_field, {}) + + doc.embedded_field.list_field = [] + self.assertEqual(doc._get_changed_fields(), + ['db_embedded_field.db_list_field']) + self.assertEqual(doc.embedded_field._delta(), + ({}, {'db_list_field': 1})) + self.assertEqual(doc._delta(), + ({}, {'db_embedded_field.db_list_field': 1})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field, []) + + embedded_2 = Embedded() + embedded_2.string_field = 'hello' + embedded_2.int_field = 1 + embedded_2.dict_field = {'hello': 'world'} + embedded_2.list_field = ['1', 2, {'hello': 'world'}] + + doc.embedded_field.list_field = ['1', 2, embedded_2] + self.assertEqual(doc._get_changed_fields(), + ['db_embedded_field.db_list_field']) + self.assertEqual(doc.embedded_field._delta(), ({ + 'db_list_field': ['1', 2, { + '_cls': 'Embedded', + 'db_string_field': 'hello', + 'db_dict_field': {'hello': 'world'}, + 'db_int_field': 1, + 'db_list_field': ['1', 2, {'hello': 'world'}], + }] + }, {})) + + self.assertEqual(doc._delta(), ({ + 'db_embedded_field.db_list_field': ['1', 2, { + '_cls': 'Embedded', + 'db_string_field': 'hello', + 'db_dict_field': {'hello': 'world'}, + 'db_int_field': 1, + 'db_list_field': ['1', 2, {'hello': 'world'}], + }] + }, {})) + doc.save() + doc = doc.reload(10) + + self.assertEqual(doc.embedded_field.list_field[0], '1') + self.assertEqual(doc.embedded_field.list_field[1], 2) + for k in doc.embedded_field.list_field[2]._fields: + self.assertEqual(doc.embedded_field.list_field[2][k], + embedded_2[k]) + + doc.embedded_field.list_field[2].string_field = 'world' + self.assertEqual(doc._get_changed_fields(), + ['db_embedded_field.db_list_field.2.db_string_field']) + self.assertEqual(doc.embedded_field._delta(), + ({'db_list_field.2.db_string_field': 'world'}, {})) + self.assertEqual(doc._delta(), + ({'db_embedded_field.db_list_field.2.db_string_field': 'world'}, + {})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field[2].string_field, + 'world') + + # Test multiple assignments + doc.embedded_field.list_field[2].string_field = 'hello world' + doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] + self.assertEqual(doc._get_changed_fields(), + ['db_embedded_field.db_list_field']) + self.assertEqual(doc.embedded_field._delta(), ({ + 'db_list_field': ['1', 2, { + '_cls': 'Embedded', + 'db_string_field': 'hello world', + 'db_int_field': 1, + 'db_list_field': ['1', 2, {'hello': 'world'}], + 'db_dict_field': {'hello': 'world'}}]}, {})) + self.assertEqual(doc._delta(), ({ + 'db_embedded_field.db_list_field': ['1', 2, { + '_cls': 'Embedded', + 'db_string_field': 'hello world', + 'db_int_field': 1, + 'db_list_field': ['1', 2, {'hello': 'world'}], + 'db_dict_field': {'hello': 'world'}} + ]}, {})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field[2].string_field, + 'hello world') + + # Test list native methods + doc.embedded_field.list_field[2].list_field.pop(0) + self.assertEqual(doc._delta(), + ({'db_embedded_field.db_list_field.2.db_list_field': + [2, {'hello': 'world'}]}, {})) + doc.save() + doc = doc.reload(10) + + doc.embedded_field.list_field[2].list_field.append(1) + self.assertEqual(doc._delta(), + ({'db_embedded_field.db_list_field.2.db_list_field': + [2, {'hello': 'world'}, 1]}, {})) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field[2].list_field, + [2, {'hello': 'world'}, 1]) + + doc.embedded_field.list_field[2].list_field.sort(key=str) + doc.save() + doc = doc.reload(10) + self.assertEqual(doc.embedded_field.list_field[2].list_field, + [1, 2, {'hello': 'world'}]) + + del(doc.embedded_field.list_field[2].list_field[2]['hello']) + self.assertEqual(doc._delta(), + ({'db_embedded_field.db_list_field.2.db_list_field': + [1, 2, {}]}, {})) + doc.save() + doc = doc.reload(10) + + del(doc.embedded_field.list_field[2].list_field) + self.assertEqual(doc._delta(), ({}, + {'db_embedded_field.db_list_field.2.db_list_field': 1})) + + def test_delta_for_dynamic_documents(self): + class Person(DynamicDocument): + name = StringField() + meta = {'allow_inheritance': True} + + Person.drop_collection() + + p = Person(name="James", age=34) + self.assertEqual(p._delta(), ({'age': 34, 'name': 'James', + '_cls': 'Person'}, {})) + + p.doc = 123 + del(p.doc) + self.assertEqual(p._delta(), ({'age': 34, 'name': 'James', + '_cls': 'Person'}, {'doc': 1})) + + p = Person() + p.name = "Dean" + p.age = 22 + p.save() + + p.age = 24 + self.assertEqual(p.age, 24) + self.assertEqual(p._get_changed_fields(), ['age']) + self.assertEqual(p._delta(), ({'age': 24}, {})) + + p = self.Person.objects(age=22).get() + p.age = 24 + self.assertEqual(p.age, 24) + self.assertEqual(p._get_changed_fields(), ['age']) + self.assertEqual(p._delta(), ({'age': 24}, {})) + + p.save() + self.assertEqual(1, self.Person.objects(age=24).count()) + + def test_dynamic_delta(self): + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + doc.save() + + doc = Doc.objects.first() + self.assertEqual(doc._get_changed_fields(), []) + self.assertEqual(doc._delta(), ({}, {})) + + doc.string_field = 'hello' + self.assertEqual(doc._get_changed_fields(), ['string_field']) + self.assertEqual(doc._delta(), ({'string_field': 'hello'}, {})) + + doc._changed_fields = [] + doc.int_field = 1 + self.assertEqual(doc._get_changed_fields(), ['int_field']) + self.assertEqual(doc._delta(), ({'int_field': 1}, {})) + + doc._changed_fields = [] + dict_value = {'hello': 'world', 'ping': 'pong'} + doc.dict_field = dict_value + self.assertEqual(doc._get_changed_fields(), ['dict_field']) + self.assertEqual(doc._delta(), ({'dict_field': dict_value}, {})) + + doc._changed_fields = [] + list_value = ['1', 2, {'hello': 'world'}] + doc.list_field = list_value + self.assertEqual(doc._get_changed_fields(), ['list_field']) + self.assertEqual(doc._delta(), ({'list_field': list_value}, {})) + + # Test unsetting + doc._changed_fields = [] + doc.dict_field = {} + self.assertEqual(doc._get_changed_fields(), ['dict_field']) + self.assertEqual(doc._delta(), ({}, {'dict_field': 1})) + + doc._changed_fields = [] + doc.list_field = [] + self.assertEqual(doc._get_changed_fields(), ['list_field']) + self.assertEqual(doc._delta(), ({}, {'list_field': 1})) diff --git a/tests/document/dynamic.py b/tests/document/dynamic.py new file mode 100644 index 0000000..ef27917 --- /dev/null +++ b/tests/document/dynamic.py @@ -0,0 +1,270 @@ +import unittest + +from mongoengine import * +from mongoengine.connection import get_db + +__all__ = ("DynamicTest", ) + + +class DynamicTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + class Person(DynamicDocument): + name = StringField() + meta = {'allow_inheritance': True} + + Person.drop_collection() + + self.Person = Person + + def test_simple_dynamic_document(self): + """Ensures simple dynamic documents are saved correctly""" + + p = self.Person() + p.name = "James" + p.age = 34 + + self.assertEqual(p.to_mongo(), {"_cls": "Person", "name": "James", + "age": 34}) + + p.save() + + self.assertEqual(self.Person.objects.first().age, 34) + + # Confirm no changes to self.Person + self.assertFalse(hasattr(self.Person, 'age')) + + def test_change_scope_of_variable(self): + """Test changing the scope of a dynamic field has no adverse effects""" + p = self.Person() + p.name = "Dean" + p.misc = 22 + p.save() + + p = self.Person.objects.get() + p.misc = {'hello': 'world'} + p.save() + + p = self.Person.objects.get() + self.assertEqual(p.misc, {'hello': 'world'}) + + def test_delete_dynamic_field(self): + """Test deleting a dynamic field works""" + self.Person.drop_collection() + p = self.Person() + p.name = "Dean" + p.misc = 22 + p.save() + + p = self.Person.objects.get() + p.misc = {'hello': 'world'} + p.save() + + p = self.Person.objects.get() + self.assertEqual(p.misc, {'hello': 'world'}) + collection = self.db[self.Person._get_collection_name()] + obj = collection.find_one() + self.assertEqual(sorted(obj.keys()), ['_cls', '_id', 'misc', 'name']) + + del(p.misc) + p.save() + + p = self.Person.objects.get() + self.assertFalse(hasattr(p, 'misc')) + + obj = collection.find_one() + self.assertEqual(sorted(obj.keys()), ['_cls', '_id', 'name']) + + def test_dynamic_document_queries(self): + """Ensure we can query dynamic fields""" + p = self.Person() + p.name = "Dean" + p.age = 22 + p.save() + + self.assertEqual(1, self.Person.objects(age=22).count()) + p = self.Person.objects(age=22) + p = p.get() + self.assertEqual(22, p.age) + + def test_complex_dynamic_document_queries(self): + class Person(DynamicDocument): + name = StringField() + + Person.drop_collection() + + p = Person(name="test") + p.age = "ten" + p.save() + + p1 = Person(name="test1") + p1.age = "less then ten and a half" + p1.save() + + p2 = Person(name="test2") + p2.age = 10 + p2.save() + + self.assertEqual(Person.objects(age__icontains='ten').count(), 2) + self.assertEqual(Person.objects(age__gte=10).count(), 1) + + def test_complex_data_lookups(self): + """Ensure you can query dynamic document dynamic fields""" + p = self.Person() + p.misc = {'hello': 'world'} + p.save() + + self.assertEqual(1, self.Person.objects(misc__hello='world').count()) + + def test_inheritance(self): + """Ensure that dynamic document plays nice with inheritance""" + class Employee(self.Person): + salary = IntField() + + Employee.drop_collection() + + self.assertTrue('name' in Employee._fields) + self.assertTrue('salary' in Employee._fields) + self.assertEqual(Employee._get_collection_name(), + self.Person._get_collection_name()) + + joe_bloggs = Employee() + joe_bloggs.name = "Joe Bloggs" + joe_bloggs.salary = 10 + joe_bloggs.age = 20 + joe_bloggs.save() + + self.assertEqual(1, self.Person.objects(age=20).count()) + self.assertEqual(1, Employee.objects(age=20).count()) + + joe_bloggs = self.Person.objects.first() + self.assertTrue(isinstance(joe_bloggs, Employee)) + + def test_embedded_dynamic_document(self): + """Test dynamic embedded documents""" + class Embedded(DynamicEmbeddedDocument): + pass + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + + embedded_1 = Embedded() + embedded_1.string_field = 'hello' + embedded_1.int_field = 1 + embedded_1.dict_field = {'hello': 'world'} + embedded_1.list_field = ['1', 2, {'hello': 'world'}] + doc.embedded_field = embedded_1 + + self.assertEqual(doc.to_mongo(), {"_cls": "Doc", + "embedded_field": { + "_cls": "Embedded", + "string_field": "hello", + "int_field": 1, + "dict_field": {"hello": "world"}, + "list_field": ['1', 2, {'hello': 'world'}] + } + }) + doc.save() + + doc = Doc.objects.first() + self.assertEqual(doc.embedded_field.__class__, Embedded) + self.assertEqual(doc.embedded_field.string_field, "hello") + self.assertEqual(doc.embedded_field.int_field, 1) + self.assertEqual(doc.embedded_field.dict_field, {'hello': 'world'}) + self.assertEqual(doc.embedded_field.list_field, + ['1', 2, {'hello': 'world'}]) + + def test_complex_embedded_documents(self): + """Test complex dynamic embedded documents setups""" + class Embedded(DynamicEmbeddedDocument): + pass + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + + embedded_1 = Embedded() + embedded_1.string_field = 'hello' + embedded_1.int_field = 1 + embedded_1.dict_field = {'hello': 'world'} + + embedded_2 = Embedded() + embedded_2.string_field = 'hello' + embedded_2.int_field = 1 + embedded_2.dict_field = {'hello': 'world'} + embedded_2.list_field = ['1', 2, {'hello': 'world'}] + + embedded_1.list_field = ['1', 2, embedded_2] + doc.embedded_field = embedded_1 + + self.assertEqual(doc.to_mongo(), {"_cls": "Doc", + "embedded_field": { + "_cls": "Embedded", + "string_field": "hello", + "int_field": 1, + "dict_field": {"hello": "world"}, + "list_field": ['1', 2, + {"_cls": "Embedded", + "string_field": "hello", + "int_field": 1, + "dict_field": {"hello": "world"}, + "list_field": ['1', 2, {'hello': 'world'}]} + ] + } + }) + doc.save() + doc = Doc.objects.first() + self.assertEqual(doc.embedded_field.__class__, Embedded) + self.assertEqual(doc.embedded_field.string_field, "hello") + self.assertEqual(doc.embedded_field.int_field, 1) + self.assertEqual(doc.embedded_field.dict_field, {'hello': 'world'}) + self.assertEqual(doc.embedded_field.list_field[0], '1') + self.assertEqual(doc.embedded_field.list_field[1], 2) + + embedded_field = doc.embedded_field.list_field[2] + + self.assertEqual(embedded_field.__class__, Embedded) + self.assertEqual(embedded_field.string_field, "hello") + self.assertEqual(embedded_field.int_field, 1) + self.assertEqual(embedded_field.dict_field, {'hello': 'world'}) + self.assertEqual(embedded_field.list_field, ['1', 2, + {'hello': 'world'}]) + + def test_dynamic_and_embedded(self): + """Ensure embedded documents play nicely""" + + class Address(EmbeddedDocument): + city = StringField() + + class Person(DynamicDocument): + name = StringField() + meta = {'allow_inheritance': True} + + Person.drop_collection() + + Person(name="Ross", address=Address(city="London")).save() + + person = Person.objects.first() + person.address.city = "Lundenne" + person.save() + + self.assertEqual(Person.objects.first().address.city, "Lundenne") + + person = Person.objects.first() + person.address = Address(city="Londinium") + person.save() + + self.assertEqual(Person.objects.first().address.city, "Londinium") + + person = Person.objects.first() + person.age = 35 + person.save() + self.assertEqual(Person.objects.first().age, 35) diff --git a/tests/document/indexes.py b/tests/document/indexes.py new file mode 100644 index 0000000..a6b74cd --- /dev/null +++ b/tests/document/indexes.py @@ -0,0 +1,637 @@ +# -*- coding: utf-8 -*- +from __future__ import with_statement +import bson +import os +import pickle +import pymongo +import sys +import unittest +import uuid +import warnings + +from nose.plugins.skip import SkipTest +from datetime import datetime + +from tests.fixtures import Base, Mixin, PickleEmbedded, PickleTest + +from mongoengine import * +from mongoengine.errors import (NotRegistered, InvalidDocumentError, + InvalidQueryError) +from mongoengine.queryset import NULLIFY, Q +from mongoengine.connection import get_db, get_connection + +TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') + +__all__ = ("InstanceTest", ) + + +class InstanceTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + class Person(Document): + name = StringField() + age = IntField() + + non_field = True + + meta = {"allow_inheritance": True} + + self.Person = Person + + def tearDown(self): + for collection in self.db.collection_names(): + if 'system.' in collection: + continue + self.db.drop_collection(collection) + + def test_indexes_document(self, ): + """Ensure that indexes are used when meta[indexes] is specified for + Documents + """ + index_test(Document) + + def test_indexes_dynamic_document(self, ): + """Ensure that indexes are used when meta[indexes] is specified for + Dynamic Documents + """ + index_test(DynamicDocument) + + def index_test(self, InheritFrom): + + class BlogPost(InheritFrom): + date = DateTimeField(db_field='addDate', default=datetime.now) + category = StringField() + tags = ListField(StringField()) + meta = { + 'indexes': [ + '-date', + 'tags', + ('category', '-date') + ], + 'allow_inheritance': True + } + + expected_specs = [{'fields': [('_cls', 1), ('addDate', -1)]}, + {'fields': [('_cls', 1), ('tags', 1)]}, + {'fields': [('_cls', 1), ('category', 1), + ('addDate', -1)]}] + self.assertEqual(expected_specs, BlogPost._meta['index_specs']) + + BlogPost.objects._ensure_indexes() + info = BlogPost.objects._collection.index_information() + # _id, '-date', 'tags', ('cat', 'date') + # NB: there is no index on _cls by itself, since + # the indices on -date and tags will both contain + # _cls as first element in the key + self.assertEqual(len(info), 4) + info = [value['key'] for key, value in info.iteritems()] + for expected in expected_specs: + self.assertTrue(expected['fields'] in info) + + class ExtendedBlogPost(BlogPost): + title = StringField() + meta = {'indexes': ['title']} + + expected_specs.append({'fields': [('_cls', 1), ('title', 1)]}) + self.assertEqual(expected_specs, ExtendedBlogPost._meta['index_specs']) + + BlogPost.drop_collection() + + ExtendedBlogPost.objects._ensure_indexes() + info = ExtendedBlogPost.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] + for expected in expected_specs: + self.assertTrue(expected['fields'] in info) + + def test_inherited_index(self): + """Ensure index specs are inhertited correctly""" + + class A(Document): + title = StringField() + meta = { + 'indexes': [ + { + 'fields': ('title',), + }, + ], + 'allow_inheritance': True, + } + + class B(A): + description = StringField() + + self.assertEqual(A._meta['index_specs'], B._meta['index_specs']) + self.assertEqual([{'fields': [('_cls', 1), ('title', 1)]}], + A._meta['index_specs']) + + def test_build_index_spec_is_not_destructive(self): + + class MyDoc(Document): + keywords = StringField() + + meta = { + 'indexes': ['keywords'], + 'allow_inheritance': False + } + + self.assertEqual(MyDoc._meta['index_specs'], + [{'fields': [('keywords', 1)]}]) + + # Force index creation + MyDoc.objects._ensure_indexes() + + self.assertEqual(MyDoc._meta['index_specs'], + [{'fields': [('keywords', 1)]}]) + + def test_embedded_document_index_meta(self): + """Ensure that embedded document indexes are created explicitly + """ + class Rank(EmbeddedDocument): + title = StringField(required=True) + + class Person(Document): + name = StringField(required=True) + rank = EmbeddedDocumentField(Rank, required=False) + + meta = { + 'indexes': [ + 'rank.title', + ], + 'allow_inheritance': False + } + + self.assertEqual([{'fields': [('rank.title', 1)]}], + Person._meta['index_specs']) + + Person.drop_collection() + + # Indexes are lazy so use list() to perform query + list(Person.objects) + info = Person.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] + self.assertTrue([('rank.title', 1)] in info) + + def test_explicit_geo2d_index(self): + """Ensure that geo2d indexes work when created via meta[indexes] + """ + class Place(Document): + location = DictField() + meta = { + 'allow_inheritance': True, + 'indexes': [ + '*location.point', + ] + } + + self.assertEqual([{'fields': [('location.point', '2d')]}], + Place._meta['index_specs']) + + Place.objects()._ensure_indexes() + info = Place._get_collection().index_information() + info = [value['key'] for key, value in info.iteritems()] + self.assertTrue([('location.point', '2d')] in info) + + def test_dictionary_indexes(self): + """Ensure that indexes are used when meta[indexes] contains + dictionaries instead of lists. + """ + class BlogPost(Document): + date = DateTimeField(db_field='addDate', default=datetime.now) + category = StringField() + tags = ListField(StringField()) + meta = { + 'indexes': [ + {'fields': ['-date'], 'unique': True, + 'sparse': True, 'types': False}, + ], + } + + self.assertEqual([{'fields': [('addDate', -1)], 'unique': True, + 'sparse': True, 'types': False}], + BlogPost._meta['index_specs']) + + BlogPost.drop_collection() + + info = BlogPost.objects._collection.index_information() + # _id, '-date' + self.assertEqual(len(info), 3) + + # Indexes are lazy so use list() to perform query + list(BlogPost.objects) + info = BlogPost.objects._collection.index_information() + info = [(value['key'], + value.get('unique', False), + value.get('sparse', False)) + for key, value in info.iteritems()] + self.assertTrue(([('addDate', -1)], True, True) in info) + + BlogPost.drop_collection() + + def test_abstract_index_inheritance(self): + + class UserBase(Document): + user_guid = StringField(required=True) + meta = { + 'abstract': True, + 'indexes': ['user_guid'], + 'allow_inheritance': True + } + + class Person(UserBase): + name = StringField() + + meta = { + 'indexes': ['name'], + } + + Person(name="test", user_guid='123').save() + + self.assertEqual(1, Person.objects.count()) + info = Person.objects._collection.index_information() + self.assertEqual(info.keys(), ['_cls_1_name_1', '_cls_1_user_guid_1', + '_id_']) + + def test_disable_index_creation(self): + """Tests setting auto_create_index to False on the connection will + disable any index generation. + """ + class User(Document): + meta = { + 'indexes': ['user_guid'], + 'auto_create_index': False + } + user_guid = StringField(required=True) + + + User.drop_collection() + + u = User(user_guid='123') + u.save() + + self.assertEqual(1, User.objects.count()) + info = User.objects._collection.index_information() + self.assertEqual(info.keys(), ['_id_']) + User.drop_collection() + + def test_embedded_document_index(self): + """Tests settings an index on an embedded document + """ + class Date(EmbeddedDocument): + year = IntField(db_field='yr') + + class BlogPost(Document): + title = StringField() + date = EmbeddedDocumentField(Date) + + meta = { + 'indexes': [ + '-date.year' + ], + } + + BlogPost.drop_collection() + + info = BlogPost.objects._collection.index_information() + self.assertEqual(info.keys(), ['_cls_1_date.yr_-1', '_id_']) + BlogPost.drop_collection() + + def test_list_embedded_document_index(self): + """Ensure list embedded documents can be indexed + """ + class Tag(EmbeddedDocument): + name = StringField(db_field='tag') + + class BlogPost(Document): + title = StringField() + tags = ListField(EmbeddedDocumentField(Tag)) + + meta = { + 'indexes': [ + 'tags.name' + ] + } + + BlogPost.drop_collection() + + info = BlogPost.objects._collection.index_information() + # we don't use _cls in with list fields by default + self.assertEqual(info.keys(), ['_id_', '_cls_1_tags.tag_1']) + + post1 = BlogPost(title="Embedded Indexes tests in place", + tags=[Tag(name="about"), Tag(name="time")] + ) + post1.save() + BlogPost.drop_collection() + + def test_recursive_embedded_objects_dont_break_indexes(self): + + class RecursiveObject(EmbeddedDocument): + obj = EmbeddedDocumentField('self') + + class RecursiveDocument(Document): + recursive_obj = EmbeddedDocumentField(RecursiveObject) + meta = {'allow_inheritance': True} + + RecursiveDocument.objects._ensure_indexes() + info = RecursiveDocument._get_collection().index_information() + self.assertEqual(info.keys(), ['_id_', '_cls_1']) + + def test_geo_indexes_recursion(self): + + class Location(Document): + name = StringField() + location = GeoPointField() + + class Parent(Document): + name = StringField() + location = ReferenceField(Location) + + Location.drop_collection() + Parent.drop_collection() + + list(Parent.objects) + + collection = Parent._get_collection() + info = collection.index_information() + + self.assertFalse('location_2d' in info) + + self.assertEqual(len(Parent._geo_indices()), 0) + self.assertEqual(len(Location._geo_indices()), 1) + + def test_covered_index(self): + """Ensure that covered indexes can be used + """ + + class Test(Document): + a = IntField() + + meta = { + 'indexes': ['a'], + 'allow_inheritance': False + } + + Test.drop_collection() + + obj = Test(a=1) + obj.save() + + # Need to be explicit about covered indexes as mongoDB doesn't know if + # the documents returned might have more keys in that here. + query_plan = Test.objects(id=obj.id).exclude('a').explain() + self.assertFalse(query_plan['indexOnly']) + + query_plan = Test.objects(id=obj.id).only('id').explain() + self.assertTrue(query_plan['indexOnly']) + + query_plan = Test.objects(a=1).only('a').exclude('id').explain() + self.assertTrue(query_plan['indexOnly']) + + def test_index_on_id(self): + + class BlogPost(Document): + meta = { + 'indexes': [ + ['categories', 'id'] + ], + 'allow_inheritance': False + } + + title = StringField(required=True) + description = StringField(required=True) + categories = ListField() + + BlogPost.drop_collection() + + indexes = BlogPost.objects._collection.index_information() + self.assertEqual(indexes['categories_1__id_1']['key'], + [('categories', 1), ('_id', 1)]) + + def test_hint(self): + + class BlogPost(Document): + tags = ListField(StringField()) + meta = { + 'indexes': [ + 'tags', + ], + } + + BlogPost.drop_collection() + + for i in xrange(0, 10): + tags = [("tag %i" % n) for n in xrange(0, i % 2)] + BlogPost(tags=tags).save() + + self.assertEqual(BlogPost.objects.count(), 10) + self.assertEqual(BlogPost.objects.hint().count(), 10) + self.assertEqual(BlogPost.objects.hint([('tags', 1)]).count(), 10) + + self.assertEqual(BlogPost.objects.hint([('ZZ', 1)]).count(), 10) + + def invalid_index(): + BlogPost.objects.hint('tags') + self.assertRaises(TypeError, invalid_index) + + def invalid_index_2(): + return BlogPost.objects.hint(('tags', 1)) + self.assertRaises(TypeError, invalid_index_2) + + def test_unique(self): + """Ensure that uniqueness constraints are applied to fields. + """ + class BlogPost(Document): + title = StringField() + slug = StringField(unique=True) + + BlogPost.drop_collection() + + post1 = BlogPost(title='test1', slug='test') + post1.save() + + # Two posts with the same slug is not allowed + post2 = BlogPost(title='test2', slug='test') + self.assertRaises(NotUniqueError, post2.save) + + # Ensure backwards compatibilty for errors + self.assertRaises(OperationError, post2.save) + + def test_unique_with(self): + """Ensure that unique_with constraints are applied to fields. + """ + class Date(EmbeddedDocument): + year = IntField(db_field='yr') + + class BlogPost(Document): + title = StringField() + date = EmbeddedDocumentField(Date) + slug = StringField(unique_with='date.year') + + BlogPost.drop_collection() + + post1 = BlogPost(title='test1', date=Date(year=2009), slug='test') + post1.save() + + # day is different so won't raise exception + post2 = BlogPost(title='test2', date=Date(year=2010), slug='test') + post2.save() + + # Now there will be two docs with the same slug and the same day: fail + post3 = BlogPost(title='test3', date=Date(year=2010), slug='test') + self.assertRaises(OperationError, post3.save) + + BlogPost.drop_collection() + + def test_unique_embedded_document(self): + """Ensure that uniqueness constraints are applied to fields on embedded documents. + """ + class SubDocument(EmbeddedDocument): + year = IntField(db_field='yr') + slug = StringField(unique=True) + + class BlogPost(Document): + title = StringField() + sub = EmbeddedDocumentField(SubDocument) + + BlogPost.drop_collection() + + post1 = BlogPost(title='test1', sub=SubDocument(year=2009, slug="test")) + post1.save() + + # sub.slug is different so won't raise exception + post2 = BlogPost(title='test2', sub=SubDocument(year=2010, slug='another-slug')) + post2.save() + + # Now there will be two docs with the same sub.slug + post3 = BlogPost(title='test3', sub=SubDocument(year=2010, slug='test')) + self.assertRaises(NotUniqueError, post3.save) + + BlogPost.drop_collection() + + def test_unique_with_embedded_document_and_embedded_unique(self): + """Ensure that uniqueness constraints are applied to fields on + embedded documents. And work with unique_with as well. + """ + class SubDocument(EmbeddedDocument): + year = IntField(db_field='yr') + slug = StringField(unique=True) + + class BlogPost(Document): + title = StringField(unique_with='sub.year') + sub = EmbeddedDocumentField(SubDocument) + + BlogPost.drop_collection() + + post1 = BlogPost(title='test1', sub=SubDocument(year=2009, slug="test")) + post1.save() + + # sub.slug is different so won't raise exception + post2 = BlogPost(title='test2', sub=SubDocument(year=2010, slug='another-slug')) + post2.save() + + # Now there will be two docs with the same sub.slug + post3 = BlogPost(title='test3', sub=SubDocument(year=2010, slug='test')) + self.assertRaises(NotUniqueError, post3.save) + + # Now there will be two docs with the same title and year + post3 = BlogPost(title='test1', sub=SubDocument(year=2009, slug='test-1')) + self.assertRaises(NotUniqueError, post3.save) + + BlogPost.drop_collection() + + def test_ttl_indexes(self): + + class Log(Document): + created = DateTimeField(default=datetime.now) + meta = { + 'indexes': [ + {'fields': ['created'], 'expireAfterSeconds': 3600} + ] + } + + Log.drop_collection() + + if pymongo.version_tuple[0] < 2 and pymongo.version_tuple[1] < 3: + raise SkipTest('pymongo needs to be 2.3 or higher for this test') + + connection = get_connection() + version_array = connection.server_info()['versionArray'] + if version_array[0] < 2 and version_array[1] < 2: + raise SkipTest('MongoDB needs to be 2.2 or higher for this test') + + # Indexes are lazy so use list() to perform query + list(Log.objects) + info = Log.objects._collection.index_information() + self.assertEqual(3600, + info['_cls_1_created_1']['expireAfterSeconds']) + + def test_unique_and_indexes(self): + """Ensure that 'unique' constraints aren't overridden by + meta.indexes. + """ + class Customer(Document): + cust_id = IntField(unique=True, required=True) + meta = { + 'indexes': ['cust_id'], + 'allow_inheritance': False, + } + + Customer.drop_collection() + cust = Customer(cust_id=1) + cust.save() + + cust_dupe = Customer(cust_id=1) + try: + cust_dupe.save() + raise AssertionError, "We saved a dupe!" + except NotUniqueError: + pass + Customer.drop_collection() + + def test_unique_and_primary(self): + """If you set a field as primary, then unexpected behaviour can occur. + You won't create a duplicate but you will update an existing document. + """ + + class User(Document): + name = StringField(primary_key=True, unique=True) + password = StringField() + + User.drop_collection() + + user = User(name='huangz', password='secret') + user.save() + + user = User(name='huangz', password='secret2') + user.save() + + self.assertEqual(User.objects.count(), 1) + self.assertEqual(User.objects.get().password, 'secret2') + + User.drop_collection() + + def test_types_index_with_pk(self): + """Ensure you can use `pk` as part of a query""" + + class Comment(EmbeddedDocument): + comment_id = IntField(required=True) + + try: + class BlogPost(Document): + comments = EmbeddedDocumentField(Comment) + meta = {'indexes': [ + {'fields': ['pk', 'comments.comment_id'], + 'unique': True}]} + except UnboundLocalError: + self.fail('Unbound local error at types index + pk definition') + + info = BlogPost.objects._collection.index_information() + info = [value['key'] for key, value in info.iteritems()] + index_item = [('_cls', 1), ('_id', 1), ('comments.comment_id', 1)] + self.assertTrue(index_item in info) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/document/inheritance.py b/tests/document/inheritance.py new file mode 100644 index 0000000..d269ac0 --- /dev/null +++ b/tests/document/inheritance.py @@ -0,0 +1,395 @@ +# -*- coding: utf-8 -*- +import unittest +import warnings + +from datetime import datetime + +from tests.fixtures import Base + +from mongoengine import Document, EmbeddedDocument, connect +from mongoengine.connection import get_db +from mongoengine.fields import (BooleanField, GenericReferenceField, + IntField, StringField) + +__all__ = ('InheritanceTest', ) + + +class InheritanceTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + def tearDown(self): + for collection in self.db.collection_names(): + if 'system.' in collection: + continue + self.db.drop_collection(collection) + + def test_superclasses(self): + """Ensure that the correct list of superclasses is assembled. + """ + class Animal(Document): + meta = {'allow_inheritance': True} + class Fish(Animal): pass + class Guppy(Fish): pass + class Mammal(Animal): pass + class Dog(Mammal): pass + class Human(Mammal): pass + + self.assertEqual(Animal._superclasses, ()) + self.assertEqual(Fish._superclasses, ('Animal',)) + self.assertEqual(Guppy._superclasses, ('Animal', 'Animal.Fish')) + self.assertEqual(Mammal._superclasses, ('Animal',)) + self.assertEqual(Dog._superclasses, ('Animal', 'Animal.Mammal')) + self.assertEqual(Human._superclasses, ('Animal', 'Animal.Mammal')) + + def test_external_superclasses(self): + """Ensure that the correct list of super classes is assembled when + importing part of the model. + """ + class Animal(Base): pass + class Fish(Animal): pass + class Guppy(Fish): pass + class Mammal(Animal): pass + class Dog(Mammal): pass + class Human(Mammal): pass + + self.assertEqual(Animal._superclasses, ('Base', )) + self.assertEqual(Fish._superclasses, ('Base', 'Base.Animal',)) + self.assertEqual(Guppy._superclasses, ('Base', 'Base.Animal', + 'Base.Animal.Fish')) + self.assertEqual(Mammal._superclasses, ('Base', 'Base.Animal',)) + self.assertEqual(Dog._superclasses, ('Base', 'Base.Animal', + 'Base.Animal.Mammal')) + self.assertEqual(Human._superclasses, ('Base', 'Base.Animal', + 'Base.Animal.Mammal')) + + def test_subclasses(self): + """Ensure that the correct list of _subclasses (subclasses) is + assembled. + """ + class Animal(Document): + meta = {'allow_inheritance': True} + class Fish(Animal): pass + class Guppy(Fish): pass + class Mammal(Animal): pass + class Dog(Mammal): pass + class Human(Mammal): pass + + self.assertEqual(Animal._subclasses, ('Animal', + 'Animal.Fish', + 'Animal.Fish.Guppy', + 'Animal.Mammal', + 'Animal.Mammal.Dog', + 'Animal.Mammal.Human')) + self.assertEqual(Fish._subclasses, ('Animal.Fish', + 'Animal.Fish.Guppy',)) + self.assertEqual(Guppy._subclasses, ('Animal.Fish.Guppy',)) + self.assertEqual(Mammal._subclasses, ('Animal.Mammal', + 'Animal.Mammal.Dog', + 'Animal.Mammal.Human')) + self.assertEqual(Human._subclasses, ('Animal.Mammal.Human',)) + + def test_external_subclasses(self): + """Ensure that the correct list of _subclasses (subclasses) is + assembled when importing part of the model. + """ + class Animal(Base): pass + class Fish(Animal): pass + class Guppy(Fish): pass + class Mammal(Animal): pass + class Dog(Mammal): pass + class Human(Mammal): pass + + self.assertEqual(Animal._subclasses, ('Base.Animal', + 'Base.Animal.Fish', + 'Base.Animal.Fish.Guppy', + 'Base.Animal.Mammal', + 'Base.Animal.Mammal.Dog', + 'Base.Animal.Mammal.Human')) + self.assertEqual(Fish._subclasses, ('Base.Animal.Fish', + 'Base.Animal.Fish.Guppy',)) + self.assertEqual(Guppy._subclasses, ('Base.Animal.Fish.Guppy',)) + self.assertEqual(Mammal._subclasses, ('Base.Animal.Mammal', + 'Base.Animal.Mammal.Dog', + 'Base.Animal.Mammal.Human')) + self.assertEqual(Human._subclasses, ('Base.Animal.Mammal.Human',)) + + def test_dynamic_declarations(self): + """Test that declaring an extra class updates meta data""" + + class Animal(Document): + meta = {'allow_inheritance': True} + + self.assertEqual(Animal._superclasses, ()) + self.assertEqual(Animal._subclasses, ('Animal',)) + + # Test dynamically adding a class changes the meta data + class Fish(Animal): + pass + + self.assertEqual(Animal._superclasses, ()) + self.assertEqual(Animal._subclasses, ('Animal', 'Animal.Fish')) + + self.assertEqual(Fish._superclasses, ('Animal', )) + self.assertEqual(Fish._subclasses, ('Animal.Fish',)) + + # Test dynamically adding an inherited class changes the meta data + class Pike(Fish): + pass + + self.assertEqual(Animal._superclasses, ()) + self.assertEqual(Animal._subclasses, ('Animal', 'Animal.Fish', + 'Animal.Fish.Pike')) + + self.assertEqual(Fish._superclasses, ('Animal', )) + self.assertEqual(Fish._subclasses, ('Animal.Fish', 'Animal.Fish.Pike')) + + self.assertEqual(Pike._superclasses, ('Animal', 'Animal.Fish')) + self.assertEqual(Pike._subclasses, ('Animal.Fish.Pike',)) + + def test_inheritance_meta_data(self): + """Ensure that document may inherit fields from a superclass document. + """ + class Person(Document): + name = StringField() + age = IntField() + + meta = {'allow_inheritance': True} + + class Employee(Person): + salary = IntField() + + self.assertEqual(['salary', 'age', 'name', 'id'], + Employee._fields.keys()) + self.assertEqual(Employee._get_collection_name(), + Person._get_collection_name()) + + + def test_polymorphic_queries(self): + """Ensure that the correct subclasses are returned from a query + """ + + class Animal(Document): + meta = {'allow_inheritance': True} + class Fish(Animal): pass + class Mammal(Animal): pass + class Dog(Mammal): pass + class Human(Mammal): pass + + Animal.drop_collection() + + Animal().save() + Fish().save() + Mammal().save() + Dog().save() + Human().save() + + classes = [obj.__class__ for obj in Animal.objects] + self.assertEqual(classes, [Animal, Fish, Mammal, Dog, Human]) + + classes = [obj.__class__ for obj in Mammal.objects] + self.assertEqual(classes, [Mammal, Dog, Human]) + + classes = [obj.__class__ for obj in Human.objects] + self.assertEqual(classes, [Human]) + + + def test_allow_inheritance(self): + """Ensure that inheritance may be disabled on simple classes and that + _cls and _subclasses will not be used. + """ + + class Animal(Document): + name = StringField() + meta = {'allow_inheritance': False} + + def create_dog_class(): + class Dog(Animal): + pass + + self.assertRaises(ValueError, create_dog_class) + + # Check that _cls etc aren't present on simple documents + dog = Animal(name='dog') + dog.save() + + collection = self.db[Animal._get_collection_name()] + obj = collection.find_one() + self.assertFalse('_cls' in obj) + + def test_cant_turn_off_inheritance_on_subclass(self): + """Ensure if inheritance is on in a subclass you cant turn it off + """ + + class Animal(Document): + name = StringField() + meta = {'allow_inheritance': True} + + def create_mammal_class(): + class Mammal(Animal): + meta = {'allow_inheritance': False} + self.assertRaises(ValueError, create_mammal_class) + + def test_allow_inheritance_abstract_document(self): + """Ensure that abstract documents can set inheritance rules and that + _cls will not be used. + """ + class FinalDocument(Document): + meta = {'abstract': True, + 'allow_inheritance': False} + + class Animal(FinalDocument): + name = StringField() + + def create_mammal_class(): + class Mammal(Animal): + pass + self.assertRaises(ValueError, create_mammal_class) + + # Check that _cls isn't present in simple documents + doc = Animal(name='dog') + self.assertFalse('_cls' in doc.to_mongo()) + + def test_allow_inheritance_embedded_document(self): + """Ensure embedded documents respect inheritance + """ + + class Comment(EmbeddedDocument): + content = StringField() + meta = {'allow_inheritance': False} + + def create_special_comment(): + class SpecialComment(Comment): + pass + + self.assertRaises(ValueError, create_special_comment) + + doc = Comment(content='test') + self.assertFalse('_cls' in doc.to_mongo()) + + class Comment(EmbeddedDocument): + content = StringField() + meta = {'allow_inheritance': True} + + doc = Comment(content='test') + self.assertTrue('_cls' in doc.to_mongo()) + + def test_document_inheritance(self): + """Ensure mutliple inheritance of abstract documents + """ + class DateCreatedDocument(Document): + meta = { + 'allow_inheritance': True, + 'abstract': True, + } + + class DateUpdatedDocument(Document): + meta = { + 'allow_inheritance': True, + 'abstract': True, + } + + try: + class MyDocument(DateCreatedDocument, DateUpdatedDocument): + pass + except: + self.assertTrue(False, "Couldn't create MyDocument class") + + def test_abstract_documents(self): + """Ensure that a document superclass can be marked as abstract + thereby not using it as the name for the collection.""" + + defaults = {'index_background': True, + 'index_drop_dups': True, + 'index_opts': {'hello': 'world'}, + 'allow_inheritance': True, + 'queryset_class': 'QuerySet', + 'db_alias': 'myDB', + 'shard_key': ('hello', 'world')} + + meta_settings = {'abstract': True} + meta_settings.update(defaults) + + class Animal(Document): + name = StringField() + meta = meta_settings + + class Fish(Animal): pass + class Guppy(Fish): pass + + class Mammal(Animal): + meta = {'abstract': True} + class Human(Mammal): pass + + for k, v in defaults.iteritems(): + for cls in [Animal, Fish, Guppy]: + self.assertEqual(cls._meta[k], v) + + self.assertFalse('collection' in Animal._meta) + self.assertFalse('collection' in Mammal._meta) + + self.assertEqual(Animal._get_collection_name(), None) + self.assertEqual(Mammal._get_collection_name(), None) + + self.assertEqual(Fish._get_collection_name(), 'fish') + self.assertEqual(Guppy._get_collection_name(), 'fish') + self.assertEqual(Human._get_collection_name(), 'human') + + def create_bad_abstract(): + class EvilHuman(Human): + evil = BooleanField(default=True) + meta = {'abstract': True} + self.assertRaises(ValueError, create_bad_abstract) + + def test_inherited_collections(self): + """Ensure that subclassed documents don't override parents' + collections + """ + + class Drink(Document): + name = StringField() + meta = {'allow_inheritance': True} + + class Drinker(Document): + drink = GenericReferenceField() + + try: + warnings.simplefilter("error") + + class AcloholicDrink(Drink): + meta = {'collection': 'booze'} + + except SyntaxWarning: + warnings.simplefilter("ignore") + + class AlcoholicDrink(Drink): + meta = {'collection': 'booze'} + + else: + raise AssertionError("SyntaxWarning should be triggered") + + warnings.resetwarnings() + + Drink.drop_collection() + AlcoholicDrink.drop_collection() + Drinker.drop_collection() + + red_bull = Drink(name='Red Bull') + red_bull.save() + + programmer = Drinker(drink=red_bull) + programmer.save() + + beer = AlcoholicDrink(name='Beer') + beer.save() + real_person = Drinker(drink=beer) + real_person.save() + + self.assertEqual(Drinker.objects[0].drink.name, red_bull.name) + self.assertEqual(Drinker.objects[1].drink.name, beer.name) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_document.py b/tests/document/instance.py similarity index 50% rename from tests/test_document.py rename to tests/document/instance.py index a09aaec..95f37d9 100644 --- a/tests/test_document.py +++ b/tests/document/instance.py @@ -15,14 +15,17 @@ from datetime import datetime from tests.fixtures import Base, Mixin, PickleEmbedded, PickleTest from mongoengine import * -from mongoengine.base import NotRegistered, InvalidDocumentError -from mongoengine.queryset import InvalidQueryError +from mongoengine.errors import (NotRegistered, InvalidDocumentError, + InvalidQueryError) +from mongoengine.queryset import NULLIFY, Q from mongoengine.connection import get_db, get_connection TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') +__all__ = ("InstanceTest",) -class DocumentTest(unittest.TestCase): + +class InstanceTest(unittest.TestCase): def setUp(self): connect(db='mongoenginetest') @@ -32,59 +35,57 @@ class DocumentTest(unittest.TestCase): name = StringField() age = IntField() - meta = {'allow_inheritance': True} + non_field = True + + meta = {"allow_inheritance": True} self.Person = Person def tearDown(self): - self.Person.drop_collection() + for collection in self.db.collection_names(): + if 'system.' in collection: + continue + self.db.drop_collection(collection) - def test_drop_collection(self): - """Ensure that the collection may be dropped from the database. + def test_capped_collection(self): + """Ensure that capped collections work properly. """ - self.Person(name='Test').save() + class Log(Document): + date = DateTimeField(default=datetime.now) + meta = { + 'max_documents': 10, + 'max_size': 90000, + } - collection = self.Person._get_collection_name() - self.assertTrue(collection in self.db.collection_names()) + Log.drop_collection() - self.Person.drop_collection() - self.assertFalse(collection in self.db.collection_names()) + # Ensure that the collection handles up to its maximum + for _ in range(10): + Log().save() - def test_queryset_resurrects_dropped_collection(self): + self.assertEqual(len(Log.objects), 10) - self.Person.objects().item_frequencies('name') - self.Person.drop_collection() + # Check that extra documents don't increase the size + Log().save() + self.assertEqual(len(Log.objects), 10) - self.assertEqual({}, self.Person.objects().item_frequencies('name')) + options = Log.objects._collection.options() + self.assertEqual(options['capped'], True) + self.assertEqual(options['max'], 10) + self.assertEqual(options['size'], 90000) - class Actor(self.Person): - pass + # Check that the document cannot be redefined with different options + def recreate_log_document(): + class Log(Document): + date = DateTimeField(default=datetime.now) + meta = { + 'max_documents': 11, + } + # Create the collection by accessing Document.objects + Log.objects + self.assertRaises(InvalidCollectionError, recreate_log_document) - # Ensure works correctly with inhertited classes - Actor.objects().item_frequencies('name') - self.Person.drop_collection() - self.assertEqual({}, Actor.objects().item_frequencies('name')) - - def test_definition(self): - """Ensure that document may be defined using fields. - """ - name_field = StringField() - age_field = IntField() - - class Person(Document): - name = name_field - age = age_field - non_field = True - - self.assertEqual(Person._fields['name'], name_field) - self.assertEqual(Person._fields['age'], age_field) - self.assertFalse('non_field' in Person._fields) - self.assertTrue('id' in Person._fields) - # Test iteration over fields - fields = list(Person()) - self.assertTrue('name' in fields and 'age' in fields) - # Ensure Document isn't treated like an actual document - self.assertFalse(hasattr(Document, '_fields')) + Log.drop_collection() def test_repr(self): """Ensure that unicode representation works @@ -95,146 +96,22 @@ class DocumentTest(unittest.TestCase): def __unicode__(self): return self.title - Article.drop_collection() + doc = Article(title=u'привет мир') - Article(title=u'привет мир').save() + self.assertEqual('', repr(doc)) - self.assertEqual('', repr(Article.objects.first())) - self.assertEqual('[]', repr(Article.objects.all())) + def test_queryset_resurrects_dropped_collection(self): + self.Person.drop_collection() - def test_collection_naming(self): - """Ensure that a collection with a specified name may be used. - """ + self.assertEqual([], list(self.Person.objects())) - class DefaultNamingTest(Document): - pass - self.assertEqual('default_naming_test', DefaultNamingTest._get_collection_name()) - - class CustomNamingTest(Document): - meta = {'collection': 'pimp_my_collection'} - - self.assertEqual('pimp_my_collection', CustomNamingTest._get_collection_name()) - - class DynamicNamingTest(Document): - meta = {'collection': lambda c: "DYNAMO"} - self.assertEqual('DYNAMO', DynamicNamingTest._get_collection_name()) - - # Use Abstract class to handle backwards compatibility - class BaseDocument(Document): - meta = { - 'abstract': True, - 'collection': lambda c: c.__name__.lower() - } - - class OldNamingConvention(BaseDocument): - pass - self.assertEqual('oldnamingconvention', OldNamingConvention._get_collection_name()) - - class InheritedAbstractNamingTest(BaseDocument): - meta = {'collection': 'wibble'} - self.assertEqual('wibble', InheritedAbstractNamingTest._get_collection_name()) - - - # Mixin tests - class BaseMixin(object): - meta = { - 'collection': lambda c: c.__name__.lower() - } - - class OldMixinNamingConvention(Document, BaseMixin): - pass - self.assertEqual('oldmixinnamingconvention', OldMixinNamingConvention._get_collection_name()) - - class BaseMixin(object): - meta = { - 'collection': lambda c: c.__name__.lower() - } - - class BaseDocument(Document, BaseMixin): - meta = {'allow_inheritance': True} - - class MyDocument(BaseDocument): + class Actor(self.Person): pass - self.assertEqual('basedocument', MyDocument._get_collection_name()) - - def test_get_superclasses(self): - """Ensure that the correct list of superclasses is assembled. - """ - class Animal(Document): - meta = {'allow_inheritance': True} - class Fish(Animal): pass - class Mammal(Animal): pass - class Human(Mammal): pass - class Dog(Mammal): pass - - mammal_superclasses = {'Animal': Animal} - self.assertEqual(Mammal._superclasses, mammal_superclasses) - - dog_superclasses = { - 'Animal': Animal, - 'Animal.Mammal': Mammal, - } - self.assertEqual(Dog._superclasses, dog_superclasses) - - def test_external_superclasses(self): - """Ensure that the correct list of sub and super classes is assembled. - when importing part of the model - """ - class Animal(Base): pass - class Fish(Animal): pass - class Mammal(Animal): pass - class Human(Mammal): pass - class Dog(Mammal): pass - - mammal_superclasses = {'Base': Base, 'Base.Animal': Animal} - self.assertEqual(Mammal._superclasses, mammal_superclasses) - - dog_superclasses = { - 'Base': Base, - 'Base.Animal': Animal, - 'Base.Animal.Mammal': Mammal, - } - self.assertEqual(Dog._superclasses, dog_superclasses) - - Base.drop_collection() - - h = Human() - h.save() - - self.assertEqual(Human.objects.count(), 1) - self.assertEqual(Mammal.objects.count(), 1) - self.assertEqual(Animal.objects.count(), 1) - self.assertEqual(Base.objects.count(), 1) - Base.drop_collection() - - def test_polymorphic_queries(self): - """Ensure that the correct subclasses are returned from a query""" - class Animal(Document): - meta = {'allow_inheritance': True} - class Fish(Animal): pass - class Mammal(Animal): pass - class Human(Mammal): pass - class Dog(Mammal): pass - - Animal.drop_collection() - - Animal().save() - Fish().save() - Mammal().save() - Human().save() - Dog().save() - - classes = [obj.__class__ for obj in Animal.objects] - self.assertEqual(classes, [Animal, Fish, Mammal, Human, Dog]) - - classes = [obj.__class__ for obj in Mammal.objects] - self.assertEqual(classes, [Mammal, Human, Dog]) - - classes = [obj.__class__ for obj in Human.objects] - self.assertEqual(classes, [Human]) - - Animal.drop_collection() + # Ensure works correctly with inhertited classes + Actor.objects() + self.Person.drop_collection() + self.assertEqual([], list(Actor.objects())) def test_polymorphic_references(self): """Ensure that the correct subclasses are returned from a query when @@ -244,8 +121,8 @@ class DocumentTest(unittest.TestCase): meta = {'allow_inheritance': True} class Fish(Animal): pass class Mammal(Animal): pass - class Human(Mammal): pass class Dog(Mammal): pass + class Human(Mammal): pass class Zoo(Document): animals = ListField(ReferenceField(Animal)) @@ -256,8 +133,8 @@ class DocumentTest(unittest.TestCase): Animal().save() Fish().save() Mammal().save() - Human().save() Dog().save() + Human().save() # Save a reference to each animal zoo = Zoo(animals=Animal.objects) @@ -265,7 +142,7 @@ class DocumentTest(unittest.TestCase): zoo.reload() classes = [a.__class__ for a in Zoo.objects.first().animals] - self.assertEqual(classes, [Animal, Fish, Mammal, Human, Dog]) + self.assertEqual(classes, [Animal, Fish, Mammal, Dog, Human]) Zoo.drop_collection() @@ -278,7 +155,7 @@ class DocumentTest(unittest.TestCase): zoo.reload() classes = [a.__class__ for a in Zoo.objects.first().animals] - self.assertEqual(classes, [Animal, Fish, Mammal, Human, Dog]) + self.assertEqual(classes, [Animal, Fish, Mammal, Dog, Human]) Zoo.drop_collection() Animal.drop_collection() @@ -308,466 +185,8 @@ class DocumentTest(unittest.TestCase): self.assertEqual(list_stats, CompareStats.objects.first().stats) - def test_inheritance(self): - """Ensure that document may inherit fields from a superclass document. - """ - class Employee(self.Person): - salary = IntField() - self.assertTrue('name' in Employee._fields) - self.assertTrue('salary' in Employee._fields) - self.assertEqual(Employee._get_collection_name(), - self.Person._get_collection_name()) - # Ensure that MRO error is not raised - class A(Document): - meta = {'allow_inheritance': True} - class B(A): pass - class C(B): pass - - def test_allow_inheritance(self): - """Ensure that inheritance may be disabled on simple classes and that - _cls and _types will not be used. - """ - - class Animal(Document): - name = StringField() - meta = {'allow_inheritance': False} - - Animal.drop_collection() - def create_dog_class(): - class Dog(Animal): - pass - self.assertRaises(ValueError, create_dog_class) - - # Check that _cls etc aren't present on simple documents - dog = Animal(name='dog') - dog.save() - collection = self.db[Animal._get_collection_name()] - obj = collection.find_one() - self.assertFalse('_cls' in obj) - self.assertFalse('_types' in obj) - - Animal.drop_collection() - - def create_employee_class(): - class Employee(self.Person): - meta = {'allow_inheritance': False} - self.assertRaises(ValueError, create_employee_class) - - def test_allow_inheritance_abstract_document(self): - """Ensure that abstract documents can set inheritance rules and that - _cls and _types will not be used. - """ - class FinalDocument(Document): - meta = {'abstract': True, - 'allow_inheritance': False} - - class Animal(FinalDocument): - name = StringField() - - Animal.drop_collection() - def create_dog_class(): - class Dog(Animal): - pass - self.assertRaises(ValueError, create_dog_class) - - # Check that _cls etc aren't present on simple documents - dog = Animal(name='dog') - dog.save() - collection = self.db[Animal._get_collection_name()] - obj = collection.find_one() - self.assertFalse('_cls' in obj) - self.assertFalse('_types' in obj) - - Animal.drop_collection() - - def test_allow_inheritance_embedded_document(self): - - # Test the same for embedded documents - class Comment(EmbeddedDocument): - content = StringField() - meta = {'allow_inheritance': False} - - def create_special_comment(): - class SpecialComment(Comment): - pass - - self.assertRaises(ValueError, create_special_comment) - - comment = Comment(content='test') - self.assertFalse('_cls' in comment.to_mongo()) - self.assertFalse('_types' in comment.to_mongo()) - - class Comment(EmbeddedDocument): - content = StringField() - meta = {'allow_inheritance': True} - - comment = Comment(content='test') - self.assertTrue('_cls' in comment.to_mongo()) - self.assertTrue('_types' in comment.to_mongo()) - - def test_document_inheritance(self): - """Ensure mutliple inheritance of abstract docs works - """ - class DateCreatedDocument(Document): - meta = { - 'allow_inheritance': True, - 'abstract': True, - } - - class DateUpdatedDocument(Document): - meta = { - 'allow_inheritance': True, - 'abstract': True, - } - - try: - class MyDocument(DateCreatedDocument, DateUpdatedDocument): - pass - except: - self.assertTrue(False, "Couldn't create MyDocument class") - - def test_how_to_turn_off_inheritance(self): - """Demonstrates migrating from allow_inheritance = True to False. - """ - class Animal(Document): - name = StringField() - meta = { - 'indexes': ['name'] - } - - self.assertEqual(Animal._meta['index_specs'], - [{'fields': [('_types', 1), ('name', 1)]}]) - - Animal.drop_collection() - - dog = Animal(name='dog') - dog.save() - - collection = self.db[Animal._get_collection_name()] - obj = collection.find_one() - self.assertTrue('_cls' in obj) - self.assertTrue('_types' in obj) - - info = collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertEqual([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], info) - - # Turn off inheritance - class Animal(Document): - name = StringField() - meta = { - 'allow_inheritance': False, - 'indexes': ['name'] - } - - self.assertEqual(Animal._meta['index_specs'], - [{'fields': [('name', 1)]}]) - collection.update({}, {"$unset": {"_types": 1, "_cls": 1}}, multi=True) - - # Confirm extra data is removed - obj = collection.find_one() - self.assertFalse('_cls' in obj) - self.assertFalse('_types' in obj) - - info = collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertEqual([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], info) - - info = collection.index_information() - indexes_to_drop = [key for key, value in info.iteritems() if '_types' in dict(value['key'])] - for index in indexes_to_drop: - collection.drop_index(index) - - info = collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertEqual([[(u'_id', 1)]], info) - - # Recreate indexes - dog = Animal.objects.first() - dog.save() - info = collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertEqual([[(u'_id', 1)], [(u'name', 1),]], info) - - Animal.drop_collection() - - def test_abstract_documents(self): - """Ensure that a document superclass can be marked as abstract - thereby not using it as the name for the collection.""" - - defaults = {'index_background': True, - 'index_drop_dups': True, - 'index_opts': {'hello': 'world'}, - 'allow_inheritance': True, - 'queryset_class': 'QuerySet', - 'db_alias': 'myDB', - 'shard_key': ('hello', 'world')} - - meta_settings = {'abstract': True} - meta_settings.update(defaults) - - class Animal(Document): - name = StringField() - meta = meta_settings - - class Fish(Animal): pass - class Guppy(Fish): pass - - class Mammal(Animal): - meta = {'abstract': True} - class Human(Mammal): pass - - for k, v in defaults.iteritems(): - for cls in [Animal, Fish, Guppy]: - self.assertEqual(cls._meta[k], v) - - self.assertFalse('collection' in Animal._meta) - self.assertFalse('collection' in Mammal._meta) - - self.assertEqual(Animal._get_collection_name(), None) - self.assertEqual(Mammal._get_collection_name(), None) - - self.assertEqual(Fish._get_collection_name(), 'fish') - self.assertEqual(Guppy._get_collection_name(), 'fish') - self.assertEqual(Human._get_collection_name(), 'human') - - def create_bad_abstract(): - class EvilHuman(Human): - evil = BooleanField(default=True) - meta = {'abstract': True} - self.assertRaises(ValueError, create_bad_abstract) - - def test_collection_name(self): - """Ensure that a collection with a specified name may be used. - """ - collection = 'personCollTest' - if collection in self.db.collection_names(): - self.db.drop_collection(collection) - - class Person(Document): - name = StringField() - meta = {'collection': collection} - - user = Person(name="Test User") - user.save() - self.assertTrue(collection in self.db.collection_names()) - - user_obj = self.db[collection].find_one() - self.assertEqual(user_obj['name'], "Test User") - - user_obj = Person.objects[0] - self.assertEqual(user_obj.name, "Test User") - - Person.drop_collection() - self.assertFalse(collection in self.db.collection_names()) - - def test_collection_name_and_primary(self): - """Ensure that a collection with a specified name may be used. - """ - - class Person(Document): - name = StringField(primary_key=True) - meta = {'collection': 'app'} - - user = Person(name="Test User") - user.save() - - user_obj = Person.objects[0] - self.assertEqual(user_obj.name, "Test User") - - Person.drop_collection() - - def test_inherited_collections(self): - """Ensure that subclassed documents don't override parents' collections. - """ - - class Drink(Document): - name = StringField() - meta = {'allow_inheritance': True} - - class Drinker(Document): - drink = GenericReferenceField() - - try: - warnings.simplefilter("error") - - class AcloholicDrink(Drink): - meta = {'collection': 'booze'} - - except SyntaxWarning, w: - warnings.simplefilter("ignore") - - class AlcoholicDrink(Drink): - meta = {'collection': 'booze'} - - else: - raise AssertionError("SyntaxWarning should be triggered") - - warnings.resetwarnings() - - Drink.drop_collection() - AlcoholicDrink.drop_collection() - Drinker.drop_collection() - - red_bull = Drink(name='Red Bull') - red_bull.save() - - programmer = Drinker(drink=red_bull) - programmer.save() - - beer = AlcoholicDrink(name='Beer') - beer.save() - real_person = Drinker(drink=beer) - real_person.save() - - self.assertEqual(Drinker.objects[0].drink.name, red_bull.name) - self.assertEqual(Drinker.objects[1].drink.name, beer.name) - - def test_capped_collection(self): - """Ensure that capped collections work properly. - """ - class Log(Document): - date = DateTimeField(default=datetime.now) - meta = { - 'max_documents': 10, - 'max_size': 90000, - } - - Log.drop_collection() - - # Ensure that the collection handles up to its maximum - for i in range(10): - Log().save() - - self.assertEqual(len(Log.objects), 10) - - # Check that extra documents don't increase the size - Log().save() - self.assertEqual(len(Log.objects), 10) - - options = Log.objects._collection.options() - self.assertEqual(options['capped'], True) - self.assertEqual(options['max'], 10) - self.assertEqual(options['size'], 90000) - - # Check that the document cannot be redefined with different options - def recreate_log_document(): - class Log(Document): - date = DateTimeField(default=datetime.now) - meta = { - 'max_documents': 11, - } - # Create the collection by accessing Document.objects - Log.objects - self.assertRaises(InvalidCollectionError, recreate_log_document) - - Log.drop_collection() - - def test_indexes(self): - """Ensure that indexes are used when meta[indexes] is specified. - """ - class BlogPost(Document): - date = DateTimeField(db_field='addDate', default=datetime.now) - category = StringField() - tags = ListField(StringField()) - meta = { - 'indexes': [ - '-date', - 'tags', - ('category', '-date') - ], - 'allow_inheritance': True - } - - self.assertEqual(BlogPost._meta['index_specs'], - [{'fields': [('_types', 1), ('addDate', -1)]}, - {'fields': [('tags', 1)]}, - {'fields': [('_types', 1), ('category', 1), - ('addDate', -1)]}]) - - BlogPost.drop_collection() - - info = BlogPost.objects._collection.index_information() - # _id, '-date', 'tags', ('cat', 'date') - # NB: there is no index on _types by itself, since - # the indices on -date and tags will both contain - # _types as first element in the key - self.assertEqual(len(info), 4) - - # Indexes are lazy so use list() to perform query - list(BlogPost.objects) - info = BlogPost.objects._collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] - in info) - self.assertTrue([('_types', 1), ('addDate', -1)] in info) - # tags is a list field so it shouldn't have _types in the index - self.assertTrue([('tags', 1)] in info) - - class ExtendedBlogPost(BlogPost): - title = StringField() - meta = {'indexes': ['title']} - - self.assertEqual(ExtendedBlogPost._meta['index_specs'], - [{'fields': [('_types', 1), ('addDate', -1)]}, - {'fields': [('tags', 1)]}, - {'fields': [('_types', 1), ('category', 1), - ('addDate', -1)]}, - {'fields': [('_types', 1), ('title', 1)]}]) - - BlogPost.drop_collection() - - list(ExtendedBlogPost.objects) - info = ExtendedBlogPost.objects._collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertTrue([('_types', 1), ('category', 1), ('addDate', -1)] - in info) - self.assertTrue([('_types', 1), ('addDate', -1)] in info) - self.assertTrue([('_types', 1), ('title', 1)] in info) - - BlogPost.drop_collection() - - def test_inherited_index(self): - """Ensure index specs are inhertited correctly""" - - class A(Document): - title = StringField() - meta = { - 'indexes': [ - { - 'fields': ('title',), - }, - ], - 'allow_inheritance': True, - } - - class B(A): - description = StringField() - - self.assertEqual(A._meta['index_specs'], B._meta['index_specs']) - self.assertEqual([{'fields': [('_types', 1), ('title', 1)]}], - A._meta['index_specs']) - - def test_build_index_spec_is_not_destructive(self): - - class MyDoc(Document): - keywords = StringField() - - meta = { - 'indexes': ['keywords'], - 'allow_inheritance': False - } - - self.assertEqual(MyDoc._meta['index_specs'], - [{'fields': [('keywords', 1)]}]) - - # Force index creation - MyDoc.objects._ensure_indexes() - - self.assertEqual(MyDoc._meta['index_specs'], - [{'fields': [('keywords', 1)]}]) def test_db_field_load(self): """Ensure we load data correctly @@ -812,477 +231,8 @@ class DocumentTest(unittest.TestCase): self.assertEqual(Person.objects.get(name="Jack").rank, "Corporal") self.assertEqual(Person.objects.get(name="Fred").rank, "Private") - def test_embedded_document_index_meta(self): - """Ensure that embedded document indexes are created explicitly - """ - class Rank(EmbeddedDocument): - title = StringField(required=True) - class Person(Document): - name = StringField(required=True) - rank = EmbeddedDocumentField(Rank, required=False) - meta = { - 'indexes': [ - 'rank.title', - ], - 'allow_inheritance': False - } - - self.assertEqual([{'fields': [('rank.title', 1)]}], - Person._meta['index_specs']) - - Person.drop_collection() - - # Indexes are lazy so use list() to perform query - list(Person.objects) - info = Person.objects._collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertTrue([('rank.title', 1)] in info) - - def test_explicit_geo2d_index(self): - """Ensure that geo2d indexes work when created via meta[indexes] - """ - class Place(Document): - location = DictField() - meta = { - 'indexes': [ - '*location.point', - ], - } - - self.assertEqual([{'fields': [('location.point', '2d')]}], - Place._meta['index_specs']) - - Place.drop_collection() - - info = Place.objects._collection.index_information() - # Indexes are lazy so use list() to perform query - list(Place.objects) - info = Place.objects._collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - - self.assertTrue([('location.point', '2d')] in info) - - def test_dictionary_indexes(self): - """Ensure that indexes are used when meta[indexes] contains dictionaries - instead of lists. - """ - class BlogPost(Document): - date = DateTimeField(db_field='addDate', default=datetime.now) - category = StringField() - tags = ListField(StringField()) - meta = { - 'indexes': [ - {'fields': ['-date'], 'unique': True, - 'sparse': True, 'types': False }, - ], - } - - self.assertEqual([{'fields': [('addDate', -1)], 'unique': True, - 'sparse': True, 'types': False}], - BlogPost._meta['index_specs']) - - BlogPost.drop_collection() - - info = BlogPost.objects._collection.index_information() - # _id, '-date' - self.assertEqual(len(info), 3) - - # Indexes are lazy so use list() to perform query - list(BlogPost.objects) - info = BlogPost.objects._collection.index_information() - info = [(value['key'], - value.get('unique', False), - value.get('sparse', False)) - for key, value in info.iteritems()] - self.assertTrue(([('addDate', -1)], True, True) in info) - - BlogPost.drop_collection() - - def test_abstract_index_inheritance(self): - - class UserBase(Document): - meta = { - 'abstract': True, - 'indexes': ['user_guid'] - } - - user_guid = StringField(required=True) - - class Person(UserBase): - meta = { - 'indexes': ['name'], - } - - name = StringField() - - Person.drop_collection() - - p = Person(name="test", user_guid='123') - p.save() - - self.assertEqual(1, Person.objects.count()) - info = Person.objects._collection.index_information() - self.assertEqual(info.keys(), ['_types_1_user_guid_1', '_id_', '_types_1_name_1']) - Person.drop_collection() - - def test_disable_index_creation(self): - """Tests setting auto_create_index to False on the connection will - disable any index generation. - """ - class User(Document): - meta = { - 'indexes': ['user_guid'], - 'auto_create_index': False - } - user_guid = StringField(required=True) - - - User.drop_collection() - - u = User(user_guid='123') - u.save() - - self.assertEqual(1, User.objects.count()) - info = User.objects._collection.index_information() - self.assertEqual(info.keys(), ['_id_']) - User.drop_collection() - - def test_embedded_document_index(self): - """Tests settings an index on an embedded document - """ - class Date(EmbeddedDocument): - year = IntField(db_field='yr') - - class BlogPost(Document): - title = StringField() - date = EmbeddedDocumentField(Date) - - meta = { - 'indexes': [ - '-date.year' - ], - } - - BlogPost.drop_collection() - - info = BlogPost.objects._collection.index_information() - self.assertEqual(info.keys(), ['_types_1_date.yr_-1', '_id_']) - BlogPost.drop_collection() - - def test_list_embedded_document_index(self): - """Ensure list embedded documents can be indexed - """ - class Tag(EmbeddedDocument): - name = StringField(db_field='tag') - - class BlogPost(Document): - title = StringField() - tags = ListField(EmbeddedDocumentField(Tag)) - - meta = { - 'indexes': [ - 'tags.name' - ], - } - - BlogPost.drop_collection() - - info = BlogPost.objects._collection.index_information() - # we don't use _types in with list fields by default - self.assertEqual(info.keys(), ['_id_', '_types_1', 'tags.tag_1']) - - post1 = BlogPost(title="Embedded Indexes tests in place", - tags=[Tag(name="about"), Tag(name="time")] - ) - post1.save() - BlogPost.drop_collection() - - def test_recursive_embedded_objects_dont_break_indexes(self): - - class RecursiveObject(EmbeddedDocument): - obj = EmbeddedDocumentField('self') - - class RecursiveDocument(Document): - recursive_obj = EmbeddedDocumentField(RecursiveObject) - - info = RecursiveDocument.objects._collection.index_information() - self.assertEqual(info.keys(), ['_id_', '_types_1']) - - def test_geo_indexes_recursion(self): - - class Location(Document): - name = StringField() - location = GeoPointField() - - class Parent(Document): - name = StringField() - location = ReferenceField(Location) - - Location.drop_collection() - Parent.drop_collection() - - list(Parent.objects) - - collection = Parent._get_collection() - info = collection.index_information() - - self.assertFalse('location_2d' in info) - - self.assertEqual(len(Parent._geo_indices()), 0) - self.assertEqual(len(Location._geo_indices()), 1) - - def test_covered_index(self): - """Ensure that covered indexes can be used - """ - - class Test(Document): - a = IntField() - - meta = { - 'indexes': ['a'], - 'allow_inheritance': False - } - - Test.drop_collection() - - obj = Test(a=1) - obj.save() - - # Need to be explicit about covered indexes as mongoDB doesn't know if - # the documents returned might have more keys in that here. - query_plan = Test.objects(id=obj.id).exclude('a').explain() - self.assertFalse(query_plan['indexOnly']) - - query_plan = Test.objects(id=obj.id).only('id').explain() - self.assertTrue(query_plan['indexOnly']) - - query_plan = Test.objects(a=1).only('a').exclude('id').explain() - self.assertTrue(query_plan['indexOnly']) - - def test_index_on_id(self): - - class BlogPost(Document): - meta = { - 'indexes': [ - ['categories', 'id'] - ], - 'allow_inheritance': False - } - - title = StringField(required=True) - description = StringField(required=True) - categories = ListField() - - BlogPost.drop_collection() - - indexes = BlogPost.objects._collection.index_information() - self.assertEqual(indexes['categories_1__id_1']['key'], - [('categories', 1), ('_id', 1)]) - - def test_hint(self): - - class BlogPost(Document): - tags = ListField(StringField()) - meta = { - 'indexes': [ - 'tags', - ], - } - - BlogPost.drop_collection() - - for i in xrange(0, 10): - tags = [("tag %i" % n) for n in xrange(0, i % 2)] - BlogPost(tags=tags).save() - - self.assertEqual(BlogPost.objects.count(), 10) - self.assertEqual(BlogPost.objects.hint().count(), 10) - self.assertEqual(BlogPost.objects.hint([('tags', 1)]).count(), 10) - - self.assertEqual(BlogPost.objects.hint([('ZZ', 1)]).count(), 10) - - def invalid_index(): - BlogPost.objects.hint('tags') - self.assertRaises(TypeError, invalid_index) - - def invalid_index_2(): - return BlogPost.objects.hint(('tags', 1)) - self.assertRaises(TypeError, invalid_index_2) - - def test_unique(self): - """Ensure that uniqueness constraints are applied to fields. - """ - class BlogPost(Document): - title = StringField() - slug = StringField(unique=True) - - BlogPost.drop_collection() - - post1 = BlogPost(title='test1', slug='test') - post1.save() - - # Two posts with the same slug is not allowed - post2 = BlogPost(title='test2', slug='test') - self.assertRaises(NotUniqueError, post2.save) - - # Ensure backwards compatibilty for errors - self.assertRaises(OperationError, post2.save) - - def test_unique_with(self): - """Ensure that unique_with constraints are applied to fields. - """ - class Date(EmbeddedDocument): - year = IntField(db_field='yr') - - class BlogPost(Document): - title = StringField() - date = EmbeddedDocumentField(Date) - slug = StringField(unique_with='date.year') - - BlogPost.drop_collection() - - post1 = BlogPost(title='test1', date=Date(year=2009), slug='test') - post1.save() - - # day is different so won't raise exception - post2 = BlogPost(title='test2', date=Date(year=2010), slug='test') - post2.save() - - # Now there will be two docs with the same slug and the same day: fail - post3 = BlogPost(title='test3', date=Date(year=2010), slug='test') - self.assertRaises(OperationError, post3.save) - - BlogPost.drop_collection() - - def test_unique_embedded_document(self): - """Ensure that uniqueness constraints are applied to fields on embedded documents. - """ - class SubDocument(EmbeddedDocument): - year = IntField(db_field='yr') - slug = StringField(unique=True) - - class BlogPost(Document): - title = StringField() - sub = EmbeddedDocumentField(SubDocument) - - BlogPost.drop_collection() - - post1 = BlogPost(title='test1', sub=SubDocument(year=2009, slug="test")) - post1.save() - - # sub.slug is different so won't raise exception - post2 = BlogPost(title='test2', sub=SubDocument(year=2010, slug='another-slug')) - post2.save() - - # Now there will be two docs with the same sub.slug - post3 = BlogPost(title='test3', sub=SubDocument(year=2010, slug='test')) - self.assertRaises(NotUniqueError, post3.save) - - BlogPost.drop_collection() - - def test_unique_with_embedded_document_and_embedded_unique(self): - """Ensure that uniqueness constraints are applied to fields on - embedded documents. And work with unique_with as well. - """ - class SubDocument(EmbeddedDocument): - year = IntField(db_field='yr') - slug = StringField(unique=True) - - class BlogPost(Document): - title = StringField(unique_with='sub.year') - sub = EmbeddedDocumentField(SubDocument) - - BlogPost.drop_collection() - - post1 = BlogPost(title='test1', sub=SubDocument(year=2009, slug="test")) - post1.save() - - # sub.slug is different so won't raise exception - post2 = BlogPost(title='test2', sub=SubDocument(year=2010, slug='another-slug')) - post2.save() - - # Now there will be two docs with the same sub.slug - post3 = BlogPost(title='test3', sub=SubDocument(year=2010, slug='test')) - self.assertRaises(NotUniqueError, post3.save) - - # Now there will be two docs with the same title and year - post3 = BlogPost(title='test1', sub=SubDocument(year=2009, slug='test-1')) - self.assertRaises(NotUniqueError, post3.save) - - BlogPost.drop_collection() - - def test_ttl_indexes(self): - - class Log(Document): - created = DateTimeField(default=datetime.now) - meta = { - 'indexes': [ - {'fields': ['created'], 'expireAfterSeconds': 3600} - ] - } - - Log.drop_collection() - - if pymongo.version_tuple[0] < 2 and pymongo.version_tuple[1] < 3: - raise SkipTest('pymongo needs to be 2.3 or higher for this test') - - connection = get_connection() - version_array = connection.server_info()['versionArray'] - if version_array[0] < 2 and version_array[1] < 2: - raise SkipTest('MongoDB needs to be 2.2 or higher for this test') - - # Indexes are lazy so use list() to perform query - list(Log.objects) - info = Log.objects._collection.index_information() - self.assertEqual(3600, - info['_types_1_created_1']['expireAfterSeconds']) - - def test_unique_and_indexes(self): - """Ensure that 'unique' constraints aren't overridden by - meta.indexes. - """ - class Customer(Document): - cust_id = IntField(unique=True, required=True) - meta = { - 'indexes': ['cust_id'], - 'allow_inheritance': False, - } - - Customer.drop_collection() - cust = Customer(cust_id=1) - cust.save() - - cust_dupe = Customer(cust_id=1) - try: - cust_dupe.save() - raise AssertionError, "We saved a dupe!" - except NotUniqueError: - pass - Customer.drop_collection() - - def test_unique_and_primary(self): - """If you set a field as primary, then unexpected behaviour can occur. - You won't create a duplicate but you will update an existing document. - """ - - class User(Document): - name = StringField(primary_key=True, unique=True) - password = StringField() - - User.drop_collection() - - user = User(name='huangz', password='secret') - user.save() - - user = User(name='huangz', password='secret2') - user.save() - - self.assertEqual(User.objects.count(), 1) - self.assertEqual(User.objects.get().password, 'secret2') - - User.drop_collection() def test_custom_id_field(self): """Ensure that documents may be created with custom primary keys. @@ -1876,7 +826,6 @@ class DocumentTest(unittest.TestCase): class Site(Document): page = EmbeddedDocumentField(Page) - Site.drop_collection() site = Site(page=Page(log_message="Warning: Dummy message")) site.save() @@ -1903,7 +852,6 @@ class DocumentTest(unittest.TestCase): class Site(Document): page = EmbeddedDocumentField(Page) - Site.drop_collection() site = Site(page=Page(log_message="Warning: Dummy message")) @@ -1917,519 +865,6 @@ class DocumentTest(unittest.TestCase): site = Site.objects.first() self.assertEqual(site.page.log_message, "Error: Dummy message") - def test_circular_reference_deltas(self): - - class Person(Document): - name = StringField() - owns = ListField(ReferenceField('Organization')) - - class Organization(Document): - name = StringField() - owner = ReferenceField('Person') - - Person.drop_collection() - Organization.drop_collection() - - person = Person(name="owner") - person.save() - organization = Organization(name="company") - organization.save() - - person.owns.append(organization) - organization.owner = person - - person.save() - organization.save() - - p = Person.objects[0].select_related() - o = Organization.objects.first() - self.assertEqual(p.owns[0], o) - self.assertEqual(o.owner, p) - - def test_circular_reference_deltas_2(self): - - class Person(Document): - name = StringField() - owns = ListField( ReferenceField( 'Organization' ) ) - employer = ReferenceField( 'Organization' ) - - class Organization( Document ): - name = StringField() - owner = ReferenceField( 'Person' ) - employees = ListField( ReferenceField( 'Person' ) ) - - Person.drop_collection() - Organization.drop_collection() - - person = Person( name="owner" ) - person.save() - - employee = Person( name="employee" ) - employee.save() - - organization = Organization( name="company" ) - organization.save() - - person.owns.append( organization ) - organization.owner = person - - organization.employees.append( employee ) - employee.employer = organization - - person.save() - organization.save() - employee.save() - - p = Person.objects.get(name="owner") - e = Person.objects.get(name="employee") - o = Organization.objects.first() - - self.assertEqual(p.owns[0], o) - self.assertEqual(o.owner, p) - self.assertEqual(e.employer, o) - - def test_delta(self): - - class Doc(Document): - string_field = StringField() - int_field = IntField() - dict_field = DictField() - list_field = ListField() - - Doc.drop_collection() - doc = Doc() - doc.save() - - doc = Doc.objects.first() - self.assertEqual(doc._get_changed_fields(), []) - self.assertEqual(doc._delta(), ({}, {})) - - doc.string_field = 'hello' - self.assertEqual(doc._get_changed_fields(), ['string_field']) - self.assertEqual(doc._delta(), ({'string_field': 'hello'}, {})) - - doc._changed_fields = [] - doc.int_field = 1 - self.assertEqual(doc._get_changed_fields(), ['int_field']) - self.assertEqual(doc._delta(), ({'int_field': 1}, {})) - - doc._changed_fields = [] - dict_value = {'hello': 'world', 'ping': 'pong'} - doc.dict_field = dict_value - self.assertEqual(doc._get_changed_fields(), ['dict_field']) - self.assertEqual(doc._delta(), ({'dict_field': dict_value}, {})) - - doc._changed_fields = [] - list_value = ['1', 2, {'hello': 'world'}] - doc.list_field = list_value - self.assertEqual(doc._get_changed_fields(), ['list_field']) - self.assertEqual(doc._delta(), ({'list_field': list_value}, {})) - - # Test unsetting - doc._changed_fields = [] - doc.dict_field = {} - self.assertEqual(doc._get_changed_fields(), ['dict_field']) - self.assertEqual(doc._delta(), ({}, {'dict_field': 1})) - - doc._changed_fields = [] - doc.list_field = [] - self.assertEqual(doc._get_changed_fields(), ['list_field']) - self.assertEqual(doc._delta(), ({}, {'list_field': 1})) - - def test_delta_recursive(self): - - class Embedded(EmbeddedDocument): - string_field = StringField() - int_field = IntField() - dict_field = DictField() - list_field = ListField() - - class Doc(Document): - string_field = StringField() - int_field = IntField() - dict_field = DictField() - list_field = ListField() - embedded_field = EmbeddedDocumentField(Embedded) - - Doc.drop_collection() - doc = Doc() - doc.save() - - doc = Doc.objects.first() - self.assertEqual(doc._get_changed_fields(), []) - self.assertEqual(doc._delta(), ({}, {})) - - embedded_1 = Embedded() - embedded_1.string_field = 'hello' - embedded_1.int_field = 1 - embedded_1.dict_field = {'hello': 'world'} - embedded_1.list_field = ['1', 2, {'hello': 'world'}] - doc.embedded_field = embedded_1 - - self.assertEqual(doc._get_changed_fields(), ['embedded_field']) - - embedded_delta = { - 'string_field': 'hello', - 'int_field': 1, - 'dict_field': {'hello': 'world'}, - 'list_field': ['1', 2, {'hello': 'world'}] - } - self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) - embedded_delta.update({ - '_types': ['Embedded'], - '_cls': 'Embedded', - }) - self.assertEqual(doc._delta(), ({'embedded_field': embedded_delta}, {})) - - doc.save() - doc = doc.reload(10) - - doc.embedded_field.dict_field = {} - self.assertEqual(doc._get_changed_fields(), ['embedded_field.dict_field']) - self.assertEqual(doc.embedded_field._delta(), ({}, {'dict_field': 1})) - self.assertEqual(doc._delta(), ({}, {'embedded_field.dict_field': 1})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.dict_field, {}) - - doc.embedded_field.list_field = [] - self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEqual(doc.embedded_field._delta(), ({}, {'list_field': 1})) - self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field': 1})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field, []) - - embedded_2 = Embedded() - embedded_2.string_field = 'hello' - embedded_2.int_field = 1 - embedded_2.dict_field = {'hello': 'world'} - embedded_2.list_field = ['1', 2, {'hello': 'world'}] - - doc.embedded_field.list_field = ['1', 2, embedded_2] - self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEqual(doc.embedded_field._delta(), ({ - 'list_field': ['1', 2, { - '_cls': 'Embedded', - '_types': ['Embedded'], - 'string_field': 'hello', - 'dict_field': {'hello': 'world'}, - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], - }] - }, {})) - - self.assertEqual(doc._delta(), ({ - 'embedded_field.list_field': ['1', 2, { - '_cls': 'Embedded', - '_types': ['Embedded'], - 'string_field': 'hello', - 'dict_field': {'hello': 'world'}, - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], - }] - }, {})) - doc.save() - doc = doc.reload(10) - - self.assertEqual(doc.embedded_field.list_field[0], '1') - self.assertEqual(doc.embedded_field.list_field[1], 2) - for k in doc.embedded_field.list_field[2]._fields: - self.assertEqual(doc.embedded_field.list_field[2][k], embedded_2[k]) - - doc.embedded_field.list_field[2].string_field = 'world' - self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field.2.string_field']) - self.assertEqual(doc.embedded_field._delta(), ({'list_field.2.string_field': 'world'}, {})) - self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.string_field': 'world'}, {})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field[2].string_field, 'world') - - # Test multiple assignments - doc.embedded_field.list_field[2].string_field = 'hello world' - doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] - self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEqual(doc.embedded_field._delta(), ({ - 'list_field': ['1', 2, { - '_types': ['Embedded'], - '_cls': 'Embedded', - 'string_field': 'hello world', - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], - 'dict_field': {'hello': 'world'}}]}, {})) - self.assertEqual(doc._delta(), ({ - 'embedded_field.list_field': ['1', 2, { - '_types': ['Embedded'], - '_cls': 'Embedded', - 'string_field': 'hello world', - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], - 'dict_field': {'hello': 'world'}} - ]}, {})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field[2].string_field, 'hello world') - - # Test list native methods - doc.embedded_field.list_field[2].list_field.pop(0) - self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}]}, {})) - doc.save() - doc = doc.reload(10) - - doc.embedded_field.list_field[2].list_field.append(1) - self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}, 1]}, {})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) - - doc.embedded_field.list_field[2].list_field.sort(key=str) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) - - del(doc.embedded_field.list_field[2].list_field[2]['hello']) - self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) - doc.save() - doc = doc.reload(10) - - del(doc.embedded_field.list_field[2].list_field) - self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field.2.list_field': 1})) - - doc.save() - doc = doc.reload(10) - - doc.dict_field['Embedded'] = embedded_1 - doc.save() - doc = doc.reload(10) - - doc.dict_field['Embedded'].string_field = 'Hello World' - self.assertEqual(doc._get_changed_fields(), ['dict_field.Embedded.string_field']) - self.assertEqual(doc._delta(), ({'dict_field.Embedded.string_field': 'Hello World'}, {})) - - - def test_delta_db_field(self): - - class Doc(Document): - string_field = StringField(db_field='db_string_field') - int_field = IntField(db_field='db_int_field') - dict_field = DictField(db_field='db_dict_field') - list_field = ListField(db_field='db_list_field') - - Doc.drop_collection() - doc = Doc() - doc.save() - - doc = Doc.objects.first() - self.assertEqual(doc._get_changed_fields(), []) - self.assertEqual(doc._delta(), ({}, {})) - - doc.string_field = 'hello' - self.assertEqual(doc._get_changed_fields(), ['db_string_field']) - self.assertEqual(doc._delta(), ({'db_string_field': 'hello'}, {})) - - doc._changed_fields = [] - doc.int_field = 1 - self.assertEqual(doc._get_changed_fields(), ['db_int_field']) - self.assertEqual(doc._delta(), ({'db_int_field': 1}, {})) - - doc._changed_fields = [] - dict_value = {'hello': 'world', 'ping': 'pong'} - doc.dict_field = dict_value - self.assertEqual(doc._get_changed_fields(), ['db_dict_field']) - self.assertEqual(doc._delta(), ({'db_dict_field': dict_value}, {})) - - doc._changed_fields = [] - list_value = ['1', 2, {'hello': 'world'}] - doc.list_field = list_value - self.assertEqual(doc._get_changed_fields(), ['db_list_field']) - self.assertEqual(doc._delta(), ({'db_list_field': list_value}, {})) - - # Test unsetting - doc._changed_fields = [] - doc.dict_field = {} - self.assertEqual(doc._get_changed_fields(), ['db_dict_field']) - self.assertEqual(doc._delta(), ({}, {'db_dict_field': 1})) - - doc._changed_fields = [] - doc.list_field = [] - self.assertEqual(doc._get_changed_fields(), ['db_list_field']) - self.assertEqual(doc._delta(), ({}, {'db_list_field': 1})) - - # Test it saves that data - doc = Doc() - doc.save() - - doc.string_field = 'hello' - doc.int_field = 1 - doc.dict_field = {'hello': 'world'} - doc.list_field = ['1', 2, {'hello': 'world'}] - doc.save() - doc = doc.reload(10) - - self.assertEqual(doc.string_field, 'hello') - self.assertEqual(doc.int_field, 1) - self.assertEqual(doc.dict_field, {'hello': 'world'}) - self.assertEqual(doc.list_field, ['1', 2, {'hello': 'world'}]) - - def test_delta_recursive_db_field(self): - - class Embedded(EmbeddedDocument): - string_field = StringField(db_field='db_string_field') - int_field = IntField(db_field='db_int_field') - dict_field = DictField(db_field='db_dict_field') - list_field = ListField(db_field='db_list_field') - - class Doc(Document): - string_field = StringField(db_field='db_string_field') - int_field = IntField(db_field='db_int_field') - dict_field = DictField(db_field='db_dict_field') - list_field = ListField(db_field='db_list_field') - embedded_field = EmbeddedDocumentField(Embedded, db_field='db_embedded_field') - - Doc.drop_collection() - doc = Doc() - doc.save() - - doc = Doc.objects.first() - self.assertEqual(doc._get_changed_fields(), []) - self.assertEqual(doc._delta(), ({}, {})) - - embedded_1 = Embedded() - embedded_1.string_field = 'hello' - embedded_1.int_field = 1 - embedded_1.dict_field = {'hello': 'world'} - embedded_1.list_field = ['1', 2, {'hello': 'world'}] - doc.embedded_field = embedded_1 - - self.assertEqual(doc._get_changed_fields(), ['db_embedded_field']) - - embedded_delta = { - 'db_string_field': 'hello', - 'db_int_field': 1, - 'db_dict_field': {'hello': 'world'}, - 'db_list_field': ['1', 2, {'hello': 'world'}] - } - self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) - embedded_delta.update({ - '_types': ['Embedded'], - '_cls': 'Embedded', - }) - self.assertEqual(doc._delta(), ({'db_embedded_field': embedded_delta}, {})) - - doc.save() - doc = doc.reload(10) - - doc.embedded_field.dict_field = {} - self.assertEqual(doc._get_changed_fields(), ['db_embedded_field.db_dict_field']) - self.assertEqual(doc.embedded_field._delta(), ({}, {'db_dict_field': 1})) - self.assertEqual(doc._delta(), ({}, {'db_embedded_field.db_dict_field': 1})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.dict_field, {}) - - doc.embedded_field.list_field = [] - self.assertEqual(doc._get_changed_fields(), ['db_embedded_field.db_list_field']) - self.assertEqual(doc.embedded_field._delta(), ({}, {'db_list_field': 1})) - self.assertEqual(doc._delta(), ({}, {'db_embedded_field.db_list_field': 1})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field, []) - - embedded_2 = Embedded() - embedded_2.string_field = 'hello' - embedded_2.int_field = 1 - embedded_2.dict_field = {'hello': 'world'} - embedded_2.list_field = ['1', 2, {'hello': 'world'}] - - doc.embedded_field.list_field = ['1', 2, embedded_2] - self.assertEqual(doc._get_changed_fields(), ['db_embedded_field.db_list_field']) - self.assertEqual(doc.embedded_field._delta(), ({ - 'db_list_field': ['1', 2, { - '_cls': 'Embedded', - '_types': ['Embedded'], - 'db_string_field': 'hello', - 'db_dict_field': {'hello': 'world'}, - 'db_int_field': 1, - 'db_list_field': ['1', 2, {'hello': 'world'}], - }] - }, {})) - - self.assertEqual(doc._delta(), ({ - 'db_embedded_field.db_list_field': ['1', 2, { - '_cls': 'Embedded', - '_types': ['Embedded'], - 'db_string_field': 'hello', - 'db_dict_field': {'hello': 'world'}, - 'db_int_field': 1, - 'db_list_field': ['1', 2, {'hello': 'world'}], - }] - }, {})) - doc.save() - doc = doc.reload(10) - - self.assertEqual(doc.embedded_field.list_field[0], '1') - self.assertEqual(doc.embedded_field.list_field[1], 2) - for k in doc.embedded_field.list_field[2]._fields: - self.assertEqual(doc.embedded_field.list_field[2][k], embedded_2[k]) - - doc.embedded_field.list_field[2].string_field = 'world' - self.assertEqual(doc._get_changed_fields(), ['db_embedded_field.db_list_field.2.db_string_field']) - self.assertEqual(doc.embedded_field._delta(), ({'db_list_field.2.db_string_field': 'world'}, {})) - self.assertEqual(doc._delta(), ({'db_embedded_field.db_list_field.2.db_string_field': 'world'}, {})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field[2].string_field, 'world') - - # Test multiple assignments - doc.embedded_field.list_field[2].string_field = 'hello world' - doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] - self.assertEqual(doc._get_changed_fields(), ['db_embedded_field.db_list_field']) - self.assertEqual(doc.embedded_field._delta(), ({ - 'db_list_field': ['1', 2, { - '_types': ['Embedded'], - '_cls': 'Embedded', - 'db_string_field': 'hello world', - 'db_int_field': 1, - 'db_list_field': ['1', 2, {'hello': 'world'}], - 'db_dict_field': {'hello': 'world'}}]}, {})) - self.assertEqual(doc._delta(), ({ - 'db_embedded_field.db_list_field': ['1', 2, { - '_types': ['Embedded'], - '_cls': 'Embedded', - 'db_string_field': 'hello world', - 'db_int_field': 1, - 'db_list_field': ['1', 2, {'hello': 'world'}], - 'db_dict_field': {'hello': 'world'}} - ]}, {})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field[2].string_field, 'hello world') - - # Test list native methods - doc.embedded_field.list_field[2].list_field.pop(0) - self.assertEqual(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [2, {'hello': 'world'}]}, {})) - doc.save() - doc = doc.reload(10) - - doc.embedded_field.list_field[2].list_field.append(1) - self.assertEqual(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [2, {'hello': 'world'}, 1]}, {})) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) - - doc.embedded_field.list_field[2].list_field.sort(key=str) - doc.save() - doc = doc.reload(10) - self.assertEqual(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) - - del(doc.embedded_field.list_field[2].list_field[2]['hello']) - self.assertEqual(doc._delta(), ({'db_embedded_field.db_list_field.2.db_list_field': [1, 2, {}]}, {})) - doc.save() - doc = doc.reload(10) - - del(doc.embedded_field.list_field[2].list_field) - self.assertEqual(doc._delta(), ({}, {'db_embedded_field.db_list_field.2.db_list_field': 1})) - def test_save_only_changed_fields(self): """Ensure save only sets / unsets changed fields """ @@ -2437,7 +872,6 @@ class DocumentTest(unittest.TestCase): class User(self.Person): active = BooleanField(default=True) - User.drop_collection() # Create person object and save it to the database @@ -2697,29 +1131,6 @@ class DocumentTest(unittest.TestCase): promoted_employee.reload() self.assertEqual(promoted_employee.details, None) - def test_mixins_dont_add_to_types(self): - - class Mixin(object): - name = StringField() - - class Person(Document, Mixin): - pass - - Person.drop_collection() - - self.assertEqual(Person._fields.keys(), ['name', 'id']) - - Person(name="Rozza").save() - - collection = self.db[Person._get_collection_name()] - obj = collection.find_one() - self.assertEqual(obj['_cls'], 'Person') - self.assertEqual(obj['_types'], ['Person']) - - self.assertEqual(Person.objects.count(), 1) - - Person.drop_collection() - def test_object_mixins(self): class NameMixin(object): @@ -2795,22 +1206,6 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() - def test_cannot_perform_joins_references(self): - - class BlogPost(Document): - author = ReferenceField(self.Person) - author2 = GenericReferenceField() - - def test_reference(): - list(BlogPost.objects(author__name="test")) - - self.assertRaises(InvalidQueryError, test_reference) - - def test_generic_reference(): - list(BlogPost.objects(author2__name="test")) - - self.assertRaises(InvalidQueryError, test_generic_reference) - def test_duplicate_db_fields_raise_invalid_document_error(self): """Ensure a InvalidDocumentError is thrown if duplicate fields declare the same db_field""" @@ -3082,17 +1477,17 @@ class DocumentTest(unittest.TestCase): for u in User.objects.all(): all_user_dic[u] = "OK" - self.assertEqual(all_user_dic.get(u1, False), "OK" ) - self.assertEqual(all_user_dic.get(u2, False), "OK" ) - self.assertEqual(all_user_dic.get(u3, False), "OK" ) - self.assertEqual(all_user_dic.get(u4, False), False ) # New object - self.assertEqual(all_user_dic.get(b1, False), False ) # Other object - self.assertEqual(all_user_dic.get(b2, False), False ) # Other object + self.assertEqual(all_user_dic.get(u1, False), "OK") + self.assertEqual(all_user_dic.get(u2, False), "OK") + self.assertEqual(all_user_dic.get(u3, False), "OK") + self.assertEqual(all_user_dic.get(u4, False), False) # New object + self.assertEqual(all_user_dic.get(b1, False), False) # Other object + self.assertEqual(all_user_dic.get(b2, False), False) # Other object # in Set all_user_set = set(User.objects.all()) - self.assertTrue(u1 in all_user_set ) + self.assertTrue(u1 in all_user_set) def test_picklable(self): @@ -3313,7 +1708,7 @@ class DocumentTest(unittest.TestCase): # Bob Book.objects.create(name="1", author=bob, extra={"a": bob.to_dbref(), "b": [karl.to_dbref(), susan.to_dbref()]}) - Book.objects.create(name="2", author=bob, extra={"a": bob.to_dbref(), "b": karl.to_dbref()} ) + Book.objects.create(name="2", author=bob, extra={"a": bob.to_dbref(), "b": karl.to_dbref()}) Book.objects.create(name="3", author=bob, extra={"a": bob.to_dbref(), "c": [jon.to_dbref(), peter.to_dbref()]}) Book.objects.create(name="4", author=bob) @@ -3325,20 +1720,20 @@ class DocumentTest(unittest.TestCase): Book.objects.create(name="9", author=jon, extra={"a": peter.to_dbref()}) # Checks - self.assertEqual(u",".join([str(b) for b in Book.objects.all()] ) , "1,2,3,4,5,6,7,8,9" ) + self.assertEqual(u",".join([str(b) for b in Book.objects.all()]) , "1,2,3,4,5,6,7,8,9") # bob related books self.assertEqual(u",".join([str(b) for b in Book.objects.filter( - Q(extra__a=bob ) | + Q(extra__a=bob) | Q(author=bob) | Q(extra__b=bob))]) , "1,2,3,4") # Susan & Karl related books self.assertEqual(u",".join([str(b) for b in Book.objects.filter( - Q(extra__a__all=[karl, susan] ) | - Q(author__all=[karl, susan ] ) | - Q(extra__b__all=[karl.to_dbref(), susan.to_dbref()] ) - ) ] ) , "1" ) + Q(extra__a__all=[karl, susan]) | + Q(author__all=[karl, susan ]) | + Q(extra__b__all=[karl.to_dbref(), susan.to_dbref()]) + ) ]) , "1") # $Where self.assertEqual(u",".join([str(b) for b in Book.objects.filter( @@ -3348,7 +1743,7 @@ class DocumentTest(unittest.TestCase): return this.name == '1' || this.name == '2';}""" } - ) ]), "1,2") + ) ]), "1,2") class ValidatorErrorTest(unittest.TestCase): @@ -3504,5 +1899,6 @@ class ValidatorErrorTest(unittest.TestCase): self.assertRaises(OperationError, change_shard_key) + if __name__ == '__main__': unittest.main() diff --git a/tests/mongoengine.png b/tests/document/mongoengine.png similarity index 100% rename from tests/mongoengine.png rename to tests/document/mongoengine.png diff --git a/tests/migration/__init__.py b/tests/migration/__init__.py new file mode 100644 index 0000000..882e737 --- /dev/null +++ b/tests/migration/__init__.py @@ -0,0 +1,4 @@ +from turn_off_inheritance import * + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/migration/test_convert_to_new_inheritance_model.py b/tests/migration/test_convert_to_new_inheritance_model.py new file mode 100644 index 0000000..0ef37f7 --- /dev/null +++ b/tests/migration/test_convert_to_new_inheritance_model.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +import unittest + +from mongoengine import Document, connect +from mongoengine.connection import get_db +from mongoengine.fields import StringField + +__all__ = ('ConvertToNewInheritanceModel', ) + + +class ConvertToNewInheritanceModel(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + def tearDown(self): + for collection in self.db.collection_names(): + if 'system.' in collection: + continue + self.db.drop_collection(collection) + + def test_how_to_convert_to_the_new_inheritance_model(self): + """Demonstrates migrating from 0.7 to 0.8 + """ + + # 1. Declaration of the class + class Animal(Document): + name = StringField() + meta = { + 'allow_inheritance': True, + 'indexes': ['name'] + } + + # 2. Remove _types + collection = Animal._get_collection() + collection.update({}, {"$unset": {"_types": 1}}, multi=True) + + # 3. Confirm extra data is removed + count = collection.find({'_types': {"$exists": True}}).count() + assert count == 0 + + # 4. Remove indexes + info = collection.index_information() + indexes_to_drop = [key for key, value in info.iteritems() + if '_types' in dict(value['key'])] + for index in indexes_to_drop: + collection.drop_index(index) + + # 5. Recreate indexes + Animal.objects._ensure_indexes() diff --git a/tests/migration/turn_off_inheritance.py b/tests/migration/turn_off_inheritance.py new file mode 100644 index 0000000..5d0f7d7 --- /dev/null +++ b/tests/migration/turn_off_inheritance.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +import unittest + +from mongoengine import Document, connect +from mongoengine.connection import get_db +from mongoengine.fields import StringField + +__all__ = ('TurnOffInheritanceTest', ) + + +class TurnOffInheritanceTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + def tearDown(self): + for collection in self.db.collection_names(): + if 'system.' in collection: + continue + self.db.drop_collection(collection) + + def test_how_to_turn_off_inheritance(self): + """Demonstrates migrating from allow_inheritance = True to False. + """ + + # 1. Old declaration of the class + + class Animal(Document): + name = StringField() + meta = { + 'allow_inheritance': True, + 'indexes': ['name'] + } + + # 2. Turn off inheritance + class Animal(Document): + name = StringField() + meta = { + 'allow_inheritance': False, + 'indexes': ['name'] + } + + # 3. Remove _types and _cls + collection = Animal._get_collection() + collection.update({}, {"$unset": {"_types": 1, "_cls": 1}}, multi=True) + + # 3. Confirm extra data is removed + count = collection.find({"$or": [{'_types': {"$exists": True}}, + {'_cls': {"$exists": True}}]}).count() + assert count == 0 + + # 4. Remove indexes + info = collection.index_information() + indexes_to_drop = [key for key, value in info.iteritems() + if '_types' in dict(value['key']) + or '_cls' in dict(value['key'])] + for index in indexes_to_drop: + collection.drop_index(index) + + # 5. Recreate indexes + Animal.objects._ensure_indexes() diff --git a/tests/test_dynamic_document.py b/tests/test_dynamic_document.py deleted file mode 100644 index 23762a3..0000000 --- a/tests/test_dynamic_document.py +++ /dev/null @@ -1,533 +0,0 @@ -import unittest - -from mongoengine import * -from mongoengine.connection import get_db - - -class DynamicDocTest(unittest.TestCase): - - def setUp(self): - connect(db='mongoenginetest') - self.db = get_db() - - class Person(DynamicDocument): - name = StringField() - meta = {'allow_inheritance': True} - - Person.drop_collection() - - self.Person = Person - - def test_simple_dynamic_document(self): - """Ensures simple dynamic documents are saved correctly""" - - p = self.Person() - p.name = "James" - p.age = 34 - - self.assertEqual(p.to_mongo(), - {"_types": ["Person"], "_cls": "Person", - "name": "James", "age": 34} - ) - - p.save() - - self.assertEqual(self.Person.objects.first().age, 34) - - # Confirm no changes to self.Person - self.assertFalse(hasattr(self.Person, 'age')) - - def test_dynamic_document_delta(self): - """Ensures simple dynamic documents can delta correctly""" - p = self.Person(name="James", age=34) - self.assertEqual(p._delta(), ({'_types': ['Person'], 'age': 34, 'name': 'James', '_cls': 'Person'}, {})) - - p.doc = 123 - del(p.doc) - self.assertEqual(p._delta(), ({'_types': ['Person'], 'age': 34, 'name': 'James', '_cls': 'Person'}, {'doc': 1})) - - def test_change_scope_of_variable(self): - """Test changing the scope of a dynamic field has no adverse effects""" - p = self.Person() - p.name = "Dean" - p.misc = 22 - p.save() - - p = self.Person.objects.get() - p.misc = {'hello': 'world'} - p.save() - - p = self.Person.objects.get() - self.assertEqual(p.misc, {'hello': 'world'}) - - def test_delete_dynamic_field(self): - """Test deleting a dynamic field works""" - self.Person.drop_collection() - p = self.Person() - p.name = "Dean" - p.misc = 22 - p.save() - - p = self.Person.objects.get() - p.misc = {'hello': 'world'} - p.save() - - p = self.Person.objects.get() - self.assertEqual(p.misc, {'hello': 'world'}) - collection = self.db[self.Person._get_collection_name()] - obj = collection.find_one() - self.assertEqual(sorted(obj.keys()), ['_cls', '_id', '_types', 'misc', 'name']) - - del(p.misc) - p.save() - - p = self.Person.objects.get() - self.assertFalse(hasattr(p, 'misc')) - - obj = collection.find_one() - self.assertEqual(sorted(obj.keys()), ['_cls', '_id', '_types', 'name']) - - def test_dynamic_document_queries(self): - """Ensure we can query dynamic fields""" - p = self.Person() - p.name = "Dean" - p.age = 22 - p.save() - - self.assertEqual(1, self.Person.objects(age=22).count()) - p = self.Person.objects(age=22) - p = p.get() - self.assertEqual(22, p.age) - - def test_complex_dynamic_document_queries(self): - class Person(DynamicDocument): - name = StringField() - - Person.drop_collection() - - p = Person(name="test") - p.age = "ten" - p.save() - - p1 = Person(name="test1") - p1.age = "less then ten and a half" - p1.save() - - p2 = Person(name="test2") - p2.age = 10 - p2.save() - - self.assertEqual(Person.objects(age__icontains='ten').count(), 2) - self.assertEqual(Person.objects(age__gte=10).count(), 1) - - def test_complex_data_lookups(self): - """Ensure you can query dynamic document dynamic fields""" - p = self.Person() - p.misc = {'hello': 'world'} - p.save() - - self.assertEqual(1, self.Person.objects(misc__hello='world').count()) - - def test_inheritance(self): - """Ensure that dynamic document plays nice with inheritance""" - class Employee(self.Person): - salary = IntField() - - Employee.drop_collection() - - self.assertTrue('name' in Employee._fields) - self.assertTrue('salary' in Employee._fields) - self.assertEqual(Employee._get_collection_name(), - self.Person._get_collection_name()) - - joe_bloggs = Employee() - joe_bloggs.name = "Joe Bloggs" - joe_bloggs.salary = 10 - joe_bloggs.age = 20 - joe_bloggs.save() - - self.assertEqual(1, self.Person.objects(age=20).count()) - self.assertEqual(1, Employee.objects(age=20).count()) - - joe_bloggs = self.Person.objects.first() - self.assertTrue(isinstance(joe_bloggs, Employee)) - - def test_embedded_dynamic_document(self): - """Test dynamic embedded documents""" - class Embedded(DynamicEmbeddedDocument): - pass - - class Doc(DynamicDocument): - pass - - Doc.drop_collection() - doc = Doc() - - embedded_1 = Embedded() - embedded_1.string_field = 'hello' - embedded_1.int_field = 1 - embedded_1.dict_field = {'hello': 'world'} - embedded_1.list_field = ['1', 2, {'hello': 'world'}] - doc.embedded_field = embedded_1 - - self.assertEqual(doc.to_mongo(), {"_types": ['Doc'], "_cls": "Doc", - "embedded_field": { - "_types": ['Embedded'], "_cls": "Embedded", - "string_field": "hello", - "int_field": 1, - "dict_field": {"hello": "world"}, - "list_field": ['1', 2, {'hello': 'world'}] - } - }) - doc.save() - - doc = Doc.objects.first() - self.assertEqual(doc.embedded_field.__class__, Embedded) - self.assertEqual(doc.embedded_field.string_field, "hello") - self.assertEqual(doc.embedded_field.int_field, 1) - self.assertEqual(doc.embedded_field.dict_field, {'hello': 'world'}) - self.assertEqual(doc.embedded_field.list_field, ['1', 2, {'hello': 'world'}]) - - def test_complex_embedded_documents(self): - """Test complex dynamic embedded documents setups""" - class Embedded(DynamicEmbeddedDocument): - pass - - class Doc(DynamicDocument): - pass - - Doc.drop_collection() - doc = Doc() - - embedded_1 = Embedded() - embedded_1.string_field = 'hello' - embedded_1.int_field = 1 - embedded_1.dict_field = {'hello': 'world'} - - embedded_2 = Embedded() - embedded_2.string_field = 'hello' - embedded_2.int_field = 1 - embedded_2.dict_field = {'hello': 'world'} - embedded_2.list_field = ['1', 2, {'hello': 'world'}] - - embedded_1.list_field = ['1', 2, embedded_2] - doc.embedded_field = embedded_1 - - self.assertEqual(doc.to_mongo(), {"_types": ['Doc'], "_cls": "Doc", - "embedded_field": { - "_types": ['Embedded'], "_cls": "Embedded", - "string_field": "hello", - "int_field": 1, - "dict_field": {"hello": "world"}, - "list_field": ['1', 2, - {"_types": ['Embedded'], "_cls": "Embedded", - "string_field": "hello", - "int_field": 1, - "dict_field": {"hello": "world"}, - "list_field": ['1', 2, {'hello': 'world'}]} - ] - } - }) - doc.save() - doc = Doc.objects.first() - self.assertEqual(doc.embedded_field.__class__, Embedded) - self.assertEqual(doc.embedded_field.string_field, "hello") - self.assertEqual(doc.embedded_field.int_field, 1) - self.assertEqual(doc.embedded_field.dict_field, {'hello': 'world'}) - self.assertEqual(doc.embedded_field.list_field[0], '1') - self.assertEqual(doc.embedded_field.list_field[1], 2) - - embedded_field = doc.embedded_field.list_field[2] - - self.assertEqual(embedded_field.__class__, Embedded) - self.assertEqual(embedded_field.string_field, "hello") - self.assertEqual(embedded_field.int_field, 1) - self.assertEqual(embedded_field.dict_field, {'hello': 'world'}) - self.assertEqual(embedded_field.list_field, ['1', 2, {'hello': 'world'}]) - - def test_delta_for_dynamic_documents(self): - p = self.Person() - p.name = "Dean" - p.age = 22 - p.save() - - p.age = 24 - self.assertEqual(p.age, 24) - self.assertEqual(p._get_changed_fields(), ['age']) - self.assertEqual(p._delta(), ({'age': 24}, {})) - - p = self.Person.objects(age=22).get() - p.age = 24 - self.assertEqual(p.age, 24) - self.assertEqual(p._get_changed_fields(), ['age']) - self.assertEqual(p._delta(), ({'age': 24}, {})) - - p.save() - self.assertEqual(1, self.Person.objects(age=24).count()) - - def test_delta(self): - - class Doc(DynamicDocument): - pass - - Doc.drop_collection() - doc = Doc() - doc.save() - - doc = Doc.objects.first() - self.assertEqual(doc._get_changed_fields(), []) - self.assertEqual(doc._delta(), ({}, {})) - - doc.string_field = 'hello' - self.assertEqual(doc._get_changed_fields(), ['string_field']) - self.assertEqual(doc._delta(), ({'string_field': 'hello'}, {})) - - doc._changed_fields = [] - doc.int_field = 1 - self.assertEqual(doc._get_changed_fields(), ['int_field']) - self.assertEqual(doc._delta(), ({'int_field': 1}, {})) - - doc._changed_fields = [] - dict_value = {'hello': 'world', 'ping': 'pong'} - doc.dict_field = dict_value - self.assertEqual(doc._get_changed_fields(), ['dict_field']) - self.assertEqual(doc._delta(), ({'dict_field': dict_value}, {})) - - doc._changed_fields = [] - list_value = ['1', 2, {'hello': 'world'}] - doc.list_field = list_value - self.assertEqual(doc._get_changed_fields(), ['list_field']) - self.assertEqual(doc._delta(), ({'list_field': list_value}, {})) - - # Test unsetting - doc._changed_fields = [] - doc.dict_field = {} - self.assertEqual(doc._get_changed_fields(), ['dict_field']) - self.assertEqual(doc._delta(), ({}, {'dict_field': 1})) - - doc._changed_fields = [] - doc.list_field = [] - self.assertEqual(doc._get_changed_fields(), ['list_field']) - self.assertEqual(doc._delta(), ({}, {'list_field': 1})) - - def test_delta_recursive(self): - """Testing deltaing works with dynamic documents""" - class Embedded(DynamicEmbeddedDocument): - pass - - class Doc(DynamicDocument): - pass - - Doc.drop_collection() - doc = Doc() - doc.save() - - doc = Doc.objects.first() - self.assertEqual(doc._get_changed_fields(), []) - self.assertEqual(doc._delta(), ({}, {})) - - embedded_1 = Embedded() - embedded_1.string_field = 'hello' - embedded_1.int_field = 1 - embedded_1.dict_field = {'hello': 'world'} - embedded_1.list_field = ['1', 2, {'hello': 'world'}] - doc.embedded_field = embedded_1 - - self.assertEqual(doc._get_changed_fields(), ['embedded_field']) - - embedded_delta = { - 'string_field': 'hello', - 'int_field': 1, - 'dict_field': {'hello': 'world'}, - 'list_field': ['1', 2, {'hello': 'world'}] - } - self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) - embedded_delta.update({ - '_types': ['Embedded'], - '_cls': 'Embedded', - }) - self.assertEqual(doc._delta(), ({'embedded_field': embedded_delta}, {})) - - doc.save() - doc.reload() - - doc.embedded_field.dict_field = {} - self.assertEqual(doc._get_changed_fields(), ['embedded_field.dict_field']) - self.assertEqual(doc.embedded_field._delta(), ({}, {'dict_field': 1})) - - self.assertEqual(doc._delta(), ({}, {'embedded_field.dict_field': 1})) - doc.save() - doc.reload() - - doc.embedded_field.list_field = [] - self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEqual(doc.embedded_field._delta(), ({}, {'list_field': 1})) - self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field': 1})) - doc.save() - doc.reload() - - embedded_2 = Embedded() - embedded_2.string_field = 'hello' - embedded_2.int_field = 1 - embedded_2.dict_field = {'hello': 'world'} - embedded_2.list_field = ['1', 2, {'hello': 'world'}] - - doc.embedded_field.list_field = ['1', 2, embedded_2] - self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEqual(doc.embedded_field._delta(), ({ - 'list_field': ['1', 2, { - '_cls': 'Embedded', - '_types': ['Embedded'], - 'string_field': 'hello', - 'dict_field': {'hello': 'world'}, - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], - }] - }, {})) - - self.assertEqual(doc._delta(), ({ - 'embedded_field.list_field': ['1', 2, { - '_cls': 'Embedded', - '_types': ['Embedded'], - 'string_field': 'hello', - 'dict_field': {'hello': 'world'}, - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], - }] - }, {})) - doc.save() - doc.reload() - - self.assertEqual(doc.embedded_field.list_field[2]._changed_fields, []) - self.assertEqual(doc.embedded_field.list_field[0], '1') - self.assertEqual(doc.embedded_field.list_field[1], 2) - for k in doc.embedded_field.list_field[2]._fields: - self.assertEqual(doc.embedded_field.list_field[2][k], embedded_2[k]) - - doc.embedded_field.list_field[2].string_field = 'world' - self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field.2.string_field']) - self.assertEqual(doc.embedded_field._delta(), ({'list_field.2.string_field': 'world'}, {})) - self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.string_field': 'world'}, {})) - doc.save() - doc.reload() - self.assertEqual(doc.embedded_field.list_field[2].string_field, 'world') - - # Test multiple assignments - doc.embedded_field.list_field[2].string_field = 'hello world' - doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] - self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) - self.assertEqual(doc.embedded_field._delta(), ({ - 'list_field': ['1', 2, { - '_types': ['Embedded'], - '_cls': 'Embedded', - 'string_field': 'hello world', - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], - 'dict_field': {'hello': 'world'}}]}, {})) - self.assertEqual(doc._delta(), ({ - 'embedded_field.list_field': ['1', 2, { - '_types': ['Embedded'], - '_cls': 'Embedded', - 'string_field': 'hello world', - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], - 'dict_field': {'hello': 'world'}} - ]}, {})) - doc.save() - doc.reload() - self.assertEqual(doc.embedded_field.list_field[2].string_field, 'hello world') - - # Test list native methods - doc.embedded_field.list_field[2].list_field.pop(0) - self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}]}, {})) - doc.save() - doc.reload() - - doc.embedded_field.list_field[2].list_field.append(1) - self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [2, {'hello': 'world'}, 1]}, {})) - doc.save() - doc.reload() - self.assertEqual(doc.embedded_field.list_field[2].list_field, [2, {'hello': 'world'}, 1]) - - doc.embedded_field.list_field[2].list_field.sort(key=str)# use str as a key to allow comparing uncomperable types - doc.save() - doc.reload() - self.assertEqual(doc.embedded_field.list_field[2].list_field, [1, 2, {'hello': 'world'}]) - - del(doc.embedded_field.list_field[2].list_field[2]['hello']) - self.assertEqual(doc._delta(), ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) - doc.save() - doc.reload() - - del(doc.embedded_field.list_field[2].list_field) - self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field.2.list_field': 1})) - - doc.save() - doc.reload() - - doc.dict_field = {'embedded': embedded_1} - doc.save() - doc.reload() - - doc.dict_field['embedded'].string_field = 'Hello World' - self.assertEqual(doc._get_changed_fields(), ['dict_field.embedded.string_field']) - self.assertEqual(doc._delta(), ({'dict_field.embedded.string_field': 'Hello World'}, {})) - - def test_indexes(self): - """Ensure that indexes are used when meta[indexes] is specified. - """ - class BlogPost(DynamicDocument): - meta = { - 'indexes': [ - '-date', - ('category', '-date') - ], - } - - BlogPost.drop_collection() - - info = BlogPost.objects._collection.index_information() - # _id, '-date', ('cat', 'date') - # NB: there is no index on _types by itself, since - # the indices on -date and tags will both contain - # _types as first element in the key - self.assertEqual(len(info), 3) - - # Indexes are lazy so use list() to perform query - list(BlogPost.objects) - info = BlogPost.objects._collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertTrue([('_types', 1), ('category', 1), ('date', -1)] - in info) - self.assertTrue([('_types', 1), ('date', -1)] in info) - - def test_dynamic_and_embedded(self): - """Ensure embedded documents play nicely""" - - class Address(EmbeddedDocument): - city = StringField() - - class Person(DynamicDocument): - name = StringField() - meta = {'allow_inheritance': True} - - Person.drop_collection() - - Person(name="Ross", address=Address(city="London")).save() - - person = Person.objects.first() - person.address.city = "Lundenne" - person.save() - - self.assertEqual(Person.objects.first().address.city, "Lundenne") - - person = Person.objects.first() - person.address = Address(city="Londinium") - person.save() - - self.assertEqual(Person.objects.first().address.city, "Londinium") - - person = Person.objects.first() - person.age = 35 - person.save() - self.assertEqual(Person.objects.first().age, 35) diff --git a/tests/test_fields.py b/tests/test_fields.py index 9806550..118521f 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -14,10 +14,11 @@ import gridfs from nose.plugins.skip import SkipTest from mongoengine import * from mongoengine.connection import get_db -from mongoengine.base import _document_registry, NotRegistered +from mongoengine.base import _document_registry +from mongoengine.errors import NotRegistered from mongoengine.python_support import PY3, b, StringIO, bin_type -TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') +TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'document/mongoengine.png') class FieldTest(unittest.TestCase): diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 690df5e..cdabadb 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -11,9 +11,12 @@ from mongoengine import * from mongoengine.connection import get_connection from mongoengine.python_support import PY3 from mongoengine.tests import query_counter -from mongoengine.queryset import (QuerySet, QuerySetManager, +from mongoengine.queryset import (Q, QuerySet, QuerySetManager, MultipleObjectsReturned, DoesNotExist, - QueryFieldList) + QueryFieldList, queryset_manager) +from mongoengine.queryset import transform +from mongoengine.errors import InvalidQueryError + class QuerySetTest(unittest.TestCase): @@ -40,19 +43,34 @@ class QuerySetTest(unittest.TestCase): def test_transform_query(self): """Ensure that the _transform_query function operates correctly. """ - self.assertEqual(QuerySet._transform_query(name='test', age=30), + self.assertEqual(transform.query(name='test', age=30), {'name': 'test', 'age': 30}) - self.assertEqual(QuerySet._transform_query(age__lt=30), + self.assertEqual(transform.query(age__lt=30), {'age': {'$lt': 30}}) - self.assertEqual(QuerySet._transform_query(age__gt=20, age__lt=50), + self.assertEqual(transform.query(age__gt=20, age__lt=50), {'age': {'$gt': 20, '$lt': 50}}) - self.assertEqual(QuerySet._transform_query(age=20, age__gt=50), + self.assertEqual(transform.query(age=20, age__gt=50), {'age': 20}) - self.assertEqual(QuerySet._transform_query(friend__age__gte=30), + self.assertEqual(transform.query(friend__age__gte=30), {'friend.age': {'$gte': 30}}) - self.assertEqual(QuerySet._transform_query(name__exists=True), + self.assertEqual(transform.query(name__exists=True), {'name': {'$exists': True}}) + def test_cannot_perform_joins_references(self): + + class BlogPost(Document): + author = ReferenceField(self.Person) + author2 = GenericReferenceField() + + def test_reference(): + list(BlogPost.objects(author__name="test")) + + self.assertRaises(InvalidQueryError, test_reference) + + def test_generic_reference(): + list(BlogPost.objects(author2__name="test")) + + def test_find(self): """Ensure that a query returns a valid set of results. """ @@ -921,10 +939,9 @@ class QuerySetTest(unittest.TestCase): # find all published blog posts before 2010-01-07 published_posts = BlogPost.published() published_posts = published_posts.filter( - published_date__lt=datetime(2010, 1, 7, 0, 0 ,0)) + published_date__lt=datetime(2010, 1, 7, 0, 0, 0)) self.assertEqual(published_posts.count(), 2) - blog_posts = BlogPost.objects blog_posts = blog_posts.filter(blog__in=[blog_1, blog_2]) blog_posts = blog_posts.filter(blog=blog_3) @@ -935,7 +952,7 @@ class QuerySetTest(unittest.TestCase): def test_raw_and_merging(self): class Doc(Document): - pass + meta = {'allow_inheritance': False} raw_query = Doc.objects(__raw__={'deleted': False, 'scraped': 'yes', @@ -943,7 +960,7 @@ class QuerySetTest(unittest.TestCase): {'attachments.views.extracted':'no'}] })._query - expected = {'deleted': False, '_types': 'Doc', 'scraped': 'yes', + expected = {'deleted': False, 'scraped': 'yes', '$nor': [{'views.extracted': 'no'}, {'attachments.views.extracted': 'no'}]} self.assertEqual(expected, raw_query) @@ -2598,68 +2615,6 @@ class QuerySetTest(unittest.TestCase): Group.drop_collection() - def test_types_index(self): - """Ensure that and index is used when '_types' is being used in a - query. - """ - class BlogPost(Document): - date = DateTimeField() - meta = {'indexes': ['-date']} - - # Indexes are lazy so use list() to perform query - list(BlogPost.objects) - info = BlogPost.objects._collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertTrue([('_types', 1)] in info) - self.assertTrue([('_types', 1), ('date', -1)] in info) - - def test_dont_index_types(self): - """Ensure that index_types will, when disabled, prevent _types - being added to all indices. - """ - class BloggPost(Document): - date = DateTimeField() - meta = {'index_types': False, - 'indexes': ['-date']} - - # Indexes are lazy so use list() to perform query - list(BloggPost.objects) - info = BloggPost.objects._collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - self.assertTrue([('_types', 1)] not in info) - self.assertTrue([('date', -1)] in info) - - BloggPost.drop_collection() - - class BloggPost(Document): - title = StringField() - meta = {'allow_inheritance': False} - - # _types is not used on objects where allow_inheritance is False - list(BloggPost.objects) - info = BloggPost.objects._collection.index_information() - self.assertFalse([('_types', 1)] in info.values()) - - BloggPost.drop_collection() - - def test_types_index_with_pk(self): - - class Comment(EmbeddedDocument): - comment_id = IntField(required=True) - - try: - class BlogPost(Document): - comments = EmbeddedDocumentField(Comment) - meta = {'indexes': [{'fields': ['pk', 'comments.comment_id'], - 'unique': True}]} - except UnboundLocalError: - self.fail('Unbound local error at types index + pk definition') - - info = BlogPost.objects._collection.index_information() - info = [value['key'] for key, value in info.iteritems()] - index_item = [(u'_types', 1), (u'_id', 1), (u'comments.comment_id', 1)] - self.assertTrue(index_item in info) - def test_dict_with_custom_baseclass(self): """Ensure DictField working with custom base clases. """ @@ -3116,6 +3071,7 @@ class QuerySetTest(unittest.TestCase): """ class Comment(Document): message = StringField() + meta = {'allow_inheritance': True} Comment.objects.ensure_index('message') @@ -3124,7 +3080,7 @@ class QuerySetTest(unittest.TestCase): value.get('unique', False), value.get('sparse', False)) for key, value in info.iteritems()] - self.assertTrue(([('_types', 1), ('message', 1)], False, False) in info) + self.assertTrue(([('_cls', 1), ('message', 1)], False, False) in info) def test_where(self): """Ensure that where clauses work. From 59826c8cfd42a13bb0e4cf9f9e0f62ab1d27a543 Mon Sep 17 00:00:00 2001 From: Marcelo Anton Date: Thu, 18 Oct 2012 11:44:18 -0300 Subject: [PATCH 0823/1279] This change in how the variable is declared DESCRIPTION corrects problems when running the command ``python setup.py bdist_rpm`` --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 6d9b51b..35ca579 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ try: except ImportError: pass -DESCRIPTION = """MongoEngine is a Python Object-Document -Mapper for working with MongoDB.""" +DESCRIPTION = 'MongoEngine is a Python Object-Document ' + \ +'Mapper for working with MongoDB.' LONG_DESCRIPTION = None try: LONG_DESCRIPTION = open('README.rst').read() From 3d5b6ae332291821d60d09e761f5e5b02529a404 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 17 Oct 2012 11:36:18 +0000 Subject: [PATCH 0824/1279] Inheritance is off by default (MongoEngine/mongoengine#122) --- docs/changelog.rst | 1 + docs/guide/defining-documents.rst | 30 +++++++------- docs/tutorial.rst | 5 ++- docs/upgrade.rst | 24 ++++++++++-- mongoengine/base/common.py | 2 +- mongoengine/base/document.py | 42 +++++++++++++------- mongoengine/base/fields.py | 31 ++++++++++----- mongoengine/base/metaclasses.py | 23 ++++++----- mongoengine/dereference.py | 5 ++- mongoengine/document.py | 19 ++++++--- mongoengine/fields.py | 51 ++++++++++++++---------- mongoengine/queryset/queryset.py | 2 +- tests/all_warnings/__init__.py | 18 +-------- tests/document/delta.py | 29 +++++++------- tests/document/dynamic.py | 12 ++++-- tests/document/inheritance.py | 2 - tests/document/instance.py | 65 +++++++++++++++++-------------- tests/test_dereference.py | 12 ++---- tests/test_fields.py | 32 +++++++++------ tests/test_queryset.py | 17 ++++---- 20 files changed, 245 insertions(+), 177 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8388b05..1970bf0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Inheritance is off by default (MongoEngine/mongoengine#122) - Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index cf3b5a6..ea8e05b 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -462,9 +462,10 @@ If a dictionary is passed then the following options are available: The fields to index. Specified in the same format as described above. :attr:`cls` (Default: True) - If you have polymorphic models that inherit and have `allow_inheritance` - turned on, you can configure whether the index should have the - :attr:`_cls` field added automatically to the start of the index. + If you have polymorphic models that inherit and have + :attr:`allow_inheritance` turned on, you can configure whether the index + should have the :attr:`_cls` field added automatically to the start of the + index. :attr:`sparse` (Default: False) Whether the index should be sparse. @@ -573,7 +574,9 @@ defined, you may subclass it and add any extra fields or methods you may need. As this is new class is not a direct subclass of :class:`~mongoengine.Document`, it will not be stored in its own collection; it will use the same collection as its superclass uses. This allows for more -convenient and efficient retrieval of related documents:: +convenient and efficient retrieval of related documents - all you need do is +set :attr:`allow_inheritance` to True in the :attr:`meta` data for a +document.:: # Stored in a collection named 'page' class Page(Document): @@ -585,25 +588,20 @@ convenient and efficient retrieval of related documents:: class DatedPage(Page): date = DateTimeField() -.. note:: From 0.7 onwards you must declare `allow_inheritance` in the document meta. +.. note:: From 0.8 onwards you must declare :attr:`allow_inheritance` defaults + to False, meaning you must set it to True to use inheritance. Working with existing data -------------------------- -To enable correct retrieval of documents involved in this kind of heirarchy, -an extra attribute is stored on each document in the database: :attr:`_cls`. -These are hidden from the user through the MongoEngine interface, but may not -be present if you are trying to use MongoEngine with an existing database. - -For this reason, you may disable this inheritance mechansim, removing the -dependency of :attr:`_cls`, enabling you to work with existing databases. -To disable inheritance on a document class, set :attr:`allow_inheritance` to -``False`` in the :attr:`meta` dictionary:: +As MongoEngine no longer defaults to needing :attr:`_cls` you can quickly and +easily get working with existing data. Just define the document to match +the expected schema in your database. If you have wildly varying schemas then +a :class:`~mongoengine.DynamicDocument` might be more appropriate. # Will work with data in an existing collection named 'cmsPage' class Page(Document): title = StringField(max_length=200, required=True) meta = { - 'collection': 'cmsPage', - 'allow_inheritance': False, + 'collection': 'cmsPage' } diff --git a/docs/tutorial.rst b/docs/tutorial.rst index a5284c8..c2fb5b9 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -84,12 +84,15 @@ using* the new fields we need to support video posts. This fits with the Object-Oriented principle of *inheritance* nicely. We can think of :class:`Post` as a base class, and :class:`TextPost`, :class:`ImagePost` and :class:`LinkPost` as subclasses of :class:`Post`. In fact, MongoEngine supports -this kind of modelling out of the box:: +this kind of modelling out of the box - all you need do is turn on inheritance +by setting :attr:`allow_inheritance` to True in the :attr:`meta`:: class Post(Document): title = StringField(max_length=120, required=True) author = ReferenceField(User) + meta = {'allow_inheritance': True} + class TextPost(Post): content = StringField() diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 99e3078..bf0a842 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -8,10 +8,13 @@ Upgrading Inheritance ----------- +Data Model +~~~~~~~~~~ + The inheritance model has changed, we no longer need to store an array of -`types` with the model we can just use the classname in `_cls`. This means -that you will have to update your indexes for each of your inherited classes -like so: +:attr:`types` with the model we can just use the classname in :attr:`_cls`. +This means that you will have to update your indexes for each of your +inherited classes like so: # 1. Declaration of the class class Animal(Document): @@ -40,6 +43,19 @@ like so: Animal.objects._ensure_indexes() +Document Definition +~~~~~~~~~~~~~~~~~~~ + +The default for inheritance has changed - its now off by default and +:attr:`_cls` will not be stored automatically with the class. So if you extend +your :class:`~mongoengine.Document` or :class:`~mongoengine.EmbeddedDocuments` +you will need to declare :attr:`allow_inheritance` in the meta data like so: + + class Animal(Document): + name = StringField() + + meta = {'allow_inheritance': True} + 0.6 to 0.7 ========== @@ -123,7 +139,7 @@ Document.objects.with_id - now raises an InvalidQueryError if used with a filter. FutureWarning - A future warning has been added to all inherited classes that -don't define `allow_inheritance` in their meta. +don't define :attr:`allow_inheritance` in their meta. You may need to update pyMongo to 2.0 for use with Sharding. diff --git a/mongoengine/base/common.py b/mongoengine/base/common.py index 648561b..82728d1 100644 --- a/mongoengine/base/common.py +++ b/mongoengine/base/common.py @@ -2,7 +2,7 @@ from mongoengine.errors import NotRegistered __all__ = ('ALLOW_INHERITANCE', 'get_document', '_document_registry') -ALLOW_INHERITANCE = True +ALLOW_INHERITANCE = False _document_registry = {} diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index af97e1f..bc509af 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -50,7 +50,6 @@ class BaseDocument(object): for key, value in values.iteritems(): key = self._reverse_db_field_map.get(key, key) setattr(self, key, value) - # Set any get_fieldname_display methods self.__set_field_display() @@ -83,6 +82,11 @@ class BaseDocument(object): if hasattr(self, '_changed_fields'): self._mark_as_changed(name) + # Check if the user has created a new instance of a class + if (self._is_document and self._initialised + and self._created and name == self._meta['id_field']): + super(BaseDocument, self).__setattr__('_created', False) + if (self._is_document and not self._created and name in self._meta.get('shard_key', tuple()) and self._data.get(name) != value): @@ -171,14 +175,24 @@ class BaseDocument(object): """Return data dictionary ready for use with MongoDB. """ data = {} - for field_name, field in self._fields.items(): - value = getattr(self, field_name, None) + for field_name, field in self._fields.iteritems(): + value = self._data.get(field_name, None) if value is not None: - data[field.db_field] = field.to_mongo(value) - # Only add _cls if allow_inheritance is not False - if not (hasattr(self, '_meta') and - self._meta.get('allow_inheritance', ALLOW_INHERITANCE) == False): + value = field.to_mongo(value) + + # Handle self generating fields + if value is None and field._auto_gen: + value = field.generate() + self._data[field_name] = value + + if value is not None: + data[field.db_field] = value + + # Only add _cls if allow_inheritance is True + if (hasattr(self, '_meta') and + self._meta.get('allow_inheritance', ALLOW_INHERITANCE) == True): data['_cls'] = self._class_name + if '_id' in data and data['_id'] is None: del data['_id'] @@ -194,7 +208,7 @@ class BaseDocument(object): are present. """ # Get a list of tuples of field names and their current values - fields = [(field, getattr(self, name)) + fields = [(field, self._data.get(name)) for name, field in self._fields.items()] # Ensure that each field is matched to a valid value @@ -207,7 +221,7 @@ class BaseDocument(object): errors[field.name] = error.errors or error except (ValueError, AttributeError, AssertionError), error: errors[field.name] = error - elif field.required: + elif field.required and not getattr(field, '_auto_gen', False): errors[field.name] = ValidationError('Field is required', field_name=field.name) if errors: @@ -313,6 +327,7 @@ class BaseDocument(object): """ # Handles cases where not loaded from_son but has _id doc = self.to_mongo() + set_fields = self._get_changed_fields() set_data = {} unset_data = {} @@ -370,7 +385,6 @@ class BaseDocument(object): if hasattr(d, '_fields'): field_name = d._reverse_db_field_map.get(db_field_name, db_field_name) - if field_name in d._fields: default = d._fields.get(field_name).default else: @@ -379,6 +393,7 @@ class BaseDocument(object): if default is not None: if callable(default): default = default() + if default != value: continue @@ -399,15 +414,12 @@ class BaseDocument(object): # get the class name from the document, falling back to the given # class if unavailable class_name = son.get('_cls', cls._class_name) - data = dict(("%s" % key, value) for key, value in son.items()) + data = dict(("%s" % key, value) for key, value in son.iteritems()) if not UNICODE_KWARGS: # python 2.6.4 and lower cannot handle unicode keys # passed to class constructor example: cls(**data) to_str_keys_recursive(data) - if '_cls' in data: - del data['_cls'] - # Return correct subclass for document type if class_name != cls._class_name: cls = get_document(class_name) @@ -415,7 +427,7 @@ class BaseDocument(object): changed_fields = [] errors_dict = {} - for field_name, field in cls._fields.items(): + for field_name, field in cls._fields.iteritems(): if field.db_field in data: value = data[field.db_field] try: diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 44f5e13..00e040c 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -21,6 +21,7 @@ class BaseField(object): name = None _geo_index = False + _auto_gen = False # Call `generate` to generate a value # These track each time a Field instance is created. Used to retain order. # The auto_creation_counter is used for fields that MongoEngine implicitly @@ -36,7 +37,6 @@ class BaseField(object): if name: msg = "Fields' 'name' attribute deprecated in favour of 'db_field'" warnings.warn(msg, DeprecationWarning) - self.name = None self.required = required or primary_key self.default = default self.unique = bool(unique or unique_with) @@ -62,7 +62,6 @@ class BaseField(object): if instance is None: # Document class being used rather than a document object return self - # Get value from document instance if available, if not use default value = instance._data.get(self.name) @@ -241,12 +240,21 @@ class ComplexBaseField(BaseField): """Convert a Python type to a MongoDB-compatible type. """ Document = _import_class("Document") + EmbeddedDocument = _import_class("EmbeddedDocument") + GenericReferenceField = _import_class("GenericReferenceField") if isinstance(value, basestring): return value if hasattr(value, 'to_mongo'): - return value.to_mongo() + if isinstance(value, Document): + return GenericReferenceField().to_mongo(value) + cls = value.__class__ + val = value.to_mongo() + # If we its a document thats not inherited add _cls + if (isinstance(value, EmbeddedDocument)): + val['_cls'] = cls.__name__ + return val is_list = False if not hasattr(value, 'items'): @@ -258,10 +266,10 @@ class ComplexBaseField(BaseField): if self.field: value_dict = dict([(key, self.field.to_mongo(item)) - for key, item in value.items()]) + for key, item in value.iteritems()]) else: value_dict = {} - for k, v in value.items(): + for k, v in value.iteritems(): if isinstance(v, Document): # We need the id from the saved object to create the DBRef if v.pk is None: @@ -274,16 +282,19 @@ class ComplexBaseField(BaseField): meta = getattr(v, '_meta', {}) allow_inheritance = ( meta.get('allow_inheritance', ALLOW_INHERITANCE) - == False) - if allow_inheritance and not self.field: - GenericReferenceField = _import_class( - "GenericReferenceField") + == True) + if not allow_inheritance and not self.field: value_dict[k] = GenericReferenceField().to_mongo(v) else: collection = v._get_collection_name() value_dict[k] = DBRef(collection, v.pk) elif hasattr(v, 'to_mongo'): - value_dict[k] = v.to_mongo() + cls = v.__class__ + val = v.to_mongo() + # If we its a document thats not inherited add _cls + if (isinstance(v, (Document, EmbeddedDocument))): + val['_cls'] = cls.__name__ + value_dict[k] = val else: value_dict[k] = self.to_mongo(v) diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index f87b03e..e68ec13 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -34,6 +34,17 @@ class DocumentMetaclass(type): if 'meta' in attrs: attrs['_meta'] = attrs.pop('meta') + # EmbeddedDocuments should inherit meta data + if '_meta' not in attrs: + meta = MetaDict() + for base in flattened_bases[::-1]: + # Add any mixin metadata from plain objects + if hasattr(base, 'meta'): + meta.merge(base.meta) + elif hasattr(base, '_meta'): + meta.merge(base._meta) + attrs['_meta'] = meta + # Handle document Fields # Merge all fields from subclasses @@ -52,6 +63,7 @@ class DocumentMetaclass(type): if not attr_value.db_field: attr_value.db_field = attr_name base_fields[attr_name] = attr_value + doc_fields.update(base_fields) # Discover any document fields @@ -98,15 +110,7 @@ class DocumentMetaclass(type): # inheritance of classes where inheritance is set to False allow_inheritance = base._meta.get('allow_inheritance', ALLOW_INHERITANCE) - if (not getattr(base, '_is_base_cls', True) - and allow_inheritance is None): - warnings.warn( - "%s uses inheritance, the default for " - "allow_inheritance is changing to off by default. " - "Please add it to the document meta." % name, - FutureWarning - ) - elif (allow_inheritance == False and + if (allow_inheritance != True and not base._meta.get('abstract')): raise ValueError('Document %s may not be subclassed' % base.__name__) @@ -353,6 +357,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): if not new_class._meta.get('id_field'): new_class._meta['id_field'] = 'id' new_class._fields['id'] = ObjectIdField(db_field='_id') + new_class._fields['id'].name = 'id' new_class.id = new_class._fields['id'] # Merge in exceptions with parent hierarchy diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 59cc0a5..25d46b4 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -121,7 +121,10 @@ class DeReference(object): for key, doc in references.iteritems(): object_map[key] = doc else: # Generic reference: use the refs data to convert to document - if doc_type and not isinstance(doc_type, (ListField, DictField, MapField,) ): + if isinstance(doc_type, (ListField, DictField, MapField,)): + continue + + if doc_type: references = doc_type._get_db()[col].find({'_id': {'$in': refs}}) for ref in references: doc = doc_type._from_son(ref) diff --git a/mongoengine/document.py b/mongoengine/document.py index b1ce13a..95dd624 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -117,6 +117,7 @@ class Document(BaseDocument): """ def fget(self): return getattr(self, self._meta['id_field']) + def fset(self, value): return setattr(self, self._meta['id_field'], value) return property(fget, fset) @@ -125,7 +126,7 @@ class Document(BaseDocument): @classmethod def _get_db(cls): """Some Model using other db_alias""" - return get_db(cls._meta.get("db_alias", DEFAULT_CONNECTION_NAME )) + return get_db(cls._meta.get("db_alias", DEFAULT_CONNECTION_NAME)) @classmethod def _get_collection(cls): @@ -212,11 +213,11 @@ class Document(BaseDocument): doc = self.to_mongo() - created = force_insert or '_id' not in doc + find_delta = ('_id' not in doc or self._created or force_insert) try: collection = self.__class__.objects._collection - if created: + if find_delta: if force_insert: object_id = collection.insert(doc, safe=safe, **write_options) @@ -271,7 +272,8 @@ class Document(BaseDocument): self._changed_fields = [] self._created = False - signals.post_save.send(self.__class__, document=self, created=created) + signals.post_save.send(self.__class__, document=self, + created=find_delta) return self def cascade_save(self, warn_cascade=None, *args, **kwargs): @@ -373,6 +375,7 @@ class Document(BaseDocument): for name in self._dynamic_fields.keys(): setattr(self, name, self._reload(name, obj._data[name])) self._changed_fields = obj._changed_fields + self._created = False return obj def _reload(self, key, value): @@ -464,7 +467,13 @@ class DynamicEmbeddedDocument(EmbeddedDocument): """Deletes the attribute by setting to None and allowing _delta to unset it""" field_name = args[0] - setattr(self, field_name, None) + if field_name in self._fields: + default = self._fields[field_name].default + if callable(default): + default = default() + setattr(self, field_name, default) + else: + setattr(self, field_name, None) class MapReduceDocument(object): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 9bcba9f..15e1626 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -16,12 +16,11 @@ from mongoengine.errors import ValidationError from mongoengine.python_support import (PY3, bin_type, txt_type, str_types, StringIO) from base import (BaseField, ComplexBaseField, ObjectIdField, - get_document, BaseDocument) + get_document, BaseDocument, ALLOW_INHERITANCE) from queryset import DO_NOTHING, QuerySet from document import Document, EmbeddedDocument from connection import get_db, DEFAULT_CONNECTION_NAME - try: from PIL import Image, ImageOps except ImportError: @@ -314,16 +313,16 @@ class DateTimeField(BaseField): usecs = 0 kwargs = {'microsecond': usecs} try: # Seconds are optional, so try converting seconds first. - return datetime.datetime(*time.strptime(value, '%Y-%m-%d %H:%M:%S')[:6], - **kwargs) + return datetime.datetime(*time.strptime(value, + '%Y-%m-%d %H:%M:%S')[:6], **kwargs) except ValueError: try: # Try without seconds. - return datetime.datetime(*time.strptime(value, '%Y-%m-%d %H:%M')[:5], - **kwargs) + return datetime.datetime(*time.strptime(value, + '%Y-%m-%d %H:%M')[:5], **kwargs) except ValueError: # Try without hour/minutes/seconds. try: - return datetime.datetime(*time.strptime(value, '%Y-%m-%d')[:3], - **kwargs) + return datetime.datetime(*time.strptime(value, + '%Y-%m-%d')[:3], **kwargs) except ValueError: return None @@ -410,6 +409,7 @@ class ComplexDateTimeField(StringField): return super(ComplexDateTimeField, self).__set__(instance, value) def validate(self, value): + value = self.to_python(value) if not isinstance(value, datetime.datetime): self.error('Only datetime objects may used in a ' 'ComplexDateTimeField') @@ -422,6 +422,7 @@ class ComplexDateTimeField(StringField): return original_value def to_mongo(self, value): + value = self.to_python(value) return self._convert_from_datetime(value) def prepare_query_value(self, op, value): @@ -529,7 +530,12 @@ class DynamicField(BaseField): return value if hasattr(value, 'to_mongo'): - return value.to_mongo() + cls = value.__class__ + val = value.to_mongo() + # If we its a document thats not inherited add _cls + if (isinstance(value, (Document, EmbeddedDocument))): + val['_cls'] = cls.__name__ + return val if not isinstance(value, (dict, list, tuple)): return value @@ -540,13 +546,12 @@ class DynamicField(BaseField): value = dict([(k, v) for k, v in enumerate(value)]) data = {} - for k, v in value.items(): + for k, v in value.iteritems(): data[k] = self.to_mongo(v) + value = data if is_list: # Convert back to a list - value = [v for k, v in sorted(data.items(), key=itemgetter(0))] - else: - value = data + value = [v for k, v in sorted(data.iteritems(), key=itemgetter(0))] return value def lookup_member(self, member_name): @@ -666,7 +671,6 @@ class DictField(ComplexBaseField): if op in match_operators and isinstance(value, basestring): return StringField().prepare_query_value(op, value) - return super(DictField, self).prepare_query_value(op, value) @@ -1323,7 +1327,8 @@ class GeoPointField(BaseField): class SequenceField(IntField): - """Provides a sequental counter (see http://www.mongodb.org/display/DOCS/Object+IDs#ObjectIDs-SequenceNumbers) + """Provides a sequental counter see: + http://www.mongodb.org/display/DOCS/Object+IDs#ObjectIDs-SequenceNumbers .. note:: @@ -1335,17 +1340,21 @@ class SequenceField(IntField): .. versionadded:: 0.5 """ - def __init__(self, collection_name=None, db_alias = None, sequence_name = None, *args, **kwargs): + _auto_gen = True + + def __init__(self, collection_name=None, db_alias=None, + sequence_name=None, *args, **kwargs): self.collection_name = collection_name or 'mongoengine.counters' self.db_alias = db_alias or DEFAULT_CONNECTION_NAME self.sequence_name = sequence_name return super(SequenceField, self).__init__(*args, **kwargs) - def generate_new_value(self): + def generate(self): """ Generate and Increment the counter """ - sequence_name = self.sequence_name or self.owner_document._get_collection_name() + sequence_name = (self.sequence_name or + self.owner_document._get_collection_name()) sequence_id = "%s.%s" % (sequence_name, self.name) collection = get_db(alias=self.db_alias)[self.collection_name] counter = collection.find_and_modify(query={"_id": sequence_id}, @@ -1365,7 +1374,7 @@ class SequenceField(IntField): value = instance._data.get(self.name) if not value and instance._initialised: - value = self.generate_new_value() + value = self.generate() instance._data[self.name] = value instance._mark_as_changed(self.name) @@ -1374,13 +1383,13 @@ class SequenceField(IntField): def __set__(self, instance, value): if value is None and instance._initialised: - value = self.generate_new_value() + value = self.generate() return super(SequenceField, self).__set__(instance, value) def to_python(self, value): if value is None: - value = self.generate_new_value() + value = self.generate() return value diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 5108066..dd7200b 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -58,7 +58,7 @@ class QuerySet(object): # If inheritance is allowed, only return instances and instances of # subclasses of the class being used - if document._meta.get('allow_inheritance') != False: + if document._meta.get('allow_inheritance') == True: self._initial_query = {"_cls": {"$in": self._document._subclasses}} self._loaded_fields = QueryFieldList(always_include=['_cls']) self._cursor_obj = None diff --git a/tests/all_warnings/__init__.py b/tests/all_warnings/__init__.py index 72de822..4609c5a 100644 --- a/tests/all_warnings/__init__.py +++ b/tests/all_warnings/__init__.py @@ -29,22 +29,6 @@ class AllWarnings(unittest.TestCase): # restore default handling of warnings warnings.showwarning = self.showwarning_default - def test_allow_inheritance_future_warning(self): - """Add FutureWarning for future allow_inhertiance default change. - """ - - class SimpleBase(Document): - a = IntField() - - class InheritedClass(SimpleBase): - b = IntField() - - InheritedClass() - self.assertEqual(len(self.warning_list), 1) - warning = self.warning_list[0] - self.assertEqual(FutureWarning, warning["category"]) - self.assertTrue("InheritedClass" in str(warning["message"])) - def test_dbref_reference_field_future_warning(self): class Person(Document): @@ -93,7 +77,7 @@ class AllWarnings(unittest.TestCase): def test_document_collection_syntax_warning(self): class NonAbstractBase(Document): - pass + meta = {'allow_inheritance': True} class InheritedDocumentFailTest(NonAbstractBase): meta = {'collection': 'fail'} diff --git a/tests/document/delta.py b/tests/document/delta.py index f8a071d..c6191d9 100644 --- a/tests/document/delta.py +++ b/tests/document/delta.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +import sys +sys.path[0:0] = [""] import unittest from mongoengine import * @@ -126,9 +128,6 @@ class DeltaTest(unittest.TestCase): 'list_field': ['1', 2, {'hello': 'world'}] } self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) - embedded_delta.update({ - '_cls': 'Embedded', - }) self.assertEqual(doc._delta(), ({'embedded_field': embedded_delta}, {})) @@ -162,6 +161,7 @@ class DeltaTest(unittest.TestCase): doc.embedded_field.list_field = ['1', 2, embedded_2] self.assertEqual(doc._get_changed_fields(), ['embedded_field.list_field']) + self.assertEqual(doc.embedded_field._delta(), ({ 'list_field': ['1', 2, { '_cls': 'Embedded', @@ -175,10 +175,10 @@ class DeltaTest(unittest.TestCase): self.assertEqual(doc._delta(), ({ 'embedded_field.list_field': ['1', 2, { '_cls': 'Embedded', - 'string_field': 'hello', - 'dict_field': {'hello': 'world'}, - 'int_field': 1, - 'list_field': ['1', 2, {'hello': 'world'}], + 'string_field': 'hello', + 'dict_field': {'hello': 'world'}, + 'int_field': 1, + 'list_field': ['1', 2, {'hello': 'world'}], }] }, {})) doc.save() @@ -467,9 +467,6 @@ class DeltaTest(unittest.TestCase): 'db_list_field': ['1', 2, {'hello': 'world'}] } self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) - embedded_delta.update({ - '_cls': 'Embedded', - }) self.assertEqual(doc._delta(), ({'db_embedded_field': embedded_delta}, {})) @@ -520,10 +517,10 @@ class DeltaTest(unittest.TestCase): self.assertEqual(doc._delta(), ({ 'db_embedded_field.db_list_field': ['1', 2, { '_cls': 'Embedded', - 'db_string_field': 'hello', - 'db_dict_field': {'hello': 'world'}, - 'db_int_field': 1, - 'db_list_field': ['1', 2, {'hello': 'world'}], + 'db_string_field': 'hello', + 'db_dict_field': {'hello': 'world'}, + 'db_int_field': 1, + 'db_list_field': ['1', 2, {'hello': 'world'}], }] }, {})) doc.save() @@ -686,3 +683,7 @@ class DeltaTest(unittest.TestCase): doc.list_field = [] self.assertEqual(doc._get_changed_fields(), ['list_field']) self.assertEqual(doc._delta(), ({}, {'list_field': 1})) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/document/dynamic.py b/tests/document/dynamic.py index ef27917..d879b54 100644 --- a/tests/document/dynamic.py +++ b/tests/document/dynamic.py @@ -1,4 +1,7 @@ import unittest +import sys + +sys.path[0:0] = [""] from mongoengine import * from mongoengine.connection import get_db @@ -161,7 +164,7 @@ class DynamicTest(unittest.TestCase): embedded_1.list_field = ['1', 2, {'hello': 'world'}] doc.embedded_field = embedded_1 - self.assertEqual(doc.to_mongo(), {"_cls": "Doc", + self.assertEqual(doc.to_mongo(), { "embedded_field": { "_cls": "Embedded", "string_field": "hello", @@ -205,7 +208,7 @@ class DynamicTest(unittest.TestCase): embedded_1.list_field = ['1', 2, embedded_2] doc.embedded_field = embedded_1 - self.assertEqual(doc.to_mongo(), {"_cls": "Doc", + self.assertEqual(doc.to_mongo(), { "embedded_field": { "_cls": "Embedded", "string_field": "hello", @@ -246,7 +249,6 @@ class DynamicTest(unittest.TestCase): class Person(DynamicDocument): name = StringField() - meta = {'allow_inheritance': True} Person.drop_collection() @@ -268,3 +270,7 @@ class DynamicTest(unittest.TestCase): person.age = 35 person.save() self.assertEqual(Person.objects.first().age, 35) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/document/inheritance.py b/tests/document/inheritance.py index d269ac0..08e2904 100644 --- a/tests/document/inheritance.py +++ b/tests/document/inheritance.py @@ -203,7 +203,6 @@ class InheritanceTest(unittest.TestCase): class Animal(Document): name = StringField() - meta = {'allow_inheritance': False} def create_dog_class(): class Dog(Animal): @@ -258,7 +257,6 @@ class InheritanceTest(unittest.TestCase): class Comment(EmbeddedDocument): content = StringField() - meta = {'allow_inheritance': False} def create_special_comment(): class SpecialComment(Comment): diff --git a/tests/document/instance.py b/tests/document/instance.py index 95f37d9..fcc43ba 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1,24 +1,22 @@ # -*- coding: utf-8 -*- from __future__ import with_statement +import sys +sys.path[0:0] = [""] + import bson import os import pickle -import pymongo -import sys import unittest import uuid -import warnings -from nose.plugins.skip import SkipTest from datetime import datetime - -from tests.fixtures import Base, Mixin, PickleEmbedded, PickleTest +from tests.fixtures import PickleEmbedded, PickleTest from mongoengine import * from mongoengine.errors import (NotRegistered, InvalidDocumentError, InvalidQueryError) from mongoengine.queryset import NULLIFY, Q -from mongoengine.connection import get_db, get_connection +from mongoengine.connection import get_db TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') @@ -461,7 +459,7 @@ class InstanceTest(unittest.TestCase): doc.validate() keys = doc._data.keys() self.assertEqual(2, len(keys)) - self.assertTrue(None in keys) + self.assertTrue('id' in keys) self.assertTrue('e' in keys) def test_save(self): @@ -656,8 +654,8 @@ class InstanceTest(unittest.TestCase): self.assertEqual(p1.name, p.parent.name) def test_update(self): - """Ensure that an existing document is updated instead of be overwritten. - """ + """Ensure that an existing document is updated instead of be + overwritten.""" # Create person object and save it to the database person = self.Person(name='Test User', age=30) person.save() @@ -753,30 +751,33 @@ class InstanceTest(unittest.TestCase): float_field = FloatField(default=1.1) boolean_field = BooleanField(default=True) datetime_field = DateTimeField(default=datetime.now) - embedded_document_field = EmbeddedDocumentField(EmbeddedDoc, default=lambda: EmbeddedDoc()) + embedded_document_field = EmbeddedDocumentField(EmbeddedDoc, + default=lambda: EmbeddedDoc()) list_field = ListField(default=lambda: [1, 2, 3]) dict_field = DictField(default=lambda: {"hello": "world"}) objectid_field = ObjectIdField(default=bson.ObjectId) - reference_field = ReferenceField(Simple, default=lambda: Simple().save()) + reference_field = ReferenceField(Simple, default=lambda: + Simple().save()) map_field = MapField(IntField(), default=lambda: {"simple": 1}) decimal_field = DecimalField(default=1.0) complex_datetime_field = ComplexDateTimeField(default=datetime.now) url_field = URLField(default="http://mongoengine.org") dynamic_field = DynamicField(default=1) - generic_reference_field = GenericReferenceField(default=lambda: Simple().save()) - sorted_list_field = SortedListField(IntField(), default=lambda: [1, 2, 3]) + generic_reference_field = GenericReferenceField( + default=lambda: Simple().save()) + sorted_list_field = SortedListField(IntField(), + default=lambda: [1, 2, 3]) email_field = EmailField(default="ross@example.com") geo_point_field = GeoPointField(default=lambda: [1, 2]) sequence_field = SequenceField() uuid_field = UUIDField(default=uuid.uuid4) - generic_embedded_document_field = GenericEmbeddedDocumentField(default=lambda: EmbeddedDoc()) - + generic_embedded_document_field = GenericEmbeddedDocumentField( + default=lambda: EmbeddedDoc()) Simple.drop_collection() Doc.drop_collection() Doc().save() - my_doc = Doc.objects.only("string_field").first() my_doc.string_field = "string" my_doc.save() @@ -1707,9 +1708,12 @@ class InstanceTest(unittest.TestCase): peter = User.objects.create(name="Peter") # Bob - Book.objects.create(name="1", author=bob, extra={"a": bob.to_dbref(), "b": [karl.to_dbref(), susan.to_dbref()]}) - Book.objects.create(name="2", author=bob, extra={"a": bob.to_dbref(), "b": karl.to_dbref()}) - Book.objects.create(name="3", author=bob, extra={"a": bob.to_dbref(), "c": [jon.to_dbref(), peter.to_dbref()]}) + Book.objects.create(name="1", author=bob, extra={ + "a": bob.to_dbref(), "b": [karl.to_dbref(), susan.to_dbref()]}) + Book.objects.create(name="2", author=bob, extra={ + "a": bob.to_dbref(), "b": karl.to_dbref()}) + Book.objects.create(name="3", author=bob, extra={ + "a": bob.to_dbref(), "c": [jon.to_dbref(), peter.to_dbref()]}) Book.objects.create(name="4", author=bob) # Jon @@ -1717,23 +1721,26 @@ class InstanceTest(unittest.TestCase): Book.objects.create(name="6", author=peter) Book.objects.create(name="7", author=jon) Book.objects.create(name="8", author=jon) - Book.objects.create(name="9", author=jon, extra={"a": peter.to_dbref()}) + Book.objects.create(name="9", author=jon, + extra={"a": peter.to_dbref()}) # Checks - self.assertEqual(u",".join([str(b) for b in Book.objects.all()]) , "1,2,3,4,5,6,7,8,9") + self.assertEqual(",".join([str(b) for b in Book.objects.all()]), + "1,2,3,4,5,6,7,8,9") # bob related books - self.assertEqual(u",".join([str(b) for b in Book.objects.filter( + self.assertEqual(",".join([str(b) for b in Book.objects.filter( Q(extra__a=bob) | Q(author=bob) | - Q(extra__b=bob))]) , + Q(extra__b=bob))]), "1,2,3,4") # Susan & Karl related books - self.assertEqual(u",".join([str(b) for b in Book.objects.filter( + self.assertEqual(",".join([str(b) for b in Book.objects.filter( Q(extra__a__all=[karl, susan]) | - Q(author__all=[karl, susan ]) | - Q(extra__b__all=[karl.to_dbref(), susan.to_dbref()]) - ) ]) , "1") + Q(author__all=[karl, susan]) | + Q(extra__b__all=[ + karl.to_dbref(), susan.to_dbref()])) + ]), "1") # $Where self.assertEqual(u",".join([str(b) for b in Book.objects.filter( @@ -1743,7 +1750,7 @@ class InstanceTest(unittest.TestCase): return this.name == '1' || this.name == '2';}""" } - ) ]), "1,2") + )]), "1,2") class ValidatorErrorTest(unittest.TestCase): diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 7b149db..c9631eb 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -331,14 +331,10 @@ class FieldTest(unittest.TestCase): return "" % self.name Person.drop_collection() - paul = Person(name="Paul") - paul.save() - maria = Person(name="Maria") - maria.save() - julia = Person(name='Julia') - julia.save() - anna = Person(name='Anna') - anna.save() + paul = Person(name="Paul").save() + maria = Person(name="Maria").save() + julia = Person(name='Julia').save() + anna = Person(name='Anna').save() paul.other.friends = [maria, julia, anna] paul.other.name = "Paul's friends" diff --git a/tests/test_fields.py b/tests/test_fields.py index 118521f..1c13a58 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -727,7 +727,7 @@ class FieldTest(unittest.TestCase): """Ensure that the list fields can handle the complex types.""" class SettingBase(EmbeddedDocument): - pass + meta = {'allow_inheritance': True} class StringSetting(SettingBase): value = StringField() @@ -743,8 +743,9 @@ class FieldTest(unittest.TestCase): e.mapping.append(StringSetting(value='foo')) e.mapping.append(IntegerSetting(value=42)) e.mapping.append({'number': 1, 'string': 'Hi!', 'float': 1.001, - 'complex': IntegerSetting(value=42), 'list': - [IntegerSetting(value=42), StringSetting(value='foo')]}) + 'complex': IntegerSetting(value=42), + 'list': [IntegerSetting(value=42), + StringSetting(value='foo')]}) e.save() e2 = Simple.objects.get(id=e.id) @@ -844,7 +845,7 @@ class FieldTest(unittest.TestCase): """Ensure that the dict field can handle the complex types.""" class SettingBase(EmbeddedDocument): - pass + meta = {'allow_inheritance': True} class StringSetting(SettingBase): value = StringField() @@ -859,9 +860,11 @@ class FieldTest(unittest.TestCase): e = Simple() e.mapping['somestring'] = StringSetting(value='foo') e.mapping['someint'] = IntegerSetting(value=42) - e.mapping['nested_dict'] = {'number': 1, 'string': 'Hi!', 'float': 1.001, - 'complex': IntegerSetting(value=42), 'list': - [IntegerSetting(value=42), StringSetting(value='foo')]} + e.mapping['nested_dict'] = {'number': 1, 'string': 'Hi!', + 'float': 1.001, + 'complex': IntegerSetting(value=42), + 'list': [IntegerSetting(value=42), + StringSetting(value='foo')]} e.save() e2 = Simple.objects.get(id=e.id) @@ -915,7 +918,7 @@ class FieldTest(unittest.TestCase): """Ensure that the MapField can handle complex declared types.""" class SettingBase(EmbeddedDocument): - pass + meta = {"allow_inheritance": True} class StringSetting(SettingBase): value = StringField() @@ -951,7 +954,8 @@ class FieldTest(unittest.TestCase): number = IntField(default=0, db_field='i') class Test(Document): - my_map = MapField(field=EmbeddedDocumentField(Embedded), db_field='x') + my_map = MapField(field=EmbeddedDocumentField(Embedded), + db_field='x') Test.drop_collection() @@ -1038,6 +1042,8 @@ class FieldTest(unittest.TestCase): class User(EmbeddedDocument): name = StringField() + meta = {'allow_inheritance': True} + class PowerUser(User): power = IntField() @@ -1046,8 +1052,10 @@ class FieldTest(unittest.TestCase): author = EmbeddedDocumentField(User) post = BlogPost(content='What I did today...') - post.author = User(name='Test User') post.author = PowerUser(name='Test User', power=47) + post.save() + + self.assertEqual(47, BlogPost.objects.first().author.power) def test_reference_validation(self): """Ensure that invalid docment objects cannot be assigned to reference @@ -2117,12 +2125,12 @@ class FieldTest(unittest.TestCase): def test_sequence_fields_reload(self): class Animal(Document): counter = SequenceField() - type = StringField() + name = StringField() self.db['mongoengine.counters'].drop() Animal.drop_collection() - a = Animal(type="Boi") + a = Animal(name="Boi") a.save() self.assertEqual(a.counter, 1) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index cdabadb..e9e78b4 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -647,7 +647,8 @@ class QuerySetTest(unittest.TestCase): self.assertRaises(NotUniqueError, throw_operation_error_not_unique) self.assertEqual(Blog.objects.count(), 2) - Blog.objects.insert([blog2, blog3], write_options={'continue_on_error': True}) + Blog.objects.insert([blog2, blog3], write_options={ + 'continue_on_error': True}) self.assertEqual(Blog.objects.count(), 3) def test_get_changed_fields_query_count(self): @@ -673,7 +674,7 @@ class QuerySetTest(unittest.TestCase): r2 = Project(name="r2").save() r3 = Project(name="r3").save() p1 = Person(name="p1", projects=[r1, r2]).save() - p2 = Person(name="p2", projects=[r2]).save() + p2 = Person(name="p2", projects=[r2, r3]).save() o1 = Organization(name="o1", employees=[p1]).save() with query_counter() as q: @@ -688,24 +689,24 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(q, 0) fresh_o1 = Organization.objects.get(id=o1.id) - fresh_o1.save() + fresh_o1.save() # No changes, does nothing - self.assertEqual(q, 2) + self.assertEqual(q, 1) with query_counter() as q: self.assertEqual(q, 0) fresh_o1 = Organization.objects.get(id=o1.id) - fresh_o1.save(cascade=False) + fresh_o1.save(cascade=False) # No changes, does nothing - self.assertEqual(q, 2) + self.assertEqual(q, 1) with query_counter() as q: self.assertEqual(q, 0) fresh_o1 = Organization.objects.get(id=o1.id) - fresh_o1.employees.append(p2) - fresh_o1.save(cascade=False) + fresh_o1.employees.append(p2) # Dereferences + fresh_o1.save(cascade=False) # Saves self.assertEqual(q, 3) From c31488add9340e7b4bf85e4d2507a1dac78620e2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 5 Nov 2012 11:14:02 +0000 Subject: [PATCH 0825/1279] Version bump --- docs/changelog.rst | 2 +- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b2a855d..aac24c6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,7 @@ Changelog ========= -Changes in 0.7.X +Changes in 0.7.6 ================ - Unicode fix for repr (MongoEngine/mongoengine#133) - Allow updates with match operators (MongoEngine/mongoengine#144) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 9044e61..cdfbfff 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 7, 5) +VERSION = (0, 7, 6) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 641b3de..d796f99 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.7.5 +Version: 0.7.6 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 7d90aa76ff7116269dea42f2c6629ea6b868b0de Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 6 Nov 2012 16:04:23 +0000 Subject: [PATCH 0826/1279] Add _instance to Embedded Documents Fixes MongoEngine/mongoengine#139 --- mongoengine/base/datastructures.py | 20 +++++++++++- mongoengine/base/fields.py | 4 +++ mongoengine/document.py | 2 ++ mongoengine/fields.py | 7 +++-- tests/document/instance.py | 50 ++++++++++++++++++++++++------ 5 files changed, 70 insertions(+), 13 deletions(-) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index 9a7620e..c750b5b 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -1,4 +1,5 @@ import weakref +from mongoengine.common import _import_class __all__ = ("BaseDict", "BaseList") @@ -16,6 +17,14 @@ class BaseDict(dict): self._name = name return super(BaseDict, self).__init__(dict_items) + def __getitem__(self, *args, **kwargs): + value = super(BaseDict, self).__getitem__(*args, **kwargs) + + EmbeddedDocument = _import_class('EmbeddedDocument') + if isinstance(value, EmbeddedDocument) and value._instance is None: + value._instance = self._instance + return value + def __setitem__(self, *args, **kwargs): self._mark_as_changed() return super(BaseDict, self).__setitem__(*args, **kwargs) @@ -75,6 +84,14 @@ class BaseList(list): self._name = name return super(BaseList, self).__init__(list_items) + def __getitem__(self, *args, **kwargs): + value = super(BaseList, self).__getitem__(*args, **kwargs) + + EmbeddedDocument = _import_class('EmbeddedDocument') + if isinstance(value, EmbeddedDocument) and value._instance is None: + value._instance = self._instance + return value + def __setitem__(self, *args, **kwargs): self._mark_as_changed() return super(BaseList, self).__setitem__(*args, **kwargs) @@ -84,7 +101,8 @@ class BaseList(list): return super(BaseList, self).__delitem__(*args, **kwargs) def __getstate__(self): - self.observer = None + self.instance = None + self._dereferenced = False return self def __setstate__(self, state): diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 00e040c..fc1a076 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -1,5 +1,6 @@ import operator import warnings +import weakref from bson import DBRef, ObjectId @@ -71,6 +72,9 @@ class BaseField(object): if callable(value): value = value() + EmbeddedDocument = _import_class('EmbeddedDocument') + if isinstance(value, EmbeddedDocument) and value._instance is None: + value._instance = weakref.proxy(instance) return value def __set__(self, instance, value): diff --git a/mongoengine/document.py b/mongoengine/document.py index 95dd624..adbdcca 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -40,6 +40,8 @@ class EmbeddedDocument(BaseDocument): my_metaclass = DocumentMetaclass __metaclass__ = DocumentMetaclass + _instance = None + def __init__(self, *args, **kwargs): super(EmbeddedDocument, self).__init__(*args, **kwargs) self._changed_fields = [] diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 15e1626..94e1155 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -625,7 +625,8 @@ class SortedListField(ListField): def to_mongo(self, value): value = super(SortedListField, self).to_mongo(value) if self._ordering is not None: - return sorted(value, key=itemgetter(self._ordering), reverse=self._order_reverse) + return sorted(value, key=itemgetter(self._ordering), + reverse=self._order_reverse) return sorted(value, reverse=self._order_reverse) @@ -655,7 +656,9 @@ class DictField(ComplexBaseField): self.error('Only dictionaries may be used in a DictField') if any(k for k in value.keys() if not isinstance(k, basestring)): - self.error('Invalid dictionary key - documents must have only string keys') + msg = ("Invalid dictionary key - documents must " + "have only string keys") + self.error(msg) if any(('.' in k or '$' in k) for k in value.keys()): self.error('Invalid dictionary key name - keys may not contain "."' ' or "$" characters') diff --git a/tests/document/instance.py b/tests/document/instance.py index fcc43ba..48ddc10 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -183,9 +183,6 @@ class InstanceTest(unittest.TestCase): self.assertEqual(list_stats, CompareStats.objects.first().stats) - - - def test_db_field_load(self): """Ensure we load data correctly """ @@ -214,24 +211,24 @@ class InstanceTest(unittest.TestCase): class Person(Document): name = StringField(required=True) - rank_ = EmbeddedDocumentField(Rank, required=False, db_field='rank') + rank_ = EmbeddedDocumentField(Rank, + required=False, + db_field='rank') @property def rank(self): - return self.rank_.title if self.rank_ is not None else "Private" + if self.rank_ is None: + return "Private" + return self.rank_.title Person.drop_collection() Person(name="Jack", rank_=Rank(title="Corporal")).save() - Person(name="Fred").save() self.assertEqual(Person.objects.get(name="Jack").rank, "Corporal") self.assertEqual(Person.objects.get(name="Fred").rank, "Private") - - - def test_custom_id_field(self): """Ensure that documents may be created with custom primary keys. """ @@ -247,7 +244,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(User._meta['id_field'], 'username') def create_invalid_user(): - User(name='test').save() # no primary key field + User(name='test').save() # no primary key field self.assertRaises(ValidationError, create_invalid_user) def define_invalid_user(): @@ -424,6 +421,36 @@ class InstanceTest(unittest.TestCase): self.assertTrue('content' in Comment._fields) self.assertFalse('id' in Comment._fields) + def test_embedded_document_instance(self): + """Ensure that embedded documents can reference parent instance + """ + class Embedded(EmbeddedDocument): + string = StringField() + + class Doc(Document): + embedded_field = EmbeddedDocumentField(Embedded) + + Doc.drop_collection() + Doc(embedded_field=Embedded(string="Hi")).save() + + doc = Doc.objects.get() + self.assertEqual(doc, doc.embedded_field._instance) + + def test_embedded_document_complex_instance(self): + """Ensure that embedded documents in complex fields can reference + parent instance""" + class Embedded(EmbeddedDocument): + string = StringField() + + class Doc(Document): + embedded_field = ListField(EmbeddedDocumentField(Embedded)) + + Doc.drop_collection() + Doc(embedded_field=[Embedded(string="Hi")]).save() + + doc = Doc.objects.get() + self.assertEqual(doc, doc.embedded_field[0]._instance) + def test_embedded_document_validation(self): """Ensure that embedded documents may be validated. """ @@ -442,6 +469,7 @@ class InstanceTest(unittest.TestCase): comment.date = datetime.now() comment.validate() + self.assertEqual(comment._instance, None) def test_embedded_db_field_validate(self): @@ -475,11 +503,13 @@ class InstanceTest(unittest.TestCase): self.assertEqual(person_obj['age'], 30) self.assertEqual(person_obj['_id'], person.id) # Test skipping validation on save + class Recipient(Document): email = EmailField(required=True) recipient = Recipient(email='root@localhost') self.assertRaises(ValidationError, recipient.save) + try: recipient.save(validate=False) except ValidationError: From f0f1308465773545a6b3cead3abe2c1f82b2f2e8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 6 Nov 2012 16:06:54 +0000 Subject: [PATCH 0827/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index ecd487f..33d22e1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Added _instance to EmbeddedDocuments pointing to the parent (MongoEngine/mongoengine#139) - Inheritance is off by default (MongoEngine/mongoengine#122) - Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) From f2049e9c1896eae2e34b9a25d6bdbf82fa8375e2 Mon Sep 17 00:00:00 2001 From: Samuel Clay Date: Tue, 6 Nov 2012 18:55:13 +0000 Subject: [PATCH 0828/1279] Adding QuerySet(read_preference=pymongo.ReadPreference.X) and QuerySet().read_preference() method to override connection-level read_preference on a per-query basis. --- mongoengine/queryset/queryset.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index dd7200b..0437395 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -53,6 +53,7 @@ class QuerySet(object): self._timeout = True self._class_check = True self._slave_okay = False + self._read_preference = None self._iter = False self._scalar = [] @@ -75,7 +76,8 @@ class QuerySet(object): copy_props = ('_initial_query', '_query_obj', '_where_clause', '_loaded_fields', '_ordering', '_snapshot', - '_timeout', '_limit', '_skip', '_slave_okay', '_hint') + '_timeout', '_limit', '_skip', '_slave_okay', '_hint', + '_read_preference') for prop in copy_props: val = getattr(self, prop) @@ -109,7 +111,8 @@ class QuerySet(object): self._collection.ensure_index(fields, **index_spec) return self - def __call__(self, q_obj=None, class_check=True, slave_okay=False, **query): + def __call__(self, q_obj=None, class_check=True, slave_okay=False, read_preference=None, + **query): """Filter the selected documents by calling the :class:`~mongoengine.queryset.QuerySet` with a query. @@ -121,6 +124,8 @@ class QuerySet(object): querying collection :param slave_okay: if True, allows this query to be run against a replica secondary. + :params read_preference: if set, overrides connection-level + read_preference from `ReplicaSetConnection`. :param query: Django-style query keyword arguments """ query = Q(**query) @@ -129,6 +134,8 @@ class QuerySet(object): self._query_obj &= query self._mongo_query = None self._cursor_obj = None + if read_preference is not None: + self._read_preference = read_preference self._class_check = class_check return self @@ -229,8 +236,10 @@ class QuerySet(object): cursor_args = { 'snapshot': self._snapshot, 'timeout': self._timeout, - 'slave_okay': self._slave_okay + 'slave_okay': self._slave_okay, } + if self._read_preference is not None: + cursor_args['read_preference'] = self._read_preference if self._loaded_fields: cursor_args['fields'] = self._loaded_fields.as_dict() return cursor_args @@ -802,6 +811,15 @@ class QuerySet(object): self._slave_okay = enabled return self + def read_preference(self, read_preference): + """Change the read_preference when querying. + + :param read_preference: override ReplicaSetConnection-level + preference. + """ + self._read_preference = read_preference + return self + def delete(self, safe=False): """Delete the documents matched by the query. From 7073b9d395429a753f95ecdb26decb4344784691 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 6 Nov 2012 18:55:14 +0000 Subject: [PATCH 0829/1279] Added validation and tests --- docs/changelog.rst | 1 + docs/guide/connecting.rst | 6 ++ mongoengine/queryset/queryset.py | 118 ++++++++++++++++++------------- tests/test_queryset.py | 16 +++++ 4 files changed, 93 insertions(+), 48 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 33d22e1..5ea1e4f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Added support setting for read prefrence at a query level (MongoEngine/mongoengine#157) - Added _instance to EmbeddedDocuments pointing to the parent (MongoEngine/mongoengine#139) - Inheritance is off by default (MongoEngine/mongoengine#122) - Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index bc45dbf..657c46c 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -33,6 +33,12 @@ MongoEngine now supports :func:`~pymongo.replica_set_connection.ReplicaSetConnec to use them please use a URI style connection and provide the `replicaSet` name in the connection kwargs. +Read preferences are supported throught the connection or via individual +queries by passing the read_preference :: + + Bar.objects().read_preference(ReadPreference.PRIMARY) + Bar.objects(read_preference=ReadPreference.PRIMARY) + Multiple Databases ================== diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 0437395..cf4b4f8 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -6,6 +6,7 @@ import operator import pymongo from bson.code import Code +from pymongo.common import validate_read_preference from mongoengine import signals from mongoengine.common import _import_class @@ -68,7 +69,8 @@ class QuerySet(object): self._hint = -1 # Using -1 as None is a valid value for hint def clone(self): - """Creates a copy of the current :class:`~mongoengine.queryset.QuerySet` + """Creates a copy of the current + :class:`~mongoengine.queryset.QuerySet` .. versionadded:: 0.5 """ @@ -111,8 +113,8 @@ class QuerySet(object): self._collection.ensure_index(fields, **index_spec) return self - def __call__(self, q_obj=None, class_check=True, slave_okay=False, read_preference=None, - **query): + def __call__(self, q_obj=None, class_check=True, slave_okay=False, + read_preference=None, **query): """Filter the selected documents by calling the :class:`~mongoengine.queryset.QuerySet` with a query. @@ -124,7 +126,7 @@ class QuerySet(object): querying collection :param slave_okay: if True, allows this query to be run against a replica secondary. - :params read_preference: if set, overrides connection-level + :params read_preference: if set, overrides connection-level read_preference from `ReplicaSetConnection`. :param query: Django-style query keyword arguments """ @@ -135,7 +137,7 @@ class QuerySet(object): self._mongo_query = None self._cursor_obj = None if read_preference is not None: - self._read_preference = read_preference + self.read_preference(read_preference) self._class_check = class_check return self @@ -282,39 +284,43 @@ class QuerySet(object): self.limit(2) self.__call__(*q_objs, **query) try: - result1 = self.next() + result = self.next() except StopIteration: - raise self._document.DoesNotExist("%s matching query does not exist." - % self._document._class_name) + msg = ("%s matching query does not exist." + % self._document._class_name) + raise self._document.DoesNotExist(msg) try: - result2 = self.next() + self.next() except StopIteration: - return result1 + return result self.rewind() message = u'%d items returned, instead of 1' % self.count() raise self._document.MultipleObjectsReturned(message) - def get_or_create(self, write_options=None, auto_save=True, *q_objs, **query): - """Retrieve unique object or create, if it doesn't exist. Returns a tuple of - ``(object, created)``, where ``object`` is the retrieved or created object - and ``created`` is a boolean specifying whether a new object was created. Raises + def get_or_create(self, write_options=None, auto_save=True, + *q_objs, **query): + """Retrieve unique object or create, if it doesn't exist. Returns a + tuple of ``(object, created)``, where ``object`` is the retrieved or + created object and ``created`` is a boolean specifying whether a new + object was created. Raises :class:`~mongoengine.queryset.MultipleObjectsReturned` or `DocumentName.MultipleObjectsReturned` if multiple results are found. A new document will be created if the document doesn't exists; a dictionary of default values for the new document may be provided as a keyword argument called :attr:`defaults`. - .. note:: This requires two separate operations and therefore a - race condition exists. Because there are no transactions in mongoDB - other approaches should be investigated, to ensure you don't - accidently duplicate data when using this method. + .. warning:: This requires two separate operations and therefore a + race condition exists. Because there are no transactions in + mongoDB other approaches should be investigated, to ensure you + don't accidently duplicate data when using this method. :param write_options: optional extra keyword arguments used if we have to create a new document. Passes any write_options onto :meth:`~mongoengine.Document.save` - :param auto_save: if the object is to be saved automatically if not found. + :param auto_save: if the object is to be saved automatically if + not found. .. versionchanged:: 0.6 - added `auto_save` .. versionadded:: 0.3 @@ -352,21 +358,24 @@ class QuerySet(object): result = None return result - def insert(self, doc_or_docs, load_bulk=True, safe=False, write_options=None): + def insert(self, doc_or_docs, load_bulk=True, safe=False, + write_options=None): """bulk insert documents If ``safe=True`` and the operation is unsuccessful, an :class:`~mongoengine.OperationError` will be raised. :param docs_or_doc: a document or list of documents to be inserted - :param load_bulk (optional): If True returns the list of document instances + :param load_bulk (optional): If True returns the list of document + instances :param safe: check if the operation succeeded before returning :param write_options: Extra keyword arguments are passed down to :meth:`~pymongo.collection.Collection.insert` - which will be used as options for the resultant ``getLastError`` command. - For example, ``insert(..., {w: 2, fsync: True})`` will wait until at least two - servers have recorded the write and will force an fsync on each server being - written to. + which will be used as options for the resultant + ``getLastError`` command. For example, + ``insert(..., {w: 2, fsync: True})`` will wait until at least + two servers have recorded the write and will force an fsync on + each server being written to. By default returns document instances, set ``load_bulk`` to False to return just ``ObjectIds`` @@ -388,7 +397,8 @@ class QuerySet(object): raw = [] for doc in docs: if not isinstance(doc, self._document): - msg = "Some documents inserted aren't instances of %s" % str(self._document) + msg = ("Some documents inserted aren't instances of %s" + % str(self._document)) raise OperationError(msg) if doc.pk: msg = "Some documents have ObjectIds use doc.update() instead" @@ -429,7 +439,8 @@ class QuerySet(object): .. versionchanged:: 0.6 Raises InvalidQueryError if filter has been set """ if not self._query_obj.empty: - raise InvalidQueryError("Cannot use a filter whilst using `with_id`") + msg = "Cannot use a filter whilst using `with_id`" + raise InvalidQueryError(msg) return self.filter(pk=object_id).first() def in_bulk(self, object_ids): @@ -503,9 +514,9 @@ class QuerySet(object): :param reduce_f: reduce function, as :class:`~bson.code.Code` or string :param output: output collection name, if set to 'inline' will try to - use :class:`~pymongo.collection.Collection.inline_map_reduce` - This can also be a dictionary containing output options - see: http://docs.mongodb.org/manual/reference/commands/#mapReduce + use :class:`~pymongo.collection.Collection.inline_map_reduce` + This can also be a dictionary containing output options + see: http://docs.mongodb.org/manual/reference/commands/#mapReduce :param finalize_f: finalize function, an optional function that performs any post-reduction processing. :param scope: values to insert into map/reduce global scope. Optional. @@ -568,7 +579,8 @@ class QuerySet(object): map_reduce_function = 'map_reduce' mr_args['out'] = output - results = getattr(self._collection, map_reduce_function)(map_f, reduce_f, **mr_args) + results = getattr(self._collection, map_reduce_function)( + map_f, reduce_f, **mr_args) if map_reduce_function == 'map_reduce': results = results.find() @@ -609,9 +621,9 @@ class QuerySet(object): """Added 'hint' support, telling Mongo the proper index to use for the query. - Judicious use of hints can greatly improve query performance. When doing - a query on multiple fields (at least one of which is indexed) pass the - indexed field as a hint to the query. + Judicious use of hints can greatly improve query performance. When + doing a query on multiple fields (at least one of which is indexed) + pass the indexed field as a hint to the query. Hinting will not do anything if the corresponding index does not exist. The last hint applied to this cursor takes precedence over all others. @@ -695,9 +707,9 @@ class QuerySet(object): Retrieving a Subrange of Array Elements: You can use the $slice operator to retrieve a subrange of elements in - an array :: + an array. For example to get the first 5 comments:: - post = BlogPost.objects(...).fields(slice__comments=5) // first 5 comments + post = BlogPost.objects(...).fields(slice__comments=5) :param kwargs: A dictionary identifying what to include @@ -724,9 +736,10 @@ class QuerySet(object): return self def all_fields(self): - """Include all fields. Reset all previously calls of .only() and .exclude(). :: + """Include all fields. Reset all previously calls of .only() or + .exclude(). :: - post = BlogPost.objects(...).exclude("comments").only("title").all_fields() + post = BlogPost.objects.exclude("comments").all_fields() .. versionadded:: 0.5 """ @@ -817,6 +830,7 @@ class QuerySet(object): :param read_preference: override ReplicaSetConnection-level preference. """ + validate_read_preference('read_preference', read_preference) self._read_preference = read_preference return self @@ -839,9 +853,10 @@ class QuerySet(object): for rule_entry in delete_rules: document_cls, field_name = rule_entry rule = doc._meta['delete_rules'][rule_entry] - if rule == DENY and document_cls.objects(**{field_name + '__in': self}).count() > 0: - msg = u'Could not delete document (at least %s.%s refers to it)' % \ - (document_cls.__name__, field_name) + if rule == DENY and document_cls.objects( + **{field_name + '__in': self}).count() > 0: + msg = ("Could not delete document (%s.%s refers to it)" + % (document_cls.__name__, field_name)) raise OperationError(msg) for rule_entry in delete_rules: @@ -864,13 +879,15 @@ class QuerySet(object): self._collection.remove(self._query, safe=safe) - def update(self, safe_update=True, upsert=False, multi=True, write_options=None, **update): + def update(self, safe_update=True, upsert=False, multi=True, + write_options=None, **update): """Perform an atomic update on the fields matched by the query. When ``safe_update`` is used, the number of affected documents is returned. :param safe_update: check if the operation succeeded before returning :param upsert: Any existing document with that "_id" is overwritten. - :param write_options: extra keyword arguments for :meth:`~pymongo.collection.Collection.update` + :param write_options: extra keyword arguments for + :meth:`~pymongo.collection.Collection.update` .. versionadded:: 0.2 """ @@ -895,13 +912,15 @@ class QuerySet(object): raise OperationError(message) raise OperationError(u'Update failed (%s)' % unicode(err)) - def update_one(self, safe_update=True, upsert=False, write_options=None, **update): + def update_one(self, safe_update=True, upsert=False, write_options=None, + **update): """Perform an atomic update on first field matched by the query. When ``safe_update`` is used, the number of affected documents is returned. :param safe_update: check if the operation succeeded before returning :param upsert: Any existing document with that "_id" is overwritten. - :param write_options: extra keyword arguments for :meth:`~pymongo.collection.Collection.update` + :param write_options: extra keyword arguments for + :meth:`~pymongo.collection.Collection.update` :param update: Django-style update keyword arguments .. versionadded:: 0.2 @@ -970,7 +989,8 @@ class QuerySet(object): return ".".join([f.db_field for f in fields]) code = re.sub(u'\[\s*~([A-z_][A-z_0-9.]+?)\s*\]', field_sub, code) - code = re.sub(u'\{\{\s*~([A-z_][A-z_0-9.]+?)\s*\}\}', field_path_sub, code) + code = re.sub(u'\{\{\s*~([A-z_][A-z_0-9.]+?)\s*\}\}', field_path_sub, + code) return code def exec_js(self, code, *fields, **options): @@ -1094,7 +1114,8 @@ class QuerySet(object): } """) - for result in self.map_reduce(map_func, reduce_func, finalize_f=finalize_func, output='inline'): + for result in self.map_reduce(map_func, reduce_func, + finalize_f=finalize_func, output='inline'): return result.value else: return 0 @@ -1122,7 +1143,8 @@ class QuerySet(object): document lookups """ if map_reduce: - return self._item_frequencies_map_reduce(field, normalize=normalize) + return self._item_frequencies_map_reduce(field, + normalize=normalize) return self._item_frequencies_exec_js(field, normalize=normalize) def _item_frequencies_map_reduce(self, field, normalize=False): diff --git a/tests/test_queryset.py b/tests/test_queryset.py index e9e78b4..dcb2524 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -1,9 +1,13 @@ from __future__ import with_statement +import sys +sys.path[0:0] = [""] import unittest from datetime import datetime, timedelta import pymongo +from pymongo.errors import ConfigurationError +from pymongo.read_preferences import ReadPreference from bson import ObjectId @@ -3648,6 +3652,18 @@ class QueryFieldListTest(unittest.TestCase): ak = list(Bar.objects(foo__match={'shape': "square", "color": "purple"})) self.assertEqual([b1], ak) + def test_read_preference(self): + class Bar(Document): + pass + + Bar.drop_collection() + bars = list(Bar.objects(read_preference=ReadPreference.PRIMARY)) + self.assertEqual([], bars) + + self.assertRaises(ConfigurationError, Bar.objects, + read_preference='Primary') + + if __name__ == '__main__': unittest.main() From 1986e82783ba7432728ee1a21a45dae7d8971d34 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Nov 2012 12:12:28 +0000 Subject: [PATCH 0830/1279] Added clean method to documents for pre validation data cleaning (MongoEngine/mongoengine#60) --- docs/changelog.rst | 1 + docs/guide/document-instances.rst | 28 ++++++++++++ mongoengine/base/document.py | 46 +++++++++++++++----- mongoengine/base/fields.py | 6 +-- mongoengine/common.py | 4 +- mongoengine/document.py | 10 +++-- mongoengine/fields.py | 8 ++-- tests/document/instance.py | 72 ++++++++++++++++++++++++++++++- 8 files changed, 150 insertions(+), 25 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5ea1e4f..ca18d3e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Added clean method to documents for pre validation data cleaning (MongoEngine/mongoengine#60) - Added support setting for read prefrence at a query level (MongoEngine/mongoengine#157) - Added _instance to EmbeddedDocuments pointing to the parent (MongoEngine/mongoengine#139) - Inheritance is off by default (MongoEngine/mongoengine#122) diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index 54fa804..b3bf687 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -38,6 +38,34 @@ already exist, then any changes will be updated atomically. For example:: .. seealso:: :ref:`guide-atomic-updates` +Pre save data validation and cleaning +------------------------------------- +MongoEngine allows you to create custom cleaning rules for your documents when +calling :meth:`~mongoengine.Document.save`. By providing a custom +:meth:`~mongoengine.Document.clean` method you can do any pre validation / data +cleaning. + +This might be useful if you want to ensure a default value based on other +document values for example:: + + class Essay(Document): + status = StringField(choices=('Published', 'Draft'), required=True) + pub_date = DateTimeField() + + def clean(self): + """Ensures that only published essays have a `pub_date` and + automatically sets the pub_date if published and not set""" + if self.status == 'Draft' and self.pub_date is not None: + msg = 'Draft entries should not have a publication date.' + raise ValidationError(msg) + # Set the pub_date for published items if not set. + if self.status == 'Published' and self.pub_date is None: + self.pub_date = datetime.now() + +.. note:: + Cleaning is only called if validation is turned on and when calling +:meth:`~mongoengine.Document.save`. + Cascading Saves --------------- If your document contains :class:`~mongoengine.ReferenceField` or diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index bc509af..46f5320 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -15,7 +15,9 @@ from .common import get_document, ALLOW_INHERITANCE from .datastructures import BaseDict, BaseList from .fields import ComplexBaseField -__all__ = ('BaseDocument', ) +__all__ = ('BaseDocument', 'NON_FIELD_ERRORS') + +NON_FIELD_ERRORS = '__all__' class BaseDocument(object): @@ -82,11 +84,6 @@ class BaseDocument(object): if hasattr(self, '_changed_fields'): self._mark_as_changed(name) - # Check if the user has created a new instance of a class - if (self._is_document and self._initialised - and self._created and name == self._meta['id_field']): - super(BaseDocument, self).__setattr__('_created', False) - if (self._is_document and not self._created and name in self._meta.get('shard_key', tuple()) and self._data.get(name) != value): @@ -94,6 +91,11 @@ class BaseDocument(object): msg = "Shard Keys are immutable. Tried to update %s" % name raise OperationError(msg) + # Check if the user has created a new instance of a class + if (self._is_document and self._initialised + and self._created and name == self._meta['id_field']): + super(BaseDocument, self).__setattr__('_created', False) + super(BaseDocument, self).__setattr__(name, value) def __getstate__(self): @@ -171,6 +173,16 @@ class BaseDocument(object): else: return hash(self.pk) + def clean(self): + """ + Hook for doing document level data cleaning before validation is run. + + Any ValidationError raised by this method will not be associated with + a particular field; it will have a special-case association with the + field defined by NON_FIELD_ERRORS. + """ + pass + def to_mongo(self): """Return data dictionary ready for use with MongoDB. """ @@ -203,20 +215,33 @@ class BaseDocument(object): data[name] = field.to_mongo(self._data.get(name, None)) return data - def validate(self): + def validate(self, clean=True): """Ensure that all fields' values are valid and that required fields are present. """ + # Ensure that each field is matched to a valid value + errors = {} + if clean: + try: + self.clean() + except ValidationError, error: + errors[NON_FIELD_ERRORS] = error + # Get a list of tuples of field names and their current values fields = [(field, self._data.get(name)) for name, field in self._fields.items()] - # Ensure that each field is matched to a valid value - errors = {} + EmbeddedDocumentField = _import_class("EmbeddedDocumentField") + GenericEmbeddedDocumentField = _import_class("GenericEmbeddedDocumentField") + for field, value in fields: if value is not None: try: - field._validate(value) + if isinstance(field, (EmbeddedDocumentField, + GenericEmbeddedDocumentField)): + field._validate(value, clean=clean) + else: + field._validate(value) except ValidationError, error: errors[field.name] = error.errors or error except (ValueError, AttributeError, AssertionError), error: @@ -224,6 +249,7 @@ class BaseDocument(object): elif field.required and not getattr(field, '_auto_gen', False): errors[field.name] = ValidationError('Field is required', field_name=field.name) + if errors: raise ValidationError('ValidationError', errors=errors) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index fc1a076..11719b5 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -105,12 +105,12 @@ class BaseField(object): """ return value - def validate(self, value): + def validate(self, value, clean=True): """Perform validation on a value. """ pass - def _validate(self, value): + def _validate(self, value, **kwargs): Document = _import_class('Document') EmbeddedDocument = _import_class('EmbeddedDocument') # check choices @@ -138,7 +138,7 @@ class BaseField(object): raise ValueError('validation argument for "%s" must be a ' 'callable.' % self.name) - self.validate(value) + self.validate(value, **kwargs) class ComplexBaseField(BaseField): diff --git a/mongoengine/common.py b/mongoengine/common.py index c284777..c76801c 100644 --- a/mongoengine/common.py +++ b/mongoengine/common.py @@ -9,8 +9,8 @@ def _import_class(cls_name): doc_classes = ('Document', 'DynamicEmbeddedDocument', 'EmbeddedDocument', 'MapReduceDocument') field_classes = ('DictField', 'DynamicField', 'EmbeddedDocumentField', - 'GenericReferenceField', 'GeoPointField', - 'ReferenceField', 'StringField') + 'GenericReferenceField', 'GenericEmbeddedDocumentField', + 'GeoPointField', 'ReferenceField', 'StringField') queryset_classes = ('OperationError',) deref_classes = ('DeReference',) diff --git a/mongoengine/document.py b/mongoengine/document.py index adbdcca..fcf8256 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -100,8 +100,8 @@ class Document(BaseDocument): Automatic index creation can be disabled by specifying attr:`auto_create_index` in the :attr:`meta` dictionary. If this is set to False then indexes will not be created by MongoEngine. This is useful in - production systems where index creation is performed as part of a deployment - system. + production systems where index creation is performed as part of a + deployment system. By default, _cls will be added to the start of every index (that doesn't contain a list) if allow_inheritance is True. This can be @@ -165,7 +165,7 @@ class Document(BaseDocument): cls._collection = db[collection_name] return cls._collection - def save(self, safe=True, force_insert=False, validate=True, + def save(self, safe=True, force_insert=False, validate=True, clean=True, write_options=None, cascade=None, cascade_kwargs=None, _refs=None): """Save the :class:`~mongoengine.Document` to the database. If the @@ -179,6 +179,8 @@ class Document(BaseDocument): :param force_insert: only try to create a new document, don't allow updates of existing documents :param validate: validates the document; set to ``False`` to skip. + :param clean: call the document clean method, requires `validate` to be + True. :param write_options: Extra keyword arguments are passed down to :meth:`~pymongo.collection.Collection.save` OR :meth:`~pymongo.collection.Collection.insert` @@ -208,7 +210,7 @@ class Document(BaseDocument): signals.pre_save.send(self.__class__, document=self) if validate: - self.validate() + self.validate(clean=clean) if not write_options: write_options = {} diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 94e1155..8aa7f64 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -461,7 +461,7 @@ class EmbeddedDocumentField(BaseField): return value return self.document_type.to_mongo(value) - def validate(self, value): + def validate(self, value, clean=True): """Make sure that the document instance is an instance of the EmbeddedDocument subclass provided when the document was defined. """ @@ -469,7 +469,7 @@ class EmbeddedDocumentField(BaseField): if not isinstance(value, self.document_type): self.error('Invalid embedded document instance provided to an ' 'EmbeddedDocumentField') - self.document_type.validate(value) + self.document_type.validate(value, clean) def lookup_member(self, member_name): return self.document_type._fields.get(member_name) @@ -499,12 +499,12 @@ class GenericEmbeddedDocumentField(BaseField): return value - def validate(self, value): + def validate(self, value, clean=True): if not isinstance(value, EmbeddedDocument): self.error('Invalid embedded document instance provided to an ' 'GenericEmbeddedDocumentField') - value.validate() + value.validate(clean=clean) def to_mongo(self, document): if document is None: diff --git a/tests/document/instance.py b/tests/document/instance.py index 48ddc10..2e07eb2 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -490,6 +490,76 @@ class InstanceTest(unittest.TestCase): self.assertTrue('id' in keys) self.assertTrue('e' in keys) + def test_document_clean(self): + class TestDocument(Document): + status = StringField() + pub_date = DateTimeField() + + def clean(self): + if self.status == 'draft' and self.pub_date is not None: + msg = 'Draft entries may not have a publication date.' + raise ValidationError(msg) + # Set the pub_date for published items if not set. + if self.status == 'published' and self.pub_date is None: + self.pub_date = datetime.now() + + TestDocument.drop_collection() + + t = TestDocument(status="draft", pub_date=datetime.now()) + + try: + t.save() + except ValidationError, e: + expect_msg = "Draft entries may not have a publication date." + self.assertTrue(expect_msg in e.message) + self.assertEqual(e.to_dict(), {'__all__': expect_msg}) + + t = TestDocument(status="published") + t.save(clean=False) + + self.assertEquals(t.pub_date, None) + + t = TestDocument(status="published") + t.save(clean=True) + + self.assertEquals(type(t.pub_date), datetime) + + def test_document_embedded_clean(self): + class TestEmbeddedDocument(EmbeddedDocument): + x = IntField(required=True) + y = IntField(required=True) + z = IntField(required=True) + + meta = {'allow_inheritance': False} + + def clean(self): + if self.z: + if self.z != self.x + self.y: + raise ValidationError('Value of z != x + y') + else: + self.z = self.x + self.y + + class TestDocument(Document): + doc = EmbeddedDocumentField(TestEmbeddedDocument) + status = StringField() + + TestDocument.drop_collection() + + t = TestDocument(doc=TestEmbeddedDocument(x=10, y=25, z=15)) + try: + t.save() + except ValidationError, e: + expect_msg = "Value of z != x + y" + self.assertTrue(expect_msg in e.message) + self.assertEqual(e.to_dict(), {'doc': {'__all__': expect_msg}}) + + t = TestDocument(doc=TestEmbeddedDocument(x=10, y=25)).save() + self.assertEquals(t.doc.z, 35) + + # Asserts not raises + t = TestDocument(doc=TestEmbeddedDocument(x=15, y=35, z=5)) + t.save(clean=False) + def test_save(self): """Ensure that a document may be saved in the database. """ @@ -1935,7 +2005,5 @@ class ValidatorErrorTest(unittest.TestCase): self.assertRaises(OperationError, change_shard_key) - - if __name__ == '__main__': unittest.main() From 99fe1da34564b29cb2505fe938b09ca4253a884c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Nov 2012 13:20:34 +0000 Subject: [PATCH 0831/1279] Add value_decorator into SequenceField Allows post processing of the calculated counter value. --- docs/changelog.rst | 1 + docs/upgrade.rst | 7 +++++++ mongoengine/fields.py | 33 ++++++++++++++++++--------------- tests/test_fields.py | 24 ++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ca18d3e..550cc8d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Updated SequenceFields to allow post processing of the calculated counter value (MongoEngine/mongoengine#141) - Added clean method to documents for pre validation data cleaning (MongoEngine/mongoengine#60) - Added support setting for read prefrence at a query level (MongoEngine/mongoengine#157) - Added _instance to EmbeddedDocuments pointing to the parent (MongoEngine/mongoengine#139) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index bf0a842..daf0912 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -57,6 +57,13 @@ you will need to declare :attr:`allow_inheritance` in the meta data like so: meta = {'allow_inheritance': True} +SequenceFields +-------------- + +:class:`~mongoengine.fields.SequenceField`s now inherit from `BaseField` to +allow flexible storage of the calculated value. As such MIN and MAX settings +are no longer handled. + 0.6 to 0.7 ========== diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 8aa7f64..e2ce33c 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1329,7 +1329,7 @@ class GeoPointField(BaseField): self.error('Both values in point must be float or int') -class SequenceField(IntField): +class SequenceField(BaseField): """Provides a sequental counter see: http://www.mongodb.org/display/DOCS/Object+IDs#ObjectIDs-SequenceNumbers @@ -1341,15 +1341,26 @@ class SequenceField(IntField): cluster of machines, it is easier to create an object ID than have global, uniformly increasing sequence numbers. + Use any callable as `value_decorator` to transform calculated counter into + any value suitable for your needs, e.g. string or hexadecimal + representation of the default integer counter value. + .. versionadded:: 0.5 + + .. versionchanged:: 0.8 added `value_decorator` """ + _auto_gen = True + COLLECTION_NAME = 'mongoengine.counters' + VALUE_DECORATOR = int def __init__(self, collection_name=None, db_alias=None, - sequence_name=None, *args, **kwargs): - self.collection_name = collection_name or 'mongoengine.counters' + sequence_name=None, value_decorator=None, *args, **kwargs): + self.collection_name = collection_name or self.COLLECTION_NAME self.db_alias = db_alias or DEFAULT_CONNECTION_NAME self.sequence_name = sequence_name + self.value_decorator = (callable(value_decorator) and + value_decorator or self.VALUE_DECORATOR) return super(SequenceField, self).__init__(*args, **kwargs) def generate(self): @@ -1364,24 +1375,16 @@ class SequenceField(IntField): update={"$inc": {"next": 1}}, new=True, upsert=True) - return counter['next'] + return self.value_decorator(counter['next']) def __get__(self, instance, owner): - - if instance is None: - return self - - if not instance._data: - return - - value = instance._data.get(self.name) - - if not value and instance._initialised: + value = super(SequenceField, self).__get__(instance, owner) + if value is None and instance._initialised: value = self.generate() instance._data[self.name] = value instance._mark_as_changed(self.name) - return int(value) if value else None + return value def __set__(self, instance, value): diff --git a/tests/test_fields.py b/tests/test_fields.py index 1c13a58..f1a36ed 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- from __future__ import with_statement +import sys +sys.path[0:0] = [""] + import datetime import os import unittest @@ -2184,6 +2187,27 @@ class FieldTest(unittest.TestCase): c = self.db['mongoengine.counters'].find_one({'_id': 'animal.id'}) self.assertEqual(c['next'], 10) + def test_sequence_field_value_decorator(self): + class Person(Document): + id = SequenceField(primary_key=True, value_decorator=str) + name = StringField() + + self.db['mongoengine.counters'].drop() + Person.drop_collection() + + for x in xrange(10): + p = Person(name="Person %s" % x) + p.save() + + c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) + self.assertEqual(c['next'], 10) + + ids = [i.id for i in Person.objects] + self.assertEqual(ids, map(str, range(1, 11))) + + c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) + self.assertEqual(c['next'], 10) + def test_generic_embedded_document(self): class Car(EmbeddedDocument): name = StringField() From 9ca96e4e17f4d954abd4163c5a3e13b0ff094f96 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Nov 2012 13:51:02 +0000 Subject: [PATCH 0832/1279] Added none() to queryset (MongoEngine/mongoengine#127) --- docs/changelog.rst | 1 + mongoengine/queryset/queryset.py | 4 ++++ tests/test_queryset.py | 8 ++++++++ 3 files changed, 13 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 550cc8d..9bd822b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Added none() to queryset (MongoEngine/mongoengine#127) - Updated SequenceFields to allow post processing of the calculated counter value (MongoEngine/mongoengine#141) - Added clean method to documents for pre validation data cleaning (MongoEngine/mongoengine#60) - Added support setting for read prefrence at a query level (MongoEngine/mongoengine#157) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index cf4b4f8..65c71e1 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -489,6 +489,10 @@ class QuerySet(object): self._iter = False self._cursor.rewind() + def none(self): + """Helper that just returns a list""" + return [] + def count(self): """Count the selected elements in the query. """ diff --git a/tests/test_queryset.py b/tests/test_queryset.py index dcb2524..a3e64d2 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -252,6 +252,14 @@ class QuerySetTest(unittest.TestCase): Blog.drop_collection() + def test_none(self): + class A(Document): + pass + + A.drop_collection() + A().save() + self.assertEqual(A.objects.none(), []) + def test_chaining(self): class A(Document): pass From 8706fbe461c1bb3f5ea8d9ee23434a6aeaf86fc5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Nov 2012 15:04:45 +0000 Subject: [PATCH 0833/1279] Updated index creation now tied to Document class ((MongoEngine/mongoengine#102) --- docs/changelog.rst | 1 + docs/upgrade.rst | 9 +- mongoengine/document.py | 83 ++++++++++++- mongoengine/queryset/queryset.py | 114 +++--------------- tests/document/indexes.py | 10 +- .../test_convert_to_new_inheritance_model.py | 2 +- tests/migration/turn_off_inheritance.py | 2 +- tests/test_queryset.py | 3 +- 8 files changed, 115 insertions(+), 109 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9bd822b..ca450f1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Updated index creation now tied to Document class ((MongoEngine/mongoengine#102) - Added none() to queryset (MongoEngine/mongoengine#127) - Updated SequenceFields to allow post processing of the calculated counter value (MongoEngine/mongoengine#141) - Added clean method to documents for pre validation data cleaning (MongoEngine/mongoengine#60) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index daf0912..44c69be 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -40,7 +40,7 @@ inherited classes like so: collection.drop_index(index) # 5. Recreate indexes - Animal.objects._ensure_indexes() + Animal.ensure_indexes() Document Definition @@ -56,6 +56,13 @@ you will need to declare :attr:`allow_inheritance` in the meta data like so: meta = {'allow_inheritance': True} +Indexes +------- + +Index methods are no longer tied to querysets but rather to the document class. +Although `QuerySet._ensure_indexes` and `QuerySet.ensure_index` still exist. +They should be replaced with :func:`~mongoengine.Document.ensure_indexes` / +:func:`~mongoengine.Document.ensure_index`. SequenceFields -------------- diff --git a/mongoengine/document.py b/mongoengine/document.py index fcf8256..cda3a9c 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -7,7 +7,7 @@ from bson.dbref import DBRef from mongoengine import signals, queryset from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, - BaseDict, BaseList) + BaseDict, BaseList, ALLOW_INHERITANCE) from queryset import OperationError, NotUniqueError from connection import get_db, DEFAULT_CONNECTION_NAME @@ -163,6 +163,8 @@ class Document(BaseDocument): ) else: cls._collection = db[collection_name] + if cls._meta.get('auto_create_index', True): + cls.ensure_indexes() return cls._collection def save(self, safe=True, force_insert=False, validate=True, clean=True, @@ -418,9 +420,86 @@ class Document(BaseDocument): """Drops the entire collection associated with this :class:`~mongoengine.Document` type from the database. """ + cls._collection = None db = cls._get_db() db.drop_collection(cls._get_collection_name()) - queryset.QuerySet._reset_already_indexed(cls) + + @classmethod + def ensure_index(cls, key_or_list, drop_dups=False, background=False, + **kwargs): + """Ensure that the given indexes are in place. + + :param key_or_list: a single index key or a list of index keys (to + construct a multi-field index); keys may be prefixed with a **+** + or a **-** to determine the index ordering + """ + index_spec = cls._build_index_spec(key_or_list) + index_spec = index_spec.copy() + fields = index_spec.pop('fields') + index_spec['drop_dups'] = drop_dups + index_spec['background'] = background + index_spec.update(kwargs) + + return cls._get_collection().ensure_index(fields, **index_spec) + + @classmethod + def ensure_indexes(cls): + """Checks the document meta data and ensures all the indexes exist. + + .. note:: You can disable automatic index creation by setting + `auto_create_index` to False in the documents meta data + """ + background = cls._meta.get('index_background', False) + drop_dups = cls._meta.get('index_drop_dups', False) + index_opts = cls._meta.get('index_opts') or {} + index_cls = cls._meta.get('index_cls', True) + + collection = cls._get_collection() + + # determine if an index which we are creating includes + # _cls as its first field; if so, we can avoid creating + # an extra index on _cls, as mongodb will use the existing + # index to service queries against _cls + cls_indexed = False + + def includes_cls(fields): + first_field = None + if len(fields): + if isinstance(fields[0], basestring): + first_field = fields[0] + elif isinstance(fields[0], (list, tuple)) and len(fields[0]): + first_field = fields[0][0] + return first_field == '_cls' + + # Ensure indexes created by uniqueness constraints + for index in cls._meta['unique_indexes']: + cls_indexed = cls_indexed or includes_cls(index) + collection.ensure_index(index, unique=True, background=background, + drop_dups=drop_dups, **index_opts) + + # Ensure document-defined indexes are created + if cls._meta['index_specs']: + index_spec = cls._meta['index_specs'] + for spec in index_spec: + spec = spec.copy() + fields = spec.pop('fields') + cls_indexed = cls_indexed or includes_cls(fields) + opts = index_opts.copy() + opts.update(spec) + collection.ensure_index(fields, background=background, **opts) + + # If _cls is being used (for polymorphism), it needs an index, + # only if another index doesn't begin with _cls + if (index_cls and not cls_indexed and + cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) == True): + collection.ensure_index('_cls', background=background, + **index_opts) + + # Add geo indicies + for field in cls._geo_indices(): + index_spec = [(field.db_field, pymongo.GEO2D)] + collection.ensure_index(index_spec, background=background, + **index_opts) class DynamicDocument(Document): diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 65c71e1..1122123 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1,11 +1,12 @@ -import pprint -import re import copy import itertools import operator +import pprint +import re +import warnings -import pymongo from bson.code import Code +import pymongo from pymongo.common import validate_read_preference from mongoengine import signals @@ -37,8 +38,6 @@ class QuerySet(object): """A set of results returned from a query. Wraps a MongoDB cursor, providing :class:`~mongoengine.Document` objects as the results. """ - - __already_indexed = set() __dereference = False def __init__(self, document, collection): @@ -95,24 +94,6 @@ class QuerySet(object): self._mongo_query.update(self._initial_query) return self._mongo_query - def ensure_index(self, key_or_list, drop_dups=False, background=False, - **kwargs): - """Ensure that the given indexes are in place. - - :param key_or_list: a single index key or a list of index keys (to - construct a multi-field index); keys may be prefixed with a **+** - or a **-** to determine the index ordering - """ - index_spec = self._document._build_index_spec(key_or_list) - index_spec = index_spec.copy() - fields = index_spec.pop('fields') - index_spec['drop_dups'] = drop_dups - index_spec['background'] = background - index_spec.update(kwargs) - - self._collection.ensure_index(fields, **index_spec) - return self - def __call__(self, q_obj=None, class_check=True, slave_okay=False, read_preference=None, **query): """Filter the selected documents by calling the @@ -150,87 +131,26 @@ class QuerySet(object): """Returns all documents.""" return self.__call__() + def ensure_index(self, **kwargs): + """Deprecated use :func:`~Document.ensure_index`""" + msg = ("Doc.objects()._ensure_index() is deprecated. " + "Use Doc.ensure_index() instead.") + warnings.warn(msg, DeprecationWarning) + self._document.__class__.ensure_index(**kwargs) + return self + def _ensure_indexes(self): - """Checks the document meta data and ensures all the indexes exist. - - .. note:: You can disable automatic index creation by setting - `auto_create_index` to False in the documents meta data - """ - background = self._document._meta.get('index_background', False) - drop_dups = self._document._meta.get('index_drop_dups', False) - index_opts = self._document._meta.get('index_opts') or {} - index_cls = self._document._meta.get('index_cls', True) - - # determine if an index which we are creating includes - # _cls as its first field; if so, we can avoid creating - # an extra index on _cls, as mongodb will use the existing - # index to service queries against _cls - cls_indexed = False - - def includes_cls(fields): - first_field = None - if len(fields): - if isinstance(fields[0], basestring): - first_field = fields[0] - elif isinstance(fields[0], (list, tuple)) and len(fields[0]): - first_field = fields[0][0] - return first_field == '_cls' - - # Ensure indexes created by uniqueness constraints - for index in self._document._meta['unique_indexes']: - cls_indexed = cls_indexed or includes_cls(index) - self._collection.ensure_index(index, unique=True, - background=background, drop_dups=drop_dups, **index_opts) - - # Ensure document-defined indexes are created - if self._document._meta['index_specs']: - index_spec = self._document._meta['index_specs'] - for spec in index_spec: - spec = spec.copy() - fields = spec.pop('fields') - cls_indexed = cls_indexed or includes_cls(fields) - opts = index_opts.copy() - opts.update(spec) - self._collection.ensure_index(fields, - background=background, **opts) - - # If _cls is being used (for polymorphism), it needs an index, - # only if another index doesn't begin with _cls - if index_cls and '_cls' in self._query and not cls_indexed: - self._collection.ensure_index('_cls', - background=background, **index_opts) - - # Add geo indicies - for field in self._document._geo_indices(): - index_spec = [(field.db_field, pymongo.GEO2D)] - self._collection.ensure_index(index_spec, - background=background, **index_opts) - - @classmethod - def _reset_already_indexed(cls, document=None): - """Helper to reset already indexed, can be useful for testing purposes - """ - if document: - cls.__already_indexed.discard(document) - cls.__already_indexed.clear() + """Deprecated use :func:`~Document.ensure_indexes`""" + msg = ("Doc.objects()._ensure_indexes() is deprecated. " + "Use Doc.ensure_indexes() instead.") + warnings.warn(msg, DeprecationWarning) + self._document.__class__.ensure_indexes() @property def _collection(self): """Property that returns the collection object. This allows us to perform operations only if the collection is accessed. """ - if self._document not in QuerySet.__already_indexed: - # Ensure collection exists - db = self._document._get_db() - if self._collection_obj.name not in db.collection_names(): - self._document._collection = None - self._collection_obj = self._document._get_collection() - - QuerySet.__already_indexed.add(self._document) - - if self._document._meta.get('auto_create_index', True): - self._ensure_indexes() - return self._collection_obj @property diff --git a/tests/document/indexes.py b/tests/document/indexes.py index a6b74cd..8f83afc 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -80,7 +80,7 @@ class InstanceTest(unittest.TestCase): ('addDate', -1)]}] self.assertEqual(expected_specs, BlogPost._meta['index_specs']) - BlogPost.objects._ensure_indexes() + BlogPost.ensure_indexes() info = BlogPost.objects._collection.index_information() # _id, '-date', 'tags', ('cat', 'date') # NB: there is no index on _cls by itself, since @@ -100,7 +100,7 @@ class InstanceTest(unittest.TestCase): BlogPost.drop_collection() - ExtendedBlogPost.objects._ensure_indexes() + ExtendedBlogPost.ensure_indexes() info = ExtendedBlogPost.objects._collection.index_information() info = [value['key'] for key, value in info.iteritems()] for expected in expected_specs: @@ -141,7 +141,7 @@ class InstanceTest(unittest.TestCase): [{'fields': [('keywords', 1)]}]) # Force index creation - MyDoc.objects._ensure_indexes() + MyDoc.ensure_indexes() self.assertEqual(MyDoc._meta['index_specs'], [{'fields': [('keywords', 1)]}]) @@ -189,7 +189,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual([{'fields': [('location.point', '2d')]}], Place._meta['index_specs']) - Place.objects()._ensure_indexes() + Place.ensure_indexes() info = Place._get_collection().index_information() info = [value['key'] for key, value in info.iteritems()] self.assertTrue([('location.point', '2d')] in info) @@ -335,7 +335,7 @@ class InstanceTest(unittest.TestCase): recursive_obj = EmbeddedDocumentField(RecursiveObject) meta = {'allow_inheritance': True} - RecursiveDocument.objects._ensure_indexes() + RecursiveDocument.ensure_indexes() info = RecursiveDocument._get_collection().index_information() self.assertEqual(info.keys(), ['_id_', '_cls_1']) diff --git a/tests/migration/test_convert_to_new_inheritance_model.py b/tests/migration/test_convert_to_new_inheritance_model.py index 0ef37f7..d4337bf 100644 --- a/tests/migration/test_convert_to_new_inheritance_model.py +++ b/tests/migration/test_convert_to_new_inheritance_model.py @@ -48,4 +48,4 @@ class ConvertToNewInheritanceModel(unittest.TestCase): collection.drop_index(index) # 5. Recreate indexes - Animal.objects._ensure_indexes() + Animal.ensure_indexes() diff --git a/tests/migration/turn_off_inheritance.py b/tests/migration/turn_off_inheritance.py index 5d0f7d7..ee461a8 100644 --- a/tests/migration/turn_off_inheritance.py +++ b/tests/migration/turn_off_inheritance.py @@ -59,4 +59,4 @@ class TurnOffInheritanceTest(unittest.TestCase): collection.drop_index(index) # 5. Recreate indexes - Animal.objects._ensure_indexes() + Animal.ensure_indexes() diff --git a/tests/test_queryset.py b/tests/test_queryset.py index a3e64d2..378b489 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -3078,7 +3078,6 @@ class QuerySetTest(unittest.TestCase): self.assertEqual([1, 2, 3], numbers) Number.drop_collection() - def test_ensure_index(self): """Ensure that manual creation of indexes works. """ @@ -3086,7 +3085,7 @@ class QuerySetTest(unittest.TestCase): message = StringField() meta = {'allow_inheritance': True} - Comment.objects.ensure_index('message') + Comment.ensure_index('message') info = Comment.objects._collection.index_information() info = [(value['key'], From e7c0da38c2b512fabe1e3dc885a4aaf9919bcdef Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Nov 2012 15:09:11 +0000 Subject: [PATCH 0834/1279] Better implementation for none - MongoEngine/mongoengine#127 --- mongoengine/queryset/queryset.py | 6 ++++-- tests/test_queryset.py | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 1122123..058bdd8 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -39,6 +39,7 @@ class QuerySet(object): providing :class:`~mongoengine.Document` objects as the results. """ __dereference = False + __none = False def __init__(self, document, collection): self._document = document @@ -391,7 +392,7 @@ class QuerySet(object): """ self._iter = True try: - if self._limit == 0: + if self._limit == 0 or self.__none: raise StopIteration if self._scalar: return self._get_scalar(self._document._from_son( @@ -411,7 +412,8 @@ class QuerySet(object): def none(self): """Helper that just returns a list""" - return [] + self.__none = True + return self def count(self): """Count the selected elements in the query. diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 378b489..09b6b3f 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -258,7 +258,9 @@ class QuerySetTest(unittest.TestCase): A.drop_collection() A().save() - self.assertEqual(A.objects.none(), []) + + self.assertEqual(list(A.objects.none()), []) + self.assertEqual(list(A.objects.none().all()), []) def test_chaining(self): class A(Document): From 4b45c0cd14182b3abf4b1af8cc9f2296e7bdbcda Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Nov 2012 15:15:04 +0000 Subject: [PATCH 0835/1279] Removed deprecation warning #55 --- mongoengine/queryset/manager.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mongoengine/queryset/manager.py b/mongoengine/queryset/manager.py index 7376e3c..08d4d3a 100644 --- a/mongoengine/queryset/manager.py +++ b/mongoengine/queryset/manager.py @@ -54,8 +54,4 @@ def queryset_manager(func): function should return a :class:`~mongoengine.queryset.QuerySet`, probably the same one that was passed in, but modified in some way. """ - if func.func_code.co_argcount == 1: - import warnings - msg = 'Methods decorated with queryset_manager should take 2 arguments' - warnings.warn(msg, DeprecationWarning) return QuerySetManager(func) From b8d53a6f0d69c81e0566f5cadbe9237e1f530131 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 8 Nov 2012 12:04:14 +0000 Subject: [PATCH 0836/1279] Added json serialisation support - Added to_json and from_json to Document (MongoEngine/mongoengine#1) - Added to_json and from_json to QuerySet (MongoEngine/mongoengine#131) --- docs/changelog.rst | 4 +- mongoengine/base/document.py | 10 ++++ mongoengine/queryset/queryset.py | 10 ++++ tests/document/__init__.py | 5 +- tests/document/instance.py | 2 +- tests/document/json_serialisation.py | 81 ++++++++++++++++++++++++++++ tests/test_fields.py | 39 ++++++++------ tests/test_queryset.py | 70 +++++++++++++++++++++++- 8 files changed, 201 insertions(+), 20 deletions(-) create mode 100644 tests/document/json_serialisation.py diff --git a/docs/changelog.rst b/docs/changelog.rst index ca450f1..26108b5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,9 @@ Changelog Changes in 0.8 ============== -- Updated index creation now tied to Document class ((MongoEngine/mongoengine#102) +- Added to_json and from_json to Document (MongoEngine/mongoengine#1) +- Added to_json and from_json to QuerySet (MongoEngine/mongoengine#131) +- Updated index creation now tied to Document class (MongoEngine/mongoengine#102) - Added none() to queryset (MongoEngine/mongoengine#127) - Updated SequenceFields to allow post processing of the calculated counter value (MongoEngine/mongoengine#141) - Added clean method to documents for pre validation data cleaning (MongoEngine/mongoengine#60) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 46f5320..939c9fb 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -2,6 +2,7 @@ import operator from functools import partial import pymongo +from bson import json_util from bson.dbref import DBRef from mongoengine import signals @@ -253,6 +254,15 @@ class BaseDocument(object): if errors: raise ValidationError('ValidationError', errors=errors) + def to_json(self): + """Converts a document to JSON""" + return json_util.dumps(self.to_mongo()) + + @classmethod + def from_json(cls, json_data): + """Converts json data to an unsaved document instance""" + return cls._from_son(json_util.loads(json_data)) + def __expand_dynamic_values(self, name, value): """expand any dynamic values to their correct types / values""" if not isinstance(value, (dict, list, tuple)): diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 058bdd8..3c44f01 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -6,6 +6,7 @@ import re import warnings from bson.code import Code +from bson import json_util import pymongo from pymongo.common import validate_read_preference @@ -1216,6 +1217,15 @@ class QuerySet(object): max_depth += 1 return self._dereference(self, max_depth=max_depth) + def to_json(self): + """Converts a queryset to JSON""" + return json_util.dumps(self._collection_obj.find(self._query)) + + def from_json(self, json_data): + """Converts json data to unsaved objects""" + son_data = json_util.loads(json_data) + return [self._document._from_son(data) for data in son_data] + @property def _dereference(self): if not self.__dereference: diff --git a/tests/document/__init__.py b/tests/document/__init__.py index 1ef2520..7774ee1 100644 --- a/tests/document/__init__.py +++ b/tests/document/__init__.py @@ -1,4 +1,6 @@ -# TODO EXPLICT IMPORTS +import sys +sys.path[0:0] = [""] +import unittest from class_methods import * from delta import * @@ -6,6 +8,7 @@ from dynamic import * from indexes import * from inheritance import * from instance import * +from json_serialisation import * if __name__ == '__main__': unittest.main() diff --git a/tests/document/instance.py b/tests/document/instance.py index 2e07eb2..2118575 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -346,7 +346,7 @@ class InstanceTest(unittest.TestCase): meta = {'shard_key': ('superphylum',)} Animal.drop_collection() - doc = Animal(superphylum = 'Deuterostomia') + doc = Animal(superphylum='Deuterostomia') doc.save() doc.reload() Animal.drop_collection() diff --git a/tests/document/json_serialisation.py b/tests/document/json_serialisation.py new file mode 100644 index 0000000..dbc09d8 --- /dev/null +++ b/tests/document/json_serialisation.py @@ -0,0 +1,81 @@ +import sys +sys.path[0:0] = [""] + +import unittest +import uuid + +from nose.plugins.skip import SkipTest +from datetime import datetime +from bson import ObjectId + +import pymongo + +from mongoengine import * + +__all__ = ("TestJson",) + + +class TestJson(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + + def test_json_simple(self): + + class Embedded(EmbeddedDocument): + string = StringField() + + class Doc(Document): + string = StringField() + embedded_field = EmbeddedDocumentField(Embedded) + + doc = Doc(string="Hi", embedded_field=Embedded(string="Hi")) + + self.assertEqual(doc, Doc.from_json(doc.to_json())) + + def test_json_complex(self): + + if pymongo.version_tuple[0] <= 2 and pymongo.version_tuple[1] <= 3: + raise SkipTest("Need pymongo 2.4 as has a fix for DBRefs") + + class EmbeddedDoc(EmbeddedDocument): + pass + + class Simple(Document): + pass + + class Doc(Document): + string_field = StringField(default='1') + int_field = IntField(default=1) + float_field = FloatField(default=1.1) + boolean_field = BooleanField(default=True) + datetime_field = DateTimeField(default=datetime.now) + embedded_document_field = EmbeddedDocumentField(EmbeddedDoc, + default=lambda: EmbeddedDoc()) + list_field = ListField(default=lambda: [1, 2, 3]) + dict_field = DictField(default=lambda: {"hello": "world"}) + objectid_field = ObjectIdField(default=ObjectId) + reference_field = ReferenceField(Simple, default=lambda: + Simple().save()) + map_field = MapField(IntField(), default=lambda: {"simple": 1}) + decimal_field = DecimalField(default=1.0) + complex_datetime_field = ComplexDateTimeField(default=datetime.now) + url_field = URLField(default="http://mongoengine.org") + dynamic_field = DynamicField(default=1) + generic_reference_field = GenericReferenceField( + default=lambda: Simple().save()) + sorted_list_field = SortedListField(IntField(), + default=lambda: [1, 2, 3]) + email_field = EmailField(default="ross@example.com") + geo_point_field = GeoPointField(default=lambda: [1, 2]) + sequence_field = SequenceField() + uuid_field = UUIDField(default=uuid.uuid4) + generic_embedded_document_field = GenericEmbeddedDocumentField( + default=lambda: EmbeddedDoc()) + + doc = Doc() + self.assertEqual(doc, Doc.from_json(doc.to_json())) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_fields.py b/tests/test_fields.py index f1a36ed..69cce87 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -606,7 +606,8 @@ class FieldTest(unittest.TestCase): name = StringField() class CategoryList(Document): - categories = SortedListField(EmbeddedDocumentField(Category), ordering='count', reverse=True) + categories = SortedListField(EmbeddedDocumentField(Category), + ordering='count', reverse=True) name = StringField() catlist = CategoryList(name="Top categories") @@ -1616,8 +1617,9 @@ class FieldTest(unittest.TestCase): """Ensure that value is in a container of allowed values. """ class Shirt(Document): - size = StringField(max_length=3, choices=(('S', 'Small'), ('M', 'Medium'), ('L', 'Large'), - ('XL', 'Extra Large'), ('XXL', 'Extra Extra Large'))) + size = StringField(max_length=3, choices=( + ('S', 'Small'), ('M', 'Medium'), ('L', 'Large'), + ('XL', 'Extra Large'), ('XXL', 'Extra Extra Large'))) Shirt.drop_collection() @@ -1633,12 +1635,15 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() def test_choices_get_field_display(self): - """Test dynamic helper for returning the display value of a choices field. + """Test dynamic helper for returning the display value of a choices + field. """ class Shirt(Document): - size = StringField(max_length=3, choices=(('S', 'Small'), ('M', 'Medium'), ('L', 'Large'), - ('XL', 'Extra Large'), ('XXL', 'Extra Extra Large'))) - style = StringField(max_length=3, choices=(('S', 'Small'), ('B', 'Baggy'), ('W', 'wide')), default='S') + size = StringField(max_length=3, choices=( + ('S', 'Small'), ('M', 'Medium'), ('L', 'Large'), + ('XL', 'Extra Large'), ('XXL', 'Extra Extra Large'))) + style = StringField(max_length=3, choices=( + ('S', 'Small'), ('B', 'Baggy'), ('W', 'wide')), default='S') Shirt.drop_collection() @@ -1665,7 +1670,8 @@ class FieldTest(unittest.TestCase): """Ensure that value is in a container of allowed values. """ class Shirt(Document): - size = StringField(max_length=3, choices=('S', 'M', 'L', 'XL', 'XXL')) + size = StringField(max_length=3, + choices=('S', 'M', 'L', 'XL', 'XXL')) Shirt.drop_collection() @@ -1681,11 +1687,15 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() def test_simple_choices_get_field_display(self): - """Test dynamic helper for returning the display value of a choices field. + """Test dynamic helper for returning the display value of a choices + field. """ class Shirt(Document): - size = StringField(max_length=3, choices=('S', 'M', 'L', 'XL', 'XXL')) - style = StringField(max_length=3, choices=('Small', 'Baggy', 'wide'), default='Small') + size = StringField(max_length=3, + choices=('S', 'M', 'L', 'XL', 'XXL')) + style = StringField(max_length=3, + choices=('Small', 'Baggy', 'wide'), + default='Small') Shirt.drop_collection() @@ -1736,7 +1746,7 @@ class FieldTest(unittest.TestCase): self.assertTrue(putfile == result) self.assertEqual(result.the_file.read(), text) self.assertEqual(result.the_file.content_type, content_type) - result.the_file.delete() # Remove file from GridFS + result.the_file.delete() # Remove file from GridFS PutFile.objects.delete() # Ensure file-like objects are stored @@ -1801,7 +1811,6 @@ class FieldTest(unittest.TestCase): the_file = FileField() DemoFile.objects.create() - def test_file_field_no_default(self): class GridDocument(Document): @@ -1817,7 +1826,6 @@ class FieldTest(unittest.TestCase): doc_a = GridDocument() doc_a.save() - doc_b = GridDocument.objects.with_id(doc_a.id) doc_b.the_file.replace(f, filename='doc_b') doc_b.save() @@ -1859,7 +1867,7 @@ class FieldTest(unittest.TestCase): # Second instance test_file_dupe = TestFile() - data = test_file_dupe.the_file.read() # Should be None + data = test_file_dupe.the_file.read() # Should be None self.assertTrue(test_file.name != test_file_dupe.name) self.assertTrue(test_file.the_file.read() != data) @@ -2328,7 +2336,6 @@ class FieldTest(unittest.TestCase): self.assertEqual(error_dict['comments'][1]['content'], u'Field is required') - post.comments[1].content = 'here we go' post.validate() diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 09b6b3f..9dfe9a2 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -1,7 +1,10 @@ from __future__ import with_statement import sys sys.path[0:0] = [""] + import unittest +import uuid +from nose.plugins.skip import SkipTest from datetime import datetime, timedelta @@ -74,7 +77,6 @@ class QuerySetTest(unittest.TestCase): def test_generic_reference(): list(BlogPost.objects(author2__name="test")) - def test_find(self): """Ensure that a query returns a valid set of results. """ @@ -3672,6 +3674,72 @@ class QueryFieldListTest(unittest.TestCase): self.assertRaises(ConfigurationError, Bar.objects, read_preference='Primary') + def test_json_simple(self): + + class Embedded(EmbeddedDocument): + string = StringField() + + class Doc(Document): + string = StringField() + embedded_field = EmbeddedDocumentField(Embedded) + + Doc.drop_collection() + Doc(string="Hi", embedded_field=Embedded(string="Hi")).save() + Doc(string="Bye", embedded_field=Embedded(string="Bye")).save() + + Doc().save() + json_data = Doc.objects.to_json() + doc_objects = list(Doc.objects) + + self.assertEqual(doc_objects, Doc.objects.from_json(json_data)) + + def test_json_complex(self): + if pymongo.version_tuple[0] <= 2 and pymongo.version_tuple[1] <= 3: + raise SkipTest("Need pymongo 2.4 as has a fix for DBRefs") + + class EmbeddedDoc(EmbeddedDocument): + pass + + class Simple(Document): + pass + + class Doc(Document): + string_field = StringField(default='1') + int_field = IntField(default=1) + float_field = FloatField(default=1.1) + boolean_field = BooleanField(default=True) + datetime_field = DateTimeField(default=datetime.now) + embedded_document_field = EmbeddedDocumentField(EmbeddedDoc, + default=lambda: EmbeddedDoc()) + list_field = ListField(default=lambda: [1, 2, 3]) + dict_field = DictField(default=lambda: {"hello": "world"}) + objectid_field = ObjectIdField(default=ObjectId) + reference_field = ReferenceField(Simple, default=lambda: + Simple().save()) + map_field = MapField(IntField(), default=lambda: {"simple": 1}) + decimal_field = DecimalField(default=1.0) + complex_datetime_field = ComplexDateTimeField(default=datetime.now) + url_field = URLField(default="http://mongoengine.org") + dynamic_field = DynamicField(default=1) + generic_reference_field = GenericReferenceField( + default=lambda: Simple().save()) + sorted_list_field = SortedListField(IntField(), + default=lambda: [1, 2, 3]) + email_field = EmailField(default="ross@example.com") + geo_point_field = GeoPointField(default=lambda: [1, 2]) + sequence_field = SequenceField() + uuid_field = UUIDField(default=uuid.uuid4) + generic_embedded_document_field = GenericEmbeddedDocumentField( + default=lambda: EmbeddedDoc()) + + Simple.drop_collection() + Doc.drop_collection() + + Doc().save() + json_data = Doc.objects.to_json() + doc_objects = list(Doc.objects) + + self.assertEqual(doc_objects, Doc.objects.from_json(json_data)) if __name__ == '__main__': From 363e50abbe1db318472de82ad583c98cef3e61c3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 8 Nov 2012 14:46:56 +0000 Subject: [PATCH 0837/1279] Updated documents with embedded documents can be created in a single operation (MongoEngine/mongoengine#6) --- docs/changelog.rst | 1 + mongoengine/base/document.py | 18 ++++++++++++++++-- mongoengine/common.py | 5 +++-- tests/document/instance.py | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 26108b5..778a047 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Added support for creating documents with embedded documents in a single operation (MongoEngine/mongoengine#6) - Added to_json and from_json to Document (MongoEngine/mongoengine#1) - Added to_json and from_json to QuerySet (MongoEngine/mongoengine#131) - Updated index creation now tied to Document class (MongoEngine/mongoengine#102) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 939c9fb..2dd4b03 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -28,7 +28,14 @@ class BaseDocument(object): _dynamic_lock = True _initialised = False - def __init__(self, **values): + def __init__(self, __auto_convert=True, **values): + """ + Initialise a document or embedded document + + :param __auto_convert: Try and will cast python objects to Object types + :param values: A dictionary of values for the document + """ + signals.pre_init.send(self.__class__, document=self, values=values) self._data = {} @@ -50,9 +57,16 @@ class BaseDocument(object): elif self._dynamic: dynamic_data[key] = value else: + FileField = _import_class('FileField') for key, value in values.iteritems(): key = self._reverse_db_field_map.get(key, key) + if (value is not None and __auto_convert and + key in self._fields): + field = self._fields.get(key) + if not isinstance(field, FileField): + value = field.to_python(value) setattr(self, key, value) + # Set any get_fieldname_display methods self.__set_field_display() @@ -487,7 +501,7 @@ class BaseDocument(object): % (cls._class_name, errors)) raise InvalidDocumentError(msg) - obj = cls(**data) + obj = cls(__auto_convert=False, **data) obj._changed_fields = changed_fields obj._created = False return obj diff --git a/mongoengine/common.py b/mongoengine/common.py index c76801c..a8422c0 100644 --- a/mongoengine/common.py +++ b/mongoengine/common.py @@ -9,8 +9,9 @@ def _import_class(cls_name): doc_classes = ('Document', 'DynamicEmbeddedDocument', 'EmbeddedDocument', 'MapReduceDocument') field_classes = ('DictField', 'DynamicField', 'EmbeddedDocumentField', - 'GenericReferenceField', 'GenericEmbeddedDocumentField', - 'GeoPointField', 'ReferenceField', 'StringField') + 'FileField', 'GenericReferenceField', + 'GenericEmbeddedDocumentField', 'GeoPointField', + 'ReferenceField', 'StringField') queryset_classes = ('OperationError',) deref_classes = ('DeReference',) diff --git a/tests/document/instance.py b/tests/document/instance.py index 2118575..8fb4fd7 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -2005,5 +2005,41 @@ class ValidatorErrorTest(unittest.TestCase): self.assertRaises(OperationError, change_shard_key) + def test_kwargs_simple(self): + + class Embedded(EmbeddedDocument): + name = StringField() + + class Doc(Document): + doc_name = StringField() + doc = EmbeddedDocumentField(Embedded) + + classic_doc = Doc(doc_name="my doc", doc=Embedded(name="embedded doc")) + dict_doc = Doc(**{"doc_name": "my doc", + "doc": {"name": "embedded doc"}}) + + self.assertEqual(classic_doc, dict_doc) + self.assertEqual(classic_doc._data, dict_doc._data) + + def test_kwargs_complex(self): + + class Embedded(EmbeddedDocument): + name = StringField() + + class Doc(Document): + doc_name = StringField() + docs = ListField(EmbeddedDocumentField(Embedded)) + + classic_doc = Doc(doc_name="my doc", docs=[ + Embedded(name="embedded doc1"), + Embedded(name="embedded doc2")]) + dict_doc = Doc(**{"doc_name": "my doc", + "docs": [{"name": "embedded doc1"}, + {"name": "embedded doc2"}]}) + + self.assertEqual(classic_doc, dict_doc) + self.assertEqual(classic_doc._data, dict_doc._data) + + if __name__ == '__main__': unittest.main() From 1a93b9b2263a71f179a35760b84b5d740fdd4f57 Mon Sep 17 00:00:00 2001 From: helduel Date: Thu, 8 Nov 2012 16:30:29 +0100 Subject: [PATCH 0838/1279] More precise "created" keyword argument signals If a document has a user given id value, the post_save signal always got the "created" keyword argument with False value (unless force_insert is True). This patch uses the result of getlasterror to check whether the save was an update or not. --- mongoengine/document.py | 19 +++++++++++++++---- tests/test_signals.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 7b3afaf..694d1ed 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -233,13 +233,24 @@ class Document(BaseDocument): actual_key = self._db_field_map.get(k, k) select_dict[actual_key] = doc[actual_key] + def is_new_object(last_error): + if last_error is not None: + updated = last_error.get("updatedExisting") + if updated is not None: + return not updated + return created + upsert = self._created if updates: - collection.update(select_dict, {"$set": updates}, - upsert=upsert, safe=safe, **write_options) + last_error = collection.update(select_dict, + {"$set": updates}, upsert=upsert, safe=safe, + **write_options) + created = is_new_object(last_error) if removals: - collection.update(select_dict, {"$unset": removals}, - upsert=upsert, safe=safe, **write_options) + last_error = collection.update(select_dict, + {"$unset": removals}, upsert=upsert, safe=safe, + **write_options) + created = created or is_new_object(last_error) warn_cascade = not cascade and 'cascade' not in self._meta cascade = (self._meta.get('cascade', True) diff --git a/tests/test_signals.py b/tests/test_signals.py index d119924..2ca820d 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -108,6 +108,20 @@ class SignalTests(unittest.TestCase): signal_output.append('post_delete Another signal, %s' % document) self.Another = Another + + class ExplicitId(Document): + id = IntField(primary_key=True) + + @classmethod + def post_save(cls, sender, document, **kwargs): + if 'created' in kwargs: + if kwargs['created']: + signal_output.append('Is created') + else: + signal_output.append('Is updated') + + self.ExplicitId = ExplicitId + self.ExplicitId.objects.delete() # Save up the number of connected signals so that we can check at the end # that all the signals we register get properly unregistered self.pre_signals = ( @@ -137,6 +151,8 @@ class SignalTests(unittest.TestCase): signals.pre_delete.connect(Another.pre_delete, sender=Another) signals.post_delete.connect(Another.post_delete, sender=Another) + signals.post_save.connect(ExplicitId.post_save, sender=ExplicitId) + def tearDown(self): signals.pre_init.disconnect(self.Author.pre_init) signals.post_init.disconnect(self.Author.post_init) @@ -154,6 +170,8 @@ class SignalTests(unittest.TestCase): signals.post_save.disconnect(self.Another.post_save) signals.pre_save.disconnect(self.Another.pre_save) + signals.post_save.disconnect(self.ExplicitId.post_save) + # Check that all our signals got disconnected properly. post_signals = ( len(signals.pre_init.receivers), @@ -166,6 +184,8 @@ class SignalTests(unittest.TestCase): len(signals.post_bulk_insert.receivers), ) + self.ExplicitId.objects.delete() + self.assertEqual(self.pre_signals, post_signals) def test_model_signals(self): @@ -228,3 +248,12 @@ class SignalTests(unittest.TestCase): ]) self.Author.objects.delete() + + def test_signals_with_explicit_doc_ids(self): + """ Model saves must have a created flag the first time.""" + ei = self.ExplicitId(id=123) + # post save must received the created flag, even if there's already + # an object id present + self.assertEqual(self.get_signal_output(ei.save), ['Is created']) + # second time, it must be an update + self.assertEqual(self.get_signal_output(ei.save), ['Is updated']) From f265915aa227f941fa07fae2df85155be9e0f3d1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 8 Nov 2012 16:35:20 +0000 Subject: [PATCH 0839/1279] Updated inheritable objects created by upsert now contain _cls (MongoEngine/mongoengine#118) --- docs/changelog.rst | 1 + mongoengine/queryset/queryset.py | 12 ++++++++++-- tests/test_queryset.py | 19 +++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 778a047..e8d3d57 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Updated inheritable objects created by upsert now contain _cls (MongoEngine/mongoengine#118) - Added support for creating documents with embedded documents in a single operation (MongoEngine/mongoengine#6) - Added to_json and from_json to Document (MongoEngine/mongoengine#1) - Added to_json and from_json to QuerySet (MongoEngine/mongoengine#131) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 3c44f01..bfd15a8 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -824,8 +824,16 @@ class QuerySet(object): if not write_options: write_options = {} - update = transform.update(self._document, **update) query = self._query + update = transform.update(self._document, **update) + + # If doing an atomic upsert on an inheritable class + # then ensure we add _cls to the update operation + if upsert and '_cls' in query: + if '$set' in update: + update["$set"]["_cls"] = self._document._class_name + else: + update["$set"] = {"_cls": self._document._class_name} try: ret = self._collection.update(query, update, multi=multi, @@ -852,7 +860,7 @@ class QuerySet(object): .. versionadded:: 0.2 """ - return self.update(safe_update=True, upsert=False, multi=False, + return self.update(safe_update=True, upsert=upsert, multi=False, write_options=None, **update) def __iter__(self): diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 9dfe9a2..a86920e 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -3741,6 +3741,25 @@ class QueryFieldListTest(unittest.TestCase): self.assertEqual(doc_objects, Doc.objects.from_json(json_data)) + def test_upsert_includes_cls(self): + """Upserts should include _cls information for inheritable classes + """ + + class Test(Document): + test = StringField() + + Test.drop_collection() + Test.objects(test='foo').update_one(upsert=True, set__test='foo') + self.assertFalse('_cls' in Test._collection.find_one()) + + class Test(Document): + meta = {'allow_inheritance': True} + test = StringField() + + Test.drop_collection() + + Test.objects(test='foo').update_one(upsert=True, set__test='foo') + self.assertTrue('_cls' in Test._collection.find_one()) if __name__ == '__main__': unittest.main() From dfdc0d92c3f95e9b6788c14c8fb9facaa4bfbfc9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 8 Nov 2012 16:40:58 +0000 Subject: [PATCH 0840/1279] Updated docs --- docs/guide/defining-documents.rst | 2 +- docs/guide/document-instances.rst | 2 +- docs/upgrade.rst | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index ea8e05b..9abea9b 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -597,7 +597,7 @@ Working with existing data As MongoEngine no longer defaults to needing :attr:`_cls` you can quickly and easily get working with existing data. Just define the document to match the expected schema in your database. If you have wildly varying schemas then -a :class:`~mongoengine.DynamicDocument` might be more appropriate. +a :class:`~mongoengine.DynamicDocument` might be more appropriate. :: # Will work with data in an existing collection named 'cmsPage' class Page(Document): diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index b3bf687..e8e7d63 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -64,7 +64,7 @@ document values for example:: .. note:: Cleaning is only called if validation is turned on and when calling -:meth:`~mongoengine.Document.save`. + :meth:`~mongoengine.Document.save`. Cascading Saves --------------- diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 44c69be..bf48527 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -14,7 +14,7 @@ Data Model The inheritance model has changed, we no longer need to store an array of :attr:`types` with the model we can just use the classname in :attr:`_cls`. This means that you will have to update your indexes for each of your -inherited classes like so: +inherited classes like so: :: # 1. Declaration of the class class Animal(Document): @@ -49,7 +49,7 @@ Document Definition The default for inheritance has changed - its now off by default and :attr:`_cls` will not be stored automatically with the class. So if you extend your :class:`~mongoengine.Document` or :class:`~mongoengine.EmbeddedDocuments` -you will need to declare :attr:`allow_inheritance` in the meta data like so: +you will need to declare :attr:`allow_inheritance` in the meta data like so: :: class Animal(Document): name = StringField() @@ -67,7 +67,7 @@ They should be replaced with :func:`~mongoengine.Document.ensure_indexes` / SequenceFields -------------- -:class:`~mongoengine.fields.SequenceField`s now inherit from `BaseField` to +:class:`~mongoengine.fields.SequenceField` now inherits from `BaseField` to allow flexible storage of the calculated value. As such MIN and MAX settings are no longer handled. From 787fc1cd8ba115aa1186952833a29e8ee3b0d45e Mon Sep 17 00:00:00 2001 From: yak Date: Tue, 13 Nov 2012 13:02:07 +0100 Subject: [PATCH 0841/1279] bug fix for RefferenceField.to_mongo when dbref=False --- mongoengine/fields.py | 2 +- tests/test_fields.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 01d3fc6..ee02906 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -777,7 +777,7 @@ class ReferenceField(BaseField): def to_mongo(self, document): if isinstance(document, DBRef): if not self.dbref: - return DBRef.id + return document.id return document elif not self.dbref and isinstance(document, basestring): return document diff --git a/tests/test_fields.py b/tests/test_fields.py index 9806550..abc50a3 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1104,6 +1104,15 @@ class FieldTest(unittest.TestCase): p = Person.objects.get(name="Ross") self.assertEqual(p.parent, p1) + + def test_dbref_to_mongo(self): + class Person(Document): + name = StringField() + parent = ReferenceField('self', dbref=False) + + p1 = Person._from_son({'name':"Yakxxx", 'parent': "50a234ea469ac1eda42d347d"}) + mongoed = p1.to_mongo() + self.assertIsInstance(mongoed['parent'], ObjectId) def test_objectid_reference_fields(self): From 0da2dfd191143dcf11d3281cea7b3da2c790c99d Mon Sep 17 00:00:00 2001 From: yak Date: Tue, 13 Nov 2012 13:04:05 +0100 Subject: [PATCH 0842/1279] addition to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 6ba2f88..bae1dcb 100644 --- a/AUTHORS +++ b/AUTHORS @@ -124,3 +124,4 @@ that much better: * Stefan Wójcik * dimonb * Garry Polley + * Jakub Kot From 28ef54986d0847dac129f5bb4b66e644e4fe947b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 21 Nov 2012 16:53:06 +0000 Subject: [PATCH 0843/1279] Deprecated `get_or_create` (MongoEngine/mongoengine#35) --- docs/changelog.rst | 1 + mongoengine/queryset/queryset.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e8d3d57..c3c6340 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Deprecated `get_or_create` (MongoEngine/mongoengine#35) - Updated inheritable objects created by upsert now contain _cls (MongoEngine/mongoengine#118) - Added support for creating documents with embedded documents in a single operation (MongoEngine/mongoengine#6) - Added to_json and from_json to Document (MongoEngine/mongoengine#1) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index bfd15a8..dde7d55 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -232,10 +232,11 @@ class QuerySet(object): dictionary of default values for the new document may be provided as a keyword argument called :attr:`defaults`. - .. warning:: This requires two separate operations and therefore a + .. note:: This requires two separate operations and therefore a race condition exists. Because there are no transactions in mongoDB other approaches should be investigated, to ensure you - don't accidently duplicate data when using this method. + don't accidently duplicate data when using this method. This is + now scheduled to be removed before 1.0 :param write_options: optional extra keyword arguments used if we have to create a new document. @@ -244,9 +245,14 @@ class QuerySet(object): :param auto_save: if the object is to be saved automatically if not found. + .. deprecated:: 0.8 .. versionchanged:: 0.6 - added `auto_save` .. versionadded:: 0.3 """ + msg = ("get_or_create is scheduled to be deprecated. The approach is " + "flawed without transactions. Upserts should be preferred.") + raise DeprecationWarning(msg) + defaults = query.get('defaults', {}) if 'defaults' in query: del query['defaults'] From aa5a9ff1f428027a5eb6adec54fa6f596ae4e32d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 21 Nov 2012 17:03:32 +0000 Subject: [PATCH 0844/1279] Documentation update for document errors (MongoEngine/mongoengine#124) --- docs/changelog.rst | 1 + docs/guide/querying.rst | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c3c6340..a13a5f1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Documentation update for document errors (MongoEngine/mongoengine#124) - Deprecated `get_or_create` (MongoEngine/mongoengine#35) - Updated inheritable objects created by upsert now contain _cls (MongoEngine/mongoengine#118) - Added support for creating documents with embedded documents in a single operation (MongoEngine/mongoengine#6) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 1449801..d582943 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -179,9 +179,11 @@ Retrieving unique results ------------------------- To retrieve a result that should be unique in the collection, use :meth:`~mongoengine.queryset.QuerySet.get`. This will raise -:class:`~mongoengine.queryset.DoesNotExist` if no document matches the query, -and :class:`~mongoengine.queryset.MultipleObjectsReturned` if more than one -document matched the query. +:class:`~mongoengine.queryset.DoesNotExist` if +no document matches the query, and +:class:`~mongoengine.queryset.MultipleObjectsReturned` +if more than one document matched the query. These exceptions are merged into +your document defintions eg: `MyDoc.DoesNotExist` A variation of this method exists, :meth:`~mongoengine.queryset.Queryset.get_or_create`, that will create a new From 003454573ce86614344e11e6252d9accaa392101 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 21 Nov 2012 17:14:53 +0000 Subject: [PATCH 0845/1279] Making django user sparse (MongoEngine/mongoengine#128) --- mongoengine/django/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 65afacf..1685f6f 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -70,7 +70,7 @@ class User(Document): meta = { 'allow_inheritance': True, 'indexes': [ - {'fields': ['username'], 'unique': True} + {'fields': ['username'], 'unique': True, 'sparse': True} ] } From 2c0fc142a385f791e6cc67b5bf624eb6e2df82ed Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 26 Nov 2012 21:04:06 +0000 Subject: [PATCH 0846/1279] Updated travis.yml --- .travis.yml | 2 +- tests/test_dereference.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c10a1f3..1aa9774 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,4 +26,4 @@ notifications: branches: only: - master - - 0.7 \ No newline at end of file + - 0.8 \ No newline at end of file diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 7b149db..0eb891c 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -42,6 +42,12 @@ class FieldTest(unittest.TestCase): group_obj = Group.objects.first() self.assertEqual(q, 1) + len(group_obj._data['members']) + self.assertEqual(q, 1) + + len(group_obj.members) + self.assertEqual(q, 2) + [m for m in group_obj.members] self.assertEqual(q, 2) From 66c6d14f7ac5838f48664b48a1903cb8e72b558f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 10:50:22 +0000 Subject: [PATCH 0847/1279] Trying to fix seesaw test on travis --- tests/test_all_warnings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_all_warnings.py b/tests/test_all_warnings.py index 9b38fa6..8297e45 100644 --- a/tests/test_all_warnings.py +++ b/tests/test_all_warnings.py @@ -53,7 +53,7 @@ class TestWarnings(unittest.TestCase): p2.parent = p1 p2.save(cascade=False) - self.assertEqual(len(self.warning_list), 1) + self.assertTrue(len(self.warning_list) > 0) warning = self.warning_list[0] self.assertEqual(FutureWarning, warning["category"]) self.assertTrue("ReferenceFields will default to using ObjectId" From 9f5ab8149f4085a305f9a7b86ce3d4e9ab497514 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 11:06:55 +0000 Subject: [PATCH 0848/1279] Adding some debugging --- tests/test_all_warnings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_all_warnings.py b/tests/test_all_warnings.py index 8297e45..7ef1f21 100644 --- a/tests/test_all_warnings.py +++ b/tests/test_all_warnings.py @@ -77,6 +77,8 @@ class TestWarnings(unittest.TestCase): p2.save() self.assertEqual(len(self.warning_list), 1) + if len(self.warning_list) > 1: + print self.warning_list warning = self.warning_list[0] self.assertEqual(FutureWarning, warning["category"]) self.assertTrue("Cascading saves will default to off in 0.8" From 653c4259eebc0f37e1e2b82d5510d5c4c9475f87 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 11:59:34 +0000 Subject: [PATCH 0849/1279] Fixed handling for old style types --- docs/changelog.rst | 4 ++++ mongoengine/__init__.py | 2 +- mongoengine/base.py | 9 ++++----- python-mongoengine.spec | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index aac24c6..7457eeb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.7.7 +================ +- Fix handling for old style _types + Changes in 0.7.6 ================ - Unicode fix for repr (MongoEngine/mongoengine#133) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index cdfbfff..9f1f552 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 7, 6) +VERSION = (0, 7, 7) def get_version(): diff --git a/mongoengine/base.py b/mongoengine/base.py index fa12e35..208e0e5 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -121,11 +121,10 @@ class ValidationError(AssertionError): def get_document(name): doc = _document_registry.get(name, None) if not doc: - # Possible old style names - end = ".%s" % name - possible_match = [k for k in _document_registry.keys() - if k.endswith(end)] - if len(possible_match) == 1: + # Possible old style name + end = name.split('.')[-1] + possible_match = [k for k in _document_registry.keys() if k == end] + if len(possible_match) == 1 and end != name: doc = _document_registry.get(possible_match.pop(), None) if not doc: raise NotRegistered(""" diff --git a/python-mongoengine.spec b/python-mongoengine.spec index d796f99..9a376ec 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.7.6 +Version: 0.7.7 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 027b3d36de9a42d31757353150dd39a94a6da584 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:01:58 +0000 Subject: [PATCH 0850/1279] Fixed deprecation warning --- mongoengine/queryset/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index dde7d55..3acee36 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -251,7 +251,7 @@ class QuerySet(object): """ msg = ("get_or_create is scheduled to be deprecated. The approach is " "flawed without transactions. Upserts should be preferred.") - raise DeprecationWarning(msg) + warnings.warn(msg, DeprecationWarning) defaults = query.get('defaults', {}) if 'defaults' in query: From b5e868655e5ff056150b358510f8aea4a8125881 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:02:49 +0000 Subject: [PATCH 0851/1279] Updated travis.yml --- .travis.yml | 2 +- tests/test_dereference.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c10a1f3..1aa9774 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,4 +26,4 @@ notifications: branches: only: - master - - 0.7 \ No newline at end of file + - 0.8 \ No newline at end of file diff --git a/tests/test_dereference.py b/tests/test_dereference.py index c9631eb..8f61792 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -42,6 +42,12 @@ class FieldTest(unittest.TestCase): group_obj = Group.objects.first() self.assertEqual(q, 1) + len(group_obj._data['members']) + self.assertEqual(q, 1) + + len(group_obj.members) + self.assertEqual(q, 2) + [m for m in group_obj.members] self.assertEqual(q, 2) From 59e7617e82a89a0afe61f24180f8873efbb316bf Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:02:49 +0000 Subject: [PATCH 0852/1279] Trying to fix seesaw test on travis --- tests/all_warnings/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/all_warnings/__init__.py b/tests/all_warnings/__init__.py index 4609c5a..7d4db0d 100644 --- a/tests/all_warnings/__init__.py +++ b/tests/all_warnings/__init__.py @@ -45,7 +45,7 @@ class AllWarnings(unittest.TestCase): p2.parent = p1 p2.save(cascade=False) - self.assertEqual(len(self.warning_list), 1) + self.assertTrue(len(self.warning_list) > 0) warning = self.warning_list[0] self.assertEqual(FutureWarning, warning["category"]) self.assertTrue("ReferenceFields will default to using ObjectId" From b849c719a8c336d388a5a341e2ffb9afd9bf86eb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:02:50 +0000 Subject: [PATCH 0853/1279] Adding some debugging --- tests/all_warnings/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/all_warnings/__init__.py b/tests/all_warnings/__init__.py index 7d4db0d..220b0bb 100644 --- a/tests/all_warnings/__init__.py +++ b/tests/all_warnings/__init__.py @@ -69,6 +69,8 @@ class AllWarnings(unittest.TestCase): p2.save() self.assertEqual(len(self.warning_list), 1) + if len(self.warning_list) > 1: + print self.warning_list warning = self.warning_list[0] self.assertEqual(FutureWarning, warning["category"]) self.assertTrue("Cascading saves will default to off in 0.8" From 68e4a27aaff24ca60b851da71f5c4aff45e87341 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:02:50 +0000 Subject: [PATCH 0854/1279] Fixed handling for old style types --- docs/changelog.rst | 4 ++++ python-mongoengine.spec | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index a13a5f1..756b1cd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,10 @@ Changes in 0.8 - Inheritance is off by default (MongoEngine/mongoengine#122) - Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) +Changes in 0.7.7 +================ +- Fix handling for old style _types + Changes in 0.7.6 ================ - Unicode fix for repr (MongoEngine/mongoengine#133) diff --git a/python-mongoengine.spec b/python-mongoengine.spec index d796f99..9a376ec 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.7.6 +Version: 0.7.7 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From f9dd051ec90c9fc17d2cc1221d43e6549427b915 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:02:50 +0000 Subject: [PATCH 0855/1279] Merged get_document fix --- mongoengine/base/common.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mongoengine/base/common.py b/mongoengine/base/common.py index 82728d1..dc43d40 100644 --- a/mongoengine/base/common.py +++ b/mongoengine/base/common.py @@ -9,11 +9,10 @@ _document_registry = {} def get_document(name): doc = _document_registry.get(name, None) - if not doc: - # Possible old style names - end = ".%s" % name - possible_match = [k for k in _document_registry.keys() - if k.endswith(end)] + if not doc and '.' in name: + # Possible old style name + end = name.split('.')[-1] + possible_match = [k for k in _document_registry.keys() if k == end] if len(possible_match) == 1: doc = _document_registry.get(possible_match.pop(), None) if not doc: From 3598fe0fb45d969ef936fedc33db4a63371266a8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:02:50 +0000 Subject: [PATCH 0856/1279] Adding _collection to _cls --- mongoengine/base/metaclasses.py | 3 +++ tests/document/instance.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index e68ec13..c6c4db1 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -169,6 +169,9 @@ class DocumentMetaclass(type): "field name" % field.name) raise InvalidDocumentError(msg) + if issubclass(new_class, Document): + new_class._collection = None + # Add class to the _document_registry _document_registry[new_class._class_name] = new_class diff --git a/tests/document/instance.py b/tests/document/instance.py index 8fb4fd7..de677e2 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1762,6 +1762,31 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Book._get_collection(), get_db("testdb-2")[Book._get_collection_name()]) self.assertEqual(AuthorBooks._get_collection(), get_db("testdb-3")[AuthorBooks._get_collection_name()]) + def test_db_alias_overrides(self): + """db_alias can be overriden + """ + # Register a connection with db_alias testdb-2 + register_connection('testdb-2', 'mongoenginetest2') + + class A(Document): + """Uses default db_alias + """ + name = StringField() + meta = {"allow_inheritance": True} + + class B(A): + """Uses testdb-2 db_alias + """ + meta = {"db_alias": "testdb-2"} + + A.objects.all() + + self.assertEquals('testdb-2', B._meta.get('db_alias')) + self.assertEquals('mongoenginetest', + A._get_collection().database.name) + self.assertEquals('mongoenginetest2', + B._get_collection().database.name) + def test_db_alias_propagates(self): """db_alias propagates? """ From 219b28c97b846b90e0d755558be526bb51773a92 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:04:57 +0000 Subject: [PATCH 0857/1279] Updated docs regarding 3598fe0fb45d969ef936fedc33db4a63371266a8 Fixed db_alias and inherited Documents (MongoEngine/mongoengine#143) --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 6ba2f88..5801981 100644 --- a/AUTHORS +++ b/AUTHORS @@ -124,3 +124,4 @@ that much better: * Stefan Wójcik * dimonb * Garry Polley + * James Slagle diff --git a/docs/changelog.rst b/docs/changelog.rst index 756b1cd..a56f33a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8 ============== +- Fixed db_alias and inherited Documents (MongoEngine/mongoengine#143) - Documentation update for document errors (MongoEngine/mongoengine#124) - Deprecated `get_or_create` (MongoEngine/mongoengine#35) - Updated inheritable objects created by upsert now contain _cls (MongoEngine/mongoengine#118) From f6f7c12f0ecadb7ae50f29287622b9a42507dacb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 27 Nov 2012 14:37:13 +0000 Subject: [PATCH 0858/1279] Added test case checking type with dbref=False Ensures when dbref=False the data is stored as the same type as the primary key of the item stored. MongoEngine/mongoengine#160 --- tests/test_dereference.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 8f61792..41f8aeb 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -123,6 +123,27 @@ class FieldTest(unittest.TestCase): User.drop_collection() Group.drop_collection() + def test_list_item_dereference_dref_false_stores_as_type(self): + """Ensure that DBRef items are stored as their type + """ + class User(Document): + my_id = IntField(primary_key=True) + name = StringField() + + class Group(Document): + members = ListField(ReferenceField(User, dbref=False)) + + User.drop_collection() + Group.drop_collection() + + user = User(my_id=1, name='user 1').save() + + Group(members=User.objects).save() + group = Group.objects.first() + + self.assertEqual(Group._get_collection().find_one()['members'], [1]) + self.assertEqual(group.members, [user]) + def test_handle_old_style_references(self): """Ensure that DBRef items in ListFields are dereferenced. """ From 9d52e1865931025ec4c10212db70b0b727e2a91a Mon Sep 17 00:00:00 2001 From: Peter Teichman Date: Wed, 21 Nov 2012 13:22:10 -0500 Subject: [PATCH 0859/1279] Don't freeze the current query state when calling .order_by() This changes order_by() to eliminate its reference to self._cursor. This meant that any parameters built by QuerySet that followed an order_by() clause were ignored. --- mongoengine/queryset.py | 6 ++++-- tests/test_queryset.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 5c7b9c8..58b1959 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -586,11 +586,13 @@ class QuerySet(object): if self._where_clause: self._cursor_obj.where(self._where_clause) - # apply default ordering if self._ordering: + # Apply query ordering self._cursor_obj.sort(self._ordering) elif self._document._meta['ordering']: + # Otherwise, apply the ordering from the document model self.order_by(*self._document._meta['ordering']) + self._cursor_obj.sort(self._ordering) if self._limit is not None: self._cursor_obj.limit(self._limit - (self._skip or 0)) @@ -1274,7 +1276,7 @@ class QuerySet(object): key_list.append((key, direction)) self._ordering = key_list - self._cursor.sort(key_list) + return self def explain(self, format=False): diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 8f846ea..48d2a26 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -1793,6 +1793,22 @@ class QuerySetTest(unittest.TestCase): ages = [p.age for p in self.Person.objects.order_by('-name')] self.assertEqual(ages, [30, 40, 20]) + def test_order_by_chaining(self): + """Ensure that an order_by query chains properly and allows .only() + """ + self.Person(name="User A", age=20).save() + self.Person(name="User B", age=40).save() + self.Person(name="User C", age=30).save() + + only_age = self.Person.objects.order_by('-age').only('age') + + names = [p.name for p in only_age] + ages = [p.age for p in only_age] + + # The .only('age') clause should mean that all names are None + self.assertEqual(names, [None, None, None]) + self.assertEqual(ages, [40, 30, 20]) + def test_confirm_order_by_reference_wont_work(self): """Ordering by reference is not possible. Use map / reduce.. or denormalise""" From 3bdc9a2f0940cf03d6f72212e4db3148f0a0cfb7 Mon Sep 17 00:00:00 2001 From: Adrian Scott Date: Thu, 29 Nov 2012 20:53:09 -0500 Subject: [PATCH 0860/1279] session collection parameter; encoding optional Added a parameter for the name of the session collection; Added the option to not encode session_data, which is useful for expiring sessions of users when a password is changed, etc.; these upgrades provided by SocialVilla Inc. --- mongoengine/django/sessions.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index f178342..0330ee1 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -15,13 +15,20 @@ MONGOENGINE_SESSION_DB_ALIAS = getattr( settings, 'MONGOENGINE_SESSION_DB_ALIAS', DEFAULT_CONNECTION_NAME) +MONGOENGINE_SESSION_COLLECTION = getattr( + settings, 'MONGOENGINE_SESSION_COLLECTION', + 'django_session') + +MONGOENGINE_SESSION_DATA_ENCODE = getattr( + settings, 'MONGOENGINE_SESSION_DATA_ENCODE', + True) class MongoSession(Document): session_key = fields.StringField(primary_key=True, max_length=40) - session_data = fields.StringField() + session_data = fields.StringField() if MONGOENGINE_SESSION_DATA_ENCODE else fields.DictField() expire_date = fields.DateTimeField() - meta = {'collection': 'django_session', + meta = {'collection': MONGOENGINE_SESSION_COLLECTION, 'db_alias': MONGOENGINE_SESSION_DB_ALIAS, 'allow_inheritance': False} @@ -34,7 +41,10 @@ class SessionStore(SessionBase): try: s = MongoSession.objects(session_key=self.session_key, expire_date__gt=datetime.now())[0] - return self.decode(force_unicode(s.session_data)) + if MONGOENGINE_SESSION_DATA_ENCODE: + return self.decode(force_unicode(s.session_data)) + else: + return s.session_data except (IndexError, SuspiciousOperation): self.create() return {} @@ -57,7 +67,10 @@ class SessionStore(SessionBase): if self.session_key is None: self._session_key = self._get_new_session_key() s = MongoSession(session_key=self.session_key) - s.session_data = self.encode(self._get_session(no_load=must_create)) + if MONGOENGINE_SESSION_DATA_ENCODE: + s.session_data = self.encode(self._get_session(no_load=must_create)) + else: + s.session_data = self._get_session(no_load=must_create) s.expire_date = self.get_expiry_date() try: s.save(force_insert=must_create, safe=True) From b10d76cf4b396edf5473a386717a31831b749500 Mon Sep 17 00:00:00 2001 From: Adrian Scott Date: Thu, 29 Nov 2012 21:28:03 -0500 Subject: [PATCH 0861/1279] split line to meet 79 char max line limit --- mongoengine/django/sessions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index 0330ee1..ca7b01f 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -25,7 +25,8 @@ MONGOENGINE_SESSION_DATA_ENCODE = getattr( class MongoSession(Document): session_key = fields.StringField(primary_key=True, max_length=40) - session_data = fields.StringField() if MONGOENGINE_SESSION_DATA_ENCODE else fields.DictField() + session_data = fields.StringField() if MONGOENGINE_SESSION_DATA_ENCODE \ + else fields.DictField() expire_date = fields.DateTimeField() meta = {'collection': MONGOENGINE_SESSION_COLLECTION, From 4fe87b40da989bdd8caecc56fe48e025f0061ade Mon Sep 17 00:00:00 2001 From: Adrian Scott Date: Thu, 29 Nov 2012 21:49:54 -0500 Subject: [PATCH 0862/1279] added comments --- mongoengine/django/sessions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index ca7b01f..810b626 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -15,10 +15,12 @@ MONGOENGINE_SESSION_DB_ALIAS = getattr( settings, 'MONGOENGINE_SESSION_DB_ALIAS', DEFAULT_CONNECTION_NAME) +# a setting for the name of the collection used to store sessions MONGOENGINE_SESSION_COLLECTION = getattr( settings, 'MONGOENGINE_SESSION_COLLECTION', 'django_session') +# a setting for whether session data is stored encoded or not MONGOENGINE_SESSION_DATA_ENCODE = getattr( settings, 'MONGOENGINE_SESSION_DATA_ENCODE', True) From 376d1c97ab9bce3df6f060bc18e9edd6c64a3e87 Mon Sep 17 00:00:00 2001 From: Shaun Duncan Date: Tue, 4 Dec 2012 13:08:49 -0500 Subject: [PATCH 0863/1279] EmailField should honor StringField validation as well --- mongoengine/fields.py | 1 + tests/test_fields.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 1f86560..368c000 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -141,6 +141,7 @@ class EmailField(StringField): def validate(self, value): if not EmailField.EMAIL_REGEX.match(value): self.error('Invalid Mail-address: %s' % value) + super(EmailField, self).validate(value) class IntField(BaseField): diff --git a/tests/test_fields.py b/tests/test_fields.py index 68c79b5..db23d93 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -2114,6 +2114,18 @@ class FieldTest(unittest.TestCase): post.comments[1].content = 'here we go' post.validate() + def test_email_field_honors_regex(self): + class User(Document): + email = EmailField(regex=r'\w+@example.com') + + # Fails regex validation + user = User(email='me@foo.com') + self.assertRaises(ValidationError, user.validate) + + # Passes regex validation + user = User(email='me@example.com') + self.assertTrue(user.validate() is None) + if __name__ == '__main__': unittest.main() From 94adc207ad86883eb22d592a5844fb82e62c4b6f Mon Sep 17 00:00:00 2001 From: Jorge Bastida Date: Fri, 7 Dec 2012 11:20:27 +0000 Subject: [PATCH 0864/1279] First as_pymongo implementation --- mongoengine/queryset.py | 12 ++++++++++++ tests/test_queryset.py | 16 ++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index c774322..4a27caf 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -353,6 +353,7 @@ class QuerySet(object): self._slave_okay = False self._iter = False self._scalar = [] + self._as_pymongo = False # If inheritance is allowed, only return instances and instances of # subclasses of the class being used @@ -1002,6 +1003,10 @@ class QuerySet(object): if self._scalar: return self._get_scalar(self._document._from_son( self._cursor.next())) + + if self._as_pymongo: + return self._cursor.next() + return self._document._from_son(self._cursor.next()) except StopIteration, e: self.rewind() @@ -1602,6 +1607,13 @@ class QuerySet(object): """An alias for scalar""" return self.scalar(*fields) + def as_pymongo(self): + """Instead of returning Document instances, return raw values from + pymongo. + """ + self._as_pymongo = True + return self + def _sub_js_fields(self, code): """When fields are specified with [~fieldname] syntax, where *fieldname* is the Python name of a field, *fieldname* will be diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 690df5e..1920d2f 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -3691,6 +3691,22 @@ class QueryFieldListTest(unittest.TestCase): ak = list(Bar.objects(foo__match={'shape': "square", "color": "purple"})) self.assertEqual([b1], ak) + def test_as_pymongo(self): + + class User(Document): + id = ObjectIdField('_id') + name = StringField() + age = IntField() + + User.drop_collection() + User(name="Bob Dole", age=89).save() + User(name="Barack Obama", age=51).save() + + users = [u for u in User.objects.only('name').as_pymongo()] + self.assertTrue(isinstance(users[0], dict)) + self.assertTrue(isinstance(users[1], dict)) + self.assertEqual(users[0]['name'], 'Bob Dole') + self.assertEqual(users[1]['name'], 'Barack Obama') if __name__ == '__main__': unittest.main() From bb15bf8d1349fdcef597f35517ed92939eb559e2 Mon Sep 17 00:00:00 2001 From: Adrian Scott Date: Fri, 7 Dec 2012 10:02:12 -0500 Subject: [PATCH 0865/1279] Update AUTHORS added me (Adrian Scott, issues 180, 181) --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 6ba2f88..80ea613 100644 --- a/AUTHORS +++ b/AUTHORS @@ -124,3 +124,4 @@ that much better: * Stefan Wójcik * dimonb * Garry Polley + * Adrian Scott From ad983dc279c25b0305dcd32f07cb380a87609f3e Mon Sep 17 00:00:00 2001 From: Jorge Bastida Date: Fri, 7 Dec 2012 15:42:10 +0000 Subject: [PATCH 0866/1279] Implement _get_as_pymongo --- mongoengine/queryset.py | 51 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 4a27caf..be60571 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -354,6 +354,7 @@ class QuerySet(object): self._iter = False self._scalar = [] self._as_pymongo = False + self._as_pymongo_coerce = False # If inheritance is allowed, only return instances and instances of # subclasses of the class being used @@ -1003,9 +1004,8 @@ class QuerySet(object): if self._scalar: return self._get_scalar(self._document._from_son( self._cursor.next())) - if self._as_pymongo: - return self._cursor.next() + return self._get_as_pymongo(self._cursor.next()) return self._document._from_son(self._cursor.next()) except StopIteration, e: @@ -1585,6 +1585,48 @@ class QuerySet(object): return tuple(data) + def _get_as_pymongo(self, row): + # Extract which fields paths we should follow if .fields(...) was + # used. If not, handle all fields. + if not getattr(self, '__as_pymongo_fields', None): + self.__as_pymongo_fields = [] + for field in self._loaded_fields.fields - set(['_cls', '_id', '_types']): + self.__as_pymongo_fields.append(field) + while '.' in field: + field, _ = field.rsplit('.', 1) + self.__as_pymongo_fields.append(field) + + all_fields = not self.__as_pymongo_fields + + def clean(data, path=None): + path = path or '' + + if isinstance(data, dict): + new_data = {} + for key, value in data.iteritems(): + new_path = '%s.%s' % (path, key) if path else key + if all_fields or new_path in self.__as_pymongo_fields: + new_data[key] = clean(value, path=new_path) + data = new_data + elif isinstance(data, list): + data = [clean(d, path=path) for d in data] + else: + if self._as_pymongo_coerce: + # If we need to coerce types, we need to determine the + # type of this field and use the corresponding .to_python(...) + from mongoengine.fields import EmbeddedDocumentField + obj = self._document + for chunk in path.split('.'): + obj = getattr(obj, chunk, None) + if obj is None: + break + elif isinstance(obj, EmbeddedDocumentField): + obj = obj.document_type + if obj and data is not None: + data = obj.to_python(data) + return data + return clean(row) + def scalar(self, *fields): """Instead of returning Document instances, return either a specific value or a tuple of values in order. @@ -1607,11 +1649,14 @@ class QuerySet(object): """An alias for scalar""" return self.scalar(*fields) - def as_pymongo(self): + def as_pymongo(self, coerce_types=False): """Instead of returning Document instances, return raw values from pymongo. + + :param coerce_type: Field types (if applicable) would be use to coerce types. """ self._as_pymongo = True + self._as_pymongo_coerce = coerce_types return self def _sub_js_fields(self, code): From d5ec3c6a31f0409731f11a3e2fefc4b9b7184a4c Mon Sep 17 00:00:00 2001 From: Jorge Bastida Date: Fri, 7 Dec 2012 15:59:09 +0000 Subject: [PATCH 0867/1279] Add as_pymongo to __getitem__ and in_bulk --- mongoengine/queryset.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index be60571..d12f3bb 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -988,6 +988,9 @@ class QuerySet(object): for doc in docs: doc_map[doc['_id']] = self._get_scalar( self._document._from_son(doc)) + elif self._as_pymongo: + for doc in docs: + doc_map[doc['_id']] = self._get_as_pymongo(doc) else: for doc in docs: doc_map[doc['_id']] = self._document._from_son(doc) @@ -1189,6 +1192,8 @@ class QuerySet(object): if self._scalar: return self._get_scalar(self._document._from_son( self._cursor[key])) + if self._as_pymongo: + return self._get_as_pymongo(self._cursor.next()) return self._document._from_son(self._cursor[key]) raise AttributeError From e62c35b040b9bcc3b9d99e6ba87b1fa34b5a5af1 Mon Sep 17 00:00:00 2001 From: Jorge Bastida Date: Fri, 7 Dec 2012 16:21:31 +0000 Subject: [PATCH 0868/1279] Add more tests --- tests/test_queryset.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 1920d2f..09a4823 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -3693,20 +3693,36 @@ class QueryFieldListTest(unittest.TestCase): def test_as_pymongo(self): + from decimal import Decimal + class User(Document): id = ObjectIdField('_id') name = StringField() age = IntField() + price = DecimalField() User.drop_collection() - User(name="Bob Dole", age=89).save() - User(name="Barack Obama", age=51).save() + User(name="Bob Dole", age=89, price=Decimal('1.11')).save() + User(name="Barack Obama", age=51, price=Decimal('2.22')).save() - users = [u for u in User.objects.only('name').as_pymongo()] - self.assertTrue(isinstance(users[0], dict)) - self.assertTrue(isinstance(users[1], dict)) - self.assertEqual(users[0]['name'], 'Bob Dole') - self.assertEqual(users[1]['name'], 'Barack Obama') + users = User.objects.only('name', 'price').as_pymongo() + results = list(users) + self.assertTrue(isinstance(results[0], dict)) + self.assertTrue(isinstance(results[1], dict)) + self.assertEqual(results[0]['name'], 'Bob Dole') + self.assertEqual(results[0]['price'], '1.11') + self.assertEqual(results[1]['name'], 'Barack Obama') + self.assertEqual(results[1]['price'], '2.22') + + # Test coerce_types + users = User.objects.only('name', 'price').as_pymongo(coerce_types=True) + results = list(users) + self.assertTrue(isinstance(results[0], dict)) + self.assertTrue(isinstance(results[1], dict)) + self.assertEqual(results[0]['name'], 'Bob Dole') + self.assertEqual(results[0]['price'], Decimal('1.11')) + self.assertEqual(results[1]['name'], 'Barack Obama') + self.assertEqual(results[1]['price'], Decimal('2.22')) if __name__ == '__main__': unittest.main() From 452cd125faf161feff10f79fe080405074d21650 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 10 Dec 2012 08:11:35 +0000 Subject: [PATCH 0869/1279] Updated Changelog --- docs/changelog.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7457eeb..355449f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,11 @@ Changelog ========= + +Changes in 0.7.8 +================ +- Added as_pymongo method to return raw or cast results from pymongo (MongoEngine/mongoengine#193) + Changes in 0.7.7 ================ - Fix handling for old style _types From 6997e0247665605ba872f82254db11328739829d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 10 Dec 2012 08:23:41 +0000 Subject: [PATCH 0870/1279] Fixed EmailField so can add extra validation (MongoEngine/mongoengine#173, MongoEngine/mongoengine#174, MongoEngine/mongoengine#187) --- AUTHORS | 2 +- docs/changelog.rst | 2 ++ mongoengine/queryset.py | 2 +- tests/test_queryset.py | 4 ++++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 6ba2f88..5e6f9e7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -106,7 +106,7 @@ that much better: * Adam Reeve * Anthony Nemitz * deignacio - * shaunduncan + * Shaun Duncan * Meir Kriheli * Andrey Fedoseev * aparajita diff --git a/docs/changelog.rst b/docs/changelog.rst index 355449f..3dff3bd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,8 @@ Changelog Changes in 0.7.8 ================ +- Fixed EmailField so can add extra validation (MongoEngine/mongoengine#173, MongoEngine/mongoengine#174, MongoEngine/mongoengine#187) +- Fixed bulk inserts can now handle custom pk's (MongoEngine/mongoengine#192) - Added as_pymongo method to return raw or cast results from pymongo (MongoEngine/mongoengine#193) Changes in 0.7.7 diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index d12f3bb..f0609ff 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -929,7 +929,7 @@ class QuerySet(object): if not isinstance(doc, self._document): msg = "Some documents inserted aren't instances of %s" % str(self._document) raise OperationError(msg) - if doc.pk: + if doc.pk and not doc._created: msg = "Some documents have ObjectIds use doc.update() instead" raise OperationError(msg) raw.append(doc.to_mongo()) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 09a4823..aed606d 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -591,6 +591,10 @@ class QuerySetTest(unittest.TestCase): self.assertRaises(OperationError, throw_operation_error) + # Test can insert new doc + new_post = Blog(title="code", id=ObjectId()) + Blog.objects.insert(new_post) + # test handles other classes being inserted def throw_operation_error_wrong_doc(): class Author(Document): From 260d9377f537590e3d0c1cfe0dfe08dff5401784 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 10 Dec 2012 08:26:42 +0000 Subject: [PATCH 0871/1279] Updated Changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3dff3bd..5fcf619 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.7.8 ================ +- Added optional encoding and collection config for Django sessions (MongoEngine/mongoengine#180, MongoEngine/mongoengine#181, MongoEngine/mongoengine#183) - Fixed EmailField so can add extra validation (MongoEngine/mongoengine#173, MongoEngine/mongoengine#174, MongoEngine/mongoengine#187) - Fixed bulk inserts can now handle custom pk's (MongoEngine/mongoengine#192) - Added as_pymongo method to return raw or cast results from pymongo (MongoEngine/mongoengine#193) From 90d22c2a28dc0946457565ca250b443c89fbe220 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 10 Dec 2012 08:50:21 +0000 Subject: [PATCH 0872/1279] Update AUTHORS & Changelog (MongoEngine/mongoengine#176) --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 1c57463..330256c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -125,3 +125,4 @@ that much better: * dimonb * Garry Polley * Adrian Scott + * Peter Teichman diff --git a/docs/changelog.rst b/docs/changelog.rst index 5fcf619..c9535a0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.7.8 ================ +- Fix query chaining with .order_by() (MongoEngine/mongoengine#176) - Added optional encoding and collection config for Django sessions (MongoEngine/mongoengine#180, MongoEngine/mongoengine#181, MongoEngine/mongoengine#183) - Fixed EmailField so can add extra validation (MongoEngine/mongoengine#173, MongoEngine/mongoengine#174, MongoEngine/mongoengine#187) - Fixed bulk inserts can now handle custom pk's (MongoEngine/mongoengine#192) From 9236f365fa95b5006fe50702c29dd118d888f2e5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 10 Dec 2012 09:11:31 +0000 Subject: [PATCH 0873/1279] Fix sequence fields in embedded documents (MongoEngine/mongoengine#166) --- docs/changelog.rst | 1 + mongoengine/fields.py | 13 +++++++++++-- tests/test_fields.py | 22 ++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c9535a0..92770af 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.7.8 ================ +- Fix sequence fields in embedded documents (MongoEngine/mongoengine#166) - Fix query chaining with .order_by() (MongoEngine/mongoengine#176) - Added optional encoding and collection config for Django sessions (MongoEngine/mongoengine#180, MongoEngine/mongoengine#181, MongoEngine/mongoengine#183) - Fixed EmailField so can add extra validation (MongoEngine/mongoengine#173, MongoEngine/mongoengine#174, MongoEngine/mongoengine#187) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 9c0bede..3f413b2 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1338,7 +1338,7 @@ class SequenceField(IntField): .. versionadded:: 0.5 """ - def __init__(self, collection_name=None, db_alias = None, sequence_name = None, *args, **kwargs): + def __init__(self, collection_name=None, db_alias=None, sequence_name=None, *args, **kwargs): self.collection_name = collection_name or 'mongoengine.counters' self.db_alias = db_alias or DEFAULT_CONNECTION_NAME self.sequence_name = sequence_name @@ -1348,7 +1348,7 @@ class SequenceField(IntField): """ Generate and Increment the counter """ - sequence_name = self.sequence_name or self.owner_document._get_collection_name() + sequence_name = self.get_sequence_name() sequence_id = "%s.%s" % (sequence_name, self.name) collection = get_db(alias=self.db_alias)[self.collection_name] counter = collection.find_and_modify(query={"_id": sequence_id}, @@ -1357,6 +1357,15 @@ class SequenceField(IntField): upsert=True) return counter['next'] + def get_sequence_name(self): + if self.sequence_name: + return self.sequence_name + owner = self.owner_document + if issubclass(owner, Document): + return owner._get_collection_name() + else: + return owner._class_name + def __get__(self, instance, owner): if instance is None: diff --git a/tests/test_fields.py b/tests/test_fields.py index 88e82f1..fdcc308 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -2175,6 +2175,28 @@ class FieldTest(unittest.TestCase): c = self.db['mongoengine.counters'].find_one({'_id': 'animal.id'}) self.assertEqual(c['next'], 10) + def test_embedded_sequence_field(self): + class Comment(EmbeddedDocument): + id = SequenceField() + content = StringField(required=True) + + class Post(Document): + title = StringField(required=True) + comments = ListField(EmbeddedDocumentField(Comment)) + + self.db['mongoengine.counters'].drop() + Post.drop_collection() + + Post(title="MongoEngine", + comments=[Comment(content="NoSQL Rocks"), + Comment(content="MongoEngine Rocks")]).save() + + c = self.db['mongoengine.counters'].find_one({'_id': 'Comment.id'}) + self.assertEqual(c['next'], 2) + post = Post.objects.first() + self.assertEqual(1, post.comments[0].id) + self.assertEqual(2, post.comments[1].id) + def test_generic_embedded_document(self): class Car(EmbeddedDocument): name = StringField() From 1bc2d2ec37ffe608ad56d7e790d087e6dfeda1bc Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 10 Dec 2012 09:54:20 +0000 Subject: [PATCH 0874/1279] Version Bump --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 9f1f552..08cabaa 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 7, 7) +VERSION = (0, 7, 8) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 9a376ec..f175546 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.7.7 +Version: 0.7.8 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 6ff1bd9b3cb623bc6c864e0856a7ec37b427fd65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johnny=20Bergstr=C3=B6m?= Date: Mon, 10 Dec 2012 11:01:08 +0100 Subject: [PATCH 0875/1279] Corrected user guide link in README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index ae6bd0e..5eab502 100644 --- a/README.rst +++ b/README.rst @@ -14,7 +14,7 @@ About MongoEngine is a Python Object-Document Mapper for working with MongoDB. Documentation available at http://mongoengine-odm.rtfd.org - there is currently a `tutorial `_, a `user guide -`_ and an `API reference +`_ and an `API reference `_. Installation From b15c3f6a3f1f54023cb7ca9855db938373f012c2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 10 Dec 2012 10:33:55 +0000 Subject: [PATCH 0876/1279] Update AUTHORS Sorry Jorge! --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 864c95c..82a1dfa 100644 --- a/AUTHORS +++ b/AUTHORS @@ -127,3 +127,4 @@ that much better: * Adrian Scott * Peter Teichman * Jakub Kot + * Jorge Bastida From 3b3738b36ba007460a9085472c5013cabc664956 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 10 Dec 2012 15:16:31 +0000 Subject: [PATCH 0877/1279] 0.7.9 --- docs/changelog.rst | 4 ++++ mongoengine/__init__.py | 2 +- mongoengine/base.py | 8 +++++--- mongoengine/fields.py | 3 ++- python-mongoengine.spec | 2 +- tests/test_document.py | 16 ++++++++++++++-- tests/test_fields.py | 2 +- 7 files changed, 28 insertions(+), 9 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 92770af..d93bf13 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.7.9 +================ +- Better fix handling for old style _types +- Embedded SequenceFields follow collection naming convention Changes in 0.7.8 ================ diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 08cabaa..b67512d 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 7, 8) +VERSION = (0, 7, 9) def get_version(): diff --git a/mongoengine/base.py b/mongoengine/base.py index 208e0e5..013afe7 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -122,9 +122,11 @@ def get_document(name): doc = _document_registry.get(name, None) if not doc: # Possible old style name - end = name.split('.')[-1] - possible_match = [k for k in _document_registry.keys() if k == end] - if len(possible_match) == 1 and end != name: + single_end = name.split('.')[-1] + compound_end = '.%s' % single_end + possible_match = [k for k in _document_registry.keys() + if k.endswith(compound_end) or k == single_end] + if len(possible_match) == 1: doc = _document_registry.get(possible_match.pop(), None) if not doc: raise NotRegistered(""" diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 213c214..de484a1 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1364,7 +1364,8 @@ class SequenceField(IntField): if issubclass(owner, Document): return owner._get_collection_name() else: - return owner._class_name + return ''.join('_%s' % c if c.isupper() else c + for c in owner._class_name).strip('_').lower() def __get__(self, instance, owner): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index f175546..b1ec336 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.7.8 +Version: 0.7.9 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB diff --git a/tests/test_document.py b/tests/test_document.py index a09aaec..cd0ab8f 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -15,7 +15,7 @@ from datetime import datetime from tests.fixtures import Base, Mixin, PickleEmbedded, PickleTest from mongoengine import * -from mongoengine.base import NotRegistered, InvalidDocumentError +from mongoengine.base import NotRegistered, InvalidDocumentError, get_document from mongoengine.queryset import InvalidQueryError from mongoengine.connection import get_db, get_connection @@ -1336,7 +1336,6 @@ class DocumentTest(unittest.TestCase): User.drop_collection() - def test_document_not_registered(self): class Place(Document): @@ -1361,6 +1360,19 @@ class DocumentTest(unittest.TestCase): print Place.objects.all() self.assertRaises(NotRegistered, query_without_importing_nice_place) + def test_document_registry_regressions(self): + + class Location(Document): + name = StringField() + meta = {'allow_inheritance': True} + + class Area(Location): + location = ReferenceField('Location', dbref=True) + + Location.drop_collection() + + self.assertEquals(Area, get_document("Area")) + self.assertEquals(Area, get_document("Location.Area")) def test_creation(self): """Ensure that document may be created using keyword arguments. diff --git a/tests/test_fields.py b/tests/test_fields.py index 0483519..28af1b2 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -2201,7 +2201,7 @@ class FieldTest(unittest.TestCase): comments=[Comment(content="NoSQL Rocks"), Comment(content="MongoEngine Rocks")]).save() - c = self.db['mongoengine.counters'].find_one({'_id': 'Comment.id'}) + c = self.db['mongoengine.counters'].find_one({'_id': 'comment.id'}) self.assertEqual(c['next'], 2) post = Post.objects.first() self.assertEqual(1, post.comments[0].id) From 148f8b8a3a2f53b2e086b48b2294b537224c4d33 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Mon, 17 Dec 2012 21:13:45 -0800 Subject: [PATCH 0878/1279] Only allow QNode instances to be passed as query objects --- mongoengine/queryset.py | 3 +++ tests/test_queryset.py | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 5a1aa71..e987da9 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -423,6 +423,9 @@ class QuerySet(object): """ query = Q(**query) if q_obj: + # make sure proper query object is passed + if not isinstance(q_obj, QNode): + raise InvalidQueryError('Not a query object: %s. Did you intend to use key=value?' % q_obj) query &= q_obj self._query_obj &= query self._mongo_query = None diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 55531a1..c49566c 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -1289,6 +1289,14 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(len(self.Person.objects(Q(age__in=[20]))), 2) self.assertEqual(len(self.Person.objects(Q(age__in=[20, 30]))), 3) + # Test invalid query objs + def wrong_query_objs(): + self.Person.objects('user1') + def wrong_query_objs_filter(): + self.Person.objects('user1') + self.assertRaises(InvalidQueryError, wrong_query_objs) + self.assertRaises(InvalidQueryError, wrong_query_objs_filter) + def test_q_regex(self): """Ensure that Q objects can be queried using regexes. """ From 3e8f02c64babd024c39a913a946f3b6e3dc6fd75 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 11:39:19 +0000 Subject: [PATCH 0879/1279] Merge sequence field changes --- mongoengine/fields.py | 3 +-- tests/test_fields.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 65996a4..fe94d35 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1368,8 +1368,7 @@ class SequenceField(BaseField): """ Generate and Increment the counter """ - sequence_name = (self.sequence_name or - self.owner_document._get_collection_name()) + sequence_name = self.get_sequence_name() sequence_id = "%s.%s" % (sequence_name, self.name) collection = get_db(alias=self.db_alias)[self.collection_name] counter = collection.find_and_modify(query={"_id": sequence_id}, diff --git a/tests/test_fields.py b/tests/test_fields.py index 6e3dc8b..97a2d5f 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -2241,7 +2241,6 @@ class FieldTest(unittest.TestCase): Post(title="MongoEngine", comments=[Comment(content="NoSQL Rocks"), Comment(content="MongoEngine Rocks")]).save() - import ipdb; ipdb.set_trace(); c = self.db['mongoengine.counters'].find_one({'_id': 'comment.id'}) self.assertEqual(c['next'], 2) post = Post.objects.first() From 1a131ff1207cc116e5b90bcf3f70268dd6f0061f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 12:16:12 +0000 Subject: [PATCH 0880/1279] Only allow QNode instances to be passed as query objects (MongoEngine/mongoengine#199) --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 23afe6f..2519c89 100644 --- a/AUTHORS +++ b/AUTHORS @@ -129,3 +129,4 @@ that much better: * Peter Teichman * Jakub Kot * Jorge Bastida + * Stefan Wójcik \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 109940e..845f3a9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -19,6 +19,7 @@ Changes in 0.8 - Added _instance to EmbeddedDocuments pointing to the parent (MongoEngine/mongoengine#139) - Inheritance is off by default (MongoEngine/mongoengine#122) - Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) +- Only allow QNode instances to be passed as query objects (MongoEngine/mongoengine#199) Changes in 0.7.9 ================ From c528ac09d6ab25fdb2bf7972d51da42995a464e2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 12:29:46 +0000 Subject: [PATCH 0881/1279] Fix merge for QNode checks --- mongoengine/queryset/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 24836eb..fda8a75 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -17,7 +17,7 @@ from mongoengine.errors import (OperationError, NotUniqueError, from . import transform from .field_list import QueryFieldList -from .visitor import Q +from .visitor import Q, QNode __all__ = ('QuerySet', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL') From 9cc02d4dbe27b6a8b12f958b4fae58573264dde0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 12:32:06 +0000 Subject: [PATCH 0882/1279] Dynamic fields are now validated on save (MongoEngine/mongoengine#153) (MongoEngine/mongoengine#154) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + mongoengine/base/document.py | 3 +++ mongoengine/fields.py | 4 ++++ tests/document/dynamic.py | 23 +++++++++++++++++++++++ 5 files changed, 33 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 2519c89..794f297 100644 --- a/AUTHORS +++ b/AUTHORS @@ -129,4 +129,5 @@ that much better: * Peter Teichman * Jakub Kot * Jorge Bastida - * Stefan Wójcik \ No newline at end of file + * Stefan Wójcik + * Pete Campton \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 845f3a9..fa0fe10 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -20,6 +20,7 @@ Changes in 0.8 - Inheritance is off by default (MongoEngine/mongoengine#122) - Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) - Only allow QNode instances to be passed as query objects (MongoEngine/mongoengine#199) +- Dynamic fields are now validated on save (MongoEngine/mongoengine#153) (MongoEngine/mongoengine#154) Changes in 0.7.9 ================ diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 2dd4b03..a3f10f5 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -245,6 +245,9 @@ class BaseDocument(object): # Get a list of tuples of field names and their current values fields = [(field, self._data.get(name)) for name, field in self._fields.items()] + if self._dynamic: + fields += [(field, self._data.get(name)) + for name, field in self._dynamic_fields.items()] EmbeddedDocumentField = _import_class("EmbeddedDocumentField") GenericEmbeddedDocumentField = _import_class("GenericEmbeddedDocumentField") diff --git a/mongoengine/fields.py b/mongoengine/fields.py index fe94d35..73c0db4 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -564,6 +564,10 @@ class DynamicField(BaseField): return StringField().prepare_query_value(op, value) return self.to_mongo(value) + def validate(self, value, clean=True): + if hasattr(value, "validate"): + value.validate(clean=clean) + class ListField(ComplexBaseField): """A list field that wraps a standard field, allowing multiple instances diff --git a/tests/document/dynamic.py b/tests/document/dynamic.py index d879b54..ca0db0a 100644 --- a/tests/document/dynamic.py +++ b/tests/document/dynamic.py @@ -122,6 +122,29 @@ class DynamicTest(unittest.TestCase): self.assertEqual(1, self.Person.objects(misc__hello='world').count()) + def test_complex_embedded_document_validation(self): + """Ensure embedded dynamic documents may be validated""" + class Embedded(DynamicEmbeddedDocument): + content = URLField() + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + + embedded_doc_1 = Embedded(content='http://mongoengine.org') + embedded_doc_1.validate() + + embedded_doc_2 = Embedded(content='this is not a url') + with self.assertRaises(ValidationError): + embedded_doc_2.validate() + + doc.embedded_field_1 = embedded_doc_1 + doc.embedded_field_2 = embedded_doc_2 + with self.assertRaises(ValidationError): + doc.validate() + def test_inheritance(self): """Ensure that dynamic document plays nice with inheritance""" class Employee(self.Person): From 7f732459a130d2c0a870c1803c9694002f325e17 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 12:34:02 +0000 Subject: [PATCH 0883/1279] Updated tickets links as now default to MongoEngine/mongoengine --- docs/changelog.rst | 104 ++++++++++++++++++++++----------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index fa0fe10..8fc279e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,23 +4,23 @@ Changelog Changes in 0.8 ============== -- Fixed db_alias and inherited Documents (MongoEngine/mongoengine#143) -- Documentation update for document errors (MongoEngine/mongoengine#124) -- Deprecated `get_or_create` (MongoEngine/mongoengine#35) -- Updated inheritable objects created by upsert now contain _cls (MongoEngine/mongoengine#118) -- Added support for creating documents with embedded documents in a single operation (MongoEngine/mongoengine#6) -- Added to_json and from_json to Document (MongoEngine/mongoengine#1) -- Added to_json and from_json to QuerySet (MongoEngine/mongoengine#131) -- Updated index creation now tied to Document class (MongoEngine/mongoengine#102) -- Added none() to queryset (MongoEngine/mongoengine#127) -- Updated SequenceFields to allow post processing of the calculated counter value (MongoEngine/mongoengine#141) -- Added clean method to documents for pre validation data cleaning (MongoEngine/mongoengine#60) -- Added support setting for read prefrence at a query level (MongoEngine/mongoengine#157) -- Added _instance to EmbeddedDocuments pointing to the parent (MongoEngine/mongoengine#139) -- Inheritance is off by default (MongoEngine/mongoengine#122) -- Remove _types and just use _cls for inheritance (MongoEngine/mongoengine#148) -- Only allow QNode instances to be passed as query objects (MongoEngine/mongoengine#199) -- Dynamic fields are now validated on save (MongoEngine/mongoengine#153) (MongoEngine/mongoengine#154) +- Fixed db_alias and inherited Documents (#143) +- Documentation update for document errors (#124) +- Deprecated `get_or_create` (#35) +- Updated inheritable objects created by upsert now contain _cls (#118) +- Added support for creating documents with embedded documents in a single operation (#6) +- Added to_json and from_json to Document (#1) +- Added to_json and from_json to QuerySet (#131) +- Updated index creation now tied to Document class (#102) +- Added none() to queryset (#127) +- Updated SequenceFields to allow post processing of the calculated counter value (#141) +- Added clean method to documents for pre validation data cleaning (#60) +- Added support setting for read prefrence at a query level (#157) +- Added _instance to EmbeddedDocuments pointing to the parent (#139) +- Inheritance is off by default (#122) +- Remove _types and just use _cls for inheritance (#148) +- Only allow QNode instances to be passed as query objects (#199) +- Dynamic fields are now validated on save (#153) (#154) Changes in 0.7.9 ================ @@ -29,12 +29,12 @@ Changes in 0.7.9 Changes in 0.7.8 ================ -- Fix sequence fields in embedded documents (MongoEngine/mongoengine#166) -- Fix query chaining with .order_by() (MongoEngine/mongoengine#176) -- Added optional encoding and collection config for Django sessions (MongoEngine/mongoengine#180, MongoEngine/mongoengine#181, MongoEngine/mongoengine#183) -- Fixed EmailField so can add extra validation (MongoEngine/mongoengine#173, MongoEngine/mongoengine#174, MongoEngine/mongoengine#187) -- Fixed bulk inserts can now handle custom pk's (MongoEngine/mongoengine#192) -- Added as_pymongo method to return raw or cast results from pymongo (MongoEngine/mongoengine#193) +- Fix sequence fields in embedded documents (#166) +- Fix query chaining with .order_by() (#176) +- Added optional encoding and collection config for Django sessions (#180, #181, #183) +- Fixed EmailField so can add extra validation (#173, #174, #187) +- Fixed bulk inserts can now handle custom pk's (#192) +- Added as_pymongo method to return raw or cast results from pymongo (#193) Changes in 0.7.7 ================ @@ -42,70 +42,70 @@ Changes in 0.7.7 Changes in 0.7.6 ================ -- Unicode fix for repr (MongoEngine/mongoengine#133) -- Allow updates with match operators (MongoEngine/mongoengine#144) -- Updated URLField - now can have a override the regex (MongoEngine/mongoengine#136) +- Unicode fix for repr (#133) +- Allow updates with match operators (#144) +- Updated URLField - now can have a override the regex (#136) - Allow Django AuthenticationBackends to work with Django user (hmarr/mongoengine#573) -- Fixed reload issue with ReferenceField where dbref=False (MongoEngine/mongoengine#138) +- Fixed reload issue with ReferenceField where dbref=False (#138) Changes in 0.7.5 ================ -- ReferenceFields with dbref=False use ObjectId instead of strings (MongoEngine/mongoengine#134) - See ticket for upgrade notes (https://github.com/MongoEngine/mongoengine/issues/134) +- ReferenceFields with dbref=False use ObjectId instead of strings (#134) + See ticket for upgrade notes (#134) Changes in 0.7.4 ================ -- Fixed index inheritance issues - firmed up testcases (MongoEngine/mongoengine#123) (MongoEngine/mongoengine#125) +- Fixed index inheritance issues - firmed up testcases (#123) (#125) Changes in 0.7.3 ================ -- Reverted EmbeddedDocuments meta handling - now can turn off inheritance (MongoEngine/mongoengine#119) +- Reverted EmbeddedDocuments meta handling - now can turn off inheritance (#119) Changes in 0.7.2 ================ -- Update index spec generation so its not destructive (MongoEngine/mongoengine#113) +- Update index spec generation so its not destructive (#113) Changes in 0.7.1 ================= -- Fixed index spec inheritance (MongoEngine/mongoengine#111) +- Fixed index spec inheritance (#111) Changes in 0.7.0 ================= -- Updated queryset.delete so you can use with skip / limit (MongoEngine/mongoengine#107) -- Updated index creation allows kwargs to be passed through refs (MongoEngine/mongoengine#104) -- Fixed Q object merge edge case (MongoEngine/mongoengine#109) +- Updated queryset.delete so you can use with skip / limit (#107) +- Updated index creation allows kwargs to be passed through refs (#104) +- Fixed Q object merge edge case (#109) - Fixed reloading on sharded documents (hmarr/mongoengine#569) -- Added NotUniqueError for duplicate keys (MongoEngine/mongoengine#62) -- Added custom collection / sequence naming for SequenceFields (MongoEngine/mongoengine#92) -- Fixed UnboundLocalError in composite index with pk field (MongoEngine/mongoengine#88) +- Added NotUniqueError for duplicate keys (#62) +- Added custom collection / sequence naming for SequenceFields (#92) +- Fixed UnboundLocalError in composite index with pk field (#88) - Updated ReferenceField's to optionally store ObjectId strings - this will become the default in 0.8 (MongoEngine/mongoengine#89) + this will become the default in 0.8 (#89) - Added FutureWarning - save will default to `cascade=False` in 0.8 -- Added example of indexing embedded document fields (MongoEngine/mongoengine#75) -- Fixed ImageField resizing when forcing size (MongoEngine/mongoengine#80) -- Add flexibility for fields handling bad data (MongoEngine/mongoengine#78) +- Added example of indexing embedded document fields (#75) +- Fixed ImageField resizing when forcing size (#80) +- Add flexibility for fields handling bad data (#78) - Embedded Documents no longer handle meta definitions -- Use weakref proxies in base lists / dicts (MongoEngine/mongoengine#74) +- Use weakref proxies in base lists / dicts (#74) - Improved queryset filtering (hmarr/mongoengine#554) - Fixed Dynamic Documents and Embedded Documents (hmarr/mongoengine#561) -- Fixed abstract classes and shard keys (MongoEngine/mongoengine#64) +- Fixed abstract classes and shard keys (#64) - Fixed Python 2.5 support - Added Python 3 support (thanks to Laine Heron) Changes in 0.6.20 ================= -- Added support for distinct and db_alias (MongoEngine/mongoengine#59) +- Added support for distinct and db_alias (#59) - Improved support for chained querysets when constraining the same fields (hmarr/mongoengine#554) -- Fixed BinaryField lookup re (MongoEngine/mongoengine#48) +- Fixed BinaryField lookup re (#48) Changes in 0.6.19 ================= -- Added Binary support to UUID (MongoEngine/mongoengine#47) -- Fixed MapField lookup for fields without declared lookups (MongoEngine/mongoengine#46) -- Fixed BinaryField python value issue (MongoEngine/mongoengine#48) -- Fixed SequenceField non numeric value lookup (MongoEngine/mongoengine#41) -- Fixed queryset manager issue (MongoEngine/mongoengine#52) +- Added Binary support to UUID (#47) +- Fixed MapField lookup for fields without declared lookups (#46) +- Fixed BinaryField python value issue (#48) +- Fixed SequenceField non numeric value lookup (#41) +- Fixed queryset manager issue (#52) - Fixed FileField comparision (hmarr/mongoengine#547) Changes in 0.6.18 From 25cdf16cc0ed045d6f69c6495d37dfe51b86c563 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 12:37:37 +0000 Subject: [PATCH 0884/1279] Updated travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1aa9774..c7cc271 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,8 @@ python: - 3.2 env: - PYMONGO=dev + - PYMONGO=2.4.1 - PYMONGO=2.3 - - PYMONGO=2.2 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install zlib1g zlib1g-dev; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo ln -s /usr/lib/i386-linux-gnu/libz.so /usr/lib/; fi From 420c3e0073c9a055b4db062863352b93a4b2911f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 12:51:42 +0000 Subject: [PATCH 0885/1279] Fixing for python2.5 closes #188 --- mongoengine/base/__init__.py | 10 +++++----- mongoengine/base/document.py | 6 +++--- mongoengine/base/fields.py | 4 ++-- mongoengine/base/metaclasses.py | 4 ++-- mongoengine/errors.py | 2 +- mongoengine/queryset/__init__.py | 10 +++++----- mongoengine/queryset/manager.py | 2 +- mongoengine/queryset/queryset.py | 8 +++++--- tests/__init__.py | 4 ++-- tests/document/dynamic.py | 6 ++---- 10 files changed, 28 insertions(+), 28 deletions(-) diff --git a/mongoengine/base/__init__.py b/mongoengine/base/__init__.py index 1d4a6eb..ce119b3 100644 --- a/mongoengine/base/__init__.py +++ b/mongoengine/base/__init__.py @@ -1,5 +1,5 @@ -from .common import * -from .datastructures import * -from .document import * -from .fields import * -from .metaclasses import * +from mongoengine.base.common import * +from mongoengine.base.datastructures import * +from mongoengine.base.document import * +from mongoengine.base.fields import * +from mongoengine.base.metaclasses import * diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index a3f10f5..affc20e 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -12,9 +12,9 @@ from mongoengine.errors import (ValidationError, InvalidDocumentError, from mongoengine.python_support import (PY3, UNICODE_KWARGS, txt_type, to_str_keys_recursive) -from .common import get_document, ALLOW_INHERITANCE -from .datastructures import BaseDict, BaseList -from .fields import ComplexBaseField +from mongoengine.base.common import get_document, ALLOW_INHERITANCE +from mongoengine.base.datastructures import BaseDict, BaseList +from mongoengine.base.fields import ComplexBaseField __all__ = ('BaseDocument', 'NON_FIELD_ERRORS') diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 11719b5..a892fbd 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -7,8 +7,8 @@ from bson import DBRef, ObjectId from mongoengine.common import _import_class from mongoengine.errors import ValidationError -from .common import ALLOW_INHERITANCE -from .datastructures import BaseDict, BaseList +from mongoengine.base.common import ALLOW_INHERITANCE +from mongoengine.base.datastructures import BaseDict, BaseList __all__ = ("BaseField", "ComplexBaseField", "ObjectIdField") diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index c6c4db1..af39e14 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -9,8 +9,8 @@ from mongoengine.queryset import (DO_NOTHING, DoesNotExist, MultipleObjectsReturned, QuerySet, QuerySetManager) -from .common import _document_registry, ALLOW_INHERITANCE -from .fields import BaseField, ComplexBaseField, ObjectIdField +from mongoengine.base.common import _document_registry, ALLOW_INHERITANCE +from mongoengine.base.fields import BaseField, ComplexBaseField, ObjectIdField __all__ = ('DocumentMetaclass', 'TopLevelDocumentMetaclass') diff --git a/mongoengine/errors.py b/mongoengine/errors.py index eb72503..9cfcd1d 100644 --- a/mongoengine/errors.py +++ b/mongoengine/errors.py @@ -1,6 +1,6 @@ from collections import defaultdict -from .python_support import txt_type +from mongoengine.python_support import txt_type __all__ = ('NotRegistered', 'InvalidDocumentError', 'ValidationError') diff --git a/mongoengine/queryset/__init__.py b/mongoengine/queryset/__init__.py index f6feeab..026a7ac 100644 --- a/mongoengine/queryset/__init__.py +++ b/mongoengine/queryset/__init__.py @@ -1,11 +1,11 @@ from mongoengine.errors import (DoesNotExist, MultipleObjectsReturned, InvalidQueryError, OperationError, NotUniqueError) -from .field_list import * -from .manager import * -from .queryset import * -from .transform import * -from .visitor import * +from mongoengine.queryset.field_list import * +from mongoengine.queryset.manager import * +from mongoengine.queryset.queryset import * +from mongoengine.queryset.transform import * +from mongoengine.queryset.visitor import * __all__ = (field_list.__all__ + manager.__all__ + queryset.__all__ + transform.__all__ + visitor.__all__) diff --git a/mongoengine/queryset/manager.py b/mongoengine/queryset/manager.py index 08d4d3a..d9f9992 100644 --- a/mongoengine/queryset/manager.py +++ b/mongoengine/queryset/manager.py @@ -1,5 +1,5 @@ from functools import partial -from .queryset import QuerySet +from mongoengine.queryset.queryset import QuerySet __all__ = ('queryset_manager', 'QuerySetManager') diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index fda8a75..3ea9f23 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + import copy import itertools import operator @@ -15,9 +17,9 @@ from mongoengine.common import _import_class from mongoengine.errors import (OperationError, NotUniqueError, InvalidQueryError) -from . import transform -from .field_list import QueryFieldList -from .visitor import Q, QNode +from mongoengine.queryset import transform +from mongoengine.queryset.field_list import QueryFieldList +from mongoengine.queryset.visitor import Q, QNode __all__ = ('QuerySet', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL') diff --git a/tests/__init__.py b/tests/__init__.py index f2a43b0..ccc90f7 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,2 +1,2 @@ -from .all_warnings import AllWarnings -from .document import * \ No newline at end of file +from all_warnings import AllWarnings +from document import * \ No newline at end of file diff --git a/tests/document/dynamic.py b/tests/document/dynamic.py index ca0db0a..4848b8f 100644 --- a/tests/document/dynamic.py +++ b/tests/document/dynamic.py @@ -137,13 +137,11 @@ class DynamicTest(unittest.TestCase): embedded_doc_1.validate() embedded_doc_2 = Embedded(content='this is not a url') - with self.assertRaises(ValidationError): - embedded_doc_2.validate() + self.assertRaises(ValidationError, embedded_doc_2.validate) doc.embedded_field_1 = embedded_doc_1 doc.embedded_field_2 = embedded_doc_2 - with self.assertRaises(ValidationError): - doc.validate() + self.assertRaises(ValidationError, doc.validate) def test_inheritance(self): """Ensure that dynamic document plays nice with inheritance""" From 50b755db0c535fd96b374be88d79410ebc356838 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 13:35:37 +0000 Subject: [PATCH 0886/1279] Split out queryset tests --- tests/__init__.py | 3 +- tests/queryset/__init__.py | 5 + tests/queryset/field_list.py | 67 ++ .../queryset.py} | 571 +----------------- tests/queryset/transform.py | 148 +++++ tests/queryset/visitor.py | 310 ++++++++++ 6 files changed, 565 insertions(+), 539 deletions(-) create mode 100644 tests/queryset/__init__.py create mode 100644 tests/queryset/field_list.py rename tests/{test_queryset.py => queryset/queryset.py} (85%) create mode 100644 tests/queryset/transform.py create mode 100644 tests/queryset/visitor.py diff --git a/tests/__init__.py b/tests/__init__.py index ccc90f7..152a8ce 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,2 +1,3 @@ from all_warnings import AllWarnings -from document import * \ No newline at end of file +from document import * +from queryset import * \ No newline at end of file diff --git a/tests/queryset/__init__.py b/tests/queryset/__init__.py new file mode 100644 index 0000000..93cb8c2 --- /dev/null +++ b/tests/queryset/__init__.py @@ -0,0 +1,5 @@ + +from transform import * +from field_list import * +from queryset import * +from visitor import * \ No newline at end of file diff --git a/tests/queryset/field_list.py b/tests/queryset/field_list.py new file mode 100644 index 0000000..f3b457b --- /dev/null +++ b/tests/queryset/field_list.py @@ -0,0 +1,67 @@ +import sys +sys.path[0:0] = [""] + +import unittest + +from mongoengine import * +from mongoengine.queryset import QueryFieldList + +__all__ = ("QueryFieldListTest",) + +class QueryFieldListTest(unittest.TestCase): + + def test_empty(self): + q = QueryFieldList() + self.assertFalse(q) + + q = QueryFieldList(always_include=['_cls']) + self.assertFalse(q) + + def test_include_include(self): + q = QueryFieldList() + q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'a': True, 'b': True}) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'b': True}) + + def test_include_exclude(self): + q = QueryFieldList() + q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'a': True, 'b': True}) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.EXCLUDE) + self.assertEqual(q.as_dict(), {'a': True}) + + def test_exclude_exclude(self): + q = QueryFieldList() + q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.EXCLUDE) + self.assertEqual(q.as_dict(), {'a': False, 'b': False}) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.EXCLUDE) + self.assertEqual(q.as_dict(), {'a': False, 'b': False, 'c': False}) + + def test_exclude_include(self): + q = QueryFieldList() + q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.EXCLUDE) + self.assertEqual(q.as_dict(), {'a': False, 'b': False}) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'c': True}) + + def test_always_include(self): + q = QueryFieldList(always_include=['x', 'y']) + q += QueryFieldList(fields=['a', 'b', 'x'], value=QueryFieldList.EXCLUDE) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'c': True}) + + def test_reset(self): + q = QueryFieldList(always_include=['x', 'y']) + q += QueryFieldList(fields=['a', 'b', 'x'], value=QueryFieldList.EXCLUDE) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'c': True}) + q.reset() + self.assertFalse(q) + q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) + self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'b': True, 'c': True}) + + def test_using_a_slice(self): + q = QueryFieldList() + q += QueryFieldList(fields=['a'], value={"$slice": 5}) + self.assertEqual(q.as_dict(), {'a': {"$slice": 5}}) diff --git a/tests/test_queryset.py b/tests/queryset/queryset.py similarity index 85% rename from tests/test_queryset.py rename to tests/queryset/queryset.py index 07725d4..e3e0215 100644 --- a/tests/test_queryset.py +++ b/tests/queryset/queryset.py @@ -18,12 +18,13 @@ from mongoengine import * from mongoengine.connection import get_connection from mongoengine.python_support import PY3 from mongoengine.tests import query_counter -from mongoengine.queryset import (Q, QuerySet, QuerySetManager, +from mongoengine.queryset import (QuerySet, QuerySetManager, MultipleObjectsReturned, DoesNotExist, QueryFieldList, queryset_manager) -from mongoengine.queryset import transform from mongoengine.errors import InvalidQueryError +__all__ = ("QuerySetTest",) + class QuerySetTest(unittest.TestCase): @@ -47,22 +48,6 @@ class QuerySetTest(unittest.TestCase): self.assertTrue(isinstance(self.Person.objects._collection, pymongo.collection.Collection)) - def test_transform_query(self): - """Ensure that the _transform_query function operates correctly. - """ - self.assertEqual(transform.query(name='test', age=30), - {'name': 'test', 'age': 30}) - self.assertEqual(transform.query(age__lt=30), - {'age': {'$lt': 30}}) - self.assertEqual(transform.query(age__gt=20, age__lt=50), - {'age': {'$gt': 20, '$lt': 50}}) - self.assertEqual(transform.query(age=20, age__gt=50), - {'age': 20}) - self.assertEqual(transform.query(friend__age__gte=30), - {'friend.age': {'$gte': 30}}) - self.assertEqual(transform.query(name__exists=True), - {'name': {'$exists': True}}) - def test_cannot_perform_joins_references(self): class BlogPost(Document): @@ -264,30 +249,6 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(list(A.objects.none()), []) self.assertEqual(list(A.objects.none().all()), []) - def test_chaining(self): - class A(Document): - pass - - class B(Document): - a = ReferenceField(A) - - A.drop_collection() - B.drop_collection() - - a1 = A().save() - a2 = A().save() - - B(a=a1).save() - - # Works - q1 = B.objects.filter(a__in=[a1, a2], a=a1)._query - - # Doesn't work - q2 = B.objects.filter(a__in=[a1, a2]) - q2 = q2.filter(a=a1)._query - - self.assertEqual(q1, q2) - def test_update_write_options(self): """Test that passing write_options works""" @@ -830,48 +791,30 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(obj, person) obj = self.Person.objects(name__contains='Van').first() self.assertEqual(obj, None) - obj = self.Person.objects(Q(name__contains='van')).first() - self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__contains='Van')).first() - self.assertEqual(obj, None) # Test icontains obj = self.Person.objects(name__icontains='Van').first() self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__icontains='Van')).first() - self.assertEqual(obj, person) # Test startswith obj = self.Person.objects(name__startswith='Guido').first() self.assertEqual(obj, person) obj = self.Person.objects(name__startswith='guido').first() self.assertEqual(obj, None) - obj = self.Person.objects(Q(name__startswith='Guido')).first() - self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__startswith='guido')).first() - self.assertEqual(obj, None) # Test istartswith obj = self.Person.objects(name__istartswith='guido').first() self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__istartswith='guido')).first() - self.assertEqual(obj, person) # Test endswith obj = self.Person.objects(name__endswith='Rossum').first() self.assertEqual(obj, person) obj = self.Person.objects(name__endswith='rossuM').first() self.assertEqual(obj, None) - obj = self.Person.objects(Q(name__endswith='Rossum')).first() - self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__endswith='rossuM')).first() - self.assertEqual(obj, None) # Test iendswith obj = self.Person.objects(name__iendswith='rossuM').first() self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__iendswith='rossuM')).first() - self.assertEqual(obj, person) # Test exact obj = self.Person.objects(name__exact='Guido van Rossum').first() @@ -880,28 +823,18 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(obj, None) obj = self.Person.objects(name__exact='Guido van Rossu').first() self.assertEqual(obj, None) - obj = self.Person.objects(Q(name__exact='Guido van Rossum')).first() - self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__exact='Guido van rossum')).first() - self.assertEqual(obj, None) - obj = self.Person.objects(Q(name__exact='Guido van Rossu')).first() - self.assertEqual(obj, None) # Test iexact obj = self.Person.objects(name__iexact='gUIDO VAN rOSSUM').first() self.assertEqual(obj, person) obj = self.Person.objects(name__iexact='gUIDO VAN rOSSU').first() self.assertEqual(obj, None) - obj = self.Person.objects(Q(name__iexact='gUIDO VAN rOSSUM')).first() - self.assertEqual(obj, person) - obj = self.Person.objects(Q(name__iexact='gUIDO VAN rOSSU')).first() - self.assertEqual(obj, None) # Test unsafe expressions person = self.Person(name='Guido van Rossum [.\'Geek\']') person.save() - obj = self.Person.objects(Q(name__icontains='[.\'Geek')).first() + obj = self.Person.objects(name__icontains='[.\'Geek').first() self.assertEqual(obj, person) def test_not(self): @@ -944,14 +877,14 @@ class QuerySetTest(unittest.TestCase): blog_3.save() blog_post_1 = BlogPost(blog=blog_1, title="Blog Post #1", - is_published = True, - published_date=datetime(2010, 1, 5, 0, 0 ,0)) + is_published=True, + published_date=datetime(2010, 1, 5, 0, 0, 0)) blog_post_2 = BlogPost(blog=blog_2, title="Blog Post #2", - is_published = True, - published_date=datetime(2010, 1, 6, 0, 0 ,0)) + is_published=True, + published_date=datetime(2010, 1, 6, 0, 0, 0)) blog_post_3 = BlogPost(blog=blog_3, title="Blog Post #3", - is_published = True, - published_date=datetime(2010, 1, 7, 0, 0 ,0)) + is_published=True, + published_date=datetime(2010, 1, 7, 0, 0, 0)) blog_post_1.save() blog_post_2.save() @@ -971,21 +904,6 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() Blog.drop_collection() - def test_raw_and_merging(self): - class Doc(Document): - meta = {'allow_inheritance': False} - - raw_query = Doc.objects(__raw__={'deleted': False, - 'scraped': 'yes', - '$nor': [{'views.extracted': 'no'}, - {'attachments.views.extracted':'no'}] - })._query - - expected = {'deleted': False, 'scraped': 'yes', - '$nor': [{'views.extracted': 'no'}, - {'attachments.views.extracted': 'no'}]} - self.assertEqual(expected, raw_query) - def test_ordering(self): """Ensure default ordering is applied and can be overridden. """ @@ -1000,11 +918,11 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() blog_post_1 = BlogPost(title="Blog Post #1", - published_date=datetime(2010, 1, 5, 0, 0 ,0)) + published_date=datetime(2010, 1, 5, 0, 0, 0)) blog_post_2 = BlogPost(title="Blog Post #2", - published_date=datetime(2010, 1, 6, 0, 0 ,0)) + published_date=datetime(2010, 1, 6, 0, 0, 0)) blog_post_3 = BlogPost(title="Blog Post #3", - published_date=datetime(2010, 1, 7, 0, 0 ,0)) + published_date=datetime(2010, 1, 7, 0, 0, 0)) blog_post_1.save() blog_post_2.save() @@ -1310,151 +1228,6 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() - def test_q(self): - """Ensure that Q objects may be used to query for documents. - """ - class BlogPost(Document): - title = StringField() - publish_date = DateTimeField() - published = BooleanField() - - BlogPost.drop_collection() - - post1 = BlogPost(title='Test 1', publish_date=datetime(2010, 1, 8), published=False) - post1.save() - - post2 = BlogPost(title='Test 2', publish_date=datetime(2010, 1, 15), published=True) - post2.save() - - post3 = BlogPost(title='Test 3', published=True) - post3.save() - - post4 = BlogPost(title='Test 4', publish_date=datetime(2010, 1, 8)) - post4.save() - - post5 = BlogPost(title='Test 1', publish_date=datetime(2010, 1, 15)) - post5.save() - - post6 = BlogPost(title='Test 1', published=False) - post6.save() - - # Check ObjectId lookup works - obj = BlogPost.objects(id=post1.id).first() - self.assertEqual(obj, post1) - - # Check Q object combination with one does not exist - q = BlogPost.objects(Q(title='Test 5') | Q(published=True)) - posts = [post.id for post in q] - - published_posts = (post2, post3) - self.assertTrue(all(obj.id in posts for obj in published_posts)) - - q = BlogPost.objects(Q(title='Test 1') | Q(published=True)) - posts = [post.id for post in q] - published_posts = (post1, post2, post3, post5, post6) - self.assertTrue(all(obj.id in posts for obj in published_posts)) - - # Check Q object combination - date = datetime(2010, 1, 10) - q = BlogPost.objects(Q(publish_date__lte=date) | Q(published=True)) - posts = [post.id for post in q] - - published_posts = (post1, post2, post3, post4) - self.assertTrue(all(obj.id in posts for obj in published_posts)) - - self.assertFalse(any(obj.id in posts for obj in [post5, post6])) - - BlogPost.drop_collection() - - # Check the 'in' operator - self.Person(name='user1', age=20).save() - self.Person(name='user2', age=20).save() - self.Person(name='user3', age=30).save() - self.Person(name='user4', age=40).save() - - self.assertEqual(len(self.Person.objects(Q(age__in=[20]))), 2) - self.assertEqual(len(self.Person.objects(Q(age__in=[20, 30]))), 3) - - # Test invalid query objs - def wrong_query_objs(): - self.Person.objects('user1') - def wrong_query_objs_filter(): - self.Person.objects('user1') - self.assertRaises(InvalidQueryError, wrong_query_objs) - self.assertRaises(InvalidQueryError, wrong_query_objs_filter) - - def test_q_regex(self): - """Ensure that Q objects can be queried using regexes. - """ - person = self.Person(name='Guido van Rossum') - person.save() - - import re - obj = self.Person.objects(Q(name=re.compile('^Gui'))).first() - self.assertEqual(obj, person) - obj = self.Person.objects(Q(name=re.compile('^gui'))).first() - self.assertEqual(obj, None) - - obj = self.Person.objects(Q(name=re.compile('^gui', re.I))).first() - self.assertEqual(obj, person) - - obj = self.Person.objects(Q(name__not=re.compile('^bob'))).first() - self.assertEqual(obj, person) - - obj = self.Person.objects(Q(name__not=re.compile('^Gui'))).first() - self.assertEqual(obj, None) - - def test_q_lists(self): - """Ensure that Q objects query ListFields correctly. - """ - class BlogPost(Document): - tags = ListField(StringField()) - - BlogPost.drop_collection() - - BlogPost(tags=['python', 'mongo']).save() - BlogPost(tags=['python']).save() - - self.assertEqual(len(BlogPost.objects(Q(tags='mongo'))), 1) - self.assertEqual(len(BlogPost.objects(Q(tags='python'))), 2) - - BlogPost.drop_collection() - - def test_raw_query_and_Q_objects(self): - """ - Test raw plays nicely - """ - class Foo(Document): - name = StringField() - a = StringField() - b = StringField() - c = StringField() - - meta = { - 'allow_inheritance': False - } - - query = Foo.objects(__raw__={'$nor': [{'name': 'bar'}]})._query - self.assertEqual(query, {'$nor': [{'name': 'bar'}]}) - - q1 = {'$or': [{'a': 1}, {'b': 1}]} - query = Foo.objects(Q(__raw__=q1) & Q(c=1))._query - self.assertEqual(query, {'$or': [{'a': 1}, {'b': 1}], 'c': 1}) - - def test_q_merge_queries_edge_case(self): - - class User(Document): - email = EmailField(required=False) - name = StringField() - - User.drop_collection() - pk = ObjectId() - User(email='example@example.com', pk=pk).save() - - self.assertEqual(1, User.objects.filter( - Q(email='example@example.com') | - Q(name='John Doe') - ).limit(2).filter(pk=pk).count()) def test_exec_js_query(self): """Ensure that queries are properly formed for use in exec_js. @@ -1491,13 +1264,6 @@ class QuerySetTest(unittest.TestCase): c = BlogPost.objects(published=False).exec_js(js_func, 'hits') self.assertEqual(c, 1) - # Ensure that Q object queries work - c = BlogPost.objects(Q(published=True)).exec_js(js_func, 'hits') - self.assertEqual(c, 2) - - c = BlogPost.objects(Q(published=False)).exec_js(js_func, 'hits') - self.assertEqual(c, 1) - BlogPost.drop_collection() def test_exec_js_field_sub(self): @@ -2558,56 +2324,6 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() - def test_query_field_name(self): - """Ensure that the correct field name is used when querying. - """ - class Comment(EmbeddedDocument): - content = StringField(db_field='commentContent') - - class BlogPost(Document): - title = StringField(db_field='postTitle') - comments = ListField(EmbeddedDocumentField(Comment), - db_field='postComments') - - - BlogPost.drop_collection() - - data = {'title': 'Post 1', 'comments': [Comment(content='test')]} - post = BlogPost(**data) - post.save() - - self.assertTrue('postTitle' in - BlogPost.objects(title=data['title'])._query) - self.assertFalse('title' in - BlogPost.objects(title=data['title'])._query) - self.assertEqual(len(BlogPost.objects(title=data['title'])), 1) - - self.assertTrue('_id' in BlogPost.objects(pk=post.id)._query) - self.assertEqual(len(BlogPost.objects(pk=post.id)), 1) - - self.assertTrue('postComments.commentContent' in - BlogPost.objects(comments__content='test')._query) - self.assertEqual(len(BlogPost.objects(comments__content='test')), 1) - - BlogPost.drop_collection() - - def test_query_pk_field_name(self): - """Ensure that the correct "primary key" field name is used when querying - """ - class BlogPost(Document): - title = StringField(primary_key=True, db_field='postTitle') - - BlogPost.drop_collection() - - data = { 'title':'Post 1' } - post = BlogPost(**data) - post.save() - - self.assertTrue('_id' in BlogPost.objects(pk=data['title'])._query) - self.assertTrue('_id' in BlogPost.objects(title=data['title'])._query) - self.assertEqual(len(BlogPost.objects(pk=data['title'])), 1) - - BlogPost.drop_collection() def test_query_value_conversion(self): """Ensure that query values are properly converted when necessary. @@ -3446,227 +3162,6 @@ class QuerySetTest(unittest.TestCase): else: self.assertEqual("[u'A1', u'A2']", "%s" % sorted(self.Person.objects.scalar('name').in_bulk(list(pks)).values())) - -class QTest(unittest.TestCase): - - def setUp(self): - connect(db='mongoenginetest') - - def test_empty_q(self): - """Ensure that empty Q objects won't hurt. - """ - q1 = Q() - q2 = Q(age__gte=18) - q3 = Q() - q4 = Q(name='test') - q5 = Q() - - class Person(Document): - name = StringField() - age = IntField() - - query = {'$or': [{'age': {'$gte': 18}}, {'name': 'test'}]} - self.assertEqual((q1 | q2 | q3 | q4 | q5).to_query(Person), query) - - query = {'age': {'$gte': 18}, 'name': 'test'} - self.assertEqual((q1 & q2 & q3 & q4 & q5).to_query(Person), query) - - def test_q_with_dbref(self): - """Ensure Q objects handle DBRefs correctly""" - connect(db='mongoenginetest') - - class User(Document): - pass - - class Post(Document): - created_user = ReferenceField(User) - - user = User.objects.create() - Post.objects.create(created_user=user) - - self.assertEqual(Post.objects.filter(created_user=user).count(), 1) - self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) - - def test_and_combination(self): - """Ensure that Q-objects correctly AND together. - """ - class TestDoc(Document): - x = IntField() - y = StringField() - - # Check than an error is raised when conflicting queries are anded - def invalid_combination(): - query = Q(x__lt=7) & Q(x__lt=3) - query.to_query(TestDoc) - self.assertRaises(InvalidQueryError, invalid_combination) - - # Check normal cases work without an error - query = Q(x__lt=7) & Q(x__gt=3) - - q1 = Q(x__lt=7) - q2 = Q(x__gt=3) - query = (q1 & q2).to_query(TestDoc) - self.assertEqual(query, {'x': {'$lt': 7, '$gt': 3}}) - - # More complex nested example - query = Q(x__lt=100) & Q(y__ne='NotMyString') - query &= Q(y__in=['a', 'b', 'c']) & Q(x__gt=-100) - mongo_query = { - 'x': {'$lt': 100, '$gt': -100}, - 'y': {'$ne': 'NotMyString', '$in': ['a', 'b', 'c']}, - } - self.assertEqual(query.to_query(TestDoc), mongo_query) - - def test_or_combination(self): - """Ensure that Q-objects correctly OR together. - """ - class TestDoc(Document): - x = IntField() - - q1 = Q(x__lt=3) - q2 = Q(x__gt=7) - query = (q1 | q2).to_query(TestDoc) - self.assertEqual(query, { - '$or': [ - {'x': {'$lt': 3}}, - {'x': {'$gt': 7}}, - ] - }) - - def test_and_or_combination(self): - """Ensure that Q-objects handle ANDing ORed components. - """ - class TestDoc(Document): - x = IntField() - y = BooleanField() - - query = (Q(x__gt=0) | Q(x__exists=False)) - query &= Q(x__lt=100) - self.assertEqual(query.to_query(TestDoc), { - '$or': [ - {'x': {'$lt': 100, '$gt': 0}}, - {'x': {'$lt': 100, '$exists': False}}, - ] - }) - - q1 = (Q(x__gt=0) | Q(x__exists=False)) - q2 = (Q(x__lt=100) | Q(y=True)) - query = (q1 & q2).to_query(TestDoc) - - self.assertEqual(['$or'], query.keys()) - conditions = [ - {'x': {'$lt': 100, '$gt': 0}}, - {'x': {'$lt': 100, '$exists': False}}, - {'x': {'$gt': 0}, 'y': True}, - {'x': {'$exists': False}, 'y': True}, - ] - self.assertEqual(len(conditions), len(query['$or'])) - for condition in conditions: - self.assertTrue(condition in query['$or']) - - def test_or_and_or_combination(self): - """Ensure that Q-objects handle ORing ANDed ORed components. :) - """ - class TestDoc(Document): - x = IntField() - y = BooleanField() - - q1 = (Q(x__gt=0) & (Q(y=True) | Q(y__exists=False))) - q2 = (Q(x__lt=100) & (Q(y=False) | Q(y__exists=False))) - query = (q1 | q2).to_query(TestDoc) - - self.assertEqual(['$or'], query.keys()) - conditions = [ - {'x': {'$gt': 0}, 'y': True}, - {'x': {'$gt': 0}, 'y': {'$exists': False}}, - {'x': {'$lt': 100}, 'y':False}, - {'x': {'$lt': 100}, 'y': {'$exists': False}}, - ] - self.assertEqual(len(conditions), len(query['$or'])) - for condition in conditions: - self.assertTrue(condition in query['$or']) - - - def test_q_clone(self): - - class TestDoc(Document): - x = IntField() - - TestDoc.drop_collection() - for i in xrange(1, 101): - t = TestDoc(x=i) - t.save() - - # Check normal cases work without an error - test = TestDoc.objects(Q(x__lt=7) & Q(x__gt=3)) - - self.assertEqual(test.count(), 3) - - test2 = test.clone() - self.assertEqual(test2.count(), 3) - self.assertFalse(test2 == test) - - test2.filter(x=6) - self.assertEqual(test2.count(), 1) - self.assertEqual(test.count(), 3) - -class QueryFieldListTest(unittest.TestCase): - def test_empty(self): - q = QueryFieldList() - self.assertFalse(q) - - q = QueryFieldList(always_include=['_cls']) - self.assertFalse(q) - - def test_include_include(self): - q = QueryFieldList() - q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'a': True, 'b': True}) - q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'b': True}) - - def test_include_exclude(self): - q = QueryFieldList() - q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'a': True, 'b': True}) - q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.EXCLUDE) - self.assertEqual(q.as_dict(), {'a': True}) - - def test_exclude_exclude(self): - q = QueryFieldList() - q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.EXCLUDE) - self.assertEqual(q.as_dict(), {'a': False, 'b': False}) - q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.EXCLUDE) - self.assertEqual(q.as_dict(), {'a': False, 'b': False, 'c': False}) - - def test_exclude_include(self): - q = QueryFieldList() - q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.EXCLUDE) - self.assertEqual(q.as_dict(), {'a': False, 'b': False}) - q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'c': True}) - - def test_always_include(self): - q = QueryFieldList(always_include=['x', 'y']) - q += QueryFieldList(fields=['a', 'b', 'x'], value=QueryFieldList.EXCLUDE) - q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'c': True}) - - def test_reset(self): - q = QueryFieldList(always_include=['x', 'y']) - q += QueryFieldList(fields=['a', 'b', 'x'], value=QueryFieldList.EXCLUDE) - q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'c': True}) - q.reset() - self.assertFalse(q) - q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'b': True, 'c': True}) - - def test_using_a_slice(self): - q = QueryFieldList() - q += QueryFieldList(fields=['a'], value={"$slice": 5}) - self.assertEqual(q.as_dict(), {'a': {"$slice": 5}}) - def test_elem_match(self): class Foo(EmbeddedDocument): shape = StringField() @@ -3691,6 +3186,26 @@ class QueryFieldListTest(unittest.TestCase): ak = list(Bar.objects(foo__match={'shape': "square", "color": "purple"})) self.assertEqual([b1], ak) + def test_upsert_includes_cls(self): + """Upserts should include _cls information for inheritable classes + """ + + class Test(Document): + test = StringField() + + Test.drop_collection() + Test.objects(test='foo').update_one(upsert=True, set__test='foo') + self.assertFalse('_cls' in Test._collection.find_one()) + + class Test(Document): + meta = {'allow_inheritance': True} + test = StringField() + + Test.drop_collection() + + Test.objects(test='foo').update_one(upsert=True, set__test='foo') + self.assertTrue('_cls' in Test._collection.find_one()) + def test_read_preference(self): class Bar(Document): pass @@ -3769,26 +3284,6 @@ class QueryFieldListTest(unittest.TestCase): self.assertEqual(doc_objects, Doc.objects.from_json(json_data)) - def test_upsert_includes_cls(self): - """Upserts should include _cls information for inheritable classes - """ - - class Test(Document): - test = StringField() - - Test.drop_collection() - Test.objects(test='foo').update_one(upsert=True, set__test='foo') - self.assertFalse('_cls' in Test._collection.find_one()) - - class Test(Document): - meta = {'allow_inheritance': True} - test = StringField() - - Test.drop_collection() - - Test.objects(test='foo').update_one(upsert=True, set__test='foo') - self.assertTrue('_cls' in Test._collection.find_one()) - def test_as_pymongo(self): from decimal import Decimal diff --git a/tests/queryset/transform.py b/tests/queryset/transform.py new file mode 100644 index 0000000..666b345 --- /dev/null +++ b/tests/queryset/transform.py @@ -0,0 +1,148 @@ +from __future__ import with_statement +import sys +sys.path[0:0] = [""] + +import unittest + +from mongoengine import * +from mongoengine.queryset import Q +from mongoengine.queryset import transform + +__all__ = ("TransformTest",) + + +class TransformTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + + def test_transform_query(self): + """Ensure that the _transform_query function operates correctly. + """ + self.assertEqual(transform.query(name='test', age=30), + {'name': 'test', 'age': 30}) + self.assertEqual(transform.query(age__lt=30), + {'age': {'$lt': 30}}) + self.assertEqual(transform.query(age__gt=20, age__lt=50), + {'age': {'$gt': 20, '$lt': 50}}) + self.assertEqual(transform.query(age=20, age__gt=50), + {'age': 20}) + self.assertEqual(transform.query(friend__age__gte=30), + {'friend.age': {'$gte': 30}}) + self.assertEqual(transform.query(name__exists=True), + {'name': {'$exists': True}}) + + def test_query_field_name(self): + """Ensure that the correct field name is used when querying. + """ + class Comment(EmbeddedDocument): + content = StringField(db_field='commentContent') + + class BlogPost(Document): + title = StringField(db_field='postTitle') + comments = ListField(EmbeddedDocumentField(Comment), + db_field='postComments') + + BlogPost.drop_collection() + + data = {'title': 'Post 1', 'comments': [Comment(content='test')]} + post = BlogPost(**data) + post.save() + + self.assertTrue('postTitle' in + BlogPost.objects(title=data['title'])._query) + self.assertFalse('title' in + BlogPost.objects(title=data['title'])._query) + self.assertEqual(len(BlogPost.objects(title=data['title'])), 1) + + self.assertTrue('_id' in BlogPost.objects(pk=post.id)._query) + self.assertEqual(len(BlogPost.objects(pk=post.id)), 1) + + self.assertTrue('postComments.commentContent' in + BlogPost.objects(comments__content='test')._query) + self.assertEqual(len(BlogPost.objects(comments__content='test')), 1) + + BlogPost.drop_collection() + + def test_query_pk_field_name(self): + """Ensure that the correct "primary key" field name is used when + querying + """ + class BlogPost(Document): + title = StringField(primary_key=True, db_field='postTitle') + + BlogPost.drop_collection() + + data = {'title': 'Post 1'} + post = BlogPost(**data) + post.save() + + self.assertTrue('_id' in BlogPost.objects(pk=data['title'])._query) + self.assertTrue('_id' in BlogPost.objects(title=data['title'])._query) + self.assertEqual(len(BlogPost.objects(pk=data['title'])), 1) + + BlogPost.drop_collection() + + def test_chaining(self): + class A(Document): + pass + + class B(Document): + a = ReferenceField(A) + + A.drop_collection() + B.drop_collection() + + a1 = A().save() + a2 = A().save() + + B(a=a1).save() + + # Works + q1 = B.objects.filter(a__in=[a1, a2], a=a1)._query + + # Doesn't work + q2 = B.objects.filter(a__in=[a1, a2]) + q2 = q2.filter(a=a1)._query + + self.assertEqual(q1, q2) + + def test_raw_query_and_Q_objects(self): + """ + Test raw plays nicely + """ + class Foo(Document): + name = StringField() + a = StringField() + b = StringField() + c = StringField() + + meta = { + 'allow_inheritance': False + } + + query = Foo.objects(__raw__={'$nor': [{'name': 'bar'}]})._query + self.assertEqual(query, {'$nor': [{'name': 'bar'}]}) + + q1 = {'$or': [{'a': 1}, {'b': 1}]} + query = Foo.objects(Q(__raw__=q1) & Q(c=1))._query + self.assertEqual(query, {'$or': [{'a': 1}, {'b': 1}], 'c': 1}) + + def test_raw_and_merging(self): + class Doc(Document): + meta = {'allow_inheritance': False} + + raw_query = Doc.objects(__raw__={'deleted': False, + 'scraped': 'yes', + '$nor': [{'views.extracted': 'no'}, + {'attachments.views.extracted':'no'}] + })._query + + expected = {'deleted': False, 'scraped': 'yes', + '$nor': [{'views.extracted': 'no'}, + {'attachments.views.extracted': 'no'}]} + self.assertEqual(expected, raw_query) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/queryset/visitor.py b/tests/queryset/visitor.py new file mode 100644 index 0000000..71c3561 --- /dev/null +++ b/tests/queryset/visitor.py @@ -0,0 +1,310 @@ +from __future__ import with_statement +import sys +sys.path[0:0] = [""] + +import unittest + +from bson import ObjectId +from datetime import datetime + +from mongoengine import * +from mongoengine.queryset import Q +from mongoengine.errors import InvalidQueryError + +__all__ = ("QTest",) + + +class QTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + + class Person(Document): + name = StringField() + age = IntField() + meta = {'allow_inheritance': True} + + Person.drop_collection() + self.Person = Person + + def test_empty_q(self): + """Ensure that empty Q objects won't hurt. + """ + q1 = Q() + q2 = Q(age__gte=18) + q3 = Q() + q4 = Q(name='test') + q5 = Q() + + class Person(Document): + name = StringField() + age = IntField() + + query = {'$or': [{'age': {'$gte': 18}}, {'name': 'test'}]} + self.assertEqual((q1 | q2 | q3 | q4 | q5).to_query(Person), query) + + query = {'age': {'$gte': 18}, 'name': 'test'} + self.assertEqual((q1 & q2 & q3 & q4 & q5).to_query(Person), query) + + def test_q_with_dbref(self): + """Ensure Q objects handle DBRefs correctly""" + connect(db='mongoenginetest') + + class User(Document): + pass + + class Post(Document): + created_user = ReferenceField(User) + + user = User.objects.create() + Post.objects.create(created_user=user) + + self.assertEqual(Post.objects.filter(created_user=user).count(), 1) + self.assertEqual(Post.objects.filter(Q(created_user=user)).count(), 1) + + def test_and_combination(self): + """Ensure that Q-objects correctly AND together. + """ + class TestDoc(Document): + x = IntField() + y = StringField() + + # Check than an error is raised when conflicting queries are anded + def invalid_combination(): + query = Q(x__lt=7) & Q(x__lt=3) + query.to_query(TestDoc) + self.assertRaises(InvalidQueryError, invalid_combination) + + # Check normal cases work without an error + query = Q(x__lt=7) & Q(x__gt=3) + + q1 = Q(x__lt=7) + q2 = Q(x__gt=3) + query = (q1 & q2).to_query(TestDoc) + self.assertEqual(query, {'x': {'$lt': 7, '$gt': 3}}) + + # More complex nested example + query = Q(x__lt=100) & Q(y__ne='NotMyString') + query &= Q(y__in=['a', 'b', 'c']) & Q(x__gt=-100) + mongo_query = { + 'x': {'$lt': 100, '$gt': -100}, + 'y': {'$ne': 'NotMyString', '$in': ['a', 'b', 'c']}, + } + self.assertEqual(query.to_query(TestDoc), mongo_query) + + def test_or_combination(self): + """Ensure that Q-objects correctly OR together. + """ + class TestDoc(Document): + x = IntField() + + q1 = Q(x__lt=3) + q2 = Q(x__gt=7) + query = (q1 | q2).to_query(TestDoc) + self.assertEqual(query, { + '$or': [ + {'x': {'$lt': 3}}, + {'x': {'$gt': 7}}, + ] + }) + + def test_and_or_combination(self): + """Ensure that Q-objects handle ANDing ORed components. + """ + class TestDoc(Document): + x = IntField() + y = BooleanField() + + query = (Q(x__gt=0) | Q(x__exists=False)) + query &= Q(x__lt=100) + self.assertEqual(query.to_query(TestDoc), { + '$or': [ + {'x': {'$lt': 100, '$gt': 0}}, + {'x': {'$lt': 100, '$exists': False}}, + ] + }) + + q1 = (Q(x__gt=0) | Q(x__exists=False)) + q2 = (Q(x__lt=100) | Q(y=True)) + query = (q1 & q2).to_query(TestDoc) + + self.assertEqual(['$or'], query.keys()) + conditions = [ + {'x': {'$lt': 100, '$gt': 0}}, + {'x': {'$lt': 100, '$exists': False}}, + {'x': {'$gt': 0}, 'y': True}, + {'x': {'$exists': False}, 'y': True}, + ] + self.assertEqual(len(conditions), len(query['$or'])) + for condition in conditions: + self.assertTrue(condition in query['$or']) + + def test_or_and_or_combination(self): + """Ensure that Q-objects handle ORing ANDed ORed components. :) + """ + class TestDoc(Document): + x = IntField() + y = BooleanField() + + q1 = (Q(x__gt=0) & (Q(y=True) | Q(y__exists=False))) + q2 = (Q(x__lt=100) & (Q(y=False) | Q(y__exists=False))) + query = (q1 | q2).to_query(TestDoc) + + self.assertEqual(['$or'], query.keys()) + conditions = [ + {'x': {'$gt': 0}, 'y': True}, + {'x': {'$gt': 0}, 'y': {'$exists': False}}, + {'x': {'$lt': 100}, 'y':False}, + {'x': {'$lt': 100}, 'y': {'$exists': False}}, + ] + self.assertEqual(len(conditions), len(query['$or'])) + for condition in conditions: + self.assertTrue(condition in query['$or']) + + def test_q_clone(self): + + class TestDoc(Document): + x = IntField() + + TestDoc.drop_collection() + for i in xrange(1, 101): + t = TestDoc(x=i) + t.save() + + # Check normal cases work without an error + test = TestDoc.objects(Q(x__lt=7) & Q(x__gt=3)) + + self.assertEqual(test.count(), 3) + + test2 = test.clone() + self.assertEqual(test2.count(), 3) + self.assertFalse(test2 == test) + + test2.filter(x=6) + self.assertEqual(test2.count(), 1) + self.assertEqual(test.count(), 3) + + def test_q(self): + """Ensure that Q objects may be used to query for documents. + """ + class BlogPost(Document): + title = StringField() + publish_date = DateTimeField() + published = BooleanField() + + BlogPost.drop_collection() + + post1 = BlogPost(title='Test 1', publish_date=datetime(2010, 1, 8), published=False) + post1.save() + + post2 = BlogPost(title='Test 2', publish_date=datetime(2010, 1, 15), published=True) + post2.save() + + post3 = BlogPost(title='Test 3', published=True) + post3.save() + + post4 = BlogPost(title='Test 4', publish_date=datetime(2010, 1, 8)) + post4.save() + + post5 = BlogPost(title='Test 1', publish_date=datetime(2010, 1, 15)) + post5.save() + + post6 = BlogPost(title='Test 1', published=False) + post6.save() + + # Check ObjectId lookup works + obj = BlogPost.objects(id=post1.id).first() + self.assertEqual(obj, post1) + + # Check Q object combination with one does not exist + q = BlogPost.objects(Q(title='Test 5') | Q(published=True)) + posts = [post.id for post in q] + + published_posts = (post2, post3) + self.assertTrue(all(obj.id in posts for obj in published_posts)) + + q = BlogPost.objects(Q(title='Test 1') | Q(published=True)) + posts = [post.id for post in q] + published_posts = (post1, post2, post3, post5, post6) + self.assertTrue(all(obj.id in posts for obj in published_posts)) + + # Check Q object combination + date = datetime(2010, 1, 10) + q = BlogPost.objects(Q(publish_date__lte=date) | Q(published=True)) + posts = [post.id for post in q] + + published_posts = (post1, post2, post3, post4) + self.assertTrue(all(obj.id in posts for obj in published_posts)) + + self.assertFalse(any(obj.id in posts for obj in [post5, post6])) + + BlogPost.drop_collection() + + # Check the 'in' operator + self.Person(name='user1', age=20).save() + self.Person(name='user2', age=20).save() + self.Person(name='user3', age=30).save() + self.Person(name='user4', age=40).save() + + self.assertEqual(len(self.Person.objects(Q(age__in=[20]))), 2) + self.assertEqual(len(self.Person.objects(Q(age__in=[20, 30]))), 3) + + # Test invalid query objs + def wrong_query_objs(): + self.Person.objects('user1') + def wrong_query_objs_filter(): + self.Person.objects('user1') + self.assertRaises(InvalidQueryError, wrong_query_objs) + self.assertRaises(InvalidQueryError, wrong_query_objs_filter) + + def test_q_regex(self): + """Ensure that Q objects can be queried using regexes. + """ + person = self.Person(name='Guido van Rossum') + person.save() + + import re + obj = self.Person.objects(Q(name=re.compile('^Gui'))).first() + self.assertEqual(obj, person) + obj = self.Person.objects(Q(name=re.compile('^gui'))).first() + self.assertEqual(obj, None) + + obj = self.Person.objects(Q(name=re.compile('^gui', re.I))).first() + self.assertEqual(obj, person) + + obj = self.Person.objects(Q(name__not=re.compile('^bob'))).first() + self.assertEqual(obj, person) + + obj = self.Person.objects(Q(name__not=re.compile('^Gui'))).first() + self.assertEqual(obj, None) + + def test_q_lists(self): + """Ensure that Q objects query ListFields correctly. + """ + class BlogPost(Document): + tags = ListField(StringField()) + + BlogPost.drop_collection() + + BlogPost(tags=['python', 'mongo']).save() + BlogPost(tags=['python']).save() + + self.assertEqual(len(BlogPost.objects(Q(tags='mongo'))), 1) + self.assertEqual(len(BlogPost.objects(Q(tags='python'))), 2) + + BlogPost.drop_collection() + + def test_q_merge_queries_edge_case(self): + + class User(Document): + email = EmailField(required=False) + name = StringField() + + User.drop_collection() + pk = ObjectId() + User(email='example@example.com', pk=pk).save() + + self.assertEqual(1, User.objects.filter( + Q(email='example@example.com') | + Q(name='John Doe') + ).limit(2).filter(pk=pk).count()) \ No newline at end of file From 42f506adc6875f31c5081b1067b13c16f71bdb29 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 13:58:04 +0000 Subject: [PATCH 0887/1279] Updates to test suite --- tests/queryset/field_list.py | 271 ++++++++++++++++++++++++++++++++++- tests/queryset/queryset.py | 251 -------------------------------- tests/queryset/visitor.py | 5 +- 3 files changed, 274 insertions(+), 253 deletions(-) diff --git a/tests/queryset/field_list.py b/tests/queryset/field_list.py index f3b457b..6a9c6a9 100644 --- a/tests/queryset/field_list.py +++ b/tests/queryset/field_list.py @@ -6,7 +6,8 @@ import unittest from mongoengine import * from mongoengine.queryset import QueryFieldList -__all__ = ("QueryFieldListTest",) +__all__ = ("QueryFieldListTest", "OnlyExcludeAllTest") + class QueryFieldListTest(unittest.TestCase): @@ -65,3 +66,271 @@ class QueryFieldListTest(unittest.TestCase): q = QueryFieldList() q += QueryFieldList(fields=['a'], value={"$slice": 5}) self.assertEqual(q.as_dict(), {'a': {"$slice": 5}}) + + +class OnlyExcludeAllTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + + class Person(Document): + name = StringField() + age = IntField() + meta = {'allow_inheritance': True} + + Person.drop_collection() + self.Person = Person + + def test_only(self): + """Ensure that QuerySet.only only returns the requested fields. + """ + person = self.Person(name='test', age=25) + person.save() + + obj = self.Person.objects.only('name').get() + self.assertEqual(obj.name, person.name) + self.assertEqual(obj.age, None) + + obj = self.Person.objects.only('age').get() + self.assertEqual(obj.name, None) + self.assertEqual(obj.age, person.age) + + obj = self.Person.objects.only('name', 'age').get() + self.assertEqual(obj.name, person.name) + self.assertEqual(obj.age, person.age) + + # Check polymorphism still works + class Employee(self.Person): + salary = IntField(db_field='wage') + + employee = Employee(name='test employee', age=40, salary=30000) + employee.save() + + obj = self.Person.objects(id=employee.id).only('age').get() + self.assertTrue(isinstance(obj, Employee)) + + # Check field names are looked up properly + obj = Employee.objects(id=employee.id).only('salary').get() + self.assertEqual(obj.salary, employee.salary) + self.assertEqual(obj.name, None) + + def test_only_with_subfields(self): + class User(EmbeddedDocument): + name = StringField() + email = StringField() + + class Comment(EmbeddedDocument): + title = StringField() + text = StringField() + + class BlogPost(Document): + content = StringField() + author = EmbeddedDocumentField(User) + comments = ListField(EmbeddedDocumentField(Comment)) + + BlogPost.drop_collection() + + post = BlogPost(content='Had a good coffee today...') + post.author = User(name='Test User') + post.comments = [Comment(title='I aggree', text='Great post!'), Comment(title='Coffee', text='I hate coffee')] + post.save() + + obj = BlogPost.objects.only('author.name',).get() + self.assertEqual(obj.content, None) + self.assertEqual(obj.author.email, None) + self.assertEqual(obj.author.name, 'Test User') + self.assertEqual(obj.comments, []) + + obj = BlogPost.objects.only('content', 'comments.title',).get() + self.assertEqual(obj.content, 'Had a good coffee today...') + self.assertEqual(obj.author, None) + self.assertEqual(obj.comments[0].title, 'I aggree') + self.assertEqual(obj.comments[1].title, 'Coffee') + self.assertEqual(obj.comments[0].text, None) + self.assertEqual(obj.comments[1].text, None) + + obj = BlogPost.objects.only('comments',).get() + self.assertEqual(obj.content, None) + self.assertEqual(obj.author, None) + self.assertEqual(obj.comments[0].title, 'I aggree') + self.assertEqual(obj.comments[1].title, 'Coffee') + self.assertEqual(obj.comments[0].text, 'Great post!') + self.assertEqual(obj.comments[1].text, 'I hate coffee') + + BlogPost.drop_collection() + + def test_exclude(self): + class User(EmbeddedDocument): + name = StringField() + email = StringField() + + class Comment(EmbeddedDocument): + title = StringField() + text = StringField() + + class BlogPost(Document): + content = StringField() + author = EmbeddedDocumentField(User) + comments = ListField(EmbeddedDocumentField(Comment)) + + BlogPost.drop_collection() + + post = BlogPost(content='Had a good coffee today...') + post.author = User(name='Test User') + post.comments = [Comment(title='I aggree', text='Great post!'), Comment(title='Coffee', text='I hate coffee')] + post.save() + + obj = BlogPost.objects.exclude('author', 'comments.text').get() + self.assertEqual(obj.author, None) + self.assertEqual(obj.content, 'Had a good coffee today...') + self.assertEqual(obj.comments[0].title, 'I aggree') + self.assertEqual(obj.comments[0].text, None) + + BlogPost.drop_collection() + + def test_exclude_only_combining(self): + class Attachment(EmbeddedDocument): + name = StringField() + content = StringField() + + class Email(Document): + sender = StringField() + to = StringField() + subject = StringField() + body = StringField() + content_type = StringField() + attachments = ListField(EmbeddedDocumentField(Attachment)) + + Email.drop_collection() + email = Email(sender='me', to='you', subject='From Russia with Love', body='Hello!', content_type='text/plain') + email.attachments = [ + Attachment(name='file1.doc', content='ABC'), + Attachment(name='file2.doc', content='XYZ'), + ] + email.save() + + obj = Email.objects.exclude('content_type').exclude('body').get() + self.assertEqual(obj.sender, 'me') + self.assertEqual(obj.to, 'you') + self.assertEqual(obj.subject, 'From Russia with Love') + self.assertEqual(obj.body, None) + self.assertEqual(obj.content_type, None) + + obj = Email.objects.only('sender', 'to').exclude('body', 'sender').get() + self.assertEqual(obj.sender, None) + self.assertEqual(obj.to, 'you') + self.assertEqual(obj.subject, None) + self.assertEqual(obj.body, None) + self.assertEqual(obj.content_type, None) + + obj = Email.objects.exclude('attachments.content').exclude('body').only('to', 'attachments.name').get() + self.assertEqual(obj.attachments[0].name, 'file1.doc') + self.assertEqual(obj.attachments[0].content, None) + self.assertEqual(obj.sender, None) + self.assertEqual(obj.to, 'you') + self.assertEqual(obj.subject, None) + self.assertEqual(obj.body, None) + self.assertEqual(obj.content_type, None) + + Email.drop_collection() + + def test_all_fields(self): + + class Email(Document): + sender = StringField() + to = StringField() + subject = StringField() + body = StringField() + content_type = StringField() + + Email.drop_collection() + + email = Email(sender='me', to='you', subject='From Russia with Love', body='Hello!', content_type='text/plain') + email.save() + + obj = Email.objects.exclude('content_type', 'body').only('to', 'body').all_fields().get() + self.assertEqual(obj.sender, 'me') + self.assertEqual(obj.to, 'you') + self.assertEqual(obj.subject, 'From Russia with Love') + self.assertEqual(obj.body, 'Hello!') + self.assertEqual(obj.content_type, 'text/plain') + + Email.drop_collection() + + def test_slicing_fields(self): + """Ensure that query slicing an array works. + """ + class Numbers(Document): + n = ListField(IntField()) + + Numbers.drop_collection() + + numbers = Numbers(n=[0,1,2,3,4,5,-5,-4,-3,-2,-1]) + numbers.save() + + # first three + numbers = Numbers.objects.fields(slice__n=3).get() + self.assertEqual(numbers.n, [0, 1, 2]) + + # last three + numbers = Numbers.objects.fields(slice__n=-3).get() + self.assertEqual(numbers.n, [-3, -2, -1]) + + # skip 2, limit 3 + numbers = Numbers.objects.fields(slice__n=[2, 3]).get() + self.assertEqual(numbers.n, [2, 3, 4]) + + # skip to fifth from last, limit 4 + numbers = Numbers.objects.fields(slice__n=[-5, 4]).get() + self.assertEqual(numbers.n, [-5, -4, -3, -2]) + + # skip to fifth from last, limit 10 + numbers = Numbers.objects.fields(slice__n=[-5, 10]).get() + self.assertEqual(numbers.n, [-5, -4, -3, -2, -1]) + + # skip to fifth from last, limit 10 dict method + numbers = Numbers.objects.fields(n={"$slice": [-5, 10]}).get() + self.assertEqual(numbers.n, [-5, -4, -3, -2, -1]) + + def test_slicing_nested_fields(self): + """Ensure that query slicing an embedded array works. + """ + + class EmbeddedNumber(EmbeddedDocument): + n = ListField(IntField()) + + class Numbers(Document): + embedded = EmbeddedDocumentField(EmbeddedNumber) + + Numbers.drop_collection() + + numbers = Numbers() + numbers.embedded = EmbeddedNumber(n=[0,1,2,3,4,5,-5,-4,-3,-2,-1]) + numbers.save() + + # first three + numbers = Numbers.objects.fields(slice__embedded__n=3).get() + self.assertEqual(numbers.embedded.n, [0, 1, 2]) + + # last three + numbers = Numbers.objects.fields(slice__embedded__n=-3).get() + self.assertEqual(numbers.embedded.n, [-3, -2, -1]) + + # skip 2, limit 3 + numbers = Numbers.objects.fields(slice__embedded__n=[2, 3]).get() + self.assertEqual(numbers.embedded.n, [2, 3, 4]) + + # skip to fifth from last, limit 4 + numbers = Numbers.objects.fields(slice__embedded__n=[-5, 4]).get() + self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2]) + + # skip to fifth from last, limit 10 + numbers = Numbers.objects.fields(slice__embedded__n=[-5, 10]).get() + self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2, -1]) + + # skip to fifth from last, limit 10 dict method + numbers = Numbers.objects.fields(embedded__n={"$slice": [-5, 10]}).get() + self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2, -1]) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index e3e0215..bad3d36 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -939,257 +939,6 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() - def test_only(self): - """Ensure that QuerySet.only only returns the requested fields. - """ - person = self.Person(name='test', age=25) - person.save() - - obj = self.Person.objects.only('name').get() - self.assertEqual(obj.name, person.name) - self.assertEqual(obj.age, None) - - obj = self.Person.objects.only('age').get() - self.assertEqual(obj.name, None) - self.assertEqual(obj.age, person.age) - - obj = self.Person.objects.only('name', 'age').get() - self.assertEqual(obj.name, person.name) - self.assertEqual(obj.age, person.age) - - # Check polymorphism still works - class Employee(self.Person): - salary = IntField(db_field='wage') - - employee = Employee(name='test employee', age=40, salary=30000) - employee.save() - - obj = self.Person.objects(id=employee.id).only('age').get() - self.assertTrue(isinstance(obj, Employee)) - - # Check field names are looked up properly - obj = Employee.objects(id=employee.id).only('salary').get() - self.assertEqual(obj.salary, employee.salary) - self.assertEqual(obj.name, None) - - def test_only_with_subfields(self): - class User(EmbeddedDocument): - name = StringField() - email = StringField() - - class Comment(EmbeddedDocument): - title = StringField() - text = StringField() - - class BlogPost(Document): - content = StringField() - author = EmbeddedDocumentField(User) - comments = ListField(EmbeddedDocumentField(Comment)) - - BlogPost.drop_collection() - - post = BlogPost(content='Had a good coffee today...') - post.author = User(name='Test User') - post.comments = [Comment(title='I aggree', text='Great post!'), Comment(title='Coffee', text='I hate coffee')] - post.save() - - obj = BlogPost.objects.only('author.name',).get() - self.assertEqual(obj.content, None) - self.assertEqual(obj.author.email, None) - self.assertEqual(obj.author.name, 'Test User') - self.assertEqual(obj.comments, []) - - obj = BlogPost.objects.only('content', 'comments.title',).get() - self.assertEqual(obj.content, 'Had a good coffee today...') - self.assertEqual(obj.author, None) - self.assertEqual(obj.comments[0].title, 'I aggree') - self.assertEqual(obj.comments[1].title, 'Coffee') - self.assertEqual(obj.comments[0].text, None) - self.assertEqual(obj.comments[1].text, None) - - obj = BlogPost.objects.only('comments',).get() - self.assertEqual(obj.content, None) - self.assertEqual(obj.author, None) - self.assertEqual(obj.comments[0].title, 'I aggree') - self.assertEqual(obj.comments[1].title, 'Coffee') - self.assertEqual(obj.comments[0].text, 'Great post!') - self.assertEqual(obj.comments[1].text, 'I hate coffee') - - BlogPost.drop_collection() - - def test_exclude(self): - class User(EmbeddedDocument): - name = StringField() - email = StringField() - - class Comment(EmbeddedDocument): - title = StringField() - text = StringField() - - class BlogPost(Document): - content = StringField() - author = EmbeddedDocumentField(User) - comments = ListField(EmbeddedDocumentField(Comment)) - - BlogPost.drop_collection() - - post = BlogPost(content='Had a good coffee today...') - post.author = User(name='Test User') - post.comments = [Comment(title='I aggree', text='Great post!'), Comment(title='Coffee', text='I hate coffee')] - post.save() - - obj = BlogPost.objects.exclude('author', 'comments.text').get() - self.assertEqual(obj.author, None) - self.assertEqual(obj.content, 'Had a good coffee today...') - self.assertEqual(obj.comments[0].title, 'I aggree') - self.assertEqual(obj.comments[0].text, None) - - BlogPost.drop_collection() - - def test_exclude_only_combining(self): - class Attachment(EmbeddedDocument): - name = StringField() - content = StringField() - - class Email(Document): - sender = StringField() - to = StringField() - subject = StringField() - body = StringField() - content_type = StringField() - attachments = ListField(EmbeddedDocumentField(Attachment)) - - Email.drop_collection() - email = Email(sender='me', to='you', subject='From Russia with Love', body='Hello!', content_type='text/plain') - email.attachments = [ - Attachment(name='file1.doc', content='ABC'), - Attachment(name='file2.doc', content='XYZ'), - ] - email.save() - - obj = Email.objects.exclude('content_type').exclude('body').get() - self.assertEqual(obj.sender, 'me') - self.assertEqual(obj.to, 'you') - self.assertEqual(obj.subject, 'From Russia with Love') - self.assertEqual(obj.body, None) - self.assertEqual(obj.content_type, None) - - obj = Email.objects.only('sender', 'to').exclude('body', 'sender').get() - self.assertEqual(obj.sender, None) - self.assertEqual(obj.to, 'you') - self.assertEqual(obj.subject, None) - self.assertEqual(obj.body, None) - self.assertEqual(obj.content_type, None) - - obj = Email.objects.exclude('attachments.content').exclude('body').only('to', 'attachments.name').get() - self.assertEqual(obj.attachments[0].name, 'file1.doc') - self.assertEqual(obj.attachments[0].content, None) - self.assertEqual(obj.sender, None) - self.assertEqual(obj.to, 'you') - self.assertEqual(obj.subject, None) - self.assertEqual(obj.body, None) - self.assertEqual(obj.content_type, None) - - Email.drop_collection() - - def test_all_fields(self): - - class Email(Document): - sender = StringField() - to = StringField() - subject = StringField() - body = StringField() - content_type = StringField() - - Email.drop_collection() - - email = Email(sender='me', to='you', subject='From Russia with Love', body='Hello!', content_type='text/plain') - email.save() - - obj = Email.objects.exclude('content_type', 'body').only('to', 'body').all_fields().get() - self.assertEqual(obj.sender, 'me') - self.assertEqual(obj.to, 'you') - self.assertEqual(obj.subject, 'From Russia with Love') - self.assertEqual(obj.body, 'Hello!') - self.assertEqual(obj.content_type, 'text/plain') - - Email.drop_collection() - - def test_slicing_fields(self): - """Ensure that query slicing an array works. - """ - class Numbers(Document): - n = ListField(IntField()) - - Numbers.drop_collection() - - numbers = Numbers(n=[0,1,2,3,4,5,-5,-4,-3,-2,-1]) - numbers.save() - - # first three - numbers = Numbers.objects.fields(slice__n=3).get() - self.assertEqual(numbers.n, [0, 1, 2]) - - # last three - numbers = Numbers.objects.fields(slice__n=-3).get() - self.assertEqual(numbers.n, [-3, -2, -1]) - - # skip 2, limit 3 - numbers = Numbers.objects.fields(slice__n=[2, 3]).get() - self.assertEqual(numbers.n, [2, 3, 4]) - - # skip to fifth from last, limit 4 - numbers = Numbers.objects.fields(slice__n=[-5, 4]).get() - self.assertEqual(numbers.n, [-5, -4, -3, -2]) - - # skip to fifth from last, limit 10 - numbers = Numbers.objects.fields(slice__n=[-5, 10]).get() - self.assertEqual(numbers.n, [-5, -4, -3, -2, -1]) - - # skip to fifth from last, limit 10 dict method - numbers = Numbers.objects.fields(n={"$slice": [-5, 10]}).get() - self.assertEqual(numbers.n, [-5, -4, -3, -2, -1]) - - def test_slicing_nested_fields(self): - """Ensure that query slicing an embedded array works. - """ - - class EmbeddedNumber(EmbeddedDocument): - n = ListField(IntField()) - - class Numbers(Document): - embedded = EmbeddedDocumentField(EmbeddedNumber) - - Numbers.drop_collection() - - numbers = Numbers() - numbers.embedded = EmbeddedNumber(n=[0,1,2,3,4,5,-5,-4,-3,-2,-1]) - numbers.save() - - # first three - numbers = Numbers.objects.fields(slice__embedded__n=3).get() - self.assertEqual(numbers.embedded.n, [0, 1, 2]) - - # last three - numbers = Numbers.objects.fields(slice__embedded__n=-3).get() - self.assertEqual(numbers.embedded.n, [-3, -2, -1]) - - # skip 2, limit 3 - numbers = Numbers.objects.fields(slice__embedded__n=[2, 3]).get() - self.assertEqual(numbers.embedded.n, [2, 3, 4]) - - # skip to fifth from last, limit 4 - numbers = Numbers.objects.fields(slice__embedded__n=[-5, 4]).get() - self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2]) - - # skip to fifth from last, limit 10 - numbers = Numbers.objects.fields(slice__embedded__n=[-5, 10]).get() - self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2, -1]) - - # skip to fifth from last, limit 10 dict method - numbers = Numbers.objects.fields(embedded__n={"$slice": [-5, 10]}).get() - self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2, -1]) - def test_find_embedded(self): """Ensure that an embedded document is properly returned from a query. """ diff --git a/tests/queryset/visitor.py b/tests/queryset/visitor.py index 71c3561..82a9913 100644 --- a/tests/queryset/visitor.py +++ b/tests/queryset/visitor.py @@ -307,4 +307,7 @@ class QTest(unittest.TestCase): self.assertEqual(1, User.objects.filter( Q(email='example@example.com') | Q(name='John Doe') - ).limit(2).filter(pk=pk).count()) \ No newline at end of file + ).limit(2).filter(pk=pk).count()) + +if __name__ == '__main__': + unittest.main() From 3074dad293f7c0e7753fd194247675c9b955a011 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 13:58:22 +0000 Subject: [PATCH 0888/1279] Test mixing only, include and exclude #191 --- tests/queryset/field_list.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/queryset/field_list.py b/tests/queryset/field_list.py index 6a9c6a9..9e71133 100644 --- a/tests/queryset/field_list.py +++ b/tests/queryset/field_list.py @@ -81,6 +81,40 @@ class OnlyExcludeAllTest(unittest.TestCase): Person.drop_collection() self.Person = Person + def test_mixing_only_exclude(self): + + class MyDoc(Document): + a = StringField() + b = StringField() + c = StringField() + d = StringField() + e = StringField() + f = StringField() + + include = ['a', 'b', 'c', 'd', 'e'] + exclude = ['d', 'e'] + only = ['b', 'c'] + + qs = MyDoc.objects.fields(**dict(((i, 1) for i in include))) + self.assertEqual(qs._loaded_fields.as_dict(), + {'a': 1, 'b': 1, 'c': 1, 'd': 1, 'e': 1}) + qs = qs.only(*only) + self.assertEqual(qs._loaded_fields.as_dict(), {'b': 1, 'c': 1}) + qs = qs.exclude(*exclude) + self.assertEqual(qs._loaded_fields.as_dict(), {'b': 1, 'c': 1}) + + qs = MyDoc.objects.fields(**dict(((i, 1) for i in include))) + qs = qs.exclude(*exclude) + self.assertEqual(qs._loaded_fields.as_dict(), {'a': 1, 'b': 1, 'c': 1}) + qs = qs.only(*only) + self.assertEqual(qs._loaded_fields.as_dict(), {'b': 1, 'c': 1}) + + qs = MyDoc.objects.exclude(*exclude) + qs = qs.fields(**dict(((i, 1) for i in include))) + self.assertEqual(qs._loaded_fields.as_dict(), {'a': 1, 'b': 1, 'c': 1}) + qs = qs.only(*only) + self.assertEqual(qs._loaded_fields.as_dict(), {'b': 1, 'c': 1}) + def test_only(self): """Ensure that QuerySet.only only returns the requested fields. """ From 1c10f3020b079247ce1d0bca269fd03635c2ca70 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 14:56:15 +0000 Subject: [PATCH 0889/1279] Added support for multiple slices Also made slicing chainable. (#170) (#190) (#191) --- docs/changelog.rst | 1 + mongoengine/queryset/field_list.py | 21 ++++++++++++++++++++- tests/queryset/field_list.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8fc279e..9e1cec8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -21,6 +21,7 @@ Changes in 0.8 - Remove _types and just use _cls for inheritance (#148) - Only allow QNode instances to be passed as query objects (#199) - Dynamic fields are now validated on save (#153) (#154) +- Added support for multiple slices and made slicing chainable. (#170) (#190) (#191) Changes in 0.7.9 ================ diff --git a/mongoengine/queryset/field_list.py b/mongoengine/queryset/field_list.py index 1c825fa..7b2b0cb 100644 --- a/mongoengine/queryset/field_list.py +++ b/mongoengine/queryset/field_list.py @@ -12,20 +12,31 @@ class QueryFieldList(object): self.fields = set(fields) self.always_include = set(always_include) self._id = None + self.slice = {} def __add__(self, f): - if not self.fields: + if isinstance(f.value, dict): + for field in f.fields: + self.slice[field] = f.value + if not self.fields: + self.fields = f.fields + elif not self.fields: self.fields = f.fields self.value = f.value + self.slice = {} elif self.value is self.ONLY and f.value is self.ONLY: + self._clean_slice() self.fields = self.fields.intersection(f.fields) elif self.value is self.EXCLUDE and f.value is self.EXCLUDE: self.fields = self.fields.union(f.fields) + self._clean_slice() elif self.value is self.ONLY and f.value is self.EXCLUDE: self.fields -= f.fields + self._clean_slice() elif self.value is self.EXCLUDE and f.value is self.ONLY: self.value = self.ONLY self.fields = f.fields - self.fields + self._clean_slice() if '_id' in f.fields: self._id = f.value @@ -42,10 +53,18 @@ class QueryFieldList(object): def as_dict(self): field_list = dict((field, self.value) for field in self.fields) + if self.slice: + field_list.update(self.slice) if self._id is not None: field_list['_id'] = self._id return field_list def reset(self): self.fields = set([]) + self.slice = {} self.value = self.ONLY + + def _clean_slice(self): + if self.slice: + for field in set(self.slice.keys()) - self.fields: + del self.slice[field] diff --git a/tests/queryset/field_list.py b/tests/queryset/field_list.py index 9e71133..4a8a72b 100644 --- a/tests/queryset/field_list.py +++ b/tests/queryset/field_list.py @@ -115,6 +115,35 @@ class OnlyExcludeAllTest(unittest.TestCase): qs = qs.only(*only) self.assertEqual(qs._loaded_fields.as_dict(), {'b': 1, 'c': 1}) + def test_slicing(self): + + class MyDoc(Document): + a = ListField() + b = ListField() + c = ListField() + d = ListField() + e = ListField() + f = ListField() + + include = ['a', 'b', 'c', 'd', 'e'] + exclude = ['d', 'e'] + only = ['b', 'c'] + + qs = MyDoc.objects.fields(**dict(((i, 1) for i in include))) + qs = qs.exclude(*exclude) + qs = qs.only(*only) + qs = qs.fields(slice__b=5) + self.assertEqual(qs._loaded_fields.as_dict(), + {'b': {'$slice': 5}, 'c': 1}) + + qs = qs.fields(slice__c=[5, 1]) + self.assertEqual(qs._loaded_fields.as_dict(), + {'b': {'$slice': 5}, 'c': {'$slice': [5, 1]}}) + + qs = qs.exclude('c') + self.assertEqual(qs._loaded_fields.as_dict(), + {'b': {'$slice': 5}}) + def test_only(self): """Ensure that QuerySet.only only returns the requested fields. """ From f335591045e01ed12b12dca4964c3a8c2200a35f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 16:55:14 +0000 Subject: [PATCH 0890/1279] Fix index build_spec #177 --- docs/changelog.rst | 4 +- mongoengine/base/document.py | 2 +- mongoengine/fields.py | 2 +- tests/document/indexes.py | 108 +++++++++++++++++++++++------------ 4 files changed, 76 insertions(+), 40 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9e1cec8..279abc9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,8 +2,8 @@ Changelog ========= -Changes in 0.8 -============== +Changes in 0.8.X +================ - Fixed db_alias and inherited Documents (#143) - Documentation update for document errors (#124) - Deprecated `get_or_create` (#35) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index affc20e..93bde8e 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -525,7 +525,7 @@ class BaseDocument(object): # Check to see if we need to include _cls allow_inheritance = cls._meta.get('allow_inheritance', - ALLOW_INHERITANCE) != False + ALLOW_INHERITANCE) include_cls = allow_inheritance and not spec.get('sparse', False) for key in spec['fields']: diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 73c0db4..3f9810f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -749,7 +749,7 @@ class ReferenceField(BaseField): if dbref is None: msg = ("ReferenceFields will default to using ObjectId " - " strings in 0.8, set DBRef=True if this isn't desired") + "in 0.8, set DBRef=True if this isn't desired") warnings.warn(msg, FutureWarning) self.dbref = dbref if dbref is not None else True # To change in 0.8 diff --git a/tests/document/indexes.py b/tests/document/indexes.py index 8f83afc..9ebd9cb 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -1,31 +1,23 @@ # -*- coding: utf-8 -*- from __future__ import with_statement -import bson -import os -import pickle -import pymongo -import sys import unittest -import uuid -import warnings +import sys + +sys.path[0:0] = [""] + +import os +import pymongo from nose.plugins.skip import SkipTest from datetime import datetime -from tests.fixtures import Base, Mixin, PickleEmbedded, PickleTest - from mongoengine import * -from mongoengine.errors import (NotRegistered, InvalidDocumentError, - InvalidQueryError) -from mongoengine.queryset import NULLIFY, Q from mongoengine.connection import get_db, get_connection -TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') - -__all__ = ("InstanceTest", ) +__all__ = ("IndexesTest", ) -class InstanceTest(unittest.TestCase): +class IndexesTest(unittest.TestCase): def setUp(self): connect(db='mongoenginetest') @@ -47,20 +39,59 @@ class InstanceTest(unittest.TestCase): continue self.db.drop_collection(collection) - def test_indexes_document(self, ): + def ztest_indexes_document(self, ): """Ensure that indexes are used when meta[indexes] is specified for Documents """ - index_test(Document) + self.index_test(Document) def test_indexes_dynamic_document(self, ): """Ensure that indexes are used when meta[indexes] is specified for Dynamic Documents """ - index_test(DynamicDocument) + self.index_test(DynamicDocument) def index_test(self, InheritFrom): + class BlogPost(InheritFrom): + date = DateTimeField(db_field='addDate', default=datetime.now) + category = StringField() + tags = ListField(StringField()) + meta = { + 'indexes': [ + '-date', + 'tags', + ('category', '-date') + ] + } + + expected_specs = [{'fields': [('addDate', -1)]}, + {'fields': [('tags', 1)]}, + {'fields': [('category', 1), ('addDate', -1)]}] + self.assertEqual(expected_specs, BlogPost._meta['index_specs']) + + BlogPost.ensure_indexes() + info = BlogPost.objects._collection.index_information() + # _id, '-date', 'tags', ('cat', 'date') + self.assertEqual(len(info), 4) + info = [value['key'] for key, value in info.iteritems()] + for expected in expected_specs: + self.assertTrue(expected['fields'] in info) + + def test_indexes_document_inheritance(self): + """Ensure that indexes are used when meta[indexes] is specified for + Documents + """ + self.index_test_inheritance(Document) + + def test_indexes_dynamic_document_inheritance(self): + """Ensure that indexes are used when meta[indexes] is specified for + Dynamic Documents + """ + self.index_test_inheritance(DynamicDocument) + + def index_test_inheritance(self, InheritFrom): + class BlogPost(InheritFrom): date = DateTimeField(db_field='addDate', default=datetime.now) category = StringField() @@ -217,7 +248,7 @@ class InstanceTest(unittest.TestCase): info = BlogPost.objects._collection.index_information() # _id, '-date' - self.assertEqual(len(info), 3) + self.assertEqual(len(info), 2) # Indexes are lazy so use list() to perform query list(BlogPost.objects) @@ -265,7 +296,6 @@ class InstanceTest(unittest.TestCase): } user_guid = StringField(required=True) - User.drop_collection() u = User(user_guid='123') @@ -295,7 +325,7 @@ class InstanceTest(unittest.TestCase): BlogPost.drop_collection() info = BlogPost.objects._collection.index_information() - self.assertEqual(info.keys(), ['_cls_1_date.yr_-1', '_id_']) + self.assertEqual(info.keys(), ['date.yr_-1', '_id_']) BlogPost.drop_collection() def test_list_embedded_document_index(self): @@ -318,7 +348,7 @@ class InstanceTest(unittest.TestCase): info = BlogPost.objects._collection.index_information() # we don't use _cls in with list fields by default - self.assertEqual(info.keys(), ['_id_', '_cls_1_tags.tag_1']) + self.assertEqual(info.keys(), ['_id_', 'tags.tag_1']) post1 = BlogPost(title="Embedded Indexes tests in place", tags=[Tag(name="about"), Tag(name="time")] @@ -347,7 +377,7 @@ class InstanceTest(unittest.TestCase): class Parent(Document): name = StringField() - location = ReferenceField(Location) + location = ReferenceField(Location, dbref=False) Location.drop_collection() Parent.drop_collection() @@ -396,8 +426,7 @@ class InstanceTest(unittest.TestCase): meta = { 'indexes': [ ['categories', 'id'] - ], - 'allow_inheritance': False + ] } title = StringField(required=True) @@ -498,15 +527,18 @@ class InstanceTest(unittest.TestCase): BlogPost.drop_collection() - post1 = BlogPost(title='test1', sub=SubDocument(year=2009, slug="test")) + post1 = BlogPost(title='test1', + sub=SubDocument(year=2009, slug="test")) post1.save() # sub.slug is different so won't raise exception - post2 = BlogPost(title='test2', sub=SubDocument(year=2010, slug='another-slug')) + post2 = BlogPost(title='test2', + sub=SubDocument(year=2010, slug='another-slug')) post2.save() # Now there will be two docs with the same sub.slug - post3 = BlogPost(title='test3', sub=SubDocument(year=2010, slug='test')) + post3 = BlogPost(title='test3', + sub=SubDocument(year=2010, slug='test')) self.assertRaises(NotUniqueError, post3.save) BlogPost.drop_collection() @@ -525,19 +557,23 @@ class InstanceTest(unittest.TestCase): BlogPost.drop_collection() - post1 = BlogPost(title='test1', sub=SubDocument(year=2009, slug="test")) + post1 = BlogPost(title='test1', + sub=SubDocument(year=2009, slug="test")) post1.save() # sub.slug is different so won't raise exception - post2 = BlogPost(title='test2', sub=SubDocument(year=2010, slug='another-slug')) + post2 = BlogPost(title='test2', + sub=SubDocument(year=2010, slug='another-slug')) post2.save() # Now there will be two docs with the same sub.slug - post3 = BlogPost(title='test3', sub=SubDocument(year=2010, slug='test')) + post3 = BlogPost(title='test3', + sub=SubDocument(year=2010, slug='test')) self.assertRaises(NotUniqueError, post3.save) # Now there will be two docs with the same title and year - post3 = BlogPost(title='test1', sub=SubDocument(year=2009, slug='test-1')) + post3 = BlogPost(title='test1', + sub=SubDocument(year=2009, slug='test-1')) self.assertRaises(NotUniqueError, post3.save) BlogPost.drop_collection() @@ -566,7 +602,7 @@ class InstanceTest(unittest.TestCase): list(Log.objects) info = Log.objects._collection.index_information() self.assertEqual(3600, - info['_cls_1_created_1']['expireAfterSeconds']) + info['created_1']['expireAfterSeconds']) def test_unique_and_indexes(self): """Ensure that 'unique' constraints aren't overridden by @@ -586,7 +622,7 @@ class InstanceTest(unittest.TestCase): cust_dupe = Customer(cust_id=1) try: cust_dupe.save() - raise AssertionError, "We saved a dupe!" + raise AssertionError("We saved a dupe!") except NotUniqueError: pass Customer.drop_collection() @@ -630,7 +666,7 @@ class InstanceTest(unittest.TestCase): info = BlogPost.objects._collection.index_information() info = [value['key'] for key, value in info.iteritems()] - index_item = [('_cls', 1), ('_id', 1), ('comments.comment_id', 1)] + index_item = [('_id', 1), ('comments.comment_id', 1)] self.assertTrue(index_item in info) if __name__ == '__main__': From 485b811bd00c4c96486b8537347bbc33f05b87d1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 19 Dec 2012 17:05:27 +0000 Subject: [PATCH 0891/1279] Test case for embedded docs and 2d indexes #183 --- tests/document/indexes.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/document/indexes.py b/tests/document/indexes.py index 9ebd9cb..285d8c6 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -225,6 +225,30 @@ class IndexesTest(unittest.TestCase): info = [value['key'] for key, value in info.iteritems()] self.assertTrue([('location.point', '2d')] in info) + def test_explicit_geo2d_index_embedded(self): + """Ensure that geo2d indexes work when created via meta[indexes] + """ + class EmbeddedLocation(EmbeddedDocument): + location = DictField() + + class Place(Document): + current = DictField( + field=EmbeddedDocumentField('EmbeddedLocation')) + meta = { + 'allow_inheritance': True, + 'indexes': [ + '*current.location.point', + ] + } + + self.assertEqual([{'fields': [('current.location.point', '2d')]}], + Place._meta['index_specs']) + + Place.ensure_indexes() + info = Place._get_collection().index_information() + info = [value['key'] for key, value in info.iteritems()] + self.assertTrue([('current.location.point', '2d')] in info) + def test_dictionary_indexes(self): """Ensure that indexes are used when meta[indexes] contains dictionaries instead of lists. From c5b047d0cde0e40c844183a5e5f0c500e63ff8f7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Dec 2012 11:55:05 +0000 Subject: [PATCH 0892/1279] Fixed GridFSProxy __getattr__ behaviour (#196) --- AUTHORS | 3 +- docs/changelog.rst | 1 + docs/guide/gridfs.rst | 16 +- mongoengine/fields.py | 2 +- tests/document/instance.py | 3 +- tests/fields/__init__.py | 2 + tests/{test_fields.py => fields/fields.py} | 305 +---------------- tests/fields/file.py | 370 +++++++++++++++++++++ tests/{document => fields}/mongoengine.png | Bin 9 files changed, 384 insertions(+), 318 deletions(-) create mode 100644 tests/fields/__init__.py rename tests/{test_fields.py => fields/fields.py} (87%) create mode 100644 tests/fields/file.py rename tests/{document => fields}/mongoengine.png (100%) diff --git a/AUTHORS b/AUTHORS index 794f297..b49ddab 100644 --- a/AUTHORS +++ b/AUTHORS @@ -130,4 +130,5 @@ that much better: * Jakub Kot * Jorge Bastida * Stefan Wójcik - * Pete Campton \ No newline at end of file + * Pete Campton + * Martyn Smith \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 279abc9..352d0c7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -22,6 +22,7 @@ Changes in 0.8.X - Only allow QNode instances to be passed as query objects (#199) - Dynamic fields are now validated on save (#153) (#154) - Added support for multiple slices and made slicing chainable. (#170) (#190) (#191) +- Fixed GridFSProxy __getattr__ behaviour (#196) Changes in 0.7.9 ================ diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst index 9c80a99..1125947 100644 --- a/docs/guide/gridfs.rst +++ b/docs/guide/gridfs.rst @@ -18,20 +18,10 @@ a document is created to store details about animals, including a photo:: family = StringField() photo = FileField() - marmot = Animal('Marmota', 'Sciuridae') - - marmot_photo = open('marmot.jpg', 'r') # Retrieve a photo from disk - marmot.photo = marmot_photo # Store photo in the document - marmot.photo.content_type = 'image/jpeg' # Store metadata - - marmot.save() - -Another way of writing to a :class:`~mongoengine.FileField` is to use the -:func:`put` method. This allows for metadata to be stored in the same call as -the file:: - - marmot.photo.put(marmot_photo, content_type='image/jpeg') + marmot = Animal(genus='Marmota', family='Sciuridae') + marmot_photo = open('marmot.jpg', 'r') + marmot.photo.put(marmot_photo, content_type = 'image/jpeg') marmot.save() Retrieval diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 3f9810f..1e1a5ce 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -969,7 +969,7 @@ class GridFSProxy(object): if name in attrs: return self.__getattribute__(name) obj = self.get() - if name in dir(obj): + if hasattr(obj, name): return getattr(obj, name) raise AttributeError diff --git a/tests/document/instance.py b/tests/document/instance.py index 5e29dc3..0054480 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -19,7 +19,8 @@ from mongoengine.queryset import NULLIFY, Q from mongoengine.connection import get_db from mongoengine.base import get_document -TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') +TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), + '../fields/mongoengine.png') __all__ = ("InstanceTest",) diff --git a/tests/fields/__init__.py b/tests/fields/__init__.py new file mode 100644 index 0000000..86dfa84 --- /dev/null +++ b/tests/fields/__init__.py @@ -0,0 +1,2 @@ +from fields import * +from file import * \ No newline at end of file diff --git a/tests/test_fields.py b/tests/fields/fields.py similarity index 87% rename from tests/test_fields.py rename to tests/fields/fields.py index 97a2d5f..a96ff0b 100644 --- a/tests/test_fields.py +++ b/tests/fields/fields.py @@ -4,24 +4,21 @@ import sys sys.path[0:0] = [""] import datetime -import os import unittest import uuid -import tempfile from decimal import Decimal from bson import Binary, DBRef, ObjectId -import gridfs -from nose.plugins.skip import SkipTest from mongoengine import * from mongoengine.connection import get_db from mongoengine.base import _document_registry from mongoengine.errors import NotRegistered -from mongoengine.python_support import PY3, b, StringIO, bin_type +from mongoengine.python_support import PY3, b, bin_type + +__all__ = ("FieldTest", ) -TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'document/mongoengine.png') class FieldTest(unittest.TestCase): @@ -1728,302 +1725,6 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() - def test_file_fields(self): - """Ensure that file fields can be written to and their data retrieved - """ - class PutFile(Document): - the_file = FileField() - - class StreamFile(Document): - the_file = FileField() - - class SetFile(Document): - the_file = FileField() - - text = b('Hello, World!') - more_text = b('Foo Bar') - content_type = 'text/plain' - - PutFile.drop_collection() - StreamFile.drop_collection() - SetFile.drop_collection() - - putfile = PutFile() - putfile.the_file.put(text, content_type=content_type) - putfile.save() - putfile.validate() - result = PutFile.objects.first() - self.assertTrue(putfile == result) - self.assertEqual(result.the_file.read(), text) - self.assertEqual(result.the_file.content_type, content_type) - result.the_file.delete() # Remove file from GridFS - PutFile.objects.delete() - - # Ensure file-like objects are stored - putfile = PutFile() - putstring = StringIO() - putstring.write(text) - putstring.seek(0) - putfile.the_file.put(putstring, content_type=content_type) - putfile.save() - putfile.validate() - result = PutFile.objects.first() - self.assertTrue(putfile == result) - self.assertEqual(result.the_file.read(), text) - self.assertEqual(result.the_file.content_type, content_type) - result.the_file.delete() - - streamfile = StreamFile() - streamfile.the_file.new_file(content_type=content_type) - streamfile.the_file.write(text) - streamfile.the_file.write(more_text) - streamfile.the_file.close() - streamfile.save() - streamfile.validate() - result = StreamFile.objects.first() - self.assertTrue(streamfile == result) - self.assertEqual(result.the_file.read(), text + more_text) - self.assertEqual(result.the_file.content_type, content_type) - result.the_file.seek(0) - self.assertEqual(result.the_file.tell(), 0) - self.assertEqual(result.the_file.read(len(text)), text) - self.assertEqual(result.the_file.tell(), len(text)) - self.assertEqual(result.the_file.read(len(more_text)), more_text) - self.assertEqual(result.the_file.tell(), len(text + more_text)) - result.the_file.delete() - - # Ensure deleted file returns None - self.assertTrue(result.the_file.read() == None) - - setfile = SetFile() - setfile.the_file = text - setfile.save() - setfile.validate() - result = SetFile.objects.first() - self.assertTrue(setfile == result) - self.assertEqual(result.the_file.read(), text) - - # Try replacing file with new one - result.the_file.replace(more_text) - result.save() - result.validate() - result = SetFile.objects.first() - self.assertTrue(setfile == result) - self.assertEqual(result.the_file.read(), more_text) - result.the_file.delete() - - PutFile.drop_collection() - StreamFile.drop_collection() - SetFile.drop_collection() - - # Make sure FileField is optional and not required - class DemoFile(Document): - the_file = FileField() - DemoFile.objects.create() - - def test_file_field_no_default(self): - - class GridDocument(Document): - the_file = FileField() - - GridDocument.drop_collection() - - with tempfile.TemporaryFile() as f: - f.write(b("Hello World!")) - f.flush() - - # Test without default - doc_a = GridDocument() - doc_a.save() - - doc_b = GridDocument.objects.with_id(doc_a.id) - doc_b.the_file.replace(f, filename='doc_b') - doc_b.save() - self.assertNotEqual(doc_b.the_file.grid_id, None) - - # Test it matches - doc_c = GridDocument.objects.with_id(doc_b.id) - self.assertEqual(doc_b.the_file.grid_id, doc_c.the_file.grid_id) - - # Test with default - doc_d = GridDocument(the_file=b('')) - doc_d.save() - - doc_e = GridDocument.objects.with_id(doc_d.id) - self.assertEqual(doc_d.the_file.grid_id, doc_e.the_file.grid_id) - - doc_e.the_file.replace(f, filename='doc_e') - doc_e.save() - - doc_f = GridDocument.objects.with_id(doc_e.id) - self.assertEqual(doc_e.the_file.grid_id, doc_f.the_file.grid_id) - - db = GridDocument._get_db() - grid_fs = gridfs.GridFS(db) - self.assertEqual(['doc_b', 'doc_e'], grid_fs.list()) - - def test_file_uniqueness(self): - """Ensure that each instance of a FileField is unique - """ - class TestFile(Document): - name = StringField() - the_file = FileField() - - # First instance - test_file = TestFile() - test_file.name = "Hello, World!" - test_file.the_file.put(b('Hello, World!')) - test_file.save() - - # Second instance - test_file_dupe = TestFile() - data = test_file_dupe.the_file.read() # Should be None - - self.assertTrue(test_file.name != test_file_dupe.name) - self.assertTrue(test_file.the_file.read() != data) - - TestFile.drop_collection() - - def test_file_boolean(self): - """Ensure that a boolean test of a FileField indicates its presence - """ - class TestFile(Document): - the_file = FileField() - - test_file = TestFile() - self.assertFalse(bool(test_file.the_file)) - test_file.the_file = b('Hello, World!') - test_file.the_file.content_type = 'text/plain' - test_file.save() - self.assertTrue(bool(test_file.the_file)) - - TestFile.drop_collection() - - def test_file_cmp(self): - """Test comparing against other types""" - class TestFile(Document): - the_file = FileField() - - test_file = TestFile() - self.assertFalse(test_file.the_file in [{"test": 1}]) - - def test_image_field(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') - - class TestImage(Document): - image = ImageField() - - TestImage.drop_collection() - - t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) - t.save() - - t = TestImage.objects.first() - - self.assertEqual(t.image.format, 'PNG') - - w, h = t.image.size - self.assertEqual(w, 371) - self.assertEqual(h, 76) - - t.image.delete() - - def test_image_field_resize(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') - - class TestImage(Document): - image = ImageField(size=(185, 37)) - - TestImage.drop_collection() - - t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) - t.save() - - t = TestImage.objects.first() - - self.assertEqual(t.image.format, 'PNG') - w, h = t.image.size - - self.assertEqual(w, 185) - self.assertEqual(h, 37) - - t.image.delete() - - def test_image_field_resize_force(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') - - class TestImage(Document): - image = ImageField(size=(185, 37, True)) - - TestImage.drop_collection() - - t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) - t.save() - - t = TestImage.objects.first() - - self.assertEqual(t.image.format, 'PNG') - w, h = t.image.size - - self.assertEqual(w, 185) - self.assertEqual(h, 37) - - t.image.delete() - - def test_image_field_thumbnail(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') - - class TestImage(Document): - image = ImageField(thumbnail_size=(92, 18)) - - TestImage.drop_collection() - - t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) - t.save() - - t = TestImage.objects.first() - - self.assertEqual(t.image.thumbnail.format, 'PNG') - self.assertEqual(t.image.thumbnail.width, 92) - self.assertEqual(t.image.thumbnail.height, 18) - - t.image.delete() - - def test_file_multidb(self): - register_connection('test_files', 'test_files') - class TestFile(Document): - name = StringField() - the_file = FileField(db_alias="test_files", - collection_name="macumba") - - TestFile.drop_collection() - - # delete old filesystem - get_db("test_files").macumba.files.drop() - get_db("test_files").macumba.chunks.drop() - - # First instance - test_file = TestFile() - test_file.name = "Hello, World!" - test_file.the_file.put(b('Hello, World!'), - name="hello.txt") - test_file.save() - - data = get_db("test_files").macumba.files.find_one() - self.assertEqual(data.get('name'), 'hello.txt') - - test_file = TestFile.objects.first() - self.assertEqual(test_file.the_file.read(), - b('Hello, World!')) - def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """ diff --git a/tests/fields/file.py b/tests/fields/file.py new file mode 100644 index 0000000..17d9ec3 --- /dev/null +++ b/tests/fields/file.py @@ -0,0 +1,370 @@ +# -*- coding: utf-8 -*- +from __future__ import with_statement +import sys +sys.path[0:0] = [""] + +import datetime +import os +import unittest +import uuid +import tempfile + +from decimal import Decimal + +from bson import Binary, DBRef, ObjectId +import gridfs + +from nose.plugins.skip import SkipTest +from mongoengine import * +from mongoengine.connection import get_db +from mongoengine.base import _document_registry +from mongoengine.errors import NotRegistered +from mongoengine.python_support import PY3, b, StringIO, bin_type + +TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') + + +class FileTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + def tearDown(self): + self.db.drop_collection('fs.files') + self.db.drop_collection('fs.chunks') + + def test_file_field_optional(self): + # Make sure FileField is optional and not required + class DemoFile(Document): + the_file = FileField() + DemoFile.objects.create() + + def test_file_fields(self): + """Ensure that file fields can be written to and their data retrieved + """ + + class PutFile(Document): + the_file = FileField() + + PutFile.drop_collection() + + text = b('Hello, World!') + more_text = b('Foo Bar') + content_type = 'text/plain' + + putfile = PutFile() + putfile.the_file.put(text, content_type=content_type) + putfile.save() + putfile.validate() + result = PutFile.objects.first() + self.assertTrue(putfile == result) + self.assertEqual(result.the_file.read(), text) + self.assertEqual(result.the_file.content_type, content_type) + result.the_file.delete() # Remove file from GridFS + PutFile.objects.delete() + + # Ensure file-like objects are stored + PutFile.drop_collection() + + putfile = PutFile() + putstring = StringIO() + putstring.write(text) + putstring.seek(0) + putfile.the_file.put(putstring, content_type=content_type) + putfile.save() + putfile.validate() + result = PutFile.objects.first() + self.assertTrue(putfile == result) + self.assertEqual(result.the_file.read(), text) + self.assertEqual(result.the_file.content_type, content_type) + result.the_file.delete() + + def test_file_fields_stream(self): + """Ensure that file fields can be written to and their data retrieved + """ + class StreamFile(Document): + the_file = FileField() + + StreamFile.drop_collection() + + text = b('Hello, World!') + more_text = b('Foo Bar') + content_type = 'text/plain' + + streamfile = StreamFile() + streamfile.the_file.new_file(content_type=content_type) + streamfile.the_file.write(text) + streamfile.the_file.write(more_text) + streamfile.the_file.close() + streamfile.save() + streamfile.validate() + result = StreamFile.objects.first() + self.assertTrue(streamfile == result) + self.assertEqual(result.the_file.read(), text + more_text) + self.assertEqual(result.the_file.content_type, content_type) + result.the_file.seek(0) + self.assertEqual(result.the_file.tell(), 0) + self.assertEqual(result.the_file.read(len(text)), text) + self.assertEqual(result.the_file.tell(), len(text)) + self.assertEqual(result.the_file.read(len(more_text)), more_text) + self.assertEqual(result.the_file.tell(), len(text + more_text)) + result.the_file.delete() + + # Ensure deleted file returns None + self.assertTrue(result.the_file.read() == None) + + def test_file_fields_set(self): + + class SetFile(Document): + the_file = FileField() + + text = b('Hello, World!') + more_text = b('Foo Bar') + + SetFile.drop_collection() + + setfile = SetFile() + setfile.the_file = text + setfile.save() + + result = SetFile.objects.first() + self.assertTrue(setfile == result) + self.assertEqual(result.the_file.read(), text) + + # Try replacing file with new one + result.the_file.replace(more_text) + result.save() + result.validate() + result = SetFile.objects.first() + self.assertTrue(setfile == result) + self.assertEqual(result.the_file.read(), more_text) + result.the_file.delete() + + def test_file_field_no_default(self): + + class GridDocument(Document): + the_file = FileField() + + GridDocument.drop_collection() + + with tempfile.TemporaryFile() as f: + f.write(b("Hello World!")) + f.flush() + + # Test without default + doc_a = GridDocument() + doc_a.save() + + doc_b = GridDocument.objects.with_id(doc_a.id) + doc_b.the_file.replace(f, filename='doc_b') + doc_b.save() + self.assertNotEqual(doc_b.the_file.grid_id, None) + + # Test it matches + doc_c = GridDocument.objects.with_id(doc_b.id) + self.assertEqual(doc_b.the_file.grid_id, doc_c.the_file.grid_id) + + # Test with default + doc_d = GridDocument(the_file=b('')) + doc_d.save() + + doc_e = GridDocument.objects.with_id(doc_d.id) + self.assertEqual(doc_d.the_file.grid_id, doc_e.the_file.grid_id) + + doc_e.the_file.replace(f, filename='doc_e') + doc_e.save() + + doc_f = GridDocument.objects.with_id(doc_e.id) + self.assertEqual(doc_e.the_file.grid_id, doc_f.the_file.grid_id) + + db = GridDocument._get_db() + grid_fs = gridfs.GridFS(db) + self.assertEqual(['doc_b', 'doc_e'], grid_fs.list()) + + def test_file_uniqueness(self): + """Ensure that each instance of a FileField is unique + """ + class TestFile(Document): + name = StringField() + the_file = FileField() + + # First instance + test_file = TestFile() + test_file.name = "Hello, World!" + test_file.the_file.put(b('Hello, World!')) + test_file.save() + + # Second instance + test_file_dupe = TestFile() + data = test_file_dupe.the_file.read() # Should be None + + self.assertTrue(test_file.name != test_file_dupe.name) + self.assertTrue(test_file.the_file.read() != data) + + TestFile.drop_collection() + + def test_file_saving(self): + """Ensure you can add meta data to file""" + + class Animal(Document): + genus = StringField() + family = StringField() + photo = FileField() + + Animal.drop_collection() + marmot = Animal(genus='Marmota', family='Sciuridae') + + marmot_photo = open(TEST_IMAGE_PATH, 'r') # Retrieve a photo from disk + marmot.photo.put(marmot_photo, content_type='image/jpeg', foo='bar') + marmot.photo.close() + marmot.save() + + marmot = Animal.objects.get() + self.assertEqual(marmot.photo.content_type, 'image/jpeg') + self.assertEqual(marmot.photo.foo, 'bar') + + def test_file_boolean(self): + """Ensure that a boolean test of a FileField indicates its presence + """ + class TestFile(Document): + the_file = FileField() + TestFile.drop_collection() + + test_file = TestFile() + self.assertFalse(bool(test_file.the_file)) + test_file.the_file.put(b('Hello, World!'), content_type='text/plain') + test_file.save() + self.assertTrue(bool(test_file.the_file)) + + test_file = TestFile.objects.first() + self.assertEqual(test_file.the_file.content_type, "text/plain") + + def test_file_cmp(self): + """Test comparing against other types""" + class TestFile(Document): + the_file = FileField() + + test_file = TestFile() + self.assertFalse(test_file.the_file in [{"test": 1}]) + + def test_image_field(self): + if PY3: + raise SkipTest('PIL does not have Python 3 support') + + class TestImage(Document): + image = ImageField() + + TestImage.drop_collection() + + t = TestImage() + t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.save() + + t = TestImage.objects.first() + + self.assertEqual(t.image.format, 'PNG') + + w, h = t.image.size + self.assertEqual(w, 371) + self.assertEqual(h, 76) + + t.image.delete() + + def test_image_field_resize(self): + if PY3: + raise SkipTest('PIL does not have Python 3 support') + + class TestImage(Document): + image = ImageField(size=(185, 37)) + + TestImage.drop_collection() + + t = TestImage() + t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.save() + + t = TestImage.objects.first() + + self.assertEqual(t.image.format, 'PNG') + w, h = t.image.size + + self.assertEqual(w, 185) + self.assertEqual(h, 37) + + t.image.delete() + + def test_image_field_resize_force(self): + if PY3: + raise SkipTest('PIL does not have Python 3 support') + + class TestImage(Document): + image = ImageField(size=(185, 37, True)) + + TestImage.drop_collection() + + t = TestImage() + t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.save() + + t = TestImage.objects.first() + + self.assertEqual(t.image.format, 'PNG') + w, h = t.image.size + + self.assertEqual(w, 185) + self.assertEqual(h, 37) + + t.image.delete() + + def test_image_field_thumbnail(self): + if PY3: + raise SkipTest('PIL does not have Python 3 support') + + class TestImage(Document): + image = ImageField(thumbnail_size=(92, 18)) + + TestImage.drop_collection() + + t = TestImage() + t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.save() + + t = TestImage.objects.first() + + self.assertEqual(t.image.thumbnail.format, 'PNG') + self.assertEqual(t.image.thumbnail.width, 92) + self.assertEqual(t.image.thumbnail.height, 18) + + t.image.delete() + + def test_file_multidb(self): + register_connection('test_files', 'test_files') + + class TestFile(Document): + name = StringField() + the_file = FileField(db_alias="test_files", + collection_name="macumba") + + TestFile.drop_collection() + + # delete old filesystem + get_db("test_files").macumba.files.drop() + get_db("test_files").macumba.chunks.drop() + + # First instance + test_file = TestFile() + test_file.name = "Hello, World!" + test_file.the_file.put(b('Hello, World!'), + name="hello.txt") + test_file.save() + + data = get_db("test_files").macumba.files.find_one() + self.assertEqual(data.get('name'), 'hello.txt') + + test_file = TestFile.objects.first() + self.assertEqual(test_file.the_file.read(), + b('Hello, World!')) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/document/mongoengine.png b/tests/fields/mongoengine.png similarity index 100% rename from tests/document/mongoengine.png rename to tests/fields/mongoengine.png From 286beca6c5798c0d5c25e7981a8a0c448743eb58 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Dec 2012 12:07:15 +0000 Subject: [PATCH 0893/1279] Added Marcelo Anton to authors #152 --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index b49ddab..4b3a297 100644 --- a/AUTHORS +++ b/AUTHORS @@ -131,4 +131,5 @@ that much better: * Jorge Bastida * Stefan Wójcik * Pete Campton - * Martyn Smith \ No newline at end of file + * Martyn Smith + * Marcelo Anton \ No newline at end of file From 0c2fb6807e425638cc92a92a0ef058932a9b1c05 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Dec 2012 12:14:32 +0000 Subject: [PATCH 0894/1279] Added Aleksey Porfirov to AUTHORS #151 --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 4b3a297..989fd68 100644 --- a/AUTHORS +++ b/AUTHORS @@ -132,4 +132,5 @@ that much better: * Stefan Wójcik * Pete Campton * Martyn Smith - * Marcelo Anton \ No newline at end of file + * Marcelo Anton + * Aleksey Porfirov \ No newline at end of file From bf74d7537cafc9f9baf15a87eb8c2d7e569dd5c9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Dec 2012 16:20:01 +0000 Subject: [PATCH 0895/1279] Fix Django timezone support - update field for callable #151 --- mongoengine/fields.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 1e1a5ce..f6c0311 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -300,6 +300,8 @@ class DateTimeField(BaseField): return value if isinstance(value, datetime.date): return datetime.datetime(value.year, value.month, value.day) + if callable(value): + return value() # Attempt to parse a datetime: # value = smart_str(value) From 3aff4610394e31c64ac2a3ed2f256fd1f3e90da6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Dec 2012 16:29:27 +0000 Subject: [PATCH 0896/1279] Fix test discovery --- tests/document/indexes.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/tests/document/indexes.py b/tests/document/indexes.py index 285d8c6..445cfe2 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -39,19 +39,19 @@ class IndexesTest(unittest.TestCase): continue self.db.drop_collection(collection) - def ztest_indexes_document(self, ): + def test_indexes_document(self): """Ensure that indexes are used when meta[indexes] is specified for Documents """ - self.index_test(Document) + self._index_test(Document) - def test_indexes_dynamic_document(self, ): + def test_indexes_dynamic_document(self): """Ensure that indexes are used when meta[indexes] is specified for Dynamic Documents """ - self.index_test(DynamicDocument) + self._index_test(DynamicDocument) - def index_test(self, InheritFrom): + def _index_test(self, InheritFrom): class BlogPost(InheritFrom): date = DateTimeField(db_field='addDate', default=datetime.now) @@ -78,19 +78,7 @@ class IndexesTest(unittest.TestCase): for expected in expected_specs: self.assertTrue(expected['fields'] in info) - def test_indexes_document_inheritance(self): - """Ensure that indexes are used when meta[indexes] is specified for - Documents - """ - self.index_test_inheritance(Document) - - def test_indexes_dynamic_document_inheritance(self): - """Ensure that indexes are used when meta[indexes] is specified for - Dynamic Documents - """ - self.index_test_inheritance(DynamicDocument) - - def index_test_inheritance(self, InheritFrom): + def _index_test_inheritance(self, InheritFrom): class BlogPost(InheritFrom): date = DateTimeField(db_field='addDate', default=datetime.now) @@ -137,6 +125,18 @@ class IndexesTest(unittest.TestCase): for expected in expected_specs: self.assertTrue(expected['fields'] in info) + def test_indexes_document_inheritance(self): + """Ensure that indexes are used when meta[indexes] is specified for + Documents + """ + self._index_test_inheritance(Document) + + def test_indexes_dynamic_document_inheritance(self): + """Ensure that indexes are used when meta[indexes] is specified for + Dynamic Documents + """ + self._index_test_inheritance(DynamicDocument) + def test_inherited_index(self): """Ensure index specs are inhertited correctly""" @@ -301,6 +301,7 @@ class IndexesTest(unittest.TestCase): meta = { 'indexes': ['name'], } + Person.drop_collection() Person(name="test", user_guid='123').save() From 1cdf71b647f31a8c7cf05b0256a9795038c8808d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Dec 2012 16:35:09 +0000 Subject: [PATCH 0897/1279] Simplified Q objects Removed QueryTreeTransformerVisitor (#98) (#171) --- docs/changelog.rst | 1 + mongoengine/queryset/visitor.py | 92 ++------------------------------- tests/queryset/visitor.py | 72 +++++++++++++++++--------- 3 files changed, 53 insertions(+), 112 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f934f5e..b9ab42c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -24,6 +24,7 @@ Changes in 0.8.X - Added support for multiple slices and made slicing chainable. (#170) (#190) (#191) - Fixed GridFSProxy __getattr__ behaviour (#196) - Fix Django timezone support (#151) +- Simplified Q objects, removed QueryTreeTransformerVisitor (#98) (#171) Changes in 0.7.9 ================ diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py index 94d6a5e..8932a54 100644 --- a/mongoengine/queryset/visitor.py +++ b/mongoengine/queryset/visitor.py @@ -55,57 +55,6 @@ class SimplificationVisitor(QNodeVisitor): return combined_query -class QueryTreeTransformerVisitor(QNodeVisitor): - """Transforms the query tree in to a form that may be used with MongoDB. - """ - - def visit_combination(self, combination): - if combination.operation == combination.AND: - # MongoDB doesn't allow us to have too many $or operations in our - # queries, so the aim is to move the ORs up the tree to one - # 'master' $or. Firstly, we must find all the necessary parts (part - # of an AND combination or just standard Q object), and store them - # separately from the OR parts. - or_groups = [] - and_parts = [] - for node in combination.children: - if isinstance(node, QCombination): - if node.operation == node.OR: - # Any of the children in an $or component may cause - # the query to succeed - or_groups.append(node.children) - elif node.operation == node.AND: - and_parts.append(node) - elif isinstance(node, Q): - and_parts.append(node) - - # Now we combine the parts into a usable query. AND together all of - # the necessary parts. Then for each $or part, create a new query - # that ANDs the necessary part with the $or part. - clauses = [] - for or_group in product(*or_groups): - q_object = reduce(lambda a, b: a & b, and_parts, Q()) - q_object = reduce(lambda a, b: a & b, or_group, q_object) - clauses.append(q_object) - # Finally, $or the generated clauses in to one query. Each of the - # clauses is sufficient for the query to succeed. - return reduce(lambda a, b: a | b, clauses, Q()) - - if combination.operation == combination.OR: - children = [] - # Crush any nested ORs in to this combination as MongoDB doesn't - # support nested $or operations - for node in combination.children: - if (isinstance(node, QCombination) and - node.operation == combination.OR): - children += node.children - else: - children.append(node) - combination.children = children - - return combination - - class QueryCompilerVisitor(QNodeVisitor): """Compiles the nodes in a query tree to a PyMongo-compatible query dictionary. @@ -115,45 +64,14 @@ class QueryCompilerVisitor(QNodeVisitor): self.document = document def visit_combination(self, combination): + operator = "$and" if combination.operation == combination.OR: - return {'$or': combination.children} - elif combination.operation == combination.AND: - return self._mongo_query_conjunction(combination.children) - return combination + operator = "$or" + return {operator: combination.children} def visit_query(self, query): return transform.query(self.document, **query.query) - def _mongo_query_conjunction(self, queries): - """Merges Mongo query dicts - effectively &ing them together. - """ - combined_query = {} - for query in queries: - for field, ops in query.items(): - if field not in combined_query: - combined_query[field] = ops - else: - # The field is already present in the query the only way - # we can merge is if both the existing value and the new - # value are operation dicts, reject anything else - if (not isinstance(combined_query[field], dict) or - not isinstance(ops, dict)): - message = 'Conflicting values for ' + field - raise InvalidQueryError(message) - - current_ops = set(combined_query[field].keys()) - new_ops = set(ops.keys()) - # Make sure that the same operation isn't applied more than - # once to a single field - intersection = current_ops.intersection(new_ops) - if intersection: - msg = 'Duplicate query conditions: ' - raise InvalidQueryError(msg + ', '.join(intersection)) - - # Right! We've got two non-overlapping dicts of operations! - combined_query[field].update(copy.deepcopy(ops)) - return combined_query - class QNode(object): """Base class for nodes in query trees. @@ -164,7 +82,6 @@ class QNode(object): def to_query(self, document): query = self.accept(SimplificationVisitor()) - query = query.accept(QueryTreeTransformerVisitor()) query = query.accept(QueryCompilerVisitor(document)) return query @@ -205,7 +122,8 @@ class QCombination(QNode): # If the child is a combination of the same type, we can merge its # children directly into this combinations children if isinstance(node, QCombination) and node.operation == operation: - self.children += node.children + # self.children += node.children + self.children.append(node) else: self.children.append(node) diff --git a/tests/queryset/visitor.py b/tests/queryset/visitor.py index 82a9913..4af39e8 100644 --- a/tests/queryset/visitor.py +++ b/tests/queryset/visitor.py @@ -115,29 +115,31 @@ class QTest(unittest.TestCase): x = IntField() y = BooleanField() + TestDoc.drop_collection() + query = (Q(x__gt=0) | Q(x__exists=False)) query &= Q(x__lt=100) - self.assertEqual(query.to_query(TestDoc), { - '$or': [ - {'x': {'$lt': 100, '$gt': 0}}, - {'x': {'$lt': 100, '$exists': False}}, - ] + self.assertEqual(query.to_query(TestDoc), {'$and': [ + {'$or': [{'x': {'$gt': 0}}, + {'x': {'$exists': False}}]}, + {'x': {'$lt': 100}}] }) q1 = (Q(x__gt=0) | Q(x__exists=False)) q2 = (Q(x__lt=100) | Q(y=True)) query = (q1 & q2).to_query(TestDoc) - self.assertEqual(['$or'], query.keys()) - conditions = [ - {'x': {'$lt': 100, '$gt': 0}}, - {'x': {'$lt': 100, '$exists': False}}, - {'x': {'$gt': 0}, 'y': True}, - {'x': {'$exists': False}, 'y': True}, - ] - self.assertEqual(len(conditions), len(query['$or'])) - for condition in conditions: - self.assertTrue(condition in query['$or']) + TestDoc(x=101).save() + TestDoc(x=10).save() + TestDoc(y=True).save() + + self.assertEqual(query, + {'$and': [ + {'$or': [{'x': {'$gt': 0}}, {'x': {'$exists': False}}]}, + {'$or': [{'x': {'$lt': 100}}, {'y': True}]} + ]}) + + self.assertEqual(2, TestDoc.objects(q1 & q2).count()) def test_or_and_or_combination(self): """Ensure that Q-objects handle ORing ANDed ORed components. :) @@ -146,20 +148,40 @@ class QTest(unittest.TestCase): x = IntField() y = BooleanField() + TestDoc.drop_collection() + TestDoc(x=-1, y=True).save() + TestDoc(x=101, y=True).save() + TestDoc(x=99, y=False).save() + TestDoc(x=101, y=False).save() + q1 = (Q(x__gt=0) & (Q(y=True) | Q(y__exists=False))) q2 = (Q(x__lt=100) & (Q(y=False) | Q(y__exists=False))) query = (q1 | q2).to_query(TestDoc) - self.assertEqual(['$or'], query.keys()) - conditions = [ - {'x': {'$gt': 0}, 'y': True}, - {'x': {'$gt': 0}, 'y': {'$exists': False}}, - {'x': {'$lt': 100}, 'y':False}, - {'x': {'$lt': 100}, 'y': {'$exists': False}}, - ] - self.assertEqual(len(conditions), len(query['$or'])) - for condition in conditions: - self.assertTrue(condition in query['$or']) + self.assertEqual(query, + {'$or': [ + {'$and': [{'x': {'$gt': 0}}, + {'$or': [{'y': True}, {'y': {'$exists': False}}]}]}, + {'$and': [{'x': {'$lt': 100}}, + {'$or': [{'y': False}, {'y': {'$exists': False}}]}]} + ]} + ) + + self.assertEqual(2, TestDoc.objects(q1 | q2).count()) + + def test_multiple_occurence_in_field(self): + class Test(Document): + name = StringField(max_length=40) + title = StringField(max_length=40) + + q1 = Q(name__contains='te') | Q(title__contains='te') + q2 = Q(name__contains='12') | Q(title__contains='12') + + q3 = q1 & q2 + + query = q3.to_query(Test) + self.assertEqual(query["$and"][0], q1.to_query(Test)) + self.assertEqual(query["$and"][1], q2.to_query(Test)) def test_q_clone(self): From b9e0f525262111433158db16307424086efb71e0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Dec 2012 17:09:10 +0000 Subject: [PATCH 0898/1279] FileFields now copyable (#198) --- docs/changelog.rst | 1 + mongoengine/fields.py | 14 +++++++++++--- tests/fields/file.py | 39 ++++++++++++++++++++++++++------------- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b9ab42c..c649303 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -25,6 +25,7 @@ Changes in 0.8.X - Fixed GridFSProxy __getattr__ behaviour (#196) - Fix Django timezone support (#151) - Simplified Q objects, removed QueryTreeTransformerVisitor (#98) (#171) +- FileFields now copyable (#198) Changes in 0.7.9 ================ diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f6c0311..5f11ae3 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -986,14 +986,22 @@ class GridFSProxy(object): self_dict['_fs'] = None return self_dict + def __copy__(self): + copied = GridFSProxy() + copied.__dict__.update(self.__getstate__()) + return copied + + def __deepcopy__(self, memo): + return self.__copy__() + def __repr__(self): return '<%s: %s>' % (self.__class__.__name__, self.grid_id) def __eq__(self, other): if isinstance(other, GridFSProxy): - return ((self.grid_id == other.grid_id) and - (self.collection_name == other.collection_name) and - (self.db_alias == other.db_alias)) + return ((self.grid_id == other.grid_id) and + (self.collection_name == other.collection_name) and + (self.db_alias == other.db_alias)) else: return False diff --git a/tests/fields/file.py b/tests/fields/file.py index 17d9ec3..a39dadb 100644 --- a/tests/fields/file.py +++ b/tests/fields/file.py @@ -3,23 +3,17 @@ from __future__ import with_statement import sys sys.path[0:0] = [""] -import datetime +import copy import os import unittest -import uuid import tempfile -from decimal import Decimal - -from bson import Binary, DBRef, ObjectId import gridfs from nose.plugins.skip import SkipTest from mongoengine import * from mongoengine.connection import get_db -from mongoengine.base import _document_registry -from mongoengine.errors import NotRegistered -from mongoengine.python_support import PY3, b, StringIO, bin_type +from mongoengine.python_support import PY3, b, StringIO TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') @@ -50,13 +44,12 @@ class FileTest(unittest.TestCase): PutFile.drop_collection() text = b('Hello, World!') - more_text = b('Foo Bar') content_type = 'text/plain' putfile = PutFile() putfile.the_file.put(text, content_type=content_type) putfile.save() - putfile.validate() + result = PutFile.objects.first() self.assertTrue(putfile == result) self.assertEqual(result.the_file.read(), text) @@ -73,7 +66,7 @@ class FileTest(unittest.TestCase): putstring.seek(0) putfile.the_file.put(putstring, content_type=content_type) putfile.save() - putfile.validate() + result = PutFile.objects.first() self.assertTrue(putfile == result) self.assertEqual(result.the_file.read(), text) @@ -98,7 +91,7 @@ class FileTest(unittest.TestCase): streamfile.the_file.write(more_text) streamfile.the_file.close() streamfile.save() - streamfile.validate() + result = StreamFile.objects.first() self.assertTrue(streamfile == result) self.assertEqual(result.the_file.read(), text + more_text) @@ -135,7 +128,7 @@ class FileTest(unittest.TestCase): # Try replacing file with new one result.the_file.replace(more_text) result.save() - result.validate() + result = SetFile.objects.first() self.assertTrue(setfile == result) self.assertEqual(result.the_file.read(), more_text) @@ -366,5 +359,25 @@ class FileTest(unittest.TestCase): self.assertEqual(test_file.the_file.read(), b('Hello, World!')) + def test_copyable(self): + class PutFile(Document): + the_file = FileField() + + PutFile.drop_collection() + + text = b('Hello, World!') + content_type = 'text/plain' + + putfile = PutFile() + putfile.the_file.put(text, content_type=content_type) + putfile.save() + + class TestFile(Document): + name = StringField() + + self.assertEqual(putfile, copy.copy(putfile)) + self.assertEqual(putfile, copy.deepcopy(putfile)) + + if __name__ == '__main__': unittest.main() From 09a5f5c8f33f036fa7c12caf614f799bdc4046c2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 3 Jan 2013 12:56:42 +0000 Subject: [PATCH 0899/1279] Added note for django and sites config issues --- docs/django.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/django.rst b/docs/django.rst index 144baab..a4f0560 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -10,6 +10,10 @@ In your **settings.py** file, ignore the standard database settings (unless you also plan to use the ORM in your project), and instead call :func:`~mongoengine.connect` somewhere in the settings module. +.. note :: If getting an ``ImproperlyConfigured: settings.DATABASES is + improperly configured`` error you may need to remove + ``django.contrib.sites`` from ``INSTALLED_APPS`` in settings.py. + Authentication ============== MongoEngine includes a Django authentication backend, which uses MongoDB. The From 9bbd8dbe624c385c68404834884f20b304b5b64f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 4 Jan 2013 09:41:08 +0000 Subject: [PATCH 0900/1279] Querysets now return clones and are no longer edit in place Fixes #56 --- docs/changelog.rst | 1 + docs/upgrade.rst | 22 + mongoengine/connection.py | 6 +- mongoengine/queryset/queryset.py | 1182 ++++++++++++++++-------------- tests/queryset/queryset.py | 11 +- tests/queryset/visitor.py | 4 +- 6 files changed, 656 insertions(+), 570 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c649303..4fd3e14 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -26,6 +26,7 @@ Changes in 0.8.X - Fix Django timezone support (#151) - Simplified Q objects, removed QueryTreeTransformerVisitor (#98) (#171) - FileFields now copyable (#198) +- Querysets now return clones and are no longer edit in place (#56) Changes in 0.7.9 ================ diff --git a/docs/upgrade.rst b/docs/upgrade.rst index bf48527..9c6c9a9 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -56,6 +56,28 @@ you will need to declare :attr:`allow_inheritance` in the meta data like so: :: meta = {'allow_inheritance': True} +Querysets +~~~~~~~~~ + +Querysets now return clones and should no longer be considered editable in +place. This brings us in line with how Django's querysets work and removes a +long running gotcha. If you edit your querysets inplace you will have to +update your code like so: :: + + # Old code: + mammals = Animal.objects(type="mammal") + mammals.filter(order="Carnivora") # Returns a cloned queryset that isn't assigned to anything - so this will break in 0.8 + [m for m in mammals] # This will return all mammals in 0.8 as the 2nd filter returned a new queryset + + # Update example a) assign queryset after a change: + mammals = Animal.objects(type="mammal") + carnivores = mammals.filter(order="Carnivora") # Reassign the new queryset so fitler can be applied + [m for m in carnivores] # This will return all carnivores + + # Update example b) chain the queryset: + mammals = Animal.objects(type="mammal").filter(order="Carnivora") # The final queryset is assgined to mammals + [m for m in mammals] # This will return all carnivores + Indexes ------- diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 1ccbbe3..87308ba 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -28,8 +28,10 @@ def register_connection(alias, name, host='localhost', port=27017, :param name: the name of the specific database to use :param host: the host name of the :program:`mongod` instance to connect to :param port: the port that the :program:`mongod` instance is running on - :param is_slave: whether the connection can act as a slave ** Depreciated pymongo 2.0.1+ - :param read_preference: The read preference for the collection ** Added pymongo 2.1 + :param is_slave: whether the connection can act as a slave + ** Depreciated pymongo 2.0.1+ + :param read_preference: The read preference for the collection + ** Added pymongo 2.1 :param slaves: a list of aliases of slave connections; each of these must be a registered connection that has :attr:`is_slave` set to ``True`` :param username: username to authenticate with diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 3ea9f23..239975f 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -42,7 +42,6 @@ class QuerySet(object): providing :class:`~mongoengine.Document` objects as the results. """ __dereference = False - __none = False def __init__(self, document, collection): self._document = document @@ -60,6 +59,7 @@ class QuerySet(object): self._read_preference = None self._iter = False self._scalar = [] + self._none = False self._as_pymongo = False self._as_pymongo_coerce = False @@ -71,35 +71,9 @@ class QuerySet(object): self._cursor_obj = None self._limit = None self._skip = None + self._slice = None self._hint = -1 # Using -1 as None is a valid value for hint - def clone(self): - """Creates a copy of the current - :class:`~mongoengine.queryset.QuerySet` - - .. versionadded:: 0.5 - """ - c = self.__class__(self._document, self._collection_obj) - - copy_props = ('_initial_query', '_query_obj', '_where_clause', - '_loaded_fields', '_ordering', '_snapshot', - '_timeout', '_limit', '_skip', '_slave_okay', '_hint', - '_read_preference') - - for prop in copy_props: - val = getattr(self, prop) - setattr(c, prop, copy.deepcopy(val)) - - return c - - @property - def _query(self): - if self._mongo_query is None: - self._mongo_query = self._query_obj.to_query(self._document) - if self._class_check: - self._mongo_query.update(self._initial_query) - return self._mongo_query - def __call__(self, q_obj=None, class_check=True, slave_okay=False, read_preference=None, **query): """Filter the selected documents by calling the @@ -121,87 +95,94 @@ class QuerySet(object): if q_obj: # make sure proper query object is passed if not isinstance(q_obj, QNode): - raise InvalidQueryError('Not a query object: %s. Did you intend to use key=value?' % q_obj) + msg = ("Not a query object: %s. " + "Did you intend to use key=value?" % q_obj) + raise InvalidQueryError(msg) query &= q_obj - self._query_obj &= query - self._mongo_query = None - self._cursor_obj = None + + queryset = self.clone() + queryset._query_obj &= query + queryset._mongo_query = None + queryset._cursor_obj = None if read_preference is not None: - self.read_preference(read_preference) - self._class_check = class_check + queryset.read_preference(read_preference) + queryset._class_check = class_check + return queryset + + + def __iter__(self): + """Support iterator protocol""" + self.rewind() return self - def filter(self, *q_objs, **query): - """An alias of :meth:`~mongoengine.queryset.QuerySet.__call__` + def __len__(self): + return self.count() + + def __getitem__(self, key): + """Support skip and limit using getitem and slicing syntax. """ - return self.__call__(*q_objs, **query) + queryset = self.clone() + + # Slice provided + if isinstance(key, slice): + try: + queryset._cursor_obj = queryset._cursor[key] + queryset._slice = key + queryset._skip, queryset._limit = key.start, key.stop + except IndexError, err: + # PyMongo raises an error if key.start == key.stop, catch it, + # bin it, kill it. + start = key.start or 0 + if start >= 0 and key.stop >= 0 and key.step is None: + if start == key.stop: + queryset.limit(0) + queryset._skip = key.start + queryset._limit = key.stop - start + return queryset + raise err + # Allow further QuerySet modifications to be performed + return queryset + # Integer index provided + elif isinstance(key, int): + if queryset._scalar: + return queryset._get_scalar( + queryset._document._from_son(queryset._cursor[key])) + if queryset._as_pymongo: + return queryset._get_as_pymongo(queryset._cursor.next()) + return queryset._document._from_son(queryset._cursor[key]) + raise AttributeError + + def __repr__(self): + """Provides the string representation of the QuerySet + + .. versionchanged:: 0.6.13 Now doesnt modify the cursor + """ + + if self._iter: + return '.. queryset mid-iteration ..' + + data = [] + for i in xrange(REPR_OUTPUT_SIZE + 1): + try: + data.append(self.next()) + except StopIteration: + break + if len(data) > REPR_OUTPUT_SIZE: + data[-1] = "...(remaining elements truncated)..." + + self.rewind() + return repr(data) + + # Core functions def all(self): """Returns all documents.""" return self.__call__() - def ensure_index(self, **kwargs): - """Deprecated use :func:`~Document.ensure_index`""" - msg = ("Doc.objects()._ensure_index() is deprecated. " - "Use Doc.ensure_index() instead.") - warnings.warn(msg, DeprecationWarning) - self._document.__class__.ensure_index(**kwargs) - return self - - def _ensure_indexes(self): - """Deprecated use :func:`~Document.ensure_indexes`""" - msg = ("Doc.objects()._ensure_indexes() is deprecated. " - "Use Doc.ensure_indexes() instead.") - warnings.warn(msg, DeprecationWarning) - self._document.__class__.ensure_indexes() - - @property - def _collection(self): - """Property that returns the collection object. This allows us to - perform operations only if the collection is accessed. + def filter(self, *q_objs, **query): + """An alias of :meth:`~mongoengine.queryset.QuerySet.__call__` """ - return self._collection_obj - - @property - def _cursor_args(self): - cursor_args = { - 'snapshot': self._snapshot, - 'timeout': self._timeout, - 'slave_okay': self._slave_okay, - } - if self._read_preference is not None: - cursor_args['read_preference'] = self._read_preference - if self._loaded_fields: - cursor_args['fields'] = self._loaded_fields.as_dict() - return cursor_args - - @property - def _cursor(self): - if self._cursor_obj is None: - - self._cursor_obj = self._collection.find(self._query, - **self._cursor_args) - # Apply where clauses to cursor - if self._where_clause: - self._cursor_obj.where(self._where_clause) - - if self._ordering: - # Apply query ordering - self._cursor_obj.sort(self._ordering) - elif self._document._meta['ordering']: - # Otherwise, apply the ordering from the document model - self.order_by(*self._document._meta['ordering']) - self._cursor_obj.sort(self._ordering) - - if self._limit is not None: - self._cursor_obj.limit(self._limit - (self._skip or 0)) - - if self._skip is not None: - self._cursor_obj.skip(self._skip) - - if self._hint != -1: - self._cursor_obj.hint(self._hint) - return self._cursor_obj + return self.__call__(*q_objs, **query) def get(self, *q_objs, **query): """Retrieve the the matching object raising @@ -212,22 +193,29 @@ class QuerySet(object): .. versionadded:: 0.3 """ - self.limit(2) - self.__call__(*q_objs, **query) + queryset = self.__call__(*q_objs, **query) + queryset = queryset.limit(2) try: - result = self.next() + result = queryset.next() except StopIteration: msg = ("%s matching query does not exist." - % self._document._class_name) - raise self._document.DoesNotExist(msg) + % queryset._document._class_name) + raise queryset._document.DoesNotExist(msg) try: - self.next() + queryset.next() except StopIteration: return result - self.rewind() - message = u'%d items returned, instead of 1' % self.count() - raise self._document.MultipleObjectsReturned(message) + queryset.rewind() + message = u'%d items returned, instead of 1' % queryset.count() + raise queryset._document.MultipleObjectsReturned(message) + + def create(self, **kwargs): + """Create new object. Returns the saved object instance. + + .. versionadded:: 0.4 + """ + return self._document(**kwargs).save() def get_or_create(self, write_options=None, auto_save=True, *q_objs, **query): @@ -277,20 +265,12 @@ class QuerySet(object): doc.save(write_options=write_options) return doc, True - def create(self, **kwargs): - """Create new object. Returns the saved object instance. - - .. versionadded:: 0.4 - """ - doc = self._document(**kwargs) - doc.save() - return doc - def first(self): """Retrieve the first object matching the query. """ + queryset = self.clone() try: - result = self[0] + result = queryset[0] except IndexError: result = None return result @@ -367,6 +347,117 @@ class QuerySet(object): self._document, documents=results, loaded=True) return return_one and results[0] or results + def count(self): + """Count the selected elements in the query. + """ + if self._limit == 0: + return 0 + return self._cursor.count(with_limit_and_skip=True) + + def delete(self, safe=False): + """Delete the documents matched by the query. + + :param safe: check if the operation succeeded before returning + """ + queryset = self.clone() + doc = queryset._document + + # Handle deletes where skips or limits have been applied + if queryset._skip or queryset._limit: + for doc in queryset: + doc.delete() + return + + delete_rules = doc._meta.get('delete_rules') or {} + # Check for DENY rules before actually deleting/nullifying any other + # references + for rule_entry in delete_rules: + document_cls, field_name = rule_entry + rule = doc._meta['delete_rules'][rule_entry] + if rule == DENY and document_cls.objects( + **{field_name + '__in': self}).count() > 0: + msg = ("Could not delete document (%s.%s refers to it)" + % (document_cls.__name__, field_name)) + raise OperationError(msg) + + for rule_entry in delete_rules: + document_cls, field_name = rule_entry + rule = doc._meta['delete_rules'][rule_entry] + if rule == CASCADE: + ref_q = document_cls.objects(**{field_name + '__in': self}) + ref_q_count = ref_q.count() + if (doc != document_cls and ref_q_count > 0 + or (doc == document_cls and ref_q_count > 0)): + ref_q.delete(safe=safe) + elif rule == NULLIFY: + document_cls.objects(**{field_name + '__in': self}).update( + safe_update=safe, + **{'unset__%s' % field_name: 1}) + elif rule == PULL: + document_cls.objects(**{field_name + '__in': self}).update( + safe_update=safe, + **{'pull_all__%s' % field_name: self}) + + queryset._collection.remove(queryset._query, safe=safe) + + def update(self, safe_update=True, upsert=False, multi=True, + write_options=None, **update): + """Perform an atomic update on the fields matched by the query. When + ``safe_update`` is used, the number of affected documents is returned. + + :param safe_update: check if the operation succeeded before returning + :param upsert: Any existing document with that "_id" is overwritten. + :param write_options: extra keyword arguments for + :meth:`~pymongo.collection.Collection.update` + + .. versionadded:: 0.2 + """ + if not update: + raise OperationError("No update parameters, would remove data") + + if not write_options: + write_options = {} + + queryset = self.clone() + query = queryset._query + update = transform.update(queryset._document, **update) + + # If doing an atomic upsert on an inheritable class + # then ensure we add _cls to the update operation + if upsert and '_cls' in query: + if '$set' in update: + update["$set"]["_cls"] = queryset._document._class_name + else: + update["$set"] = {"_cls": queryset._document._class_name} + + try: + ret = queryset._collection.update(query, update, multi=multi, + upsert=upsert, safe=safe_update, + **write_options) + if ret is not None and 'n' in ret: + return ret['n'] + except pymongo.errors.OperationFailure, err: + if unicode(err) == u'multi not coded yet': + message = u'update() method requires MongoDB 1.1.3+' + raise OperationError(message) + raise OperationError(u'Update failed (%s)' % unicode(err)) + + def update_one(self, safe_update=True, upsert=False, write_options=None, + **update): + """Perform an atomic update on first field matched by the query. When + ``safe_update`` is used, the number of affected documents is returned. + + :param safe_update: check if the operation succeeded before returning + :param upsert: Any existing document with that "_id" is overwritten. + :param write_options: extra keyword arguments for + :meth:`~pymongo.collection.Collection.update` + :param update: Django-style update keyword arguments + + .. versionadded:: 0.2 + """ + return self.update(safe_update=True, upsert=upsert, multi=False, + write_options=None, **update) + def with_id(self, object_id): """Retrieve the object matching the id provided. Uses `object_id` only and raises InvalidQueryError if a filter has been applied. @@ -375,10 +466,11 @@ class QuerySet(object): .. versionchanged:: 0.6 Raises InvalidQueryError if filter has been set """ - if not self._query_obj.empty: + queryset = self.clone() + if not queryset._query_obj.empty: msg = "Cannot use a filter whilst using `with_id`" raise InvalidQueryError(msg) - return self.filter(pk=object_id).first() + return queryset.filter(pk=object_id).first() def in_bulk(self, object_ids): """Retrieve a set of documents by their ids. @@ -406,139 +498,48 @@ class QuerySet(object): return doc_map - def next(self): - """Wrap the result in a :class:`~mongoengine.Document` object. - """ - self._iter = True - try: - if self._limit == 0 or self.__none: - raise StopIteration - if self._scalar: - return self._get_scalar(self._document._from_son( - self._cursor.next())) - if self._as_pymongo: - return self._get_as_pymongo(self._cursor.next()) - - return self._document._from_son(self._cursor.next()) - except StopIteration, e: - self.rewind() - raise e - - def rewind(self): - """Rewind the cursor to its unevaluated state. - - .. versionadded:: 0.3 - """ - self._iter = False - self._cursor.rewind() - def none(self): """Helper that just returns a list""" - self.__none = True - return self + queryset = self.clone() + queryset._none = True + return queryset - def count(self): - """Count the selected elements in the query. + def clone(self): + """Creates a copy of the current + :class:`~mongoengine.queryset.QuerySet` + + .. versionadded:: 0.5 """ - if self._limit == 0: - return 0 - return self._cursor.count(with_limit_and_skip=True) + c = self.__class__(self._document, self._collection_obj) - def __len__(self): - return self.count() + copy_props = ('_mongo_query', '_initial_query', '_none', '_query_obj', + '_where_clause', '_loaded_fields', '_ordering', '_snapshot', + '_timeout', '_class_check', '_slave_okay', '_read_preference', + '_iter', '_scalar', '_as_pymongo', '_as_pymongo_coerce', + '_limit', '_skip', '_slice', '_hint') - def map_reduce(self, map_f, reduce_f, output, finalize_f=None, limit=None, - scope=None): - """Perform a map/reduce query using the current query spec - and ordering. While ``map_reduce`` respects ``QuerySet`` chaining, - it must be the last call made, as it does not return a maleable - ``QuerySet``. + for prop in copy_props: + val = getattr(self, prop) + setattr(c, prop, copy.copy(val)) - See the :meth:`~mongoengine.tests.QuerySetTest.test_map_reduce` - and :meth:`~mongoengine.tests.QuerySetTest.test_map_advanced` - tests in ``tests.queryset.QuerySetTest`` for usage examples. + if self._cursor_obj: + c._cursor_obj = self._cursor_obj.clone() - :param map_f: map function, as :class:`~bson.code.Code` or string - :param reduce_f: reduce function, as - :class:`~bson.code.Code` or string - :param output: output collection name, if set to 'inline' will try to - use :class:`~pymongo.collection.Collection.inline_map_reduce` - This can also be a dictionary containing output options - see: http://docs.mongodb.org/manual/reference/commands/#mapReduce - :param finalize_f: finalize function, an optional function that - performs any post-reduction processing. - :param scope: values to insert into map/reduce global scope. Optional. - :param limit: number of objects from current query to provide - to map/reduce method + if self._slice: + c._cursor_obj[self._slice] - Returns an iterator yielding - :class:`~mongoengine.document.MapReduceDocument`. + return c - .. note:: + def select_related(self, max_depth=1): + """Handles dereferencing of :class:`~bson.dbref.DBRef` objects to + a maximum depth in order to cut down the number queries to mongodb. - Map/Reduce changed in server version **>= 1.7.4**. The PyMongo - :meth:`~pymongo.collection.Collection.map_reduce` helper requires - PyMongo version **>= 1.11**. - - .. versionchanged:: 0.5 - - removed ``keep_temp`` keyword argument, which was only relevant - for MongoDB server versions older than 1.7.4 - - .. versionadded:: 0.3 + .. versionadded:: 0.5 """ - MapReduceDocument = _import_class('MapReduceDocument') - - if not hasattr(self._collection, "map_reduce"): - raise NotImplementedError("Requires MongoDB >= 1.7.1") - - map_f_scope = {} - if isinstance(map_f, Code): - map_f_scope = map_f.scope - map_f = unicode(map_f) - map_f = Code(self._sub_js_fields(map_f), map_f_scope) - - reduce_f_scope = {} - if isinstance(reduce_f, Code): - reduce_f_scope = reduce_f.scope - reduce_f = unicode(reduce_f) - reduce_f_code = self._sub_js_fields(reduce_f) - reduce_f = Code(reduce_f_code, reduce_f_scope) - - mr_args = {'query': self._query} - - if finalize_f: - finalize_f_scope = {} - if isinstance(finalize_f, Code): - finalize_f_scope = finalize_f.scope - finalize_f = unicode(finalize_f) - finalize_f_code = self._sub_js_fields(finalize_f) - finalize_f = Code(finalize_f_code, finalize_f_scope) - mr_args['finalize'] = finalize_f - - if scope: - mr_args['scope'] = scope - - if limit: - mr_args['limit'] = limit - - if output == 'inline' and not self._ordering: - map_reduce_function = 'inline_map_reduce' - else: - map_reduce_function = 'map_reduce' - mr_args['out'] = output - - results = getattr(self._collection, map_reduce_function)( - map_f, reduce_f, **mr_args) - - if map_reduce_function == 'map_reduce': - results = results.find() - - if self._ordering: - results = results.sort(self._ordering) - - for doc in results: - yield MapReduceDocument(self._document, self._collection, - doc['_id'], doc['value']) + # Make select related work the same for querysets + max_depth += 1 + queryset = self.clone() + return queryset._dereference(queryset, max_depth=max_depth) def limit(self, n): """Limit the number of returned documents to `n`. This may also be @@ -546,14 +547,15 @@ class QuerySet(object): :param n: the maximum number of objects to return """ + queryset = self.clone() if n == 0: - self._cursor.limit(1) + queryset._cursor.limit(1) else: - self._cursor.limit(n) - self._limit = n + queryset._cursor.limit(n) + queryset._limit = n # Return self to allow chaining - return self + return queryset def skip(self, n): """Skip `n` documents before returning the results. This may also be @@ -561,9 +563,10 @@ class QuerySet(object): :param n: the number of objects to skip before returning results """ - self._cursor.skip(n) - self._skip = n - return self + queryset = self.clone() + queryset._cursor.skip(n) + queryset._skip = n + return queryset def hint(self, index=None): """Added 'hint' support, telling Mongo the proper index to use for the @@ -578,39 +581,10 @@ class QuerySet(object): .. versionadded:: 0.5 """ - self._cursor.hint(index) - self._hint = index - return self - - def __getitem__(self, key): - """Support skip and limit using getitem and slicing syntax. - """ - # Slice provided - if isinstance(key, slice): - try: - self._cursor_obj = self._cursor[key] - self._skip, self._limit = key.start, key.stop - except IndexError, err: - # PyMongo raises an error if key.start == key.stop, catch it, - # bin it, kill it. - start = key.start or 0 - if start >= 0 and key.stop >= 0 and key.step is None: - if start == key.stop: - self.limit(0) - self._skip, self._limit = key.start, key.stop - start - return self - raise err - # Allow further QuerySet modifications to be performed - return self - # Integer index provided - elif isinstance(key, int): - if self._scalar: - return self._get_scalar(self._document._from_son( - self._cursor[key])) - if self._as_pymongo: - return self._get_as_pymongo(self._cursor.next()) - return self._document._from_son(self._cursor[key]) - raise AttributeError + queryset = self.clone() + queryset._cursor.hint(index) + queryset._hint = index + return queryset def distinct(self, field): """Return a list of distinct values for a given field. @@ -621,8 +595,9 @@ class QuerySet(object): .. versionchanged:: 0.5 - Fixed handling references .. versionchanged:: 0.6 - Improved db_field refrence handling """ - return self._dereference(self._cursor.distinct(field), 1, - name=field, instance=self._document) + queryset = self.clone() + return queryset._dereference(queryset._cursor.distinct(field), 1, + name=field, instance=queryset._document) def only(self, *fields): """Load only a subset of this document's fields. :: @@ -679,11 +654,12 @@ class QuerySet(object): cleaned_fields.append((key, value)) fields = sorted(cleaned_fields, key=operator.itemgetter(1)) + queryset = self.clone() for value, group in itertools.groupby(fields, lambda x: x[1]): fields = [field for field, value in group] - fields = self._fields_to_dbfields(fields) - self._loaded_fields += QueryFieldList(fields, value=value) - return self + fields = queryset._fields_to_dbfields(fields) + queryset._loaded_fields += QueryFieldList(fields, value=value) + return queryset def all_fields(self): """Include all fields. Reset all previously calls of .only() or @@ -693,18 +669,10 @@ class QuerySet(object): .. versionadded:: 0.5 """ - self._loaded_fields = QueryFieldList( - always_include=self._loaded_fields.always_include) - return self - - def _fields_to_dbfields(self, fields): - """Translate fields paths to its db equivalents""" - ret = [] - for field in fields: - field = ".".join(f.db_field for f in - self._document._lookup_field(field.split('.'))) - ret.append(field) - return ret + queryset = self.clone() + queryset._loaded_fields = QueryFieldList( + always_include=queryset._loaded_fields.always_include) + return queryset def order_by(self, *keys): """Order the :class:`~mongoengine.queryset.QuerySet` by the keys. The @@ -714,25 +682,9 @@ class QuerySet(object): :param keys: fields to order the query results by; keys may be prefixed with **+** or **-** to determine the ordering direction """ - key_list = [] - for key in keys: - if not key: - continue - direction = pymongo.ASCENDING - if key[0] == '-': - direction = pymongo.DESCENDING - if key[0] in ('-', '+'): - key = key[1:] - key = key.replace('__', '.') - try: - key = self._document._translate_field_name(key) - except: - pass - key_list.append((key, direction)) - - self._ordering = key_list - - return self + queryset = self.clone() + queryset._ordering = self._get_order_by(keys) + return queryset def explain(self, format=False): """Return an explain plan record for the @@ -740,7 +692,6 @@ class QuerySet(object): :param format: format the plan before returning it """ - plan = self._cursor.explain() if format: plan = pprint.pformat(plan) @@ -753,8 +704,9 @@ class QuerySet(object): ..versionchanged:: 0.5 - made chainable """ - self._snapshot = enabled - return self + queryset = self.clone() + queryset._snapshot = enabled + return queryset def timeout(self, enabled): """Enable or disable the default mongod timeout when querying. @@ -763,16 +715,18 @@ class QuerySet(object): ..versionchanged:: 0.5 - made chainable """ - self._timeout = enabled - return self + queryset = self.clone() + queryset._timeout = enabled + return queryset def slave_okay(self, enabled): """Enable or disable the slave_okay when querying. :param enabled: whether or not the slave_okay is enabled """ - self._slave_okay = enabled - return self + queryset = self.clone() + queryset._slave_okay = enabled + return queryset def read_preference(self, read_preference): """Change the read_preference when querying. @@ -781,170 +735,9 @@ class QuerySet(object): preference. """ validate_read_preference('read_preference', read_preference) - self._read_preference = read_preference - return self - - def delete(self, safe=False): - """Delete the documents matched by the query. - - :param safe: check if the operation succeeded before returning - """ - doc = self._document - - # Handle deletes where skips or limits have been applied - if self._skip or self._limit: - for doc in self: - doc.delete() - return - - delete_rules = doc._meta.get('delete_rules') or {} - # Check for DENY rules before actually deleting/nullifying any other - # references - for rule_entry in delete_rules: - document_cls, field_name = rule_entry - rule = doc._meta['delete_rules'][rule_entry] - if rule == DENY and document_cls.objects( - **{field_name + '__in': self}).count() > 0: - msg = ("Could not delete document (%s.%s refers to it)" - % (document_cls.__name__, field_name)) - raise OperationError(msg) - - for rule_entry in delete_rules: - document_cls, field_name = rule_entry - rule = doc._meta['delete_rules'][rule_entry] - if rule == CASCADE: - ref_q = document_cls.objects(**{field_name + '__in': self}) - ref_q_count = ref_q.count() - if (doc != document_cls and ref_q_count > 0 - or (doc == document_cls and ref_q_count > 0)): - ref_q.delete(safe=safe) - elif rule == NULLIFY: - document_cls.objects(**{field_name + '__in': self}).update( - safe_update=safe, - **{'unset__%s' % field_name: 1}) - elif rule == PULL: - document_cls.objects(**{field_name + '__in': self}).update( - safe_update=safe, - **{'pull_all__%s' % field_name: self}) - - self._collection.remove(self._query, safe=safe) - - def update(self, safe_update=True, upsert=False, multi=True, - write_options=None, **update): - """Perform an atomic update on the fields matched by the query. When - ``safe_update`` is used, the number of affected documents is returned. - - :param safe_update: check if the operation succeeded before returning - :param upsert: Any existing document with that "_id" is overwritten. - :param write_options: extra keyword arguments for - :meth:`~pymongo.collection.Collection.update` - - .. versionadded:: 0.2 - """ - if not update: - raise OperationError("No update parameters, would remove data") - - if not write_options: - write_options = {} - - query = self._query - update = transform.update(self._document, **update) - - # If doing an atomic upsert on an inheritable class - # then ensure we add _cls to the update operation - if upsert and '_cls' in query: - if '$set' in update: - update["$set"]["_cls"] = self._document._class_name - else: - update["$set"] = {"_cls": self._document._class_name} - - try: - ret = self._collection.update(query, update, multi=multi, - upsert=upsert, safe=safe_update, - **write_options) - if ret is not None and 'n' in ret: - return ret['n'] - except pymongo.errors.OperationFailure, err: - if unicode(err) == u'multi not coded yet': - message = u'update() method requires MongoDB 1.1.3+' - raise OperationError(message) - raise OperationError(u'Update failed (%s)' % unicode(err)) - - def update_one(self, safe_update=True, upsert=False, write_options=None, - **update): - """Perform an atomic update on first field matched by the query. When - ``safe_update`` is used, the number of affected documents is returned. - - :param safe_update: check if the operation succeeded before returning - :param upsert: Any existing document with that "_id" is overwritten. - :param write_options: extra keyword arguments for - :meth:`~pymongo.collection.Collection.update` - :param update: Django-style update keyword arguments - - .. versionadded:: 0.2 - """ - return self.update(safe_update=True, upsert=upsert, multi=False, - write_options=None, **update) - - def __iter__(self): - self.rewind() - return self - - def _get_scalar(self, doc): - - def lookup(obj, name): - chunks = name.split('__') - for chunk in chunks: - obj = getattr(obj, chunk) - return obj - - data = [lookup(doc, n) for n in self._scalar] - if len(data) == 1: - return data[0] - - return tuple(data) - - def _get_as_pymongo(self, row): - # Extract which fields paths we should follow if .fields(...) was - # used. If not, handle all fields. - if not getattr(self, '__as_pymongo_fields', None): - self.__as_pymongo_fields = [] - for field in self._loaded_fields.fields - set(['_cls', '_id', '_types']): - self.__as_pymongo_fields.append(field) - while '.' in field: - field, _ = field.rsplit('.', 1) - self.__as_pymongo_fields.append(field) - - all_fields = not self.__as_pymongo_fields - - def clean(data, path=None): - path = path or '' - - if isinstance(data, dict): - new_data = {} - for key, value in data.iteritems(): - new_path = '%s.%s' % (path, key) if path else key - if all_fields or new_path in self.__as_pymongo_fields: - new_data[key] = clean(value, path=new_path) - data = new_data - elif isinstance(data, list): - data = [clean(d, path=path) for d in data] - else: - if self._as_pymongo_coerce: - # If we need to coerce types, we need to determine the - # type of this field and use the corresponding .to_python(...) - from mongoengine.fields import EmbeddedDocumentField - obj = self._document - for chunk in path.split('.'): - obj = getattr(obj, chunk, None) - if obj is None: - break - elif isinstance(obj, EmbeddedDocumentField): - obj = obj.document_type - if obj and data is not None: - data = obj.to_python(data) - return data - return clean(row) + queryset = self.clone() + queryset._read_preference = read_preference + return queryset def scalar(self, *fields): """Instead of returning Document instances, return either a specific @@ -955,14 +748,15 @@ class QuerySet(object): :param fields: One or more fields to return instead of a Document. """ - self._scalar = list(fields) + queryset = self.clone() + queryset._scalar = list(fields) if fields: - self.only(*fields) + queryset = queryset.only(*fields) else: - self.all_fields() + queryset = queryset.all_fields() - return self + return queryset def values_list(self, *fields): """An alias for scalar""" @@ -972,36 +766,122 @@ class QuerySet(object): """Instead of returning Document instances, return raw values from pymongo. - :param coerce_type: Field types (if applicable) would be use to coerce types. + :param coerce_type: Field types (if applicable) would be use to + coerce types. """ - self._as_pymongo = True - self._as_pymongo_coerce = coerce_types - return self + queryset = self.clone() + queryset._as_pymongo = True + queryset._as_pymongo_coerce = coerce_types + return queryset - def _sub_js_fields(self, code): - """When fields are specified with [~fieldname] syntax, where - *fieldname* is the Python name of a field, *fieldname* will be - substituted for the MongoDB name of the field (specified using the - :attr:`name` keyword argument in a field's constructor). + # JSON Helpers + + def to_json(self): + """Converts a queryset to JSON""" + queryset = self.clone() + return json_util.dumps(queryset._collection_obj.find(queryset._query)) + + def from_json(self, json_data): + """Converts json data to unsaved objects""" + son_data = json_util.loads(json_data) + return [self._document._from_son(data) for data in son_data] + + # JS functionality + + def map_reduce(self, map_f, reduce_f, output, finalize_f=None, limit=None, + scope=None): + """Perform a map/reduce query using the current query spec + and ordering. While ``map_reduce`` respects ``QuerySet`` chaining, + it must be the last call made, as it does not return a maleable + ``QuerySet``. + + See the :meth:`~mongoengine.tests.QuerySetTest.test_map_reduce` + and :meth:`~mongoengine.tests.QuerySetTest.test_map_advanced` + tests in ``tests.queryset.QuerySetTest`` for usage examples. + + :param map_f: map function, as :class:`~bson.code.Code` or string + :param reduce_f: reduce function, as + :class:`~bson.code.Code` or string + :param output: output collection name, if set to 'inline' will try to + use :class:`~pymongo.collection.Collection.inline_map_reduce` + This can also be a dictionary containing output options + see: http://docs.mongodb.org/manual/reference/commands/#mapReduce + :param finalize_f: finalize function, an optional function that + performs any post-reduction processing. + :param scope: values to insert into map/reduce global scope. Optional. + :param limit: number of objects from current query to provide + to map/reduce method + + Returns an iterator yielding + :class:`~mongoengine.document.MapReduceDocument`. + + .. note:: + + Map/Reduce changed in server version **>= 1.7.4**. The PyMongo + :meth:`~pymongo.collection.Collection.map_reduce` helper requires + PyMongo version **>= 1.11**. + + .. versionchanged:: 0.5 + - removed ``keep_temp`` keyword argument, which was only relevant + for MongoDB server versions older than 1.7.4 + + .. versionadded:: 0.3 """ - def field_sub(match): - # Extract just the field name, and look up the field objects - field_name = match.group(1).split('.') - fields = self._document._lookup_field(field_name) - # Substitute the correct name for the field into the javascript - return u'["%s"]' % fields[-1].db_field + queryset = self.clone() - def field_path_sub(match): - # Extract just the field name, and look up the field objects - field_name = match.group(1).split('.') - fields = self._document._lookup_field(field_name) - # Substitute the correct name for the field into the javascript - return ".".join([f.db_field for f in fields]) + MapReduceDocument = _import_class('MapReduceDocument') - code = re.sub(u'\[\s*~([A-z_][A-z_0-9.]+?)\s*\]', field_sub, code) - code = re.sub(u'\{\{\s*~([A-z_][A-z_0-9.]+?)\s*\}\}', field_path_sub, - code) - return code + if not hasattr(self._collection, "map_reduce"): + raise NotImplementedError("Requires MongoDB >= 1.7.1") + + map_f_scope = {} + if isinstance(map_f, Code): + map_f_scope = map_f.scope + map_f = unicode(map_f) + map_f = Code(queryset._sub_js_fields(map_f), map_f_scope) + + reduce_f_scope = {} + if isinstance(reduce_f, Code): + reduce_f_scope = reduce_f.scope + reduce_f = unicode(reduce_f) + reduce_f_code = queryset._sub_js_fields(reduce_f) + reduce_f = Code(reduce_f_code, reduce_f_scope) + + mr_args = {'query': queryset._query} + + if finalize_f: + finalize_f_scope = {} + if isinstance(finalize_f, Code): + finalize_f_scope = finalize_f.scope + finalize_f = unicode(finalize_f) + finalize_f_code = queryset._sub_js_fields(finalize_f) + finalize_f = Code(finalize_f_code, finalize_f_scope) + mr_args['finalize'] = finalize_f + + if scope: + mr_args['scope'] = scope + + if limit: + mr_args['limit'] = limit + + if output == 'inline' and not queryset._ordering: + map_reduce_function = 'inline_map_reduce' + else: + map_reduce_function = 'map_reduce' + mr_args['out'] = output + + results = getattr(queryset._collection, map_reduce_function)( + map_f, reduce_f, **mr_args) + + if map_reduce_function == 'map_reduce': + results = results.find() + + if queryset._ordering: + results = results.sort(queryset._ordering) + + for doc in results: + yield MapReduceDocument(queryset._document, queryset._collection, + doc['_id'], doc['value']) def exec_js(self, code, *fields, **options): """Execute a Javascript function on the server. A list of fields may be @@ -1025,24 +905,26 @@ class QuerySet(object): :param options: options that you want available to the function (accessed in Javascript through the ``options`` object) """ - code = self._sub_js_fields(code) + queryset = self.clone() - fields = [self._document._translate_field_name(f) for f in fields] - collection = self._document._get_collection_name() + code = queryset._sub_js_fields(code) + + fields = [queryset._document._translate_field_name(f) for f in fields] + collection = queryset._document._get_collection_name() scope = { 'collection': collection, 'options': options or {}, } - query = self._query - if self._where_clause: - query['$where'] = self._where_clause + query = queryset._query + if queryset._where_clause: + query['$where'] = queryset._where_clause scope['query'] = query code = Code(code, scope=scope) - db = self._document._get_db() + db = queryset._document._get_db() return db.eval(code, *fields) def where(self, where_clause): @@ -1056,9 +938,10 @@ class QuerySet(object): .. versionadded:: 0.5 """ - where_clause = self._sub_js_fields(where_clause) - self._where_clause = where_clause - return self + queryset = self.clone() + where_clause = queryset._sub_js_fields(where_clause) + queryset._where_clause = where_clause + return queryset def sum(self, field): """Sum over the values of the specified field. @@ -1157,6 +1040,101 @@ class QuerySet(object): normalize=normalize) return self._item_frequencies_exec_js(field, normalize=normalize) + # Iterator helpers + + def next(self): + """Wrap the result in a :class:`~mongoengine.Document` object. + """ + self._iter = True + try: + if self._limit == 0 or self._none: + raise StopIteration + if self._scalar: + return self._get_scalar(self._document._from_son( + self._cursor.next())) + if self._as_pymongo: + return self._get_as_pymongo(self._cursor.next()) + + return self._document._from_son(self._cursor.next()) + except StopIteration, e: + self.rewind() + raise e + + def rewind(self): + """Rewind the cursor to its unevaluated state. + + .. versionadded:: 0.3 + """ + self._iter = False + self._cursor.rewind() + + # Properties + + @property + def _collection(self): + """Property that returns the collection object. This allows us to + perform operations only if the collection is accessed. + """ + return self._collection_obj + + @property + def _cursor_args(self): + cursor_args = { + 'snapshot': self._snapshot, + 'timeout': self._timeout, + 'slave_okay': self._slave_okay, + } + if self._read_preference is not None: + cursor_args['read_preference'] = self._read_preference + if self._loaded_fields: + cursor_args['fields'] = self._loaded_fields.as_dict() + return cursor_args + + @property + def _cursor(self): + if self._cursor_obj is None: + + self._cursor_obj = self._collection.find(self._query, + **self._cursor_args) + # Apply where clauses to cursor + if self._where_clause: + where_clause = self._sub_js_fields(self._where_clause) + self._cursor_obj.where(where_clause) + + if self._ordering: + # Apply query ordering + self._cursor_obj.sort(self._ordering) + elif self._document._meta['ordering']: + # Otherwise, apply the ordering from the document model + order = self._get_order_by(self._document._meta['ordering']) + self._cursor_obj.sort(order) + + if self._limit is not None: + self._cursor_obj.limit(self._limit - (self._skip or 0)) + + if self._skip is not None: + self._cursor_obj.skip(self._skip) + + if self._hint != -1: + self._cursor_obj.hint(self._hint) + return self._cursor_obj + + @property + def _query(self): + if self._mongo_query is None: + self._mongo_query = self._query_obj.to_query(self._document) + if self._class_check: + self._mongo_query.update(self._initial_query) + return self._mongo_query + + @property + def _dereference(self): + if not self.__dereference: + self.__dereference = _import_class('DeReference')() + return self.__dereference + + # Helper Functions + def _item_frequencies_map_reduce(self, field, normalize=False): map_func = """ function() { @@ -1269,48 +1247,130 @@ class QuerySet(object): return frequencies - def __repr__(self): - """Provides the string representation of the QuerySet + def _fields_to_dbfields(self, fields): + """Translate fields paths to its db equivalents""" + ret = [] + for field in fields: + field = ".".join(f.db_field for f in + self._document._lookup_field(field.split('.'))) + ret.append(field) + return ret - .. versionchanged:: 0.6.13 Now doesnt modify the cursor + def _get_order_by(self, keys): + """Creates a list of order by fields """ - - if self._iter: - return '.. queryset mid-iteration ..' - - data = [] - for i in xrange(REPR_OUTPUT_SIZE + 1): + key_list = [] + for key in keys: + if not key: + continue + direction = pymongo.ASCENDING + if key[0] == '-': + direction = pymongo.DESCENDING + if key[0] in ('-', '+'): + key = key[1:] + key = key.replace('__', '.') try: - data.append(self.next()) - except StopIteration: - break - if len(data) > REPR_OUTPUT_SIZE: - data[-1] = "...(remaining elements truncated)..." + key = self._document._translate_field_name(key) + except: + pass + key_list.append((key, direction)) + return key_list - self.rewind() - return repr(data) + def _get_scalar(self, doc): - def select_related(self, max_depth=1): - """Handles dereferencing of :class:`~bson.dbref.DBRef` objects to - a maximum depth in order to cut down the number queries to mongodb. + def lookup(obj, name): + chunks = name.split('__') + for chunk in chunks: + obj = getattr(obj, chunk) + return obj - .. versionadded:: 0.5 + data = [lookup(doc, n) for n in self._scalar] + if len(data) == 1: + return data[0] + + return tuple(data) + + def _get_as_pymongo(self, row): + # Extract which fields paths we should follow if .fields(...) was + # used. If not, handle all fields. + if not getattr(self, '__as_pymongo_fields', None): + self.__as_pymongo_fields = [] + for field in self._loaded_fields.fields - set(['_cls', '_id']): + self.__as_pymongo_fields.append(field) + while '.' in field: + field, _ = field.rsplit('.', 1) + self.__as_pymongo_fields.append(field) + + all_fields = not self.__as_pymongo_fields + + def clean(data, path=None): + path = path or '' + + if isinstance(data, dict): + new_data = {} + for key, value in data.iteritems(): + new_path = '%s.%s' % (path, key) if path else key + if all_fields or new_path in self.__as_pymongo_fields: + new_data[key] = clean(value, path=new_path) + data = new_data + elif isinstance(data, list): + data = [clean(d, path=path) for d in data] + else: + if self._as_pymongo_coerce: + # If we need to coerce types, we need to determine the + # type of this field and use the corresponding + # .to_python(...) + from mongoengine.fields import EmbeddedDocumentField + obj = self._document + for chunk in path.split('.'): + obj = getattr(obj, chunk, None) + if obj is None: + break + elif isinstance(obj, EmbeddedDocumentField): + obj = obj.document_type + if obj and data is not None: + data = obj.to_python(data) + return data + return clean(row) + + def _sub_js_fields(self, code): + """When fields are specified with [~fieldname] syntax, where + *fieldname* is the Python name of a field, *fieldname* will be + substituted for the MongoDB name of the field (specified using the + :attr:`name` keyword argument in a field's constructor). """ - # Make select related work the same for querysets - max_depth += 1 - return self._dereference(self, max_depth=max_depth) + def field_sub(match): + # Extract just the field name, and look up the field objects + field_name = match.group(1).split('.') + fields = self._document._lookup_field(field_name) + # Substitute the correct name for the field into the javascript + return u'["%s"]' % fields[-1].db_field - def to_json(self): - """Converts a queryset to JSON""" - return json_util.dumps(self._collection_obj.find(self._query)) + def field_path_sub(match): + # Extract just the field name, and look up the field objects + field_name = match.group(1).split('.') + fields = self._document._lookup_field(field_name) + # Substitute the correct name for the field into the javascript + return ".".join([f.db_field for f in fields]) - def from_json(self, json_data): - """Converts json data to unsaved objects""" - son_data = json_util.loads(json_data) - return [self._document._from_son(data) for data in son_data] + code = re.sub(u'\[\s*~([A-z_][A-z_0-9.]+?)\s*\]', field_sub, code) + code = re.sub(u'\{\{\s*~([A-z_][A-z_0-9.]+?)\s*\}\}', field_path_sub, + code) + return code - @property - def _dereference(self): - if not self.__dereference: - self.__dereference = _import_class('DeReference')() - return self.__dereference + # Deprecated + + def ensure_index(self, **kwargs): + """Deprecated use :func:`~Document.ensure_index`""" + msg = ("Doc.objects()._ensure_index() is deprecated. " + "Use Doc.ensure_index() instead.") + warnings.warn(msg, DeprecationWarning) + self._document.__class__.ensure_index(**kwargs) + return self + + def _ensure_indexes(self): + """Deprecated use :func:`~Document.ensure_indexes`""" + msg = ("Doc.objects()._ensure_indexes() is deprecated. " + "Use Doc.ensure_indexes() instead.") + warnings.warn(msg, DeprecationWarning) + self._document.__class__.ensure_indexes() diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index bad3d36..bf64a56 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -713,19 +713,19 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(p._cursor_args, {'snapshot': False, 'slave_okay': False, 'timeout': True}) - p.snapshot(False).slave_okay(False).timeout(False) + p = p.snapshot(False).slave_okay(False).timeout(False) self.assertEqual(p._cursor_args, {'snapshot': False, 'slave_okay': False, 'timeout': False}) - p.snapshot(True).slave_okay(False).timeout(False) + p = p.snapshot(True).slave_okay(False).timeout(False) self.assertEqual(p._cursor_args, {'snapshot': True, 'slave_okay': False, 'timeout': False}) - p.snapshot(True).slave_okay(True).timeout(False) + p = p.snapshot(True).slave_okay(True).timeout(False) self.assertEqual(p._cursor_args, {'snapshot': True, 'slave_okay': True, 'timeout': False}) - p.snapshot(True).slave_okay(True).timeout(True) + p = p.snapshot(True).slave_okay(True).timeout(True) self.assertEqual(p._cursor_args, {'snapshot': True, 'slave_okay': True, 'timeout': True}) @@ -773,7 +773,8 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(len(docs), 1000) # Limit and skip - self.assertEqual('[, , ]', "%s" % docs[1:4]) + docs = docs[1:4] + self.assertEqual('[, , ]', "%s" % docs) self.assertEqual(docs.count(), 3) self.assertEqual(len(docs), 3) diff --git a/tests/queryset/visitor.py b/tests/queryset/visitor.py index 4af39e8..98815db 100644 --- a/tests/queryset/visitor.py +++ b/tests/queryset/visitor.py @@ -202,8 +202,8 @@ class QTest(unittest.TestCase): self.assertEqual(test2.count(), 3) self.assertFalse(test2 == test) - test2.filter(x=6) - self.assertEqual(test2.count(), 1) + test3 = test2.filter(x=6) + self.assertEqual(test3.count(), 1) self.assertEqual(test.count(), 3) def test_q(self): From e537369d9885df724537974e9c44710110cfe9e7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 4 Jan 2013 14:02:34 +0000 Subject: [PATCH 0901/1279] Trying to get travis to build the 0.8 branch --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c7cc271..806d8b4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,4 +26,4 @@ notifications: branches: only: - master - - 0.8 \ No newline at end of file + - "0.8" From e6ac8cab53a3269d2dee91d529f646d59f1b98c8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 4 Jan 2013 14:28:42 +0000 Subject: [PATCH 0902/1279] Fixing python 2.5 support --- mongoengine/queryset/queryset.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 239975f..e637370 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -516,12 +516,15 @@ class QuerySet(object): '_where_clause', '_loaded_fields', '_ordering', '_snapshot', '_timeout', '_class_check', '_slave_okay', '_read_preference', '_iter', '_scalar', '_as_pymongo', '_as_pymongo_coerce', - '_limit', '_skip', '_slice', '_hint') + '_limit', '_skip', '_hint') for prop in copy_props: val = getattr(self, prop) setattr(c, prop, copy.copy(val)) + if self._slice: + c._slice = self._slice + if self._cursor_obj: c._cursor_obj = self._cursor_obj.clone() From d9ed33d1b191f075a5370a7f9f199205496ea953 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 4 Jan 2013 14:33:08 +0000 Subject: [PATCH 0903/1279] Added python 3.3 support to travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 806d8b4..c5f3961 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ python: - 2.7 - 3.1 - 3.2 + - 3.3 env: - PYMONGO=dev - PYMONGO=2.4.1 From 85173d188ba8038cf6713e94d25c43e53e49db1d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 4 Jan 2013 15:57:08 +0000 Subject: [PATCH 0904/1279] Add simplejson to python 2.5 build --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c5f3961..bf34bab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo ln -s /usr/lib/i386-linux-gnu/libz.so /usr/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install PIL --use-mirrors ; true; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install PIL --use-mirrors ; true; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then pip install simplejson --use-mirrors ; true; fi - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - python setup.py install From 0f9e4ef352fa14b52ddc37b3bf9f17f03f8f0ab5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 4 Jan 2013 16:27:58 +0000 Subject: [PATCH 0905/1279] Add mongoengine.png asset in the build --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 35ca579..4b42e71 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ if sys.version_info[0] == 3: extra_opts['packages'] = find_packages(exclude=('tests',)) if "test" in sys.argv or "nosetests" in sys.argv: extra_opts['packages'].append("tests") - extra_opts['package_data'] = {"tests": ["mongoengine.png"]} + extra_opts['package_data'] = {"tests": ["fields/mongoengine.png"]} else: extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.3', 'PIL'] extra_opts['packages'] = find_packages(exclude=('tests',)) From 5c45eee817a680851f990e0ca0e1fc26c5cb6e9c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 4 Jan 2013 16:28:26 +0000 Subject: [PATCH 0906/1279] Whitespace --- tests/document/instance.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index 0054480..c8a1b11 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -654,8 +654,7 @@ class InstanceTest(unittest.TestCase): Foo.drop_collection() - a = Foo(name='hello') - a.save() + a = Foo(name='hello').save() a.bar = a with open(TEST_IMAGE_PATH, 'rb') as test_image: @@ -665,7 +664,7 @@ class InstanceTest(unittest.TestCase): # Confirm can save and it resets the changed fields without hitting # max recursion error b = Foo.objects.with_id(a.id) - b.name='world' + b.name = 'world' b.save() self.assertEqual(b.picture, b.bar.picture, b.bar.bar.picture) From 7bb9c7d47f18cdee5c68eceb7cbe6792a962dbae Mon Sep 17 00:00:00 2001 From: Nick Joyce Date: Mon, 7 Jan 2013 14:49:39 +0000 Subject: [PATCH 0907/1279] Ensure that the update actions are grouped rather than serial. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a performance update. When multiple properties of the same entity have been deleted and modified, 2 calls to update the entity are made, one {"$set": … } and another {"$unset": … }. This is 2 network interface calls which is a performance killer (even at lan speeds). Fixes: #210 --- mongoengine/document.py | 11 ++++++++--- tests/test_document.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 7b3afaf..2bd1d22 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -234,11 +234,16 @@ class Document(BaseDocument): select_dict[actual_key] = doc[actual_key] upsert = self._created + work = {} + if updates: - collection.update(select_dict, {"$set": updates}, - upsert=upsert, safe=safe, **write_options) + work["$set"] = updates + if removals: - collection.update(select_dict, {"$unset": removals}, + work["$unset"] = removals + + if work: + collection.update(select_dict, work, upsert=upsert, safe=safe, **write_options) warn_cascade = not cascade and 'cascade' not in self._meta diff --git a/tests/test_document.py b/tests/test_document.py index cd0ab8f..fa6eb28 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -3515,6 +3515,36 @@ class ValidatorErrorTest(unittest.TestCase): self.assertRaises(OperationError, change_shard_key) + def test_set_unset_one_operation(self): + """Ensure that $set and $unset actions are performed in the same + operation. + """ + class FooBar(Document): + meta = { + 'collection': 'foobar', + } + + foo = StringField(default=None) + bar = StringField(default=None) + + FooBar.drop_collection() + + # write an entity with a single prop + foo = FooBar(foo='foo') + foo.save() + + self.assertEqual(foo.foo, 'foo') + + # set foo to the default causing mongoengine to $unset foo. + foo.foo = None + foo.bar = 'bar' + + foo.save() + foo.reload() + + self.assertIsNone(foo.foo) + self.assertEqual(foo.bar, 'bar') + if __name__ == '__main__': unittest.main() From 50905ab459e729d2136a47c0e7be0c9d10daef14 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 Jan 2013 08:41:03 +0000 Subject: [PATCH 0908/1279] Test update --- tests/test_dereference.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 41f8aeb..8557ec5 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -198,6 +198,10 @@ class FieldTest(unittest.TestCase): raw_data = Group._get_collection().find_one() self.assertTrue(isinstance(raw_data['author'], DBRef)) self.assertTrue(isinstance(raw_data['members'][0], DBRef)) + group = Group.objects.first() + + self.assertEqual(group.author, user) + self.assertEqual(group.members, [user]) # Migrate the model definition class Group(Document): From 8e038dd5634d85a57ebb3560cd735aff0eaa44ed Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 Jan 2013 08:51:45 +0000 Subject: [PATCH 0909/1279] Updated travis python-3.1 no longer supported --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 806d8b4..550cc6c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,11 +2,11 @@ language: python services: mongodb python: - - 2.5 - - 2.6 - - 2.7 - - 3.1 - - 3.2 + - "2.5" + - "2.6" + - "2.7" + - "3.2" + - "3.3" env: - PYMONGO=dev - PYMONGO=2.4.1 From 06681a453fe18e3bf91912630a979672e489821b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 Jan 2013 13:15:21 +0000 Subject: [PATCH 0910/1279] python 3.3 test fixes --- tests/test_document.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_document.py b/tests/test_document.py index cd0ab8f..9fc79f5 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -924,7 +924,7 @@ class DocumentTest(unittest.TestCase): self.assertEqual(1, Person.objects.count()) info = Person.objects._collection.index_information() - self.assertEqual(info.keys(), ['_types_1_user_guid_1', '_id_', '_types_1_name_1']) + self.assertEqual(sorted(info.keys()), ['_id_', '_types_1_name_1', '_types_1_user_guid_1']) Person.drop_collection() def test_disable_index_creation(self): @@ -968,7 +968,7 @@ class DocumentTest(unittest.TestCase): BlogPost.drop_collection() info = BlogPost.objects._collection.index_information() - self.assertEqual(info.keys(), ['_types_1_date.yr_-1', '_id_']) + self.assertEqual(sorted(info.keys()), [ '_id_', '_types_1_date.yr_-1']) BlogPost.drop_collection() def test_list_embedded_document_index(self): @@ -991,7 +991,8 @@ class DocumentTest(unittest.TestCase): info = BlogPost.objects._collection.index_information() # we don't use _types in with list fields by default - self.assertEqual(info.keys(), ['_id_', '_types_1', 'tags.tag_1']) + self.assertEqual(sorted(info.keys()), + ['_id_', '_types_1', 'tags.tag_1']) post1 = BlogPost(title="Embedded Indexes tests in place", tags=[Tag(name="about"), Tag(name="time")] @@ -1008,7 +1009,7 @@ class DocumentTest(unittest.TestCase): recursive_obj = EmbeddedDocumentField(RecursiveObject) info = RecursiveDocument.objects._collection.index_information() - self.assertEqual(info.keys(), ['_id_', '_types_1']) + self.assertEqual(sorted(info.keys()), ['_id_', '_types_1']) def test_geo_indexes_recursion(self): @@ -2719,7 +2720,7 @@ class DocumentTest(unittest.TestCase): Person.drop_collection() - self.assertEqual(Person._fields.keys(), ['name', 'id']) + self.assertEqual(sorted(Person._fields.keys()), ['id', 'name']) Person(name="Rozza").save() From 72dd9daa23ea4e9adf44df6a3c3ab2499e971ad6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 9 Jan 2013 16:16:48 +0000 Subject: [PATCH 0911/1279] Fixing py3.3 tests --- mongoengine/queryset/transform.py | 2 +- tests/queryset/transform.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 8ee84ee..9fe8780 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -26,7 +26,7 @@ def query(_doc_cls=None, _field_operation=False, **query): """ mongo_query = {} merge_query = defaultdict(list) - for key, value in query.items(): + for key, value in sorted(query.items()): if key == "__raw__": mongo_query.update(value) continue diff --git a/tests/queryset/transform.py b/tests/queryset/transform.py index 666b345..d38cbfd 100644 --- a/tests/queryset/transform.py +++ b/tests/queryset/transform.py @@ -26,7 +26,7 @@ class TransformTest(unittest.TestCase): self.assertEqual(transform.query(age__gt=20, age__lt=50), {'age': {'$gt': 20, '$lt': 50}}) self.assertEqual(transform.query(age=20, age__gt=50), - {'age': 20}) + {'$and': [{'age': {'$gt': 50}}, {'age': 20}]}) self.assertEqual(transform.query(friend__age__gte=30), {'friend.age': {'$gte': 30}}) self.assertEqual(transform.query(name__exists=True), From 87c965edd347138fd071da0cd5fa82a2377f7205 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 10 Jan 2013 11:08:07 +0000 Subject: [PATCH 0912/1279] Fixing PY3.3 test cases --- mongoengine/queryset.py | 12 +++++++++--- tests/test_document.py | 17 +++++++++++------ tests/test_queryset.py | 2 +- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 160201e..bff05fc 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -7,10 +7,11 @@ import operator from collections import defaultdict from functools import partial -from mongoengine.python_support import product, reduce +from mongoengine.python_support import product, reduce, PY3 import pymongo from bson.code import Code +from bson.son import SON from mongoengine import signals @@ -388,7 +389,12 @@ class QuerySet(object): if self._mongo_query is None: self._mongo_query = self._query_obj.to_query(self._document) if self._class_check: - self._mongo_query.update(self._initial_query) + if PY3: + query = SON(self._initial_query.items()) + query.update(self._mongo_query) + self._mongo_query = query + else: + self._mongo_query.update(self._initial_query) return self._mongo_query def ensure_index(self, key_or_list, drop_dups=False, background=False, @@ -704,7 +710,7 @@ class QuerySet(object): mongo_query = {} merge_query = defaultdict(list) - for key, value in query.items(): + for key, value in sorted(query.items()): if key == "__raw__": mongo_query.update(value) continue diff --git a/tests/test_document.py b/tests/test_document.py index 9fc79f5..3e8d813 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -8,6 +8,7 @@ import sys import unittest import uuid import warnings +import operator from nose.plugins.skip import SkipTest from datetime import datetime @@ -452,7 +453,8 @@ class DocumentTest(unittest.TestCase): info = collection.index_information() info = [value['key'] for key, value in info.iteritems()] - self.assertEqual([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], info) + self.assertEqual([[('_id', 1)], [('_types', 1), ('name', 1)]], + sorted(info, key=operator.itemgetter(0))) # Turn off inheritance class Animal(Document): @@ -473,7 +475,8 @@ class DocumentTest(unittest.TestCase): info = collection.index_information() info = [value['key'] for key, value in info.iteritems()] - self.assertEqual([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], info) + self.assertEqual([[(u'_id', 1)], [(u'_types', 1), (u'name', 1)]], + sorted(info, key=operator.itemgetter(0))) info = collection.index_information() indexes_to_drop = [key for key, value in info.iteritems() if '_types' in dict(value['key'])] @@ -482,14 +485,16 @@ class DocumentTest(unittest.TestCase): info = collection.index_information() info = [value['key'] for key, value in info.iteritems()] - self.assertEqual([[(u'_id', 1)]], info) + self.assertEqual([[(u'_id', 1)]], + sorted(info, key=operator.itemgetter(0))) # Recreate indexes dog = Animal.objects.first() dog.save() info = collection.index_information() info = [value['key'] for key, value in info.iteritems()] - self.assertEqual([[(u'_id', 1)], [(u'name', 1),]], info) + self.assertEqual([[(u'_id', 1)], [(u'name', 1),]], + sorted(info, key=operator.itemgetter(0))) Animal.drop_collection() @@ -3412,8 +3417,8 @@ class ValidatorErrorTest(unittest.TestCase): try: User().validate() except ValidationError, e: - expected_error_message = """ValidationError(Field is required: ['username', 'name'])""" - self.assertEqual(e.message, expected_error_message) + expected_error_message = """ValidationError(Field is required""" + self.assertTrue(expected_error_message in e.message) self.assertEqual(e.to_dict(), { 'username': 'Field is required', 'name': 'Field is required'}) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index b09eafb..5234cea 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -47,7 +47,7 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(QuerySet._transform_query(age__gt=20, age__lt=50), {'age': {'$gt': 20, '$lt': 50}}) self.assertEqual(QuerySet._transform_query(age=20, age__gt=50), - {'age': 20}) + {'$and': [{'age': {'$gt': 50}}, {'age': 20}]}) self.assertEqual(QuerySet._transform_query(friend__age__gte=30), {'friend.age': {'$gte': 30}}) self.assertEqual(QuerySet._transform_query(name__exists=True), From e508625935ceee3c8becc387df7a5c396153dc1a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 14 Jan 2013 16:37:40 +0000 Subject: [PATCH 0913/1279] Update docs/upgrade.rst --- docs/upgrade.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 82ac7ca..901c251 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -183,3 +183,9 @@ Alternatively, you can rename your collections eg :: else: print "Upgraded collection names" + +mongodb 1.8 > 2.0 + +=================== + +Its been reported that indexes may need to be recreated to the newer version of indexes. +To do this drop indexes and call ``ensure_indexes`` on each model. From f5d02e1b1015856d013e73d01c7035359f0e9edb Mon Sep 17 00:00:00 2001 From: Martin Alderete Date: Tue, 15 Jan 2013 02:40:15 -0300 Subject: [PATCH 0914/1279] Fixed issue with choices validation when they are simple list/tuple, after model.validate() did not get any error message. Added test to ensure that model.validate() set the correct error messages. --- AUTHORS | 1 + mongoengine/base.py | 2 +- tests/test_fields.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 82a1dfa..ac48d7b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -128,3 +128,4 @@ that much better: * Peter Teichman * Jakub Kot * Jorge Bastida + * Martin Alderete https://github.com/malderete diff --git a/mongoengine/base.py b/mongoengine/base.py index 013afe7..f73af4c 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -252,7 +252,7 @@ class BaseField(object): elif value_to_check not in self.choices: msg = ('Value must be %s of %s' % (err_msg, unicode(self.choices))) - self.error() + self.error(msg) # check validation argument if self.validation is not None: diff --git a/tests/test_fields.py b/tests/test_fields.py index 28af1b2..fa7e396 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -1706,6 +1706,41 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() + + def test_simple_choices_validation_invalid_value(self): + """Ensure that error messages are correct. + """ + SIZES = ('S', 'M', 'L', 'XL', 'XXL') + COLORS = (('R', 'Red'), ('B', 'Blue')) + SIZE_MESSAGE = u"Value must be one of ('S', 'M', 'L', 'XL', 'XXL')" + COLOR_MESSAGE = u"Value must be one of ['R', 'B']" + + class Shirt(Document): + size = StringField(max_length=3, choices=SIZES) + color = StringField(max_length=1, choices=COLORS) + + Shirt.drop_collection() + + shirt = Shirt() + shirt.validate() + + shirt.size = "S" + shirt.color = "R" + shirt.validate() + + shirt.size = "XS" + shirt.color = "G" + + try: + shirt.validate() + except ValidationError, error: + # get the validation rules + error_dict = error.to_dict() + self.assertEqual(error_dict['size'], SIZE_MESSAGE) + self.assertEqual(error_dict['color'], COLOR_MESSAGE) + + Shirt.drop_collection() + def test_file_fields(self): """Ensure that file fields can be written to and their data retrieved """ From 17eeeb75366ab4625ee5c1a11089aaf9c21fee18 Mon Sep 17 00:00:00 2001 From: lcya86 Date: Thu, 17 Jan 2013 15:31:27 +0800 Subject: [PATCH 0915/1279] Update mongoengine/django/sessions.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit added the "get_decoded" method to the MongoSession class --- mongoengine/django/sessions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index 810b626..8a35e89 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -34,6 +34,9 @@ class MongoSession(Document): meta = {'collection': MONGOENGINE_SESSION_COLLECTION, 'db_alias': MONGOENGINE_SESSION_DB_ALIAS, 'allow_inheritance': False} + + def get_decoded(self): + return SessionStore().decode(self.session_data) class SessionStore(SessionBase): From 2c7b12c022130727c25c3c0686a53a3762639f8e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Jan 2013 13:31:53 +0000 Subject: [PATCH 0916/1279] Added support for $maxDistance (#179) --- docs/changelog.rst | 1 + docs/guide/querying.rst | 2 ++ mongoengine/queryset/transform.py | 5 ++++- tests/queryset/queryset.py | 10 ++++++++++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4fd3e14..3e8f782 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -27,6 +27,7 @@ Changes in 0.8.X - Simplified Q objects, removed QueryTreeTransformerVisitor (#98) (#171) - FileFields now copyable (#198) - Querysets now return clones and are no longer edit in place (#56) +- Added support for $maxDistance (#179) Changes in 0.7.9 ================ diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index d582943..40e36e3 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -92,6 +92,8 @@ may used with :class:`~mongoengine.GeoPointField`\ s: * ``within_polygon`` -- filter documents to those within a given polygon (e.g. [(41.91,-87.69), (41.92,-87.68), (41.91,-87.65), (41.89,-87.65)]). .. note:: Requires Mongo Server 2.0 +* ``max_distance`` -- can be added to your location queries to set a maximum +distance. Querying lists diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 9fe8780..5707cec 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -9,7 +9,8 @@ __all__ = ('query', 'update') COMPARISON_OPERATORS = ('ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', 'all', 'size', 'exists', 'not') GEO_OPERATORS = ('within_distance', 'within_spherical_distance', - 'within_box', 'within_polygon', 'near', 'near_sphere') + 'within_box', 'within_polygon', 'near', 'near_sphere', + 'max_distance') STRING_OPERATORS = ('contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith', 'exact', 'iexact') @@ -97,6 +98,8 @@ def query(_doc_cls=None, _field_operation=False, **query): value = {'$nearSphere': value} elif op == 'within_box': value = {'$within': {'$box': value}} + elif op == "max_distance": + value = {'$maxDistance': value} else: raise NotImplementedError("Geo method '%s' has not " "been implemented" % op) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index bf64a56..b5b0b28 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -2238,6 +2238,12 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(events.count(), 3) self.assertEqual(list(events), [event3, event1, event2]) + # find events within 10 degrees of san francisco + point = [37.7566023, -122.415579] + events = Event.objects(location__near=point, location__max_distance=10) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0], event2) + # find events within 10 degrees of san francisco point_and_distance = [[37.7566023, -122.415579], 10] events = Event.objects(location__within_distance=point_and_distance) @@ -2317,6 +2323,10 @@ class QuerySetTest(unittest.TestCase): ); self.assertEqual(points.count(), 2) + points = Point.objects(location__near_sphere=[-122, 37.5], + location__max_distance=60 / earth_radius); + self.assertEqual(points.count(), 2) + # Finds both points, but orders the north point first because it's # closer to the reference point to the north. points = Point.objects(location__near_sphere=[-122, 38.5]) From 3ba58ebaae51a708ccf02bd3bd7f6e75e6ce3258 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Jan 2013 13:39:53 +0000 Subject: [PATCH 0917/1279] Added Nicolas Trippar to AUTHORS #179 --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 989fd68..7c2f8c8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -133,4 +133,5 @@ that much better: * Pete Campton * Martyn Smith * Marcelo Anton - * Aleksey Porfirov \ No newline at end of file + * Aleksey Porfirov + * Nicolas Trippar \ No newline at end of file From 344dc64df85cbcbcfccaad4cbce7351465170c43 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Jan 2013 14:05:06 +0000 Subject: [PATCH 0918/1279] Updated authors and changelog #163 --- AUTHORS | 3 ++- docs/changelog.rst | 1 + mongoengine/document.py | 7 +++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/AUTHORS b/AUTHORS index 7c2f8c8..aa7f833 100644 --- a/AUTHORS +++ b/AUTHORS @@ -134,4 +134,5 @@ that much better: * Martyn Smith * Marcelo Anton * Aleksey Porfirov - * Nicolas Trippar \ No newline at end of file + * Nicolas Trippar + * Manuel Hermann \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 3e8f782..1905a9d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -28,6 +28,7 @@ Changes in 0.8.X - FileFields now copyable (#198) - Querysets now return clones and are no longer edit in place (#56) - Added support for $maxDistance (#179) +- Uses getlasterror to test created on updated saves (#163) Changes in 0.7.9 ================ diff --git a/mongoengine/document.py b/mongoengine/document.py index 03a838b..69d4d40 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -219,11 +219,11 @@ class Document(BaseDocument): doc = self.to_mongo() - find_delta = ('_id' not in doc or self._created or force_insert) + created = ('_id' not in doc or self._created or force_insert) try: collection = self.__class__.objects._collection - if find_delta: + if created: if force_insert: object_id = collection.insert(doc, safe=safe, **write_options) @@ -289,8 +289,7 @@ class Document(BaseDocument): self._changed_fields = [] self._created = False - signals.post_save.send(self.__class__, document=self, - created=find_delta) + signals.post_save.send(self.__class__, document=self, created=created) return self def cascade_save(self, warn_cascade=None, *args, **kwargs): From 692f00864d982a8c54f08bcda3712b43d9708751 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Jan 2013 15:16:58 +0000 Subject: [PATCH 0919/1279] Fixed inheritance and unique index creation (#140) --- docs/changelog.rst | 1 + mongoengine/base/document.py | 79 +++++++++++++++++++++++---------- mongoengine/base/metaclasses.py | 5 +-- mongoengine/document.py | 17 ++----- tests/document/indexes.py | 9 ++-- 5 files changed, 65 insertions(+), 46 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1905a9d..cb0ac6c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -29,6 +29,7 @@ Changes in 0.8.X - Querysets now return clones and are no longer edit in place (#56) - Added support for $maxDistance (#179) - Uses getlasterror to test created on updated saves (#163) +- Fixed inheritance and unique index creation (#140) Changes in 0.7.9 ================ diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 93bde8e..9f40061 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -509,6 +509,34 @@ class BaseDocument(object): obj._created = False return obj + @classmethod + def _build_index_specs(cls, meta_indexes): + """Generate and merge the full index specs + """ + + geo_indices = cls._geo_indices() + unique_indices = cls._unique_with_indexes() + index_specs = [cls._build_index_spec(spec) + for spec in meta_indexes] + + def merge_index_specs(index_specs, indices): + if not indices: + return index_specs + + spec_fields = [v['fields'] + for k, v in enumerate(index_specs)] + # Merge unqiue_indexes with existing specs + for k, v in enumerate(indices): + if v['fields'] in spec_fields: + index_specs[spec_fields.index(v['fields'])].update(v) + else: + index_specs.append(v) + return index_specs + + index_specs = merge_index_specs(index_specs, geo_indices) + index_specs = merge_index_specs(index_specs, unique_indices) + return index_specs + @classmethod def _build_index_spec(cls, spec): """Build a PyMongo index spec from a MongoEngine index spec. @@ -576,6 +604,7 @@ class BaseDocument(object): """ unique_indexes = [] for field_name, field in cls._fields.items(): + sparse = False # Generate a list of indexes needed by uniqueness constraints if field.unique: field.required = True @@ -596,11 +625,14 @@ class BaseDocument(object): unique_with.append('.'.join(name_parts)) # Unique field should be required parts[-1].required = True + sparse = (not sparse and + parts[-1].name not in cls.__dict__) unique_fields += unique_with # Add the new index to the list - index = [("%s%s" % (namespace, f), pymongo.ASCENDING) + fields = [("%s%s" % (namespace, f), pymongo.ASCENDING) for f in unique_fields] + index = {'fields': fields, 'unique': True, 'sparse': sparse} unique_indexes.append(index) # Grab any embedded document field unique indexes @@ -612,6 +644,29 @@ class BaseDocument(object): return unique_indexes + @classmethod + def _geo_indices(cls, inspected=None): + inspected = inspected or [] + geo_indices = [] + inspected.append(cls) + + EmbeddedDocumentField = _import_class("EmbeddedDocumentField") + GeoPointField = _import_class("GeoPointField") + + for field in cls._fields.values(): + if not isinstance(field, (EmbeddedDocumentField, GeoPointField)): + continue + if hasattr(field, 'document_type'): + field_cls = field.document_type + if field_cls in inspected: + continue + if hasattr(field_cls, '_geo_indices'): + geo_indices += field_cls._geo_indices(inspected) + elif field._geo_index: + geo_indices.append({'fields': + [(field.db_field, pymongo.GEO2D)]}) + return geo_indices + @classmethod def _lookup_field(cls, parts): """Lookup a field based on its attribute and return a list containing @@ -671,28 +726,6 @@ class BaseDocument(object): parts = [f.db_field for f in cls._lookup_field(parts)] return '.'.join(parts) - @classmethod - def _geo_indices(cls, inspected=None): - inspected = inspected or [] - geo_indices = [] - inspected.append(cls) - - EmbeddedDocumentField = _import_class("EmbeddedDocumentField") - GeoPointField = _import_class("GeoPointField") - - for field in cls._fields.values(): - if not isinstance(field, (EmbeddedDocumentField, GeoPointField)): - continue - if hasattr(field, 'document_type'): - field_cls = field.document_type - if field_cls in inspected: - continue - if hasattr(field_cls, '_geo_indices'): - geo_indices += field_cls._geo_indices(inspected) - elif field._geo_index: - geo_indices.append(field) - return geo_indices - def __set_field_display(self): """Dynamically set the display value for a field with choices""" for attr_name, field in self._fields.items(): diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index af39e14..2b63bfa 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -329,10 +329,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): meta = new_class._meta # Set index specifications - meta['index_specs'] = [new_class._build_index_spec(spec) - for spec in meta['indexes']] - unique_indexes = new_class._unique_with_indexes() - new_class._meta['unique_indexes'] = unique_indexes + meta['index_specs'] = new_class._build_index_specs(meta['indexes']) # If collection is a callable - call it and set the value collection = meta.get('collection') diff --git a/mongoengine/document.py b/mongoengine/document.py index 69d4d40..fff7efa 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -105,7 +105,7 @@ class Document(BaseDocument): By default, _cls will be added to the start of every index (that doesn't contain a list) if allow_inheritance is True. This can be - disabled by either setting types to False on the specific index or + disabled by either setting cls to False on the specific index or by setting index_cls to False on the meta dictionary for the document. """ @@ -481,12 +481,6 @@ class Document(BaseDocument): first_field = fields[0][0] return first_field == '_cls' - # Ensure indexes created by uniqueness constraints - for index in cls._meta['unique_indexes']: - cls_indexed = cls_indexed or includes_cls(index) - collection.ensure_index(index, unique=True, background=background, - drop_dups=drop_dups, **index_opts) - # Ensure document-defined indexes are created if cls._meta['index_specs']: index_spec = cls._meta['index_specs'] @@ -496,7 +490,8 @@ class Document(BaseDocument): cls_indexed = cls_indexed or includes_cls(fields) opts = index_opts.copy() opts.update(spec) - collection.ensure_index(fields, background=background, **opts) + collection.ensure_index(fields, background=background, + drop_dups=drop_dups, **opts) # If _cls is being used (for polymorphism), it needs an index, # only if another index doesn't begin with _cls @@ -505,12 +500,6 @@ class Document(BaseDocument): collection.ensure_index('_cls', background=background, **index_opts) - # Add geo indicies - for field in cls._geo_indices(): - index_spec = [(field.db_field, pymongo.GEO2D)] - collection.ensure_index(index_spec, background=background, - **index_opts) - class DynamicDocument(Document): """A Dynamic Document class allowing flexible, expandable and uncontrolled diff --git a/tests/document/indexes.py b/tests/document/indexes.py index cf25f61..fb278aa 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -259,13 +259,12 @@ class IndexesTest(unittest.TestCase): tags = ListField(StringField()) meta = { 'indexes': [ - {'fields': ['-date'], 'unique': True, - 'sparse': True, 'types': False}, + {'fields': ['-date'], 'unique': True, 'sparse': True}, ], } self.assertEqual([{'fields': [('addDate', -1)], 'unique': True, - 'sparse': True, 'types': False}], + 'sparse': True}], BlogPost._meta['index_specs']) BlogPost.drop_collection() @@ -674,7 +673,7 @@ class IndexesTest(unittest.TestCase): User.drop_collection() - def test_types_index_with_pk(self): + def test_index_with_pk(self): """Ensure you can use `pk` as part of a query""" class Comment(EmbeddedDocument): @@ -687,7 +686,7 @@ class IndexesTest(unittest.TestCase): {'fields': ['pk', 'comments.comment_id'], 'unique': True}]} except UnboundLocalError: - self.fail('Unbound local error at types index + pk definition') + self.fail('Unbound local error at index + pk definition') info = BlogPost.objects._collection.index_information() info = [value['key'] for key, value in info.iteritems()] From 3364e040c83412444dd19c00f714d33c27a0e526 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Jan 2013 16:05:44 +0000 Subject: [PATCH 0920/1279] Ensure $maxDistance is always the last part of the query (#179) --- mongoengine/queryset/transform.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 5707cec..71f12e3 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -1,5 +1,7 @@ from collections import defaultdict +from bson import SON + from mongoengine.common import _import_class from mongoengine.errors import InvalidQueryError, LookUpError @@ -123,6 +125,16 @@ def query(_doc_cls=None, _field_operation=False, **query): elif key in mongo_query: if key in mongo_query and isinstance(mongo_query[key], dict): mongo_query[key].update(value) + # $maxDistance needs to come last - convert to SON + if '$maxDistance' in mongo_query[key]: + value_dict = mongo_query[key] + value_son = SON() + for k, v in value_dict.iteritems(): + if k == '$maxDistance': + continue + value_son[k] = v + value_son['$maxDistance'] = value_dict['$maxDistance'] + mongo_query[key] = value_son else: # Store for manually merging later merge_query[key].append(value) From 445f9453c4d305c394bb833dcd16ddce4b3ab2b8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Jan 2013 16:38:07 +0000 Subject: [PATCH 0921/1279] Fixed reverse delete rule with inheritance (#197) --- mongoengine/document.py | 17 +++++++++++++---- tests/document/instance.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index fff7efa..f40f1c9 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -7,7 +7,7 @@ from bson.dbref import DBRef from mongoengine import signals, queryset from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, - BaseDict, BaseList, ALLOW_INHERITANCE) + BaseDict, BaseList, ALLOW_INHERITANCE, get_document) from queryset import OperationError, NotUniqueError from connection import get_db, DEFAULT_CONNECTION_NAME @@ -421,9 +421,18 @@ class Document(BaseDocument): """This method registers the delete rules to apply when removing this object. """ - delete_rules = cls._meta.get('delete_rules') or {} - delete_rules[(document_cls, field_name)] = rule - cls._meta['delete_rules'] = delete_rules + classes = [get_document(class_name) + for class_name in cls._subclasses + if class_name != cls.__name__] + [cls] + documents = [get_document(class_name) + for class_name in document_cls._subclasses + if class_name != document_cls.__name__] + [document_cls] + + for cls in classes: + for document_cls in documents: + delete_rules = cls._meta.get('delete_rules') or {} + delete_rules[(document_cls, field_name)] = rule + cls._meta['delete_rules'] = delete_rules @classmethod def drop_collection(cls): diff --git a/tests/document/instance.py b/tests/document/instance.py index c8a1b11..07c4f0e 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1378,6 +1378,42 @@ class InstanceTest(unittest.TestCase): author.delete() self.assertEqual(len(BlogPost.objects), 0) + def test_reverse_delete_rule_with_document_inheritance(self): + """Ensure that a referenced document is also deleted upon deletion + of a child document. + """ + + class Writer(self.Person): + pass + + class BlogPost(Document): + content = StringField() + author = ReferenceField(self.Person, reverse_delete_rule=CASCADE) + reviewer = ReferenceField(self.Person, reverse_delete_rule=NULLIFY) + + self.Person.drop_collection() + BlogPost.drop_collection() + + author = Writer(name='Test User') + author.save() + + reviewer = Writer(name='Re Viewer') + reviewer.save() + + post = BlogPost(content='Watched some TV') + post.author = author + post.reviewer = reviewer + post.save() + + reviewer.delete() + self.assertEqual(len(BlogPost.objects), 1) + self.assertEqual(BlogPost.objects.get().reviewer, None) + + # Delete the Writer should lead to deletion of the BlogPost + author.delete() + self.assertEqual(len(BlogPost.objects), 0) + + def test_reverse_delete_rule_cascade_and_nullify_complex_field(self): """Ensure that a referenced document is also deleted upon deletion for complex fields. From c44b98a7e15d1c82f54e5c90ccd17e8d834cb4df Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Jan 2013 17:54:35 +0000 Subject: [PATCH 0922/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index cb0ac6c..c0757fb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -30,6 +30,7 @@ Changes in 0.8.X - Added support for $maxDistance (#179) - Uses getlasterror to test created on updated saves (#163) - Fixed inheritance and unique index creation (#140) +- Fixed reverse delete rule with inheritance (#197) Changes in 0.7.9 ================ From 6d68ad735cfcb6f37b964612867f31ff188b622f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 22 Jan 2013 17:56:15 +0000 Subject: [PATCH 0923/1279] Fixed validation for GenericReferences Where the references haven't been dereferenced --- docs/changelog.rst | 1 + mongoengine/fields.py | 8 ++- tests/test_dereference.py | 105 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c0757fb..ba2c04c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -31,6 +31,7 @@ Changes in 0.8.X - Uses getlasterror to test created on updated saves (#163) - Fixed inheritance and unique index creation (#140) - Fixed reverse delete rule with inheritance (#197) +- Fixed validation for GenericReferences which havent been dereferenced Changes in 0.7.9 ================ diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 5f11ae3..f781774 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -865,11 +865,15 @@ class GenericReferenceField(BaseField): return super(GenericReferenceField, self).__get__(instance, owner) def validate(self, value): - if not isinstance(value, (Document, DBRef)): + if not isinstance(value, (Document, DBRef, dict, SON)): self.error('GenericReferences can only contain documents') + if isinstance(value, (dict, SON)): + if '_ref' not in value or '_cls' not in value: + self.error('GenericReferences can only contain documents') + # We need the id from the saved object to create the DBRef - if isinstance(value, Document) and value.id is None: + elif isinstance(value, Document) and value.id is None: self.error('You can only reference documents once they have been' ' saved to the database') diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 8557ec5..f42482d 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1,4 +1,7 @@ from __future__ import with_statement +import sys +sys.path[0:0] = [""] + import unittest from bson import DBRef, ObjectId @@ -1018,3 +1021,105 @@ class FieldTest(unittest.TestCase): msg = Message.objects.get(id=1) self.assertEqual(0, msg.comments[0].id) self.assertEqual(1, msg.comments[1].id) + + def test_list_item_dereference_dref_false_save_doesnt_cause_extra_queries(self): + """Ensure that DBRef items in ListFields are dereferenced. + """ + class User(Document): + name = StringField() + + class Group(Document): + name = StringField() + members = ListField(ReferenceField(User, dbref=False)) + + User.drop_collection() + Group.drop_collection() + + for i in xrange(1, 51): + User(name='user %s' % i).save() + + Group(name="Test", members=User.objects).save() + + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first() + self.assertEqual(q, 1) + + group_obj.name = "new test" + group_obj.save() + + self.assertEqual(q, 2) + + def test_list_item_dereference_dref_true_save_doesnt_cause_extra_queries(self): + """Ensure that DBRef items in ListFields are dereferenced. + """ + class User(Document): + name = StringField() + + class Group(Document): + name = StringField() + members = ListField(ReferenceField(User, dbref=True)) + + User.drop_collection() + Group.drop_collection() + + for i in xrange(1, 51): + User(name='user %s' % i).save() + + Group(name="Test", members=User.objects).save() + + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first() + self.assertEqual(q, 1) + + group_obj.name = "new test" + group_obj.save() + + self.assertEqual(q, 2) + + def test_generic_reference_save_doesnt_cause_extra_queries(self): + + class UserA(Document): + name = StringField() + + class UserB(Document): + name = StringField() + + class UserC(Document): + name = StringField() + + class Group(Document): + name = StringField() + members = ListField(GenericReferenceField()) + + UserA.drop_collection() + UserB.drop_collection() + UserC.drop_collection() + Group.drop_collection() + + members = [] + for i in xrange(1, 51): + a = UserA(name='User A %s' % i).save() + b = UserB(name='User B %s' % i).save() + c = UserC(name='User C %s' % i).save() + + members += [a, b, c] + + Group(name="test", members=members).save() + + with query_counter() as q: + self.assertEqual(q, 0) + + group_obj = Group.objects.first() + self.assertEqual(q, 1) + + group_obj.name = "new test" + group_obj.save() + + self.assertEqual(q, 2) + +if __name__ == '__main__': + unittest.main() From e5e88d792e457e17040d803ab8c0592b174989bb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 23 Jan 2013 12:54:14 +0000 Subject: [PATCH 0924/1279] Added SwitchDB context manager (#106) --- docs/apireference.rst | 1 + docs/changelog.rst | 1 + docs/guide/connecting.rst | 18 ++++++++++++++++ mongoengine/connection.py | 43 ++++++++++++++++++++++++++++++++++++++- tests/test_connection.py | 23 +++++++++++++++++++++ 5 files changed, 85 insertions(+), 1 deletion(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 0f8901a..69b1db0 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -7,6 +7,7 @@ Connecting .. autofunction:: mongoengine.connect .. autofunction:: mongoengine.register_connection +.. autoclass:: mongoengine.SwitchDB Documents ========= diff --git a/docs/changelog.rst b/docs/changelog.rst index ba2c04c..354d471 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -32,6 +32,7 @@ Changes in 0.8.X - Fixed inheritance and unique index creation (#140) - Fixed reverse delete rule with inheritance (#197) - Fixed validation for GenericReferences which havent been dereferenced +- Added SwitchDB context manager (#106) Changes in 0.7.9 ================ diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index 657c46c..b39ccda 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -69,3 +69,21 @@ to point across databases and collections. Below is an example schema, using book = ReferenceField(Book) meta = {"db_alias": "users-books-db"} + + +Switch Database Context Manager +=============================== + +Sometimes you might want to switch the database to query against for a class. +The SwitchDB context manager allows you to change the database alias for a +class eg :: + + from mongoengine import SwitchDB + + class User(Document): + name = StringField() + + meta = {"db_alias": "user-db"} + + with SwitchDB(User, 'archive-user-db') as User: + User(name="Ross").save() # Saves the 'archive-user-db' diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 87308ba..b6c78e8 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -3,7 +3,7 @@ from pymongo import Connection, ReplicaSetConnection, uri_parser __all__ = ['ConnectionError', 'connect', 'register_connection', - 'DEFAULT_CONNECTION_NAME'] + 'DEFAULT_CONNECTION_NAME', 'SwitchDB'] DEFAULT_CONNECTION_NAME = 'default' @@ -163,6 +163,47 @@ def connect(db, alias=DEFAULT_CONNECTION_NAME, **kwargs): return get_connection(alias) + +class SwitchDB(object): + """ SwitchDB alias contextmanager. + + Example :: + # Register connections + register_connection('default', 'mongoenginetest') + register_connection('testdb-1', 'mongoenginetest2') + + class Group(Document): + name = StringField() + + Group(name="test").save() # Saves in the default db + + with SwitchDB(Group, 'testdb-1') as Group: + Group(name="hello testdb!").save() # Saves in testdb-1 + + """ + + def __init__(self, cls, db_alias): + """ Construct the query_counter. + + :param cls: the class to change the registered db + :param db_alias: the name of the specific database to use + """ + self.cls = cls + self.collection = cls._get_collection() + self.db_alias = db_alias + self.ori_db_alias = cls._meta.get("db_alias", DEFAULT_CONNECTION_NAME) + + def __enter__(self): + """ change the db_alias and clear the cached collection """ + self.cls._meta["db_alias"] = self.db_alias + self.cls._collection = None + return self.cls + + def __exit__(self, t, value, traceback): + """ Reset the db_alias and collection """ + self.cls._meta["db_alias"] = self.ori_db_alias + self.cls._collection = self.collection + # Support old naming convention _get_connection = get_connection _get_db = get_db diff --git a/tests/test_connection.py b/tests/test_connection.py index cd03df0..4931dc9 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -93,6 +93,29 @@ class ConnectionTest(unittest.TestCase): date_doc = DateDoc.objects.first() self.assertEqual(d, date_doc.the_date) + def test_switch_db_context_manager(self): + register_connection('testdb-1', 'mongoenginetest2') + + class Group(Document): + name = StringField() + + Group.drop_collection() + + Group(name="hello - default").save() + self.assertEqual(1, Group.objects.count()) + + with SwitchDB(Group, 'testdb-1') as Group: + + self.assertEqual(0, Group.objects.count()) + + Group(name="hello").save() + + self.assertEqual(1, Group.objects.count()) + + Group.drop_collection() + self.assertEqual(0, Group.objects.count()) + + self.assertEqual(1, Group.objects.count()) if __name__ == '__main__': unittest.main() From ea46edf50a712021d12249fd9e784bffbb505520 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 23 Jan 2013 16:07:07 +0000 Subject: [PATCH 0925/1279] Added switch_db method to document instances (#106) --- docs/changelog.rst | 1 + mongoengine/connection.py | 1 + mongoengine/document.py | 46 +++++++++++++++++++++++++++++++---- tests/document/instance.py | 50 ++++++++++++++++++++++++++++++++++++++ tests/test_connection.py | 1 + 5 files changed, 94 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 354d471..65e1103 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -33,6 +33,7 @@ Changes in 0.8.X - Fixed reverse delete rule with inheritance (#197) - Fixed validation for GenericReferences which havent been dereferenced - Added SwitchDB context manager (#106) +- Added switch_db method to document instances (#106) Changes in 0.7.9 ================ diff --git a/mongoengine/connection.py b/mongoengine/connection.py index b6c78e8..80791e5 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -168,6 +168,7 @@ class SwitchDB(object): """ SwitchDB alias contextmanager. Example :: + # Register connections register_connection('default', 'mongoenginetest') register_connection('testdb-1', 'mongoenginetest2') diff --git a/mongoengine/document.py b/mongoengine/document.py index f40f1c9..3bc4cae 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -9,7 +9,7 @@ from mongoengine import signals, queryset from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, BaseDict, BaseList, ALLOW_INHERITANCE, get_document) from queryset import OperationError, NotUniqueError -from connection import get_db, DEFAULT_CONNECTION_NAME +from connection import get_db, DEFAULT_CONNECTION_NAME, SwitchDB __all__ = ('Document', 'EmbeddedDocument', 'DynamicDocument', 'DynamicEmbeddedDocument', 'OperationError', @@ -222,7 +222,7 @@ class Document(BaseDocument): created = ('_id' not in doc or self._created or force_insert) try: - collection = self.__class__.objects._collection + collection = self._get_collection() if created: if force_insert: object_id = collection.insert(doc, safe=safe, @@ -321,6 +321,16 @@ class Document(BaseDocument): ref.save(**kwargs) ref._changed_fields = [] + @property + def _qs(self): + """ + Returns the queryset to use for updating / reloading / deletions + """ + qs = self.__class__.objects + if hasattr(self, '_objects'): + qs = self._objects + return qs + @property def _object_key(self): """Dict to identify object in collection @@ -342,7 +352,7 @@ class Document(BaseDocument): raise OperationError('attempt to update a document not yet saved') # Need to add shard key to query, or you get an error - return self.__class__.objects(**self._object_key).update_one(**kwargs) + return self._qs.filter(**self._object_key).update_one(**kwargs) def delete(self, safe=False): """Delete the :class:`~mongoengine.Document` from the database. This @@ -353,13 +363,39 @@ class Document(BaseDocument): signals.pre_delete.send(self.__class__, document=self) try: - self.__class__.objects(**self._object_key).delete(safe=safe) + self._qs.filter(**self._object_key).delete(safe=safe) except pymongo.errors.OperationFailure, err: message = u'Could not delete document (%s)' % err.message raise OperationError(message) signals.post_delete.send(self.__class__, document=self) + def switch_db(self, db_alias): + """ + Temporarily switch the database for a document instance. + + Only really useful for archiving off data and calling `save()`:: + + user = User.objects.get(id=user_id) + user.switch_db('archive-db') + user.save() + + If you need to read from another database see + :class:`~mongoengine.SwitchDB` + + :param db_alias: The database alias to use for saving the document + """ + with SwitchDB(self.__class__, db_alias) as cls: + collection = cls._get_collection() + db = cls._get_db + self._get_collection = lambda: collection + self._get_db = lambda: db + self._collection = collection + self._created = True + self._objects = self.__class__.objects + self._objects._collection_obj = collection + return self + def select_related(self, max_depth=1): """Handles dereferencing of :class:`~bson.dbref.DBRef` objects to a maximum depth in order to cut down the number queries to mongodb. @@ -377,7 +413,7 @@ class Document(BaseDocument): .. versionchanged:: 0.6 Now chainable """ id_field = self._meta['id_field'] - obj = self.__class__.objects( + obj = self._qs.filter( **{id_field: self[id_field]} ).limit(1).select_related(max_depth=max_depth) if obj: diff --git a/tests/document/instance.py b/tests/document/instance.py index 07c4f0e..3b5a4bd 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -2114,6 +2114,56 @@ class ValidatorErrorTest(unittest.TestCase): self.assertEqual(classic_doc, dict_doc) self.assertEqual(classic_doc._data, dict_doc._data) + def test_switch_db_instance(self): + register_connection('testdb-1', 'mongoenginetest2') + + class Group(Document): + name = StringField() + + Group.drop_collection() + with SwitchDB(Group, 'testdb-1') as Group: + Group.drop_collection() + + Group(name="hello - default").save() + self.assertEqual(1, Group.objects.count()) + + group = Group.objects.first() + group.switch_db('testdb-1') + group.name = "hello - testdb!" + group.save() + + with SwitchDB(Group, 'testdb-1') as Group: + group = Group.objects.first() + self.assertEqual("hello - testdb!", group.name) + + group = Group.objects.first() + self.assertEqual("hello - default", group.name) + + # Slightly contrived now - perform an update + # Only works as they have the same object_id + group.switch_db('testdb-1') + group.update(set__name="hello - update") + + with SwitchDB(Group, 'testdb-1') as Group: + group = Group.objects.first() + self.assertEqual("hello - update", group.name) + Group.drop_collection() + self.assertEqual(0, Group.objects.count()) + + group = Group.objects.first() + self.assertEqual("hello - default", group.name) + + # Totally contrived now - perform a delete + # Only works as they have the same object_id + group.switch_db('testdb-1') + group.delete() + + with SwitchDB(Group, 'testdb-1') as Group: + self.assertEqual(0, Group.objects.count()) + + group = Group.objects.first() + self.assertEqual("hello - default", group.name) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_connection.py b/tests/test_connection.py index 4931dc9..7ff18a3 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -94,6 +94,7 @@ class ConnectionTest(unittest.TestCase): self.assertEqual(d, date_doc.the_date) def test_switch_db_context_manager(self): + connect('mongoenginetest') register_connection('testdb-1', 'mongoenginetest2') class Group(Document): From 4f70c27b5695216839bd92d42eadd04518d95e58 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 23 Jan 2013 16:19:07 +0000 Subject: [PATCH 0926/1279] Updated doc string --- mongoengine/connection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 80791e5..9f906a2 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -165,7 +165,7 @@ def connect(db, alias=DEFAULT_CONNECTION_NAME, **kwargs): class SwitchDB(object): - """ SwitchDB alias contextmanager. + """ SwitchDB alias context manager. Example :: @@ -184,7 +184,7 @@ class SwitchDB(object): """ def __init__(self, cls, db_alias): - """ Construct the query_counter. + """ Construct the SwitchDB context manager :param cls: the class to change the registered db :param db_alias: the name of the specific database to use From 3a6dc77d3630ca87033374ae8c094f5ee8c9cda2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 23 Jan 2013 19:05:44 +0000 Subject: [PATCH 0927/1279] Added no_dereference context manager (#82) Reorganised the context_managers as well --- docs/apireference.rst | 8 +- docs/changelog.rst | 3 +- docs/guide/connecting.rst | 8 +- docs/guide/querying.rst | 18 +++- mongoengine/base/fields.py | 12 ++- mongoengine/common.py | 2 +- mongoengine/connection.py | 43 +-------- mongoengine/context_managers.py | 159 +++++++++++++++++++++++++++++++ mongoengine/document.py | 18 ++-- mongoengine/fields.py | 2 +- mongoengine/queryset/manager.py | 4 +- mongoengine/queryset/queryset.py | 1 - mongoengine/tests.py | 59 ------------ tests/queryset/queryset.py | 2 +- tests/test_connection.py | 4 +- tests/test_dereference.py | 74 +++++++++++++- 16 files changed, 289 insertions(+), 128 deletions(-) create mode 100644 mongoengine/context_managers.py delete mode 100644 mongoengine/tests.py diff --git a/docs/apireference.rst b/docs/apireference.rst index 69b1db0..049cc30 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -7,7 +7,6 @@ Connecting .. autofunction:: mongoengine.connect .. autofunction:: mongoengine.register_connection -.. autoclass:: mongoengine.SwitchDB Documents ========= @@ -35,6 +34,13 @@ Documents .. autoclass:: mongoengine.ValidationError :members: +Context Managers +================ + +.. autoclass:: mongoengine.context_managers.switch_db +.. autoclass:: mongoengine.context_managers.no_dereference +.. autoclass:: mongoengine.context_managers.query_counter + Querying ======== diff --git a/docs/changelog.rst b/docs/changelog.rst index 65e1103..bead693 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -32,8 +32,9 @@ Changes in 0.8.X - Fixed inheritance and unique index creation (#140) - Fixed reverse delete rule with inheritance (#197) - Fixed validation for GenericReferences which havent been dereferenced -- Added SwitchDB context manager (#106) +- Added switch_db context manager (#106) - Added switch_db method to document instances (#106) +- Added no_dereference context manager (#82) Changes in 0.7.9 ================ diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index b39ccda..ebd61a9 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -75,15 +75,15 @@ Switch Database Context Manager =============================== Sometimes you might want to switch the database to query against for a class. -The SwitchDB context manager allows you to change the database alias for a -class eg :: +The :class:`~mongoengine.context_managers.switch_db` context manager allows +you to change the database alias for a class eg :: - from mongoengine import SwitchDB + from mongoengine.context_managers import switch_db class User(Document): name = StringField() meta = {"db_alias": "user-db"} - with SwitchDB(User, 'archive-user-db') as User: + with switch_db(User, 'archive-user-db') as User: User(name="Ross").save() # Saves the 'archive-user-db' diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 40e36e3..7ccf143 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -93,7 +93,7 @@ may used with :class:`~mongoengine.GeoPointField`\ s: [(41.91,-87.69), (41.92,-87.68), (41.91,-87.65), (41.89,-87.65)]). .. note:: Requires Mongo Server 2.0 * ``max_distance`` -- can be added to your location queries to set a maximum -distance. + distance. Querying lists @@ -369,6 +369,22 @@ references to the depth of 1 level. If you have more complicated documents and want to dereference more of the object at once then increasing the :attr:`max_depth` will dereference more levels of the document. +Turning off dereferencing +------------------------- + +Sometimes for performance reasons you don't want to automatically dereference +data . To turn off all dereferencing you can use the +:class:`~mongoengine.context_managers.no_dereference` context manager:: + + with no_dereference(Post) as Post: + post = Post.objects.first() + assert(isinstance(post.author, ObjectId)) + +.. note:: + + :class:`~mongoengine.context_managers.no_dereference` only works on the + Default QuerySet manager. + Advanced queries ================ Sometimes calling a :class:`~mongoengine.queryset.QuerySet` object with keyword diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index a892fbd..82981e2 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -23,6 +23,7 @@ class BaseField(object): name = None _geo_index = False _auto_gen = False # Call `generate` to generate a value + _auto_dereference = True # These track each time a Field instance is created. Used to retain order. # The auto_creation_counter is used for fields that MongoEngine implicitly @@ -163,9 +164,11 @@ class ComplexBaseField(BaseField): ReferenceField = _import_class('ReferenceField') GenericReferenceField = _import_class('GenericReferenceField') - dereference = self.field is None or isinstance(self.field, - (GenericReferenceField, ReferenceField)) - if not self._dereference and instance._initialised and dereference: + dereference = (self._auto_dereference and + (self.field is None or isinstance(self.field, + (GenericReferenceField, ReferenceField)))) + + if not self.__dereference and instance._initialised and dereference: instance._data[self.name] = self._dereference( instance._data.get(self.name), max_depth=1, instance=instance, name=self.name @@ -182,7 +185,8 @@ class ComplexBaseField(BaseField): value = BaseDict(value, instance, self.name) instance._data[self.name] = value - if (instance._initialised and isinstance(value, (BaseList, BaseDict)) + if (self._auto_dereference and instance._initialised and + isinstance(value, (BaseList, BaseDict)) and not value._dereferenced): value = self._dereference( value, max_depth=1, instance=instance, name=self.name diff --git a/mongoengine/common.py b/mongoengine/common.py index a8422c0..718ac0b 100644 --- a/mongoengine/common.py +++ b/mongoengine/common.py @@ -11,7 +11,7 @@ def _import_class(cls_name): field_classes = ('DictField', 'DynamicField', 'EmbeddedDocumentField', 'FileField', 'GenericReferenceField', 'GenericEmbeddedDocumentField', 'GeoPointField', - 'ReferenceField', 'StringField') + 'ReferenceField', 'StringField', 'ComplexBaseField') queryset_classes = ('OperationError',) deref_classes = ('DeReference',) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 9f906a2..a47be44 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -3,7 +3,7 @@ from pymongo import Connection, ReplicaSetConnection, uri_parser __all__ = ['ConnectionError', 'connect', 'register_connection', - 'DEFAULT_CONNECTION_NAME', 'SwitchDB'] + 'DEFAULT_CONNECTION_NAME'] DEFAULT_CONNECTION_NAME = 'default' @@ -164,47 +164,6 @@ def connect(db, alias=DEFAULT_CONNECTION_NAME, **kwargs): return get_connection(alias) -class SwitchDB(object): - """ SwitchDB alias context manager. - - Example :: - - # Register connections - register_connection('default', 'mongoenginetest') - register_connection('testdb-1', 'mongoenginetest2') - - class Group(Document): - name = StringField() - - Group(name="test").save() # Saves in the default db - - with SwitchDB(Group, 'testdb-1') as Group: - Group(name="hello testdb!").save() # Saves in testdb-1 - - """ - - def __init__(self, cls, db_alias): - """ Construct the SwitchDB context manager - - :param cls: the class to change the registered db - :param db_alias: the name of the specific database to use - """ - self.cls = cls - self.collection = cls._get_collection() - self.db_alias = db_alias - self.ori_db_alias = cls._meta.get("db_alias", DEFAULT_CONNECTION_NAME) - - def __enter__(self): - """ change the db_alias and clear the cached collection """ - self.cls._meta["db_alias"] = self.db_alias - self.cls._collection = None - return self.cls - - def __exit__(self, t, value, traceback): - """ Reset the db_alias and collection """ - self.cls._meta["db_alias"] = self.ori_db_alias - self.cls._collection = self.collection - # Support old naming convention _get_connection = get_connection _get_db = get_db diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py new file mode 100644 index 0000000..7255d51 --- /dev/null +++ b/mongoengine/context_managers.py @@ -0,0 +1,159 @@ +from mongoengine.common import _import_class +from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db +from mongoengine.queryset import OperationError, QuerySet + +__all__ = ("switch_db", "no_dereference", "query_counter") + + +class switch_db(object): + """ switch_db alias context manager. + + Example :: + + # Register connections + register_connection('default', 'mongoenginetest') + register_connection('testdb-1', 'mongoenginetest2') + + class Group(Document): + name = StringField() + + Group(name="test").save() # Saves in the default db + + with switch_db(Group, 'testdb-1') as Group: + Group(name="hello testdb!").save() # Saves in testdb-1 + + """ + + def __init__(self, cls, db_alias): + """ Construct the switch_db context manager + + :param cls: the class to change the registered db + :param db_alias: the name of the specific database to use + """ + self.cls = cls + self.collection = cls._get_collection() + self.db_alias = db_alias + self.ori_db_alias = cls._meta.get("db_alias", DEFAULT_CONNECTION_NAME) + + def __enter__(self): + """ change the db_alias and clear the cached collection """ + self.cls._meta["db_alias"] = self.db_alias + self.cls._collection = None + return self.cls + + def __exit__(self, t, value, traceback): + """ Reset the db_alias and collection """ + self.cls._meta["db_alias"] = self.ori_db_alias + self.cls._collection = self.collection + + +class no_dereference(object): + """ no_dereference context manager. + + Turns off all dereferencing in Documents:: + + with no_dereference(Group) as Group: + Group.objects.find() + + """ + + def __init__(self, cls): + """ Construct the no_dereference context manager. + + :param cls: the class to turn dereferencing off on + """ + self.cls = cls + + ReferenceField = _import_class('ReferenceField') + GenericReferenceField = _import_class('GenericReferenceField') + ComplexBaseField = _import_class('ComplexBaseField') + + self.deref_fields = [k for k, v in self.cls._fields.iteritems() + if isinstance(v, (ReferenceField, + GenericReferenceField, + ComplexBaseField))] + + def __enter__(self): + """ change the objects default and _auto_dereference values""" + if 'queryset_class' in self.cls._meta: + raise OperationError("no_dereference context manager only works on" + " default queryset classes") + objects = self.cls.__dict__['objects'] + objects.default = QuerySetNoDeRef + self.cls.objects = objects + for field in self.deref_fields: + self.cls._fields[field]._auto_dereference = False + return self.cls + + def __exit__(self, t, value, traceback): + """ Reset the default and _auto_dereference values""" + objects = self.cls.__dict__['objects'] + objects.default = QuerySet + self.cls.objects = objects + for field in self.deref_fields: + self.cls._fields[field]._auto_dereference = True + return self.cls + + +class QuerySetNoDeRef(QuerySet): + """Special no_dereference QuerySet""" + def __dereference(items, max_depth=1, instance=None, name=None): + return items + + +class query_counter(object): + """ Query_counter contextmanager to get the number of queries. """ + + def __init__(self): + """ Construct the query_counter. """ + self.counter = 0 + self.db = get_db() + + def __enter__(self): + """ On every with block we need to drop the profile collection. """ + self.db.set_profiling_level(0) + self.db.system.profile.drop() + self.db.set_profiling_level(2) + return self + + def __exit__(self, t, value, traceback): + """ Reset the profiling level. """ + self.db.set_profiling_level(0) + + def __eq__(self, value): + """ == Compare querycounter. """ + return value == self._get_count() + + def __ne__(self, value): + """ != Compare querycounter. """ + return not self.__eq__(value) + + def __lt__(self, value): + """ < Compare querycounter. """ + return self._get_count() < value + + def __le__(self, value): + """ <= Compare querycounter. """ + return self._get_count() <= value + + def __gt__(self, value): + """ > Compare querycounter. """ + return self._get_count() > value + + def __ge__(self, value): + """ >= Compare querycounter. """ + return self._get_count() >= value + + def __int__(self): + """ int representation. """ + return self._get_count() + + def __repr__(self): + """ repr query_counter as the number of queries. """ + return u"%s" % self._get_count() + + def _get_count(self): + """ Get the number of queries. """ + count = self.db.system.profile.find().count() - self.counter + self.counter += 1 + return count diff --git a/mongoengine/document.py b/mongoengine/document.py index 3bc4cae..9d4a1e6 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,15 +1,17 @@ +from __future__ import with_statement import warnings import pymongo import re from bson.dbref import DBRef -from mongoengine import signals, queryset - -from base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, - BaseDict, BaseList, ALLOW_INHERITANCE, get_document) -from queryset import OperationError, NotUniqueError -from connection import get_db, DEFAULT_CONNECTION_NAME, SwitchDB +from mongoengine import signals +from mongoengine.base import (DocumentMetaclass, TopLevelDocumentMetaclass, + BaseDocument, BaseDict, BaseList, + ALLOW_INHERITANCE, get_document) +from mongoengine.queryset import OperationError, NotUniqueError +from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME +from mongoengine.context_managers import switch_db __all__ = ('Document', 'EmbeddedDocument', 'DynamicDocument', 'DynamicEmbeddedDocument', 'OperationError', @@ -381,11 +383,11 @@ class Document(BaseDocument): user.save() If you need to read from another database see - :class:`~mongoengine.SwitchDB` + :class:`~mongoengine.context_managers.switch_db` :param db_alias: The database alias to use for saving the document """ - with SwitchDB(self.__class__, db_alias) as cls: + with switch_db(self.__class__, db_alias) as cls: collection = cls._get_collection() db = cls._get_db self._get_collection = lambda: collection diff --git a/mongoengine/fields.py b/mongoengine/fields.py index f781774..1ccdb65 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -779,7 +779,7 @@ class ReferenceField(BaseField): value = instance._data.get(self.name) # Dereference DBRefs - if isinstance(value, DBRef): + if self._auto_dereference and isinstance(value, DBRef): value = self.document_type._get_db().dereference(value) if value is not None: instance._data[self.name] = self.document_type._from_son(value) diff --git a/mongoengine/queryset/manager.py b/mongoengine/queryset/manager.py index d9f9992..47c2143 100644 --- a/mongoengine/queryset/manager.py +++ b/mongoengine/queryset/manager.py @@ -18,11 +18,11 @@ class QuerySetManager(object): """ get_queryset = None + default = QuerySet def __init__(self, queryset_func=None): if queryset_func: self.get_queryset = queryset_func - self._collections = {} def __get__(self, instance, owner): """Descriptor for instantiating a new QuerySet object when @@ -33,7 +33,7 @@ class QuerySetManager(object): return self # owner is the document that contains the QuerySetManager - queryset_class = owner._meta.get('queryset_class') or QuerySet + queryset_class = owner._meta.get('queryset_class', self.default) queryset = queryset_class(owner, owner._get_collection()) if self.get_queryset: arg_count = self.get_queryset.func_code.co_argcount diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index e637370..a9ff6e7 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -109,7 +109,6 @@ class QuerySet(object): queryset._class_check = class_check return queryset - def __iter__(self): """Support iterator protocol""" self.rewind() diff --git a/mongoengine/tests.py b/mongoengine/tests.py deleted file mode 100644 index 6866377..0000000 --- a/mongoengine/tests.py +++ /dev/null @@ -1,59 +0,0 @@ -from mongoengine.connection import get_db - - -class query_counter(object): - """ Query_counter contextmanager to get the number of queries. """ - - def __init__(self): - """ Construct the query_counter. """ - self.counter = 0 - self.db = get_db() - - def __enter__(self): - """ On every with block we need to drop the profile collection. """ - self.db.set_profiling_level(0) - self.db.system.profile.drop() - self.db.set_profiling_level(2) - return self - - def __exit__(self, t, value, traceback): - """ Reset the profiling level. """ - self.db.set_profiling_level(0) - - def __eq__(self, value): - """ == Compare querycounter. """ - return value == self._get_count() - - def __ne__(self, value): - """ != Compare querycounter. """ - return not self.__eq__(value) - - def __lt__(self, value): - """ < Compare querycounter. """ - return self._get_count() < value - - def __le__(self, value): - """ <= Compare querycounter. """ - return self._get_count() <= value - - def __gt__(self, value): - """ > Compare querycounter. """ - return self._get_count() > value - - def __ge__(self, value): - """ >= Compare querycounter. """ - return self._get_count() >= value - - def __int__(self): - """ int representation. """ - return self._get_count() - - def __repr__(self): - """ repr query_counter as the number of queries. """ - return u"%s" % self._get_count() - - def _get_count(self): - """ Get the number of queries. """ - count = self.db.system.profile.find().count() - self.counter - self.counter += 1 - return count diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index b5b0b28..3594044 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -17,7 +17,7 @@ from bson import ObjectId from mongoengine import * from mongoengine.connection import get_connection from mongoengine.python_support import PY3 -from mongoengine.tests import query_counter +from mongoengine.context_managers import query_counter from mongoengine.queryset import (QuerySet, QuerySetManager, MultipleObjectsReturned, DoesNotExist, QueryFieldList, queryset_manager) diff --git a/tests/test_connection.py b/tests/test_connection.py index 7ff18a3..2a216fe 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1,3 +1,4 @@ +from __future__ import with_statement import datetime import pymongo import unittest @@ -8,6 +9,7 @@ from bson.tz_util import utc from mongoengine import * from mongoengine.connection import get_db, get_connection, ConnectionError +from mongoengine.context_managers import switch_db class ConnectionTest(unittest.TestCase): @@ -105,7 +107,7 @@ class ConnectionTest(unittest.TestCase): Group(name="hello - default").save() self.assertEqual(1, Group.objects.count()) - with SwitchDB(Group, 'testdb-1') as Group: + with switch_db(Group, 'testdb-1') as Group: self.assertEqual(0, Group.objects.count()) diff --git a/tests/test_dereference.py b/tests/test_dereference.py index f42482d..8e4ffdd 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -8,7 +8,7 @@ from bson import DBRef, ObjectId from mongoengine import * from mongoengine.connection import get_db -from mongoengine.tests import query_counter +from mongoengine.context_managers import query_counter, no_dereference class FieldTest(unittest.TestCase): @@ -1121,5 +1121,77 @@ class FieldTest(unittest.TestCase): self.assertEqual(q, 2) + def test_no_dereference_context_manager_object_id(self): + """Ensure that DBRef items in ListFields aren't dereferenced. + """ + class User(Document): + name = StringField() + + class Group(Document): + ref = ReferenceField(User, dbref=False) + generic = GenericReferenceField() + members = ListField(ReferenceField(User, dbref=False)) + + User.drop_collection() + Group.drop_collection() + + for i in xrange(1, 51): + User(name='user %s' % i).save() + + user = User.objects.first() + Group(ref=user, members=User.objects, generic=user).save() + + with no_dereference(Group) as NoDeRefGroup: + self.assertTrue(Group._fields['members']._auto_dereference) + self.assertFalse(NoDeRefGroup._fields['members']._auto_dereference) + + with no_dereference(Group) as Group: + group = Group.objects.first() + self.assertTrue(all([not isinstance(m, User) + for m in group.members])) + self.assertFalse(isinstance(group.ref, User)) + self.assertFalse(isinstance(group.generic, User)) + + self.assertTrue(all([isinstance(m, User) + for m in group.members])) + self.assertTrue(isinstance(group.ref, User)) + self.assertTrue(isinstance(group.generic, User)) + + def test_no_dereference_context_manager_dbref(self): + """Ensure that DBRef items in ListFields aren't dereferenced. + """ + class User(Document): + name = StringField() + + class Group(Document): + ref = ReferenceField(User, dbref=True) + generic = GenericReferenceField() + members = ListField(ReferenceField(User, dbref=True)) + + User.drop_collection() + Group.drop_collection() + + for i in xrange(1, 51): + User(name='user %s' % i).save() + + user = User.objects.first() + Group(ref=user, members=User.objects, generic=user).save() + + with no_dereference(Group) as NoDeRefGroup: + self.assertTrue(Group._fields['members']._auto_dereference) + self.assertFalse(NoDeRefGroup._fields['members']._auto_dereference) + + with no_dereference(Group) as Group: + group = Group.objects.first() + self.assertTrue(all([not isinstance(m, User) + for m in group.members])) + self.assertFalse(isinstance(group.ref, User)) + self.assertFalse(isinstance(group.generic, User)) + + self.assertTrue(all([isinstance(m, User) + for m in group.members])) + self.assertTrue(isinstance(group.ref, User)) + self.assertTrue(isinstance(group.generic, User)) + if __name__ == '__main__': unittest.main() From c8b65317effab9631b786e78f4592382a32c3658 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 23 Jan 2013 20:15:05 +0000 Subject: [PATCH 0928/1279] Updated documentation instance tests --- tests/document/instance.py | 103 +++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 51 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index 3b5a4bd..4c67046 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -18,11 +18,12 @@ from mongoengine.errors import (NotRegistered, InvalidDocumentError, from mongoengine.queryset import NULLIFY, Q from mongoengine.connection import get_db from mongoengine.base import get_document +from mongoengine.context_managers import switch_db TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), '../fields/mongoengine.png') -__all__ = ("InstanceTest",) +__all__ = ("InstanceTest", "ValidatorErrorTest") class InstanceTest(unittest.TestCase): @@ -1926,6 +1927,56 @@ class InstanceTest(unittest.TestCase): } )]), "1,2") + def test_switch_db_instance(self): + register_connection('testdb-1', 'mongoenginetest2') + + class Group(Document): + name = StringField() + + Group.drop_collection() + with switch_db(Group, 'testdb-1') as Group: + Group.drop_collection() + + Group(name="hello - default").save() + self.assertEqual(1, Group.objects.count()) + + group = Group.objects.first() + group.switch_db('testdb-1') + group.name = "hello - testdb!" + group.save() + + with switch_db(Group, 'testdb-1') as Group: + group = Group.objects.first() + self.assertEqual("hello - testdb!", group.name) + + group = Group.objects.first() + self.assertEqual("hello - default", group.name) + + # Slightly contrived now - perform an update + # Only works as they have the same object_id + group.switch_db('testdb-1') + group.update(set__name="hello - update") + + with switch_db(Group, 'testdb-1') as Group: + group = Group.objects.first() + self.assertEqual("hello - update", group.name) + Group.drop_collection() + self.assertEqual(0, Group.objects.count()) + + group = Group.objects.first() + self.assertEqual("hello - default", group.name) + + # Totally contrived now - perform a delete + # Only works as they have the same object_id + group.switch_db('testdb-1') + group.delete() + + with switch_db(Group, 'testdb-1') as Group: + self.assertEqual(0, Group.objects.count()) + + group = Group.objects.first() + self.assertEqual("hello - default", group.name) + class ValidatorErrorTest(unittest.TestCase): @@ -2114,56 +2165,6 @@ class ValidatorErrorTest(unittest.TestCase): self.assertEqual(classic_doc, dict_doc) self.assertEqual(classic_doc._data, dict_doc._data) - def test_switch_db_instance(self): - register_connection('testdb-1', 'mongoenginetest2') - - class Group(Document): - name = StringField() - - Group.drop_collection() - with SwitchDB(Group, 'testdb-1') as Group: - Group.drop_collection() - - Group(name="hello - default").save() - self.assertEqual(1, Group.objects.count()) - - group = Group.objects.first() - group.switch_db('testdb-1') - group.name = "hello - testdb!" - group.save() - - with SwitchDB(Group, 'testdb-1') as Group: - group = Group.objects.first() - self.assertEqual("hello - testdb!", group.name) - - group = Group.objects.first() - self.assertEqual("hello - default", group.name) - - # Slightly contrived now - perform an update - # Only works as they have the same object_id - group.switch_db('testdb-1') - group.update(set__name="hello - update") - - with SwitchDB(Group, 'testdb-1') as Group: - group = Group.objects.first() - self.assertEqual("hello - update", group.name) - Group.drop_collection() - self.assertEqual(0, Group.objects.count()) - - group = Group.objects.first() - self.assertEqual("hello - default", group.name) - - # Totally contrived now - perform a delete - # Only works as they have the same object_id - group.switch_db('testdb-1') - group.delete() - - with SwitchDB(Group, 'testdb-1') as Group: - self.assertEqual(0, Group.objects.count()) - - group = Group.objects.first() - self.assertEqual("hello - default", group.name) - if __name__ == '__main__': unittest.main() From 9797d7a7fb9c21684f360d03b06800b99b8093c4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 23 Jan 2013 21:19:21 +0000 Subject: [PATCH 0929/1279] Added switch_collection context manager and method (#220) --- mongoengine/context_managers.py | 45 +++++++++- mongoengine/document.py | 27 +++++- tests/test_connection.py | 24 ----- tests/test_context_managers.py | 153 ++++++++++++++++++++++++++++++++ tests/test_dereference.py | 71 --------------- 5 files changed, 223 insertions(+), 97 deletions(-) create mode 100644 tests/test_context_managers.py diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index 7255d51..e73d4a2 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -2,7 +2,7 @@ from mongoengine.common import _import_class from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db from mongoengine.queryset import OperationError, QuerySet -__all__ = ("switch_db", "no_dereference", "query_counter") +__all__ = ("switch_db", "switch_collection", "no_dereference", "query_counter") class switch_db(object): @@ -47,6 +47,49 @@ class switch_db(object): self.cls._collection = self.collection +class switch_collection(object): + """ switch_collection alias context manager. + + Example :: + + class Group(Document): + name = StringField() + + Group(name="test").save() # Saves in the default db + + with switch_collection(Group, 'group1') as Group: + Group(name="hello testdb!").save() # Saves in group1 collection + + """ + + def __init__(self, cls, collection_name): + """ Construct the switch_collection context manager + + :param cls: the class to change the registered db + :param collection_name: the name of the collection to use + """ + self.cls = cls + self.ori_collection = cls._get_collection() + self.ori_get_collection_name = cls._get_collection_name + self.collection_name = collection_name + + def __enter__(self): + """ change the _get_collection_name and clear the cached collection """ + + @classmethod + def _get_collection_name(cls): + return self.collection_name + + self.cls._get_collection_name = _get_collection_name + self.cls._collection = None + return self.cls + + def __exit__(self, t, value, traceback): + """ Reset the collection """ + self.cls._collection = self.ori_collection + self.cls._get_collection_name = self.ori_get_collection_name + + class no_dereference(object): """ no_dereference context manager. diff --git a/mongoengine/document.py b/mongoengine/document.py index 9d4a1e6..75873b4 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -11,7 +11,7 @@ from mongoengine.base import (DocumentMetaclass, TopLevelDocumentMetaclass, ALLOW_INHERITANCE, get_document) from mongoengine.queryset import OperationError, NotUniqueError from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME -from mongoengine.context_managers import switch_db +from mongoengine.context_managers import switch_db, switch_collection __all__ = ('Document', 'EmbeddedDocument', 'DynamicDocument', 'DynamicEmbeddedDocument', 'OperationError', @@ -398,6 +398,31 @@ class Document(BaseDocument): self._objects._collection_obj = collection return self + def switch_collection(self, collection_name): + """ + Temporarily switch the collection for a document instance. + + Only really useful for archiving off data and calling `save()`:: + + user = User.objects.get(id=user_id) + user.switch_collection('old-users') + user.save() + + If you need to read from another database see + :class:`~mongoengine.context_managers.switch_collection` + + :param collection_name: The database alias to use for saving the + document + """ + with switch_collection(self.__class__, collection_name) as cls: + collection = cls._get_collection() + self._get_collection = lambda: collection + self._collection = collection + self._created = True + self._objects = self.__class__.objects + self._objects._collection_obj = collection + return self + def select_related(self, max_depth=1): """Handles dereferencing of :class:`~bson.dbref.DBRef` objects to a maximum depth in order to cut down the number queries to mongodb. diff --git a/tests/test_connection.py b/tests/test_connection.py index 2a216fe..c32d231 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -95,30 +95,6 @@ class ConnectionTest(unittest.TestCase): date_doc = DateDoc.objects.first() self.assertEqual(d, date_doc.the_date) - def test_switch_db_context_manager(self): - connect('mongoenginetest') - register_connection('testdb-1', 'mongoenginetest2') - - class Group(Document): - name = StringField() - - Group.drop_collection() - - Group(name="hello - default").save() - self.assertEqual(1, Group.objects.count()) - - with switch_db(Group, 'testdb-1') as Group: - - self.assertEqual(0, Group.objects.count()) - - Group(name="hello").save() - - self.assertEqual(1, Group.objects.count()) - - Group.drop_collection() - self.assertEqual(0, Group.objects.count()) - - self.assertEqual(1, Group.objects.count()) if __name__ == '__main__': unittest.main() diff --git a/tests/test_context_managers.py b/tests/test_context_managers.py new file mode 100644 index 0000000..10fe7b8 --- /dev/null +++ b/tests/test_context_managers.py @@ -0,0 +1,153 @@ +from __future__ import with_statement +import unittest + +from mongoengine import * +from mongoengine.connection import get_db +from mongoengine.context_managers import (switch_db, switch_collection, + no_dereference, query_counter) + + +class ContextManagersTest(unittest.TestCase): + + def test_switch_db_context_manager(self): + connect('mongoenginetest') + register_connection('testdb-1', 'mongoenginetest2') + + class Group(Document): + name = StringField() + + Group.drop_collection() + + Group(name="hello - default").save() + self.assertEqual(1, Group.objects.count()) + + with switch_db(Group, 'testdb-1') as Group: + + self.assertEqual(0, Group.objects.count()) + + Group(name="hello").save() + + self.assertEqual(1, Group.objects.count()) + + Group.drop_collection() + self.assertEqual(0, Group.objects.count()) + + self.assertEqual(1, Group.objects.count()) + + def test_switch_collection_context_manager(self): + connect('mongoenginetest') + register_connection('testdb-1', 'mongoenginetest2') + + class Group(Document): + name = StringField() + + Group.drop_collection() + with switch_collection(Group, 'group1') as Group: + Group.drop_collection() + + Group(name="hello - group").save() + self.assertEqual(1, Group.objects.count()) + + with switch_collection(Group, 'group1') as Group: + + self.assertEqual(0, Group.objects.count()) + + Group(name="hello - group1").save() + + self.assertEqual(1, Group.objects.count()) + + Group.drop_collection() + self.assertEqual(0, Group.objects.count()) + + self.assertEqual(1, Group.objects.count()) + + def test_no_dereference_context_manager_object_id(self): + """Ensure that DBRef items in ListFields aren't dereferenced. + """ + connect('mongoenginetest') + + class User(Document): + name = StringField() + + class Group(Document): + ref = ReferenceField(User, dbref=False) + generic = GenericReferenceField() + members = ListField(ReferenceField(User, dbref=False)) + + User.drop_collection() + Group.drop_collection() + + for i in xrange(1, 51): + User(name='user %s' % i).save() + + user = User.objects.first() + Group(ref=user, members=User.objects, generic=user).save() + + with no_dereference(Group) as NoDeRefGroup: + self.assertTrue(Group._fields['members']._auto_dereference) + self.assertFalse(NoDeRefGroup._fields['members']._auto_dereference) + + with no_dereference(Group) as Group: + group = Group.objects.first() + self.assertTrue(all([not isinstance(m, User) + for m in group.members])) + self.assertFalse(isinstance(group.ref, User)) + self.assertFalse(isinstance(group.generic, User)) + + self.assertTrue(all([isinstance(m, User) + for m in group.members])) + self.assertTrue(isinstance(group.ref, User)) + self.assertTrue(isinstance(group.generic, User)) + + def test_no_dereference_context_manager_dbref(self): + """Ensure that DBRef items in ListFields aren't dereferenced. + """ + connect('mongoenginetest') + + class User(Document): + name = StringField() + + class Group(Document): + ref = ReferenceField(User, dbref=True) + generic = GenericReferenceField() + members = ListField(ReferenceField(User, dbref=True)) + + User.drop_collection() + Group.drop_collection() + + for i in xrange(1, 51): + User(name='user %s' % i).save() + + user = User.objects.first() + Group(ref=user, members=User.objects, generic=user).save() + + with no_dereference(Group) as NoDeRefGroup: + self.assertTrue(Group._fields['members']._auto_dereference) + self.assertFalse(NoDeRefGroup._fields['members']._auto_dereference) + + with no_dereference(Group) as Group: + group = Group.objects.first() + self.assertTrue(all([not isinstance(m, User) + for m in group.members])) + self.assertFalse(isinstance(group.ref, User)) + self.assertFalse(isinstance(group.generic, User)) + + self.assertTrue(all([isinstance(m, User) + for m in group.members])) + self.assertTrue(isinstance(group.ref, User)) + self.assertTrue(isinstance(group.generic, User)) + + def test_query_counter(self): + connect('mongoenginetest') + db = get_db() + + with query_counter() as q: + self.assertEqual(0, q) + + for i in xrange(1, 51): + db.test.find({}).count() + + self.assertEqual(50, q) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 8e4ffdd..adbc519 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1121,77 +1121,6 @@ class FieldTest(unittest.TestCase): self.assertEqual(q, 2) - def test_no_dereference_context_manager_object_id(self): - """Ensure that DBRef items in ListFields aren't dereferenced. - """ - class User(Document): - name = StringField() - - class Group(Document): - ref = ReferenceField(User, dbref=False) - generic = GenericReferenceField() - members = ListField(ReferenceField(User, dbref=False)) - - User.drop_collection() - Group.drop_collection() - - for i in xrange(1, 51): - User(name='user %s' % i).save() - - user = User.objects.first() - Group(ref=user, members=User.objects, generic=user).save() - - with no_dereference(Group) as NoDeRefGroup: - self.assertTrue(Group._fields['members']._auto_dereference) - self.assertFalse(NoDeRefGroup._fields['members']._auto_dereference) - - with no_dereference(Group) as Group: - group = Group.objects.first() - self.assertTrue(all([not isinstance(m, User) - for m in group.members])) - self.assertFalse(isinstance(group.ref, User)) - self.assertFalse(isinstance(group.generic, User)) - - self.assertTrue(all([isinstance(m, User) - for m in group.members])) - self.assertTrue(isinstance(group.ref, User)) - self.assertTrue(isinstance(group.generic, User)) - - def test_no_dereference_context_manager_dbref(self): - """Ensure that DBRef items in ListFields aren't dereferenced. - """ - class User(Document): - name = StringField() - - class Group(Document): - ref = ReferenceField(User, dbref=True) - generic = GenericReferenceField() - members = ListField(ReferenceField(User, dbref=True)) - - User.drop_collection() - Group.drop_collection() - - for i in xrange(1, 51): - User(name='user %s' % i).save() - - user = User.objects.first() - Group(ref=user, members=User.objects, generic=user).save() - - with no_dereference(Group) as NoDeRefGroup: - self.assertTrue(Group._fields['members']._auto_dereference) - self.assertFalse(NoDeRefGroup._fields['members']._auto_dereference) - - with no_dereference(Group) as Group: - group = Group.objects.first() - self.assertTrue(all([not isinstance(m, User) - for m in group.members])) - self.assertFalse(isinstance(group.ref, User)) - self.assertFalse(isinstance(group.generic, User)) - - self.assertTrue(all([isinstance(m, User) - for m in group.members])) - self.assertTrue(isinstance(group.ref, User)) - self.assertTrue(isinstance(group.generic, User)) if __name__ == '__main__': unittest.main() From d58f594c173fa57bb0b16e77c18724ea63dbf536 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 23 Jan 2013 21:21:46 +0000 Subject: [PATCH 0930/1279] Updated changelog --- docs/changelog.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index bead693..53dbeb9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -35,6 +35,8 @@ Changes in 0.8.X - Added switch_db context manager (#106) - Added switch_db method to document instances (#106) - Added no_dereference context manager (#82) +- Added switch_collection context manager (#220) +- Added switch_collection method to document instances (#220) Changes in 0.7.9 ================ From fff27f9b8744a747eb4abd85061fb60bbff5071e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jan 2013 10:37:54 +0000 Subject: [PATCH 0931/1279] Added support for compound primary keys (#149) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + docs/django.rst | 4 ++-- docs/guide/defining-documents.rst | 22 ++++++++++++-------- tests/document/indexes.py | 34 +++++++++++++++++++++++++++++++ 5 files changed, 52 insertions(+), 12 deletions(-) diff --git a/AUTHORS b/AUTHORS index aa7f833..3f80dca 100644 --- a/AUTHORS +++ b/AUTHORS @@ -135,4 +135,5 @@ that much better: * Marcelo Anton * Aleksey Porfirov * Nicolas Trippar - * Manuel Hermann \ No newline at end of file + * Manuel Hermann + * Gustavo Gawryszewski \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 53dbeb9..7486ae9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -37,6 +37,7 @@ Changes in 0.8.X - Added no_dereference context manager (#82) - Added switch_collection context manager (#220) - Added switch_collection method to document instances (#220) +- Added support for compound primary keys (#149) Changes in 0.7.9 ================ diff --git a/docs/django.rst b/docs/django.rst index a4f0560..ba93432 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -2,7 +2,7 @@ Using MongoEngine with Django ============================= -.. note :: Updated to support Django 1.4 +.. note:: Updated to support Django 1.4 Connecting ========== @@ -10,7 +10,7 @@ In your **settings.py** file, ignore the standard database settings (unless you also plan to use the ORM in your project), and instead call :func:`~mongoengine.connect` somewhere in the settings module. -.. note :: If getting an ``ImproperlyConfigured: settings.DATABASES is +.. note:: If getting an ``ImproperlyConfigured: settings.DATABASES is improperly configured`` error you may need to remove ``django.contrib.sites`` from ``INSTALLED_APPS`` in settings.py. diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 9abea9b..c698285 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -135,7 +135,8 @@ arguments can be set on all fields: field, will not have two documents in the collection with the same value. :attr:`primary_key` (Default: False) - When True, use this field as a primary key for the collection. + When True, use this field as a primary key for the collection. `DictField` + and `EmbeddedDocuments` both support being the primary key for a document. :attr:`choices` (Default: None) An iterable (e.g. a list or tuple) of choices to which the value of this @@ -441,6 +442,7 @@ The following example shows a :class:`Log` document that will be limited to Indexes ======= + You can specify indexes on collections to make querying faster. This is done by creating a list of index specifications called :attr:`indexes` in the :attr:`~mongoengine.Document.meta` dictionary, where an index specification may @@ -473,20 +475,22 @@ If a dictionary is passed then the following options are available: :attr:`unique` (Default: False) Whether the index should be unique. -.. note :: +.. note:: - To index embedded files / dictionary fields use 'dot' notation eg: - `rank.title` + Inheritance adds extra fields indices see: :ref:`document-inheritance`. -.. warning:: +Compound Indexes and Indexing sub documents +------------------------------------------- - Inheritance adds extra indices. - If don't need inheritance for a document turn inheritance off - - see :ref:`document-inheritance`. +Compound indexes can be created by adding the Embedded field or dictionary +field name to the index definition. +Sometimes its more efficient to index parts of Embeedded / dictionary fields, +in this case use 'dot' notation to identify the value to index eg: `rank.title` Geospatial indexes ---------------------------- +------------------ + Geospatial indexes will be automatically created for all :class:`~mongoengine.GeoPointField`\ s diff --git a/tests/document/indexes.py b/tests/document/indexes.py index fb278aa..c059590 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -693,5 +693,39 @@ class IndexesTest(unittest.TestCase): index_item = [('_id', 1), ('comments.comment_id', 1)] self.assertTrue(index_item in info) + def test_compound_key_embedded(self): + + class CompoundKey(EmbeddedDocument): + name = StringField(required=True) + term = StringField(required=True) + + class Report(Document): + key = EmbeddedDocumentField(CompoundKey, primary_key=True) + text = StringField() + + Report.drop_collection() + + my_key = CompoundKey(name="n", term="ok") + report = Report(text="OK", key=my_key).save() + + self.assertEqual({'text': 'OK', '_id': {'term': 'ok', 'name': 'n'}}, + report.to_mongo()) + self.assertEqual(report, Report.objects.get(pk=my_key)) + + def test_compound_key_dictfield(self): + + class Report(Document): + key = DictField(primary_key=True) + text = StringField() + + Report.drop_collection() + + my_key = {"name": "n", "term": "ok"} + report = Report(text="OK", key=my_key).save() + + self.assertEqual({'text': 'OK', '_id': {'term': 'ok', 'name': 'n'}}, + report.to_mongo()) + self.assertEqual(report, Report.objects.get(pk=my_key)) + if __name__ == '__main__': unittest.main() From e7ba5eb160e8f05385d8849eef031984ad916b04 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jan 2013 10:41:01 +0000 Subject: [PATCH 0932/1279] Added #121 to changelog --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7486ae9..219e935 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -37,7 +37,7 @@ Changes in 0.8.X - Added no_dereference context manager (#82) - Added switch_collection context manager (#220) - Added switch_collection method to document instances (#220) -- Added support for compound primary keys (#149) +- Added support for compound primary keys (#149) (#121) Changes in 0.7.9 ================ From e38bf63be0c51c6243010f255bd45e9e2a8ddcec Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jan 2013 11:29:51 +0000 Subject: [PATCH 0933/1279] Fixed overriding objects with custom manager (#58) --- AUTHORS | 3 ++- docs/changelog.rst | 3 ++- mongoengine/document.py | 17 ++++++++--------- tests/queryset/queryset.py | 26 ++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/AUTHORS b/AUTHORS index 3f80dca..c32ab9f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -136,4 +136,5 @@ that much better: * Aleksey Porfirov * Nicolas Trippar * Manuel Hermann - * Gustavo Gawryszewski \ No newline at end of file + * Gustavo Gawryszewski + * Max Countryman \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 219e935..0d164b5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -34,10 +34,11 @@ Changes in 0.8.X - Fixed validation for GenericReferences which havent been dereferenced - Added switch_db context manager (#106) - Added switch_db method to document instances (#106) -- Added no_dereference context manager (#82) +- Added no_dereference context manager (#82) (#61) - Added switch_collection context manager (#220) - Added switch_collection method to document instances (#220) - Added support for compound primary keys (#149) (#121) +- Fixed overriding objects with custom manager (#58) Changes in 0.7.9 ================ diff --git a/mongoengine/document.py b/mongoengine/document.py index 75873b4..edc819c 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -9,7 +9,7 @@ from mongoengine import signals from mongoengine.base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, BaseDict, BaseList, ALLOW_INHERITANCE, get_document) -from mongoengine.queryset import OperationError, NotUniqueError +from mongoengine.queryset import OperationError, NotUniqueError, QuerySet from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME from mongoengine.context_managers import switch_db, switch_collection @@ -328,10 +328,9 @@ class Document(BaseDocument): """ Returns the queryset to use for updating / reloading / deletions """ - qs = self.__class__.objects - if hasattr(self, '_objects'): - qs = self._objects - return qs + if not hasattr(self, '__objects'): + self.__objects = QuerySet(self, self._get_collection()) + return self.__objects @property def _object_key(self): @@ -394,8 +393,8 @@ class Document(BaseDocument): self._get_db = lambda: db self._collection = collection self._created = True - self._objects = self.__class__.objects - self._objects._collection_obj = collection + self.__objects = self._qs + self.__objects._collection_obj = collection return self def switch_collection(self, collection_name): @@ -419,8 +418,8 @@ class Document(BaseDocument): self._get_collection = lambda: collection self._collection = collection self._created = True - self._objects = self.__class__.objects - self._objects._collection_obj = collection + self.__objects = self._qs + self.__objects._collection_obj = collection return self def select_related(self, max_depth=1): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 3594044..0ad3092 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -2074,6 +2074,32 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() + def test_custom_manager_overriding_objects_works(self): + + class Foo(Document): + bar = StringField(default='bar') + active = BooleanField(default=False) + + @queryset_manager + def objects(doc_cls, queryset): + return queryset(active=True) + + @queryset_manager + def with_inactive(doc_cls, queryset): + return queryset(active=False) + + Foo.drop_collection() + + Foo(active=True).save() + Foo(active=False).save() + + self.assertEqual(1, Foo.objects.count()) + self.assertEqual(1, Foo.with_inactive.count()) + + Foo.with_inactive.first().delete() + self.assertEqual(0, Foo.with_inactive.count()) + self.assertEqual(1, Foo.objects.count()) + def test_query_value_conversion(self): """Ensure that query values are properly converted when necessary. From eefbd3f5974e6fd8eb3810d01de168a2a280b0da Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jan 2013 12:52:16 +0000 Subject: [PATCH 0934/1279] Updated wobbly python 3.3 test --- tests/document/instance.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index 4c67046..247f627 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -2026,8 +2026,6 @@ class ValidatorErrorTest(unittest.TestCase): try: User().validate() except ValidationError, e: - expected_error_message = """ValidationError(Field is required: ['username', 'name'])""" - self.assertEqual(e.message, expected_error_message) self.assertEqual(e.to_dict(), { 'username': 'Field is required', 'name': 'Field is required'}) From ed2ea24b75ffe70258882d1cf53a86e90b6ec1e4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jan 2013 13:10:51 +0000 Subject: [PATCH 0935/1279] More test edge case fixing --- tests/test_context_managers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_context_managers.py b/tests/test_context_managers.py index 10fe7b8..c9efe8b 100644 --- a/tests/test_context_managers.py +++ b/tests/test_context_managers.py @@ -140,6 +140,7 @@ class ContextManagersTest(unittest.TestCase): def test_query_counter(self): connect('mongoenginetest') db = get_db() + db.test.find({}) with query_counter() as q: self.assertEqual(0, q) From ba48dfb4bf283c5b3f20d5a9f47d69ec09e6f2f7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jan 2013 17:33:10 +0000 Subject: [PATCH 0936/1279] Added no_dereference method for querysets (#82) (#61) --- docs/changelog.rst | 1 + docs/guide/querying.rst | 13 +++++++++---- mongoengine/base/document.py | 13 +++++++++++-- mongoengine/base/fields.py | 1 + mongoengine/context_managers.py | 14 +++----------- mongoengine/fields.py | 5 +++-- mongoengine/queryset/queryset.py | 15 ++++++++++++--- tests/queryset/queryset.py | 21 +++++++++++++++++++++ 8 files changed, 61 insertions(+), 22 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0d164b5..e24eaf4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -39,6 +39,7 @@ Changes in 0.8.X - Added switch_collection method to document instances (#220) - Added support for compound primary keys (#149) (#121) - Fixed overriding objects with custom manager (#58) +- Added no_dereference method for querysets (#82) (#61) Changes in 0.7.9 ================ diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 7ccf143..3279853 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -373,17 +373,22 @@ Turning off dereferencing ------------------------- Sometimes for performance reasons you don't want to automatically dereference -data . To turn off all dereferencing you can use the +data. To turn off dereferencing of the results of a query use +:func:`~mongoengine.queryset.QuerySet.no_dereference` on the queryset like so:: + + post = Post.objects.no_dereference().first() + assert(isinstance(post.author, ObjectId)) + +You can also turn off all dereferencing for a fixed period by using the :class:`~mongoengine.context_managers.no_dereference` context manager:: with no_dereference(Post) as Post: post = Post.objects.first() assert(isinstance(post.author, ObjectId)) -.. note:: + # Outside the context manager dereferencing occurs. + assert(isinstance(post.author, User)) - :class:`~mongoengine.context_managers.no_dereference` only works on the - Default QuerySet manager. Advanced queries ================ diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 9f40061..7c1597e 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -1,3 +1,4 @@ +import copy import operator from functools import partial @@ -461,9 +462,10 @@ class BaseDocument(object): return cls._meta.get('collection', None) @classmethod - def _from_son(cls, son): + def _from_son(cls, son, _auto_dereference=True): """Create an instance of a Document (subclass) from a PyMongo SON. """ + # get the class name from the document, falling back to the given # class if unavailable class_name = son.get('_cls', cls._class_name) @@ -480,7 +482,12 @@ class BaseDocument(object): changed_fields = [] errors_dict = {} - for field_name, field in cls._fields.iteritems(): + fields = cls._fields + if not _auto_dereference: + fields = copy.copy(fields) + + for field_name, field in fields.iteritems(): + field._auto_dereference = _auto_dereference if field.db_field in data: value = data[field.db_field] try: @@ -507,6 +514,8 @@ class BaseDocument(object): obj = cls(__auto_convert=False, **data) obj._changed_fields = changed_fields obj._created = False + if not _auto_dereference: + obj._fields = fields return obj @classmethod diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 82981e2..25f86af 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -168,6 +168,7 @@ class ComplexBaseField(BaseField): (self.field is None or isinstance(self.field, (GenericReferenceField, ReferenceField)))) + self._auto_dereference = instance._fields[self.name]._auto_dereference if not self.__dereference and instance._initialised and dereference: instance._data[self.name] = self._dereference( instance._data.get(self.name), max_depth=1, instance=instance, diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index e73d4a2..76d5fbf 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -93,7 +93,8 @@ class switch_collection(object): class no_dereference(object): """ no_dereference context manager. - Turns off all dereferencing in Documents:: + Turns off all dereferencing in Documents for the duration of the context + manager:: with no_dereference(Group) as Group: Group.objects.find() @@ -118,21 +119,12 @@ class no_dereference(object): def __enter__(self): """ change the objects default and _auto_dereference values""" - if 'queryset_class' in self.cls._meta: - raise OperationError("no_dereference context manager only works on" - " default queryset classes") - objects = self.cls.__dict__['objects'] - objects.default = QuerySetNoDeRef - self.cls.objects = objects for field in self.deref_fields: self.cls._fields[field]._auto_dereference = False return self.cls def __exit__(self, t, value, traceback): """ Reset the default and _auto_dereference values""" - objects = self.cls.__dict__['objects'] - objects.default = QuerySet - self.cls.objects = objects for field in self.deref_fields: self.cls._fields[field]._auto_dereference = True return self.cls @@ -145,7 +137,7 @@ class QuerySetNoDeRef(QuerySet): class query_counter(object): - """ Query_counter contextmanager to get the number of queries. """ + """ Query_counter context manager to get the number of queries. """ def __init__(self): """ Construct the query_counter. """ diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 1ccdb65..11e9d3f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -777,7 +777,7 @@ class ReferenceField(BaseField): # Get value from document instance if available value = instance._data.get(self.name) - + self._auto_dereference = instance._fields[self.name]._auto_dereference # Dereference DBRefs if self._auto_dereference and isinstance(value, DBRef): value = self.document_type._get_db().dereference(value) @@ -859,7 +859,8 @@ class GenericReferenceField(BaseField): return self value = instance._data.get(self.name) - if isinstance(value, (dict, SON)): + self._auto_dereference = instance._fields[self.name]._auto_dereference + if self._auto_dereference and isinstance(value, (dict, SON)): instance._data[self.name] = self.dereference(value) return super(GenericReferenceField, self).__get__(instance, owner) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index a9ff6e7..f73b0e7 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -42,6 +42,7 @@ class QuerySet(object): providing :class:`~mongoengine.Document` objects as the results. """ __dereference = False + _auto_dereference = True def __init__(self, document, collection): self._document = document @@ -145,10 +146,12 @@ class QuerySet(object): elif isinstance(key, int): if queryset._scalar: return queryset._get_scalar( - queryset._document._from_son(queryset._cursor[key])) + queryset._document._from_son(queryset._cursor[key], + _auto_dereference=self._auto_dereference)) if queryset._as_pymongo: return queryset._get_as_pymongo(queryset._cursor.next()) - return queryset._document._from_son(queryset._cursor[key]) + return queryset._document._from_son(queryset._cursor[key], + _auto_dereference=self._auto_dereference) raise AttributeError def __repr__(self): @@ -515,7 +518,7 @@ class QuerySet(object): '_where_clause', '_loaded_fields', '_ordering', '_snapshot', '_timeout', '_class_check', '_slave_okay', '_read_preference', '_iter', '_scalar', '_as_pymongo', '_as_pymongo_coerce', - '_limit', '_skip', '_hint') + '_limit', '_skip', '_hint', '_auto_dereference') for prop in copy_props: val = getattr(self, prop) @@ -1135,6 +1138,12 @@ class QuerySet(object): self.__dereference = _import_class('DeReference')() return self.__dereference + def no_dereference(self): + """Turn off any dereferencing.""" + queryset = self.clone() + queryset._auto_dereference = False + return queryset + # Helper Functions def _item_frequencies_map_reduce(self, field, normalize=False): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 0ad3092..c6b7c0e 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3103,5 +3103,26 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(results[1]['name'], 'Barack Obama') self.assertEqual(results[1]['price'], Decimal('2.22')) + def test_no_dereference(self): + + class Organization(Document): + name = StringField() + + class User(Document): + name = StringField() + organization = ReferenceField(Organization) + + User.drop_collection() + Organization.drop_collection() + + whitehouse = Organization(name="White House").save() + User(name="Bob Dole", organization=whitehouse).save() + + qs = User.objects() + self.assertTrue(isinstance(qs.first().organization, Organization)) + self.assertFalse(isinstance(qs.no_dereference().first().organization, + Organization)) + self.assertTrue(isinstance(qs.first().organization, Organization)) + if __name__ == '__main__': unittest.main() From 9f551121fbbe852bd639ce4999ad0835db3fd8a6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jan 2013 17:41:21 +0000 Subject: [PATCH 0937/1279] Added docs for no_dereference and scalar (#68) --- mongoengine/queryset/queryset.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index f73b0e7..b5e3351 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -748,8 +748,12 @@ class QuerySet(object): """Instead of returning Document instances, return either a specific value or a tuple of values in order. - This effects all results and can be unset by calling ``scalar`` - without arguments. Calls ``only`` automatically. + Can be used along with + :func:`~mongoengine.queryset.QuerySet.no_dereference` to turn off + dereferencing. + + .. note:: This effects all results and can be unset by calling + ``scalar`` without arguments. Calls ``only`` automatically. :param fields: One or more fields to return instead of a Document. """ @@ -1139,7 +1143,8 @@ class QuerySet(object): return self.__dereference def no_dereference(self): - """Turn off any dereferencing.""" + """Turn off any dereferencing for the results of this queryset. + """ queryset = self.clone() queryset._auto_dereference = False return queryset From 83da08ef7dda6b40f0e288e98b6560107279e3c7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 24 Jan 2013 17:43:57 +0000 Subject: [PATCH 0938/1279] Documentation fixes --- docs/guide/defining-documents.rst | 2 +- docs/tutorial.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index c698285..3fdb9a6 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -47,7 +47,7 @@ be saved :: >>> Page.objects(tags='mongoengine').count() >>> 1 -..note:: +.. note:: There is one caveat on Dynamic Documents: fields cannot start with `_` diff --git a/docs/tutorial.rst b/docs/tutorial.rst index c2fb5b9..c4b69c4 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -170,7 +170,7 @@ To delete all the posts if a user is deleted set the rule:: See :class:`~mongoengine.ReferenceField` for more information. -..note:: +.. note:: MapFields and DictFields currently don't support automatic handling of deleted references From 621b2b3f72e142356e06e06845faea8caa8e9279 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 25 Jan 2013 11:28:20 +0000 Subject: [PATCH 0939/1279] Undefined data should not override instance methods (#49) --- docs/changelog.rst | 1 + docs/guide/defining-documents.rst | 10 +- mongoengine/base/document.py | 16 ++- tests/document/__init__.py | 1 + tests/document/instance.py | 193 +++-------------------------- tests/document/validation.py | 195 ++++++++++++++++++++++++++++++ 6 files changed, 230 insertions(+), 186 deletions(-) create mode 100644 tests/document/validation.py diff --git a/docs/changelog.rst b/docs/changelog.rst index e24eaf4..5bdd0f8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -40,6 +40,7 @@ Changes in 0.8.X - Added support for compound primary keys (#149) (#121) - Fixed overriding objects with custom manager (#58) - Added no_dereference method for querysets (#82) (#61) +- Undefined data should not override instance methods (#49) Changes in 0.7.9 ================ diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 3fdb9a6..350ba67 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -600,8 +600,7 @@ Working with existing data -------------------------- As MongoEngine no longer defaults to needing :attr:`_cls` you can quickly and easily get working with existing data. Just define the document to match -the expected schema in your database. If you have wildly varying schemas then -a :class:`~mongoengine.DynamicDocument` might be more appropriate. :: +the expected schema in your database :: # Will work with data in an existing collection named 'cmsPage' class Page(Document): @@ -609,3 +608,10 @@ a :class:`~mongoengine.DynamicDocument` might be more appropriate. :: meta = { 'collection': 'cmsPage' } + +If you have wildly varying schemas then using a +:class:`~mongoengine.DynamicDocument` might be more appropriate, instead of +defining all possible field types. + +If you use :class:`~mongoengine.Document` and the database contains data that +isn't defined then that data will be stored in the `document._data` dictionary. diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 7c1597e..a88a38b 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -60,13 +60,17 @@ class BaseDocument(object): else: FileField = _import_class('FileField') for key, value in values.iteritems(): + if key == '__auto_convert': + continue key = self._reverse_db_field_map.get(key, key) - if (value is not None and __auto_convert and - key in self._fields): - field = self._fields.get(key) - if not isinstance(field, FileField): - value = field.to_python(value) - setattr(self, key, value) + if key in self._fields or key in ('id', 'pk', '_cls'): + if __auto_convert and value is not None: + field = self._fields.get(key) + if field and not isinstance(field, FileField): + value = field.to_python(value) + setattr(self, key, value) + else: + self._data[key] = value # Set any get_fieldname_display methods self.__set_field_display() diff --git a/tests/document/__init__.py b/tests/document/__init__.py index 7774ee1..1acc9f4 100644 --- a/tests/document/__init__.py +++ b/tests/document/__init__.py @@ -9,6 +9,7 @@ from indexes import * from inheritance import * from instance import * from json_serialisation import * +from validation import * if __name__ == '__main__': unittest.main() diff --git a/tests/document/instance.py b/tests/document/instance.py index 247f627..99e4edb 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -23,7 +23,7 @@ from mongoengine.context_managers import switch_db TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), '../fields/mongoengine.png') -__all__ = ("InstanceTest", "ValidatorErrorTest") +__all__ = ("InstanceTest",) class InstanceTest(unittest.TestCase): @@ -1977,192 +1977,29 @@ class InstanceTest(unittest.TestCase): group = Group.objects.first() self.assertEqual("hello - default", group.name) - -class ValidatorErrorTest(unittest.TestCase): - - def test_to_dict(self): - """Ensure a ValidationError handles error to_dict correctly. - """ - error = ValidationError('root') - self.assertEqual(error.to_dict(), {}) - - # 1st level error schema - error.errors = {'1st': ValidationError('bad 1st'), } - self.assertTrue('1st' in error.to_dict()) - self.assertEqual(error.to_dict()['1st'], 'bad 1st') - - # 2nd level error schema - error.errors = {'1st': ValidationError('bad 1st', errors={ - '2nd': ValidationError('bad 2nd'), - })} - self.assertTrue('1st' in error.to_dict()) - self.assertTrue(isinstance(error.to_dict()['1st'], dict)) - self.assertTrue('2nd' in error.to_dict()['1st']) - self.assertEqual(error.to_dict()['1st']['2nd'], 'bad 2nd') - - # moar levels - error.errors = {'1st': ValidationError('bad 1st', errors={ - '2nd': ValidationError('bad 2nd', errors={ - '3rd': ValidationError('bad 3rd', errors={ - '4th': ValidationError('Inception'), - }), - }), - })} - self.assertTrue('1st' in error.to_dict()) - self.assertTrue('2nd' in error.to_dict()['1st']) - self.assertTrue('3rd' in error.to_dict()['1st']['2nd']) - self.assertTrue('4th' in error.to_dict()['1st']['2nd']['3rd']) - self.assertEqual(error.to_dict()['1st']['2nd']['3rd']['4th'], - 'Inception') - - self.assertEqual(error.message, "root(2nd.3rd.4th.Inception: ['1st'])") - - def test_model_validation(self): + def test_no_overwritting_no_data_loss(self): class User(Document): username = StringField(primary_key=True) - name = StringField(required=True) - - try: - User().validate() - except ValidationError, e: - self.assertEqual(e.to_dict(), { - 'username': 'Field is required', - 'name': 'Field is required'}) - - def test_spaces_in_keys(self): - - class Embedded(DynamicEmbeddedDocument): - pass - - class Doc(DynamicDocument): - pass - - Doc.drop_collection() - doc = Doc() - setattr(doc, 'hello world', 1) - doc.save() - - one = Doc.objects.filter(**{'hello world': 1}).count() - self.assertEqual(1, one) - - def test_fields_rewrite(self): - class BasePerson(Document): - name = StringField() - age = IntField() - meta = {'abstract': True} - - class Person(BasePerson): - name = StringField(required=True) - - p = Person(age=15) - self.assertRaises(ValidationError, p.validate) - - def test_cascaded_save_wrong_reference(self): - - class ADocument(Document): - val = IntField() - - class BDocument(Document): - a = ReferenceField(ADocument) - - ADocument.drop_collection() - BDocument.drop_collection() - - a = ADocument() - a.val = 15 - a.save() - - b = BDocument() - b.a = a - b.save() - - a.delete() - - b = BDocument.objects.first() - b.save(cascade=True) - - def test_shard_key(self): - class LogEntry(Document): - machine = StringField() - log = StringField() - - meta = { - 'shard_key': ('machine',) - } - - LogEntry.drop_collection() - - log = LogEntry() - log.machine = "Localhost" - log.save() - - log.log = "Saving" - log.save() - - def change_shard_key(): - log.machine = "127.0.0.1" - - self.assertRaises(OperationError, change_shard_key) - - def test_shard_key_primary(self): - class LogEntry(Document): - machine = StringField(primary_key=True) - log = StringField() - - meta = { - 'shard_key': ('machine',) - } - - LogEntry.drop_collection() - - log = LogEntry() - log.machine = "Localhost" - log.save() - - log.log = "Saving" - log.save() - - def change_shard_key(): - log.machine = "127.0.0.1" - - self.assertRaises(OperationError, change_shard_key) - - def test_kwargs_simple(self): - - class Embedded(EmbeddedDocument): name = StringField() - class Doc(Document): - doc_name = StringField() - doc = EmbeddedDocumentField(Embedded) + @property + def foo(self): + return True - classic_doc = Doc(doc_name="my doc", doc=Embedded(name="embedded doc")) - dict_doc = Doc(**{"doc_name": "my doc", - "doc": {"name": "embedded doc"}}) + User.drop_collection() - self.assertEqual(classic_doc, dict_doc) - self.assertEqual(classic_doc._data, dict_doc._data) + user = User(username="Ross", foo="bar") + self.assertTrue(user.foo) - def test_kwargs_complex(self): - - class Embedded(EmbeddedDocument): - name = StringField() - - class Doc(Document): - doc_name = StringField() - docs = ListField(EmbeddedDocumentField(Embedded)) - - classic_doc = Doc(doc_name="my doc", docs=[ - Embedded(name="embedded doc1"), - Embedded(name="embedded doc2")]) - dict_doc = Doc(**{"doc_name": "my doc", - "docs": [{"name": "embedded doc1"}, - {"name": "embedded doc2"}]}) - - self.assertEqual(classic_doc, dict_doc) - self.assertEqual(classic_doc._data, dict_doc._data) + User._get_collection().save({"_id": "Ross", "foo": "Bar", + "data": [1, 2, 3]}) + user = User.objects.first() + self.assertEqual("Ross", user.username) + self.assertEqual(True, user.foo) + self.assertEqual("Bar", user._data["foo"]) + self.assertEqual([1, 2, 3], user._data["data"]) if __name__ == '__main__': unittest.main() diff --git a/tests/document/validation.py b/tests/document/validation.py new file mode 100644 index 0000000..dafb3a3 --- /dev/null +++ b/tests/document/validation.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +import sys +sys.path[0:0] = [""] + +import unittest + +from mongoengine import * + +__all__ = ("ValidatorErrorTest",) + + +class ValidatorErrorTest(unittest.TestCase): + + def test_to_dict(self): + """Ensure a ValidationError handles error to_dict correctly. + """ + error = ValidationError('root') + self.assertEqual(error.to_dict(), {}) + + # 1st level error schema + error.errors = {'1st': ValidationError('bad 1st'), } + self.assertTrue('1st' in error.to_dict()) + self.assertEqual(error.to_dict()['1st'], 'bad 1st') + + # 2nd level error schema + error.errors = {'1st': ValidationError('bad 1st', errors={ + '2nd': ValidationError('bad 2nd'), + })} + self.assertTrue('1st' in error.to_dict()) + self.assertTrue(isinstance(error.to_dict()['1st'], dict)) + self.assertTrue('2nd' in error.to_dict()['1st']) + self.assertEqual(error.to_dict()['1st']['2nd'], 'bad 2nd') + + # moar levels + error.errors = {'1st': ValidationError('bad 1st', errors={ + '2nd': ValidationError('bad 2nd', errors={ + '3rd': ValidationError('bad 3rd', errors={ + '4th': ValidationError('Inception'), + }), + }), + })} + self.assertTrue('1st' in error.to_dict()) + self.assertTrue('2nd' in error.to_dict()['1st']) + self.assertTrue('3rd' in error.to_dict()['1st']['2nd']) + self.assertTrue('4th' in error.to_dict()['1st']['2nd']['3rd']) + self.assertEqual(error.to_dict()['1st']['2nd']['3rd']['4th'], + 'Inception') + + self.assertEqual(error.message, "root(2nd.3rd.4th.Inception: ['1st'])") + + def test_model_validation(self): + + class User(Document): + username = StringField(primary_key=True) + name = StringField(required=True) + + try: + User().validate() + except ValidationError, e: + self.assertEqual(e.to_dict(), { + 'username': 'Field is required', + 'name': 'Field is required'}) + + def test_spaces_in_keys(self): + + class Embedded(DynamicEmbeddedDocument): + pass + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + setattr(doc, 'hello world', 1) + doc.save() + + one = Doc.objects.filter(**{'hello world': 1}).count() + self.assertEqual(1, one) + + def test_fields_rewrite(self): + class BasePerson(Document): + name = StringField() + age = IntField() + meta = {'abstract': True} + + class Person(BasePerson): + name = StringField(required=True) + + p = Person(age=15) + self.assertRaises(ValidationError, p.validate) + + def test_cascaded_save_wrong_reference(self): + + class ADocument(Document): + val = IntField() + + class BDocument(Document): + a = ReferenceField(ADocument) + + ADocument.drop_collection() + BDocument.drop_collection() + + a = ADocument() + a.val = 15 + a.save() + + b = BDocument() + b.a = a + b.save() + + a.delete() + + b = BDocument.objects.first() + b.save(cascade=True) + + def test_shard_key(self): + class LogEntry(Document): + machine = StringField() + log = StringField() + + meta = { + 'shard_key': ('machine',) + } + + LogEntry.drop_collection() + + log = LogEntry() + log.machine = "Localhost" + log.save() + + log.log = "Saving" + log.save() + + def change_shard_key(): + log.machine = "127.0.0.1" + + self.assertRaises(OperationError, change_shard_key) + + def test_shard_key_primary(self): + class LogEntry(Document): + machine = StringField(primary_key=True) + log = StringField() + + meta = { + 'shard_key': ('machine',) + } + + LogEntry.drop_collection() + + log = LogEntry() + log.machine = "Localhost" + log.save() + + log.log = "Saving" + log.save() + + def change_shard_key(): + log.machine = "127.0.0.1" + + self.assertRaises(OperationError, change_shard_key) + + def test_kwargs_simple(self): + + class Embedded(EmbeddedDocument): + name = StringField() + + class Doc(Document): + doc_name = StringField() + doc = EmbeddedDocumentField(Embedded) + + classic_doc = Doc(doc_name="my doc", doc=Embedded(name="embedded doc")) + dict_doc = Doc(**{"doc_name": "my doc", + "doc": {"name": "embedded doc"}}) + + self.assertEqual(classic_doc, dict_doc) + self.assertEqual(classic_doc._data, dict_doc._data) + + def test_kwargs_complex(self): + + class Embedded(EmbeddedDocument): + name = StringField() + + class Doc(Document): + doc_name = StringField() + docs = ListField(EmbeddedDocumentField(Embedded)) + + classic_doc = Doc(doc_name="my doc", docs=[ + Embedded(name="embedded doc1"), + Embedded(name="embedded doc2")]) + dict_doc = Doc(**{"doc_name": "my doc", + "docs": [{"name": "embedded doc1"}, + {"name": "embedded doc2"}]}) + + self.assertEqual(classic_doc, dict_doc) + self.assertEqual(classic_doc._data, dict_doc._data) From eb1b6e34c71abe45d96cbf356f1253a3ca5c51a6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 25 Jan 2013 11:51:58 +0000 Subject: [PATCH 0940/1279] Updated upgrade docs (#49) --- docs/upgrade.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 9c6c9a9..d328248 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -56,6 +56,26 @@ you will need to declare :attr:`allow_inheritance` in the meta data like so: :: meta = {'allow_inheritance': True} +Previously, if you had data the database that wasn't defined in the Document +definition, it would set it as an attribute on the document. This is no longer +the case and the data is set only in the ``document._data`` dictionary: :: + + >>> from mongoengine import * + >>> class Animal(Document): + ... name = StringField() + ... + >>> cat = Animal(name="kit", size="small") + + # 0.7 + >>> cat.size + u'small' + + # 0.8 + >>> cat.size + Traceback (most recent call last): + File "", line 1, in + AttributeError: 'Animal' object has no attribute 'size' + Querysets ~~~~~~~~~ From 0ea363c7fc760e210bee5b83b3ab83f657a6e0ae Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 25 Jan 2013 12:13:46 +0000 Subject: [PATCH 0941/1279] Updated authors and changelof (#142) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index c32ab9f..ac811dd 100644 --- a/AUTHORS +++ b/AUTHORS @@ -137,4 +137,5 @@ that much better: * Nicolas Trippar * Manuel Hermann * Gustavo Gawryszewski - * Max Countryman \ No newline at end of file + * Max Countryman + * caitifbrito \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 5bdd0f8..19bc446 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -41,6 +41,7 @@ Changes in 0.8.X - Fixed overriding objects with custom manager (#58) - Added no_dereference method for querysets (#82) (#61) - Undefined data should not override instance methods (#49) +- Added Django Group and Permission (#142) Changes in 0.7.9 ================ From 9d9a4afee9aef2e11beb37207c914a8a76329370 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 12:05:09 +0000 Subject: [PATCH 0942/1279] Added Doc class and pk to Validation messages (#69) --- docs/changelog.rst | 1 + mongoengine/base/document.py | 8 +- tests/document/instance.py | 137 +++++++++++++++++++++--------- tests/document/validation.py | 159 ++++++++++++----------------------- 4 files changed, 161 insertions(+), 144 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 19bc446..601c2db 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -42,6 +42,7 @@ Changes in 0.8.X - Added no_dereference method for querysets (#82) (#61) - Undefined data should not override instance methods (#49) - Added Django Group and Permission (#142) +- Added Doc class and pk to Validation messages (#69) Changes in 0.7.9 ================ diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index a88a38b..4f5a87e 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -274,7 +274,13 @@ class BaseDocument(object): field_name=field.name) if errors: - raise ValidationError('ValidationError', errors=errors) + pk = "None" + if hasattr(self, 'pk'): + pk = self.pk + elif self._instance: + pk = self._instance.pk + message = "ValidationError (%s:%s) " % (self._class_name, pk) + raise ValidationError(message, errors=errors) def to_json(self): """Converts a document to JSON""" diff --git a/tests/document/instance.py b/tests/document/instance.py index 99e4edb..3d4e8a9 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -466,45 +466,6 @@ class InstanceTest(unittest.TestCase): doc = Doc.objects.get() self.assertEqual(doc, doc.embedded_field[0]._instance) - def test_embedded_document_validation(self): - """Ensure that embedded documents may be validated. - """ - class Comment(EmbeddedDocument): - date = DateTimeField() - content = StringField(required=True) - - comment = Comment() - self.assertRaises(ValidationError, comment.validate) - - comment.content = 'test' - comment.validate() - - comment.date = 4 - self.assertRaises(ValidationError, comment.validate) - - comment.date = datetime.now() - comment.validate() - self.assertEqual(comment._instance, None) - - def test_embedded_db_field_validate(self): - - class SubDoc(EmbeddedDocument): - val = IntField() - - class Doc(Document): - e = EmbeddedDocumentField(SubDoc, db_field='eb') - - Doc.drop_collection() - - Doc(e=SubDoc(val=15)).save() - - doc = Doc.objects.first() - doc.validate() - keys = doc._data.keys() - self.assertEqual(2, len(keys)) - self.assertTrue('id' in keys) - self.assertTrue('e' in keys) - def test_document_clean(self): class TestDocument(Document): status = StringField() @@ -2001,5 +1962,103 @@ class InstanceTest(unittest.TestCase): self.assertEqual("Bar", user._data["foo"]) self.assertEqual([1, 2, 3], user._data["data"]) + + def test_spaces_in_keys(self): + + class Embedded(DynamicEmbeddedDocument): + pass + + class Doc(DynamicDocument): + pass + + Doc.drop_collection() + doc = Doc() + setattr(doc, 'hello world', 1) + doc.save() + + one = Doc.objects.filter(**{'hello world': 1}).count() + self.assertEqual(1, one) + + def test_shard_key(self): + class LogEntry(Document): + machine = StringField() + log = StringField() + + meta = { + 'shard_key': ('machine',) + } + + LogEntry.drop_collection() + + log = LogEntry() + log.machine = "Localhost" + log.save() + + log.log = "Saving" + log.save() + + def change_shard_key(): + log.machine = "127.0.0.1" + + self.assertRaises(OperationError, change_shard_key) + + def test_shard_key_primary(self): + class LogEntry(Document): + machine = StringField(primary_key=True) + log = StringField() + + meta = { + 'shard_key': ('machine',) + } + + LogEntry.drop_collection() + + log = LogEntry() + log.machine = "Localhost" + log.save() + + log.log = "Saving" + log.save() + + def change_shard_key(): + log.machine = "127.0.0.1" + + self.assertRaises(OperationError, change_shard_key) + + def test_kwargs_simple(self): + + class Embedded(EmbeddedDocument): + name = StringField() + + class Doc(Document): + doc_name = StringField() + doc = EmbeddedDocumentField(Embedded) + + classic_doc = Doc(doc_name="my doc", doc=Embedded(name="embedded doc")) + dict_doc = Doc(**{"doc_name": "my doc", + "doc": {"name": "embedded doc"}}) + + self.assertEqual(classic_doc, dict_doc) + self.assertEqual(classic_doc._data, dict_doc._data) + + def test_kwargs_complex(self): + + class Embedded(EmbeddedDocument): + name = StringField() + + class Doc(Document): + doc_name = StringField() + docs = ListField(EmbeddedDocumentField(Embedded)) + + classic_doc = Doc(doc_name="my doc", docs=[ + Embedded(name="embedded doc1"), + Embedded(name="embedded doc2")]) + dict_doc = Doc(**{"doc_name": "my doc", + "docs": [{"name": "embedded doc1"}, + {"name": "embedded doc2"}]}) + + self.assertEqual(classic_doc, dict_doc) + self.assertEqual(classic_doc._data, dict_doc._data) + if __name__ == '__main__': unittest.main() diff --git a/tests/document/validation.py b/tests/document/validation.py index dafb3a3..aaf6b0c 100644 --- a/tests/document/validation.py +++ b/tests/document/validation.py @@ -3,6 +3,7 @@ import sys sys.path[0:0] = [""] import unittest +from datetime import datetime from mongoengine import * @@ -11,6 +12,9 @@ __all__ = ("ValidatorErrorTest",) class ValidatorErrorTest(unittest.TestCase): + def setUp(self): + connect(db='mongoenginetest') + def test_to_dict(self): """Ensure a ValidationError handles error to_dict correctly. """ @@ -57,25 +61,19 @@ class ValidatorErrorTest(unittest.TestCase): try: User().validate() except ValidationError, e: + self.assertTrue("User:None" in e.message) self.assertEqual(e.to_dict(), { 'username': 'Field is required', 'name': 'Field is required'}) - def test_spaces_in_keys(self): - - class Embedded(DynamicEmbeddedDocument): - pass - - class Doc(DynamicDocument): - pass - - Doc.drop_collection() - doc = Doc() - setattr(doc, 'hello world', 1) - doc.save() - - one = Doc.objects.filter(**{'hello world': 1}).count() - self.assertEqual(1, one) + user = User(username="RossC0", name="Ross").save() + user.name = None + try: + user.save() + except ValidationError, e: + self.assertTrue("User:RossC0" in e.message) + self.assertEqual(e.to_dict(), { + 'name': 'Field is required'}) def test_fields_rewrite(self): class BasePerson(Document): @@ -89,107 +87,60 @@ class ValidatorErrorTest(unittest.TestCase): p = Person(age=15) self.assertRaises(ValidationError, p.validate) - def test_cascaded_save_wrong_reference(self): + def test_embedded_document_validation(self): + """Ensure that embedded documents may be validated. + """ + class Comment(EmbeddedDocument): + date = DateTimeField() + content = StringField(required=True) - class ADocument(Document): - val = IntField() + comment = Comment() + self.assertRaises(ValidationError, comment.validate) - class BDocument(Document): - a = ReferenceField(ADocument) + comment.content = 'test' + comment.validate() - ADocument.drop_collection() - BDocument.drop_collection() + comment.date = 4 + self.assertRaises(ValidationError, comment.validate) - a = ADocument() - a.val = 15 - a.save() + comment.date = datetime.now() + comment.validate() + self.assertEqual(comment._instance, None) - b = BDocument() - b.a = a - b.save() + def test_embedded_db_field_validate(self): - a.delete() - - b = BDocument.objects.first() - b.save(cascade=True) - - def test_shard_key(self): - class LogEntry(Document): - machine = StringField() - log = StringField() - - meta = { - 'shard_key': ('machine',) - } - - LogEntry.drop_collection() - - log = LogEntry() - log.machine = "Localhost" - log.save() - - log.log = "Saving" - log.save() - - def change_shard_key(): - log.machine = "127.0.0.1" - - self.assertRaises(OperationError, change_shard_key) - - def test_shard_key_primary(self): - class LogEntry(Document): - machine = StringField(primary_key=True) - log = StringField() - - meta = { - 'shard_key': ('machine',) - } - - LogEntry.drop_collection() - - log = LogEntry() - log.machine = "Localhost" - log.save() - - log.log = "Saving" - log.save() - - def change_shard_key(): - log.machine = "127.0.0.1" - - self.assertRaises(OperationError, change_shard_key) - - def test_kwargs_simple(self): - - class Embedded(EmbeddedDocument): - name = StringField() + class SubDoc(EmbeddedDocument): + val = IntField(required=True) class Doc(Document): - doc_name = StringField() - doc = EmbeddedDocumentField(Embedded) + id = StringField(primary_key=True) + e = EmbeddedDocumentField(SubDoc, db_field='eb') - classic_doc = Doc(doc_name="my doc", doc=Embedded(name="embedded doc")) - dict_doc = Doc(**{"doc_name": "my doc", - "doc": {"name": "embedded doc"}}) + try: + Doc(id="bad").validate() + except ValidationError, e: + self.assertTrue("SubDoc:None" in e.message) + self.assertEqual(e.to_dict(), { + 'e.val': 'Field is required'}) - self.assertEqual(classic_doc, dict_doc) - self.assertEqual(classic_doc._data, dict_doc._data) + Doc.drop_collection() - def test_kwargs_complex(self): + Doc(id="test", e=SubDoc(val=15)).save() - class Embedded(EmbeddedDocument): - name = StringField() + doc = Doc.objects.first() + keys = doc._data.keys() + self.assertEqual(2, len(keys)) + self.assertTrue('id' in keys) + self.assertTrue('e' in keys) - class Doc(Document): - doc_name = StringField() - docs = ListField(EmbeddedDocumentField(Embedded)) + doc.e.val = "OK" + try: + doc.save() + except ValidationError, e: + self.assertTrue("SubDoc:test" in e.message) + self.assertEqual(e.to_dict(), { + 'e.val': 'Field is required'}) - classic_doc = Doc(doc_name="my doc", docs=[ - Embedded(name="embedded doc1"), - Embedded(name="embedded doc2")]) - dict_doc = Doc(**{"doc_name": "my doc", - "docs": [{"name": "embedded doc1"}, - {"name": "embedded doc2"}]}) - self.assertEqual(classic_doc, dict_doc) - self.assertEqual(classic_doc._data, dict_doc._data) +if __name__ == '__main__': + unittest.main() From de2f774e8533ac18c161182cd7c37754e8d68844 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 13:29:44 +0000 Subject: [PATCH 0943/1279] Fix validation test --- tests/document/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/document/validation.py b/tests/document/validation.py index aaf6b0c..0f67f50 100644 --- a/tests/document/validation.py +++ b/tests/document/validation.py @@ -137,7 +137,7 @@ class ValidatorErrorTest(unittest.TestCase): try: doc.save() except ValidationError, e: - self.assertTrue("SubDoc:test" in e.message) + self.assertTrue("Doc:test" in e.message) self.assertEqual(e.to_dict(), { 'e.val': 'Field is required'}) From f182daa85eae94ccfffaa40e04f6370c007d10c4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 13:32:21 +0000 Subject: [PATCH 0944/1279] Fixed Documents deleted via a queryset don't call any signals (#105) --- docs/changelog.rst | 1 + mongoengine/queryset/queryset.py | 11 ++++++--- tests/document/instance.py | 40 ++++++++++++++++++++++++++++++-- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 601c2db..8b41d6e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -43,6 +43,7 @@ Changes in 0.8.X - Undefined data should not override instance methods (#49) - Added Django Group and Permission (#142) - Added Doc class and pk to Validation messages (#69) +- Fixed Documents deleted via a queryset don't call any signals (#105) Changes in 0.7.9 ================ diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index b5e3351..65703c3 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -364,10 +364,15 @@ class QuerySet(object): queryset = self.clone() doc = queryset._document - # Handle deletes where skips or limits have been applied - if queryset._skip or queryset._limit: + has_delete_signal = ( + signals.pre_delete.has_receivers_for(self._document) or + signals.post_delete.has_receivers_for(self._document)) + + # Handle deletes where skips or limits have been applied or has a + # delete signal + if queryset._skip or queryset._limit or has_delete_signal: for doc in queryset: - doc.delete() + doc.delete(safe=safe) return delete_rules = doc._meta.get('delete_rules') or {} diff --git a/tests/document/instance.py b/tests/document/instance.py index 3d4e8a9..172f0cc 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -19,6 +19,7 @@ from mongoengine.queryset import NULLIFY, Q from mongoengine.connection import get_db from mongoengine.base import get_document from mongoengine.context_managers import switch_db +from mongoengine import signals TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), '../fields/mongoengine.png') @@ -1375,7 +1376,6 @@ class InstanceTest(unittest.TestCase): author.delete() self.assertEqual(len(BlogPost.objects), 0) - def test_reverse_delete_rule_cascade_and_nullify_complex_field(self): """Ensure that a referenced document is also deleted upon deletion for complex fields. @@ -1410,6 +1410,43 @@ class InstanceTest(unittest.TestCase): author.delete() self.assertEqual(len(BlogPost.objects), 0) + + def test_reverse_delete_rule_cascade_triggers_pre_delete_signal(self): + ''' ensure the pre_delete signal is triggered upon a cascading deletion + setup a blog post with content, an author and editor + delete the author which triggers deletion of blogpost via cascade + blog post's pre_delete signal alters an editor attribute + ''' + class Editor(self.Person): + review_queue = IntField(default=0) + + class BlogPost(Document): + content = StringField() + author = ReferenceField(self.Person, reverse_delete_rule=CASCADE) + editor = ReferenceField(Editor) + + @classmethod + def pre_delete(cls, sender, document, **kwargs): + # decrement the docs-to-review count + document.editor.update(dec__review_queue=1) + + signals.pre_delete.connect(BlogPost.pre_delete, sender=BlogPost) + + self.Person.drop_collection() + BlogPost.drop_collection() + Editor.drop_collection() + + author = self.Person(name='Will S.').save() + editor = Editor(name='Max P.', review_queue=1).save() + BlogPost(content='wrote some books', author=author, + editor=editor).save() + + # delete the author, the post is also deleted due to the CASCADE rule + author.delete() + # the pre-delete signal should have decremented the editor's queue + editor = Editor.objects(name='Max P.').get() + self.assertEqual(editor.review_queue, 0) + def test_two_way_reverse_delete_rule(self): """Ensure that Bi-Directional relationships work with reverse_delete_rule @@ -1426,7 +1463,6 @@ class InstanceTest(unittest.TestCase): Bar.register_delete_rule(Foo, 'bar', NULLIFY) Foo.register_delete_rule(Bar, 'foo', NULLIFY) - Bar.drop_collection() Foo.drop_collection() From 0cbd3663e47d677aa4921070888d32ae4823a9f1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 13:40:28 +0000 Subject: [PATCH 0945/1279] Updated tests --- tests/all_warnings/__init__.py | 6 +++++- tests/document/class_methods.py | 2 ++ tests/document/dynamic.py | 1 - tests/document/indexes.py | 1 - tests/document/inheritance.py | 2 ++ tests/document/validation.py | 4 ++-- tests/test_connection.py | 9 +++++---- tests/test_context_managers.py | 2 ++ tests/test_dereference.py | 1 - tests/test_django.py | 5 +++++ tests/test_replicaset_connection.py | 2 ++ tests/test_signals.py | 15 ++++++++++----- 12 files changed, 35 insertions(+), 15 deletions(-) diff --git a/tests/all_warnings/__init__.py b/tests/all_warnings/__init__.py index 220b0bb..8cbe22d 100644 --- a/tests/all_warnings/__init__.py +++ b/tests/all_warnings/__init__.py @@ -3,7 +3,8 @@ This test has been put into a module. This is because it tests warnings that only get triggered on first hit. This way we can ensure its imported into the top level and called first by the test suite. """ - +import sys +sys.path[0:0] = [""] import unittest import warnings @@ -88,3 +89,6 @@ class AllWarnings(unittest.TestCase): self.assertEqual(SyntaxWarning, warning["category"]) self.assertEqual('non_abstract_base', InheritedDocumentFailTest._get_collection_name()) + +import sys +sys.path[0:0] = [""] \ No newline at end of file diff --git a/tests/document/class_methods.py b/tests/document/class_methods.py index 8e9a877..83e68ff 100644 --- a/tests/document/class_methods.py +++ b/tests/document/class_methods.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import with_statement +import sys +sys.path[0:0] = [""] import unittest from mongoengine import * diff --git a/tests/document/dynamic.py b/tests/document/dynamic.py index 4848b8f..5881cd0 100644 --- a/tests/document/dynamic.py +++ b/tests/document/dynamic.py @@ -1,6 +1,5 @@ import unittest import sys - sys.path[0:0] = [""] from mongoengine import * diff --git a/tests/document/indexes.py b/tests/document/indexes.py index c059590..ff08ef1 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -2,7 +2,6 @@ from __future__ import with_statement import unittest import sys - sys.path[0:0] = [""] import os diff --git a/tests/document/inheritance.py b/tests/document/inheritance.py index c5e1860..3b550f1 100644 --- a/tests/document/inheritance.py +++ b/tests/document/inheritance.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +import sys +sys.path[0:0] = [""] import unittest import warnings diff --git a/tests/document/validation.py b/tests/document/validation.py index 0f67f50..24ffed6 100644 --- a/tests/document/validation.py +++ b/tests/document/validation.py @@ -121,7 +121,7 @@ class ValidatorErrorTest(unittest.TestCase): except ValidationError, e: self.assertTrue("SubDoc:None" in e.message) self.assertEqual(e.to_dict(), { - 'e.val': 'Field is required'}) + "e": {'val': 'OK could not be converted to int'}}) Doc.drop_collection() @@ -139,7 +139,7 @@ class ValidatorErrorTest(unittest.TestCase): except ValidationError, e: self.assertTrue("Doc:test" in e.message) self.assertEqual(e.to_dict(), { - 'e.val': 'Field is required'}) + "e": {'val': 'OK could not be converted to int'}}) if __name__ == '__main__': diff --git a/tests/test_connection.py b/tests/test_connection.py index c32d231..5b9743d 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1,13 +1,14 @@ from __future__ import with_statement -import datetime -import pymongo +import sys +sys.path[0:0] = [""] import unittest +import datetime -import mongoengine.connection - +import pymongo from bson.tz_util import utc from mongoengine import * +import mongoengine.connection from mongoengine.connection import get_db, get_connection, ConnectionError from mongoengine.context_managers import switch_db diff --git a/tests/test_context_managers.py b/tests/test_context_managers.py index c9efe8b..eef63be 100644 --- a/tests/test_context_managers.py +++ b/tests/test_context_managers.py @@ -1,4 +1,6 @@ from __future__ import with_statement +import sys +sys.path[0:0] = [""] import unittest from mongoengine import * diff --git a/tests/test_dereference.py b/tests/test_dereference.py index adbc519..4198f3c 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1,7 +1,6 @@ from __future__ import with_statement import sys sys.path[0:0] = [""] - import unittest from bson import DBRef, ObjectId diff --git a/tests/test_django.py b/tests/test_django.py index 3b0b04f..563f407 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -1,4 +1,6 @@ from __future__ import with_statement +import sys +sys.path[0:0] = [""] import unittest from nose.plugins.skip import SkipTest from mongoengine.python_support import PY3 @@ -163,3 +165,6 @@ class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): key = session.session_key session = SessionStore(key) self.assertTrue('test_expire' in session, 'Session has expired before it is expected') + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_replicaset_connection.py b/tests/test_replicaset_connection.py index 3118c5a..d27960f 100644 --- a/tests/test_replicaset_connection.py +++ b/tests/test_replicaset_connection.py @@ -1,3 +1,5 @@ +import sys +sys.path[0:0] = [""] import unittest import pymongo diff --git a/tests/test_signals.py b/tests/test_signals.py index 2ca820d..fc638cf 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +import sys +sys.path[0:0] = [""] import unittest from mongoengine import * @@ -21,6 +23,7 @@ class SignalTests(unittest.TestCase): def setUp(self): connect(db='mongoenginetest') + class Author(Document): name = StringField() @@ -70,7 +73,6 @@ class SignalTests(unittest.TestCase): signal_output.append('Not loaded') self.Author = Author - class Another(Document): name = StringField() @@ -122,8 +124,8 @@ class SignalTests(unittest.TestCase): self.ExplicitId = ExplicitId self.ExplicitId.objects.delete() - # Save up the number of connected signals so that we can check at the end - # that all the signals we register get properly unregistered + # Save up the number of connected signals so that we can check at the + # end that all the signals we register get properly unregistered self.pre_signals = ( len(signals.pre_init.receivers), len(signals.post_init.receivers), @@ -192,7 +194,7 @@ class SignalTests(unittest.TestCase): """ Model saves should throw some signals. """ def create_author(): - a1 = self.Author(name='Bill Shakespeare') + self.Author(name='Bill Shakespeare') def bulk_create_author_with_load(): a1 = self.Author(name='Bill Shakespeare') @@ -216,7 +218,7 @@ class SignalTests(unittest.TestCase): ]) a1.reload() - a1.name='William Shakespeare' + a1.name = 'William Shakespeare' self.assertEqual(self.get_signal_output(a1.save), [ "pre_save signal, William Shakespeare", "post_save signal, William Shakespeare", @@ -257,3 +259,6 @@ class SignalTests(unittest.TestCase): self.assertEqual(self.get_signal_output(ei.save), ['Is created']) # second time, it must be an update self.assertEqual(self.get_signal_output(ei.save), ['Is updated']) + +if __name__ == '__main__': + unittest.main() From 8c1f8e54cdb97a6bb4bf8523c05ec1b783e7a283 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 14:12:47 +0000 Subject: [PATCH 0946/1279] Added the "get_decoded" method to the MongoSession class (#216) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + docs/upgrade.rst | 2 +- mongoengine/django/sessions.py | 3 ++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index ac811dd..64f0b0a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -138,4 +138,5 @@ that much better: * Manuel Hermann * Gustavo Gawryszewski * Max Countryman - * caitifbrito \ No newline at end of file + * caitifbrito + * lcya86 刘春洋 \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 8b41d6e..6556154 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -44,6 +44,7 @@ Changes in 0.8.X - Added Django Group and Permission (#142) - Added Doc class and pk to Validation messages (#69) - Fixed Documents deleted via a queryset don't call any signals (#105) +- Added the "get_decoded" method to the MongoSession class (#216) Changes in 0.7.9 ================ diff --git a/docs/upgrade.rst b/docs/upgrade.rst index fcd5f71..8724503 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -298,5 +298,5 @@ Alternatively, you can rename your collections eg :: mongodb 1.8 > 2.0 + =================== -Its been reported that indexes may need to be recreated to the newer version of indexes. +Its been reported that indexes may need to be recreated to the newer version of indexes. To do this drop indexes and call ``ensure_indexes`` on each model. diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index 17cae8a..1c9288e 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -25,6 +25,7 @@ MONGOENGINE_SESSION_DATA_ENCODE = getattr( settings, 'MONGOENGINE_SESSION_DATA_ENCODE', True) + class MongoSession(Document): session_key = fields.StringField(primary_key=True, max_length=40) session_data = fields.StringField() if MONGOENGINE_SESSION_DATA_ENCODE \ @@ -34,7 +35,7 @@ class MongoSession(Document): meta = {'collection': MONGOENGINE_SESSION_COLLECTION, 'db_alias': MONGOENGINE_SESSION_DB_ALIAS, 'allow_inheritance': False} - + def get_decoded(self): return SessionStore().decode(self.session_data) From 5b161b7445eb3d53b067ecab6b25a92a6001930f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 14:17:54 +0000 Subject: [PATCH 0947/1279] ReadPreference that overrides slave_okay (#218) --- mongoengine/queryset/queryset.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 65703c3..4c66461 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1095,11 +1095,12 @@ class QuerySet(object): def _cursor_args(self): cursor_args = { 'snapshot': self._snapshot, - 'timeout': self._timeout, - 'slave_okay': self._slave_okay, + 'timeout': self._timeout } if self._read_preference is not None: cursor_args['read_preference'] = self._read_preference + else: + cursor_args['slave_okay'] = self._slave_okay if self._loaded_fields: cursor_args['fields'] = self._loaded_fields.as_dict() return cursor_args From 3208a7f15dbe8d99253c3a397c354812ba883953 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 14:28:40 +0000 Subject: [PATCH 0948/1279] Merge fix tests --- tests/fields/fields.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 8cf3bb5..7c4e785 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -1725,9 +1725,6 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() -<<<<<<< HEAD:tests/fields/fields.py -======= - def test_simple_choices_validation_invalid_value(self): """Ensure that error messages are correct. """ @@ -2060,7 +2057,6 @@ class FieldTest(unittest.TestCase): self.assertEqual(test_file.the_file.read(), b('Hello, World!')) ->>>>>>> de5fbfde2ca96b93490e0bc96e04f3aa4affcfb5:tests/test_fields.py def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """ From 1ca098c40291ff205e64080dbb1877e8083c3c67 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 14:40:26 +0000 Subject: [PATCH 0949/1279] Fixed invalid choices error bubbling (#214) --- docs/changelog.rst | 1 + tests/fields/__init__.py | 2 +- tests/fields/{file.py => file_tests.py} | 0 3 files changed, 2 insertions(+), 1 deletion(-) rename tests/fields/{file.py => file_tests.py} (100%) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6556154..f289b39 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -45,6 +45,7 @@ Changes in 0.8.X - Added Doc class and pk to Validation messages (#69) - Fixed Documents deleted via a queryset don't call any signals (#105) - Added the "get_decoded" method to the MongoSession class (#216) +- Fixed invalid choices error bubbling (#214) Changes in 0.7.9 ================ diff --git a/tests/fields/__init__.py b/tests/fields/__init__.py index 86dfa84..0731838 100644 --- a/tests/fields/__init__.py +++ b/tests/fields/__init__.py @@ -1,2 +1,2 @@ from fields import * -from file import * \ No newline at end of file +from file_tests import * \ No newline at end of file diff --git a/tests/fields/file.py b/tests/fields/file_tests.py similarity index 100% rename from tests/fields/file.py rename to tests/fields/file_tests.py From 4177fc6df2800908c0b868bd577b9bce1bc85cfa Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 15:57:33 +0000 Subject: [PATCH 0950/1279] Can call del Doc.attr to delete field value --- mongoengine/document.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 66aa263..525d964 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -48,17 +48,6 @@ class EmbeddedDocument(BaseDocument): super(EmbeddedDocument, self).__init__(*args, **kwargs) self._changed_fields = [] - def __delattr__(self, *args, **kwargs): - """Handle deletions of fields""" - field_name = args[0] - if field_name in self._fields: - default = self._fields[field_name].default - if callable(default): - default = default() - setattr(self, field_name, default) - else: - super(EmbeddedDocument, self).__delattr__(*args, **kwargs) - def __eq__(self, other): if isinstance(other, self.__class__): return self._data == other._data From 9ca632d5184cac9ea5f7d660a63e79658881309e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 16:00:38 +0000 Subject: [PATCH 0951/1279] Updated Save so it calls $set and $unset in a single operation (#211) --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index ff58024..fb895d7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -141,3 +141,4 @@ that much better: * caitifbrito * lcya86 刘春洋 * Martin Alderete (https://github.com/malderete) + * Nick Joyce diff --git a/docs/changelog.rst b/docs/changelog.rst index f289b39..213e6fb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -46,6 +46,7 @@ Changes in 0.8.X - Fixed Documents deleted via a queryset don't call any signals (#105) - Added the "get_decoded" method to the MongoSession class (#216) - Fixed invalid choices error bubbling (#214) +- Updated Save so it calls $set and $unset in a single operation (#211) Changes in 0.7.9 ================ From 39dac7d4dbe809e7eacac190d348326718ec0c1b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 16:26:01 +0000 Subject: [PATCH 0952/1279] Fix file open rules --- tests/fields/fields.py | 8 ++++---- tests/fields/file_tests.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 7c4e785..124c953 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -1950,7 +1950,7 @@ class FieldTest(unittest.TestCase): TestImage.drop_collection() t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() t = TestImage.objects.first() @@ -1973,7 +1973,7 @@ class FieldTest(unittest.TestCase): TestImage.drop_collection() t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() t = TestImage.objects.first() @@ -1996,7 +1996,7 @@ class FieldTest(unittest.TestCase): TestImage.drop_collection() t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() t = TestImage.objects.first() @@ -2019,7 +2019,7 @@ class FieldTest(unittest.TestCase): TestImage.drop_collection() t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() t = TestImage.objects.first() diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py index a39dadb..44d2862 100644 --- a/tests/fields/file_tests.py +++ b/tests/fields/file_tests.py @@ -208,7 +208,7 @@ class FileTest(unittest.TestCase): Animal.drop_collection() marmot = Animal(genus='Marmota', family='Sciuridae') - marmot_photo = open(TEST_IMAGE_PATH, 'r') # Retrieve a photo from disk + marmot_photo = open(TEST_IMAGE_PATH, 'rb') # Retrieve a photo from disk marmot.photo.put(marmot_photo, content_type='image/jpeg', foo='bar') marmot.photo.close() marmot.save() @@ -251,7 +251,7 @@ class FileTest(unittest.TestCase): TestImage.drop_collection() t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() t = TestImage.objects.first() @@ -274,7 +274,7 @@ class FileTest(unittest.TestCase): TestImage.drop_collection() t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() t = TestImage.objects.first() @@ -297,7 +297,7 @@ class FileTest(unittest.TestCase): TestImage.drop_collection() t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() t = TestImage.objects.first() @@ -320,7 +320,7 @@ class FileTest(unittest.TestCase): TestImage.drop_collection() t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'r')) + t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() t = TestImage.objects.first() From 156ca44a135a0bc832645b0292ca3ff0e02a8148 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 28 Jan 2013 16:49:34 +0000 Subject: [PATCH 0953/1279] Doc fix thanks to @jabapyth (#206) --- AUTHORS | 1 + mongoengine/queryset/queryset.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index fb895d7..7efd3ad 100644 --- a/AUTHORS +++ b/AUTHORS @@ -142,3 +142,4 @@ that much better: * lcya86 刘春洋 * Martin Alderete (https://github.com/malderete) * Nick Joyce + * Jared Forsyth diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 4c66461..d313740 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -467,7 +467,8 @@ class QuerySet(object): def with_id(self, object_id): """Retrieve the object matching the id provided. Uses `object_id` only - and raises InvalidQueryError if a filter has been applied. + and raises InvalidQueryError if a filter has been applied. Returns + `None` if no document exists with that id. :param object_id: the value for the id of the document to look up From 025e17701bd6b403ea5aec41f484918d3c718e4e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 29 Jan 2013 10:33:13 +0000 Subject: [PATCH 0954/1279] Fixed inner queryset looping (#204) --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/queryset/queryset.py | 10 ++++---- tests/queryset/queryset.py | 41 ++++++++++++++++++++++++++++++++ tests/test_django.py | 18 ++++++++++++++ 5 files changed, 67 insertions(+), 4 deletions(-) diff --git a/AUTHORS b/AUTHORS index 7efd3ad..903542a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -143,3 +143,4 @@ that much better: * Martin Alderete (https://github.com/malderete) * Nick Joyce * Jared Forsyth + * Kenneth Falck diff --git a/docs/changelog.rst b/docs/changelog.rst index 213e6fb..e9783cd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -47,6 +47,7 @@ Changes in 0.8.X - Added the "get_decoded" method to the MongoSession class (#216) - Fixed invalid choices error bubbling (#214) - Updated Save so it calls $set and $unset in a single operation (#211) +- Fixed inner queryset looping (#204) Changes in 0.7.9 ================ diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index d313740..ba6134f 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -112,8 +112,11 @@ class QuerySet(object): def __iter__(self): """Support iterator protocol""" - self.rewind() - return self + queryset = self + if queryset._iter: + queryset = self.clone() + queryset.rewind() + return queryset def __len__(self): return self.count() @@ -159,7 +162,6 @@ class QuerySet(object): .. versionchanged:: 0.6.13 Now doesnt modify the cursor """ - if self._iter: return '.. queryset mid-iteration ..' @@ -537,7 +539,7 @@ class QuerySet(object): c._cursor_obj = self._cursor_obj.clone() if self._slice: - c._cursor_obj[self._slice] + c._cursor[self._slice] return c diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 3d5c659..d5e80be 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3125,5 +3125,46 @@ class QuerySetTest(unittest.TestCase): Organization)) self.assertTrue(isinstance(qs.first().organization, Organization)) + def test_nested_queryset_iterator(self): + # Try iterating the same queryset twice, nested. + names = ['Alice', 'Bob', 'Chuck', 'David', 'Eric', 'Francis', 'George'] + + class User(Document): + name = StringField() + + def __unicode__(self): + return self.name + + User.drop_collection() + + for name in names: + User(name=name).save() + + users = User.objects.all().order_by('name') + + outer_count = 0 + inner_count = 0 + inner_total_count = 0 + + self.assertEqual(len(users), 7) + + for i, outer_user in enumerate(users): + self.assertEqual(outer_user.name, names[i]) + outer_count += 1 + inner_count = 0 + + # Calling len might disrupt the inner loop if there are bugs + self.assertEqual(len(users), 7) + + for j, inner_user in enumerate(users): + self.assertEqual(inner_user.name, names[j]) + inner_count += 1 + inner_total_count += 1 + + self.assertEqual(inner_count, 7) # inner loop should always be executed seven times + + self.assertEqual(outer_count, 7) # outer loop should be executed seven times total + self.assertEqual(inner_total_count, 7 * 7) # inner loop should be executed fourtynine times total + if __name__ == '__main__': unittest.main() diff --git a/tests/test_django.py b/tests/test_django.py index 563f407..dceeba2 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -136,7 +136,25 @@ class QuerySetTest(unittest.TestCase): start = end - 1 self.assertEqual(t.render(Context(d)), u'%d:%d:' % (start, end)) + def test_nested_queryset_template_iterator(self): + # Try iterating the same queryset twice, nested, in a Django template. + names = ['A', 'B', 'C', 'D'] + class User(Document): + name = StringField() + + def __unicode__(self): + return self.name + + User.drop_collection() + + for name in names: + User(name=name).save() + + users = User.objects.all().order_by('name') + template = Template("{% for user in users %}{{ user.name }}{% ifequal forloop.counter 2 %} {% for inner_user in users %}{{ inner_user.name }}{% endfor %} {% endifequal %}{% endfor %}") + rendered = template.render(Context({'users': users})) + self.assertEqual(rendered, 'AB ABCD CD') class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): backend = SessionStore From 2e18199eb2cad168f07caefd68f4cdd5f3078135 Mon Sep 17 00:00:00 2001 From: hellysmile Date: Fri, 1 Feb 2013 04:17:16 +0200 Subject: [PATCH 0955/1279] Django sessions TTL support --- docs/django.rst | 3 +++ mongoengine/django/sessions.py | 14 +++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/django.rst b/docs/django.rst index 144baab..58eadc6 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -45,6 +45,9 @@ into you settings module:: SESSION_ENGINE = 'mongoengine.django.sessions' +Django provides session cookie, which expires after ```SESSION_COOKIE_AGE``` seconds, but doesnt delete cookie at sessions backend, so ```'mongoengine.django.sessions'``` supports `mongodb TTL +`_. + .. versionadded:: 0.2.1 Storage diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index 810b626..20e6b62 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -31,9 +31,17 @@ class MongoSession(Document): else fields.DictField() expire_date = fields.DateTimeField() - meta = {'collection': MONGOENGINE_SESSION_COLLECTION, - 'db_alias': MONGOENGINE_SESSION_DB_ALIAS, - 'allow_inheritance': False} + meta = { + 'collection': MONGOENGINE_SESSION_COLLECTION, + 'db_alias': MONGOENGINE_SESSION_DB_ALIAS, + 'allow_inheritance': False, + 'indexes': [ + { + 'fields': ['expire_date'], + 'expireAfterSeconds': settings.SESSION_COOKIE_AGE + } + ] + } class SessionStore(SessionBase): From d6b4ca7a985c1ba4c25692d9731b8b6cd12e7fb0 Mon Sep 17 00:00:00 2001 From: hellysmile Date: Fri, 1 Feb 2013 04:19:55 +0200 Subject: [PATCH 0956/1279] Fix docs quotes --- docs/django.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/django.rst b/docs/django.rst index 58eadc6..b2acee7 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -45,7 +45,7 @@ into you settings module:: SESSION_ENGINE = 'mongoengine.django.sessions' -Django provides session cookie, which expires after ```SESSION_COOKIE_AGE``` seconds, but doesnt delete cookie at sessions backend, so ```'mongoengine.django.sessions'``` supports `mongodb TTL +Django provides session cookie, which expires after ```SESSION_COOKIE_AGE``` seconds, but doesnt delete cookie at sessions backend, so ``'mongoengine.django.sessions'`` supports `mongodb TTL `_. .. versionadded:: 0.2.1 From 8df9ff90cb90aebc21b82fd90d2dabd084097d6e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 14 Feb 2013 08:26:36 +0000 Subject: [PATCH 0957/1279] Update LICENSE --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index cef91cc..45f233c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2009-2012 See AUTHORS +Copyright (c) 2009 See AUTHORS Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation From 3477b0107a227b88a1936fb23fa24ad7b26524a6 Mon Sep 17 00:00:00 2001 From: Loic Raucy Date: Tue, 26 Feb 2013 11:12:37 +0100 Subject: [PATCH 0958/1279] Added regression test for numerical string keys. --- tests/test_fields.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_fields.py b/tests/test_fields.py index 28af1b2..4766900 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -965,6 +965,24 @@ class FieldTest(unittest.TestCase): doc = self.db.test.find_one() self.assertEqual(doc['x']['DICTIONARY_KEY']['i'], 2) + def test_mapfield_numerical_index(self): + """Ensure that MapField accept numeric strings as indexes.""" + class Embedded(EmbeddedDocument): + name = StringField() + + class Test(Document): + my_map = MapField(EmbeddedDocumentField(Embedded)) + + Test.drop_collection() + + test = Test() + test.my_map['1'] = Embedded(name='test') + test.save() + test.my_map['1'].name = 'test updated' + test.save() + + Test.drop_collection() + def test_map_field_lookup(self): """Ensure MapField lookups succeed on Fields without a lookup method""" From d0245bb5ba3b0f4ca4ce654fd199c167ce8c5e96 Mon Sep 17 00:00:00 2001 From: Loic Raucy Date: Tue, 26 Feb 2013 11:14:47 +0100 Subject: [PATCH 0959/1279] Fixed #238: dictfields handle numerical strings indexes. --- mongoengine/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 013afe7..4f302a8 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1193,7 +1193,7 @@ class BaseDocument(object): for p in parts: if isinstance(d, DBRef): break - elif p.isdigit(): + elif isinstance(d, list) and p.isdigit(): d = d[int(p)] elif hasattr(d, 'get'): d = d.get(p) @@ -1224,7 +1224,7 @@ class BaseDocument(object): parts = path.split('.') db_field_name = parts.pop() for p in parts: - if p.isdigit(): + if isinstance(d, list) and p.isdigit(): d = d[int(p)] elif (hasattr(d, '__getattribute__') and not isinstance(d, dict)): From 3c78757778551bfe28495e3630b3e6b991e2a86f Mon Sep 17 00:00:00 2001 From: Benoit Louy Date: Tue, 26 Feb 2013 09:55:29 -0500 Subject: [PATCH 0960/1279] fix travis build: builds were failing because libz.so location changed. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 550cc6c..2dc8894 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ env: - PYMONGO=2.3 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install zlib1g zlib1g-dev; fi - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo ln -s /usr/lib/i386-linux-gnu/libz.so /usr/lib/; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo ln -s /usr/lib/x86_64-linux-gnu/libz.so /usr/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install PIL --use-mirrors ; true; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install PIL --use-mirrors ; true; fi - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi From 0d2e84b16b286fd7724676794a68aa0610285afc Mon Sep 17 00:00:00 2001 From: benoitlouy Date: Thu, 28 Feb 2013 00:37:34 -0500 Subject: [PATCH 0961/1279] Fix for issue #237: clearing changed fields recursively in EmbeddedDocuments after saving a Document --- mongoengine/base.py | 16 ++++++++++++++++ mongoengine/document.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 013afe7..521756c 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1123,6 +1123,22 @@ class BaseDocument(object): key not in self._changed_fields): self._changed_fields.append(key) + def _clear_changed_fields(self): + self._changed_fields = [] + EmbeddedDocumentField = _import_class("EmbeddedDocumentField") + for field_name, field in self._fields.iteritems(): + if (isinstance(field, ComplexBaseField) and + isinstance(field.field, EmbeddedDocumentField)): + field_value = getattr(self, field_name, None) + if field_value: + for idx in (field_value if isinstance(field_value, dict) + else xrange(len(field_value))): + field_value[idx]._clear_changed_fields() + elif isinstance(field, EmbeddedDocumentField): + field_value = getattr(self, field_name, None) + if field_value: + field_value._clear_changed_fields() + def _get_changed_fields(self, key='', inspected=None): """Returns a list of all fields that have explicitly been changed. """ diff --git a/mongoengine/document.py b/mongoengine/document.py index 7b3afaf..0cf07a9 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -269,7 +269,7 @@ class Document(BaseDocument): if id_field not in self._meta.get('shard_key', []): self[id_field] = self._fields[id_field].to_python(object_id) - self._changed_fields = [] + self._clear_changed_fields() self._created = False signals.post_save.send(self.__class__, document=self, created=created) return self From 43327ea4e1ca2041de248b438d77b392bd59f6c3 Mon Sep 17 00:00:00 2001 From: benoitlouy Date: Fri, 1 Mar 2013 07:38:28 -0500 Subject: [PATCH 0962/1279] Add testcase for issue #237 --- tests/test_document.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_document.py b/tests/test_document.py index 3e8d813..134ba34 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -3368,6 +3368,40 @@ class DocumentTest(unittest.TestCase): } ) ]), "1,2") + def test_complex_nesting_document_and_embeddeddocument(self): + class Macro(EmbeddedDocument): + value = DynamicField(default="UNDEFINED") + + class Parameter(EmbeddedDocument): + macros = MapField(EmbeddedDocumentField(Macro)) + + def expand(self): + self.macros["test"] = Macro() + + class Node(Document): + parameters = MapField(EmbeddedDocumentField(Parameter)) + + def expand(self): + self.flattened_parameter = {} + for parameter_name, parameter in self.parameters.iteritems(): + parameter.expand() + + class System(Document): + name = StringField(required=True) + nodes = MapField(ReferenceField(Node, dbref=False)) + + def save(self, *args, **kwargs): + for node_name, node in self.nodes.iteritems(): + node.expand() + node.save(*args, **kwargs) + super(System, self).save(*args, **kwargs) + + system = System(name="system") + system.save() + system.nodes["node"] = Node() + system.save() + system.nodes["node"].parameters["param"] = Parameter() + system.save() class ValidatorErrorTest(unittest.TestCase): From 3e4a900279758667ebcaaa1683c86324b06ab6ec Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 7 Mar 2013 15:25:18 +0000 Subject: [PATCH 0963/1279] Adding google analytics --- docs/_templates/layout.html | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 docs/_templates/layout.html diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 0000000..8172f21 --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,16 @@ +{% extends "!layout.html" %} + +{% block footer %} +{{ super() }} + +{% endblock %} From eeb672feb9e35416d452b386c98313134f27cc2e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 7 Mar 2013 15:37:01 +0000 Subject: [PATCH 0964/1279] Removing custom layout --- docs/_templates/layout.html | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 docs/_templates/layout.html diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html deleted file mode 100644 index 8172f21..0000000 --- a/docs/_templates/layout.html +++ /dev/null @@ -1,16 +0,0 @@ -{% extends "!layout.html" %} - -{% block footer %} -{{ super() }} - -{% endblock %} From d36f6e7f24d5fc22076fc192d6f3174b0c1ee58c Mon Sep 17 00:00:00 2001 From: zmolodchenko Date: Sat, 9 Mar 2013 21:08:10 +0200 Subject: [PATCH 0965/1279] fix error reporting, where choices is list of flat values --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 013afe7..f73af4c 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -252,7 +252,7 @@ class BaseField(object): elif value_to_check not in self.choices: msg = ('Value must be %s of %s' % (err_msg, unicode(self.choices))) - self.error() + self.error(msg) # check validation argument if self.validation is not None: From 41a698b442881b7f40cef5b3c6954097d0965a10 Mon Sep 17 00:00:00 2001 From: Russ Weeks Date: Tue, 12 Mar 2013 10:28:29 -0700 Subject: [PATCH 0966/1279] Changed dereference.py to keep tuples as tuples --- mongoengine/dereference.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 386dbf4..fcb6d89 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -171,6 +171,7 @@ class DeReference(object): if not hasattr(items, 'items'): is_list = True + as_tuple = isinstance(items, tuple) iterator = enumerate(items) data = [] else: @@ -205,7 +206,7 @@ class DeReference(object): if instance and name: if is_list: - return BaseList(data, instance, name) + return tuple(data) if as_tuple else BaseList(data, instance, name) return BaseDict(data, instance, name) depth += 1 return data From f9cd8b1841631c693ebf40479d215bee4ed13d30 Mon Sep 17 00:00:00 2001 From: Russ Weeks Date: Tue, 12 Mar 2013 12:45:38 -0700 Subject: [PATCH 0967/1279] added unit test for dereference patch --- tests/test_dereference.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 0eb891c..d7438d2 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -997,3 +997,34 @@ class FieldTest(unittest.TestCase): msg = Message.objects.get(id=1) self.assertEqual(0, msg.comments[0].id) self.assertEqual(1, msg.comments[1].id) + + def test_tuples_as_tuples(self): + """ + Ensure that tuples remain tuples when they are + inside a ComplexBaseField + """ + from mongoengine.base import BaseField + class EnumField(BaseField): + def __init__(self, **kwargs): + super(EnumField,self).__init__(**kwargs) + + def to_mongo(self, value): + return value + + def to_python(self, value): + return tuple(value) + + class TestDoc(Document): + items = ListField(EnumField()) + + TestDoc.drop_collection() + tuples = [(100,'Testing')] + doc = TestDoc() + doc.items = tuples + doc.save() + x = TestDoc.objects().get() + self.assertTrue(x is not None) + self.assertTrue(len(x.items) == 1) + self.assertTrue(tuple(x.items[0]) in tuples) + self.assertTrue(x.items[0] in tuples) + From 2d6ae1691204b1036ee5b7595721303a8533a103 Mon Sep 17 00:00:00 2001 From: Jaepil Jeong Date: Thu, 14 Mar 2013 23:25:22 +0900 Subject: [PATCH 0968/1279] Added LongField to support 64-bit integer type. --- mongoengine/fields.py | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index de484a1..c40491b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -27,7 +27,7 @@ except ImportError: Image = None ImageOps = None -__all__ = ['StringField', 'IntField', 'FloatField', 'BooleanField', +__all__ = ['StringField', 'IntField', 'LongField', 'FloatField', 'BooleanField', 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', 'ObjectIdField', 'ReferenceField', 'ValidationError', 'MapField', 'DecimalField', 'ComplexDateTimeField', 'URLField', 'DynamicField', @@ -153,7 +153,7 @@ class EmailField(StringField): class IntField(BaseField): - """An integer field. + """An 32-bit integer field. """ def __init__(self, min_value=None, max_value=None, **kwargs): @@ -186,6 +186,40 @@ class IntField(BaseField): return int(value) +class LongField(BaseField): + """An 64-bit integer field. + """ + + def __init__(self, min_value=None, max_value=None, **kwargs): + self.min_value, self.max_value = min_value, max_value + super(LongField, self).__init__(**kwargs) + + def to_python(self, value): + try: + value = long(value) + except ValueError: + pass + return value + + def validate(self, value): + try: + value = long(value) + except: + self.error('%s could not be converted to long' % value) + + if self.min_value is not None and value < self.min_value: + self.error('Long value is too small') + + if self.max_value is not None and value > self.max_value: + self.error('Long value is too large') + + def prepare_query_value(self, op, value): + if value is None: + return value + + return long(value) + + class FloatField(BaseField): """An floating point number field. """ From e9464e32db4bcd10b332fb4764b3f9189e096f36 Mon Sep 17 00:00:00 2001 From: Jaepil Jeong Date: Thu, 14 Mar 2013 23:59:50 +0900 Subject: [PATCH 0969/1279] Added test cases for LongField. --- tests/test_fields.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_fields.py b/tests/test_fields.py index 28af1b2..3ceff8d 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -144,6 +144,17 @@ class FieldTest(unittest.TestCase): self.assertEqual(1, TestDocument.objects(int_fld__ne=None).count()) self.assertEqual(1, TestDocument.objects(float_fld__ne=None).count()) + def test_long_ne_operator(self): + class TestDocument(Document): + long_fld = LongField() + + TestDocument.drop_collection() + + TestDocument(long_fld=None).save() + TestDocument(long_fld=1).save() + + self.assertEqual(1, TestDocument.objects(long_fld__ne=None).count()) + def test_object_id_validation(self): """Ensure that invalid values cannot be assigned to string fields. """ @@ -217,6 +228,23 @@ class FieldTest(unittest.TestCase): person.age = 'ten' self.assertRaises(ValidationError, person.validate) + def test_long_validation(self): + """Ensure that invalid values cannot be assigned to long fields. + """ + class TestDocument(Document): + value = LongField(min_value=0, max_value=110) + + doc = TestDocument() + doc.value = 50 + doc.validate() + + doc.value = -1 + self.assertRaises(ValidationError, doc.validate) + doc.age = 120 + self.assertRaises(ValidationError, doc.validate) + doc.age = 'ten' + self.assertRaises(ValidationError, doc.validate) + def test_float_validation(self): """Ensure that invalid values cannot be assigned to float fields. """ From 67182713d96233d3d2feb0f67d39ddaf1789c692 Mon Sep 17 00:00:00 2001 From: Jaepil Jeong Date: Fri, 15 Mar 2013 00:12:48 +0900 Subject: [PATCH 0970/1279] Fixed potential overflow error. --- mongoengine/fields.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index c40491b..d7c7cf1 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -213,6 +213,9 @@ class LongField(BaseField): if self.max_value is not None and value > self.max_value: self.error('Long value is too large') + if value > 0x7FFFFFFFFFFFFFFF: + self.error('Long value is too large') + def prepare_query_value(self, op, value): if value is None: return value From a192029901c6c21b3cc949a6221a1090083ebf86 Mon Sep 17 00:00:00 2001 From: Aleksandr Sorokoumov Date: Sat, 16 Mar 2013 16:47:22 +0100 Subject: [PATCH 0971/1279] ReferenceField query chaining bug fixed. --- mongoengine/queryset.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index bff05fc..6c61ab9 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -367,6 +367,10 @@ class QuerySet(object): self._skip = None self._hint = -1 # Using -1 as None is a valid value for hint + def __deepcopy__(self, memo): + """Essential for chained queries with ReferenceFields involved""" + return self.clone() + def clone(self): """Creates a copy of the current :class:`~mongoengine.queryset.QuerySet` @@ -814,7 +818,6 @@ class QuerySet(object): mongo_query['$and'].append(value) else: mongo_query['$and'] = value - return mongo_query def get(self, *q_objs, **query): From a762a10decef552ccf6fbee413819d18bede05ea Mon Sep 17 00:00:00 2001 From: Jaepil Jeong Date: Mon, 18 Mar 2013 19:30:04 +0900 Subject: [PATCH 0972/1279] Revert "Fixed potential overflow error." This reverts commit 67182713d96233d3d2feb0f67d39ddaf1789c692. --- mongoengine/fields.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index d7c7cf1..c40491b 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -213,9 +213,6 @@ class LongField(BaseField): if self.max_value is not None and value > self.max_value: self.error('Long value is too large') - if value > 0x7FFFFFFFFFFFFFFF: - self.error('Long value is too large') - def prepare_query_value(self, op, value): if value is None: return value From f7515cfca80d870728ae5f03535dcedcd2a6a5b1 Mon Sep 17 00:00:00 2001 From: Aleksandr Sorokoumov Date: Mon, 18 Mar 2013 12:22:55 +0100 Subject: [PATCH 0973/1279] add myself to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 82a1dfa..fe62026 100644 --- a/AUTHORS +++ b/AUTHORS @@ -128,3 +128,4 @@ that much better: * Peter Teichman * Jakub Kot * Jorge Bastida + * Aleksandr Sorokoumov \ No newline at end of file From 165bea5bb97f24c4b06fedfe38ebb3d925052eaa Mon Sep 17 00:00:00 2001 From: Aleksandr Sorokoumov Date: Mon, 18 Mar 2013 12:32:49 +0100 Subject: [PATCH 0974/1279] QuerySet chaining test was supplemented with ReferenceField chaining test --- tests/test_queryset.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 5234cea..43bb70b 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -232,28 +232,33 @@ class QuerySetTest(unittest.TestCase): def test_chaining(self): class A(Document): - pass + s = StringField() class B(Document): - a = ReferenceField(A) + ref = ReferenceField(A) + boolfield = BooleanField(default=False) A.drop_collection() B.drop_collection() - a1 = A().save() - a2 = A().save() + a1 = A(s="test1").save() + a2 = A(s="test2").save() - B(a=a1).save() + B(ref=a1, boolfield=True).save() # Works - q1 = B.objects.filter(a__in=[a1, a2], a=a1)._query + q1 = B.objects.filter(ref__in=[a1, a2], ref=a1)._query # Doesn't work - q2 = B.objects.filter(a__in=[a1, a2]) - q2 = q2.filter(a=a1)._query - + q2 = B.objects.filter(ref__in=[a1, a2]) + q2 = q2.filter(ref=a1)._query self.assertEqual(q1, q2) + a_objects = A.objects(s='test1') + query = B.objects(ref__in=a_objects) + query = query.filter(boolfield=True) + self.assertEquals(query.count(), 1) + def test_update_write_options(self): """Test that passing write_options works""" From faf840f924c6d8432c88c70ae949c8b14f0d72ed Mon Sep 17 00:00:00 2001 From: Paul Swartz Date: Mon, 25 Mar 2013 10:59:31 -0400 Subject: [PATCH 0975/1279] only mark a field as changed if the value has changed Prevents spurious changes from being recorded. --- AUTHORS | 1 + mongoengine/base.py | 14 ++++++-------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/AUTHORS b/AUTHORS index 82a1dfa..3d05cc4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -128,3 +128,4 @@ that much better: * Peter Teichman * Jakub Kot * Jorge Bastida + * Paul Swartz \ No newline at end of file diff --git a/mongoengine/base.py b/mongoengine/base.py index f73af4c..0e46248 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -205,8 +205,12 @@ class BaseField(object): def __set__(self, instance, value): """Descriptor for assigning a value to a field in a document. """ - instance._data[self.name] = value - if instance._initialised: + changed = False + if (self.name not in instance._data or + instance._data[self.name] != value): + changed = True + instance._data[self.name] = value + if changed and instance._initialised: instance._mark_as_changed(self.name) def error(self, message="", errors=None, field_name=None): @@ -317,12 +321,6 @@ class ComplexBaseField(BaseField): return value - def __set__(self, instance, value): - """Descriptor for assigning a value to a field in a document. - """ - instance._data[self.name] = value - instance._mark_as_changed(self.name) - def to_python(self, value): """Convert a MongoDB-compatible type to a Python type. """ From 20cb0285f004def84e83b501af35672d544afe0f Mon Sep 17 00:00:00 2001 From: Paul Swartz Date: Wed, 27 Mar 2013 14:53:47 -0400 Subject: [PATCH 0976/1279] explicitly check for Document instances when dereferencing In particular, `collections.namedtuple` instances also have a `_fields` attribute which confuses the dereferencing. --- mongoengine/dereference.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 386dbf4..b227ed3 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -33,7 +33,7 @@ class DeReference(object): self.max_depth = max_depth doc_type = None - if instance and instance._fields: + if instance and isinstance(instance, Document): doc_type = instance._fields.get(name) if hasattr(doc_type, 'field'): doc_type = doc_type.field @@ -84,7 +84,7 @@ class DeReference(object): # Recursively find dbreferences depth += 1 for k, item in iterator: - if hasattr(item, '_fields'): + if isinstance(item, Document): for field_name, field in item._fields.iteritems(): v = item._data.get(field_name, None) if isinstance(v, (DBRef)): @@ -187,7 +187,7 @@ class DeReference(object): if k in self.object_map and not is_list: data[k] = self.object_map[k] - elif hasattr(v, '_fields'): + elif isinstance(v, Document): for field_name, field in v._fields.iteritems(): v = data[k]._data.get(field_name, None) if isinstance(v, (DBRef)): From 74f3f4eb158ab5bd980578582af8454b0198577a Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Mon, 1 Apr 2013 16:17:17 -0700 Subject: [PATCH 0977/1279] more ordering unit tests --- tests/test_queryset.py | 91 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 10 deletions(-) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 5234cea..a87611c 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -952,6 +952,11 @@ class QuerySetTest(unittest.TestCase): {'attachments.views.extracted': 'no'}]} self.assertEqual(expected, raw_query) + def assertSequence(self, qs, expected): + self.assertEqual(len(qs), len(expected)) + for i in range(len(qs)): + self.assertEqual(qs[i], expected[i]) + def test_ordering(self): """Ensure default ordering is applied and can be overridden. """ @@ -965,10 +970,10 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() - blog_post_1 = BlogPost(title="Blog Post #1", - published_date=datetime(2010, 1, 5, 0, 0 ,0)) blog_post_2 = BlogPost(title="Blog Post #2", published_date=datetime(2010, 1, 6, 0, 0 ,0)) + blog_post_1 = BlogPost(title="Blog Post #1", + published_date=datetime(2010, 1, 5, 0, 0 ,0)) blog_post_3 = BlogPost(title="Blog Post #3", published_date=datetime(2010, 1, 7, 0, 0 ,0)) @@ -978,14 +983,13 @@ class QuerySetTest(unittest.TestCase): # get the "first" BlogPost using default ordering # from BlogPost.meta.ordering - latest_post = BlogPost.objects.first() - self.assertEqual(latest_post.title, "Blog Post #3") + expected = [blog_post_3, blog_post_2, blog_post_1] + self.assertSequence(BlogPost.objects.all(), expected) # override default ordering, order BlogPosts by "published_date" - first_post = BlogPost.objects.order_by("+published_date").first() - self.assertEqual(first_post.title, "Blog Post #1") - - BlogPost.drop_collection() + qs = BlogPost.objects.order_by("+published_date") + expected = [blog_post_1, blog_post_2, blog_post_3] + self.assertSequence(qs, expected) def test_only(self): """Ensure that QuerySet.only only returns the requested fields. @@ -1921,8 +1925,8 @@ class QuerySetTest(unittest.TestCase): def test_order_by(self): """Ensure that QuerySets may be ordered. """ - self.Person(name="User A", age=20).save() self.Person(name="User B", age=40).save() + self.Person(name="User A", age=20).save() self.Person(name="User C", age=30).save() names = [p.name for p in self.Person.objects.order_by('-age')] @@ -1937,11 +1941,67 @@ class QuerySetTest(unittest.TestCase): ages = [p.age for p in self.Person.objects.order_by('-name')] self.assertEqual(ages, [30, 40, 20]) + def test_order_by_optional(self): + class BlogPost(Document): + title = StringField() + published_date = DateTimeField(required=False) + + BlogPost.drop_collection() + + blog_post_3 = BlogPost(title="Blog Post #3", + published_date=datetime(2010, 1, 6, 0, 0 ,0)) + blog_post_2 = BlogPost(title="Blog Post #2", + published_date=datetime(2010, 1, 5, 0, 0 ,0)) + blog_post_4 = BlogPost(title="Blog Post #4", + published_date=datetime(2010, 1, 7, 0, 0 ,0)) + blog_post_1 = BlogPost(title="Blog Post #1", published_date=None) + + blog_post_3.save() + blog_post_1.save() + blog_post_4.save() + blog_post_2.save() + + expected = [blog_post_1, blog_post_2, blog_post_3, blog_post_4] + self.assertSequence(BlogPost.objects.order_by('published_date'), + expected) + self.assertSequence(BlogPost.objects.order_by('+published_date'), + expected) + + expected.reverse() + self.assertSequence(BlogPost.objects.order_by('-published_date'), + expected) + + def test_order_by_list(self): + class BlogPost(Document): + title = StringField() + published_date = DateTimeField(required=False) + + BlogPost.drop_collection() + + blog_post_1 = BlogPost(title="A", + published_date=datetime(2010, 1, 6, 0, 0 ,0)) + blog_post_2 = BlogPost(title="B", + published_date=datetime(2010, 1, 6, 0, 0 ,0)) + blog_post_3 = BlogPost(title="C", + published_date=datetime(2010, 1, 7, 0, 0 ,0)) + + blog_post_2.save() + blog_post_3.save() + blog_post_1.save() + + qs = BlogPost.objects.order_by('published_date', 'title') + expected = [blog_post_1, blog_post_2, blog_post_3] + self.assertSequence(qs, expected) + + qs = BlogPost.objects.order_by('-published_date', '-title') + expected.reverse() + self.assertSequence(qs, expected) + def test_order_by_chaining(self): """Ensure that an order_by query chains properly and allows .only() """ - self.Person(name="User A", age=20).save() self.Person(name="User B", age=40).save() + self.Person(name="User A", age=20).save() self.Person(name="User C", age=30).save() only_age = self.Person.objects.order_by('-age').only('age') @@ -1953,6 +2013,17 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(names, [None, None, None]) self.assertEqual(ages, [40, 30, 20]) + qs = self.Person.objects.all().limit(10) + qs = qs.order_by('-age') + ages = [p.age for p in qs] + self.assertEqual(ages, [40, 30, 20]) + + qs = self.Person.objects.all().skip(0) + qs = qs.order_by('-age') + ages = [p.age for p in qs] + self.assertEqual(ages, [40, 30, 20]) + + def test_confirm_order_by_reference_wont_work(self): """Ordering by reference is not possible. Use map / reduce.. or denormalise""" From dfabfce01bd91f9104fde9cd1892f9ca58fbe41b Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Mon, 1 Apr 2013 17:17:01 -0700 Subject: [PATCH 0978/1279] show that order_by followed by limit works, but not the other way around --- tests/test_queryset.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index a87611c..56177ec 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -2013,6 +2013,11 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(names, [None, None, None]) self.assertEqual(ages, [40, 30, 20]) + qs = self.Person.objects.all().order_by('-age') + qs = qs.limit(10) + ages = [p.age for p in qs] + self.assertEqual(ages, [40, 30, 20]) + qs = self.Person.objects.all().limit(10) qs = qs.order_by('-age') ages = [p.age for p in qs] @@ -2023,7 +2028,6 @@ class QuerySetTest(unittest.TestCase): ages = [p.age for p in qs] self.assertEqual(ages, [40, 30, 20]) - def test_confirm_order_by_reference_wont_work(self): """Ordering by reference is not possible. Use map / reduce.. or denormalise""" From 32d5c0c946f68a9328e23c688ad37b4472033e1c Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 3 Apr 2013 15:00:34 -0400 Subject: [PATCH 0979/1279] Store ordered list of field names, and return the ordered list when iterating a document instance. --- mongoengine/base.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index f73af4c..2f956cc 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -560,8 +560,11 @@ class DocumentMetaclass(type): # Set _fields and db_field maps attrs['_fields'] = doc_fields - attrs['_db_field_map'] = dict([(k, getattr(v, 'db_field', k)) - for k, v in doc_fields.iteritems()]) + attrs['_fields_ordered'] = tuple(i[1] + for i in sorted((v.creation_counter, v.name) + for v in doc_fields.itervalues())) + attrs['_db_field_map'] = dict((k, getattr(v, 'db_field', k)) + for k, v in doc_fields.iteritems()) attrs['_reverse_db_field_map'] = dict( (v, k) for k, v in attrs['_db_field_map'].iteritems()) @@ -1302,7 +1305,10 @@ class BaseDocument(object): return value def __iter__(self): - return iter(self._fields) + if 'id' in self._fields and 'id' not in self._fields_ordered: + return iter(('id', ) + self._fields_ordered) + + return iter(self._fields_ordered) def __getitem__(self, name): """Dictionary-style field access, return a field's value if present. From fc1ce6d39bd5dac86111276eae26de407a194ce6 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 3 Apr 2013 15:00:51 -0400 Subject: [PATCH 0980/1279] Allow construction of document instances using positional arguments. --- mongoengine/base.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 2f956cc..36d7c29 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -907,7 +907,17 @@ class BaseDocument(object): _dynamic_lock = True _initialised = False - def __init__(self, **values): + def __init__(self, *args, **values): + if args: + # Combine positional arguments with named arguments. + # We only want named arguments. + field = iter(self._fields_ordered) + for value in args: + name = next(field) + if name in values: + raise TypeError("Multiple values for keyword argument '" + name + "'") + values[name] = value + signals.pre_init.send(self.__class__, document=self, values=values) self._data = {} From 07d3e52e6a461eeca1251911e5440486e0832c2a Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 3 Apr 2013 15:03:33 -0400 Subject: [PATCH 0981/1279] Tests for construction using positional parameters. --- tests/test_document.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_document.py b/tests/test_document.py index 3e8d813..00059fa 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1386,6 +1386,28 @@ class DocumentTest(unittest.TestCase): person = self.Person(name="Test User", age=30) self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 30) + + def test_positional_creation(self): + """Ensure that document may be created using positional arguments. + """ + person = self.Person("Test User", 42) + self.assertEqual(person.name, "Test User") + self.assertEqual(person.age, 42) + + def test_mixed_creation(self): + """Ensure that document may be created using mixed arguments. + """ + person = self.Person("Test User", age=42) + self.assertEqual(person.name, "Test User") + self.assertEqual(person.age, 42) + + def test_bad_mixed_creation(self): + """Ensure that document gives correct error when duplicating arguments + """ + def construct_bad_instance(): + return self.Person("Test User", 42, name="Bad User") + + self.assertRaises(TypeError, construct_bad_instance) def test_to_dbref(self): """Ensure that you can get a dbref of a document""" From 782d48594a3fabaad8dfe4b5444f9d18839b309a Mon Sep 17 00:00:00 2001 From: "bool.dev" Date: Thu, 4 Apr 2013 08:24:35 +0530 Subject: [PATCH 0982/1279] Fixes resolving to db_field from class field name, in distinct() query. --- mongoengine/queryset.py | 2 ++ tests/test_queryset.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index bff05fc..ff168c7 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1214,6 +1214,8 @@ class QuerySet(object): .. versionchanged:: 0.5 - Fixed handling references .. versionchanged:: 0.6 - Improved db_field refrence handling """ + field = [field] + field = self._fields_to_dbfields(field).pop() return self._dereference(self._cursor.distinct(field), 1, name=field, instance=self._document) diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 5234cea..df58e21 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -2481,6 +2481,23 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(Foo.objects.distinct("bar"), [bar]) + def test_distinct_handles_db_field(self): + """Ensure that distinct resolves field name to db_field as expected. + """ + class Product(Document): + product_id=IntField(db_field='pid') + + Product.drop_collection() + + product_one = Product(product_id=1).save() + product_two = Product(product_id=2).save() + product_one_dup = Product(product_id=1).save() + + self.assertEqual(set(Product.objects.distinct('product_id')), + set([1, 2])) + + Product.drop_collection() + def test_custom_manager(self): """Ensure that custom QuerySetManager instances work as expected. """ From dd006a502eede24bad93361c74d6239fc3be4998 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristinn=20O=CC=88rn=20Sigur=C3=B0sson?= Date: Thu, 4 Apr 2013 17:09:05 +0200 Subject: [PATCH 0983/1279] Don't run unset on IntField if the value is 0 (zero). --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index f73af4c..c2d74f3 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1207,7 +1207,7 @@ class BaseDocument(object): # Determine if any changed items were actually unset. for path, value in set_data.items(): - if value or isinstance(value, bool): + if value or isinstance(value, bool) or isinstance(value, int): continue # If we've set a value that ain't the default value dont unset it. From 47df8deb58add4a7b93dedcc9da61be0be722d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristinn=20O=CC=88rn=20Sigur=C3=B0sson?= Date: Thu, 4 Apr 2013 17:30:21 +0200 Subject: [PATCH 0984/1279] Fix the implementation. --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index c2d74f3..4209156 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1207,7 +1207,7 @@ class BaseDocument(object): # Determine if any changed items were actually unset. for path, value in set_data.items(): - if value or isinstance(value, bool) or isinstance(value, int): + if value or type(value) in [bool, int]: continue # If we've set a value that ain't the default value dont unset it. From 7e980a16d088d6b73f7e3e6b4bdb3e5b75bd759b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristinn=20O=CC=88rn=20Sigur=C3=B0sson?= Date: Fri, 5 Apr 2013 11:01:46 +0200 Subject: [PATCH 0985/1279] Don't run unset on IntField if the value is 0 (zero). The IntField in unset if the IntField value doesn't validate to "truthify" (therefore, is set as 0) and the default value of the IntField in question is 0. This is not a logical functionality in my opinion. Take this example. You have an IntField that is a counter which can be incremented and decremented. This counter has the default value of 0 and is a required field. Every time the counter reaches 0, the field is unset. --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 4209156..fa6f825 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -1207,7 +1207,7 @@ class BaseDocument(object): # Determine if any changed items were actually unset. for path, value in set_data.items(): - if value or type(value) in [bool, int]: + if value or isinstance(value, (bool, int)): continue # If we've set a value that ain't the default value dont unset it. From d58341d7ae8e499f127dfc8ad0ce20be70b4c043 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 11 Apr 2013 13:15:17 +0000 Subject: [PATCH 0986/1279] Fix doc generation path (#230) Add Lukaszb to Authors --- AUTHORS | 1 + docs/conf.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 903542a..c64aaec 100644 --- a/AUTHORS +++ b/AUTHORS @@ -144,3 +144,4 @@ that much better: * Nick Joyce * Jared Forsyth * Kenneth Falck + * Lukasz Balcerzak diff --git a/docs/conf.py b/docs/conf.py index 62fa150..3cfcef5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,7 +16,7 @@ import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.append(os.path.abspath('..')) +sys.path.insert(0, os.path.abspath('..')) # -- General configuration ----------------------------------------------------- @@ -38,7 +38,7 @@ master_doc = 'index' # General information about the project. project = u'MongoEngine' -copyright = u'2009-2012, MongoEngine Authors' +copyright = u'2009, MongoEngine Authors' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From e9ff655b0ea1941837d3b9b155f530a023be81d1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 11 Apr 2013 14:58:42 +0000 Subject: [PATCH 0987/1279] Trying to fix travis --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2dc8894..a85aac4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,7 @@ env: install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install zlib1g zlib1g-dev; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo ln -s /usr/lib/x86_64-linux-gnu/libz.so /usr/lib/; fi - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install PIL --use-mirrors ; true; fi - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install PIL --use-mirrors ; true; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pillow --use-mirrors ; true; fi - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - python setup.py install From d9b8ee7895b4c0539a134e998dc6a756da502548 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 11 Apr 2013 15:47:53 +0000 Subject: [PATCH 0988/1279] next test --- .travis.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a85aac4..5e20e65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,9 +12,8 @@ env: - PYMONGO=2.4.1 - PYMONGO=2.3 install: - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install zlib1g zlib1g-dev; fi - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo ln -s /usr/lib/x86_64-linux-gnu/libz.so /usr/lib/; fi - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pillow --use-mirrors ; true; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - python setup.py install @@ -26,3 +25,8 @@ branches: only: - master - "0.8" +# # Get development headers for PIL +# before_install: +# - sudo apt-get update -qq +# - sudo apt-get build-dep -qq python-imaging +# - sudo apt-get install libjpeg-dev From b06f9dbf8d8da9b1f68007c9d9fd6e39de30b135 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 12 Apr 2013 10:02:55 +0000 Subject: [PATCH 0989/1279] Travis travis travis --- .travis.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5e20e65..3968166 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ env: - PYMONGO=2.3 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pillow --use-mirrors ; true; fi - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - python setup.py install diff --git a/setup.py b/setup.py index 6d9b51b..bcdb183 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ if sys.version_info[0] == 3: extra_opts['packages'].append("tests") extra_opts['package_data'] = {"tests": ["mongoengine.png"]} else: - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.3', 'PIL'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django=1.4.2', 'pillow'] extra_opts['packages'] = find_packages(exclude=('tests',)) setup(name='mongoengine', From cc5b60b004b19922ce2d59b33217059729134fe9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 12 Apr 2013 10:30:52 +0000 Subject: [PATCH 0990/1279] Updated pymongo versions and pillow wont work --- .travis.yml | 11 +++-------- setup.py | 2 +- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3968166..7fb55e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,11 +9,11 @@ python: - "3.3" env: - PYMONGO=dev - - PYMONGO=2.4.1 - - PYMONGO=2.3 + - PYMONGO=2.5 + - PYMONGO=2.4.2 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pillow --use-mirrors ; true; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - python setup.py install @@ -25,8 +25,3 @@ branches: only: - master - "0.8" -# # Get development headers for PIL -# before_install: -# - sudo apt-get update -qq -# - sudo apt-get build-dep -qq python-imaging -# - sudo apt-get install libjpeg-dev diff --git a/setup.py b/setup.py index bcdb183..1863df5 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ if sys.version_info[0] == 3: extra_opts['packages'].append("tests") extra_opts['package_data'] = {"tests": ["mongoengine.png"]} else: - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django=1.4.2', 'pillow'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django=1.4.2', 'PIL'] extra_opts['packages'] = find_packages(exclude=('tests',)) setup(name='mongoengine', From 19a7372ff9cbf8c192217bf6665794b6d210623b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 12 Apr 2013 10:32:50 +0000 Subject: [PATCH 0991/1279] Fix test_require for Django --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1863df5..54c2cdc 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ if sys.version_info[0] == 3: extra_opts['packages'].append("tests") extra_opts['package_data'] = {"tests": ["mongoengine.png"]} else: - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django=1.4.2', 'PIL'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django==1.4.2', 'PIL'] extra_opts['packages'] = find_packages(exclude=('tests',)) setup(name='mongoengine', From 5f1d5ea056a16d1df76463869d694c2238d13459 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 12 Apr 2013 10:35:09 +0000 Subject: [PATCH 0992/1279] Try and fix wobbly test --- tests/test_all_warnings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_all_warnings.py b/tests/test_all_warnings.py index 7ef1f21..9b09034 100644 --- a/tests/test_all_warnings.py +++ b/tests/test_all_warnings.py @@ -76,7 +76,7 @@ class TestWarnings(unittest.TestCase): p2.parent.name = "Poppa Wilson" p2.save() - self.assertEqual(len(self.warning_list), 1) + self.assertTrue(len(self.warning_list) > 0) if len(self.warning_list) > 1: print self.warning_list warning = self.warning_list[0] From 49a7542b148686b2c4ab43ed1f87195138c651da Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 12 Apr 2013 12:55:03 +0000 Subject: [PATCH 0993/1279] Fixing cloning in python 3 --- docs/changelog.rst | 5 +++++ mongoengine/queryset.py | 13 ++++--------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d93bf13..7957560 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,11 @@ Changelog ========= +Changes in 0.7.10 +================= +- Int fields no longer unset in save when changed to 0 (#272) +- Fixed ReferenceField query chaining bug fixed (#254) + Changes in 0.7.9 ================ - Better fix handling for old style _types diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 6c61ab9..20b18b7 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -379,8 +379,8 @@ class QuerySet(object): c = self.__class__(self._document, self._collection_obj) copy_props = ('_initial_query', '_query_obj', '_where_clause', - '_loaded_fields', '_ordering', '_snapshot', - '_timeout', '_limit', '_skip', '_slave_okay', '_hint') + '_loaded_fields', '_ordering', '_snapshot', '_timeout', + '_limit', '_skip', '_slave_okay', '_hint') for prop in copy_props: val = getattr(self, prop) @@ -393,16 +393,11 @@ class QuerySet(object): if self._mongo_query is None: self._mongo_query = self._query_obj.to_query(self._document) if self._class_check: - if PY3: - query = SON(self._initial_query.items()) - query.update(self._mongo_query) - self._mongo_query = query - else: - self._mongo_query.update(self._initial_query) + self._mongo_query.update(self._initial_query) return self._mongo_query def ensure_index(self, key_or_list, drop_dups=False, background=False, - **kwargs): + **kwargs): """Ensure that the given indexes are in place. :param key_or_list: a single index key or a list of index keys (to From 836dc96f67a2773671da9844508843242a5d23d6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 12 Apr 2013 12:56:15 +0000 Subject: [PATCH 0994/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7957560..ccf099f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Fixed cloning querysets in PY3 - Int fields no longer unset in save when changed to 0 (#272) - Fixed ReferenceField query chaining bug fixed (#254) From 37740dc01042ec7c8ce60ee365eae591de283032 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 12 Apr 2013 14:05:08 +0000 Subject: [PATCH 0995/1279] Added kwargs to doc.save to help interop with django (#223, #270) --- docs/changelog.rst | 1 + mongoengine/document.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ccf099f..8619a63 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Added kwargs to doc.save to help interop with django (#223, #270) - Fixed cloning querysets in PY3 - Int fields no longer unset in save when changed to 0 (#272) - Fixed ReferenceField query chaining bug fixed (#254) diff --git a/mongoengine/document.py b/mongoengine/document.py index 7b3afaf..a251f58 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -164,7 +164,7 @@ class Document(BaseDocument): def save(self, safe=True, force_insert=False, validate=True, write_options=None, cascade=None, cascade_kwargs=None, - _refs=None): + _refs=None, **kwargs): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be created. From 63edd16a925b775251e04bbf362d3540748e138d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 12 Apr 2013 14:20:44 +0000 Subject: [PATCH 0996/1279] Resolve field name to db field name when using distinct(#260, #264, #269) --- AUTHORS | 4 +++- docs/changelog.rst | 1 + mongoengine/queryset.py | 7 +++++-- tests/test_queryset.py | 19 +++++++++++++++++++ 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index fe62026..9d31c95 100644 --- a/AUTHORS +++ b/AUTHORS @@ -128,4 +128,6 @@ that much better: * Peter Teichman * Jakub Kot * Jorge Bastida - * Aleksandr Sorokoumov \ No newline at end of file + * Aleksandr Sorokoumov + * Yohan Graterol + * bool-dev \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 8619a63..667413e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Resolve field name to db field name when using distinct(#260, #264, #269) - Added kwargs to doc.save to help interop with django (#223, #270) - Fixed cloning querysets in PY3 - Int fields no longer unset in save when changed to 0 (#272) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 20b18b7..4aeff83 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1212,8 +1212,11 @@ class QuerySet(object): .. versionchanged:: 0.5 - Fixed handling references .. versionchanged:: 0.6 - Improved db_field refrence handling """ - return self._dereference(self._cursor.distinct(field), 1, - name=field, instance=self._document) + try: + field = self._fields_to_dbfields([field]).pop() + finally: + return self._dereference(self._cursor.distinct(field), 1, + name=field, instance=self._document) def only(self, *fields): """Load only a subset of this document's fields. :: diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 43bb70b..88daa5f 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -2486,6 +2486,25 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(Foo.objects.distinct("bar"), [bar]) + def test_distinct_handles_db_field(self): + """Ensure that distinct resolves field name to db_field as expected. + """ + class Product(Document): + product_id = IntField(db_field='pid') + + Product.drop_collection() + + Product(product_id=1).save() + Product(product_id=2).save() + Product(product_id=1).save() + + self.assertEqual(set(Product.objects.distinct('product_id')), + set([1, 2])) + self.assertEqual(set(Product.objects.distinct('pid')), + set([1, 2])) + + Product.drop_collection() + def test_custom_manager(self): """Ensure that custom QuerySetManager instances work as expected. """ From 2f19b22bb227097beabbc37ef5abcd86bb621074 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 12 Apr 2013 14:25:43 +0000 Subject: [PATCH 0997/1279] Added dereference support for tuples (#250) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 9d31c95..aeb672c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -130,4 +130,5 @@ that much better: * Jorge Bastida * Aleksandr Sorokoumov * Yohan Graterol - * bool-dev \ No newline at end of file + * bool-dev + * Russ Weeks \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 667413e..5ecae62 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Added dereference support for tuples (#250) - Resolve field name to db field name when using distinct(#260, #264, #269) - Added kwargs to doc.save to help interop with django (#223, #270) - Fixed cloning querysets in PY3 From c9a5710554cca690f6ee2a574d03be05d93a3e78 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 12 Apr 2013 15:56:40 +0000 Subject: [PATCH 0998/1279] Fixed order_by chaining issue (#265) --- docs/changelog.rst | 1 + mongoengine/queryset.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 5ecae62..65a5aaf 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Fixed order_by chaining issue (#265) - Added dereference support for tuples (#250) - Resolve field name to db field name when using distinct(#260, #264, #269) - Added kwargs to doc.save to help interop with django (#223, #270) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 4aeff83..727f56e 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1321,7 +1321,8 @@ class QuerySet(object): key_list.append((key, direction)) self._ordering = key_list - + if self._cursor_obj: + self._cursor_obj.sort(key_list) return self def explain(self, format=False): From d92f992c011cec92e983cf7a1da45dc31624cc92 Mon Sep 17 00:00:00 2001 From: "bool.dev" Date: Sun, 14 Apr 2013 13:48:11 +0530 Subject: [PATCH 0999/1279] Removed merge trackers in code, merged correctly now. --- mongoengine/queryset.py | 7 ------- tests/test_queryset.py | 13 ------------- 2 files changed, 20 deletions(-) diff --git a/mongoengine/queryset.py b/mongoengine/queryset.py index 8d18942..727f56e 100644 --- a/mongoengine/queryset.py +++ b/mongoengine/queryset.py @@ -1212,18 +1212,11 @@ class QuerySet(object): .. versionchanged:: 0.5 - Fixed handling references .. versionchanged:: 0.6 - Improved db_field refrence handling """ -<<<<<<< HEAD - field = [field] - field = self._fields_to_dbfields(field).pop() - return self._dereference(self._cursor.distinct(field), 1, - name=field, instance=self._document) -======= try: field = self._fields_to_dbfields([field]).pop() finally: return self._dereference(self._cursor.distinct(field), 1, name=field, instance=self._document) ->>>>>>> upstream/master def only(self, *fields): """Load only a subset of this document's fields. :: diff --git a/tests/test_queryset.py b/tests/test_queryset.py index 7bb55b0..6b56926 100644 --- a/tests/test_queryset.py +++ b/tests/test_queryset.py @@ -2565,18 +2565,6 @@ class QuerySetTest(unittest.TestCase): """Ensure that distinct resolves field name to db_field as expected. """ class Product(Document): -<<<<<<< HEAD - product_id=IntField(db_field='pid') - - Product.drop_collection() - - product_one = Product(product_id=1).save() - product_two = Product(product_id=2).save() - product_one_dup = Product(product_id=1).save() - - self.assertEqual(set(Product.objects.distinct('product_id')), - set([1, 2])) -======= product_id = IntField(db_field='pid') Product.drop_collection() @@ -2589,7 +2577,6 @@ class QuerySetTest(unittest.TestCase): set([1, 2])) self.assertEqual(set(Product.objects.distinct('pid')), set([1, 2])) ->>>>>>> upstream/master Product.drop_collection() From b6977a88ea02fba91809c19276852bb691e1c6ca Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 15 Apr 2013 07:32:04 +0000 Subject: [PATCH 1000/1279] Explicitly check for Document instances when dereferencing (#261) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index aeb672c..991b10a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -131,4 +131,5 @@ that much better: * Aleksandr Sorokoumov * Yohan Graterol * bool-dev - * Russ Weeks \ No newline at end of file + * Russ Weeks + * Paul Swartz \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 65a5aaf..2ff2f6b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Explicitly check for Document instances when dereferencing (#261) - Fixed order_by chaining issue (#265) - Added dereference support for tuples (#250) - Resolve field name to db field name when using distinct(#260, #264, #269) From da7a8939dfce47d866641cdaa7ab29fc444db2c9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 15 Apr 2013 07:41:04 +0000 Subject: [PATCH 1001/1279] Also check if a TopLevelMetaclass instance (#261) --- mongoengine/dereference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 4df1fe8..997b785 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -33,7 +33,7 @@ class DeReference(object): self.max_depth = max_depth doc_type = None - if instance and isinstance(instance, Document): + if instance and isinstance(instance, (Document, TopLevelDocumentMetaclass)): doc_type = instance._fields.get(name) if hasattr(doc_type, 'field'): doc_type = doc_type.field From 97a98f004530c078a3a0ce5e687808b3d250f362 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 15 Apr 2013 07:52:04 +0000 Subject: [PATCH 1002/1279] Only mark a field as changed if the value has changed (#258) --- tests/test_dereference.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_dereference.py b/tests/test_dereference.py index d7438d2..0900154 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -185,8 +185,9 @@ class FieldTest(unittest.TestCase): # Migrate the data for g in Group.objects(): - g.author = g.author - g.members = g.members + # Explicitly mark as changed so resets + g._mark_as_changed('author') + g._mark_as_changed('members') g.save() group = Group.objects.first() @@ -997,7 +998,7 @@ class FieldTest(unittest.TestCase): msg = Message.objects.get(id=1) self.assertEqual(0, msg.comments[0].id) self.assertEqual(1, msg.comments[1].id) - + def test_tuples_as_tuples(self): """ Ensure that tuples remain tuples when they are @@ -1007,16 +1008,16 @@ class FieldTest(unittest.TestCase): class EnumField(BaseField): def __init__(self, **kwargs): super(EnumField,self).__init__(**kwargs) - + def to_mongo(self, value): return value - + def to_python(self, value): return tuple(value) - + class TestDoc(Document): items = ListField(EnumField()) - + TestDoc.drop_collection() tuples = [(100,'Testing')] doc = TestDoc() From 757ff31661282d0296e1927b694a7951bfee6e9a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 15 Apr 2013 07:53:57 +0000 Subject: [PATCH 1003/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2ff2f6b..45b2e09 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Only mark a field as changed if the value has changed (#258) - Explicitly check for Document instances when dereferencing (#261) - Fixed order_by chaining issue (#265) - Added dereference support for tuples (#250) From b451cc567d1d3d461a1d15dace37fb79f88a5279 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 15 Apr 2013 07:59:24 +0000 Subject: [PATCH 1004/1279] Return '_id' as the key for document.id in _data dictionary * Re #146 Conflicts: mongoengine/base.py --- mongoengine/base.py | 6 +++--- tests/test_document.py | 20 ++++++++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index 25c3bbd..7e6d0aa 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -192,7 +192,7 @@ class BaseField(object): return self # Get value from document instance if available, if not use default - value = instance._data.get(self.name) + value = instance._data.get(self.name or self.db_field) if value is None: value = self.default @@ -207,9 +207,9 @@ class BaseField(object): """ changed = False if (self.name not in instance._data or - instance._data[self.name] != value): + instance._data[self.name or self.db_field] != value): changed = True - instance._data[self.name] = value + instance._data[self.name or self.db_field] = value if changed and instance._initialised: instance._mark_as_changed(self.name) diff --git a/tests/test_document.py b/tests/test_document.py index 3e8d813..b5542c2 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1529,8 +1529,10 @@ class DocumentTest(unittest.TestCase): doc.validate() keys = doc._data.keys() self.assertEqual(2, len(keys)) - self.assertTrue(None in keys) self.assertTrue('e' in keys) + # Ensure that the _id field has the right id + self.assertTrue('_id' in keys) + self.assertEqual(doc._data.get('_id'), doc.id) def test_save(self): """Ensure that a document may be saved in the database. @@ -3368,6 +3370,21 @@ class DocumentTest(unittest.TestCase): } ) ]), "1,2") + def test_data_contains_idfield(self): + """Ensure that asking for _data returns 'id' + """ + class Person(Document): + name = StringField() + + Person.drop_collection() + person = Person() + person.name = "Harry Potter" + person.save(cascade=False) + + person = Person.objects.first() + self.assertTrue('_id' in person._data.keys()) + self.assertEqual(person._data.get('_id'), person.id) + class ValidatorErrorTest(unittest.TestCase): @@ -3521,6 +3538,5 @@ class ValidatorErrorTest(unittest.TestCase): self.assertRaises(OperationError, change_shard_key) - if __name__ == '__main__': unittest.main() From 6186691259898c8e5d9da8983a74792d440622cb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 15 Apr 2013 08:01:24 +0000 Subject: [PATCH 1005/1279] Updated changelog and AUTHORS (#255) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 991b10a..aa223ed 100644 --- a/AUTHORS +++ b/AUTHORS @@ -132,4 +132,5 @@ that much better: * Yohan Graterol * bool-dev * Russ Weeks - * Paul Swartz \ No newline at end of file + * Paul Swartz + * Sundar Raman \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 45b2e09..90d4d66 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Added "_id" to _data dictionary (#255) - Only mark a field as changed if the value has changed (#258) - Explicitly check for Document instances when dereferencing (#261) - Fixed order_by chaining issue (#265) From d80b1a774934ae1d79e8b43ded5fbff8e86c9d8a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 15 Apr 2013 08:03:51 +0000 Subject: [PATCH 1006/1279] Test clean up (#255) --- tests/test_document.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_document.py b/tests/test_document.py index b5542c2..051dc2a 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -3377,9 +3377,7 @@ class DocumentTest(unittest.TestCase): name = StringField() Person.drop_collection() - person = Person() - person.name = "Harry Potter" - person.save(cascade=False) + Person(name="Harry Potter").save() person = Person.objects.first() self.assertTrue('_id' in person._data.keys()) From add0b463f5fc191161814a0eebf5d3e133ede6b0 Mon Sep 17 00:00:00 2001 From: Daniil Sharou Date: Tue, 16 Apr 2013 21:12:57 +0400 Subject: [PATCH 1007/1279] fix UnicodeEncodeError for dbref Fix "UnicodeEncodeError: 'ascii' codec can't encode character ..." error in case dbref contains non-ascii characters --- mongoengine/dereference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index 997b785..e1b0a03 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -115,7 +115,7 @@ class DeReference(object): object_map = {} for col, dbrefs in self.reference_map.iteritems(): keys = object_map.keys() - refs = list(set([dbref for dbref in dbrefs if str(dbref) not in keys])) + refs = list(set([dbref for dbref in dbrefs if dbref.encode('utf-8') not in keys])) if hasattr(col, 'objects'): # We have a document class for the refs references = col.objects.in_bulk(refs) for key, doc in references.iteritems(): From cc0a2cbc6f20756c5edac3c0785ac132be68df45 Mon Sep 17 00:00:00 2001 From: Daniil Sharou Date: Tue, 16 Apr 2013 22:34:33 +0400 Subject: [PATCH 1008/1279] fix UnicodeEncodeError for dbref Fix "UnicodeEncodeError: 'ascii' codec can't encode character ..." error in case dbref contains non-ascii characters --- mongoengine/dereference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index e1b0a03..ed75615 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -115,7 +115,7 @@ class DeReference(object): object_map = {} for col, dbrefs in self.reference_map.iteritems(): keys = object_map.keys() - refs = list(set([dbref for dbref in dbrefs if dbref.encode('utf-8') not in keys])) + refs = list(set([dbref for dbref in dbrefs if unicode(dbref).encode('utf-8') not in keys])) if hasattr(col, 'objects'): # We have a document class for the refs references = col.objects.in_bulk(refs) for key, doc in references.iteritems(): From a5257643596836f2f58fccce55157e83dc4b4f27 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Apr 2013 20:12:01 +0000 Subject: [PATCH 1009/1279] Fixed clearing _changed_fields for complex nested embedded documents (#237, #239, #242) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index aa223ed..1c72b31 100644 --- a/AUTHORS +++ b/AUTHORS @@ -133,4 +133,5 @@ that much better: * bool-dev * Russ Weeks * Paul Swartz - * Sundar Raman \ No newline at end of file + * Sundar Raman + * Benoit Louy \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 90d4d66..99aa49a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Fixed clearing _changed_fields for complex nested embedded documents (#237, #239, #242) - Added "_id" to _data dictionary (#255) - Only mark a field as changed if the value has changed (#258) - Explicitly check for Document instances when dereferencing (#261) From 6fe074fb13606ad82bb392f523acf412f65a66df Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Apr 2013 20:21:11 +0000 Subject: [PATCH 1010/1279] Fixed issue with numerical keys in MapField(EmbeddedDocumentField()) (#240) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 1c72b31..01f6f22 100644 --- a/AUTHORS +++ b/AUTHORS @@ -134,4 +134,5 @@ that much better: * Russ Weeks * Paul Swartz * Sundar Raman - * Benoit Louy \ No newline at end of file + * Benoit Louy + * lraucy \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 99aa49a..269a422 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Fixed issue with numerical keys in MapField(EmbeddedDocumentField()) (#240) - Fixed clearing _changed_fields for complex nested embedded documents (#237, #239, #242) - Added "_id" to _data dictionary (#255) - Only mark a field as changed if the value has changed (#258) From d02de0798f8ee0b04bbb33e2ca0c71651f32af09 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Apr 2013 20:26:23 +0000 Subject: [PATCH 1011/1279] Documentation fix explaining adding a dummy backend for django (#172) --- docs/django.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/django.rst b/docs/django.rst index 144baab..5859746 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -10,6 +10,16 @@ In your **settings.py** file, ignore the standard database settings (unless you also plan to use the ORM in your project), and instead call :func:`~mongoengine.connect` somewhere in the settings module. +.. note :: + If you are not using another Database backend make sure you add a dummy + backend, by adding the following to ``settings.py``:: + + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.dummy' + } + } + Authentication ============== MongoEngine includes a Django authentication backend, which uses MongoDB. The From 1f9ec0c888f6a5ea5a6e83267d361b917e7df104 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Apr 2013 20:30:40 +0000 Subject: [PATCH 1012/1279] Added Django sessions TTL support (#224) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 01f6f22..a689ec6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -135,4 +135,5 @@ that much better: * Paul Swartz * Sundar Raman * Benoit Louy - * lraucy \ No newline at end of file + * lraucy + * hellysmile \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 269a422..df15855 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Added Django sessions TTL support (#224) - Fixed issue with numerical keys in MapField(EmbeddedDocumentField()) (#240) - Fixed clearing _changed_fields for complex nested embedded documents (#237, #239, #242) - Added "_id" to _data dictionary (#255) From 3a85422e8f332484984a0b519f8cd614b7445148 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Apr 2013 20:35:29 +0000 Subject: [PATCH 1013/1279] Added 64-bit integer support (#251) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index a689ec6..370f082 100644 --- a/AUTHORS +++ b/AUTHORS @@ -136,4 +136,5 @@ that much better: * Sundar Raman * Benoit Louy * lraucy - * hellysmile \ No newline at end of file + * hellysmile + * Jaepil Jeong \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index df15855..8ff95b5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Added 64-bit integer support (#251) - Added Django sessions TTL support (#224) - Fixed issue with numerical keys in MapField(EmbeddedDocumentField()) (#240) - Fixed clearing _changed_fields for complex nested embedded documents (#237, #239, #242) From b562e209d17afda19413c2971e69e2633f1f09e5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Apr 2013 20:46:02 +0000 Subject: [PATCH 1014/1279] Updated EmailField length to support long domains (#243) --- docs/changelog.rst | 1 + mongoengine/fields.py | 2 +- tests/test_fields.py | 18 ++++++++++++++++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8ff95b5..91d4da0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Updated EmailField length to support long domains (#243) - Added 64-bit integer support (#251) - Added Django sessions TTL support (#224) - Fixed issue with numerical keys in MapField(EmbeddedDocumentField()) (#240) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index c40491b..2d1ee71 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -143,7 +143,7 @@ class EmailField(StringField): EMAIL_REGEX = re.compile( r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*" # dot-atom r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"' # quoted-string - r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$', re.IGNORECASE # domain + r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,253}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$', re.IGNORECASE # domain ) def validate(self, value): diff --git a/tests/test_fields.py b/tests/test_fields.py index 1e693e8..55ac6fb 100644 --- a/tests/test_fields.py +++ b/tests/test_fields.py @@ -2371,12 +2371,26 @@ class FieldTest(unittest.TestCase): self.assertTrue(1 in error_dict['comments']) self.assertTrue('content' in error_dict['comments'][1]) self.assertEqual(error_dict['comments'][1]['content'], - u'Field is required') - + u'Field is required') post.comments[1].content = 'here we go' post.validate() + def test_email_field(self): + class User(Document): + email = EmailField() + + user = User(email="ross@example.com") + self.assertTrue(user.validate() is None) + + user = User(email=("Kofq@rhom0e4klgauOhpbpNdogawnyIKvQS0wk2mjqrgGQ5S" + "ucictfqpdkK9iS1zeFw8sg7s7cwAF7suIfUfeyueLpfosjn3" + "aJIazqqWkm7.net")) + self.assertTrue(user.validate() is None) + + user = User(email='me@localhost') + self.assertRaises(ValidationError, user.validate) + def test_email_field_honors_regex(self): class User(Document): email = EmailField(regex=r'\w+@example.com') From b4d87d91282247be21d46107935f31891a1802d6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 16 Apr 2013 20:50:34 +0000 Subject: [PATCH 1015/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 91d4da0..0443f74 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Allow construction using positional parameters (#268) - Updated EmailField length to support long domains (#243) - Added 64-bit integer support (#251) - Added Django sessions TTL support (#224) From c2d77f51bba6e75d427f11845ad61ccca85b9803 Mon Sep 17 00:00:00 2001 From: daniil Date: Wed, 17 Apr 2013 12:14:07 +0400 Subject: [PATCH 1016/1279] test for #278 issue --- tests/test_dereference.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 0900154..8caefd3 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from __future__ import with_statement import unittest @@ -1029,3 +1030,26 @@ class FieldTest(unittest.TestCase): self.assertTrue(tuple(x.items[0]) in tuples) self.assertTrue(x.items[0] in tuples) + def test_non_ascii_pk(self): + """ + Ensure that dbref conversion to string does not fail when + non-ascii characters are used in primary key + """ + class Brand(Document): + title = StringField(max_length=255, primary_key=True) + + class BrandGroup(Document): + title = StringField(max_length=255, primary_key=True) + brands = SortedListField(ReferenceField("Brand", dbref=True)) + + Brand.drop_collection() + BrandGroup.drop_collection() + + brand1 = Brand(title="Moschino").save() + brand2 = Brand(title=u"Денис Симачёв").save() + + BrandGroup(title="top_brands", brands=[brand1, brand2]).save() + brand_groups = BrandGroup.objects().all() + + self.assertEqual(2, len([brand for bg in brand_groups for brand in bg.brands])) + From 420376d036708a1077b535ca7dfaa9471c168496 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 17 Apr 2013 14:27:33 +0000 Subject: [PATCH 1017/1279] Merge fixes --- mongoengine/base/document.py | 4 ++-- mongoengine/queryset/queryset.py | 8 ++++++++ tests/queryset/queryset.py | 20 ++++++++++++-------- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index ebb3410..7ec672f 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -30,7 +30,7 @@ class BaseDocument(object): _dynamic_lock = True _initialised = False - def __init__(self, __auto_convert=True, *args, **values): + def __init__(self, *args, **values): """ Initialise a document or embedded document @@ -46,7 +46,7 @@ class BaseDocument(object): if name in values: raise TypeError("Multiple values for keyword argument '" + name + "'") values[name] = value - + __auto_convert = values.pop("__auto_convert", True) signals.pre_init.send(self.__class__, document=self, values=values) self._data = {} diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index c299190..28a9618 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1138,8 +1138,13 @@ class QuerySet(object): if self._hint != -1: self._cursor_obj.hint(self._hint) + return self._cursor_obj + def __deepcopy__(self, memo): + """Essential for chained queries with ReferenceFields involved""" + return self.clone() + @property def _query(self): if self._mongo_query is None: @@ -1302,6 +1307,9 @@ class QuerySet(object): except: pass key_list.append((key, direction)) + + if self._cursor_obj: + self._cursor_obj.sort(key_list) return key_list def _get_scalar(self, doc): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index da0e89a..1dccdb1 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -249,6 +249,10 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(list(A.objects.none()), []) self.assertEqual(list(A.objects.none().all()), []) + def test_chaining(self): + class A(Document): + s = StringField() + class B(Document): ref = ReferenceField(A) boolfield = BooleanField(default=False) @@ -282,7 +286,7 @@ class QuerySetTest(unittest.TestCase): write_options = {"fsync": True} author, created = self.Person.objects.get_or_create( - name='Test User', write_options=write_options) + name='Test User', write_options=write_options) author.save(write_options=write_options) self.Person.objects.update(set__name='Ross', @@ -1475,7 +1479,6 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() - def test_set_list_embedded_documents(self): class Author(EmbeddedDocument): @@ -1533,11 +1536,11 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() blog_post_3 = BlogPost(title="Blog Post #3", - published_date=datetime(2010, 1, 6, 0, 0 ,0)) + published_date=datetime(2010, 1, 6, 0, 0, 0)) blog_post_2 = BlogPost(title="Blog Post #2", - published_date=datetime(2010, 1, 5, 0, 0 ,0)) + published_date=datetime(2010, 1, 5, 0, 0, 0)) blog_post_4 = BlogPost(title="Blog Post #4", - published_date=datetime(2010, 1, 7, 0, 0 ,0)) + published_date=datetime(2010, 1, 7, 0, 0, 0)) blog_post_1 = BlogPost(title="Blog Post #1", published_date=None) blog_post_3.save() @@ -1563,11 +1566,11 @@ class QuerySetTest(unittest.TestCase): BlogPost.drop_collection() blog_post_1 = BlogPost(title="A", - published_date=datetime(2010, 1, 6, 0, 0 ,0)) + published_date=datetime(2010, 1, 6, 0, 0, 0)) blog_post_2 = BlogPost(title="B", - published_date=datetime(2010, 1, 6, 0, 0 ,0)) + published_date=datetime(2010, 1, 6, 0, 0, 0)) blog_post_3 = BlogPost(title="C", - published_date=datetime(2010, 1, 7, 0, 0 ,0)) + published_date=datetime(2010, 1, 7, 0, 0, 0)) blog_post_2.save() blog_post_3.save() @@ -1604,6 +1607,7 @@ class QuerySetTest(unittest.TestCase): qs = self.Person.objects.all().limit(10) qs = qs.order_by('-age') + ages = [p.age for p in qs] self.assertEqual(ages, [40, 30, 20]) From ec639cd6e9acb10f43228487b634e74f11b54bcd Mon Sep 17 00:00:00 2001 From: Nicolas Cortot Date: Wed, 17 Apr 2013 16:19:53 +0200 Subject: [PATCH 1018/1279] Fix datetime call in UserManager --- mongoengine/django/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 024ae9c..d22f086 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -142,7 +142,7 @@ class UserManager(models.Manager): """ Creates and saves a User with the given username, e-mail and password. """ - now = datetime.datetime.now() + now = datetime_now() # Normalize the address by lowercasing the domain part of the email # address. From dcf3c86dce243962e9edc232c0ac4bea63f81ec5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 17 Apr 2013 15:07:57 +0000 Subject: [PATCH 1019/1279] Using "id" in data not "_id" as its a mapping of fieldnames (#255) --- docs/changelog.rst | 2 +- mongoengine/base.py | 13 +++++++------ tests/test_document.py | 16 ++++++++-------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0443f74..1496495 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -10,7 +10,7 @@ Changes in 0.7.10 - Added Django sessions TTL support (#224) - Fixed issue with numerical keys in MapField(EmbeddedDocumentField()) (#240) - Fixed clearing _changed_fields for complex nested embedded documents (#237, #239, #242) -- Added "_id" to _data dictionary (#255) +- Added "id" back to _data dictionary (#255) - Only mark a field as changed if the value has changed (#258) - Explicitly check for Document instances when dereferencing (#261) - Fixed order_by chaining issue (#265) diff --git a/mongoengine/base.py b/mongoengine/base.py index 0f13643..a7eb17b 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -192,7 +192,7 @@ class BaseField(object): return self # Get value from document instance if available, if not use default - value = instance._data.get(self.name or self.db_field) + value = instance._data.get(self.name) if value is None: value = self.default @@ -207,9 +207,9 @@ class BaseField(object): """ changed = False if (self.name not in instance._data or - instance._data[self.name or self.db_field] != value): + instance._data[self.name] != value): changed = True - instance._data[self.name or self.db_field] = value + instance._data[self.name] = value if changed and instance._initialised: instance._mark_as_changed(self.name) @@ -825,6 +825,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): if not new_class._meta.get('id_field'): new_class._meta['id_field'] = 'id' new_class._fields['id'] = ObjectIdField(db_field='_id') + new_class._fields['id'].name = 'id' new_class.id = new_class._fields['id'] # Merge in exceptions with parent hierarchy @@ -915,7 +916,7 @@ class BaseDocument(object): if name in values: raise TypeError("Multiple values for keyword argument '" + name + "'") values[name] = value - + signals.pre_init.send(self.__class__, document=self, values=values) self._data = {} @@ -1138,7 +1139,7 @@ class BaseDocument(object): self._changed_fields = [] EmbeddedDocumentField = _import_class("EmbeddedDocumentField") for field_name, field in self._fields.iteritems(): - if (isinstance(field, ComplexBaseField) and + if (isinstance(field, ComplexBaseField) and isinstance(field.field, EmbeddedDocumentField)): field_value = getattr(self, field_name, None) if field_value: @@ -1331,7 +1332,7 @@ class BaseDocument(object): def __iter__(self): if 'id' in self._fields and 'id' not in self._fields_ordered: return iter(('id', ) + self._fields_ordered) - + return iter(self._fields_ordered) def __getitem__(self, name): diff --git a/tests/test_document.py b/tests/test_document.py index ce00703..5b30a96 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1386,27 +1386,27 @@ class DocumentTest(unittest.TestCase): person = self.Person(name="Test User", age=30) self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 30) - + def test_positional_creation(self): """Ensure that document may be created using positional arguments. """ person = self.Person("Test User", 42) self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 42) - + def test_mixed_creation(self): """Ensure that document may be created using mixed arguments. """ person = self.Person("Test User", age=42) self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 42) - + def test_bad_mixed_creation(self): """Ensure that document gives correct error when duplicating arguments """ def construct_bad_instance(): return self.Person("Test User", 42, name="Bad User") - + self.assertRaises(TypeError, construct_bad_instance) def test_to_dbref(self): @@ -1553,8 +1553,8 @@ class DocumentTest(unittest.TestCase): self.assertEqual(2, len(keys)) self.assertTrue('e' in keys) # Ensure that the _id field has the right id - self.assertTrue('_id' in keys) - self.assertEqual(doc._data.get('_id'), doc.id) + self.assertTrue('id' in keys) + self.assertEqual(doc._data.get('id'), doc.id) def test_save(self): """Ensure that a document may be saved in the database. @@ -3402,8 +3402,8 @@ class DocumentTest(unittest.TestCase): Person(name="Harry Potter").save() person = Person.objects.first() - self.assertTrue('_id' in person._data.keys()) - self.assertEqual(person._data.get('_id'), person.id) + self.assertTrue('id' in person._data.keys()) + self.assertEqual(person._data.get('id'), person.id) def test_complex_nesting_document_and_embedded_document(self): From 03bfd018621233629467d131c8af52764c05e559 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 17 Apr 2013 15:54:32 +0000 Subject: [PATCH 1020/1279] Updated field iteration for py2.5 --- mongoengine/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base.py b/mongoengine/base.py index a7eb17b..1e1a021 100644 --- a/mongoengine/base.py +++ b/mongoengine/base.py @@ -912,7 +912,7 @@ class BaseDocument(object): # We only want named arguments. field = iter(self._fields_ordered) for value in args: - name = next(field) + name = field.next() if name in values: raise TypeError("Multiple values for keyword argument '" + name + "'") values[name] = value From 073091a06e823b95234679d8f7a349714bf67bae Mon Sep 17 00:00:00 2001 From: Nicolas Cortot Date: Wed, 17 Apr 2013 21:45:54 +0200 Subject: [PATCH 1021/1279] Do not fail on delete() when blinker is not available --- mongoengine/queryset/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 28a9618..15c8e63 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -366,7 +366,7 @@ class QuerySet(object): queryset = self.clone() doc = queryset._document - has_delete_signal = ( + has_delete_signal = signals.signals_available and ( signals.pre_delete.has_receivers_for(self._document) or signals.post_delete.has_receivers_for(self._document)) From 3f4992329831d063448c18d7a3005204e289eb65 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 18 Apr 2013 13:21:36 +0000 Subject: [PATCH 1022/1279] Update AUTHORS and changelog (#278) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 370f082..1262669 100644 --- a/AUTHORS +++ b/AUTHORS @@ -137,4 +137,5 @@ that much better: * Benoit Louy * lraucy * hellysmile - * Jaepil Jeong \ No newline at end of file + * Jaepil Jeong + * Daniil Sharou diff --git a/docs/changelog.rst b/docs/changelog.rst index 1496495..386daa5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.7.10 ================= +- Fix UnicodeEncodeError for dbref (#278) - Allow construction using positional parameters (#268) - Updated EmailField length to support long domains (#243) - Added 64-bit integer support (#251) From 11085863033b71aa4026dcd3fe2c26adea54f8a4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 18 Apr 2013 13:26:35 +0000 Subject: [PATCH 1023/1279] Updated queryset --- tests/queryset/queryset.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 1dccdb1..37670b0 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -389,6 +389,8 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(post.comments[1].by, 'jane') self.assertEqual(post.comments[1].votes, 8) + def test_update_using_positional_operator_matches_first(self): + # Currently the $ operator only applies to the first matched item in # the query From 5de4812477841e6458752562b0a936126c0ced6c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 18 Apr 2013 13:38:36 +0000 Subject: [PATCH 1024/1279] Updating AUTHORS (#283) --- AUTHORS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index b00b069..e388a04 100644 --- a/AUTHORS +++ b/AUTHORS @@ -156,3 +156,5 @@ that much better: * Jared Forsyth * Kenneth Falck * Lukasz Balcerzak + * Nicolas Cortot + From 6dcd7006d05d7ce749786a9680496c5b221a85fe Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 19 Apr 2013 12:47:19 +0000 Subject: [PATCH 1025/1279] Fix test --- tests/test_dereference.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_dereference.py b/tests/test_dereference.py index 8caefd3..a3517d2 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1020,7 +1020,7 @@ class FieldTest(unittest.TestCase): items = ListField(EnumField()) TestDoc.drop_collection() - tuples = [(100,'Testing')] + tuples = [(100, 'Testing')] doc = TestDoc() doc.items = tuples doc.save() @@ -1040,7 +1040,7 @@ class FieldTest(unittest.TestCase): class BrandGroup(Document): title = StringField(max_length=255, primary_key=True) - brands = SortedListField(ReferenceField("Brand", dbref=True)) + brands = ListField(ReferenceField("Brand", dbref=True)) Brand.drop_collection() BrandGroup.drop_collection() From e3600ef4de6ab13090f18a6155d5f515b7ae2e8d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 19 Apr 2013 12:53:46 +0000 Subject: [PATCH 1026/1279] Updated version --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index b67512d..64adb92 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -12,7 +12,7 @@ from signals import * __all__ = (document.__all__ + fields.__all__ + connection.__all__ + queryset.__all__ + signals.__all__) -VERSION = (0, 7, 9) +VERSION = (0, 7, 10) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index b1ec336..ed4a872 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.7.9 +Version: 0.7.10 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 6affbbe86570a47d8110dd9c967ea2babe856bec Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 19 Apr 2013 13:08:46 +0000 Subject: [PATCH 1027/1279] Update changelog location --- python-mongoengine.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-mongoengine.spec b/python-mongoengine.spec index ed4a872..eaf478d 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -51,4 +51,4 @@ rm -rf $RPM_BUILD_ROOT # %{python_sitearch}/* %changelog -* See: http://readthedocs.org/docs/mongoengine-odm/en/latest/changelog.html \ No newline at end of file +* See: http://docs.mongoengine.org/en/latest/changelog.html \ No newline at end of file From 485047f20b1f9b3a52ce0b11a426505ad03437eb Mon Sep 17 00:00:00 2001 From: Nicolas Cortot Date: Wed, 17 Apr 2013 21:38:11 +0200 Subject: [PATCH 1028/1279] Custom User Model for Django 1.5 --- docs/django.rst | 36 +++++++++ mongoengine/django/auth.py | 3 + mongoengine/django/mongo_auth/__init__.py | 0 mongoengine/django/mongo_auth/models.py | 90 +++++++++++++++++++++++ tests/test_django.py | 52 ++++++++++++- 5 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 mongoengine/django/mongo_auth/__init__.py create mode 100644 mongoengine/django/mongo_auth/models.py diff --git a/docs/django.rst b/docs/django.rst index 6f27b90..2d58f95 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -42,6 +42,42 @@ The :mod:`~mongoengine.django.auth` module also contains a .. versionadded:: 0.1.3 +Custom User model +================= +Django 1.5 introduced `Custom user Models +` +which can be used as an alternative the Mongoengine authentication backend. + +The main advantage of this option is that other components relying on +:mod:`django.contrib.auth` and supporting the new swappable user model are more +likely to work. For example, you can use the ``createsuperuser`` management +command as usual. + +To enable the custom User model in Django, add ``'mongoengine.django.mongo_auth'`` +in your ``INSTALLED_APPS`` and set ``'mongo_auth.MongoUser'`` as the custom user +user model to use. In your **settings.py** file you will have:: + + INSTALLED_APPS = ( + ... + 'django.contrib.auth', + 'mongoengine.django.mongo_auth', + ... + ) + + AUTH_USER_MODEL = 'mongo_auth.MongoUser' + +An additional ``MONGOENGINE_USER_DOCUMENT`` setting enables you to replace the +:class:`~mongoengine.django.auth.User` class with another class of your choice:: + + MONGOENGINE_USER_DOCUMENT = 'mongoengine.django.auth.User' + +The custom :class:`User` must be a :class:`~mongoengine.Document` class, but +otherwise has the same requirements as a standard custom user model, +as specified in the `Django Documentation +`. +In particular, the custom class must define :attr:`USERNAME_FIELD` and +:attr:`REQUIRED_FIELDS` attributes. + Sessions ======== Django allows the use of different backend stores for its sessions. MongoEngine diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index d22f086..371f0e3 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -209,6 +209,9 @@ class User(Document): date_joined = DateTimeField(default=datetime_now, verbose_name=_('date joined')) + USERNAME_FIELD = 'username' + REQUIRED_FIELDS = ['email'] + meta = { 'allow_inheritance': True, 'indexes': [ diff --git a/mongoengine/django/mongo_auth/__init__.py b/mongoengine/django/mongo_auth/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mongoengine/django/mongo_auth/models.py b/mongoengine/django/mongo_auth/models.py new file mode 100644 index 0000000..9629e64 --- /dev/null +++ b/mongoengine/django/mongo_auth/models.py @@ -0,0 +1,90 @@ +from importlib import import_module + +from django.conf import settings +from django.contrib.auth.models import UserManager +from django.core.exceptions import ImproperlyConfigured +from django.db import models +from django.utils.translation import ugettext_lazy as _ + + +MONGOENGINE_USER_DOCUMENT = getattr( + settings, 'MONGOENGINE_USER_DOCUMENT', 'mongoengine.django.auth.User') + + +class MongoUserManager(UserManager): + """A User manager wich allows the use of MongoEngine documents in Django. + + To use the manager, you must tell django.contrib.auth to use MongoUser as + the user model. In you settings.py, you need: + + INSTALLED_APPS = ( + ... + 'django.contrib.auth', + 'mongoengine.django.mongo_auth', + ... + ) + AUTH_USER_MODEL = 'mongo_auth.MongoUser' + + Django will use the model object to access the custom Manager, which will + replace the original queryset with MongoEngine querysets. + + By default, mongoengine.django.auth.User will be used to store users. You + can specify another document class in MONGOENGINE_USER_DOCUMENT in your + settings.py. + + The User Document class has the same requirements as a standard custom user + model: https://docs.djangoproject.com/en/dev/topics/auth/customizing/ + + In particular, the User Document class must define USERNAME_FIELD and + REQUIRED_FIELDS. + + `AUTH_USER_MODEL` has been added in Django 1.5. + + """ + + def contribute_to_class(self, model, name): + super(MongoUserManager, self).contribute_to_class(model, name) + self.dj_model = self.model + self.model = self._get_user_document() + + self.dj_model.USERNAME_FIELD = self.model.USERNAME_FIELD + username = models.CharField(_('username'), max_length=30, unique=True) + username.contribute_to_class(self.dj_model, self.dj_model.USERNAME_FIELD) + + self.dj_model.REQUIRED_FIELDS = self.model.REQUIRED_FIELDS + for name in self.dj_model.REQUIRED_FIELDS: + field = models.CharField(_(name), max_length=30) + field.contribute_to_class(self.dj_model, name) + + def _get_user_document(self): + try: + name = MONGOENGINE_USER_DOCUMENT + dot = name.rindex('.') + module = import_module(name[:dot]) + return getattr(module, name[dot + 1:]) + except ImportError: + raise ImproperlyConfigured("Error importing %s, please check " + "settings.MONGOENGINE_USER_DOCUMENT" + % name) + + def get(self, *args, **kwargs): + try: + return self.get_query_set().get(*args, **kwargs) + except self.model.DoesNotExist: + # ModelBackend expects this exception + raise self.dj_model.DoesNotExist + + @property + def db(self): + raise NotImplementedError + + def get_empty_query_set(self): + return self.model.objects.none() + + def get_query_set(self): + return self.model.objects + + +class MongoUser(models.Model): + objects = MongoUserManager() + diff --git a/tests/test_django.py b/tests/test_django.py index dceeba2..6f4b6ea 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -14,9 +14,16 @@ try: from django.conf import settings from django.core.paginator import Paginator - settings.configure(USE_TZ=True) + settings.configure( + USE_TZ=True, + INSTALLED_APPS=('django.contrib.auth', 'mongoengine.django.mongo_auth'), + AUTH_USER_MODEL=('mongo_auth.MongoUser'), + ) + from django.contrib.auth import authenticate, get_user_model from django.contrib.sessions.tests import SessionTestsMixin + from mongoengine.django.auth import User + from mongoengine.django.mongo_auth.models import MongoUser, MongoUserManager from mongoengine.django.sessions import SessionStore, MongoSession except Exception, err: if PY3: @@ -156,6 +163,7 @@ class QuerySetTest(unittest.TestCase): rendered = template.render(Context({'users': users})) self.assertEqual(rendered, 'AB ABCD CD') + class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): backend = SessionStore @@ -184,5 +192,47 @@ class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): session = SessionStore(key) self.assertTrue('test_expire' in session, 'Session has expired before it is expected') + +class MongoAuthTest(unittest.TestCase): + user_data = { + 'username': 'user', + 'email': 'user@example.com', + 'password': 'test', + } + + def setUp(self): + if PY3: + raise SkipTest('django does not have Python 3 support') + connect(db='mongoenginetest') + User.drop_collection() + super(MongoAuthTest, self).setUp() + + def test_user_model(self): + self.assertEqual(get_user_model(), MongoUser) + + def test_user_manager(self): + manager = get_user_model()._default_manager + self.assertIsInstance(manager, MongoUserManager) + + def test_user_manager_exception(self): + manager = get_user_model()._default_manager + self.assertRaises(MongoUser.DoesNotExist, manager.get, + username='not found') + + def test_create_user(self): + manager = get_user_model()._default_manager + user = manager.create_user(**self.user_data) + self.assertIsInstance(user, User) + db_user = User.objects.get(username='user') + self.assertEqual(user.id, db_user.id) + + def test_authenticate(self): + get_user_model()._default_manager.create_user(**self.user_data) + user = authenticate(username='user', password='fail') + self.assertIsNone(user) + user = authenticate(username='user', password='test') + db_user = User.objects.get(username='user') + self.assertEqual(user.id, db_user.id) + if __name__ == '__main__': unittest.main() From dff44ef74e8e03bc82b82fa86582af45848ceabf Mon Sep 17 00:00:00 2001 From: Nicolas Cortot Date: Fri, 19 Apr 2013 17:50:15 +0200 Subject: [PATCH 1029/1279] Fixing warning which prevented tests from succeeding Now that we're importing the auth classes in the tests, no warning can be raised or test_dbref_reference_field_future_warning will fail. --- mongoengine/django/auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 371f0e3..0839b14 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -96,7 +96,7 @@ class Permission(Document): Three basic permissions -- add, change and delete -- are automatically created for each Django model. """ name = StringField(max_length=50, verbose_name=_('username')) - content_type = ReferenceField(ContentType) + content_type = ReferenceField(ContentType, dbref=True) codename = StringField(max_length=100, verbose_name=_('codename')) # FIXME: don't access field of the other class # unique_with=['content_type__app_label', 'content_type__model']) @@ -128,7 +128,7 @@ class Group(Document): """ name = StringField(max_length=80, unique=True, verbose_name=_('name')) # permissions = models.ManyToManyField(Permission, verbose_name=_('permissions'), blank=True) - permissions = ListField(ReferenceField(Permission, verbose_name=_('permissions'), required=False)) + permissions = ListField(ReferenceField(Permission, verbose_name=_('permissions'), required=False, dbref=True)) class Meta: verbose_name = _('group') From d39d10b9fb09c0a69dc00eb226db0429048e6b42 Mon Sep 17 00:00:00 2001 From: Nicolas Cortot Date: Fri, 19 Apr 2013 18:28:45 +0200 Subject: [PATCH 1030/1279] Tests should not require Django 1.5 to run --- tests/test_django.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_django.py b/tests/test_django.py index 6f4b6ea..01a105a 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -20,10 +20,14 @@ try: AUTH_USER_MODEL=('mongo_auth.MongoUser'), ) - from django.contrib.auth import authenticate, get_user_model + try: + from django.contrib.auth import authenticate, get_user_model + from mongoengine.django.auth import User + from mongoengine.django.mongo_auth.models import MongoUser, MongoUserManager + DJ15 = True + except Exception: + DJ15 = False from django.contrib.sessions.tests import SessionTestsMixin - from mongoengine.django.auth import User - from mongoengine.django.mongo_auth.models import MongoUser, MongoUserManager from mongoengine.django.sessions import SessionStore, MongoSession except Exception, err: if PY3: @@ -203,6 +207,8 @@ class MongoAuthTest(unittest.TestCase): def setUp(self): if PY3: raise SkipTest('django does not have Python 3 support') + if not DJ15: + raise SkipTest('mongo_auth requires Django 1.5') connect(db='mongoenginetest') User.drop_collection() super(MongoAuthTest, self).setUp() From 681b74a41cd040e5e6b7e7a9cf1fb505964289f7 Mon Sep 17 00:00:00 2001 From: Nicolas Cortot Date: Fri, 19 Apr 2013 18:53:42 +0200 Subject: [PATCH 1031/1279] Travis: adding Django-1.5.1 to env --- .travis.yml | 15 ++++++++++++--- setup.py | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1b9f5b7..ad2678f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,12 +7,21 @@ python: - "3.2" - "3.3" env: - - PYMONGO=dev - - PYMONGO=2.5 - - PYMONGO=2.4.2 + - PYMONGO=dev DJANGO=1.5.1 + - PYMONGO=dev DJANGO=1.4.2 + - PYMONGO=2.5 DJANGO=1.5.1 + - PYMONGO=2.5 DJANGO=1.4.2 + - PYMONGO=2.4.2 DJANGO=1.4.2 +matrix: + exclude: + - python: "2.6" + env: PYMONGO=dev DJANGO=1.5.1 + - python: "2.6" + env: PYMONGO=2.5 DJANGO=1.5.1 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install django==$DJANGO --use-mirrors ; true; fi - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - python setup.py install diff --git a/setup.py b/setup.py index ba538fa..c6270d9 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,7 @@ if sys.version_info[0] == 3: extra_opts['packages'].append("tests") extra_opts['package_data'] = {"tests": ["fields/mongoengine.png"]} else: - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django==1.4.2', 'PIL'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL'] extra_opts['packages'] = find_packages(exclude=('tests',)) setup(name='mongoengine', From 80db9e771671135e9260d088d4139e01b04b678c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 22 Apr 2013 13:06:29 +0000 Subject: [PATCH 1032/1279] Updated travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1b9f5b7..0e47cb7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,4 +23,3 @@ notifications: branches: only: - master - - "0.8" From c16e6d74e6fe1271b628af476c4f8db2027aff66 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 22 Apr 2013 15:07:15 +0000 Subject: [PATCH 1033/1279] Updated connection to use MongoClient (#262, #274) --- docs/changelog.rst | 1 + docs/guide/connecting.rst | 2 +- docs/upgrade.rst | 79 ++++++++++++++++------ mongoengine/connection.py | 8 +-- mongoengine/django/sessions.py | 2 +- mongoengine/document.py | 47 ++++++------- mongoengine/queryset/queryset.py | 111 ++++++++++++++++--------------- tests/document/indexes.py | 14 +++- tests/queryset/queryset.py | 33 +++++---- tests/test_connection.py | 9 ++- 10 files changed, 181 insertions(+), 125 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4547000..d7d010c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Updated connection to use MongoClient (#262, #274) - Fixed db_alias and inherited Documents (#143) - Documentation update for document errors (#124) - Deprecated `get_or_create` (#35) diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index ebd61a9..de6794c 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -29,7 +29,7 @@ name - just supply the uri as the :attr:`host` to ReplicaSets =========== -MongoEngine now supports :func:`~pymongo.replica_set_connection.ReplicaSetConnection` +MongoEngine supports :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient` to use them please use a URI style connection and provide the `replicaSet` name in the connection kwargs. diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 8724503..356f510 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -1,15 +1,15 @@ -========= +######### Upgrading -========= +######### 0.7 to 0.8 -========== +********** Inheritance ------------ +=========== Data Model -~~~~~~~~~~ +---------- The inheritance model has changed, we no longer need to store an array of :attr:`types` with the model we can just use the classname in :attr:`_cls`. @@ -44,7 +44,7 @@ inherited classes like so: :: Document Definition -~~~~~~~~~~~~~~~~~~~ +------------------- The default for inheritance has changed - its now off by default and :attr:`_cls` will not be stored automatically with the class. So if you extend @@ -77,7 +77,7 @@ the case and the data is set only in the ``document._data`` dictionary: :: AttributeError: 'Animal' object has no attribute 'size' Querysets -~~~~~~~~~ +========= Querysets now return clones and should no longer be considered editable in place. This brings us in line with how Django's querysets work and removes a @@ -98,8 +98,47 @@ update your code like so: :: mammals = Animal.objects(type="mammal").filter(order="Carnivora") # The final queryset is assgined to mammals [m for m in mammals] # This will return all carnivores +Client +====== +PyMongo 2.4 came with a new connection client; MongoClient_ and started the +depreciation of the old :class:`~pymongo.connection.Connection`. MongoEngine +now uses the latest `MongoClient` for connections. By default operations were +`safe` but if you turned them off or used the connection directly this will +impact your queries. + +Querysets +--------- + +Safe +^^^^ + +`safe` has been depreciated in the new MongoClient connection. Please use +`write_concern` instead. As `safe` always defaulted as `True` normally no code +change is required. To disable confirmation of the write just pass `{"w": 0}` +eg: :: + + # Old + Animal(name="Dinasour").save(safe=False) + + # new code: + Animal(name="Dinasour").save(write_concern={"w": 0}) + +Write Concern +^^^^^^^^^^^^^ + +`write_options` has been replaced with `write_concern` to bring it inline with +pymongo. To upgrade simply rename any instances where you used the `write_option` +keyword to `write_concern` like so:: + + # Old code: + Animal(name="Dinasour").save(write_options={"w": 2}) + + # new code: + Animal(name="Dinasour").save(write_concern={"w": 2}) + + Indexes -------- +======= Index methods are no longer tied to querysets but rather to the document class. Although `QuerySet._ensure_indexes` and `QuerySet.ensure_index` still exist. @@ -107,17 +146,19 @@ They should be replaced with :func:`~mongoengine.Document.ensure_indexes` / :func:`~mongoengine.Document.ensure_index`. SequenceFields --------------- +============== :class:`~mongoengine.fields.SequenceField` now inherits from `BaseField` to allow flexible storage of the calculated value. As such MIN and MAX settings are no longer handled. +.. _MongoClient: http://blog.mongodb.org/post/36666163412/introducing-mongoclient + 0.6 to 0.7 -========== +********** Cascade saves -------------- +============= Saves will raise a `FutureWarning` if they cascade and cascade hasn't been set to True. This is because in 0.8 it will default to False. If you require @@ -135,7 +176,7 @@ via `save` eg :: Remember: cascading saves **do not** cascade through lists. ReferenceFields ---------------- +=============== ReferenceFields now can store references as ObjectId strings instead of DBRefs. This will become the default in 0.8 and if `dbref` is not set a `FutureWarning` @@ -164,7 +205,7 @@ migrate :: item_frequencies ----------------- +================ In the 0.6 series we added support for null / zero / false values in item_frequencies. A side effect was to return keys in the value they are @@ -173,14 +214,14 @@ updated to handle native types rather than strings keys for the results of item frequency queries. BinaryFields ------------- +============ Binary fields have been updated so that they are native binary types. If you previously were doing `str` comparisons with binary field values you will have to update and wrap the value in a `str`. 0.5 to 0.6 -========== +********** Embedded Documents - if you had a `pk` field you will have to rename it from `_id` to `pk` as pk is no longer a property of Embedded Documents. @@ -200,13 +241,13 @@ don't define :attr:`allow_inheritance` in their meta. You may need to update pyMongo to 2.0 for use with Sharding. 0.4 to 0.5 -=========== +********** There have been the following backwards incompatibilities from 0.4 to 0.5. The main areas of changed are: choices in fields, map_reduce and collection names. Choice options: ---------------- +=============== Are now expected to be an iterable of tuples, with the first element in each tuple being the actual value to be stored. The second element is the @@ -214,7 +255,7 @@ human-readable name for the option. PyMongo / MongoDB ------------------ +================= map reduce now requires pymongo 1.11+- The pymongo `merge_output` and `reduce_output` parameters, have been depreciated. @@ -228,7 +269,7 @@ such the following have been changed: Default collection naming -------------------------- +========================= Previously it was just lowercase, its now much more pythonic and readable as its lowercase and underscores, previously :: diff --git a/mongoengine/connection.py b/mongoengine/connection.py index a47be44..3c53ea3 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,5 +1,5 @@ import pymongo -from pymongo import Connection, ReplicaSetConnection, uri_parser +from pymongo import MongoClient, MongoReplicaSetClient, uri_parser __all__ = ['ConnectionError', 'connect', 'register_connection', @@ -112,15 +112,15 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): conn_settings['slaves'] = slaves conn_settings.pop('read_preference', None) - connection_class = Connection + connection_class = MongoClient if 'replicaSet' in conn_settings: conn_settings['hosts_or_uri'] = conn_settings.pop('host', None) - # Discard port since it can't be used on ReplicaSetConnection + # Discard port since it can't be used on MongoReplicaSetClient conn_settings.pop('port', None) # Discard replicaSet if not base string if not isinstance(conn_settings['replicaSet'], basestring): conn_settings.pop('replicaSet', None) - connection_class = ReplicaSetConnection + connection_class = MongoReplicaSetClient try: _connections[alias] = connection_class(**conn_settings) diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index 0d199a6..29583f5 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -88,7 +88,7 @@ class SessionStore(SessionBase): s.session_data = self._get_session(no_load=must_create) s.expire_date = self.get_expiry_date() try: - s.save(force_insert=must_create, safe=True) + s.save(force_insert=must_create) except OperationError: if must_create: raise CreateError diff --git a/mongoengine/document.py b/mongoengine/document.py index 9057075..54b55df 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -142,7 +142,7 @@ class Document(BaseDocument): options.get('size') != max_size: msg = (('Cannot create collection "%s" as a capped ' 'collection as it already exists') - % cls._collection) + % cls._collection) raise InvalidCollectionError(msg) else: # Create the collection as a capped collection @@ -158,28 +158,24 @@ class Document(BaseDocument): cls.ensure_indexes() return cls._collection - def save(self, safe=True, force_insert=False, validate=True, clean=True, - write_options=None, cascade=None, cascade_kwargs=None, + def save(self, force_insert=False, validate=True, clean=True, + write_concern=None, cascade=None, cascade_kwargs=None, _refs=None, **kwargs): """Save the :class:`~mongoengine.Document` to the database. If the document already exists, it will be updated, otherwise it will be created. - If ``safe=True`` and the operation is unsuccessful, an - :class:`~mongoengine.OperationError` will be raised. - - :param safe: check if the operation succeeded before returning :param force_insert: only try to create a new document, don't allow updates of existing documents :param validate: validates the document; set to ``False`` to skip. :param clean: call the document clean method, requires `validate` to be True. - :param write_options: Extra keyword arguments are passed down to + :param write_concern: Extra keyword arguments are passed down to :meth:`~pymongo.collection.Collection.save` OR :meth:`~pymongo.collection.Collection.insert` which will be used as options for the resultant ``getLastError`` command. For example, - ``save(..., write_options={w: 2, fsync: True}, ...)`` will + ``save(..., write_concern={w: 2, fsync: True}, ...)`` will wait until at least two servers have recorded the write and will force an fsync on the primary server. :param cascade: Sets the flag for cascading saves. You can set a @@ -205,8 +201,8 @@ class Document(BaseDocument): if validate: self.validate(clean=clean) - if not write_options: - write_options = {} + if not write_concern: + write_concern = {} doc = self.to_mongo() @@ -216,11 +212,9 @@ class Document(BaseDocument): collection = self._get_collection() if created: if force_insert: - object_id = collection.insert(doc, safe=safe, - **write_options) + object_id = collection.insert(doc, **write_concern) else: - object_id = collection.save(doc, safe=safe, - **write_options) + object_id = collection.save(doc, **write_concern) else: object_id = doc['_id'] updates, removals = self._delta() @@ -247,7 +241,7 @@ class Document(BaseDocument): update_query["$unset"] = removals if updates or removals: last_error = collection.update(select_dict, update_query, - upsert=upsert, safe=safe, **write_options) + upsert=upsert, **write_concern) created = is_new_object(last_error) warn_cascade = not cascade and 'cascade' not in self._meta @@ -255,10 +249,9 @@ class Document(BaseDocument): if cascade is None else cascade) if cascade: kwargs = { - "safe": safe, "force_insert": force_insert, "validate": validate, - "write_options": write_options, + "write_concern": write_concern, "cascade": cascade } if cascade_kwargs: # Allow granular control over cascades @@ -305,7 +298,7 @@ class Document(BaseDocument): if ref and ref_id not in _refs: if warn_cascade: msg = ("Cascading saves will default to off in 0.8, " - "please explicitly set `.save(cascade=True)`") + "please explicitly set `.save(cascade=True)`") warnings.warn(msg, FutureWarning) _refs.append(ref_id) kwargs["_refs"] = _refs @@ -344,16 +337,21 @@ class Document(BaseDocument): # Need to add shard key to query, or you get an error return self._qs.filter(**self._object_key).update_one(**kwargs) - def delete(self, safe=False): + def delete(self, **write_concern): """Delete the :class:`~mongoengine.Document` from the database. This will only take effect if the document has been previously saved. - :param safe: check if the operation succeeded before returning + :param write_concern: Extra keyword arguments are passed down which + will be used as options for the resultant + ``getLastError`` command. For example, + ``save(..., write_concern={w: 2, fsync: True}, ...)`` will + wait until at least two servers have recorded the write and + will force an fsync on the primary server. """ signals.pre_delete.send(self.__class__, document=self) try: - self._qs.filter(**self._object_key).delete(safe=safe) + self._qs.filter(**self._object_key).delete(write_concern=write_concern) except pymongo.errors.OperationFailure, err: message = u'Could not delete document (%s)' % err.message raise OperationError(message) @@ -428,9 +426,8 @@ class Document(BaseDocument): .. versionchanged:: 0.6 Now chainable """ id_field = self._meta['id_field'] - obj = self._qs.filter( - **{id_field: self[id_field]} - ).limit(1).select_related(max_depth=max_depth) + obj = self._qs.filter(**{id_field: self[id_field]} + ).limit(1).select_related(max_depth=max_depth) if obj: obj = obj[0] else: diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 15c8e63..71332b9 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -221,7 +221,7 @@ class QuerySet(object): """ return self._document(**kwargs).save() - def get_or_create(self, write_options=None, auto_save=True, + def get_or_create(self, write_concern=None, auto_save=True, *q_objs, **query): """Retrieve unique object or create, if it doesn't exist. Returns a tuple of ``(object, created)``, where ``object`` is the retrieved or @@ -239,9 +239,9 @@ class QuerySet(object): don't accidently duplicate data when using this method. This is now scheduled to be removed before 1.0 - :param write_options: optional extra keyword arguments used if we + :param write_concern: optional extra keyword arguments used if we have to create a new document. - Passes any write_options onto :meth:`~mongoengine.Document.save` + Passes any write_concern onto :meth:`~mongoengine.Document.save` :param auto_save: if the object is to be saved automatically if not found. @@ -266,7 +266,7 @@ class QuerySet(object): doc = self._document(**query) if auto_save: - doc.save(write_options=write_options) + doc.save(write_concern=write_concern) return doc, True def first(self): @@ -279,18 +279,13 @@ class QuerySet(object): result = None return result - def insert(self, doc_or_docs, load_bulk=True, safe=False, - write_options=None): + def insert(self, doc_or_docs, load_bulk=True, write_concern=None): """bulk insert documents - If ``safe=True`` and the operation is unsuccessful, an - :class:`~mongoengine.OperationError` will be raised. - :param docs_or_doc: a document or list of documents to be inserted :param load_bulk (optional): If True returns the list of document instances - :param safe: check if the operation succeeded before returning - :param write_options: Extra keyword arguments are passed down to + :param write_concern: Extra keyword arguments are passed down to :meth:`~pymongo.collection.Collection.insert` which will be used as options for the resultant ``getLastError`` command. For example, @@ -305,9 +300,8 @@ class QuerySet(object): """ Document = _import_class('Document') - if not write_options: - write_options = {} - write_options.update({'safe': safe}) + if not write_concern: + write_concern = {} docs = doc_or_docs return_one = False @@ -319,7 +313,7 @@ class QuerySet(object): for doc in docs: if not isinstance(doc, self._document): msg = ("Some documents inserted aren't instances of %s" - % str(self._document)) + % str(self._document)) raise OperationError(msg) if doc.pk and not doc._created: msg = "Some documents have ObjectIds use doc.update() instead" @@ -328,7 +322,7 @@ class QuerySet(object): signals.pre_bulk_insert.send(self._document, documents=docs) try: - ids = self._collection.insert(raw, **write_options) + ids = self._collection.insert(raw, **write_concern) except pymongo.errors.OperationFailure, err: message = 'Could not save document (%s)' if re.match('^E1100[01] duplicate key', unicode(err)): @@ -340,7 +334,7 @@ class QuerySet(object): if not load_bulk: signals.post_bulk_insert.send( - self._document, documents=docs, loaded=False) + self._document, documents=docs, loaded=False) return return_one and ids[0] or ids documents = self.in_bulk(ids) @@ -348,7 +342,7 @@ class QuerySet(object): for obj_id in ids: results.append(documents.get(obj_id)) signals.post_bulk_insert.send( - self._document, documents=results, loaded=True) + self._document, documents=results, loaded=True) return return_one and results[0] or results def count(self): @@ -358,10 +352,15 @@ class QuerySet(object): return 0 return self._cursor.count(with_limit_and_skip=True) - def delete(self, safe=False): + def delete(self, write_concern=None): """Delete the documents matched by the query. - :param safe: check if the operation succeeded before returning + :param write_concern: Extra keyword arguments are passed down which + will be used as options for the resultant + ``getLastError`` command. For example, + ``save(..., write_concern={w: 2, fsync: True}, ...)`` will + wait until at least two servers have recorded the write and + will force an fsync on the primary server. """ queryset = self.clone() doc = queryset._document @@ -370,11 +369,14 @@ class QuerySet(object): signals.pre_delete.has_receivers_for(self._document) or signals.post_delete.has_receivers_for(self._document)) + if not write_concern: + write_concern = {} + # Handle deletes where skips or limits have been applied or has a # delete signal if queryset._skip or queryset._limit or has_delete_signal: for doc in queryset: - doc.delete(safe=safe) + doc.delete(write_concern=write_concern) return delete_rules = doc._meta.get('delete_rules') or {} @@ -386,7 +388,7 @@ class QuerySet(object): if rule == DENY and document_cls.objects( **{field_name + '__in': self}).count() > 0: msg = ("Could not delete document (%s.%s refers to it)" - % (document_cls.__name__, field_name)) + % (document_cls.__name__, field_name)) raise OperationError(msg) for rule_entry in delete_rules: @@ -396,36 +398,38 @@ class QuerySet(object): ref_q = document_cls.objects(**{field_name + '__in': self}) ref_q_count = ref_q.count() if (doc != document_cls and ref_q_count > 0 - or (doc == document_cls and ref_q_count > 0)): - ref_q.delete(safe=safe) + or (doc == document_cls and ref_q_count > 0)): + ref_q.delete(write_concern=write_concern) elif rule == NULLIFY: document_cls.objects(**{field_name + '__in': self}).update( - safe_update=safe, - **{'unset__%s' % field_name: 1}) + write_concern=write_concern, **{'unset__%s' % field_name: 1}) elif rule == PULL: document_cls.objects(**{field_name + '__in': self}).update( - safe_update=safe, - **{'pull_all__%s' % field_name: self}) + write_concern=write_concern, + **{'pull_all__%s' % field_name: self}) - queryset._collection.remove(queryset._query, safe=safe) + queryset._collection.remove(queryset._query, write_concern=write_concern) - def update(self, safe_update=True, upsert=False, multi=True, - write_options=None, **update): - """Perform an atomic update on the fields matched by the query. When - ``safe_update`` is used, the number of affected documents is returned. + def update(self, upsert=False, multi=True, write_concern=None, **update): + """Perform an atomic update on the fields matched by the query. - :param safe_update: check if the operation succeeded before returning :param upsert: Any existing document with that "_id" is overwritten. - :param write_options: extra keyword arguments for - :meth:`~pymongo.collection.Collection.update` + :param multi: Update multiple documents. + :param write_concern: Extra keyword arguments are passed down which + will be used as options for the resultant + ``getLastError`` command. For example, + ``save(..., write_concern={w: 2, fsync: True}, ...)`` will + wait until at least two servers have recorded the write and + will force an fsync on the primary server. + :param update: Django-style update keyword arguments .. versionadded:: 0.2 """ if not update: raise OperationError("No update parameters, would remove data") - if not write_options: - write_options = {} + if not write_concern: + write_concern = {} queryset = self.clone() query = queryset._query @@ -441,8 +445,7 @@ class QuerySet(object): try: ret = queryset._collection.update(query, update, multi=multi, - upsert=upsert, safe=safe_update, - **write_options) + upsert=upsert, **write_concern) if ret is not None and 'n' in ret: return ret['n'] except pymongo.errors.OperationFailure, err: @@ -451,21 +454,21 @@ class QuerySet(object): raise OperationError(message) raise OperationError(u'Update failed (%s)' % unicode(err)) - def update_one(self, safe_update=True, upsert=False, write_options=None, - **update): - """Perform an atomic update on first field matched by the query. When - ``safe_update`` is used, the number of affected documents is returned. + def update_one(self, upsert=False, write_concern=None, **update): + """Perform an atomic update on first field matched by the query. - :param safe_update: check if the operation succeeded before returning :param upsert: Any existing document with that "_id" is overwritten. - :param write_options: extra keyword arguments for - :meth:`~pymongo.collection.Collection.update` + :param write_concern: Extra keyword arguments are passed down which + will be used as options for the resultant + ``getLastError`` command. For example, + ``save(..., write_concern={w: 2, fsync: True}, ...)`` will + wait until at least two servers have recorded the write and + will force an fsync on the primary server. :param update: Django-style update keyword arguments .. versionadded:: 0.2 """ - return self.update(safe_update=True, upsert=upsert, multi=False, - write_options=None, **update) + return self.update(upsert=upsert, multi=False, write_concern=None, **update) def with_id(self, object_id): """Retrieve the object matching the id provided. Uses `object_id` only @@ -498,7 +501,7 @@ class QuerySet(object): if self._scalar: for doc in docs: doc_map[doc['_id']] = self._get_scalar( - self._document._from_son(doc)) + self._document._from_son(doc)) elif self._as_pymongo: for doc in docs: doc_map[doc['_id']] = self._get_as_pymongo(doc) @@ -523,10 +526,10 @@ class QuerySet(object): c = self.__class__(self._document, self._collection_obj) copy_props = ('_mongo_query', '_initial_query', '_none', '_query_obj', - '_where_clause', '_loaded_fields', '_ordering', '_snapshot', - '_timeout', '_class_check', '_slave_okay', '_read_preference', - '_iter', '_scalar', '_as_pymongo', '_as_pymongo_coerce', - '_limit', '_skip', '_hint', '_auto_dereference') + '_where_clause', '_loaded_fields', '_ordering', '_snapshot', + '_timeout', '_class_check', '_slave_okay', '_read_preference', + '_iter', '_scalar', '_as_pymongo', '_as_pymongo_coerce', + '_limit', '_skip', '_hint', '_auto_dereference') for prop in copy_props: val = getattr(self, prop) diff --git a/tests/document/indexes.py b/tests/document/indexes.py index ff08ef1..fea63a5 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -314,19 +314,27 @@ class IndexesTest(unittest.TestCase): """ class User(Document): meta = { + 'allow_inheritance': True, 'indexes': ['user_guid'], 'auto_create_index': False } user_guid = StringField(required=True) + class MongoUser(User): + pass + User.drop_collection() - u = User(user_guid='123') - u.save() + User(user_guid='123').save() + MongoUser(user_guid='123').save() - self.assertEqual(1, User.objects.count()) + self.assertEqual(2, User.objects.count()) info = User.objects._collection.index_information() self.assertEqual(info.keys(), ['_id_']) + + User.ensure_indexes() + info = User.objects._collection.index_information() + self.assertEqual(info.keys(), ['_cls_1_user_guid_1', '_id_']) User.drop_collection() def test_embedded_document_index(self): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 37670b0..42e98ae 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -278,24 +278,24 @@ class QuerySetTest(unittest.TestCase): query = query.filter(boolfield=True) self.assertEquals(query.count(), 1) - def test_update_write_options(self): - """Test that passing write_options works""" + def test_update_write_concern(self): + """Test that passing write_concern works""" self.Person.drop_collection() - write_options = {"fsync": True} + write_concern = {"fsync": True} author, created = self.Person.objects.get_or_create( - name='Test User', write_options=write_options) - author.save(write_options=write_options) + name='Test User', write_concern=write_concern) + author.save(write_concern=write_concern) self.Person.objects.update(set__name='Ross', - write_options=write_options) + write_concern=write_concern) author = self.Person.objects.first() self.assertEqual(author.name, 'Ross') - self.Person.objects.update_one(set__name='Test User', write_options=write_options) + self.Person.objects.update_one(set__name='Test User', write_concern=write_concern) author = self.Person.objects.first() self.assertEqual(author.name, 'Test User') @@ -592,10 +592,17 @@ class QuerySetTest(unittest.TestCase): blogs.append(Blog(title="post %s" % i, posts=[post1, post2])) Blog.objects.insert(blogs, load_bulk=False) - self.assertEqual(q, 1) # 1 for the insert + self.assertEqual(q, 1) # 1 for the insert + + Blog.drop_collection() + with query_counter() as q: + self.assertEqual(q, 0) + + Blog.ensure_indexes() + self.assertEqual(q, 1) Blog.objects.insert(blogs) - self.assertEqual(q, 3) # 1 for insert, and 1 for in bulk fetch (3 in total) + self.assertEqual(q, 3) # 1 for insert, and 1 for in bulk fetch (3 in total) Blog.drop_collection() @@ -619,7 +626,7 @@ class QuerySetTest(unittest.TestCase): self.assertRaises(OperationError, throw_operation_error) # Test can insert new doc - new_post = Blog(title="code", id=ObjectId()) + new_post = Blog(title="code123", id=ObjectId()) Blog.objects.insert(new_post) # test handles other classes being inserted @@ -655,13 +662,13 @@ class QuerySetTest(unittest.TestCase): Blog.objects.insert([blog1, blog2]) def throw_operation_error_not_unique(): - Blog.objects.insert([blog2, blog3], safe=True) + Blog.objects.insert([blog2, blog3]) self.assertRaises(NotUniqueError, throw_operation_error_not_unique) self.assertEqual(Blog.objects.count(), 2) - Blog.objects.insert([blog2, blog3], write_options={ - 'continue_on_error': True}) + Blog.objects.insert([blog2, blog3], write_concern={"w": 0, + 'continue_on_error': True}) self.assertEqual(Blog.objects.count(), 3) def test_get_changed_fields_query_count(self): diff --git a/tests/test_connection.py b/tests/test_connection.py index 5b9743d..4b8a3d1 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -10,7 +10,6 @@ from bson.tz_util import utc from mongoengine import * import mongoengine.connection from mongoengine.connection import get_db, get_connection, ConnectionError -from mongoengine.context_managers import switch_db class ConnectionTest(unittest.TestCase): @@ -26,7 +25,7 @@ class ConnectionTest(unittest.TestCase): connect('mongoenginetest') conn = get_connection() - self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient)) db = get_db() self.assertTrue(isinstance(db, pymongo.database.Database)) @@ -34,7 +33,7 @@ class ConnectionTest(unittest.TestCase): connect('mongoenginetest2', alias='testdb') conn = get_connection('testdb') - self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient)) def test_connect_uri(self): """Ensure that the connect() method works properly with uri's @@ -52,7 +51,7 @@ class ConnectionTest(unittest.TestCase): connect("testdb_uri", host='mongodb://username:password@localhost/mongoenginetest') conn = get_connection() - self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient)) db = get_db() self.assertTrue(isinstance(db, pymongo.database.Database)) @@ -65,7 +64,7 @@ class ConnectionTest(unittest.TestCase): self.assertRaises(ConnectionError, get_connection) conn = get_connection('testdb') - self.assertTrue(isinstance(conn, pymongo.connection.Connection)) + self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient)) db = get_db('testdb') self.assertTrue(isinstance(db, pymongo.database.Database)) From efad628a87e3bb3e4ec55f1ddcaab68853917e36 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 22 Apr 2013 15:32:11 +0000 Subject: [PATCH 1034/1279] Objects queryset manager now inherited (#256) --- docs/changelog.rst | 1 + mongoengine/base/metaclasses.py | 10 ++++----- tests/queryset/queryset.py | 36 +++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d7d010c..01f5a54 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Objects manager now inherited (#256) - Updated connection to use MongoClient (#262, #274) - Fixed db_alias and inherited Documents (#143) - Documentation update for document errors (#124) diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index a53744d..2704011 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -315,8 +315,8 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): # may set allow_inheritance to False simple_class = all([b._meta.get('abstract') for b in flattened_bases if hasattr(b, '_meta')]) - if (not simple_class and meta['allow_inheritance'] == False and - not meta['abstract']): + if (not simple_class and meta['allow_inheritance'] is False and + not meta['abstract']): raise ValueError('Only direct subclasses of Document may set ' '"allow_inheritance" to False') @@ -339,9 +339,9 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): if callable(collection): new_class._meta['collection'] = collection(new_class) - # Provide a default queryset unless one has been set - manager = attrs.get('objects', QuerySetManager()) - new_class.objects = manager + # Provide a default queryset unless exists or one has been set + if not hasattr(new_class, 'objects'): + new_class.objects = QuerySetManager() # Validate the fields and set primary key if needed for field_name, field in new_class._fields.iteritems(): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 42e98ae..0d3ebf3 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -2233,6 +2233,42 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(0, Foo.with_inactive.count()) self.assertEqual(1, Foo.objects.count()) + def test_inherit_objects(self): + + class Foo(Document): + meta = {'allow_inheritance': True} + active = BooleanField(default=True) + + @queryset_manager + def objects(klass, queryset): + return queryset(active=True) + + class Bar(Foo): + pass + + Bar.drop_collection() + Bar.objects.create(active=False) + self.assertEqual(0, Bar.objects.count()) + + def test_inherit_objects_override(self): + + class Foo(Document): + meta = {'allow_inheritance': True} + active = BooleanField(default=True) + + @queryset_manager + def objects(klass, queryset): + return queryset(active=True) + + class Bar(Foo): + @queryset_manager + def objects(klass, queryset): + return queryset(active=False) + + Bar.drop_collection() + Bar.objects.create(active=False) + self.assertEqual(0, Foo.objects.count()) + self.assertEqual(1, Bar.objects.count()) def test_query_value_conversion(self): """Ensure that query values are properly converted when necessary. From 0d0befe23e7ec141e4c81d7f01a3ba7e57c43865 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 22 Apr 2013 16:19:55 +0000 Subject: [PATCH 1035/1279] Removed __len__ from queryset (#247) --- docs/changelog.rst | 3 +- docs/code/tumblelog.py | 2 +- docs/upgrade.rst | 16 +++++++ mongoengine/queryset/queryset.py | 9 ++-- tests/document/instance.py | 32 +++++++------- tests/queryset/queryset.py | 74 ++++++++++++++++---------------- tests/queryset/transform.py | 8 ++-- tests/queryset/visitor.py | 8 ++-- 8 files changed, 82 insertions(+), 70 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 01f5a54..da1424e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,8 @@ Changelog Changes in 0.8.X ================ -- Objects manager now inherited (#256) +- Removed __len__ from queryset (#247) +- Objects queryset manager now inherited (#256) - Updated connection to use MongoClient (#262, #274) - Fixed db_alias and inherited Documents (#143) - Documentation update for document errors (#124) diff --git a/docs/code/tumblelog.py b/docs/code/tumblelog.py index 6ba1eee..0e40e89 100644 --- a/docs/code/tumblelog.py +++ b/docs/code/tumblelog.py @@ -45,7 +45,7 @@ print 'ALL POSTS' print for post in Post.objects: print post.title - print '=' * len(post.title) + print '=' * post.title.count() if isinstance(post, TextPost): print post.content diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 356f510..4f549d6 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -79,6 +79,9 @@ the case and the data is set only in the ``document._data`` dictionary: :: Querysets ========= +Attack of the clones +-------------------- + Querysets now return clones and should no longer be considered editable in place. This brings us in line with how Django's querysets work and removes a long running gotcha. If you edit your querysets inplace you will have to @@ -98,6 +101,19 @@ update your code like so: :: mammals = Animal.objects(type="mammal").filter(order="Carnivora") # The final queryset is assgined to mammals [m for m in mammals] # This will return all carnivores +No more len +----------- + +If you ever did len(queryset) it previously did a count() under the covers, this +caused some unusual issues - so now it has been removed in favour of the +explicit `queryset.count()` to update:: + + # Old code + len(Animal.objects(type="mammal")) + + # New code + Animal.objects(type="mammal").count()) + Client ====== PyMongo 2.4 came with a new connection client; MongoClient_ and started the diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 71332b9..cc0b70f 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -118,9 +118,6 @@ class QuerySet(object): queryset.rewind() return queryset - def __len__(self): - return self.count() - def __getitem__(self, key): """Support skip and limit using getitem and slicing syntax. """ @@ -149,12 +146,12 @@ class QuerySet(object): elif isinstance(key, int): if queryset._scalar: return queryset._get_scalar( - queryset._document._from_son(queryset._cursor[key], - _auto_dereference=self._auto_dereference)) + queryset._document._from_son(queryset._cursor[key], + _auto_dereference=self._auto_dereference)) if queryset._as_pymongo: return queryset._get_as_pymongo(queryset._cursor.next()) return queryset._document._from_son(queryset._cursor[key], - _auto_dereference=self._auto_dereference) + _auto_dereference=self._auto_dereference) raise AttributeError def __repr__(self): diff --git a/tests/document/instance.py b/tests/document/instance.py index 07991c1..1adc140 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -65,11 +65,11 @@ class InstanceTest(unittest.TestCase): for _ in range(10): Log().save() - self.assertEqual(len(Log.objects), 10) + self.assertEqual(Log.objects.count(), 10) # Check that extra documents don't increase the size Log().save() - self.assertEqual(len(Log.objects), 10) + self.assertEqual(Log.objects.count(), 10) options = Log.objects._collection.options() self.assertEqual(options['capped'], True) @@ -1040,9 +1040,9 @@ class InstanceTest(unittest.TestCase): """ person = self.Person(name="Test User", age=30) person.save() - self.assertEqual(len(self.Person.objects), 1) + self.assertEqual(self.Person.objects.count(), 1) person.delete() - self.assertEqual(len(self.Person.objects), 0) + self.assertEqual(self.Person.objects.count(), 0) def test_save_custom_id(self): """Ensure that a document may be saved with a custom _id. @@ -1356,12 +1356,12 @@ class InstanceTest(unittest.TestCase): post.save() reviewer.delete() - self.assertEqual(len(BlogPost.objects), 1) # No effect on the BlogPost + self.assertEqual(BlogPost.objects.count(), 1) # No effect on the BlogPost self.assertEqual(BlogPost.objects.get().reviewer, None) # Delete the Person, which should lead to deletion of the BlogPost, too author.delete() - self.assertEqual(len(BlogPost.objects), 0) + self.assertEqual(BlogPost.objects.count(), 0) def test_reverse_delete_rule_with_document_inheritance(self): """Ensure that a referenced document is also deleted upon deletion @@ -1391,12 +1391,12 @@ class InstanceTest(unittest.TestCase): post.save() reviewer.delete() - self.assertEqual(len(BlogPost.objects), 1) + self.assertEqual(BlogPost.objects.count(), 1) self.assertEqual(BlogPost.objects.get().reviewer, None) # Delete the Writer should lead to deletion of the BlogPost author.delete() - self.assertEqual(len(BlogPost.objects), 0) + self.assertEqual(BlogPost.objects.count(), 0) def test_reverse_delete_rule_cascade_and_nullify_complex_field(self): """Ensure that a referenced document is also deleted upon deletion for @@ -1425,12 +1425,12 @@ class InstanceTest(unittest.TestCase): # Deleting the reviewer should have no effect on the BlogPost reviewer.delete() - self.assertEqual(len(BlogPost.objects), 1) + self.assertEqual(BlogPost.objects.count(), 1) self.assertEqual(BlogPost.objects.get().reviewers, []) # Delete the Person, which should lead to deletion of the BlogPost, too author.delete() - self.assertEqual(len(BlogPost.objects), 0) + self.assertEqual(BlogPost.objects.count(), 0) def test_reverse_delete_rule_cascade_triggers_pre_delete_signal(self): ''' ensure the pre_delete signal is triggered upon a cascading deletion @@ -1498,7 +1498,7 @@ class InstanceTest(unittest.TestCase): f.delete() - self.assertEqual(len(Bar.objects), 1) # No effect on the BlogPost + self.assertEqual(Bar.objects.count(), 1) # No effect on the BlogPost self.assertEqual(Bar.objects.get().foo, None) def test_invalid_reverse_delete_rules_raise_errors(self): @@ -1549,7 +1549,7 @@ class InstanceTest(unittest.TestCase): # Delete the Person, which should lead to deletion of the BlogPost, and, # recursively to the Comment, too author.delete() - self.assertEqual(len(Comment.objects), 0) + self.assertEqual(Comment.objects.count(), 0) self.Person.drop_collection() BlogPost.drop_collection() @@ -1576,16 +1576,16 @@ class InstanceTest(unittest.TestCase): # Delete the Person should be denied self.assertRaises(OperationError, author.delete) # Should raise denied error - self.assertEqual(len(BlogPost.objects), 1) # No objects may have been deleted - self.assertEqual(len(self.Person.objects), 1) + self.assertEqual(BlogPost.objects.count(), 1) # No objects may have been deleted + self.assertEqual(self.Person.objects.count(), 1) # Other users, that don't have BlogPosts must be removable, like normal author = self.Person(name='Another User') author.save() - self.assertEqual(len(self.Person.objects), 2) + self.assertEqual(self.Person.objects.count(), 2) author.delete() - self.assertEqual(len(self.Person.objects), 1) + self.assertEqual(self.Person.objects.count(), 1) self.Person.drop_collection() BlogPost.drop_collection() diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 0d3ebf3..f5aec7e 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -72,7 +72,7 @@ class QuerySetTest(unittest.TestCase): # Find all people in the collection people = self.Person.objects - self.assertEqual(len(people), 2) + self.assertEqual(people.count(), 2) results = list(people) self.assertTrue(isinstance(results[0], self.Person)) self.assertTrue(isinstance(results[0].id, (ObjectId, str, unicode))) @@ -83,7 +83,7 @@ class QuerySetTest(unittest.TestCase): # Use a query to filter the people found to just person1 people = self.Person.objects(age=20) - self.assertEqual(len(people), 1) + self.assertEqual(people.count(), 1) person = people.next() self.assertEqual(person.name, "User A") self.assertEqual(person.age, 20) @@ -130,7 +130,7 @@ class QuerySetTest(unittest.TestCase): for i in xrange(55): self.Person(name='A%s' % i, age=i).save() - self.assertEqual(len(self.Person.objects), 55) + self.assertEqual(self.Person.objects.count(), 55) self.assertEqual("Person object", "%s" % self.Person.objects[0]) self.assertEqual("[, ]", "%s" % self.Person.objects[1:3]) self.assertEqual("[, ]", "%s" % self.Person.objects[51:53]) @@ -211,10 +211,10 @@ class QuerySetTest(unittest.TestCase): Blog.drop_collection() Blog.objects.create(tags=['a', 'b']) - self.assertEqual(len(Blog.objects(tags__0='a')), 1) - self.assertEqual(len(Blog.objects(tags__0='b')), 0) - self.assertEqual(len(Blog.objects(tags__1='a')), 0) - self.assertEqual(len(Blog.objects(tags__1='b')), 1) + self.assertEqual(Blog.objects(tags__0='a').count(), 1) + self.assertEqual(Blog.objects(tags__0='b').count(), 0) + self.assertEqual(Blog.objects(tags__1='a').count(), 0) + self.assertEqual(Blog.objects(tags__1='b').count(), 1) Blog.drop_collection() @@ -229,13 +229,13 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(blog, blog1) query = Blog.objects(posts__1__comments__1__name='testb') - self.assertEqual(len(query), 2) + self.assertEqual(query.count(), 2) query = Blog.objects(posts__1__comments__1__name='testa') - self.assertEqual(len(query), 0) + self.assertEqual(query.count(), 0) query = Blog.objects(posts__0__comments__1__name='testa') - self.assertEqual(len(query), 0) + self.assertEqual(query.count(), 0) Blog.drop_collection() @@ -344,7 +344,7 @@ class QuerySetTest(unittest.TestCase): # Update all of the first comments of second posts of all blogs blog = Blog.objects().update(set__posts__1__comments__0__name="testc") testc_blogs = Blog.objects(posts__1__comments__0__name="testc") - self.assertEqual(len(testc_blogs), 2) + self.assertEqual(testc_blogs.count(), 2) Blog.drop_collection() @@ -355,7 +355,7 @@ class QuerySetTest(unittest.TestCase): blog = Blog.objects().update_one( set__posts__1__comments__1__name="testc") testc_blogs = Blog.objects(posts__1__comments__1__name="testc") - self.assertEqual(len(testc_blogs), 1) + self.assertEqual(testc_blogs.count(), 1) # Check that using this indexing syntax on a non-list fails def non_list_indexing(): @@ -793,7 +793,7 @@ class QuerySetTest(unittest.TestCase): number = IntField() def __repr__(self): - return "" % self.number + return "" % self.number Doc.drop_collection() @@ -803,20 +803,17 @@ class QuerySetTest(unittest.TestCase): docs = Doc.objects.order_by('number') self.assertEqual(docs.count(), 1000) - self.assertEqual(len(docs), 1000) docs_string = "%s" % docs self.assertTrue("Doc: 0" in docs_string) self.assertEqual(docs.count(), 1000) - self.assertEqual(len(docs), 1000) # Limit and skip docs = docs[1:4] self.assertEqual('[, , ]', "%s" % docs) self.assertEqual(docs.count(), 3) - self.assertEqual(len(docs), 3) for doc in docs: self.assertEqual('.. queryset mid-iteration ..', repr(docs)) @@ -945,8 +942,10 @@ class QuerySetTest(unittest.TestCase): Blog.drop_collection() def assertSequence(self, qs, expected): + qs = list(qs) + expected = list(expected) self.assertEqual(len(qs), len(expected)) - for i in range(len(qs)): + for i in xrange(len(qs)): self.assertEqual(qs[i], expected[i]) def test_ordering(self): @@ -1124,13 +1123,13 @@ class QuerySetTest(unittest.TestCase): self.Person(name="User B", age=30).save() self.Person(name="User C", age=40).save() - self.assertEqual(len(self.Person.objects), 3) + self.assertEqual(self.Person.objects.count(), 3) self.Person.objects(age__lt=30).delete() - self.assertEqual(len(self.Person.objects), 2) + self.assertEqual(self.Person.objects.count(), 2) self.Person.objects.delete() - self.assertEqual(len(self.Person.objects), 0) + self.assertEqual(self.Person.objects.count(), 0) def test_reverse_delete_rule_cascade(self): """Ensure cascading deletion of referring documents from the database. @@ -2332,8 +2331,8 @@ class QuerySetTest(unittest.TestCase): t = Test(testdict={'f': 'Value'}) t.save() - self.assertEqual(len(Test.objects(testdict__f__startswith='Val')), 1) - self.assertEqual(len(Test.objects(testdict__f='Value')), 1) + self.assertEqual(Test.objects(testdict__f__startswith='Val').count(), 1) + self.assertEqual(Test.objects(testdict__f='Value').count(), 1) Test.drop_collection() class Test(Document): @@ -2342,8 +2341,8 @@ class QuerySetTest(unittest.TestCase): t = Test(testdict={'f': 'Value'}) t.save() - self.assertEqual(len(Test.objects(testdict__f='Value')), 1) - self.assertEqual(len(Test.objects(testdict__f__startswith='Val')), 1) + self.assertEqual(Test.objects(testdict__f='Value').count(), 1) + self.assertEqual(Test.objects(testdict__f__startswith='Val').count(), 1) Test.drop_collection() def test_bulk(self): @@ -2539,8 +2538,7 @@ class QuerySetTest(unittest.TestCase): # Finds only one point because only the first point is within 60km of # the reference point to the south. points = Point.objects( - location__within_spherical_distance=[[-122, 36.5], 60/earth_radius] - ); + location__within_spherical_distance=[[-122, 36.5], 60/earth_radius]) self.assertEqual(points.count(), 1) self.assertEqual(points[0].id, south_point.id) @@ -2551,7 +2549,7 @@ class QuerySetTest(unittest.TestCase): """ class CustomQuerySet(QuerySet): def not_empty(self): - return len(self) > 0 + return self.count() > 0 class Post(Document): meta = {'queryset_class': CustomQuerySet} @@ -2572,7 +2570,7 @@ class QuerySetTest(unittest.TestCase): class CustomQuerySet(QuerySet): def not_empty(self): - return len(self) > 0 + return self.count() > 0 class CustomQuerySetManager(QuerySetManager): queryset_class = CustomQuerySet @@ -2619,7 +2617,7 @@ class QuerySetTest(unittest.TestCase): class CustomQuerySet(QuerySet): def not_empty(self): - return len(self) > 0 + return self.count() > 0 class Base(Document): meta = {'abstract': True, 'queryset_class': CustomQuerySet} @@ -2642,7 +2640,7 @@ class QuerySetTest(unittest.TestCase): class CustomQuerySet(QuerySet): def not_empty(self): - return len(self) > 0 + return self.count() > 0 class CustomQuerySetManager(QuerySetManager): queryset_class = CustomQuerySet @@ -3044,14 +3042,14 @@ class QuerySetTest(unittest.TestCase): # Find all people in the collection people = self.Person.objects.scalar('name') - self.assertEqual(len(people), 2) + self.assertEqual(people.count(), 2) results = list(people) self.assertEqual(results[0], "User A") self.assertEqual(results[1], "User B") # Use a query to filter the people found to just person1 people = self.Person.objects(age=20).scalar('name') - self.assertEqual(len(people), 1) + self.assertEqual(people.count(), 1) person = people.next() self.assertEqual(person, "User A") @@ -3097,7 +3095,7 @@ class QuerySetTest(unittest.TestCase): for i in xrange(55): self.Person(name='A%s' % i, age=i).save() - self.assertEqual(len(self.Person.objects.scalar('name')), 55) + self.assertEqual(self.Person.objects.scalar('name').count(), 55) self.assertEqual("A0", "%s" % self.Person.objects.order_by('name').scalar('name').first()) self.assertEqual("A0", "%s" % self.Person.objects.scalar('name').order_by('name')[0]) if PY3: @@ -3314,7 +3312,7 @@ class QuerySetTest(unittest.TestCase): inner_count = 0 inner_total_count = 0 - self.assertEqual(len(users), 7) + self.assertEqual(users.count(), 7) for i, outer_user in enumerate(users): self.assertEqual(outer_user.name, names[i]) @@ -3322,17 +3320,17 @@ class QuerySetTest(unittest.TestCase): inner_count = 0 # Calling len might disrupt the inner loop if there are bugs - self.assertEqual(len(users), 7) + self.assertEqual(users.count(), 7) for j, inner_user in enumerate(users): self.assertEqual(inner_user.name, names[j]) inner_count += 1 inner_total_count += 1 - self.assertEqual(inner_count, 7) # inner loop should always be executed seven times + self.assertEqual(inner_count, 7) # inner loop should always be executed seven times - self.assertEqual(outer_count, 7) # outer loop should be executed seven times total - self.assertEqual(inner_total_count, 7 * 7) # inner loop should be executed fourtynine times total + self.assertEqual(outer_count, 7) # outer loop should be executed seven times total + self.assertEqual(inner_total_count, 7 * 7) # inner loop should be executed fourtynine times total if __name__ == '__main__': unittest.main() diff --git a/tests/queryset/transform.py b/tests/queryset/transform.py index d38cbfd..bde4b6f 100644 --- a/tests/queryset/transform.py +++ b/tests/queryset/transform.py @@ -53,14 +53,14 @@ class TransformTest(unittest.TestCase): BlogPost.objects(title=data['title'])._query) self.assertFalse('title' in BlogPost.objects(title=data['title'])._query) - self.assertEqual(len(BlogPost.objects(title=data['title'])), 1) + self.assertEqual(BlogPost.objects(title=data['title']).count(), 1) self.assertTrue('_id' in BlogPost.objects(pk=post.id)._query) - self.assertEqual(len(BlogPost.objects(pk=post.id)), 1) + self.assertEqual(BlogPost.objects(pk=post.id).count(), 1) self.assertTrue('postComments.commentContent' in BlogPost.objects(comments__content='test')._query) - self.assertEqual(len(BlogPost.objects(comments__content='test')), 1) + self.assertEqual(BlogPost.objects(comments__content='test').count(), 1) BlogPost.drop_collection() @@ -79,7 +79,7 @@ class TransformTest(unittest.TestCase): self.assertTrue('_id' in BlogPost.objects(pk=data['title'])._query) self.assertTrue('_id' in BlogPost.objects(title=data['title'])._query) - self.assertEqual(len(BlogPost.objects(pk=data['title'])), 1) + self.assertEqual(BlogPost.objects(pk=data['title']).count(), 1) BlogPost.drop_collection() diff --git a/tests/queryset/visitor.py b/tests/queryset/visitor.py index 98815db..bd81a65 100644 --- a/tests/queryset/visitor.py +++ b/tests/queryset/visitor.py @@ -268,8 +268,8 @@ class QTest(unittest.TestCase): self.Person(name='user3', age=30).save() self.Person(name='user4', age=40).save() - self.assertEqual(len(self.Person.objects(Q(age__in=[20]))), 2) - self.assertEqual(len(self.Person.objects(Q(age__in=[20, 30]))), 3) + self.assertEqual(self.Person.objects(Q(age__in=[20])).count(), 2) + self.assertEqual(self.Person.objects(Q(age__in=[20, 30])).count(), 3) # Test invalid query objs def wrong_query_objs(): @@ -311,8 +311,8 @@ class QTest(unittest.TestCase): BlogPost(tags=['python', 'mongo']).save() BlogPost(tags=['python']).save() - self.assertEqual(len(BlogPost.objects(Q(tags='mongo'))), 1) - self.assertEqual(len(BlogPost.objects(Q(tags='python'))), 2) + self.assertEqual(BlogPost.objects(Q(tags='mongo')).count(), 1) + self.assertEqual(BlogPost.objects(Q(tags='python')).count(), 2) BlogPost.drop_collection() From 14b6c471cf519df6df5c737faaff512a338ef918 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 22 Apr 2013 16:37:09 +0000 Subject: [PATCH 1036/1279] Fix PY3 hasattr connecting to the db at define time --- mongoengine/base/metaclasses.py | 2 +- tests/document/instance.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index 2704011..def8a05 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -340,7 +340,7 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): new_class._meta['collection'] = collection(new_class) # Provide a default queryset unless exists or one has been set - if not hasattr(new_class, 'objects'): + if 'objects' not in dir(new_class): new_class.objects = QuerySetManager() # Validate the fields and set primary key if needed diff --git a/tests/document/instance.py b/tests/document/instance.py index 1adc140..5513ed8 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1857,6 +1857,8 @@ class InstanceTest(unittest.TestCase): def test_db_alias_propagates(self): """db_alias propagates? """ + register_connection('testdb-1', 'mongoenginetest2') + class A(Document): name = StringField() meta = {"db_alias": "testdb-1", "allow_inheritance": True} From e4f38b5665a3281eb26e365e1b5c8b050820efa1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 22 Apr 2013 16:46:59 +0000 Subject: [PATCH 1037/1279] Fragile test fix --- tests/document/indexes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/document/indexes.py b/tests/document/indexes.py index fea63a5..61e3c0e 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -334,7 +334,7 @@ class IndexesTest(unittest.TestCase): User.ensure_indexes() info = User.objects._collection.index_information() - self.assertEqual(info.keys(), ['_cls_1_user_guid_1', '_id_']) + self.assertEqual(sorted(info.keys()), ['_cls_1_user_guid_1', '_id_']) User.drop_collection() def test_embedded_document_index(self): From 81c7007f80a1147e8577ac2682f802f209e3338a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 10:38:32 +0000 Subject: [PATCH 1038/1279] Added with_limit_and_skip support to count() (#235) --- docs/changelog.rst | 1 + mongoengine/queryset/queryset.py | 8 +++-- tests/queryset/queryset.py | 52 ++++++++++++++++++-------------- 3 files changed, 36 insertions(+), 25 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index da1424e..4c0da7f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Added with_limit_and_skip support to count() (#235) - Removed __len__ from queryset (#247) - Objects queryset manager now inherited (#256) - Updated connection to use MongoClient (#262, #274) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index cc0b70f..37d07a8 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -342,12 +342,16 @@ class QuerySet(object): self._document, documents=results, loaded=True) return return_one and results[0] or results - def count(self): + def count(self, with_limit_and_skip=True): """Count the selected elements in the query. + + :param with_limit_and_skip (optional): take any :meth:`limit` or + :meth:`skip` that has been applied to this cursor into account when + getting the count """ if self._limit == 0: return 0 - return self._cursor.count(with_limit_and_skip=True) + return self._cursor.count(with_limit_and_skip=with_limit_and_skip) def delete(self, write_concern=None): """Delete the documents matched by the query. diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index f5aec7e..c7c4c7c 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -20,7 +20,7 @@ from mongoengine.python_support import PY3 from mongoengine.context_managers import query_counter from mongoengine.queryset import (QuerySet, QuerySetManager, MultipleObjectsReturned, DoesNotExist, - QueryFieldList, queryset_manager) + queryset_manager) from mongoengine.errors import InvalidQueryError __all__ = ("QuerySetTest",) @@ -65,10 +65,8 @@ class QuerySetTest(unittest.TestCase): def test_find(self): """Ensure that a query returns a valid set of results. """ - person1 = self.Person(name="User A", age=20) - person1.save() - person2 = self.Person(name="User B", age=30) - person2.save() + self.Person(name="User A", age=20).save() + self.Person(name="User B", age=30).save() # Find all people in the collection people = self.Person.objects @@ -338,21 +336,20 @@ class QuerySetTest(unittest.TestCase): comment2 = Comment(name='testb') post1 = Post(comments=[comment1, comment2]) post2 = Post(comments=[comment2, comment2]) - blog1 = Blog.objects.create(posts=[post1, post2]) - blog2 = Blog.objects.create(posts=[post2, post1]) + Blog.objects.create(posts=[post1, post2]) + Blog.objects.create(posts=[post2, post1]) # Update all of the first comments of second posts of all blogs - blog = Blog.objects().update(set__posts__1__comments__0__name="testc") + Blog.objects().update(set__posts__1__comments__0__name="testc") testc_blogs = Blog.objects(posts__1__comments__0__name="testc") self.assertEqual(testc_blogs.count(), 2) Blog.drop_collection() - - blog1 = Blog.objects.create(posts=[post1, post2]) - blog2 = Blog.objects.create(posts=[post2, post1]) + Blog.objects.create(posts=[post1, post2]) + Blog.objects.create(posts=[post2, post1]) # Update only the first blog returned by the query - blog = Blog.objects().update_one( + Blog.objects().update_one( set__posts__1__comments__1__name="testc") testc_blogs = Blog.objects(posts__1__comments__1__name="testc") self.assertEqual(testc_blogs.count(), 1) @@ -2661,6 +2658,19 @@ class QuerySetTest(unittest.TestCase): Post.drop_collection() + def test_count_limit_and_skip(self): + class Post(Document): + title = StringField() + + Post.drop_collection() + + for i in xrange(10): + Post(title="Post %s" % i).save() + + self.assertEqual(5, Post.objects.limit(5).skip(5).count()) + + self.assertEqual(10, Post.objects.limit(5).skip(5).count(with_limit_and_skip=False)) + def test_call_after_limits_set(self): """Ensure that re-filtering after slicing works """ @@ -2669,10 +2679,8 @@ class QuerySetTest(unittest.TestCase): Post.drop_collection() - post1 = Post(title="Post 1") - post1.save() - post2 = Post(title="Post 2") - post2.save() + Post(title="Post 1").save() + Post(title="Post 2").save() posts = Post.objects.all()[0:1] self.assertEqual(len(list(posts())), 1) @@ -3205,20 +3213,18 @@ class QuerySetTest(unittest.TestCase): float_field = FloatField(default=1.1) boolean_field = BooleanField(default=True) datetime_field = DateTimeField(default=datetime.now) - embedded_document_field = EmbeddedDocumentField(EmbeddedDoc, - default=lambda: EmbeddedDoc()) + embedded_document_field = EmbeddedDocumentField( + EmbeddedDoc, default=lambda: EmbeddedDoc()) list_field = ListField(default=lambda: [1, 2, 3]) dict_field = DictField(default=lambda: {"hello": "world"}) objectid_field = ObjectIdField(default=ObjectId) - reference_field = ReferenceField(Simple, default=lambda: - Simple().save()) + reference_field = ReferenceField(Simple, default=lambda: Simple().save()) map_field = MapField(IntField(), default=lambda: {"simple": 1}) decimal_field = DecimalField(default=1.0) complex_datetime_field = ComplexDateTimeField(default=datetime.now) url_field = URLField(default="http://mongoengine.org") dynamic_field = DynamicField(default=1) - generic_reference_field = GenericReferenceField( - default=lambda: Simple().save()) + generic_reference_field = GenericReferenceField(default=lambda: Simple().save()) sorted_list_field = SortedListField(IntField(), default=lambda: [1, 2, 3]) email_field = EmailField(default="ross@example.com") @@ -3226,7 +3232,7 @@ class QuerySetTest(unittest.TestCase): sequence_field = SequenceField() uuid_field = UUIDField(default=uuid.uuid4) generic_embedded_document_field = GenericEmbeddedDocumentField( - default=lambda: EmbeddedDoc()) + default=lambda: EmbeddedDoc()) Simple.drop_collection() Doc.drop_collection() From e2f3406e897fe0a033ae340665427c2b15ad9ce1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 14:06:29 +0000 Subject: [PATCH 1039/1279] Updated .only() behaviour - now like exclude it is chainable (#202) --- docs/changelog.rst | 1 + docs/upgrade.rst | 15 +++++++++++++ mongoengine/queryset/field_list.py | 23 +++++++++++++++---- mongoengine/queryset/queryset.py | 23 ++++++++++++++++--- tests/queryset/field_list.py | 36 +++++++++++++++--------------- 5 files changed, 73 insertions(+), 25 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4c0da7f..fb9c35d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Updated .only() behaviour - now like exclude it is chainable (#202) - Added with_limit_and_skip support to count() (#235) - Removed __len__ from queryset (#247) - Objects queryset manager now inherited (#256) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 4f549d6..c4273f0 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -114,6 +114,21 @@ explicit `queryset.count()` to update:: # New code Animal.objects(type="mammal").count()) + +.only() now inline with .exclude() +---------------------------------- + +The behaviour of `.only()` was highly ambious, now it works in the mirror fashion +to `.exclude()`. Chaining `.only()` calls will increase the fields required:: + + # Old code + Animal.objects().only(['type', 'name']).only('name', 'order') # Would have returned just `name` + + # New code + Animal.objects().only('name') + Animal.objects().only(['name']).only('order') # Would return `name` and `order` + + Client ====== PyMongo 2.4 came with a new connection client; MongoClient_ and started the diff --git a/mongoengine/queryset/field_list.py b/mongoengine/queryset/field_list.py index 7b2b0cb..73d3cc2 100644 --- a/mongoengine/queryset/field_list.py +++ b/mongoengine/queryset/field_list.py @@ -7,11 +7,20 @@ class QueryFieldList(object): ONLY = 1 EXCLUDE = 0 - def __init__(self, fields=[], value=ONLY, always_include=[]): + def __init__(self, fields=None, value=ONLY, always_include=None, _only_called=False): + """The QueryFieldList builder + + :param fields: A list of fields used in `.only()` or `.exclude()` + :param value: How to handle the fields; either `ONLY` or `EXCLUDE` + :param always_include: Any fields to always_include eg `_cls` + :param _only_called: Has `.only()` been called? If so its a set of fields + otherwise it performs a union. + """ self.value = value - self.fields = set(fields) - self.always_include = set(always_include) + self.fields = set(fields or []) + self.always_include = set(always_include or []) self._id = None + self._only_called = _only_called self.slice = {} def __add__(self, f): @@ -26,7 +35,10 @@ class QueryFieldList(object): self.slice = {} elif self.value is self.ONLY and f.value is self.ONLY: self._clean_slice() - self.fields = self.fields.intersection(f.fields) + if self._only_called: + self.fields = self.fields.union(f.fields) + else: + self.fields = f.fields elif self.value is self.EXCLUDE and f.value is self.EXCLUDE: self.fields = self.fields.union(f.fields) self._clean_slice() @@ -46,6 +58,9 @@ class QueryFieldList(object): self.fields = self.fields.union(self.always_include) else: self.fields -= self.always_include + + if getattr(f, '_only_called', False): + self._only_called = True return self def __nonzero__(self): diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 37d07a8..dcfb240 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -624,19 +624,35 @@ class QuerySet(object): post = BlogPost.objects(...).only("title", "author.name") + .. note :: `only()` is chainable and will perform a union :: + So with the following it will fetch both: `title` and `author.name`:: + + post = BlogPost.objects.only("title").only("author.name") + + :func:`~mongoengine.queryset.QuerySet.all_fields` will reset any + field filters. + :param fields: fields to include .. versionadded:: 0.3 .. versionchanged:: 0.5 - Added subfield support """ fields = dict([(f, QueryFieldList.ONLY) for f in fields]) - return self.fields(**fields) + return self.fields(True, **fields) def exclude(self, *fields): """Opposite to .only(), exclude some document's fields. :: post = BlogPost.objects(...).exclude("comments") + .. note :: `exclude()` is chainable and will perform a union :: + So with the following it will exclude both: `title` and `author.name`:: + + post = BlogPost.objects.exclude("title").exclude("author.name") + + :func:`~mongoengine.queryset.QuerySet.all_fields` will reset any + field filters. + :param fields: fields to exclude .. versionadded:: 0.5 @@ -644,7 +660,7 @@ class QuerySet(object): fields = dict([(f, QueryFieldList.EXCLUDE) for f in fields]) return self.fields(**fields) - def fields(self, **kwargs): + def fields(self, _only_called=False, **kwargs): """Manipulate how you load this document's fields. Used by `.only()` and `.exclude()` to manipulate which fields to retrieve. Fields also allows for a greater level of control for example: @@ -678,7 +694,8 @@ class QuerySet(object): for value, group in itertools.groupby(fields, lambda x: x[1]): fields = [field for field, value in group] fields = queryset._fields_to_dbfields(fields) - queryset._loaded_fields += QueryFieldList(fields, value=value) + queryset._loaded_fields += QueryFieldList(fields, value=value, _only_called=_only_called) + return queryset def all_fields(self): diff --git a/tests/queryset/field_list.py b/tests/queryset/field_list.py index 4a8a72b..2bdfce1 100644 --- a/tests/queryset/field_list.py +++ b/tests/queryset/field_list.py @@ -20,47 +20,47 @@ class QueryFieldListTest(unittest.TestCase): def test_include_include(self): q = QueryFieldList() - q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'a': True, 'b': True}) + q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.ONLY, _only_called=True) + self.assertEqual(q.as_dict(), {'a': 1, 'b': 1}) q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'b': True}) + self.assertEqual(q.as_dict(), {'a': 1, 'b': 1, 'c': 1}) def test_include_exclude(self): q = QueryFieldList() q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'a': True, 'b': True}) + self.assertEqual(q.as_dict(), {'a': 1, 'b': 1}) q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.EXCLUDE) - self.assertEqual(q.as_dict(), {'a': True}) + self.assertEqual(q.as_dict(), {'a': 1}) def test_exclude_exclude(self): q = QueryFieldList() q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.EXCLUDE) - self.assertEqual(q.as_dict(), {'a': False, 'b': False}) + self.assertEqual(q.as_dict(), {'a': 0, 'b': 0}) q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.EXCLUDE) - self.assertEqual(q.as_dict(), {'a': False, 'b': False, 'c': False}) + self.assertEqual(q.as_dict(), {'a': 0, 'b': 0, 'c': 0}) def test_exclude_include(self): q = QueryFieldList() q += QueryFieldList(fields=['a', 'b'], value=QueryFieldList.EXCLUDE) - self.assertEqual(q.as_dict(), {'a': False, 'b': False}) + self.assertEqual(q.as_dict(), {'a': 0, 'b': 0}) q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'c': True}) + self.assertEqual(q.as_dict(), {'c': 1}) def test_always_include(self): q = QueryFieldList(always_include=['x', 'y']) q += QueryFieldList(fields=['a', 'b', 'x'], value=QueryFieldList.EXCLUDE) q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'c': True}) + self.assertEqual(q.as_dict(), {'x': 1, 'y': 1, 'c': 1}) def test_reset(self): q = QueryFieldList(always_include=['x', 'y']) q += QueryFieldList(fields=['a', 'b', 'x'], value=QueryFieldList.EXCLUDE) q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'c': True}) + self.assertEqual(q.as_dict(), {'x': 1, 'y': 1, 'c': 1}) q.reset() self.assertFalse(q) q += QueryFieldList(fields=['b', 'c'], value=QueryFieldList.ONLY) - self.assertEqual(q.as_dict(), {'x': True, 'y': True, 'b': True, 'c': True}) + self.assertEqual(q.as_dict(), {'x': 1, 'y': 1, 'b': 1, 'c': 1}) def test_using_a_slice(self): q = QueryFieldList() @@ -97,7 +97,7 @@ class OnlyExcludeAllTest(unittest.TestCase): qs = MyDoc.objects.fields(**dict(((i, 1) for i in include))) self.assertEqual(qs._loaded_fields.as_dict(), - {'a': 1, 'b': 1, 'c': 1, 'd': 1, 'e': 1}) + {'a': 1, 'b': 1, 'c': 1, 'd': 1, 'e': 1}) qs = qs.only(*only) self.assertEqual(qs._loaded_fields.as_dict(), {'b': 1, 'c': 1}) qs = qs.exclude(*exclude) @@ -134,15 +134,15 @@ class OnlyExcludeAllTest(unittest.TestCase): qs = qs.only(*only) qs = qs.fields(slice__b=5) self.assertEqual(qs._loaded_fields.as_dict(), - {'b': {'$slice': 5}, 'c': 1}) + {'b': {'$slice': 5}, 'c': 1}) qs = qs.fields(slice__c=[5, 1]) self.assertEqual(qs._loaded_fields.as_dict(), - {'b': {'$slice': 5}, 'c': {'$slice': [5, 1]}}) + {'b': {'$slice': 5}, 'c': {'$slice': [5, 1]}}) qs = qs.exclude('c') self.assertEqual(qs._loaded_fields.as_dict(), - {'b': {'$slice': 5}}) + {'b': {'$slice': 5}}) def test_only(self): """Ensure that QuerySet.only only returns the requested fields. @@ -328,7 +328,7 @@ class OnlyExcludeAllTest(unittest.TestCase): Numbers.drop_collection() - numbers = Numbers(n=[0,1,2,3,4,5,-5,-4,-3,-2,-1]) + numbers = Numbers(n=[0, 1, 2, 3, 4, 5, -5, -4, -3, -2, -1]) numbers.save() # first three @@ -368,7 +368,7 @@ class OnlyExcludeAllTest(unittest.TestCase): Numbers.drop_collection() numbers = Numbers() - numbers.embedded = EmbeddedNumber(n=[0,1,2,3,4,5,-5,-4,-3,-2,-1]) + numbers.embedded = EmbeddedNumber(n=[0, 1, 2, 3, 4, 5, -5, -4, -3, -2, -1]) numbers.save() # first three From a692316293c1cdea8c8d10089e0651e7478dcc28 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 14:09:41 +0000 Subject: [PATCH 1040/1279] Update to upgrade docs --- docs/upgrade.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index c4273f0..5490757 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -126,7 +126,9 @@ to `.exclude()`. Chaining `.only()` calls will increase the fields required:: # New code Animal.objects().only('name') - Animal.objects().only(['name']).only('order') # Would return `name` and `order` + + # Note: + Animal.objects().only(['name']).only('order') # Now returns `name` *and* `order` Client From 94d1e566c018bc26875b1cbc585ba2215d88f80d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 14:44:17 +0000 Subject: [PATCH 1041/1279] Added SequenceField.set_next_value(value) helper (#159) --- docs/changelog.rst | 1 + mongoengine/fields.py | 19 +++++++++++++++---- tests/fields/fields.py | 40 +++++++++++++++++++++++++--------------- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index fb9c35d..d22fc60 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Added SequenceField.set_next_value(value) helper (#159) - Updated .only() behaviour - now like exclude it is chainable (#202) - Added with_limit_and_skip support to count() (#235) - Removed __len__ from queryset (#247) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 690e7ac..4fc65c7 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1408,13 +1408,13 @@ class SequenceField(BaseField): COLLECTION_NAME = 'mongoengine.counters' VALUE_DECORATOR = int - def __init__(self, collection_name=None, db_alias=None, - sequence_name=None, value_decorator=None, *args, **kwargs): + def __init__(self, collection_name=None, db_alias=None, sequence_name=None, + value_decorator=None, *args, **kwargs): self.collection_name = collection_name or self.COLLECTION_NAME self.db_alias = db_alias or DEFAULT_CONNECTION_NAME self.sequence_name = sequence_name self.value_decorator = (callable(value_decorator) and - value_decorator or self.VALUE_DECORATOR) + value_decorator or self.VALUE_DECORATOR) return super(SequenceField, self).__init__(*args, **kwargs) def generate(self): @@ -1430,6 +1430,17 @@ class SequenceField(BaseField): upsert=True) return self.value_decorator(counter['next']) + def set_next_value(self, value): + """Helper method to set the next sequence value""" + sequence_name = self.get_sequence_name() + sequence_id = "%s.%s" % (sequence_name, self.name) + collection = get_db(alias=self.db_alias)[self.collection_name] + counter = collection.find_and_modify(query={"_id": sequence_id}, + update={"$set": {"next": value}}, + new=True, + upsert=True) + return self.value_decorator(counter['next']) + def get_sequence_name(self): if self.sequence_name: return self.sequence_name @@ -1438,7 +1449,7 @@ class SequenceField(BaseField): return owner._get_collection_name() else: return ''.join('_%s' % c if c.isupper() else c - for c in owner._class_name).strip('_').lower() + for c in owner._class_name).strip('_').lower() def __get__(self, instance, owner): value = super(SequenceField, self).__get__(instance, owner) diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 9a7b82f..ade44b8 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -2164,18 +2164,21 @@ class FieldTest(unittest.TestCase): Person.drop_collection() for x in xrange(10): - p = Person(name="Person %s" % x) - p.save() + Person(name="Person %s" % x).save() c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) ids = [i.id for i in Person.objects] - self.assertEqual(ids, range(1, 11)) + self.assertEqual(ids, xrange(1, 11)) c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) + Person.id.set_next_value(1000) + c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) + self.assertEqual(c['next'], 1000) + def test_sequence_field_sequence_name(self): class Person(Document): id = SequenceField(primary_key=True, sequence_name='jelly') @@ -2185,8 +2188,7 @@ class FieldTest(unittest.TestCase): Person.drop_collection() for x in xrange(10): - p = Person(name="Person %s" % x) - p.save() + Person(name="Person %s" % x).save() c = self.db['mongoengine.counters'].find_one({'_id': 'jelly.id'}) self.assertEqual(c['next'], 10) @@ -2197,6 +2199,10 @@ class FieldTest(unittest.TestCase): c = self.db['mongoengine.counters'].find_one({'_id': 'jelly.id'}) self.assertEqual(c['next'], 10) + Person.id.set_next_value(1000) + c = self.db['mongoengine.counters'].find_one({'_id': 'jelly.id'}) + self.assertEqual(c['next'], 1000) + def test_multiple_sequence_fields(self): class Person(Document): id = SequenceField(primary_key=True) @@ -2207,21 +2213,28 @@ class FieldTest(unittest.TestCase): Person.drop_collection() for x in xrange(10): - p = Person(name="Person %s" % x) - p.save() + Person(name="Person %s" % x).save() c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) ids = [i.id for i in Person.objects] - self.assertEqual(ids, range(1, 11)) + self.assertEqual(ids, xrange(1, 11)) counters = [i.counter for i in Person.objects] - self.assertEqual(counters, range(1, 11)) + self.assertEqual(counters, xrange(1, 11)) c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) + Person.id.set_next_value(1000) + c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) + self.assertEqual(c['next'], 1000) + + Person.counter.set_next_value(999) + c = self.db['mongoengine.counters'].find_one({'_id': 'person.counter'}) + self.assertEqual(c['next'], 999) + def test_sequence_fields_reload(self): class Animal(Document): counter = SequenceField() @@ -2230,8 +2243,7 @@ class FieldTest(unittest.TestCase): self.db['mongoengine.counters'].drop() Animal.drop_collection() - a = Animal(name="Boi") - a.save() + a = Animal(name="Boi").save() self.assertEqual(a.counter, 1) a.reload() @@ -2261,10 +2273,8 @@ class FieldTest(unittest.TestCase): Person.drop_collection() for x in xrange(10): - a = Animal(name="Animal %s" % x) - a.save() - p = Person(name="Person %s" % x) - p.save() + Animal(name="Animal %s" % x).save() + Person(name="Person %s" % x).save() c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) From 3653981416146bdceb1ff6106c9c3d35d9c7db0e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 15:12:57 +0000 Subject: [PATCH 1042/1279] Added ImageField support for inline replacements (#86) --- docs/changelog.rst | 1 + mongoengine/fields.py | 24 ++++++++++-------------- tests/fields/file_tests.py | 30 ++++++++++++++++++++++++++++++ tests/fields/mongodb_leaf.png | Bin 0 -> 4971 bytes 4 files changed, 41 insertions(+), 14 deletions(-) create mode 100644 tests/fields/mongodb_leaf.png diff --git a/docs/changelog.rst b/docs/changelog.rst index d22fc60..476753d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- ImageFields now support inline replacements (#86) - Added SequenceField.set_next_value(value) helper (#159) - Updated .only() behaviour - now like exclude it is chainable (#202) - Added with_limit_and_skip support to count() (#235) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 4fc65c7..4530429 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1165,13 +1165,11 @@ class FileField(BaseField): grid_file.delete() except: pass - # Create a new file with the new data - grid_file.put(value) - else: - # Create a new proxy object as we don't already have one - instance._data[key] = self.proxy_class(key=key, instance=instance, - collection_name=self.collection_name) - instance._data[key].put(value) + + # Create a new proxy object as we don't already have one + instance._data[key] = self.proxy_class(key=key, instance=instance, + collection_name=self.collection_name) + instance._data[key].put(value) else: instance._data[key] = value @@ -1208,6 +1206,8 @@ class ImageGridFsProxy(GridFSProxy): Insert a image in database applying field properties (size, thumbnail_size) """ + if not self.instance: + import ipdb; ipdb.set_trace(); field = self.instance._fields[self.key] try: @@ -1235,10 +1235,7 @@ class ImageGridFsProxy(GridFSProxy): size = field.thumbnail_size if size['force']: - thumbnail = ImageOps.fit(img, - (size['width'], - size['height']), - Image.ANTIALIAS) + thumbnail = ImageOps.fit(img, (size['width'], size['height']), Image.ANTIALIAS) else: thumbnail = img.copy() thumbnail.thumbnail((size['width'], @@ -1246,8 +1243,7 @@ class ImageGridFsProxy(GridFSProxy): Image.ANTIALIAS) if thumbnail: - thumb_id = self._put_thumbnail(thumbnail, - img_format) + thumb_id = self._put_thumbnail(thumbnail, img_format) else: thumb_id = None @@ -1350,7 +1346,7 @@ class ImageField(FileField): if isinstance(att, (tuple, list)): if PY3: value = dict(itertools.zip_longest(params_size, att, - fillvalue=None)) + fillvalue=None)) else: value = dict(map(None, params_size, att)) diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py index 44d2862..c5842d8 100644 --- a/tests/fields/file_tests.py +++ b/tests/fields/file_tests.py @@ -16,6 +16,7 @@ from mongoengine.connection import get_db from mongoengine.python_support import PY3, b, StringIO TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') +TEST_IMAGE2_PATH = os.path.join(os.path.dirname(__file__), 'mongodb_leaf.png') class FileTest(unittest.TestCase): @@ -217,6 +218,19 @@ class FileTest(unittest.TestCase): self.assertEqual(marmot.photo.content_type, 'image/jpeg') self.assertEqual(marmot.photo.foo, 'bar') + def test_file_reassigning(self): + class TestFile(Document): + the_file = FileField() + TestFile.drop_collection() + + test_file = TestFile(the_file=open(TEST_IMAGE_PATH, 'rb')).save() + self.assertEqual(test_file.the_file.get().length, 8313) + + test_file = TestFile.objects.first() + test_file.the_file = open(TEST_IMAGE2_PATH, 'rb') + test_file.save() + self.assertEqual(test_file.the_file.get().length, 4971) + def test_file_boolean(self): """Ensure that a boolean test of a FileField indicates its presence """ @@ -264,6 +278,22 @@ class FileTest(unittest.TestCase): t.image.delete() + def test_image_field_reassigning(self): + if PY3: + raise SkipTest('PIL does not have Python 3 support') + + class TestFile(Document): + the_file = ImageField() + TestFile.drop_collection() + + test_file = TestFile(the_file=open(TEST_IMAGE_PATH, 'rb')).save() + self.assertEqual(test_file.the_file.size, (371, 76)) + + test_file = TestFile.objects.first() + test_file.the_file = open(TEST_IMAGE2_PATH, 'rb') + test_file.save() + self.assertEqual(test_file.the_file.size, (45, 101)) + def test_image_field_resize(self): if PY3: raise SkipTest('PIL does not have Python 3 support') diff --git a/tests/fields/mongodb_leaf.png b/tests/fields/mongodb_leaf.png new file mode 100644 index 0000000000000000000000000000000000000000..36661cefc824b1707a9a6063d133cefe3566e1a9 GIT binary patch literal 4971 zcmV-x6O`4Tx05}naRo`#hR1`jmZ&IWdKOk5~hl<6oRa0BJ8yc;~21%2p?MfD<>DVeH z9(p*dx19w`~g7O0}n_%Aq@s%d)fBDv`JHkDym6Hd+5XuAtvnwRpGmK zVkc9?T=n|PIo~X-eVh__(Z?q}P9Z-Dj?gOW6|D%o20XmjW-qs4UjrD(li^iv8@eK9k+ZFm zVRFymFOPAzG5-%Pn|1W;U4vNroTa&AxDScmEA~{ri9gr1^c?U@uwSpaNnw8l_>cP1 zd;)kMQS_;jeRSUEM_*s96y65j1$)tOrwdK{YIQMt92l|D^(E_=$Rjw{b!QT@q!)ni zR`|5oW9X5n$Wv+HVc@|^eX5yXnsHX8PF3UX~a6)MwxDE0HaPjyrlI!;jX{6Kvuh*8ej?;85ekN$?5uuCiS zBTvvVG+XTxAO{m@bvM#Jr)z6J><&E22D|vq?Y?Vkbo_DijopiF$2PET#mZ8eu=y$(ArYkv7@Ex`GL?QCc!_*KFrd&;n1r7 zqW-CFs9&fT)ZaU5gc&=gBz-DaCw(vdOp0__x+47~U6sC(E(JNe@4cTT*n6*E zVH4eoU1-&7pEV~_PRe`a7v+@vy!^5}8?Y3)UmlaER00009a7bBm000XU000XU0RWnu7ytkc?MXyIRCoc^ zTx*OTRT(~W`M$liv?9=Q-7ZBg?UscIL7{S~{J>Oftboy=iB=P-L@Ce^2(~VzNNU(d z)MEHiLIP?a0#-wiU<~39ei#f=%GRb7soO493zb{Tx8EF}=RN1lH#6US*PTt^Y-i5x zy*%eV&zv)t-72lMD*CwU@vA&UCcAY(6%|IIH?>+sr&yCzP1E$5~jDW6IrmiiO zN*^82*KGUJ@~YMsmV)rug&T)g&qTBfjr7XLSDqDV^+bg0q+AM=^8C9#)Sy>9wqm)` z>P1iU0*qAP1GG|e7T&dFOVPj!iS(6^T=`uU>E8ovNdvV4`oMRlz3Ya4G;%?az@6o; zZ7%=y@*gTs{|q=m2fzhC5NH*K5^1sPMf#Zqca98o?Ifc(Ezno|>WX#Ri`Fx;S^%R2 z@TiHKhyyUiS-z)VICpz$ z z8eMW9)jCaWgI@Z;#cN>vQobA<&|M0 zRQ}W-^ZFXKwFG_s{TEyh!@~|JZ-!)*=#(Oj5U^dVQ%~J6{L^M4$-btbhqqidOZjTE z0EVQllF@WxV~ng3MTn%`M9D~d*AH%3yto~mrl8B=yPIJ!q#+tk!f=r{ZKgL#5$=fD zJ34kcmH{(0OhxcQ~=6-gaOwUMd;0fav*DKcODdq>(hts zSn~b+-dbxz(3PsU$v8=HxGjo69w_&R|$9Xg> z**tMYzOD36L^jgYCc`V4R6*HkJq1FJ5nZQ&|f1 z8C%XhAHDPFj7dwxk&6d_l?gyEw3=-+GB814?8IjpZgk|V>$p?AF4j#HQlRmYWt9O2 z+hUwRQRspq1{`%0KogGJDtda};OL^uv$84Bi0s#l0BOM=`m}lM9EEl$lJP|K&J(6~ z%DXkI7lS_SzOz3GF#Q4&9-NbIDI&A&jKFIhhqFj*U02~77qC093N6cu#-OVebrrcv z;ub=Y9F&PM;F_oba{!s=jc&oZVXSDi6oW?73L32$HpDP(`4q9l&s{~##Rw)Pqn0e^ z45|hzThghg8v*T&UWWH?>MXfSBVYLgv~B>Th}6Xq`jB(klgF&Y?WzordMt=cZs4+r z>grZqf=-uskWRf;6VRu8?BFFv6XFdhBD(~ELIh_S5RLPQlb|&%Mk4NMSRkn6L4$U| zGQPx#R)gynU*n7jXsxQ3h!gS|J;p6X3_R0jfjR*KbXzv$k{eAP;3P1X&3+9JFq1sk zl}us_3J{AaVs7xYx=`jELzIgGS{m;UF)h+eBQZtnF!AaG{g4lkb2mdy@1~vdX3_+l zaY{3_B*<%3qhadQO(v;p{VCTb5LD!y$2sAMML9tcdS^hTvyF|$7=bNoO6H#BIB|(! zfA+_~7QI+DS@4c^uhV?Ko`U)ub{j`&iqN!i;#?s&O{UO*B2GFBO=c{KC0mYdC;b?3 z)yk2tQrti3s3nUf8W47oE^HOa0!r2s;A~ttPDx}A4^Fr%+x${A;rYa`o+V_h8ARC4_U?p*&m;n^m(UnPtQfM4v2R~4vU?@Q&wCg2S5w^Jyn5pH2^CFZnnm&w( zdX(ef_{LOjUnzrd85<(dQ$GO12d>v`;9q>qy~oTxB5o$aDA96yxzWunF>&6r@#(9QYWU?09GRG!hvJ_#n3?^x} z9-Of%!tp7V;*Cqo7Wlph)3c=)4ge;-`Jc^J?qzbLnEYXFG-4vREGZHP& z;$^Bj*9VB%hFEl0i-^<`tBA2^U~yokWCmg?XyMXf!BNIcN3G4+= zia02UBJN%!0(F8w_#^JxJsEIDTGsfON2ZNct{#Wwt$ku*If%Sfq$NcdR2UV@{e9K2DCd_5! zVrwEIJV%om4BX00V1=-45>S~L^dWb0)FVjIM^3Bma5w&DEj4DA^(%IW&Ro z$$wqMoR0AxM~BTs%3R8^Fo9Zt>`Mw*ibx|n+|da*@smE$#on>K?}vDU^onfbG|1M1 zR*|&3&T0a~s<51h|C?;~y|DVCL`e{{+olrIsR{P^ zlQ-;q)y%Ec65LdG@NM29MI5IckZdjv+=+m)0<6cwFrBzzc#W;RyqNyRaqwlM@RU~@ zlMTE9fIe{5e8Hle9gJNfr7mi`7lAHi#i>{_r(`&jAT!`!&Fq#kq-)+;`aeroZ;v~7Y-WMp(TIa=)` zi8CnVX@lBpdp2#KbRscmHsID~OH6@|C}Lh6$V{2h$Lw{3u6roA3-DYKGK!xX+x*}4gKfEJpCk6W7H|*FC-&hNqQ-si3Z8T=(m4lh!-x1tD*iA4@MUPaOr!S6g#C{Cgh*AfK_PHYND0;O3`?qsnj z-Vg${{Pa_2OZ3Ml z?%w&A6c$wKKvMv4@EhjSqe#`FN(Pd)IY=JyfY_K!_WnfPxlAekT3ubr1wDE5_#OoJ z7j~ZDfo`lrMx7p|38w}W5za%d6}Wg5w8z6LfE+LH8yip0IJ%i~L9?U(o%h$^A@4nw zCAC2jBVs5LZ{ZG1HqRLiMNC3TYy0)swbk;5p!clb!&eL6XtvR0R}q>O$GnK##!~au zvXmId;}iGnitjbDiVZk$ntJ1N^Ke+beK+m+GZv5^z`eBa zOu%xbOA}7CWH%aq2%TH)G4~&J;H=S12FUyOy}om0ra8CT7HDb$_?^J?JDE$=1-Q)x ztSj4mo*IA*e_ADxs)T;!68#8&H=%WE8#GN#p0wkO2;z1z+k_$v?tE`RXi^Jw?WvY~ zsl0@TF>V2Vqa?_;1_58jExUnp(SW*?s5(72y}Oyf*|m+5`jnY|H)Ck{Sp~s@E?W-cvD? z3~~I8`^I)|&UCe`b_|*p=!02UHkpW#+6@0v2(1&kOkgq@Fj_}TX_l)1( z5@C)&SDMTumMUrlv;RhN7102DbhS@zE6<*FJnrVwx2V*`$INw;WBn%L=BxhDtk)86ObH}b?Z0PyLw9hE*0ey= z#NIXA5B7S8&jF~X<-$Vf2l16pzPAAQwS%8V7dfO3dfNXC?kToT2QYQ9@-Rfjq``Yf z?wX@}Ze8`ki1K)E0J(@x=_X%x}jE%YdrR4EL0v#rQ9BcQ`l@OKtz=z1(O0vdo1;sR8qatYDy z>l$1m#Tfxz_QJRDe<1*LkLv6Dh=BIWex(Fzl(8fB^z`*MideVpGZN{Zp3 Date: Tue, 23 Apr 2013 15:32:10 +0000 Subject: [PATCH 1043/1279] Update build assets for PY3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ba538fa..e545ba2 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ if sys.version_info[0] == 3: extra_opts['packages'] = find_packages(exclude=('tests',)) if "test" in sys.argv or "nosetests" in sys.argv: extra_opts['packages'].append("tests") - extra_opts['package_data'] = {"tests": ["fields/mongoengine.png"]} + extra_opts['package_data'] = {"tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"]} else: extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django==1.4.2', 'PIL'] extra_opts['packages'] = find_packages(exclude=('tests',)) From 1e1e48732a92cc5e355301e7178b2337f2e48626 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 15:33:38 +0000 Subject: [PATCH 1044/1279] Update setup.py python versions / langauge in changelog --- docs/changelog.rst | 2 +- setup.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 476753d..49ee0ec 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,7 @@ Changelog Changes in 0.8.X ================ -- ImageFields now support inline replacements (#86) +- Added ImageField support for inline replacements (#86) - Added SequenceField.set_next_value(value) helper (#159) - Updated .only() behaviour - now like exclude it is chainable (#202) - Added with_limit_and_skip support to count() (#235) diff --git a/setup.py b/setup.py index e545ba2..13c11a9 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,6 @@ CLASSIFIERS = [ 'Operating System :: OS Independent', 'Programming Language :: Python', "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.5", "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", From 88f96b08386fcf9607600a79f6d5768e81c65a67 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 15:59:23 +0000 Subject: [PATCH 1045/1279] Ensure all field params are documented (#97) --- docs/apireference.rst | 49 +++++++++++++++++++++++------------------ docs/django.rst | 2 +- docs/guide/querying.rst | 4 ++-- docs/upgrade.rst | 2 +- mongoengine/__init__.py | 4 +++- mongoengine/errors.py | 4 +++- mongoengine/fields.py | 23 ++++++++++--------- 7 files changed, 50 insertions(+), 38 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 049cc30..0040f45 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -54,28 +54,33 @@ Querying Fields ====== -.. autoclass:: mongoengine.BinaryField -.. autoclass:: mongoengine.BooleanField -.. autoclass:: mongoengine.ComplexDateTimeField -.. autoclass:: mongoengine.DateTimeField -.. autoclass:: mongoengine.DecimalField -.. autoclass:: mongoengine.DictField -.. autoclass:: mongoengine.DynamicField -.. autoclass:: mongoengine.EmailField -.. autoclass:: mongoengine.EmbeddedDocumentField -.. autoclass:: mongoengine.FileField -.. autoclass:: mongoengine.FloatField -.. autoclass:: mongoengine.GenericEmbeddedDocumentField -.. autoclass:: mongoengine.GenericReferenceField -.. autoclass:: mongoengine.GeoPointField -.. autoclass:: mongoengine.ImageField -.. autoclass:: mongoengine.IntField -.. autoclass:: mongoengine.ListField -.. autoclass:: mongoengine.MapField -.. autoclass:: mongoengine.ObjectIdField -.. autoclass:: mongoengine.ReferenceField -.. autoclass:: mongoengine.SequenceField -.. autoclass:: mongoengine.SortedListField .. autoclass:: mongoengine.StringField .. autoclass:: mongoengine.URLField +.. autoclass:: mongoengine.EmailField +.. autoclass:: mongoengine.IntField +.. autoclass:: mongoengine.LongField +.. autoclass:: mongoengine.FloatField +.. autoclass:: mongoengine.DecimalField +.. autoclass:: mongoengine.BooleanField +.. autoclass:: mongoengine.DateTimeField +.. autoclass:: mongoengine.ComplexDateTimeField +.. autoclass:: mongoengine.EmbeddedDocumentField +.. autoclass:: mongoengine.GenericEmbeddedDocumentField +.. autoclass:: mongoengine.DynamicField +.. autoclass:: mongoengine.ListField +.. autoclass:: mongoengine.SortedListField +.. autoclass:: mongoengine.DictField +.. autoclass:: mongoengine.MapField +.. autoclass:: mongoengine.ReferenceField +.. autoclass:: mongoengine.GenericReferenceField +.. autoclass:: mongoengine.BinaryField +.. autoclass:: mongoengine.FileField +.. autoclass:: mongoengine.ImageField +.. autoclass:: mongoengine.GeoPointField +.. autoclass:: mongoengine.SequenceField +.. autoclass:: mongoengine.ObjectIdField .. autoclass:: mongoengine.UUIDField +.. autoclass:: mongoengine.GridFSError +.. autoclass:: mongoengine.GridFSProxy +.. autoclass:: mongoengine.ImageGridFsProxy +.. autoclass:: mongoengine.ImproperlyConfigured diff --git a/docs/django.rst b/docs/django.rst index 6f27b90..5c9e7bf 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -10,7 +10,7 @@ In your **settings.py** file, ignore the standard database settings (unless you also plan to use the ORM in your project), and instead call :func:`~mongoengine.connect` somewhere in the settings module. -.. note :: +.. note:: If you are not using another Database backend you may need to add a dummy database backend to ``settings.py`` eg:: diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 3279853..5e250ce 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -450,7 +450,7 @@ modifier comes before the field, not after it:: >>> post.tags ['database', 'nosql'] -.. note :: +.. note:: In version 0.5 the :meth:`~mongoengine.Document.save` runs atomic updates on changed documents by tracking changes to that document. @@ -466,7 +466,7 @@ cannot use the `$` syntax in keyword arguments it has been mapped to `S`:: >>> post.tags ['database', 'mongodb'] -.. note :: +.. note:: Currently only top level lists are handled, future versions of mongodb / pymongo plan to support nested positional operators. See `The $ positional operator `_. diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 5490757..564b7f6 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -205,7 +205,7 @@ via `save` eg :: # Or in code: my_document.save(cascade=True) -.. note :: +.. note:: Remember: cascading saves **do not** cascade through lists. ReferenceFields diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index da72e53..6fe6d08 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -8,10 +8,12 @@ import queryset from queryset import * import signals from signals import * +from errors import * +import errors import django __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + - list(queryset.__all__) + signals.__all__) + list(queryset.__all__) + signals.__all__ + list(errors.__all__)) VERSION = (0, 8, 0, '+') diff --git a/mongoengine/errors.py b/mongoengine/errors.py index 9cfcd1d..4b6b562 100644 --- a/mongoengine/errors.py +++ b/mongoengine/errors.py @@ -3,7 +3,9 @@ from collections import defaultdict from mongoengine.python_support import txt_type -__all__ = ('NotRegistered', 'InvalidDocumentError', 'ValidationError') +__all__ = ('NotRegistered', 'InvalidDocumentError', 'LookUpError', + 'DoesNotExist', 'MultipleObjectsReturned', 'InvalidQueryError', + 'OperationError', 'NotUniqueError', 'ValidationError') class NotRegistered(Exception): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 4530429..e23b90a 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -4,7 +4,6 @@ import itertools import re import time import urllib2 -import urlparse import uuid import warnings from operator import itemgetter @@ -16,7 +15,7 @@ from mongoengine.errors import ValidationError from mongoengine.python_support import (PY3, bin_type, txt_type, str_types, StringIO) from base import (BaseField, ComplexBaseField, ObjectIdField, - get_document, BaseDocument, ALLOW_INHERITANCE) + get_document, BaseDocument) from queryset import DO_NOTHING, QuerySet from document import Document, EmbeddedDocument from connection import get_db, DEFAULT_CONNECTION_NAME @@ -27,13 +26,17 @@ except ImportError: Image = None ImageOps = None -__all__ = ['StringField', 'IntField', 'LongField', 'FloatField', 'BooleanField', - 'DateTimeField', 'EmbeddedDocumentField', 'ListField', 'DictField', - 'ObjectIdField', 'ReferenceField', 'ValidationError', 'MapField', - 'DecimalField', 'ComplexDateTimeField', 'URLField', 'DynamicField', - 'GenericReferenceField', 'FileField', 'BinaryField', - 'SortedListField', 'EmailField', 'GeoPointField', 'ImageField', - 'SequenceField', 'UUIDField', 'GenericEmbeddedDocumentField'] +__all__ = ['StringField', 'URLField', 'EmailField', 'IntField', 'LongField', + 'FloatField', 'DecimalField', 'BooleanField', 'DateTimeField', + 'ComplexDateTimeField', 'EmbeddedDocumentField', 'ObjectIdField', + 'GenericEmbeddedDocumentField', 'DynamicField', 'ListField', + 'SortedListField', 'DictField', 'MapField', 'ReferenceField', + 'GenericReferenceField', 'BinaryField', 'GridFSError', + 'GridFSProxy', 'FileField', 'ImageGridFsProxy', + 'ImproperlyConfigured', 'ImageField', 'GeoPointField', + 'SequenceField', 'UUIDField'] + + RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -351,7 +354,7 @@ class DateTimeField(BaseField): kwargs = {'microsecond': usecs} try: # Seconds are optional, so try converting seconds first. return datetime.datetime(*time.strptime(value, - '%Y-%m-%d %H:%M:%S')[:6], **kwargs) + '%Y-%m-%d %H:%M:%S')[:6], **kwargs) except ValueError: try: # Try without seconds. return datetime.datetime(*time.strptime(value, From 8a7b619b77417bd2fa4e0dbc1fe14a25320018c0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 20:37:05 +0000 Subject: [PATCH 1046/1279] Tutorial updates --- docs/guide/connecting.rst | 11 +++++- docs/index.rst | 2 +- docs/tutorial.rst | 81 +++++++++++++++++++++++++-------------- 3 files changed, 63 insertions(+), 31 deletions(-) diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index de6794c..dea6a3d 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -74,9 +74,13 @@ to point across databases and collections. Below is an example schema, using Switch Database Context Manager =============================== -Sometimes you might want to switch the database to query against for a class. +Sometimes you may want to switch the database to query against for a class, +for example, you archive older data into a separate database for performance +reasons. + The :class:`~mongoengine.context_managers.switch_db` context manager allows -you to change the database alias for a class eg :: +you to change the database alias for a given class allowing quick and easy +access to the same User document across databases.eg :: from mongoengine.context_managers import switch_db @@ -87,3 +91,6 @@ you to change the database alias for a class eg :: with switch_db(User, 'archive-user-db') as User: User(name="Ross").save() # Saves the 'archive-user-db' + +.. note:: Make sure any aliases have been registered with + :func:`~mongoengine.register_connection` before using the context manager. diff --git a/docs/index.rst b/docs/index.rst index f6d44b5..4d0d211 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,7 +7,7 @@ MongoDB. To install it, simply run .. code-block:: console - # pip install -U mongoengine + $ pip install -U mongoengine :doc:`tutorial` Start here for a quick overview. diff --git a/docs/tutorial.rst b/docs/tutorial.rst index c4b69c4..423df9b 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -1,6 +1,7 @@ ======== Tutorial ======== + This tutorial introduces **MongoEngine** by means of example --- we will walk through how to create a simple **Tumblelog** application. A Tumblelog is a type of blog where posts are not constrained to being conventional text-based posts. @@ -12,23 +13,29 @@ interface. Getting started =============== + Before we start, make sure that a copy of MongoDB is running in an accessible location --- running it locally will be easier, but if that is not an option -then it may be run on a remote server. +then it may be run on a remote server. If you haven't installed mongoengine, +simply use pip to install it like so:: + + $ pip install mongoengine Before we can start using MongoEngine, we need to tell it how to connect to our instance of :program:`mongod`. For this we use the :func:`~mongoengine.connect` -function. The only argument we need to provide is the name of the MongoDB -database to use:: +function. If running locally the only argument we need to provide is the name +of the MongoDB database to use:: from mongoengine import * connect('tumblelog') -For more information about connecting to MongoDB see :ref:`guide-connecting`. +There are lots of options for connecting to MongoDB, for more information about +them see the :ref:`guide-connecting` guide. Defining our documents ====================== + MongoDB is *schemaless*, which means that no schema is enforced by the database --- we may add and remove fields however we want and MongoDB won't complain. This makes life a lot easier in many regards, especially when there is a change @@ -39,17 +46,19 @@ define utility methods on our documents in the same way that traditional In our Tumblelog application we need to store several different types of information. We will need to have a collection of **users**, so that we may -link posts to an individual. We also need to store our different types -**posts** (text, image and link) in the database. To aid navigation of our +link posts to an individual. We also need to store our different types of +**posts** (eg: text, image and link) in the database. To aid navigation of our Tumblelog, posts may have **tags** associated with them, so that the list of posts shown to the user may be limited to posts that have been assigned a -specified tag. Finally, it would be nice if **comments** could be added to -posts. We'll start with **users**, as the others are slightly more involved. +specific tag. Finally, it would be nice if **comments** could be added to +posts. We'll start with **users**, as the other document models are slightly +more involved. Users ----- + Just as if we were using a relational database with an ORM, we need to define -which fields a :class:`User` may have, and what their types will be:: +which fields a :class:`User` may have, and what types of data they might store:: class User(Document): email = StringField(required=True) @@ -58,11 +67,13 @@ which fields a :class:`User` may have, and what their types will be:: This looks similar to how a the structure of a table would be defined in a regular ORM. The key difference is that this schema will never be passed on to -MongoDB --- this will only be enforced at the application level. Also, the User -documents will be stored in a MongoDB *collection* rather than a table. +MongoDB --- this will only be enforced at the application level, making future +changes easy to manage. Also, the User documents will be stored in a +MongoDB *collection* rather than a table. Posts, Comments and Tags ------------------------ + Now we'll think about how to store the rest of the information. If we were using a relational database, we would most likely have a table of **posts**, a table of **comments** and a table of **tags**. To associate the comments with @@ -75,16 +86,17 @@ of them stand out as particularly intuitive solutions. Posts ^^^^^ -But MongoDB *isn't* a relational database, so we're not going to do it that + +Happily mongoDB *isn't* a relational database, so we're not going to do it that way. As it turns out, we can use MongoDB's schemaless nature to provide us with -a much nicer solution. We will store all of the posts in *one collection* --- -each post type will just have the fields it needs. If we later want to add +a much nicer solution. We will store all of the posts in *one collection* and +each post type will only store the fields it needs. If we later want to add video posts, we don't have to modify the collection at all, we just *start using* the new fields we need to support video posts. This fits with the Object-Oriented principle of *inheritance* nicely. We can think of :class:`Post` as a base class, and :class:`TextPost`, :class:`ImagePost` and :class:`LinkPost` as subclasses of :class:`Post`. In fact, MongoEngine supports -this kind of modelling out of the box - all you need do is turn on inheritance +this kind of modelling out of the box --- all you need do is turn on inheritance by setting :attr:`allow_inheritance` to True in the :attr:`meta`:: class Post(Document): @@ -109,6 +121,7 @@ when they are saved, and dereferenced when they are loaded. Tags ^^^^ + Now that we have our Post models figured out, how will we attach tags to them? MongoDB allows us to store lists of items natively, so rather than having a link table, we can just store a list of tags in each post. So, for both @@ -126,11 +139,14 @@ size of our database. So let's take a look that the code our modified The :class:`~mongoengine.ListField` object that is used to define a Post's tags takes a field object as its first argument --- this means that you can have -lists of any type of field (including lists). Note that we don't need to -modify the specialised post types as they all inherit from :class:`Post`. +lists of any type of field (including lists). + +.. note:: We don't need to modify the specialised post types as they all + inherit from :class:`Post`. Comments ^^^^^^^^ + A comment is typically associated with *one* post. In a relational database, to display a post with its comments, we would have to retrieve the post from the database, then query the database again for the comments associated with the @@ -181,15 +197,15 @@ Now that we've defined how our documents will be structured, let's start adding some documents to the database. Firstly, we'll need to create a :class:`User` object:: - john = User(email='jdoe@example.com', first_name='John', last_name='Doe') - john.save() + ross = User(email='ross@example.com', first_name='Ross', last_name='Lawley').save() -Note that we could have also defined our user using attribute syntax:: +.. note:: + We could have also defined our user using attribute syntax:: - john = User(email='jdoe@example.com') - john.first_name = 'John' - john.last_name = 'Doe' - john.save() + ross = User(email='ross@example.com') + ross.first_name = 'Ross' + ross.last_name = 'Lawley' + ross.save() Now that we've got our user in the database, let's add a couple of posts:: @@ -198,16 +214,17 @@ Now that we've got our user in the database, let's add a couple of posts:: post1.tags = ['mongodb', 'mongoengine'] post1.save() - post2 = LinkPost(title='MongoEngine Documentation', author=john) - post2.link_url = 'http://tractiondigital.com/labs/mongoengine/docs' + post2 = LinkPost(title='MongoEngine Documentation', author=ross) + post2.link_url = 'http://docs.mongoengine.com/' post2.tags = ['mongoengine'] post2.save() -Note that if you change a field on a object that has already been saved, then -call :meth:`save` again, the document will be updated. +.. note:: If you change a field on a object that has already been saved, then + call :meth:`save` again, the document will be updated. Accessing our data ================== + So now we've got a couple of posts in our database, how do we display them? Each document class (i.e. any class that inherits either directly or indirectly from :class:`~mongoengine.Document`) has an :attr:`objects` attribute, which is @@ -219,6 +236,7 @@ class. So let's see how we can get our posts' titles:: Retrieving type-specific information ------------------------------------ + This will print the titles of our posts, one on each line. But What if we want to access the type-specific data (link_url, content, etc.)? One way is simply to use the :attr:`objects` attribute of a subclass of :class:`Post`:: @@ -257,6 +275,7 @@ text post, and "Link: " if it was a link post. Searching our posts by tag -------------------------- + The :attr:`objects` attribute of a :class:`~mongoengine.Document` is actually a :class:`~mongoengine.queryset.QuerySet` object. This lazily queries the database only when you need the data. It may also be filtered to narrow down @@ -275,3 +294,9 @@ used on :class:`~mongoengine.queryset.QuerySet` objects:: num_posts = Post.objects(tags='mongodb').count() print 'Found %d posts with tag "mongodb"' % num_posts +Learning more about mongoengine +------------------------------- + +If you got this far you've made a great start, so well done! The next step on +your mongoengine journey is the `full user guide `_, where you +can learn indepth about how to use mongoengine and mongodb. \ No newline at end of file From 5271f3b4a0a75753d6e984267b177b22f67c3b5e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 20:49:22 +0000 Subject: [PATCH 1047/1279] More doc updates --- docs/guide/installing.rst | 4 ++-- docs/index.rst | 45 ++++++++++++++++++++++----------------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/docs/guide/installing.rst b/docs/guide/installing.rst index f15d3db..e93f048 100644 --- a/docs/guide/installing.rst +++ b/docs/guide/installing.rst @@ -22,10 +22,10 @@ Alternatively, if you don't have setuptools installed, `download it from PyPi $ python setup.py install To use the bleeding-edge version of MongoEngine, you can get the source from -`GitHub `_ and install it as above: +`GitHub `_ and install it as above: .. code-block:: console - $ git clone git://github.com/hmarr/mongoengine + $ git clone git://github.com/mongoengine/mongoengine $ cd mongoengine $ python setup.py install diff --git a/docs/index.rst b/docs/index.rst index 4d0d211..4aca82d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,13 +10,15 @@ MongoDB. To install it, simply run $ pip install -U mongoengine :doc:`tutorial` - Start here for a quick overview. + A quick tutorial building a tumblelog to get you up and running with + MongoEngine. :doc:`guide/index` - The Full guide to MongoEngine + The Full guide to MongoEngine - from modeling documents to storing files, + from querying for data to firing signals and *everything* between. :doc:`apireference` - The complete API documentation. + The complete API documentation --- the innards of documents, querysets and fields. :doc:`upgrade` How to upgrade MongoEngine. @@ -28,35 +30,40 @@ Community --------- To get help with using MongoEngine, use the `MongoEngine Users mailing list -`_ or come chat on the -`#mongoengine IRC channel `_. +`_ or the ever popular +`stackoverflow `_. Contributing ------------ -The source is available on `GitHub `_ and -contributions are always encouraged. Contributions can be as simple as -minor tweaks to this documentation. To contribute, fork the project on +**Yes please!** We are always looking for contributions, additions and improvements. + +The source is available on `GitHub `_ +and contributions are always encouraged. Contributions can be as simple as +minor tweaks to this documentation, the website or the core. + +To contribute, fork the project on `GitHub `_ and send a pull request. -Also, you can join the developers' `mailing list -`_. - Changes ------- + See the :doc:`changelog` for a full list of changes to MongoEngine and :doc:`upgrade` for upgrade information. -.. toctree:: - :hidden: +.. note:: Always read and test the `upgrade `_ documentation before + putting updates live in production **;)** - tutorial - guide/index - apireference - django - changelog - upgrade +.. toctree:: + :hidden: + + tutorial + guide/index + apireference + django + changelog + upgrade Indices and tables ------------------ From 9bd8b3e9a536f07020bfeb36624ac844c253f6e5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Apr 2013 20:55:37 +0000 Subject: [PATCH 1048/1279] Connection doc updates --- docs/guide/connecting.rst | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index dea6a3d..8674b5e 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -6,20 +6,23 @@ Connecting to MongoDB To connect to a running instance of :program:`mongod`, use the :func:`~mongoengine.connect` function. The first argument is the name of the -database to connect to. If the database does not exist, it will be created. If -the database requires authentication, :attr:`username` and :attr:`password` -arguments may be provided:: +database to connect to:: from mongoengine import connect - connect('project1', username='webapp', password='pwd123') + connect('project1') By default, MongoEngine assumes that the :program:`mongod` instance is running -on **localhost** on port **27017**. If MongoDB is running elsewhere, you may -provide :attr:`host` and :attr:`port` arguments to +on **localhost** on port **27017**. If MongoDB is running elsewhere, you should +provide the :attr:`host` and :attr:`port` arguments to :func:`~mongoengine.connect`:: connect('project1', host='192.168.1.35', port=12345) +If the database requires authentication, :attr:`username` and :attr:`password` +arguments should be provided:: + + connect('project1', username='webapp', password='pwd123') + Uri style connections are also supported as long as you include the database name - just supply the uri as the :attr:`host` to :func:`~mongoengine.connect`:: @@ -74,8 +77,8 @@ to point across databases and collections. Below is an example schema, using Switch Database Context Manager =============================== -Sometimes you may want to switch the database to query against for a class, -for example, you archive older data into a separate database for performance +Sometimes you may want to switch the database to query against for a class +for example, archiving older data into a separate database for performance reasons. The :class:`~mongoengine.context_managers.switch_db` context manager allows From c59ea268451246a8693c4dd0987ec9bd39cdcd96 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 24 Apr 2013 11:17:21 +0000 Subject: [PATCH 1049/1279] Updated docs to clarify or != | (##288) --- docs/guide/querying.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 5e250ce..60702ec 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -392,6 +392,7 @@ You can also turn off all dereferencing for a fixed period by using the Advanced queries ================ + Sometimes calling a :class:`~mongoengine.queryset.QuerySet` object with keyword arguments can't fully express the query you want to use -- for example if you need to combine a number of constraints using *and* and *or*. This is made @@ -410,6 +411,11 @@ calling it with keyword arguments:: # Get top posts Post.objects((Q(featured=True) & Q(hits__gte=1000)) | Q(hits__gte=5000)) +.. warning:: You have to use bitwise operators. You cannot use ``or``, ``and`` + to combine queries as ``Q(a=a) or Q(b=b)`` is not the same as + ``Q(a=a) | Q(b=b)``. As ``Q(a=a)`` equates to true ``Q(a=a) or Q(b=b)`` is + the same as ``Q(a=a)``. + .. _guide-atomic-updates: Atomic updates From c60ea4082807a35412e936d5cd6049cedae9e164 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 24 Apr 2013 12:14:34 +0000 Subject: [PATCH 1050/1279] ReferenceField now store ObjectId's by default rather than DBRef (#290) --- docs/changelog.rst | 1 + docs/upgrade.rst | 29 +++++++++++ mongoengine/fields.py | 9 +--- tests/all_warnings/__init__.py | 22 -------- tests/migration/__init__.py | 4 +- ...py => convert_to_new_inheritance_model.py} | 2 +- .../refrencefield_dbref_to_object_id.py | 52 +++++++++++++++++++ tests/test_signals.py | 5 +- 8 files changed, 92 insertions(+), 32 deletions(-) rename tests/migration/{test_convert_to_new_inheritance_model.py => convert_to_new_inheritance_model.py} (97%) create mode 100644 tests/migration/refrencefield_dbref_to_object_id.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 49ee0ec..7b51a79 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- ReferenceField now store ObjectId's by default rather than DBRef (#290) - Added ImageField support for inline replacements (#86) - Added SequenceField.set_next_value(value) helper (#159) - Updated .only() behaviour - now like exclude it is chainable (#202) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 564b7f6..eec9d62 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -76,6 +76,35 @@ the case and the data is set only in the ``document._data`` dictionary: :: File "", line 1, in AttributeError: 'Animal' object has no attribute 'size' +ReferenceField +-------------- + +ReferenceFields now store ObjectId's by default - this is more efficient than +DBRefs as we already know what Document types they reference. + + # Old code + class Animal(Document): + name = ReferenceField('self') + + # New code to keep dbrefs + class Animal(Document): + name = ReferenceField('self', dbref=True) + +To migrate all the references you need to touch each object and mark it as dirty +eg:: + + # Doc definition + class Person(Document): + name = StringField() + parent = ReferenceField('self') + friends = ListField(ReferenceField('self')) + + # Mark all ReferenceFields as dirty and save + for p in Person.objects: + p._mark_as_dirty('parent') + p._mark_as_dirty('friends') + p.save() + Querysets ========= diff --git a/mongoengine/fields.py b/mongoengine/fields.py index e23b90a..979699c 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -772,7 +772,7 @@ class ReferenceField(BaseField): .. versionchanged:: 0.5 added `reverse_delete_rule` """ - def __init__(self, document_type, dbref=None, + def __init__(self, document_type, dbref=False, reverse_delete_rule=DO_NOTHING, **kwargs): """Initialises the Reference Field. @@ -786,12 +786,7 @@ class ReferenceField(BaseField): self.error('Argument to ReferenceField constructor must be a ' 'document class or a string') - if dbref is None: - msg = ("ReferenceFields will default to using ObjectId " - "in 0.8, set DBRef=True if this isn't desired") - warnings.warn(msg, FutureWarning) - - self.dbref = dbref if dbref is not None else True # To change in 0.8 + self.dbref = dbref self.document_type_obj = document_type self.reverse_delete_rule = reverse_delete_rule super(ReferenceField, self).__init__(**kwargs) diff --git a/tests/all_warnings/__init__.py b/tests/all_warnings/__init__.py index 74533de..d74d39e 100644 --- a/tests/all_warnings/__init__.py +++ b/tests/all_warnings/__init__.py @@ -30,28 +30,6 @@ class AllWarnings(unittest.TestCase): # restore default handling of warnings warnings.showwarning = self.showwarning_default - def test_dbref_reference_field_future_warning(self): - - class Person(Document): - name = StringField() - parent = ReferenceField('self') - - Person.drop_collection() - - p1 = Person() - p1.parent = None - p1.save() - - p2 = Person(name="Wilson Jr") - p2.parent = p1 - p2.save(cascade=False) - - self.assertTrue(len(self.warning_list) > 0) - warning = self.warning_list[0] - self.assertEqual(FutureWarning, warning["category"]) - self.assertTrue("ReferenceFields will default to using ObjectId" - in str(warning["message"])) - def test_document_save_cascade_future_warning(self): class Person(Document): diff --git a/tests/migration/__init__.py b/tests/migration/__init__.py index 882e737..f7ad674 100644 --- a/tests/migration/__init__.py +++ b/tests/migration/__init__.py @@ -1,4 +1,6 @@ +from convert_to_new_inheritance_model import * +from refrencefield_dbref_to_object_id import * from turn_off_inheritance import * if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/migration/test_convert_to_new_inheritance_model.py b/tests/migration/convert_to_new_inheritance_model.py similarity index 97% rename from tests/migration/test_convert_to_new_inheritance_model.py rename to tests/migration/convert_to_new_inheritance_model.py index d4337bf..89ee9e9 100644 --- a/tests/migration/test_convert_to_new_inheritance_model.py +++ b/tests/migration/convert_to_new_inheritance_model.py @@ -38,7 +38,7 @@ class ConvertToNewInheritanceModel(unittest.TestCase): # 3. Confirm extra data is removed count = collection.find({'_types': {"$exists": True}}).count() - assert count == 0 + self.assertEqual(0, count) # 4. Remove indexes info = collection.index_information() diff --git a/tests/migration/refrencefield_dbref_to_object_id.py b/tests/migration/refrencefield_dbref_to_object_id.py new file mode 100644 index 0000000..d3acbe9 --- /dev/null +++ b/tests/migration/refrencefield_dbref_to_object_id.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +import unittest + +from mongoengine import Document, connect +from mongoengine.connection import get_db +from mongoengine.fields import StringField, ReferenceField, ListField + +__all__ = ('ConvertToObjectIdsModel', ) + + +class ConvertToObjectIdsModel(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + def test_how_to_convert_to_object_id_reference_fields(self): + """Demonstrates migrating from 0.7 to 0.8 + """ + + # 1. Old definition - using dbrefs + class Person(Document): + name = StringField() + parent = ReferenceField('self', dbref=True) + friends = ListField(ReferenceField('self', dbref=True)) + + Person.drop_collection() + + p1 = Person(name="Wilson", parent=None).save() + f1 = Person(name="John", parent=None).save() + f2 = Person(name="Paul", parent=None).save() + f3 = Person(name="George", parent=None).save() + f4 = Person(name="Ringo", parent=None).save() + Person(name="Wilson Jr", parent=p1, friends=[f1, f2, f3, f4]).save() + + # 2. Start the migration by changing the schema + # Change ReferenceField as now dbref defaults to False + class Person(Document): + name = StringField() + parent = ReferenceField('self') + friends = ListField(ReferenceField('self')) + + # 3. Loop all the objects and mark parent as changed + for p in Person.objects: + p._mark_as_changed('parent') + p._mark_as_changed('friends') + p.save() + + # 4. Confirmation of the fix! + wilson = Person.objects(name="Wilson Jr").as_pymongo()[0] + self.assertEqual(p1.id, wilson['parent']) + self.assertEqual([f1.id, f2.id, f3.id, f4.id], wilson['friends']) diff --git a/tests/test_signals.py b/tests/test_signals.py index fc638cf..32517dd 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -72,6 +72,7 @@ class SignalTests(unittest.TestCase): else: signal_output.append('Not loaded') self.Author = Author + Author.drop_collection() class Another(Document): name = StringField() @@ -110,6 +111,7 @@ class SignalTests(unittest.TestCase): signal_output.append('post_delete Another signal, %s' % document) self.Another = Another + Another.drop_collection() class ExplicitId(Document): id = IntField(primary_key=True) @@ -123,7 +125,8 @@ class SignalTests(unittest.TestCase): signal_output.append('Is updated') self.ExplicitId = ExplicitId - self.ExplicitId.objects.delete() + ExplicitId.drop_collection() + # Save up the number of connected signals so that we can check at the # end that all the signals we register get properly unregistered self.pre_signals = ( From fe62c3aacb0f9a3e06f96f758bf1e3496f753d7d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 10:24:33 +0000 Subject: [PATCH 1051/1279] Cascading saves now default to off (#291) --- docs/changelog.rst | 1 + docs/upgrade.rst | 31 ++++++++++++++++++++++++++++--- mongoengine/document.py | 9 ++------- tests/all_warnings/__init__.py | 30 +----------------------------- tests/document/instance.py | 27 ++++++++++++++++++++++++++- 5 files changed, 58 insertions(+), 40 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7b51a79..6c19933 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Cascading saves now default to off (#291) - ReferenceField now store ObjectId's by default rather than DBRef (#290) - Added ImageField support for inline replacements (#86) - Added SequenceField.set_next_value(value) helper (#159) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index eec9d62..86d9f9d 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -5,11 +5,21 @@ Upgrading 0.7 to 0.8 ********** -Inheritance -=========== +There have been numerous backwards breaking changes in 0.8. The reasons for +these are ensure that MongoEngine has sane defaults going forward and +performs the best it can out the box. Where possible there have been +FutureWarnings to help get you ready for the change, but that hasn't been +possible for the whole of the release. + +.. warning:: Breaking changes - test upgrading on a test system before putting +live. There maybe multiple manual steps in migrating and these are best honed +on a staging / test system. Data Model ----------- +========== + +Inheritance +----------- The inheritance model has changed, we no longer need to store an array of :attr:`types` with the model we can just use the classname in :attr:`_cls`. @@ -105,6 +115,21 @@ eg:: p._mark_as_dirty('friends') p.save() + +Cascading Saves +--------------- +To improve performance document saves will no longer automatically cascade. +Any changes to a Documents references will either have to be saved manually or +you will have to explicitly tell it to cascade on save:: + + # At the class level: + class Person(Document): + meta = {'cascade': True} + + # Or on save: + my_document.save(cascade=True) + + Querysets ========= diff --git a/mongoengine/document.py b/mongoengine/document.py index 54b55df..d0cafa3 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -244,7 +244,6 @@ class Document(BaseDocument): upsert=upsert, **write_concern) created = is_new_object(last_error) - warn_cascade = not cascade and 'cascade' not in self._meta cascade = (self._meta.get('cascade', True) if cascade is None else cascade) if cascade: @@ -257,7 +256,7 @@ class Document(BaseDocument): if cascade_kwargs: # Allow granular control over cascades kwargs.update(cascade_kwargs) kwargs['_refs'] = _refs - self.cascade_save(warn_cascade=warn_cascade, **kwargs) + self.cascade_save(**kwargs) except pymongo.errors.OperationFailure, err: message = 'Could not save document (%s)' @@ -276,7 +275,7 @@ class Document(BaseDocument): signals.post_save.send(self.__class__, document=self, created=created) return self - def cascade_save(self, warn_cascade=None, *args, **kwargs): + def cascade_save(self, *args, **kwargs): """Recursively saves any references / generic references on an objects""" import fields @@ -296,10 +295,6 @@ class Document(BaseDocument): ref_id = "%s,%s" % (ref.__class__.__name__, str(ref._data)) if ref and ref_id not in _refs: - if warn_cascade: - msg = ("Cascading saves will default to off in 0.8, " - "please explicitly set `.save(cascade=True)`") - warnings.warn(msg, FutureWarning) _refs.append(ref_id) kwargs["_refs"] = _refs ref.save(**kwargs) diff --git a/tests/all_warnings/__init__.py b/tests/all_warnings/__init__.py index d74d39e..53ce638 100644 --- a/tests/all_warnings/__init__.py +++ b/tests/all_warnings/__init__.py @@ -17,7 +17,7 @@ __all__ = ('AllWarnings', ) class AllWarnings(unittest.TestCase): def setUp(self): - conn = connect(db='mongoenginetest') + connect(db='mongoenginetest') self.warning_list = [] self.showwarning_default = warnings.showwarning warnings.showwarning = self.append_to_warning_list @@ -30,31 +30,6 @@ class AllWarnings(unittest.TestCase): # restore default handling of warnings warnings.showwarning = self.showwarning_default - def test_document_save_cascade_future_warning(self): - - class Person(Document): - name = StringField() - parent = ReferenceField('self') - - Person.drop_collection() - - p1 = Person(name="Wilson Snr") - p1.parent = None - p1.save() - - p2 = Person(name="Wilson Jr") - p2.parent = p1 - p2.parent.name = "Poppa Wilson" - p2.save() - - self.assertTrue(len(self.warning_list) > 0) - if len(self.warning_list) > 1: - print self.warning_list - warning = self.warning_list[0] - self.assertEqual(FutureWarning, warning["category"]) - self.assertTrue("Cascading saves will default to off in 0.8" - in str(warning["message"])) - def test_document_collection_syntax_warning(self): class NonAbstractBase(Document): @@ -67,6 +42,3 @@ class AllWarnings(unittest.TestCase): self.assertEqual(SyntaxWarning, warning["category"]) self.assertEqual('non_abstract_base', InheritedDocumentFailTest._get_collection_name()) - -import sys -sys.path[0:0] = [""] diff --git a/tests/document/instance.py b/tests/document/instance.py index 5513ed8..a75cc4d 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -678,7 +678,7 @@ class InstanceTest(unittest.TestCase): p1.reload() self.assertEqual(p1.name, p.parent.name) - def test_save_cascade_meta(self): + def test_save_cascade_meta_false(self): class Person(Document): name = StringField() @@ -707,6 +707,31 @@ class InstanceTest(unittest.TestCase): p1.reload() self.assertEqual(p1.name, p.parent.name) + def test_save_cascade_meta_true(self): + + class Person(Document): + name = StringField() + parent = ReferenceField('self') + + meta = {'cascade': False} + + Person.drop_collection() + + p1 = Person(name="Wilson Snr") + p1.parent = None + p1.save() + + p2 = Person(name="Wilson Jr") + p2.parent = p1 + p2.save(cascade=True) + + p = Person.objects(name="Wilson Jr").get() + p.parent.name = "Daddy Wilson" + p.save() + + p1.reload() + self.assertNotEqual(p1.name, p.parent.name) + def test_save_cascades_generically(self): class Person(Document): From cb9166aba41b045d6868bb381d75517b05db8758 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 11:04:33 +0000 Subject: [PATCH 1052/1279] Auth cleanups - removed duplicates --- mongoengine/django/auth.py | 162 ++++++++++++++++--------------------- 1 file changed, 68 insertions(+), 94 deletions(-) diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index d22f086..6582244 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -1,8 +1,7 @@ from mongoengine import * from django.utils.encoding import smart_str -from django.contrib.auth.models import _user_get_all_permissions -from django.contrib.auth.models import _user_has_perm +from django.contrib.auth.models import _user_has_perm, _user_get_all_permissions, _user_has_module_perms from django.db import models from django.contrib.contenttypes.models import ContentTypeManager from django.contrib import auth @@ -38,11 +37,12 @@ from .utils import datetime_now REDIRECT_FIELD_NAME = 'next' + class ContentType(Document): name = StringField(max_length=100) app_label = StringField(max_length=100) model = StringField(max_length=100, verbose_name=_('python model class name'), - unique_with='app_label') + unique_with='app_label') objects = ContentTypeManager() class Meta: @@ -72,9 +72,11 @@ class ContentType(Document): def natural_key(self): return (self.app_label, self.model) + class SiteProfileNotAvailable(Exception): pass + class PermissionManager(models.Manager): def get_by_natural_key(self, codename, app_label, model): return self.get( @@ -82,18 +84,28 @@ class PermissionManager(models.Manager): content_type=ContentType.objects.get_by_natural_key(app_label, model) ) + class Permission(Document): - """The permissions system provides a way to assign permissions to specific users and groups of users. + """The permissions system provides a way to assign permissions to specific + users and groups of users. - The permission system is used by the Django admin site, but may also be useful in your own code. The Django admin site uses permissions as follows: + The permission system is used by the Django admin site, but may also be + useful in your own code. The Django admin site uses permissions as follows: - - The "add" permission limits the user's ability to view the "add" form and add an object. - - The "change" permission limits a user's ability to view the change list, view the "change" form and change an object. + - The "add" permission limits the user's ability to view the "add" + form and add an object. + - The "change" permission limits a user's ability to view the change + list, view the "change" form and change an object. - The "delete" permission limits the ability to delete an object. - Permissions are set globally per type of object, not per specific object instance. It is possible to say "Mary may change news stories," but it's not currently possible to say "Mary may change news stories, but only the ones she created herself" or "Mary may only change news stories that have a certain status or publication date." + Permissions are set globally per type of object, not per specific object + instance. It is possible to say "Mary may change news stories," but it's + not currently possible to say "Mary may change news stories, but only the + ones she created herself" or "Mary may only change news stories that have + a certain status or publication date." - Three basic permissions -- add, change and delete -- are automatically created for each Django model. + Three basic permissions -- add, change and delete -- are automatically + created for each Django model. """ name = StringField(max_length=50, verbose_name=_('username')) content_type = ReferenceField(ContentType) @@ -119,12 +131,22 @@ class Permission(Document): return (self.codename,) + self.content_type.natural_key() natural_key.dependencies = ['contenttypes.contenttype'] + class Group(Document): - """Groups are a generic way of categorizing users to apply permissions, or some other label, to those users. A user can belong to any number of groups. + """Groups are a generic way of categorizing users to apply permissions, + or some other label, to those users. A user can belong to any number of + groups. - A user in a group automatically has all the permissions granted to that group. For example, if the group Site editors has the permission can_edit_home_page, any user in that group will have that permission. + A user in a group automatically has all the permissions granted to that + group. For example, if the group Site editors has the permission + can_edit_home_page, any user in that group will have that permission. - Beyond permissions, groups are a convenient way to categorize users to apply some label, or extended functionality, to them. For example, you could create a group 'Special users', and you could write code that would do special things to those users -- such as giving them access to a members-only portion of your site, or sending them members-only e-mail messages. + Beyond permissions, groups are a convenient way to categorize users to + apply some label, or extended functionality, to them. For example, you + could create a group 'Special users', and you could write code that would + do special things to those users -- such as giving them access to a + members-only portion of your site, or sending them members-only + e-mail messages. """ name = StringField(max_length=80, unique=True, verbose_name=_('name')) # permissions = models.ManyToManyField(Permission, verbose_name=_('permissions'), blank=True) @@ -137,6 +159,7 @@ class Group(Document): def __unicode__(self): return self.name + class UserManager(models.Manager): def create_user(self, username, email, password=None): """ @@ -154,8 +177,8 @@ class UserManager(models.Manager): email = '@'.join([email_name, domain_part.lower()]) user = self.model(username=username, email=email, is_staff=False, - is_active=True, is_superuser=False, last_login=now, - date_joined=now) + is_active=True, is_superuser=False, last_login=now, + date_joined=now) user.set_password(password) user.save(using=self._db) @@ -177,7 +200,6 @@ class UserManager(models.Manager): return ''.join([choice(allowed_chars) for i in range(length)]) - class User(Document): """A User document that aims to mirror most of the API specified by Django at http://docs.djangoproject.com/en/dev/topics/auth/#users @@ -248,25 +270,6 @@ class User(Document): """ return check_password(raw_password, self.password) - def get_all_permissions(self, obj=None): - return _user_get_all_permissions(self, obj) - - def has_perm(self, perm, obj=None): - """ - Returns True if the user has the specified permission. This method - queries all available auth backends, but returns immediately if any - backend returns True. Thus, a user who has permission from a single - auth backend is assumed to have permission in general. If an object is - provided, permissions for this specific object are checked. - """ - - # Active superusers have all permissions. - if self.is_active and self.is_superuser: - return True - - # Otherwise we need to check the backends. - return _user_has_perm(self, perm, obj) - @classmethod def create_user(cls, username, password, email=None): """Create (and save) a new user with the given username, password and @@ -289,68 +292,47 @@ class User(Document): user.save() return user - def get_all_permissions(self, obj=None): + def get_group_permissions(self, obj=None): + """ + Returns a list of permission strings that this user has through his/her + groups. This method queries all available auth backends. If an object + is passed in, only permissions matching this object are returned. + """ permissions = set() - anon = self.is_anonymous() for backend in auth.get_backends(): - if not anon or backend.supports_anonymous_user: - if hasattr(backend, "get_all_permissions"): - if obj is not None: - if backend.supports_object_permissions: - permissions.update( - backend.get_all_permissions(user, obj) - ) - else: - permissions.update(backend.get_all_permissions(self)) + if hasattr(backend, "get_group_permissions"): + permissions.update(backend.get_group_permissions(self, obj)) return permissions - def get_and_delete_messages(self): - return [] + def get_all_permissions(self, obj=None): + return _user_get_all_permissions(self, obj) def has_perm(self, perm, obj=None): - anon = self.is_anonymous() - active = self.is_active - for backend in auth.get_backends(): - if (not active and not anon and backend.supports_inactive_user) or \ - (not anon or backend.supports_anonymous_user): - if hasattr(backend, "has_perm"): - if obj is not None: - if (backend.supports_object_permissions and - backend.has_perm(self, perm, obj)): - return True - else: - if backend.has_perm(self, perm): - return True - return False + """ + Returns True if the user has the specified permission. This method + queries all available auth backends, but returns immediately if any + backend returns True. Thus, a user who has permission from a single + auth backend is assumed to have permission in general. If an object is + provided, permissions for this specific object are checked. + """ - def has_perms(self, perm_list, obj=None): - """ - Returns True if the user has each of the specified permissions. - If object is passed, it checks if the user has all required perms - for this object. - """ - for perm in perm_list: - if not self.has_perm(perm, obj): - return False - return True + # Active superusers have all permissions. + if self.is_active and self.is_superuser: + return True + + # Otherwise we need to check the backends. + return _user_has_perm(self, perm, obj) def has_module_perms(self, app_label): - anon = self.is_anonymous() - active = self.is_active - for backend in auth.get_backends(): - if (not active and not anon and backend.supports_inactive_user) or \ - (not anon or backend.supports_anonymous_user): - if hasattr(backend, "has_module_perms"): - if backend.has_module_perms(self, app_label): - return True - return False + """ + Returns True if the user has any permissions in the given app label. + Uses pretty much the same logic as has_perm, above. + """ + # Active superusers have all permissions. + if self.is_active and self.is_superuser: + return True - def get_and_delete_messages(self): - messages = [] - for m in self.message_set.all(): - messages.append(m.message) - m.delete() - return messages + return _user_has_module_perms(self, app_label) def email_user(self, subject, message, from_email=None): "Sends an e-mail to this User." @@ -386,14 +368,6 @@ class User(Document): raise SiteProfileNotAvailable return self._profile_cache - def _get_message_set(self): - import warnings - warnings.warn('The user messaging API is deprecated. Please update' - ' your code to use the new messages framework.', - category=DeprecationWarning) - return self._message_set - message_set = property(_get_message_set) - class MongoEngineBackend(object): """Authenticate using MongoEngine and mongoengine.django.auth.User. From df4dc3492cf9cf8613eb13f8fedf78224fa70a37 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 11:41:01 +0000 Subject: [PATCH 1053/1279] Upgrade changelog, docs and django/auth.py --- AUTHORS | 1 - docs/changelog.rst | 1 + docs/upgrade.rst | 6 +++--- mongoengine/django/auth.py | 5 ++--- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/AUTHORS b/AUTHORS index e388a04..44e19bf 100644 --- a/AUTHORS +++ b/AUTHORS @@ -157,4 +157,3 @@ that much better: * Kenneth Falck * Lukasz Balcerzak * Nicolas Cortot - diff --git a/docs/changelog.rst b/docs/changelog.rst index 6c19933..bd3821f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Added Custom User Model for Django 1.5 (#285) - Cascading saves now default to off (#291) - ReferenceField now store ObjectId's by default rather than DBRef (#290) - Added ImageField support for inline replacements (#86) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 86d9f9d..738a949 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -12,8 +12,8 @@ FutureWarnings to help get you ready for the change, but that hasn't been possible for the whole of the release. .. warning:: Breaking changes - test upgrading on a test system before putting -live. There maybe multiple manual steps in migrating and these are best honed -on a staging / test system. + live. There maybe multiple manual steps in migrating and these are best honed + on a staging / test system. Data Model ========== @@ -90,7 +90,7 @@ ReferenceField -------------- ReferenceFields now store ObjectId's by default - this is more efficient than -DBRefs as we already know what Document types they reference. +DBRefs as we already know what Document types they reference:: # Old code class Animal(Document): diff --git a/mongoengine/django/auth.py b/mongoengine/django/auth.py index 8fcbca9..cff4b74 100644 --- a/mongoengine/django/auth.py +++ b/mongoengine/django/auth.py @@ -108,7 +108,7 @@ class Permission(Document): created for each Django model. """ name = StringField(max_length=50, verbose_name=_('username')) - content_type = ReferenceField(ContentType, dbref=True) + content_type = ReferenceField(ContentType) codename = StringField(max_length=100, verbose_name=_('codename')) # FIXME: don't access field of the other class # unique_with=['content_type__app_label', 'content_type__model']) @@ -149,8 +149,7 @@ class Group(Document): e-mail messages. """ name = StringField(max_length=80, unique=True, verbose_name=_('name')) - # permissions = models.ManyToManyField(Permission, verbose_name=_('permissions'), blank=True) - permissions = ListField(ReferenceField(Permission, verbose_name=_('permissions'), required=False, dbref=True)) + permissions = ListField(ReferenceField(Permission, verbose_name=_('permissions'), required=False)) class Meta: verbose_name = _('group') From 3fc5dc852335317ae024cac81ae448d985ef9764 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 11:46:18 +0000 Subject: [PATCH 1054/1279] Testing if travis 2.6 is >= 2.6.6 --- .travis.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index c5fe62e..e78bda5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,12 +12,6 @@ env: - PYMONGO=2.5 DJANGO=1.5.1 - PYMONGO=2.5 DJANGO=1.4.2 - PYMONGO=2.4.2 DJANGO=1.4.2 -matrix: - exclude: - - python: "2.6" - env: PYMONGO=dev DJANGO=1.5.1 - - python: "2.6" - env: PYMONGO=2.5 DJANGO=1.5.1 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi From bafdf0381adbfa2f9a626d5d0ad4720366af3a03 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 11:59:56 +0000 Subject: [PATCH 1055/1279] Updates --- .travis.yml | 6 ++++++ docs/upgrade.rst | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/.travis.yml b/.travis.yml index e78bda5..c5fe62e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,12 @@ env: - PYMONGO=2.5 DJANGO=1.5.1 - PYMONGO=2.5 DJANGO=1.4.2 - PYMONGO=2.4.2 DJANGO=1.4.2 +matrix: + exclude: + - python: "2.6" + env: PYMONGO=dev DJANGO=1.5.1 + - python: "2.6" + env: PYMONGO=2.5 DJANGO=1.5.1 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 738a949..6138bb4 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -15,6 +15,11 @@ possible for the whole of the release. live. There maybe multiple manual steps in migrating and these are best honed on a staging / test system. +Python +======= + +Support for python 2.5 has been dropped. + Data Model ========== From f7bc58a767c80495ca46bd05a8b2f04aeaae462e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 12:03:44 +0000 Subject: [PATCH 1056/1279] Added assertIn / assertNotIn for python 2.6 --- .travis.yml | 6 ------ tests/test_django.py | 8 +++++++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index c5fe62e..e78bda5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,12 +12,6 @@ env: - PYMONGO=2.5 DJANGO=1.5.1 - PYMONGO=2.5 DJANGO=1.4.2 - PYMONGO=2.4.2 DJANGO=1.4.2 -matrix: - exclude: - - python: "2.6" - env: PYMONGO=dev DJANGO=1.5.1 - - python: "2.6" - env: PYMONGO=2.5 DJANGO=1.5.1 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi diff --git a/tests/test_django.py b/tests/test_django.py index 01a105a..573c072 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -178,6 +178,12 @@ class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): MongoSession.drop_collection() super(MongoDBSessionTest, self).setUp() + def assertIn(self, first, second, msg=None): + self.assertTrue(first in second, msg) + + def assertNotIn(self, first, second, msg=None): + self.assertFalse(first in second, msg) + def test_first_save(self): session = SessionStore() session['test'] = True @@ -188,7 +194,7 @@ class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): activate_timezone(FixedOffset(60, 'UTC+1')) # create and save new session session = SessionStore() - session.set_expiry(600) # expire in 600 seconds + session.set_expiry(600) # expire in 600 seconds session['test_expire'] = True session.save() # reload session with key From d0d9c3ea26ba7f053969fd84518477a4e6544be9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 12:21:25 +0000 Subject: [PATCH 1057/1279] Test to ensure that pickled complex fields work with save() (#228) --- tests/document/instance.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index a75cc4d..b800d90 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1694,11 +1694,19 @@ class InstanceTest(unittest.TestCase): self.assertEqual(resurrected, pickle_doc) + # Test pickling changed data + pickle_doc.lists.append("3") + pickled_doc = pickle.dumps(pickle_doc) + resurrected = pickle.loads(pickled_doc) + + self.assertEqual(resurrected, pickle_doc) resurrected.string = "Two" resurrected.save() - pickle_doc = pickle_doc.reload() + pickle_doc = PickleTest.objects.first() self.assertEqual(resurrected, pickle_doc) + self.assertEqual(pickle_doc.string, "Two") + self.assertEqual(pickle_doc.lists, ["1", "2", "3"]) def test_throw_invalid_document_error(self): From ac6e793bbe8e907b7f1469a18709e18d045306a2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 13:43:56 +0000 Subject: [PATCH 1058/1279] UUIDField now stores as a binary by default (#292) --- docs/changelog.rst | 1 + docs/upgrade.rst | 24 ++ mongoengine/fields.py | 12 +- tests/__init__.py | 4 +- tests/document/delta.py | 47 ++-- tests/fields/fields.py | 305 +------------------------ tests/migration/__init__.py | 1 + tests/migration/uuidfield_to_binary.py | 48 ++++ 8 files changed, 109 insertions(+), 333 deletions(-) create mode 100644 tests/migration/uuidfield_to_binary.py diff --git a/docs/changelog.rst b/docs/changelog.rst index bd3821f..9ea4ad5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- UUIDField now stores as a binary by default (#292) - Added Custom User Model for Django 1.5 (#285) - Cascading saves now default to off (#291) - ReferenceField now store ObjectId's by default rather than DBRef (#290) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 6138bb4..bcdc110 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -120,6 +120,30 @@ eg:: p._mark_as_dirty('friends') p.save() +UUIDField +--------- + +UUIDFields now default to storing binary values:: + + # Old code + class Animal(Document): + uuid = UUIDField() + + # New code + class Animal(Document): + uuid = UUIDField(binary=False) + +To migrate all the uuid's you need to touch each object and mark it as dirty +eg:: + + # Doc definition + class Animal(Document): + uuid = UUIDField() + + # Mark all ReferenceFields as dirty and save + for a in Animal.objects: + a._mark_as_dirty('uuid') + a.save() Cascading Saves --------------- diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 979699c..bea827c 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1474,19 +1474,15 @@ class UUIDField(BaseField): """ _binary = None - def __init__(self, binary=None, **kwargs): + def __init__(self, binary=True, **kwargs): """ Store UUID data in the database - :param binary: (optional) boolean store as binary. + :param binary: if False store as a string. + .. versionchanged:: 0.8.0 .. versionchanged:: 0.6.19 """ - if binary is None: - binary = False - msg = ("UUIDFields will soon default to store as binary, please " - "configure binary=False if you wish to store as a string") - warnings.warn(msg, FutureWarning) self._binary = binary super(UUIDField, self).__init__(**kwargs) @@ -1504,6 +1500,8 @@ class UUIDField(BaseField): def to_mongo(self, value): if not self._binary: return unicode(value) + elif isinstance(value, basestring): + return uuid.UUID(value) return value def prepare_query_value(self, op, value): diff --git a/tests/__init__.py b/tests/__init__.py index 152a8ce..b24df5d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,3 +1,5 @@ from all_warnings import AllWarnings from document import * -from queryset import * \ No newline at end of file +from queryset import * +from fields import * +from migration import * diff --git a/tests/document/delta.py b/tests/document/delta.py index c6191d9..16ab609 100644 --- a/tests/document/delta.py +++ b/tests/document/delta.py @@ -129,14 +129,14 @@ class DeltaTest(unittest.TestCase): } self.assertEqual(doc.embedded_field._delta(), (embedded_delta, {})) self.assertEqual(doc._delta(), - ({'embedded_field': embedded_delta}, {})) + ({'embedded_field': embedded_delta}, {})) doc.save() doc = doc.reload(10) doc.embedded_field.dict_field = {} self.assertEqual(doc._get_changed_fields(), - ['embedded_field.dict_field']) + ['embedded_field.dict_field']) self.assertEqual(doc.embedded_field._delta(), ({}, {'dict_field': 1})) self.assertEqual(doc._delta(), ({}, {'embedded_field.dict_field': 1})) doc.save() @@ -145,7 +145,7 @@ class DeltaTest(unittest.TestCase): doc.embedded_field.list_field = [] self.assertEqual(doc._get_changed_fields(), - ['embedded_field.list_field']) + ['embedded_field.list_field']) self.assertEqual(doc.embedded_field._delta(), ({}, {'list_field': 1})) self.assertEqual(doc._delta(), ({}, {'embedded_field.list_field': 1})) doc.save() @@ -160,7 +160,7 @@ class DeltaTest(unittest.TestCase): doc.embedded_field.list_field = ['1', 2, embedded_2] self.assertEqual(doc._get_changed_fields(), - ['embedded_field.list_field']) + ['embedded_field.list_field']) self.assertEqual(doc.embedded_field._delta(), ({ 'list_field': ['1', 2, { @@ -192,11 +192,11 @@ class DeltaTest(unittest.TestCase): doc.embedded_field.list_field[2].string_field = 'world' self.assertEqual(doc._get_changed_fields(), - ['embedded_field.list_field.2.string_field']) + ['embedded_field.list_field.2.string_field']) self.assertEqual(doc.embedded_field._delta(), - ({'list_field.2.string_field': 'world'}, {})) + ({'list_field.2.string_field': 'world'}, {})) self.assertEqual(doc._delta(), - ({'embedded_field.list_field.2.string_field': 'world'}, {})) + ({'embedded_field.list_field.2.string_field': 'world'}, {})) doc.save() doc = doc.reload(10) self.assertEqual(doc.embedded_field.list_field[2].string_field, @@ -206,7 +206,7 @@ class DeltaTest(unittest.TestCase): doc.embedded_field.list_field[2].string_field = 'hello world' doc.embedded_field.list_field[2] = doc.embedded_field.list_field[2] self.assertEqual(doc._get_changed_fields(), - ['embedded_field.list_field']) + ['embedded_field.list_field']) self.assertEqual(doc.embedded_field._delta(), ({ 'list_field': ['1', 2, { '_cls': 'Embedded', @@ -225,40 +225,40 @@ class DeltaTest(unittest.TestCase): doc.save() doc = doc.reload(10) self.assertEqual(doc.embedded_field.list_field[2].string_field, - 'hello world') + 'hello world') # Test list native methods doc.embedded_field.list_field[2].list_field.pop(0) self.assertEqual(doc._delta(), - ({'embedded_field.list_field.2.list_field': - [2, {'hello': 'world'}]}, {})) + ({'embedded_field.list_field.2.list_field': + [2, {'hello': 'world'}]}, {})) doc.save() doc = doc.reload(10) doc.embedded_field.list_field[2].list_field.append(1) self.assertEqual(doc._delta(), - ({'embedded_field.list_field.2.list_field': - [2, {'hello': 'world'}, 1]}, {})) + ({'embedded_field.list_field.2.list_field': + [2, {'hello': 'world'}, 1]}, {})) doc.save() doc = doc.reload(10) self.assertEqual(doc.embedded_field.list_field[2].list_field, - [2, {'hello': 'world'}, 1]) + [2, {'hello': 'world'}, 1]) doc.embedded_field.list_field[2].list_field.sort(key=str) doc.save() doc = doc.reload(10) self.assertEqual(doc.embedded_field.list_field[2].list_field, - [1, 2, {'hello': 'world'}]) + [1, 2, {'hello': 'world'}]) del(doc.embedded_field.list_field[2].list_field[2]['hello']) self.assertEqual(doc._delta(), - ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) + ({'embedded_field.list_field.2.list_field': [1, 2, {}]}, {})) doc.save() doc = doc.reload(10) del(doc.embedded_field.list_field[2].list_field) self.assertEqual(doc._delta(), - ({}, {'embedded_field.list_field.2.list_field': 1})) + ({}, {'embedded_field.list_field.2.list_field': 1})) doc.save() doc = doc.reload(10) @@ -269,9 +269,9 @@ class DeltaTest(unittest.TestCase): doc.dict_field['Embedded'].string_field = 'Hello World' self.assertEqual(doc._get_changed_fields(), - ['dict_field.Embedded.string_field']) + ['dict_field.Embedded.string_field']) self.assertEqual(doc._delta(), - ({'dict_field.Embedded.string_field': 'Hello World'}, {})) + ({'dict_field.Embedded.string_field': 'Hello World'}, {})) def test_circular_reference_deltas(self): self.circular_reference_deltas(Document, Document) @@ -289,10 +289,11 @@ class DeltaTest(unittest.TestCase): name = StringField() owner = ReferenceField('Person') - person = Person(name="owner") - person.save() - organization = Organization(name="company") - organization.save() + Person.drop_collection() + Organization.drop_collection() + + person = Person(name="owner").save() + organization = Organization(name="company").save() person.owns.append(organization) organization.owner = person diff --git a/tests/fields/fields.py b/tests/fields/fields.py index ade44b8..7eae3f4 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -354,7 +354,6 @@ class FieldTest(unittest.TestCase): person.api_key = api_key self.assertRaises(ValidationError, person.validate) - def test_datetime_validation(self): """Ensure that invalid values cannot be assigned to datetime fields. """ @@ -1805,304 +1804,6 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() - def test_file_fields(self): - """Ensure that file fields can be written to and their data retrieved - """ - class PutFile(Document): - the_file = FileField() - - class StreamFile(Document): - the_file = FileField() - - class SetFile(Document): - the_file = FileField() - - text = b('Hello, World!') - more_text = b('Foo Bar') - content_type = 'text/plain' - - PutFile.drop_collection() - StreamFile.drop_collection() - SetFile.drop_collection() - - putfile = PutFile() - putfile.the_file.put(text, content_type=content_type) - putfile.save() - putfile.validate() - result = PutFile.objects.first() - self.assertTrue(putfile == result) - self.assertEqual(result.the_file.read(), text) - self.assertEqual(result.the_file.content_type, content_type) - result.the_file.delete() # Remove file from GridFS - PutFile.objects.delete() - - # Ensure file-like objects are stored - putfile = PutFile() - putstring = StringIO() - putstring.write(text) - putstring.seek(0) - putfile.the_file.put(putstring, content_type=content_type) - putfile.save() - putfile.validate() - result = PutFile.objects.first() - self.assertTrue(putfile == result) - self.assertEqual(result.the_file.read(), text) - self.assertEqual(result.the_file.content_type, content_type) - result.the_file.delete() - - streamfile = StreamFile() - streamfile.the_file.new_file(content_type=content_type) - streamfile.the_file.write(text) - streamfile.the_file.write(more_text) - streamfile.the_file.close() - streamfile.save() - streamfile.validate() - result = StreamFile.objects.first() - self.assertTrue(streamfile == result) - self.assertEqual(result.the_file.read(), text + more_text) - self.assertEqual(result.the_file.content_type, content_type) - result.the_file.seek(0) - self.assertEqual(result.the_file.tell(), 0) - self.assertEqual(result.the_file.read(len(text)), text) - self.assertEqual(result.the_file.tell(), len(text)) - self.assertEqual(result.the_file.read(len(more_text)), more_text) - self.assertEqual(result.the_file.tell(), len(text + more_text)) - result.the_file.delete() - - # Ensure deleted file returns None - self.assertTrue(result.the_file.read() == None) - - setfile = SetFile() - setfile.the_file = text - setfile.save() - setfile.validate() - result = SetFile.objects.first() - self.assertTrue(setfile == result) - self.assertEqual(result.the_file.read(), text) - - # Try replacing file with new one - result.the_file.replace(more_text) - result.save() - result.validate() - result = SetFile.objects.first() - self.assertTrue(setfile == result) - self.assertEqual(result.the_file.read(), more_text) - result.the_file.delete() - - PutFile.drop_collection() - StreamFile.drop_collection() - SetFile.drop_collection() - - # Make sure FileField is optional and not required - class DemoFile(Document): - the_file = FileField() - DemoFile.objects.create() - - - def test_file_field_no_default(self): - - class GridDocument(Document): - the_file = FileField() - - GridDocument.drop_collection() - - with tempfile.TemporaryFile() as f: - f.write(b("Hello World!")) - f.flush() - - # Test without default - doc_a = GridDocument() - doc_a.save() - - - doc_b = GridDocument.objects.with_id(doc_a.id) - doc_b.the_file.replace(f, filename='doc_b') - doc_b.save() - self.assertNotEqual(doc_b.the_file.grid_id, None) - - # Test it matches - doc_c = GridDocument.objects.with_id(doc_b.id) - self.assertEqual(doc_b.the_file.grid_id, doc_c.the_file.grid_id) - - # Test with default - doc_d = GridDocument(the_file=b('')) - doc_d.save() - - doc_e = GridDocument.objects.with_id(doc_d.id) - self.assertEqual(doc_d.the_file.grid_id, doc_e.the_file.grid_id) - - doc_e.the_file.replace(f, filename='doc_e') - doc_e.save() - - doc_f = GridDocument.objects.with_id(doc_e.id) - self.assertEqual(doc_e.the_file.grid_id, doc_f.the_file.grid_id) - - db = GridDocument._get_db() - grid_fs = gridfs.GridFS(db) - self.assertEqual(['doc_b', 'doc_e'], grid_fs.list()) - - def test_file_uniqueness(self): - """Ensure that each instance of a FileField is unique - """ - class TestFile(Document): - name = StringField() - the_file = FileField() - - # First instance - test_file = TestFile() - test_file.name = "Hello, World!" - test_file.the_file.put(b('Hello, World!')) - test_file.save() - - # Second instance - test_file_dupe = TestFile() - data = test_file_dupe.the_file.read() # Should be None - - self.assertTrue(test_file.name != test_file_dupe.name) - self.assertTrue(test_file.the_file.read() != data) - - TestFile.drop_collection() - - def test_file_boolean(self): - """Ensure that a boolean test of a FileField indicates its presence - """ - class TestFile(Document): - the_file = FileField() - - test_file = TestFile() - self.assertFalse(bool(test_file.the_file)) - test_file.the_file = b('Hello, World!') - test_file.the_file.content_type = 'text/plain' - test_file.save() - self.assertTrue(bool(test_file.the_file)) - - TestFile.drop_collection() - - def test_file_cmp(self): - """Test comparing against other types""" - class TestFile(Document): - the_file = FileField() - - test_file = TestFile() - self.assertFalse(test_file.the_file in [{"test": 1}]) - - def test_image_field(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') - - class TestImage(Document): - image = ImageField() - - TestImage.drop_collection() - - t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'rb')) - t.save() - - t = TestImage.objects.first() - - self.assertEqual(t.image.format, 'PNG') - - w, h = t.image.size - self.assertEqual(w, 371) - self.assertEqual(h, 76) - - t.image.delete() - - def test_image_field_resize(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') - - class TestImage(Document): - image = ImageField(size=(185, 37)) - - TestImage.drop_collection() - - t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'rb')) - t.save() - - t = TestImage.objects.first() - - self.assertEqual(t.image.format, 'PNG') - w, h = t.image.size - - self.assertEqual(w, 185) - self.assertEqual(h, 37) - - t.image.delete() - - def test_image_field_resize_force(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') - - class TestImage(Document): - image = ImageField(size=(185, 37, True)) - - TestImage.drop_collection() - - t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'rb')) - t.save() - - t = TestImage.objects.first() - - self.assertEqual(t.image.format, 'PNG') - w, h = t.image.size - - self.assertEqual(w, 185) - self.assertEqual(h, 37) - - t.image.delete() - - def test_image_field_thumbnail(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') - - class TestImage(Document): - image = ImageField(thumbnail_size=(92, 18)) - - TestImage.drop_collection() - - t = TestImage() - t.image.put(open(TEST_IMAGE_PATH, 'rb')) - t.save() - - t = TestImage.objects.first() - - self.assertEqual(t.image.thumbnail.format, 'PNG') - self.assertEqual(t.image.thumbnail.width, 92) - self.assertEqual(t.image.thumbnail.height, 18) - - t.image.delete() - - def test_file_multidb(self): - register_connection('test_files', 'test_files') - class TestFile(Document): - name = StringField() - the_file = FileField(db_alias="test_files", - collection_name="macumba") - - TestFile.drop_collection() - - # delete old filesystem - get_db("test_files").macumba.files.drop() - get_db("test_files").macumba.chunks.drop() - - # First instance - test_file = TestFile() - test_file.name = "Hello, World!" - test_file.the_file.put(b('Hello, World!'), - name="hello.txt") - test_file.save() - - data = get_db("test_files").macumba.files.find_one() - self.assertEqual(data.get('name'), 'hello.txt') - - test_file = TestFile.objects.first() - self.assertEqual(test_file.the_file.read(), - b('Hello, World!')) - def test_geo_indexes(self): """Ensure that indexes are created automatically for GeoPointFields. """ @@ -2170,7 +1871,7 @@ class FieldTest(unittest.TestCase): self.assertEqual(c['next'], 10) ids = [i.id for i in Person.objects] - self.assertEqual(ids, xrange(1, 11)) + self.assertEqual(ids, range(1, 11)) c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) @@ -2219,10 +1920,10 @@ class FieldTest(unittest.TestCase): self.assertEqual(c['next'], 10) ids = [i.id for i in Person.objects] - self.assertEqual(ids, xrange(1, 11)) + self.assertEqual(ids, range(1, 11)) counters = [i.counter for i in Person.objects] - self.assertEqual(counters, xrange(1, 11)) + self.assertEqual(counters, range(1, 11)) c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 10) diff --git a/tests/migration/__init__.py b/tests/migration/__init__.py index f7ad674..bff50c3 100644 --- a/tests/migration/__init__.py +++ b/tests/migration/__init__.py @@ -1,6 +1,7 @@ from convert_to_new_inheritance_model import * from refrencefield_dbref_to_object_id import * from turn_off_inheritance import * +from uuidfield_to_binary import * if __name__ == '__main__': unittest.main() diff --git a/tests/migration/uuidfield_to_binary.py b/tests/migration/uuidfield_to_binary.py new file mode 100644 index 0000000..a535e91 --- /dev/null +++ b/tests/migration/uuidfield_to_binary.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +import unittest +import uuid + +from mongoengine import Document, connect +from mongoengine.connection import get_db +from mongoengine.fields import StringField, UUIDField, ListField + +__all__ = ('ConvertToBinaryUUID', ) + + +class ConvertToBinaryUUID(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + def test_how_to_convert_to_binary_uuid_fields(self): + """Demonstrates migrating from 0.7 to 0.8 + """ + + # 1. Old definition - using dbrefs + class Person(Document): + name = StringField() + uuid = UUIDField(binary=False) + uuids = ListField(UUIDField(binary=False)) + + Person.drop_collection() + Person(name="Wilson Jr", uuid=uuid.uuid4(), + uuids=[uuid.uuid4(), uuid.uuid4()]).save() + + # 2. Start the migration by changing the schema + # Change UUIDFIeld as now binary defaults to True + class Person(Document): + name = StringField() + uuid = UUIDField() + uuids = ListField(UUIDField()) + + # 3. Loop all the objects and mark parent as changed + for p in Person.objects: + p._mark_as_changed('uuid') + p._mark_as_changed('uuids') + p.save() + + # 4. Confirmation of the fix! + wilson = Person.objects(name="Wilson Jr").as_pymongo()[0] + self.assertTrue(isinstance(wilson['uuid'], uuid.UUID)) + self.assertTrue(all([isinstance(u, uuid.UUID) for u in wilson['uuids']])) From 5e94637adc2d0fcb89b5569cba5ffec13d511147 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 25 Apr 2013 15:39:57 +0000 Subject: [PATCH 1059/1279] DecimalField now stores as float not string (#289) --- docs/apireference.rst | 60 ++++++++++++------------ docs/changelog.rst | 1 + docs/conf.py | 7 ++- docs/upgrade.rst | 29 ++++++++++++ mongoengine/fields.py | 54 ++++++++++++++++----- mongoengine/queryset/transform.py | 6 +-- tests/fields/fields.py | 45 ++++++++++++++++-- tests/migration/__init__.py | 1 + tests/migration/decimalfield_as_float.py | 50 ++++++++++++++++++++ tests/queryset/queryset.py | 4 +- 10 files changed, 204 insertions(+), 53 deletions(-) create mode 100644 tests/migration/decimalfield_as_float.py diff --git a/docs/apireference.rst b/docs/apireference.rst index 0040f45..3a15629 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -54,33 +54,33 @@ Querying Fields ====== -.. autoclass:: mongoengine.StringField -.. autoclass:: mongoengine.URLField -.. autoclass:: mongoengine.EmailField -.. autoclass:: mongoengine.IntField -.. autoclass:: mongoengine.LongField -.. autoclass:: mongoengine.FloatField -.. autoclass:: mongoengine.DecimalField -.. autoclass:: mongoengine.BooleanField -.. autoclass:: mongoengine.DateTimeField -.. autoclass:: mongoengine.ComplexDateTimeField -.. autoclass:: mongoengine.EmbeddedDocumentField -.. autoclass:: mongoengine.GenericEmbeddedDocumentField -.. autoclass:: mongoengine.DynamicField -.. autoclass:: mongoengine.ListField -.. autoclass:: mongoengine.SortedListField -.. autoclass:: mongoengine.DictField -.. autoclass:: mongoengine.MapField -.. autoclass:: mongoengine.ReferenceField -.. autoclass:: mongoengine.GenericReferenceField -.. autoclass:: mongoengine.BinaryField -.. autoclass:: mongoengine.FileField -.. autoclass:: mongoengine.ImageField -.. autoclass:: mongoengine.GeoPointField -.. autoclass:: mongoengine.SequenceField -.. autoclass:: mongoengine.ObjectIdField -.. autoclass:: mongoengine.UUIDField -.. autoclass:: mongoengine.GridFSError -.. autoclass:: mongoengine.GridFSProxy -.. autoclass:: mongoengine.ImageGridFsProxy -.. autoclass:: mongoengine.ImproperlyConfigured +.. autoclass:: mongoengine.fields.StringField +.. autoclass:: mongoengine.fields.URLField +.. autoclass:: mongoengine.fields.EmailField +.. autoclass:: mongoengine.fields.IntField +.. autoclass:: mongoengine.fields.LongField +.. autoclass:: mongoengine.fields.FloatField +.. autoclass:: mongoengine.fields.DecimalField +.. autoclass:: mongoengine.fields.BooleanField +.. autoclass:: mongoengine.fields.DateTimeField +.. autoclass:: mongoengine.fields.ComplexDateTimeField +.. autoclass:: mongoengine.fields.EmbeddedDocumentField +.. autoclass:: mongoengine.fields.GenericEmbeddedDocumentField +.. autoclass:: mongoengine.fields.DynamicField +.. autoclass:: mongoengine.fields.ListField +.. autoclass:: mongoengine.fields.SortedListField +.. autoclass:: mongoengine.fields.DictField +.. autoclass:: mongoengine.fields.MapField +.. autoclass:: mongoengine.fields.ReferenceField +.. autoclass:: mongoengine.fields.GenericReferenceField +.. autoclass:: mongoengine.fields.BinaryField +.. autoclass:: mongoengine.fields.FileField +.. autoclass:: mongoengine.fields.ImageField +.. autoclass:: mongoengine.fields.GeoPointField +.. autoclass:: mongoengine.fields.SequenceField +.. autoclass:: mongoengine.fields.ObjectIdField +.. autoclass:: mongoengine.fields.UUIDField +.. autoclass:: mongoengine.fields.GridFSError +.. autoclass:: mongoengine.fields.GridFSProxy +.. autoclass:: mongoengine.fields.ImageGridFsProxy +.. autoclass:: mongoengine.fields.ImproperlyConfigured diff --git a/docs/changelog.rst b/docs/changelog.rst index 9ea4ad5..d0167c5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- DecimalField now stores as float not string (#289) - UUIDField now stores as a binary by default (#292) - Added Custom User Model for Django 1.5 (#285) - Cascading saves now default to off (#291) diff --git a/docs/conf.py b/docs/conf.py index 3cfcef5..8bcb9ec 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -173,8 +173,8 @@ latex_paper_size = 'a4' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'MongoEngine.tex', u'MongoEngine Documentation', - u'Harry Marr', 'manual'), + ('index', 'MongoEngine.tex', 'MongoEngine Documentation', + 'Ross Lawley', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -193,3 +193,6 @@ latex_documents = [ # If false, no module index is generated. #latex_use_modindex = True + +autoclass_content = 'both' + diff --git a/docs/upgrade.rst b/docs/upgrade.rst index bcdc110..dddce91 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -145,6 +145,35 @@ eg:: a._mark_as_dirty('uuid') a.save() +DecimalField +------------ + +DecimalField now store floats - previous it was storing strings and that +made it impossible to do comparisons when querying correctly.:: + + # Old code + class Person(Document): + balance = DecimalField() + + # New code + class Person(Document): + balance = DecimalField(force_string=True) + +To migrate all the uuid's you need to touch each object and mark it as dirty +eg:: + + # Doc definition + class Person(Document): + balance = DecimalField() + + # Mark all ReferenceFields as dirty and save + for p in Person.objects: + p._mark_as_dirty('balance') + p.save() + +.. note:: DecimalField's have also been improved with the addition of precision + and rounding. See :class:`~mongoengine.DecimalField` for more information. + Cascading Saves --------------- To improve performance document saves will no longer automatically cascade. diff --git a/mongoengine/fields.py b/mongoengine/fields.py index bea827c..2e14933 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -260,30 +260,57 @@ class FloatField(BaseField): class DecimalField(BaseField): """A fixed-point decimal number field. + .. versionchanged:: 0.8 .. versionadded:: 0.3 """ - def __init__(self, min_value=None, max_value=None, **kwargs): - self.min_value, self.max_value = min_value, max_value + def __init__(self, min_value=None, max_value=None, force_string=False, + precision=2, rounding=decimal.ROUND_HALF_UP, **kwargs): + """ + :param min_value: Validation rule for the minimum acceptable value. + :param max_value: Validation rule for the maximum acceptable value. + :param force_string: Store as a string. + :param precision: Number of decimal places to store. + :param rounding: The rounding rule from the python decimal libary: + + - decimial.ROUND_CEILING (towards Infinity) + - decimial.ROUND_DOWN (towards zero) + - decimial.ROUND_FLOOR (towards -Infinity) + - decimial.ROUND_HALF_DOWN (to nearest with ties going towards zero) + - decimial.ROUND_HALF_EVEN (to nearest with ties going to nearest even integer) + - decimial.ROUND_HALF_UP (to nearest with ties going away from zero) + - decimial.ROUND_UP (away from zero) + - decimial.ROUND_05UP (away from zero if last digit after rounding towards zero would have been 0 or 5; otherwise towards zero) + + Defaults to: ``decimal.ROUND_HALF_UP`` + + """ + self.min_value = min_value + self.max_value = max_value + self.force_string = force_string + self.precision = decimal.Decimal(".%s" % ("0" * precision)) + self.rounding = rounding + super(DecimalField, self).__init__(**kwargs) def to_python(self, value): - original_value = value - if not isinstance(value, basestring): - value = unicode(value) - try: - value = decimal.Decimal(value) - except ValueError: - return original_value - return value + if value is None: + return value + + return decimal.Decimal(value).quantize(self.precision, + rounding=self.rounding) def to_mongo(self, value): - return unicode(value) + if value is None: + return value + if self.force_string: + return unicode(value) + return float(self.to_python(value)) def validate(self, value): if not isinstance(value, decimal.Decimal): if not isinstance(value, basestring): - value = str(value) + value = unicode(value) try: value = decimal.Decimal(value) except Exception, exc: @@ -295,6 +322,9 @@ class DecimalField(BaseField): if self.max_value is not None and value > self.max_value: self.error('Decimal value is too large') + def prepare_query_value(self, op, value): + return self.to_mongo(value) + class BooleanField(BaseField): """A boolean field type. diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 71f12e3..3da2693 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -9,7 +9,7 @@ __all__ = ('query', 'update') COMPARISON_OPERATORS = ('ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', - 'all', 'size', 'exists', 'not') + 'all', 'size', 'exists', 'not') GEO_OPERATORS = ('within_distance', 'within_spherical_distance', 'within_box', 'within_polygon', 'near', 'near_sphere', 'max_distance') @@ -74,7 +74,7 @@ def query(_doc_cls=None, _field_operation=False, **query): if op in singular_ops: if isinstance(field, basestring): if (op in STRING_OPERATORS and - isinstance(value, basestring)): + isinstance(value, basestring)): StringField = _import_class('StringField') value = StringField.prepare_query_value(op, value) else: @@ -144,7 +144,7 @@ def query(_doc_cls=None, _field_operation=False, **query): merge_query[k].append(mongo_query[k]) del mongo_query[k] if isinstance(v, list): - value = [{k:val} for val in v] + value = [{k: val} for val in v] if '$and' in mongo_query.keys(): mongo_query['$and'].append(value) else: diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 7eae3f4..4fa6989 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -272,10 +272,8 @@ class FieldTest(unittest.TestCase): Person.drop_collection() - person = Person() - person.height = Decimal('1.89') - person.save() - person.reload() + Person(height=Decimal('1.89')).save() + person = Person.objects.first() self.assertEqual(person.height, Decimal('1.89')) person.height = '2.0' @@ -289,6 +287,45 @@ class FieldTest(unittest.TestCase): Person.drop_collection() + def test_decimal_comparison(self): + + class Person(Document): + money = DecimalField() + + Person.drop_collection() + + Person(money=6).save() + Person(money=8).save() + Person(money=10).save() + + self.assertEqual(2, Person.objects(money__gt=Decimal("7")).count()) + self.assertEqual(2, Person.objects(money__gt=7).count()) + self.assertEqual(2, Person.objects(money__gt="7").count()) + + def test_decimal_storage(self): + class Person(Document): + btc = DecimalField(precision=4) + + Person.drop_collection() + Person(btc=10).save() + Person(btc=10.1).save() + Person(btc=10.11).save() + Person(btc="10.111").save() + Person(btc=Decimal("10.1111")).save() + Person(btc=Decimal("10.11111")).save() + + # How its stored + expected = [{'btc': 10.0}, {'btc': 10.1}, {'btc': 10.11}, + {'btc': 10.111}, {'btc': 10.1111}, {'btc': 10.1111}] + actual = list(Person.objects.exclude('id').as_pymongo()) + self.assertEqual(expected, actual) + + # How it comes out locally + expected = [Decimal('10.0000'), Decimal('10.1000'), Decimal('10.1100'), + Decimal('10.1110'), Decimal('10.1111'), Decimal('10.1111')] + actual = list(Person.objects().scalar('btc')) + self.assertEqual(expected, actual) + def test_boolean_validation(self): """Ensure that invalid values cannot be assigned to boolean fields. """ diff --git a/tests/migration/__init__.py b/tests/migration/__init__.py index bff50c3..6fc83e0 100644 --- a/tests/migration/__init__.py +++ b/tests/migration/__init__.py @@ -1,4 +1,5 @@ from convert_to_new_inheritance_model import * +from decimalfield_as_float import * from refrencefield_dbref_to_object_id import * from turn_off_inheritance import * from uuidfield_to_binary import * diff --git a/tests/migration/decimalfield_as_float.py b/tests/migration/decimalfield_as_float.py new file mode 100644 index 0000000..3903c91 --- /dev/null +++ b/tests/migration/decimalfield_as_float.py @@ -0,0 +1,50 @@ + # -*- coding: utf-8 -*- +import unittest +import decimal +from decimal import Decimal + +from mongoengine import Document, connect +from mongoengine.connection import get_db +from mongoengine.fields import StringField, DecimalField, ListField + +__all__ = ('ConvertDecimalField', ) + + +class ConvertDecimalField(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + def test_how_to_convert_decimal_fields(self): + """Demonstrates migrating from 0.7 to 0.8 + """ + + # 1. Old definition - using dbrefs + class Person(Document): + name = StringField() + money = DecimalField(force_string=True) + monies = ListField(DecimalField(force_string=True)) + + Person.drop_collection() + Person(name="Wilson Jr", money=Decimal("2.50"), + monies=[Decimal("2.10"), Decimal("5.00")]).save() + + # 2. Start the migration by changing the schema + # Change DecimalField - add precision and rounding settings + class Person(Document): + name = StringField() + money = DecimalField(precision=2, rounding=decimal.ROUND_HALF_UP) + monies = ListField(DecimalField(precision=2, + rounding=decimal.ROUND_HALF_UP)) + + # 3. Loop all the objects and mark parent as changed + for p in Person.objects: + p._mark_as_changed('money') + p._mark_as_changed('monies') + p.save() + + # 4. Confirmation of the fix! + wilson = Person.objects(name="Wilson Jr").as_pymongo()[0] + self.assertTrue(isinstance(wilson['money'], float)) + self.assertTrue(all([isinstance(m, float) for m in wilson['monies']])) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index c7c4c7c..5e403c4 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3262,9 +3262,9 @@ class QuerySetTest(unittest.TestCase): self.assertTrue(isinstance(results[0], dict)) self.assertTrue(isinstance(results[1], dict)) self.assertEqual(results[0]['name'], 'Bob Dole') - self.assertEqual(results[0]['price'], '1.11') + self.assertEqual(results[0]['price'], 1.11) self.assertEqual(results[1]['name'], 'Barack Obama') - self.assertEqual(results[1]['price'], '2.22') + self.assertEqual(results[1]['price'], 2.22) # Test coerce_types users = User.objects.only('name', 'price').as_pymongo(coerce_types=True) From 13d8dfdb5fc44ce92ed3373111d4e2e74cf2e66b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 26 Apr 2013 08:43:38 +0000 Subject: [PATCH 1060/1279] Save py2.6 from Decimal Float fun --- mongoengine/fields.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 2e14933..a5dbf5d 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -297,8 +297,9 @@ class DecimalField(BaseField): if value is None: return value - return decimal.Decimal(value).quantize(self.precision, - rounding=self.rounding) + # Convert to string for python 2.6 before casting to Decimal + value = decimal.Decimal("%s" % value) + return value.quantize(self.precision, rounding=self.rounding) def to_mongo(self, value): if value is None: From 7765f272ac74f1b25e9e057d13b54408031d1594 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 26 Apr 2013 08:46:46 +0000 Subject: [PATCH 1061/1279] Documentation api and reference cleanups --- docs/django.rst | 2 +- docs/guide/defining-documents.rst | 82 +++++++++++++++---------------- docs/guide/document-instances.rst | 4 +- docs/guide/gridfs.rst | 6 +-- docs/guide/querying.rst | 16 +++--- docs/tutorial.rst | 8 +-- docs/upgrade.rst | 11 ++++- mongoengine/document.py | 2 +- mongoengine/fields.py | 2 +- mongoengine/queryset/queryset.py | 8 +-- 10 files changed, 75 insertions(+), 66 deletions(-) diff --git a/docs/django.rst b/docs/django.rst index e3a1c6b..d60e55d 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -98,7 +98,7 @@ Django provides session cookie, which expires after ```SESSION_COOKIE_AGE``` sec Storage ======= -With MongoEngine's support for GridFS via the :class:`~mongoengine.FileField`, +With MongoEngine's support for GridFS via the :class:`~mongoengine.fields.FileField`, it is useful to have a Django file storage backend that wraps this. The new storage module is called :class:`~mongoengine.django.storage.GridFSStorage`. Using it is very similar to using the default FileSystemStorage.:: diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 350ba67..d18606a 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -62,31 +62,31 @@ not provided. Default values may optionally be a callable, which will be called to retrieve the value (such as in the above example). The field types available are as follows: -* :class:`~mongoengine.BinaryField` -* :class:`~mongoengine.BooleanField` -* :class:`~mongoengine.ComplexDateTimeField` -* :class:`~mongoengine.DateTimeField` -* :class:`~mongoengine.DecimalField` -* :class:`~mongoengine.DictField` -* :class:`~mongoengine.DynamicField` -* :class:`~mongoengine.EmailField` -* :class:`~mongoengine.EmbeddedDocumentField` -* :class:`~mongoengine.FileField` -* :class:`~mongoengine.FloatField` -* :class:`~mongoengine.GenericEmbeddedDocumentField` -* :class:`~mongoengine.GenericReferenceField` -* :class:`~mongoengine.GeoPointField` -* :class:`~mongoengine.ImageField` -* :class:`~mongoengine.IntField` -* :class:`~mongoengine.ListField` -* :class:`~mongoengine.MapField` -* :class:`~mongoengine.ObjectIdField` -* :class:`~mongoengine.ReferenceField` -* :class:`~mongoengine.SequenceField` -* :class:`~mongoengine.SortedListField` -* :class:`~mongoengine.StringField` -* :class:`~mongoengine.URLField` -* :class:`~mongoengine.UUIDField` +* :class:`~mongoengine.fields.BinaryField` +* :class:`~mongoengine.fields.BooleanField` +* :class:`~mongoengine.fields.ComplexDateTimeField` +* :class:`~mongoengine.fields.DateTimeField` +* :class:`~mongoengine.fields.DecimalField` +* :class:`~mongoengine.fields.DictField` +* :class:`~mongoengine.fields.DynamicField` +* :class:`~mongoengine.fields.EmailField` +* :class:`~mongoengine.fields.EmbeddedDocumentField` +* :class:`~mongoengine.fields.FileField` +* :class:`~mongoengine.fields.FloatField` +* :class:`~mongoengine.fields.GenericEmbeddedDocumentField` +* :class:`~mongoengine.fields.GenericReferenceField` +* :class:`~mongoengine.fields.GeoPointField` +* :class:`~mongoengine.fields.ImageField` +* :class:`~mongoengine.fields.IntField` +* :class:`~mongoengine.fields.ListField` +* :class:`~mongoengine.fields.MapField` +* :class:`~mongoengine.fields.ObjectIdField` +* :class:`~mongoengine.fields.ReferenceField` +* :class:`~mongoengine.fields.SequenceField` +* :class:`~mongoengine.fields.SortedListField` +* :class:`~mongoengine.fields.StringField` +* :class:`~mongoengine.fields.URLField` +* :class:`~mongoengine.fields.UUIDField` Field arguments --------------- @@ -110,7 +110,7 @@ arguments can be set on all fields: The definion of default parameters follow `the general rules on Python `__, which means that some care should be taken when dealing with default mutable objects - (like in :class:`~mongoengine.ListField` or :class:`~mongoengine.DictField`):: + (like in :class:`~mongoengine.fields.ListField` or :class:`~mongoengine.fields.DictField`):: class ExampleFirst(Document): # Default an empty list @@ -172,8 +172,8 @@ arguments can be set on all fields: List fields ----------- MongoDB allows the storage of lists of items. To add a list of items to a -:class:`~mongoengine.Document`, use the :class:`~mongoengine.ListField` field -type. :class:`~mongoengine.ListField` takes another field object as its first +:class:`~mongoengine.Document`, use the :class:`~mongoengine.fields.ListField` field +type. :class:`~mongoengine.fields.ListField` takes another field object as its first argument, which specifies which type elements may be stored within the list:: class Page(Document): @@ -191,7 +191,7 @@ inherit from :class:`~mongoengine.EmbeddedDocument` rather than content = StringField() To embed the document within another document, use the -:class:`~mongoengine.EmbeddedDocumentField` field type, providing the embedded +:class:`~mongoengine.fields.EmbeddedDocumentField` field type, providing the embedded document class as the first argument:: class Page(Document): @@ -206,7 +206,7 @@ Dictionary Fields Often, an embedded document may be used instead of a dictionary -- generally this is recommended as dictionaries don't support validation or custom field types. However, sometimes you will not know the structure of what you want to -store; in this situation a :class:`~mongoengine.DictField` is appropriate:: +store; in this situation a :class:`~mongoengine.fields.DictField` is appropriate:: class SurveyResponse(Document): date = DateTimeField() @@ -224,7 +224,7 @@ other objects, so are the most flexible field type available. Reference fields ---------------- References may be stored to other documents in the database using the -:class:`~mongoengine.ReferenceField`. Pass in another document class as the +:class:`~mongoengine.fields.ReferenceField`. Pass in another document class as the first argument to the constructor, then simply assign document objects to the field:: @@ -245,9 +245,9 @@ field:: The :class:`User` object is automatically turned into a reference behind the scenes, and dereferenced when the :class:`Page` object is retrieved. -To add a :class:`~mongoengine.ReferenceField` that references the document +To add a :class:`~mongoengine.fields.ReferenceField` that references the document being defined, use the string ``'self'`` in place of the document class as the -argument to :class:`~mongoengine.ReferenceField`'s constructor. To reference a +argument to :class:`~mongoengine.fields.ReferenceField`'s constructor. To reference a document that has not yet been defined, use the name of the undefined document as the constructor's argument:: @@ -325,7 +325,7 @@ Its value can take any of the following constants: :const:`mongoengine.PULL` Removes the reference to the object (using MongoDB's "pull" operation) from any object's fields of - :class:`~mongoengine.ListField` (:class:`~mongoengine.ReferenceField`). + :class:`~mongoengine.fields.ListField` (:class:`~mongoengine.fields.ReferenceField`). .. warning:: @@ -352,7 +352,7 @@ Its value can take any of the following constants: Generic reference fields '''''''''''''''''''''''' A second kind of reference field also exists, -:class:`~mongoengine.GenericReferenceField`. This allows you to reference any +:class:`~mongoengine.fields.GenericReferenceField`. This allows you to reference any kind of :class:`~mongoengine.Document`, and hence doesn't take a :class:`~mongoengine.Document` subclass as a constructor argument:: @@ -376,15 +376,15 @@ kind of :class:`~mongoengine.Document`, and hence doesn't take a .. note:: - Using :class:`~mongoengine.GenericReferenceField`\ s is slightly less - efficient than the standard :class:`~mongoengine.ReferenceField`\ s, so if + Using :class:`~mongoengine.fields.GenericReferenceField`\ s is slightly less + efficient than the standard :class:`~mongoengine.fields.ReferenceField`\ s, so if you will only be referencing one document type, prefer the standard - :class:`~mongoengine.ReferenceField`. + :class:`~mongoengine.fields.ReferenceField`. Uniqueness constraints ---------------------- MongoEngine allows you to specify that a field should be unique across a -collection by providing ``unique=True`` to a :class:`~mongoengine.Field`\ 's +collection by providing ``unique=True`` to a :class:`~mongoengine.fields.Field`\ 's constructor. If you try to save a document that has the same value for a unique field as a document that is already in the database, a :class:`~mongoengine.OperationError` will be raised. You may also specify @@ -492,11 +492,11 @@ Geospatial indexes ------------------ Geospatial indexes will be automatically created for all -:class:`~mongoengine.GeoPointField`\ s +:class:`~mongoengine.fields.GeoPointField`\ s It is also possible to explicitly define geospatial indexes. This is useful if you need to define a geospatial index on a subfield of a -:class:`~mongoengine.DictField` or a custom field that contains a +:class:`~mongoengine.fields.DictField` or a custom field that contains a point. To create a geospatial index you must prefix the field with the ***** sign. :: diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index e8e7d63..619f3e8 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -68,8 +68,8 @@ document values for example:: Cascading Saves --------------- -If your document contains :class:`~mongoengine.ReferenceField` or -:class:`~mongoengine.GenericReferenceField` objects, then by default the +If your document contains :class:`~mongoengine.fields.ReferenceField` or +:class:`~mongoengine.fields.GenericReferenceField` objects, then by default the :meth:`~mongoengine.Document.save` method will automatically save any changes to those objects as well. If this is not desired passing :attr:`cascade` as False to the save method turns this feature off. diff --git a/docs/guide/gridfs.rst b/docs/guide/gridfs.rst index 1125947..d81bb92 100644 --- a/docs/guide/gridfs.rst +++ b/docs/guide/gridfs.rst @@ -7,7 +7,7 @@ GridFS Writing ------- -GridFS support comes in the form of the :class:`~mongoengine.FileField` field +GridFS support comes in the form of the :class:`~mongoengine.fields.FileField` field object. This field acts as a file-like object and provides a couple of different ways of inserting and retrieving data. Arbitrary metadata such as content type can also be stored alongside the files. In the following example, @@ -27,7 +27,7 @@ a document is created to store details about animals, including a photo:: Retrieval --------- -So using the :class:`~mongoengine.FileField` is just like using any other +So using the :class:`~mongoengine.fields.FileField` is just like using any other field. The file can also be retrieved just as easily:: marmot = Animal.objects(genus='Marmota').first() @@ -37,7 +37,7 @@ field. The file can also be retrieved just as easily:: Streaming --------- -Streaming data into a :class:`~mongoengine.FileField` is achieved in a +Streaming data into a :class:`~mongoengine.fields.FileField` is achieved in a slightly different manner. First, a new file must be created by calling the :func:`new_file` method. Data can then be written using :func:`write`:: diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 60702ec..3a25c28 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -79,7 +79,7 @@ expressions: * ``match`` -- performs an $elemMatch so you can match an entire document within an array There are a few special operators for performing geographical queries, that -may used with :class:`~mongoengine.GeoPointField`\ s: +may used with :class:`~mongoengine.fields.GeoPointField`\ s: * ``within_distance`` -- provide a list containing a point and a maximum distance (e.g. [(41.342, -87.653), 5]) @@ -100,7 +100,7 @@ Querying lists -------------- On most fields, this syntax will look up documents where the field specified matches the given value exactly, but when the field refers to a -:class:`~mongoengine.ListField`, a single item may be provided, in which case +:class:`~mongoengine.fields.ListField`, a single item may be provided, in which case lists that contain that item will be matched:: class Page(Document): @@ -319,7 +319,7 @@ Retrieving a subset of fields Sometimes a subset of fields on a :class:`~mongoengine.Document` is required, and for efficiency only these should be retrieved from the database. This issue is especially important for MongoDB, as fields may often be extremely large -(e.g. a :class:`~mongoengine.ListField` of +(e.g. a :class:`~mongoengine.fields.ListField` of :class:`~mongoengine.EmbeddedDocument`\ s, which represent the comments on a blog post. To select only a subset of fields, use :meth:`~mongoengine.queryset.QuerySet.only`, specifying the fields you want to @@ -351,14 +351,14 @@ If you later need the missing fields, just call Getting related data -------------------- -When iterating the results of :class:`~mongoengine.ListField` or -:class:`~mongoengine.DictField` we automatically dereference any +When iterating the results of :class:`~mongoengine.fields.ListField` or +:class:`~mongoengine.fields.DictField` we automatically dereference any :class:`~pymongo.dbref.DBRef` objects as efficiently as possible, reducing the number the queries to mongo. There are times when that efficiency is not enough, documents that have -:class:`~mongoengine.ReferenceField` objects or -:class:`~mongoengine.GenericReferenceField` objects at the top level are +:class:`~mongoengine.fields.ReferenceField` objects or +:class:`~mongoengine.fields.GenericReferenceField` objects at the top level are expensive as the number of queries to MongoDB can quickly rise. To limit the number of queries use @@ -541,7 +541,7 @@ Javascript code. When accessing a field on a collection object, use square-bracket notation, and prefix the MongoEngine field name with a tilde. The field name that follows the tilde will be translated to the name used in the database. Note that when referring to fields on embedded documents, -the name of the :class:`~mongoengine.EmbeddedDocumentField`, followed by a dot, +the name of the :class:`~mongoengine.fields.EmbeddedDocumentField`, followed by a dot, should be used before the name of the field on the embedded document. The following example shows how the substitutions are made:: diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 423df9b..c2f481b 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -115,7 +115,7 @@ by setting :attr:`allow_inheritance` to True in the :attr:`meta`:: link_url = StringField() We are storing a reference to the author of the posts using a -:class:`~mongoengine.ReferenceField` object. These are similar to foreign key +:class:`~mongoengine.fields.ReferenceField` object. These are similar to foreign key fields in traditional ORMs, and are automatically translated into references when they are saved, and dereferenced when they are loaded. @@ -137,7 +137,7 @@ size of our database. So let's take a look that the code our modified author = ReferenceField(User) tags = ListField(StringField(max_length=30)) -The :class:`~mongoengine.ListField` object that is used to define a Post's tags +The :class:`~mongoengine.fields.ListField` object that is used to define a Post's tags takes a field object as its first argument --- this means that you can have lists of any type of field (including lists). @@ -174,7 +174,7 @@ We can then store a list of comment documents in our post document:: Handling deletions of references ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The :class:`~mongoengine.ReferenceField` object takes a keyword +The :class:`~mongoengine.fields.ReferenceField` object takes a keyword `reverse_delete_rule` for handling deletion rules if the reference is deleted. To delete all the posts if a user is deleted set the rule:: @@ -184,7 +184,7 @@ To delete all the posts if a user is deleted set the rule:: tags = ListField(StringField(max_length=30)) comments = ListField(EmbeddedDocumentField(Comment)) -See :class:`~mongoengine.ReferenceField` for more information. +See :class:`~mongoengine.fields.ReferenceField` for more information. .. note:: MapFields and DictFields currently don't support automatic handling of diff --git a/docs/upgrade.rst b/docs/upgrade.rst index dddce91..0ae65f3 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -120,6 +120,9 @@ eg:: p._mark_as_dirty('friends') p.save() +`An example test migration is available on github +`_. + UUIDField --------- @@ -145,6 +148,9 @@ eg:: a._mark_as_dirty('uuid') a.save() +`An example test migration is available on github +`_. + DecimalField ------------ @@ -172,7 +178,10 @@ eg:: p.save() .. note:: DecimalField's have also been improved with the addition of precision - and rounding. See :class:`~mongoengine.DecimalField` for more information. + and rounding. See :class:`~mongoengine.fields.DecimalField` for more information. + +`An example test migration is available on github +`_. Cascading Saves --------------- diff --git a/mongoengine/document.py b/mongoengine/document.py index d0cafa3..c4542a2 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -559,7 +559,7 @@ class DynamicDocument(Document): way as an ordinary document but has expando style properties. Any data passed or set against the :class:`~mongoengine.DynamicDocument` that is not a field is automatically converted into a - :class:`~mongoengine.DynamicField` and data can be attributed to that + :class:`~mongoengine.fields.DynamicField` and data can be attributed to that field. .. note:: diff --git a/mongoengine/fields.py b/mongoengine/fields.py index a5dbf5d..cf2c802 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -782,7 +782,7 @@ class ReferenceField(BaseField): * NULLIFY - Updates the reference to null. * CASCADE - Deletes the documents associated with the reference. * DENY - Prevent the deletion of the reference object. - * PULL - Pull the reference from a :class:`~mongoengine.ListField` + * PULL - Pull the reference from a :class:`~mongoengine.fields.ListField` of references Alternative syntax for registering delete rules (useful when implementing diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index dcfb240..769cf68 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1049,7 +1049,7 @@ class QuerySet(object): """) for result in self.map_reduce(map_func, reduce_func, - finalize_f=finalize_func, output='inline'): + finalize_f=finalize_func, output='inline'): return result.value else: return 0 @@ -1062,11 +1062,11 @@ class QuerySet(object): .. note:: Can only do direct simple mappings and cannot map across - :class:`~mongoengine.ReferenceField` or - :class:`~mongoengine.GenericReferenceField` for more complex + :class:`~mongoengine.fields.ReferenceField` or + :class:`~mongoengine.fields.GenericReferenceField` for more complex counting a manual map reduce call would is required. - If the field is a :class:`~mongoengine.ListField`, the items within + If the field is a :class:`~mongoengine.fields.ListField`, the items within each list will be counted individually. :param field: the field to use From 2447349383ca86dd57ff83403ce6bf9aebef90f6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 26 Apr 2013 09:59:43 +0000 Subject: [PATCH 1062/1279] Added a note about distinct being a command --- mongoengine/queryset/queryset.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 769cf68..5c7c7c8 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -608,6 +608,9 @@ class QuerySet(object): :param field: the field to select distinct values from + .. note:: This is a command and won't take ordering or limit into + account. + .. versionadded:: 0.4 .. versionchanged:: 0.5 - Fixed handling references .. versionchanged:: 0.6 - Improved db_field refrence handling From 36993097b4668e20e809a9bbb9a575b45b004939 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 26 Apr 2013 11:38:45 +0000 Subject: [PATCH 1063/1279] Document serialization uses field order to ensure a strict order is set (#296) --- docs/changelog.rst | 1 + docs/guide/defining-documents.rst | 4 ++++ docs/guide/document-instances.rst | 14 ++++++++----- docs/upgrade.rst | 19 ++++++++++++++--- mongoengine/base/document.py | 34 ++++++++++++++++++++++--------- tests/document/dynamic.py | 3 ++- tests/document/inheritance.py | 27 ++++++++++++++++++++---- tests/document/instance.py | 15 ++++++++++++++ 8 files changed, 94 insertions(+), 23 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d0167c5..f786c1d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Document serialization uses field order to ensure a strict order is set (#296) - DecimalField now stores as float not string (#289) - UUIDField now stores as a binary by default (#292) - Added Custom User Model for Django 1.5 (#285) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index d18606a..36e0efe 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -24,6 +24,9 @@ objects** as class attributes to the document class:: title = StringField(max_length=200, required=True) date_modified = DateTimeField(default=datetime.datetime.now) +As BSON (the binary format for storing data in mongodb) is order dependent, +documents are serialized based on their field order. + Dynamic document schemas ======================== One of the benefits of MongoDb is dynamic schemas for a collection, whilst data @@ -51,6 +54,7 @@ be saved :: There is one caveat on Dynamic Documents: fields cannot start with `_` +Dynamic fields are stored in alphabetical order *after* any declared fields. Fields ====== diff --git a/docs/guide/document-instances.rst b/docs/guide/document-instances.rst index 619f3e8..f9a6610 100644 --- a/docs/guide/document-instances.rst +++ b/docs/guide/document-instances.rst @@ -30,11 +30,14 @@ already exist, then any changes will be updated atomically. For example:: .. note:: - Changes to documents are tracked and on the whole perform `set` operations. + Changes to documents are tracked and on the whole perform ``set`` operations. - * ``list_field.pop(0)`` - *sets* the resulting list + * ``list_field.push(0)`` - *sets* the resulting list * ``del(list_field)`` - *unsets* whole list + With lists its preferable to use ``Doc.update(push__list_field=0)`` as + this stops the whole list being updated - stopping any race conditions. + .. seealso:: :ref:`guide-atomic-updates` @@ -70,9 +73,10 @@ Cascading Saves --------------- If your document contains :class:`~mongoengine.fields.ReferenceField` or :class:`~mongoengine.fields.GenericReferenceField` objects, then by default the -:meth:`~mongoengine.Document.save` method will automatically save any changes to -those objects as well. If this is not desired passing :attr:`cascade` as False -to the save method turns this feature off. +:meth:`~mongoengine.Document.save` method will not save any changes to +those objects. If you want all references to also be saved also, noting each +save is a separate query, then passing :attr:`cascade` as True +to the save method will cascade any saves. Deleting documents ------------------ diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 0ae65f3..bb5705c 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -120,7 +120,7 @@ eg:: p._mark_as_dirty('friends') p.save() -`An example test migration is available on github +`An example test migration for ReferenceFields is available on github `_. UUIDField @@ -148,7 +148,7 @@ eg:: a._mark_as_dirty('uuid') a.save() -`An example test migration is available on github +`An example test migration for UUIDFields is available on github `_. DecimalField @@ -180,7 +180,7 @@ eg:: .. note:: DecimalField's have also been improved with the addition of precision and rounding. See :class:`~mongoengine.fields.DecimalField` for more information. -`An example test migration is available on github +`An example test migration for DecimalFields is available on github `_. Cascading Saves @@ -196,6 +196,19 @@ you will have to explicitly tell it to cascade on save:: # Or on save: my_document.save(cascade=True) +Storage +------- + +Document and Embedded Documents are now serialized based on declared field order. +Previously, the data was passed to mongodb as a dictionary and which meant that +order wasn't guaranteed - so things like ``$addToSet`` operations on +:class:`~mongoengine.EmbeddedDocument` could potentially fail in unexpected +ways. + +If this impacts you, you may want to rewrite the objects using the +``doc.mark_as_dirty('field')`` pattern described above. If you are using a +compound primary key then you will need to ensure the order is fixed and match +your EmbeddedDocument to that order. Querysets ========= diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 7ec672f..53686b2 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -6,6 +6,7 @@ from functools import partial import pymongo from bson import json_util from bson.dbref import DBRef +from bson.son import SON from mongoengine import signals from mongoengine.common import _import_class @@ -228,11 +229,16 @@ class BaseDocument(object): pass def to_mongo(self): - """Return data dictionary ready for use with MongoDB. + """Return as SON data ready for use with MongoDB. """ - data = {} - for field_name, field in self._fields.iteritems(): + data = SON() + data["_id"] = None + data['_cls'] = self._class_name + + for field_name in self: value = self._data.get(field_name, None) + field = self._fields.get(field_name) + if value is not None: value = field.to_mongo(value) @@ -244,19 +250,27 @@ class BaseDocument(object): if value is not None: data[field.db_field] = value - # Only add _cls if allow_inheritance is True - if (hasattr(self, '_meta') and - self._meta.get('allow_inheritance', ALLOW_INHERITANCE) == True): - data['_cls'] = self._class_name + # If "_id" has not been set, then try and set it + if data["_id"] is None: + data["_id"] = self._data.get("id", None) - if '_id' in data and data['_id'] is None: - del data['_id'] + if data['_id'] is None: + data.pop('_id') + + # Only add _cls if allow_inheritance is True + if (not hasattr(self, '_meta') or + not self._meta.get('allow_inheritance', ALLOW_INHERITANCE)): + data.pop('_cls') if not self._dynamic: return data - for name, field in self._dynamic_fields.items(): + # Sort dynamic fields by key + dynamic_fields = sorted(self._dynamic_fields.iteritems(), + key=operator.itemgetter(0)) + for name, field in dynamic_fields: data[name] = field.to_mongo(self._data.get(name, None)) + return data def validate(self, clean=True): diff --git a/tests/document/dynamic.py b/tests/document/dynamic.py index 5881cd0..6263e68 100644 --- a/tests/document/dynamic.py +++ b/tests/document/dynamic.py @@ -31,8 +31,9 @@ class DynamicTest(unittest.TestCase): self.assertEqual(p.to_mongo(), {"_cls": "Person", "name": "James", "age": 34}) - + self.assertEqual(p.to_mongo().keys(), ["_cls", "name", "age"]) p.save() + self.assertEqual(p.to_mongo().keys(), ["_id", "_cls", "name", "age"]) self.assertEqual(self.Person.objects.first().age, 34) diff --git a/tests/document/inheritance.py b/tests/document/inheritance.py index 3b550f1..f011631 100644 --- a/tests/document/inheritance.py +++ b/tests/document/inheritance.py @@ -143,7 +143,7 @@ class InheritanceTest(unittest.TestCase): self.assertEqual(Animal._superclasses, ()) self.assertEqual(Animal._subclasses, ('Animal', 'Animal.Fish', - 'Animal.Fish.Pike')) + 'Animal.Fish.Pike')) self.assertEqual(Fish._superclasses, ('Animal', )) self.assertEqual(Fish._subclasses, ('Animal.Fish', 'Animal.Fish.Pike')) @@ -168,6 +168,26 @@ class InheritanceTest(unittest.TestCase): self.assertEqual(Employee._get_collection_name(), Person._get_collection_name()) + def test_inheritance_to_mongo_keys(self): + """Ensure that document may inherit fields from a superclass document. + """ + class Person(Document): + name = StringField() + age = IntField() + + meta = {'allow_inheritance': True} + + class Employee(Person): + salary = IntField() + + self.assertEqual(['age', 'id', 'name', 'salary'], + sorted(Employee._fields.keys())) + self.assertEqual(Person(name="Bob", age=35).to_mongo().keys(), + ['_cls', 'name', 'age']) + self.assertEqual(Employee(name="Bob", age=35, salary=0).to_mongo().keys(), + ['_cls', 'name', 'age', 'salary']) + self.assertEqual(Employee._get_collection_name(), + Person._get_collection_name()) def test_polymorphic_queries(self): """Ensure that the correct subclasses are returned from a query @@ -197,7 +217,6 @@ class InheritanceTest(unittest.TestCase): classes = [obj.__class__ for obj in Human.objects] self.assertEqual(classes, [Human]) - def test_allow_inheritance(self): """Ensure that inheritance may be disabled on simple classes and that _cls and _subclasses will not be used. @@ -213,8 +232,8 @@ class InheritanceTest(unittest.TestCase): self.assertRaises(ValueError, create_dog_class) # Check that _cls etc aren't present on simple documents - dog = Animal(name='dog') - dog.save() + dog = Animal(name='dog').save() + self.assertEqual(dog.to_mongo().keys(), ['_id', 'name']) collection = self.db[Animal._get_collection_name()] obj = collection.find_one() diff --git a/tests/document/instance.py b/tests/document/instance.py index b800d90..06744ab 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -428,6 +428,21 @@ class InstanceTest(unittest.TestCase): self.assertFalse('age' in person) self.assertFalse('nationality' in person) + def test_embedded_document_to_mongo(self): + class Person(EmbeddedDocument): + name = StringField() + age = IntField() + + meta = {"allow_inheritance": True} + + class Employee(Person): + salary = IntField() + + self.assertEqual(Person(name="Bob", age=35).to_mongo().keys(), + ['_cls', 'name', 'age']) + self.assertEqual(Employee(name="Bob", age=35, salary=0).to_mongo().keys(), + ['_cls', 'name', 'age', 'salary']) + def test_embedded_document(self): """Ensure that embedded documents are set up correctly. """ From 5e65d278324c86bc615747108658957b4e9dff03 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 26 Apr 2013 11:46:12 +0000 Subject: [PATCH 1064/1279] PEP8 x == True should be x is True --- mongoengine/base/fields.py | 2 +- mongoengine/document.py | 2 +- mongoengine/queryset/queryset.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 6ebba36..3929a3a 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -295,7 +295,7 @@ class ComplexBaseField(BaseField): meta = getattr(v, '_meta', {}) allow_inheritance = ( meta.get('allow_inheritance', ALLOW_INHERITANCE) - == True) + is True) if not allow_inheritance and not self.field: value_dict[k] = GenericReferenceField().to_mongo(v) else: diff --git a/mongoengine/document.py b/mongoengine/document.py index c4542a2..bd6ce19 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -548,7 +548,7 @@ class Document(BaseDocument): # If _cls is being used (for polymorphism), it needs an index, # only if another index doesn't begin with _cls if (index_cls and not cls_indexed and - cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) == True): + cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) is True): collection.ensure_index('_cls', background=background, **index_opts) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 5c7c7c8..65d6553 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -66,7 +66,7 @@ class QuerySet(object): # If inheritance is allowed, only return instances and instances of # subclasses of the class being used - if document._meta.get('allow_inheritance') == True: + if document._meta.get('allow_inheritance') is True: self._initial_query = {"_cls": {"$in": self._document._subclasses}} self._loaded_fields = QueryFieldList(always_include=['_cls']) self._cursor_obj = None From 6e2d2f33deeaee2b69685a9c2389cacdfefefa38 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 26 Apr 2013 14:33:40 +0000 Subject: [PATCH 1065/1279] Updated benchmarks for #27 --- benchmark.py | 143 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 113 insertions(+), 30 deletions(-) diff --git a/benchmark.py b/benchmark.py index 0197e1d..16b2fd4 100644 --- a/benchmark.py +++ b/benchmark.py @@ -86,17 +86,43 @@ def main(): ---------------------------------------------------------------------------------------------------- Creating 10000 dictionaries - MongoEngine, force=True 8.36906409264 + 0.8.X + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - Pymongo + 3.69964408875 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - Pymongo write_concern={"w": 0} + 3.5526599884 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine + 7.00959801674 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries without continual assign - MongoEngine + 5.60943293571 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade=True + 6.715102911 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True + 5.50644683838 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False + 4.69851183891 + ---------------------------------------------------------------------------------------------------- + Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False + 4.68946313858 + ---------------------------------------------------------------------------------------------------- """ setup = """ -from pymongo import Connection -connection = Connection() +from pymongo import MongoClient +connection = MongoClient() connection.drop_database('timeit_test') """ stmt = """ -from pymongo import Connection -connection = Connection() +from pymongo import MongoClient +connection = MongoClient() db = connection.timeit_test noddy = db.noddy @@ -106,7 +132,7 @@ for i in xrange(10000): for j in range(20): example['fields']["key"+str(j)] = "value "+str(j) - noddy.insert(example) + noddy.save(example) myNoddys = noddy.find() [n for n in myNoddys] # iterate @@ -117,9 +143,32 @@ myNoddys = noddy.find() t = timeit.Timer(stmt=stmt, setup=setup) print t.timeit(1) + stmt = """ +from pymongo import MongoClient +connection = MongoClient() + +db = connection.timeit_test +noddy = db.noddy + +for i in xrange(10000): + example = {'fields': {}} + for j in range(20): + example['fields']["key"+str(j)] = "value "+str(j) + + noddy.save(example, write_concern={"w": 0}) + +myNoddys = noddy.find() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - Pymongo write_concern={"w": 0}""" + t = timeit.Timer(stmt=stmt, setup=setup) + print t.timeit(1) + setup = """ -from pymongo import Connection -connection = Connection() +from pymongo import MongoClient +connection = MongoClient() connection.drop_database('timeit_test') connection.disconnect() @@ -149,33 +198,18 @@ myNoddys = Noddy.objects() stmt = """ for i in xrange(10000): noddy = Noddy() + fields = {} for j in range(20): - noddy.fields["key"+str(j)] = "value "+str(j) - noddy.save(safe=False, validate=False) + fields["key"+str(j)] = "value "+str(j) + noddy.fields = fields + noddy.save() myNoddys = Noddy.objects() [n for n in myNoddys] # iterate """ print "-" * 100 - print """Creating 10000 dictionaries - MongoEngine, safe=False, validate=False""" - t = timeit.Timer(stmt=stmt, setup=setup) - print t.timeit(1) - - - stmt = """ -for i in xrange(10000): - noddy = Noddy() - for j in range(20): - noddy.fields["key"+str(j)] = "value "+str(j) - noddy.save(safe=False, validate=False, cascade=False) - -myNoddys = Noddy.objects() -[n for n in myNoddys] # iterate -""" - - print "-" * 100 - print """Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False""" + print """Creating 10000 dictionaries without continual assign - MongoEngine""" t = timeit.Timer(stmt=stmt, setup=setup) print t.timeit(1) @@ -184,16 +218,65 @@ for i in xrange(10000): noddy = Noddy() for j in range(20): noddy.fields["key"+str(j)] = "value "+str(j) - noddy.save(force_insert=True, safe=False, validate=False, cascade=False) + noddy.save(write_concern={"w": 0}, cascade=True) myNoddys = Noddy.objects() [n for n in myNoddys] # iterate """ print "-" * 100 - print """Creating 10000 dictionaries - MongoEngine, force=True""" + print """Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True""" t = timeit.Timer(stmt=stmt, setup=setup) print t.timeit(1) + stmt = """ +for i in xrange(10000): + noddy = Noddy() + for j in range(20): + noddy.fields["key"+str(j)] = "value "+str(j) + noddy.save(write_concern={"w": 0}, validate=False, cascade=True) + +myNoddys = Noddy.objects() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True""" + t = timeit.Timer(stmt=stmt, setup=setup) + print t.timeit(1) + + stmt = """ +for i in xrange(10000): + noddy = Noddy() + for j in range(20): + noddy.fields["key"+str(j)] = "value "+str(j) + noddy.save(validate=False, write_concern={"w": 0}) + +myNoddys = Noddy.objects() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False""" + t = timeit.Timer(stmt=stmt, setup=setup) + print t.timeit(1) + + stmt = """ +for i in xrange(10000): + noddy = Noddy() + for j in range(20): + noddy.fields["key"+str(j)] = "value "+str(j) + noddy.save(force_insert=True, write_concern={"w": 0}, validate=False) + +myNoddys = Noddy.objects() +[n for n in myNoddys] # iterate +""" + + print "-" * 100 + print """Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False""" + t = timeit.Timer(stmt=stmt, setup=setup) + print t.timeit(1) + + if __name__ == "__main__": - main() + main() \ No newline at end of file From b0c1ec04b5f39be0f08d2765e1070c7ed0652d60 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 29 Apr 2013 07:38:31 +0000 Subject: [PATCH 1066/1279] Improvements to indexing documentation (#130) --- docs/guide/defining-documents.rst | 20 ++++++++++++++++++++ mongoengine/fields.py | 6 +++--- tests/document/indexes.py | 11 +++++------ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 36e0efe..c404101 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -479,6 +479,10 @@ If a dictionary is passed then the following options are available: :attr:`unique` (Default: False) Whether the index should be unique. +:attr:`expireAfterSeconds` (Optional) + Allows you to automatically expire data from a collection by setting the + time in seconds to expire the a field. + .. note:: Inheritance adds extra fields indices see: :ref:`document-inheritance`. @@ -512,6 +516,22 @@ point. To create a geospatial index you must prefix the field with the ], } +Time To Live indexes +-------------------- + +A special index type that allows you to automatically expire data from a +collection after a given period. See the official +`ttl `_ +documentation for more information. A common usecase might be session data:: + + class Session(Document): + created = DateTimeField(default=datetime.now) + meta = { + 'indexes': [ + {'fields': ['created'], 'expireAfterSeconds': 3600} + ] + } + Ordering ======== A default ordering can be specified for your diff --git a/mongoengine/fields.py b/mongoengine/fields.py index cf2c802..bb2539c 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -8,6 +8,7 @@ import uuid import warnings from operator import itemgetter +import pymongo import gridfs from bson import Binary, DBRef, SON, ObjectId @@ -37,7 +38,6 @@ __all__ = ['StringField', 'URLField', 'EmailField', 'IntField', 'LongField', 'SequenceField', 'UUIDField'] - RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -1392,7 +1392,7 @@ class GeoPointField(BaseField): .. versionadded:: 0.4 """ - _geo_index = True + _geo_index = pymongo.GEO2D def validate(self, value): """Make sure that a geo-value is of type (x, y) @@ -1404,7 +1404,7 @@ class GeoPointField(BaseField): if not len(value) == 2: self.error('Value must be a two-dimensional point') if (not isinstance(value[0], (float, int)) and - not isinstance(value[1], (float, int))): + not isinstance(value[1], (float, int))): self.error('Both values in point must be float or int') diff --git a/tests/document/indexes.py b/tests/document/indexes.py index 61e3c0e..99aeca6 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -217,7 +217,7 @@ class IndexesTest(unittest.TestCase): } self.assertEqual([{'fields': [('location.point', '2d')]}], - Place._meta['index_specs']) + Place._meta['index_specs']) Place.ensure_indexes() info = Place._get_collection().index_information() @@ -231,8 +231,7 @@ class IndexesTest(unittest.TestCase): location = DictField() class Place(Document): - current = DictField( - field=EmbeddedDocumentField('EmbeddedLocation')) + current = DictField(field=EmbeddedDocumentField('EmbeddedLocation')) meta = { 'allow_inheritance': True, 'indexes': [ @@ -241,7 +240,7 @@ class IndexesTest(unittest.TestCase): } self.assertEqual([{'fields': [('current.location.point', '2d')]}], - Place._meta['index_specs']) + Place._meta['index_specs']) Place.ensure_indexes() info = Place._get_collection().index_information() @@ -264,7 +263,7 @@ class IndexesTest(unittest.TestCase): self.assertEqual([{'fields': [('addDate', -1)], 'unique': True, 'sparse': True}], - BlogPost._meta['index_specs']) + BlogPost._meta['index_specs']) BlogPost.drop_collection() @@ -633,7 +632,7 @@ class IndexesTest(unittest.TestCase): list(Log.objects) info = Log.objects._collection.index_information() self.assertEqual(3600, - info['created_1']['expireAfterSeconds']) + info['created_1']['expireAfterSeconds']) def test_unique_and_indexes(self): """Ensure that 'unique' constraints aren't overridden by From 5d7444c115c043ed0a262f03193fd2f99dd55f1d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 29 Apr 2013 09:38:21 +0000 Subject: [PATCH 1067/1279] Ensure as_pymongo() and to_json honour only() and exclude() (#293) --- docs/changelog.rst | 1 + mongoengine/queryset/queryset.py | 15 +++++++++++---- tests/queryset/queryset.py | 22 ++++++++++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index f786c1d..699c5a7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Ensure as_pymongo() and to_json honour only() and exclude() (#293) - Document serialization uses field order to ensure a strict order is set (#296) - DecimalField now stores as float not string (#289) - UUIDField now stores as a binary by default (#292) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 65d6553..5ae889c 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -822,8 +822,7 @@ class QuerySet(object): def to_json(self): """Converts a queryset to JSON""" - queryset = self.clone() - return json_util.dumps(queryset._collection_obj.find(queryset._query)) + return json_util.dumps(self.as_pymongo()) def from_json(self, json_data): """Converts json data to unsaved objects""" @@ -1095,7 +1094,7 @@ class QuerySet(object): raise StopIteration if self._scalar: return self._get_scalar(self._document._from_son( - self._cursor.next())) + self._cursor.next())) if self._as_pymongo: return self._get_as_pymongo(self._cursor.next()) @@ -1370,7 +1369,15 @@ class QuerySet(object): new_data = {} for key, value in data.iteritems(): new_path = '%s.%s' % (path, key) if path else key - if all_fields or new_path in self.__as_pymongo_fields: + + if all_fields: + include_field = True + elif self._loaded_fields.value == QueryFieldList.ONLY: + include_field = new_path in self.__as_pymongo_fields + else: + include_field = new_path not in self.__as_pymongo_fields + + if include_field: new_data[key] = clean(value, path=new_path) data = new_data elif isinstance(data, list): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 5e403c4..5bf8183 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3276,6 +3276,28 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(results[1]['name'], 'Barack Obama') self.assertEqual(results[1]['price'], Decimal('2.22')) + def test_as_pymongo_json_limit_fields(self): + + class User(Document): + email = EmailField(unique=True, required=True) + password_hash = StringField(db_field='password_hash', required=True) + password_salt = StringField(db_field='password_salt', required=True) + + User.drop_collection() + User(email="ross@example.com", password_salt="SomeSalt", password_hash="SomeHash").save() + + serialized_user = User.objects.exclude('password_salt', 'password_hash').as_pymongo()[0] + self.assertEqual(set(['_id', 'email']), set(serialized_user.keys())) + + serialized_user = User.objects.exclude('id', 'password_salt', 'password_hash').to_json() + self.assertEqual('[{"email": "ross@example.com"}]', serialized_user) + + serialized_user = User.objects.exclude('password_salt').only('email').as_pymongo()[0] + self.assertEqual(set(['email']), set(serialized_user.keys())) + + serialized_user = User.objects.exclude('password_salt').only('email').to_json() + self.assertEqual('[{"email": "ross@example.com"}]', serialized_user) + def test_no_dereference(self): class Organization(Document): From 85b81fb12a3e6fd4a1129602c433ce381d45e925 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 29 Apr 2013 10:36:11 +0000 Subject: [PATCH 1068/1279] If values cant be compared mark as changed (#287) --- docs/changelog.rst | 1 + mongoengine/base/fields.py | 17 ++++++++++------- tests/fields/fields.py | 21 +++++++++++++++++++++ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 699c5a7..ffe94d1 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- If values cant be compared mark as changed (#287) - Ensure as_pymongo() and to_json honour only() and exclude() (#293) - Document serialization uses field order to ensure a strict order is set (#296) - DecimalField now stores as float not string (#289) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 3929a3a..d9ed278 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -81,13 +81,16 @@ class BaseField(object): def __set__(self, instance, value): """Descriptor for assigning a value to a field in a document. """ - changed = False - if (self.name not in instance._data or - instance._data[self.name] != value): - changed = True - instance._data[self.name] = value - if changed and instance._initialised: - instance._mark_as_changed(self.name) + if instance._initialised: + try: + if (self.name not in instance._data or + instance._data[self.name] != value): + instance._mark_as_changed(self.name) + except: + # Values cant be compared eg: naive and tz datetimes + # So mark it as changed + instance._mark_as_changed(self.name) + instance._data[self.name] = value def error(self, message="", errors=None, field_name=None): """Raises a ValidationError. diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 4fa6989..5474aa6 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -409,6 +409,27 @@ class FieldTest(unittest.TestCase): log.time = '1pm' self.assertRaises(ValidationError, log.validate) + def test_datetime_tz_aware_mark_as_changed(self): + from mongoengine import connection + + # Reset the connections + connection._connection_settings = {} + connection._connections = {} + connection._dbs = {} + + connect(db='mongoenginetest', tz_aware=True) + + class LogEntry(Document): + time = DateTimeField() + + LogEntry.drop_collection() + + LogEntry(time=datetime.datetime(2013, 1, 1, 0, 0, 0)).save() + + log = LogEntry.objects.first() + log.time = datetime.datetime(2013, 1, 1, 0, 0, 0) + self.assertEqual(['time'], log._changed_fields) + def test_datetime(self): """Tests showing pymongo datetime fields handling of microseconds. Microseconds are rounded to the nearest millisecond and pre UTC From 9c1cd81adb4d240b9783ce80cef275858b96d5ca Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 30 Apr 2013 14:46:23 +0000 Subject: [PATCH 1069/1279] Add support for new geojson fields, indexes and queries (#299) --- docs/apireference.rst | 5 +- docs/changelog.rst | 1 + docs/conf.py | 6 +- docs/django.rst | 8 +- docs/guide/defining-documents.rst | 29 +++ docs/guide/querying.rst | 72 ++++- docs/index.rst | 4 +- mongoengine/base/document.py | 24 +- mongoengine/base/fields.py | 112 +++++++- mongoengine/common.py | 3 +- mongoengine/document.py | 1 - mongoengine/fields.py | 108 ++++++-- mongoengine/queryset/queryset.py | 7 +- mongoengine/queryset/transform.py | 98 +++++-- tests/document/indexes.py | 28 +- tests/fields/__init__.py | 3 +- tests/fields/fields.py | 39 --- tests/fields/geo.py | 274 ++++++++++++++++++++ tests/queryset/__init__.py | 4 +- tests/queryset/geo.py | 418 ++++++++++++++++++++++++++++++ tests/queryset/queryset.py | 161 ------------ 21 files changed, 1101 insertions(+), 304 deletions(-) create mode 100644 tests/fields/geo.py create mode 100644 tests/queryset/geo.py diff --git a/docs/apireference.rst b/docs/apireference.rst index 3a15629..37370e2 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -76,10 +76,13 @@ Fields .. autoclass:: mongoengine.fields.BinaryField .. autoclass:: mongoengine.fields.FileField .. autoclass:: mongoengine.fields.ImageField -.. autoclass:: mongoengine.fields.GeoPointField .. autoclass:: mongoengine.fields.SequenceField .. autoclass:: mongoengine.fields.ObjectIdField .. autoclass:: mongoengine.fields.UUIDField +.. autoclass:: mongoengine.fields.GeoPointField +.. autoclass:: mongoengine.fields.PointField +.. autoclass:: mongoengine.fields.LineStringField +.. autoclass:: mongoengine.fields.PolygonField .. autoclass:: mongoengine.fields.GridFSError .. autoclass:: mongoengine.fields.GridFSProxy .. autoclass:: mongoengine.fields.ImageGridFsProxy diff --git a/docs/changelog.rst b/docs/changelog.rst index ffe94d1..207f0dd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Add support for new geojson fields, indexes and queries (#299) - If values cant be compared mark as changed (#287) - Ensure as_pymongo() and to_json honour only() and exclude() (#293) - Document serialization uses field order to ensure a strict order is set (#296) diff --git a/docs/conf.py b/docs/conf.py index 8bcb9ec..40c1f43 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -132,7 +132,11 @@ html_theme_path = ['_themes'] html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +html_sidebars = { + 'index': ['globaltoc.html', 'searchbox.html'], + '**': ['localtoc.html', 'relations.html', 'searchbox.html'] +} + # Additional templates that should be rendered to pages, maps page names to # template names. diff --git a/docs/django.rst b/docs/django.rst index d60e55d..09c91e7 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -1,8 +1,8 @@ -============================= -Using MongoEngine with Django -============================= +============== +Django Support +============== -.. note:: Updated to support Django 1.4 +.. note:: Updated to support Django 1.5 Connecting ========== diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index c404101..2c744b7 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -499,6 +499,35 @@ in this case use 'dot' notation to identify the value to index eg: `rank.title` Geospatial indexes ------------------ + +The best geo index for mongodb is the new "2dsphere", which has an improved +spherical model and provides better performance and more options when querying. +The following fields will explicitly add a "2dsphere" index: + + - :class:`~mongoengine.fields.PointField` + - :class:`~mongoengine.fields.LineStringField` + - :class:`~mongoengine.fields.PolygonField` + +As "2dsphere" indexes can be part of a compound index, you may not want the +automatic index but would prefer a compound index. In this example we turn off +auto indexing and explicitly declare a compound index on ``location`` and ``datetime``:: + + class Log(Document): + location = PointField(auto_index=False) + datetime = DateTimeField() + + meta = { + 'indexes': [[("location", "2dsphere"), ("datetime", 1)]] + } + + +Pre MongoDB 2.4 Geo +''''''''''''''''''' + +.. note:: For MongoDB < 2.4 this is still current, however the new 2dsphere + index is a big improvement over the previous 2D model - so upgrading is + advised. + Geospatial indexes will be automatically created for all :class:`~mongoengine.fields.GeoPointField`\ s diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 3a25c28..f1b6470 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -65,6 +65,9 @@ Available operators are as follows: * ``size`` -- the size of the array is * ``exists`` -- value for field exists +String queries +-------------- + The following operators are available as shortcuts to querying with regular expressions: @@ -78,8 +81,71 @@ expressions: * ``iendswith`` -- string field ends with value (case insensitive) * ``match`` -- performs an $elemMatch so you can match an entire document within an array -There are a few special operators for performing geographical queries, that -may used with :class:`~mongoengine.fields.GeoPointField`\ s: + +Geo queries +----------- + +There are a few special operators for performing geographical queries. The following +were added in 0.8 for: :class:`~mongoengine.fields.PointField`, +:class:`~mongoengine.fields.LineStringField` and +:class:`~mongoengine.fields.PolygonField`: + +* ``geo_within`` -- Check if a geometry is within a polygon. For ease of use + it accepts either a geojson geometry or just the polygon coordinates eg:: + + loc.objects(point__geo_with=[[[40, 5], [40, 6], [41, 6], [40, 5]]]) + loc.objects(point__geo_with={"type": "Polygon", + "coordinates": [[[40, 5], [40, 6], [41, 6], [40, 5]]]}) + +* ``geo_within_box`` - simplified geo_within searching with a box eg:: + + loc.objects(point__geo_within_box=[(-125.0, 35.0), (-100.0, 40.0)]) + loc.objects(point__geo_within_box=[, ]) + +* ``geo_within_polygon`` -- simplified geo_within searching within a simple polygon eg:: + + loc.objects(point__geo_within_polygon=[[40, 5], [40, 6], [41, 6], [40, 5]]) + loc.objects(point__geo_within_polygon=[ [ , ] , + [ , ] , + [ , ] ]) + +* ``geo_within_center`` -- simplified geo_within the flat circle radius of a point eg:: + + loc.objects(point__geo_within_center=[(-125.0, 35.0), 1]) + loc.objects(point__geo_within_center=[ [ , ] , ]) + +* ``geo_within_sphere`` -- simplified geo_within the spherical circle radius of a point eg:: + + loc.objects(point__geo_within_sphere=[(-125.0, 35.0), 1]) + loc.objects(point__geo_within_sphere=[ [ , ] , ]) + +* ``geo_intersects`` -- selects all locations that intersect with a geometry eg:: + + # Inferred from provided points lists: + loc.objects(poly__geo_intersects=[40, 6]) + loc.objects(poly__geo_intersects=[[40, 5], [40, 6]]) + loc.objects(poly__geo_intersects=[[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]]) + + # With geoJson style objects + loc.objects(poly__geo_intersects={"type": "Point", "coordinates": [40, 6]}) + loc.objects(poly__geo_intersects={"type": "LineString", + "coordinates": [[40, 5], [40, 6]]}) + loc.objects(poly__geo_intersects={"type": "Polygon", + "coordinates": [[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]]}) + +* ``near`` -- Find all the locations near a given point:: + + loc.objects(point__near=[40, 5]) + loc.objects(point__near={"type": "Point", "coordinates": [40, 5]}) + + + You can also set the maximum distance in meters as well:: + + loc.objects(point__near=[40, 5], point__max_distance=1000) + + +The older 2D indexes are still supported with the +:class:`~mongoengine.fields.GeoPointField`: * ``within_distance`` -- provide a list containing a point and a maximum distance (e.g. [(41.342, -87.653), 5]) @@ -91,7 +157,9 @@ may used with :class:`~mongoengine.fields.GeoPointField`\ s: [(35.0, -125.0), (40.0, -100.0)]) * ``within_polygon`` -- filter documents to those within a given polygon (e.g. [(41.91,-87.69), (41.92,-87.68), (41.91,-87.65), (41.89,-87.65)]). + .. note:: Requires Mongo Server 2.0 + * ``max_distance`` -- can be added to your location queries to set a maximum distance. diff --git a/docs/index.rst b/docs/index.rst index 4aca82d..6358a31 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -56,14 +56,16 @@ See the :doc:`changelog` for a full list of changes to MongoEngine and putting updates live in production **;)** .. toctree:: + :maxdepth: 1 + :numbered: :hidden: tutorial guide/index apireference - django changelog upgrade + django Indices and tables ------------------ diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 53686b2..c2ccc48 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -662,7 +662,8 @@ class BaseDocument(object): if include_cls and direction is not pymongo.GEO2D: index_list.insert(0, ('_cls', 1)) - spec['fields'] = index_list + if index_list: + spec['fields'] = index_list if spec.get('sparse', False) and len(spec['fields']) > 1: raise ValueError( 'Sparse indexes can only have one field in them. ' @@ -704,13 +705,13 @@ class BaseDocument(object): # Add the new index to the list fields = [("%s%s" % (namespace, f), pymongo.ASCENDING) - for f in unique_fields] + for f in unique_fields] index = {'fields': fields, 'unique': True, 'sparse': sparse} unique_indexes.append(index) # Grab any embedded document field unique indexes if (field.__class__.__name__ == "EmbeddedDocumentField" and - field.document_type != cls): + field.document_type != cls): field_namespace = "%s." % field_name doc_cls = field.document_type unique_indexes += doc_cls._unique_with_indexes(field_namespace) @@ -718,26 +719,31 @@ class BaseDocument(object): return unique_indexes @classmethod - def _geo_indices(cls, inspected=None): + def _geo_indices(cls, inspected=None, parent_field=None): inspected = inspected or [] geo_indices = [] inspected.append(cls) - EmbeddedDocumentField = _import_class("EmbeddedDocumentField") - GeoPointField = _import_class("GeoPointField") + geo_field_type_names = ["EmbeddedDocumentField", "GeoPointField", + "PointField", "LineStringField", "PolygonField"] + + geo_field_types = tuple([_import_class(field) for field in geo_field_type_names]) for field in cls._fields.values(): - if not isinstance(field, (EmbeddedDocumentField, GeoPointField)): + if not isinstance(field, geo_field_types): continue if hasattr(field, 'document_type'): field_cls = field.document_type if field_cls in inspected: continue if hasattr(field_cls, '_geo_indices'): - geo_indices += field_cls._geo_indices(inspected) + geo_indices += field_cls._geo_indices(inspected, parent_field=field.db_field) elif field._geo_index: + field_name = field.db_field + if parent_field: + field_name = "%s.%s" % (parent_field, field_name) geo_indices.append({'fields': - [(field.db_field, pymongo.GEO2D)]}) + [(field_name, field._geo_index)]}) return geo_indices @classmethod diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index d9ed278..fa0b134 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -2,7 +2,8 @@ import operator import warnings import weakref -from bson import DBRef, ObjectId +from bson import DBRef, ObjectId, SON +import pymongo from mongoengine.common import _import_class from mongoengine.errors import ValidationError @@ -10,7 +11,7 @@ from mongoengine.errors import ValidationError from mongoengine.base.common import ALLOW_INHERITANCE from mongoengine.base.datastructures import BaseDict, BaseList -__all__ = ("BaseField", "ComplexBaseField", "ObjectIdField") +__all__ = ("BaseField", "ComplexBaseField", "ObjectIdField", "GeoJsonBaseField") class BaseField(object): @@ -186,7 +187,7 @@ class ComplexBaseField(BaseField): # Convert lists / values so we can watch for any changes on them if (isinstance(value, (list, tuple)) and - not isinstance(value, BaseList)): + not isinstance(value, BaseList)): value = BaseList(value, instance, self.name) instance._data[self.name] = value elif isinstance(value, dict) and not isinstance(value, BaseDict): @@ -194,8 +195,8 @@ class ComplexBaseField(BaseField): instance._data[self.name] = value if (self._auto_dereference and instance._initialised and - isinstance(value, (BaseList, BaseDict)) - and not value._dereferenced): + isinstance(value, (BaseList, BaseDict)) + and not value._dereferenced): value = self._dereference( value, max_depth=1, instance=instance, name=self.name ) @@ -231,7 +232,7 @@ class ComplexBaseField(BaseField): if self.field: value_dict = dict([(key, self.field.to_python(item)) - for key, item in value.items()]) + for key, item in value.items()]) else: value_dict = {} for k, v in value.items(): @@ -282,7 +283,7 @@ class ComplexBaseField(BaseField): if self.field: value_dict = dict([(key, self.field.to_mongo(item)) - for key, item in value.iteritems()]) + for key, item in value.iteritems()]) else: value_dict = {} for k, v in value.iteritems(): @@ -396,3 +397,100 @@ class ObjectIdField(BaseField): ObjectId(unicode(value)) except: self.error('Invalid Object ID') + + +class GeoJsonBaseField(BaseField): + """A geo json field storing a geojson style object. + .. versionadded:: 0.8 + """ + + _geo_index = pymongo.GEOSPHERE + _type = "GeoBase" + + def __init__(self, auto_index=True, *args, **kwargs): + """ + :param auto_index: Automatically create a "2dsphere" index. Defaults + to `True`. + """ + self._name = "%sField" % self._type + if not auto_index: + self._geo_index = False + super(GeoJsonBaseField, self).__init__(*args, **kwargs) + + def validate(self, value): + """Validate the GeoJson object based on its type + """ + if isinstance(value, dict): + if set(value.keys()) == set(['type', 'coordinates']): + if value['type'] != self._type: + self.error('%s type must be "%s"' % (self._name, self._type)) + return self.validate(value['coordinates']) + else: + self.error('%s can only accept a valid GeoJson dictionary' + ' or lists of (x, y)' % self._name) + return + elif not isinstance(value, (list, tuple)): + self.error('%s can only accept lists of [x, y]' % self._name) + return + + validate = getattr(self, "_validate_%s" % self._type.lower()) + error = validate(value) + if error: + self.error(error) + + def _validate_polygon(self, value): + if not isinstance(value, (list, tuple)): + return 'Polygons must contain list of linestrings' + + # Quick and dirty validator + try: + value[0][0][0] + except: + return "Invalid Polygon must contain at least one valid linestring" + + errors = [] + for val in value: + error = self._validate_linestring(val, False) + if not error and val[0] != val[-1]: + error = 'LineStrings must start and end at the same point' + if error and error not in errors: + errors.append(error) + if errors: + return "Invalid Polygon:\n%s" % ", ".join(set(errors)) + + def _validate_linestring(self, value, top_level=True): + """Validates a linestring""" + if not isinstance(value, (list, tuple)): + return 'LineStrings must contain list of coordinate pairs' + + # Quick and dirty validator + try: + value[0][0] + except: + return "Invalid LineString must contain at least one valid point" + + errors = [] + for val in value: + error = self._validate_point(val) + if error and error not in errors: + errors.append(error) + if errors: + if top_level: + return "Invalid LineString:\n%s" % ", ".join(errors) + else: + return "%s" % ", ".join(set(errors)) + + def _validate_point(self, value): + """Validate each set of coords""" + if not isinstance(value, (list, tuple)): + return 'Points must be a list of coordinate pairs' + elif not len(value) == 2: + return "Value (%s) must be a two-dimensional point" % repr(value) + elif (not isinstance(value[0], (float, int)) or + not isinstance(value[1], (float, int))): + return "Both values (%s) in point must be float or int" % repr(value) + + def to_mongo(self, value): + if isinstance(value, dict): + return value + return SON([("type", self._type), ("coordinates", value)]) diff --git a/mongoengine/common.py b/mongoengine/common.py index 718ac0b..bff55ac 100644 --- a/mongoengine/common.py +++ b/mongoengine/common.py @@ -11,6 +11,7 @@ def _import_class(cls_name): field_classes = ('DictField', 'DynamicField', 'EmbeddedDocumentField', 'FileField', 'GenericReferenceField', 'GenericEmbeddedDocumentField', 'GeoPointField', + 'PointField', 'LineStringField', 'PolygonField', 'ReferenceField', 'StringField', 'ComplexBaseField') queryset_classes = ('OperationError',) deref_classes = ('DeReference',) @@ -33,4 +34,4 @@ def _import_class(cls_name): for cls in import_classes: _class_registry_cache[cls] = getattr(module, cls) - return _class_registry_cache.get(cls_name) \ No newline at end of file + return _class_registry_cache.get(cls_name) diff --git a/mongoengine/document.py b/mongoengine/document.py index bd6ce19..143802c 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -523,7 +523,6 @@ class Document(BaseDocument): # an extra index on _cls, as mongodb will use the existing # index to service queries against _cls cls_indexed = False - def includes_cls(fields): first_field = None if len(fields): diff --git a/mongoengine/fields.py b/mongoengine/fields.py index bb2539c..274ad3c 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -15,7 +15,7 @@ from bson import Binary, DBRef, SON, ObjectId from mongoengine.errors import ValidationError from mongoengine.python_support import (PY3, bin_type, txt_type, str_types, StringIO) -from base import (BaseField, ComplexBaseField, ObjectIdField, +from base import (BaseField, ComplexBaseField, ObjectIdField, GeoJsonBaseField, get_document, BaseDocument) from queryset import DO_NOTHING, QuerySet from document import Document, EmbeddedDocument @@ -34,8 +34,8 @@ __all__ = ['StringField', 'URLField', 'EmailField', 'IntField', 'LongField', 'SortedListField', 'DictField', 'MapField', 'ReferenceField', 'GenericReferenceField', 'BinaryField', 'GridFSError', 'GridFSProxy', 'FileField', 'ImageGridFsProxy', - 'ImproperlyConfigured', 'ImageField', 'GeoPointField', - 'SequenceField', 'UUIDField'] + 'ImproperlyConfigured', 'ImageField', 'GeoPointField', 'PointField', + 'LineStringField', 'PolygonField', 'SequenceField', 'UUIDField'] RECURSIVE_REFERENCE_CONSTANT = 'self' @@ -1386,28 +1386,6 @@ class ImageField(FileField): **kwargs) -class GeoPointField(BaseField): - """A list storing a latitude and longitude. - - .. versionadded:: 0.4 - """ - - _geo_index = pymongo.GEO2D - - def validate(self, value): - """Make sure that a geo-value is of type (x, y) - """ - if not isinstance(value, (list, tuple)): - self.error('GeoPointField can only accept tuples or lists ' - 'of (x, y)') - - if not len(value) == 2: - self.error('Value must be a two-dimensional point') - if (not isinstance(value[0], (float, int)) and - not isinstance(value[1], (float, int))): - self.error('Both values in point must be float or int') - - class SequenceField(BaseField): """Provides a sequental counter see: http://www.mongodb.org/display/DOCS/Object+IDs#ObjectIDs-SequenceNumbers @@ -1548,3 +1526,83 @@ class UUIDField(BaseField): value = uuid.UUID(value) except Exception, exc: self.error('Could not convert to UUID: %s' % exc) + + +class GeoPointField(BaseField): + """A list storing a latitude and longitude. + + .. versionadded:: 0.4 + """ + + _geo_index = pymongo.GEO2D + + def validate(self, value): + """Make sure that a geo-value is of type (x, y) + """ + if not isinstance(value, (list, tuple)): + self.error('GeoPointField can only accept tuples or lists ' + 'of (x, y)') + + if not len(value) == 2: + self.error("Value (%s) must be a two-dimensional point" % repr(value)) + elif (not isinstance(value[0], (float, int)) or + not isinstance(value[1], (float, int))): + self.error("Both values (%s) in point must be float or int" % repr(value)) + + +class PointField(GeoJsonBaseField): + """A geo json field storing a latitude and longitude. + + The data is represented as: + + .. code-block:: js + + { "type" : "Point" , + "coordinates" : [x, y]} + + You can either pass a dict with the full information or a list + to set the value. + + Requires mongodb >= 2.4 + .. versionadded:: 0.8 + """ + _type = "Point" + + +class LineStringField(GeoJsonBaseField): + """A geo json field storing a line of latitude and longitude coordinates. + + The data is represented as: + + .. code-block:: js + + { "type" : "LineString" , + "coordinates" : [[x1, y1], [x1, y1] ... [xn, yn]]} + + You can either pass a dict with the full information or a list of points. + + Requires mongodb >= 2.4 + .. versionadded:: 0.8 + """ + _type = "LineString" + + +class PolygonField(GeoJsonBaseField): + """A geo json field storing a polygon of latitude and longitude coordinates. + + The data is represented as: + + .. code-block:: js + + { "type" : "Polygon" , + "coordinates" : [[[x1, y1], [x1, y1] ... [xn, yn]], + [[x1, y1], [x1, y1] ... [xn, yn]]} + + You can either pass a dict with the full information or a list + of LineStrings. The first LineString being the outside and the rest being + holes. + + Requires mongodb >= 2.4 + .. versionadded:: 0.8 + """ + _type = "Polygon" diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 5ae889c..bfb5a48 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1422,15 +1422,14 @@ class QuerySet(object): code = re.sub(u'\[\s*~([A-z_][A-z_0-9.]+?)\s*\]', field_sub, code) code = re.sub(u'\{\{\s*~([A-z_][A-z_0-9.]+?)\s*\}\}', field_path_sub, - code) + code) return code # Deprecated - def ensure_index(self, **kwargs): """Deprecated use :func:`~Document.ensure_index`""" msg = ("Doc.objects()._ensure_index() is deprecated. " - "Use Doc.ensure_index() instead.") + "Use Doc.ensure_index() instead.") warnings.warn(msg, DeprecationWarning) self._document.__class__.ensure_index(**kwargs) return self @@ -1438,6 +1437,6 @@ class QuerySet(object): def _ensure_indexes(self): """Deprecated use :func:`~Document.ensure_indexes`""" msg = ("Doc.objects()._ensure_indexes() is deprecated. " - "Use Doc.ensure_indexes() instead.") + "Use Doc.ensure_indexes() instead.") warnings.warn(msg, DeprecationWarning) self._document.__class__.ensure_indexes() diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 3da2693..96d9904 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -1,5 +1,6 @@ from collections import defaultdict +import pymongo from bson import SON from mongoengine.common import _import_class @@ -12,7 +13,9 @@ COMPARISON_OPERATORS = ('ne', 'gt', 'gte', 'lt', 'lte', 'in', 'nin', 'mod', 'all', 'size', 'exists', 'not') GEO_OPERATORS = ('within_distance', 'within_spherical_distance', 'within_box', 'within_polygon', 'near', 'near_sphere', - 'max_distance') + 'max_distance', 'geo_within', 'geo_within_box', + 'geo_within_polygon', 'geo_within_center', + 'geo_within_sphere', 'geo_intersects') STRING_OPERATORS = ('contains', 'icontains', 'startswith', 'istartswith', 'endswith', 'iendswith', 'exact', 'iexact') @@ -81,30 +84,14 @@ def query(_doc_cls=None, _field_operation=False, **query): value = field else: value = field.prepare_query_value(op, value) - elif op in ('in', 'nin', 'all', 'near'): + elif op in ('in', 'nin', 'all', 'near') and not isinstance(value, dict): # 'in', 'nin' and 'all' require a list of values value = [field.prepare_query_value(op, v) for v in value] # if op and op not in COMPARISON_OPERATORS: if op: if op in GEO_OPERATORS: - if op == "within_distance": - value = {'$within': {'$center': value}} - elif op == "within_spherical_distance": - value = {'$within': {'$centerSphere': value}} - elif op == "within_polygon": - value = {'$within': {'$polygon': value}} - elif op == "near": - value = {'$near': value} - elif op == "near_sphere": - value = {'$nearSphere': value} - elif op == 'within_box': - value = {'$within': {'$box': value}} - elif op == "max_distance": - value = {'$maxDistance': value} - else: - raise NotImplementedError("Geo method '%s' has not " - "been implemented" % op) + value = _geo_operator(field, op, value) elif op in CUSTOM_OPERATORS: if op == 'match': value = {"$elemMatch": value} @@ -250,3 +237,76 @@ def update(_doc_cls=None, **update): mongo_update[key].update(value) return mongo_update + + +def _geo_operator(field, op, value): + """Helper to return the query for a given geo query""" + if field._geo_index == pymongo.GEO2D: + if op == "within_distance": + value = {'$within': {'$center': value}} + elif op == "within_spherical_distance": + value = {'$within': {'$centerSphere': value}} + elif op == "within_polygon": + value = {'$within': {'$polygon': value}} + elif op == "near": + value = {'$near': value} + elif op == "near_sphere": + value = {'$nearSphere': value} + elif op == 'within_box': + value = {'$within': {'$box': value}} + elif op == "max_distance": + value = {'$maxDistance': value} + else: + raise NotImplementedError("Geo method '%s' has not " + "been implemented for a GeoPointField" % op) + else: + if op == "geo_within": + value = {"$geoWithin": _infer_geometry(value)} + elif op == "geo_within_box": + value = {"$geoWithin": {"$box": value}} + elif op == "geo_within_polygon": + value = {"$geoWithin": {"$polygon": value}} + elif op == "geo_within_center": + value = {"$geoWithin": {"$center": value}} + elif op == "geo_within_sphere": + value = {"$geoWithin": {"$centerSphere": value}} + elif op == "geo_intersects": + value = {"$geoIntersects": _infer_geometry(value)} + elif op == "near": + value = {'$near': _infer_geometry(value)} + elif op == "max_distance": + value = {'$maxDistance': value} + else: + raise NotImplementedError("Geo method '%s' has not " + "been implemented for a %s " % (op, field._name)) + return value + + +def _infer_geometry(value): + """Helper method that tries to infer the $geometry shape for a given value""" + if isinstance(value, dict): + if "$geometry" in value: + return value + elif 'coordinates' in value and 'type' in value: + return {"$geometry": value} + raise InvalidQueryError("Invalid $geometry dictionary should have " + "type and coordinates keys") + elif isinstance(value, (list, set)): + try: + value[0][0][0] + return {"$geometry": {"type": "Polygon", "coordinates": value}} + except: + pass + try: + value[0][0] + return {"$geometry": {"type": "LineString", "coordinates": value}} + except: + pass + try: + value[0] + return {"$geometry": {"type": "Point", "coordinates": value}} + except: + pass + + raise InvalidQueryError("Invalid $geometry data. Can be either a dictionary " + "or (nested) lists of coordinate(s)") diff --git a/tests/document/indexes.py b/tests/document/indexes.py index 99aeca6..ddc147b 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -381,8 +381,7 @@ class IndexesTest(unittest.TestCase): self.assertEqual(sorted(info.keys()), ['_id_', 'tags.tag_1']) post1 = BlogPost(title="Embedded Indexes tests in place", - tags=[Tag(name="about"), Tag(name="time")] - ) + tags=[Tag(name="about"), Tag(name="time")]) post1.save() BlogPost.drop_collection() @@ -399,29 +398,6 @@ class IndexesTest(unittest.TestCase): info = RecursiveDocument._get_collection().index_information() self.assertEqual(sorted(info.keys()), ['_cls_1', '_id_']) - def test_geo_indexes_recursion(self): - - class Location(Document): - name = StringField() - location = GeoPointField() - - class Parent(Document): - name = StringField() - location = ReferenceField(Location, dbref=False) - - Location.drop_collection() - Parent.drop_collection() - - list(Parent.objects) - - collection = Parent._get_collection() - info = collection.index_information() - - self.assertFalse('location_2d' in info) - - self.assertEqual(len(Parent._geo_indices()), 0) - self.assertEqual(len(Location._geo_indices()), 1) - def test_covered_index(self): """Ensure that covered indexes can be used """ @@ -432,7 +408,7 @@ class IndexesTest(unittest.TestCase): meta = { 'indexes': ['a'], 'allow_inheritance': False - } + } Test.drop_collection() diff --git a/tests/fields/__init__.py b/tests/fields/__init__.py index 0731838..be70aaa 100644 --- a/tests/fields/__init__.py +++ b/tests/fields/__init__.py @@ -1,2 +1,3 @@ from fields import * -from file_tests import * \ No newline at end of file +from file_tests import * +from geo import * \ No newline at end of file diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 5474aa6..f7ab63e 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -1862,45 +1862,6 @@ class FieldTest(unittest.TestCase): Shirt.drop_collection() - def test_geo_indexes(self): - """Ensure that indexes are created automatically for GeoPointFields. - """ - class Event(Document): - title = StringField() - location = GeoPointField() - - Event.drop_collection() - event = Event(title="Coltrane Motion @ Double Door", - location=[41.909889, -87.677137]) - event.save() - - info = Event.objects._collection.index_information() - self.assertTrue(u'location_2d' in info) - self.assertTrue(info[u'location_2d']['key'] == [(u'location', u'2d')]) - - Event.drop_collection() - - def test_geo_embedded_indexes(self): - """Ensure that indexes are created automatically for GeoPointFields on - embedded documents. - """ - class Venue(EmbeddedDocument): - location = GeoPointField() - name = StringField() - - class Event(Document): - title = StringField() - venue = EmbeddedDocumentField(Venue) - - Event.drop_collection() - venue = Venue(name="Double Door", location=[41.909889, -87.677137]) - event = Event(title="Coltrane Motion", venue=venue) - event.save() - - info = Event.objects._collection.index_information() - self.assertTrue(u'location_2d' in info) - self.assertTrue(info[u'location_2d']['key'] == [(u'location', u'2d')]) - def test_ensure_unique_default_instances(self): """Ensure that every field has it's own unique default instance.""" class D(Document): diff --git a/tests/fields/geo.py b/tests/fields/geo.py new file mode 100644 index 0000000..2936f72 --- /dev/null +++ b/tests/fields/geo.py @@ -0,0 +1,274 @@ +# -*- coding: utf-8 -*- +import sys +sys.path[0:0] = [""] + +import unittest + +from mongoengine import * +from mongoengine.connection import get_db + +__all__ = ("GeoFieldTest", ) + + +class GeoFieldTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + self.db = get_db() + + def _test_for_expected_error(self, Cls, loc, expected): + try: + Cls(loc=loc).validate() + self.fail() + except ValidationError, e: + self.assertEqual(expected, e.to_dict()['loc']) + + def test_geopoint_validation(self): + class Location(Document): + loc = GeoPointField() + + invalid_coords = [{"x": 1, "y": 2}, 5, "a"] + expected = 'GeoPointField can only accept tuples or lists of (x, y)' + + for coord in invalid_coords: + self._test_for_expected_error(Location, coord, expected) + + invalid_coords = [[], [1], [1, 2, 3]] + for coord in invalid_coords: + expected = "Value (%s) must be a two-dimensional point" % repr(coord) + self._test_for_expected_error(Location, coord, expected) + + invalid_coords = [[{}, {}], ("a", "b")] + for coord in invalid_coords: + expected = "Both values (%s) in point must be float or int" % repr(coord) + self._test_for_expected_error(Location, coord, expected) + + def test_point_validation(self): + class Location(Document): + loc = PointField() + + invalid_coords = {"x": 1, "y": 2} + expected = 'PointField can only accept a valid GeoJson dictionary or lists of (x, y)' + self._test_for_expected_error(Location, invalid_coords, expected) + + invalid_coords = {"type": "MadeUp", "coordinates": []} + expected = 'PointField type must be "Point"' + self._test_for_expected_error(Location, invalid_coords, expected) + + invalid_coords = {"type": "Point", "coordinates": [1, 2, 3]} + expected = "Value ([1, 2, 3]) must be a two-dimensional point" + self._test_for_expected_error(Location, invalid_coords, expected) + + invalid_coords = [5, "a"] + expected = "PointField can only accept lists of [x, y]" + for coord in invalid_coords: + self._test_for_expected_error(Location, coord, expected) + + invalid_coords = [[], [1], [1, 2, 3]] + for coord in invalid_coords: + expected = "Value (%s) must be a two-dimensional point" % repr(coord) + self._test_for_expected_error(Location, coord, expected) + + invalid_coords = [[{}, {}], ("a", "b")] + for coord in invalid_coords: + expected = "Both values (%s) in point must be float or int" % repr(coord) + self._test_for_expected_error(Location, coord, expected) + + Location(loc=[1, 2]).validate() + + def test_linestring_validation(self): + class Location(Document): + loc = LineStringField() + + invalid_coords = {"x": 1, "y": 2} + expected = 'LineStringField can only accept a valid GeoJson dictionary or lists of (x, y)' + self._test_for_expected_error(Location, invalid_coords, expected) + + invalid_coords = {"type": "MadeUp", "coordinates": [[]]} + expected = 'LineStringField type must be "LineString"' + self._test_for_expected_error(Location, invalid_coords, expected) + + invalid_coords = {"type": "LineString", "coordinates": [[1, 2, 3]]} + expected = "Invalid LineString:\nValue ([1, 2, 3]) must be a two-dimensional point" + self._test_for_expected_error(Location, invalid_coords, expected) + + invalid_coords = [5, "a"] + expected = "Invalid LineString must contain at least one valid point" + self._test_for_expected_error(Location, invalid_coords, expected) + + invalid_coords = [[1]] + expected = "Invalid LineString:\nValue (%s) must be a two-dimensional point" % repr(invalid_coords[0]) + self._test_for_expected_error(Location, invalid_coords, expected) + + invalid_coords = [[1, 2, 3]] + expected = "Invalid LineString:\nValue (%s) must be a two-dimensional point" % repr(invalid_coords[0]) + self._test_for_expected_error(Location, invalid_coords, expected) + + invalid_coords = [[[{}, {}]], [("a", "b")]] + for coord in invalid_coords: + expected = "Invalid LineString:\nBoth values (%s) in point must be float or int" % repr(coord[0]) + self._test_for_expected_error(Location, coord, expected) + + Location(loc=[[1, 2], [3, 4], [5, 6], [1,2]]).validate() + + def test_polygon_validation(self): + class Location(Document): + loc = PolygonField() + + invalid_coords = {"x": 1, "y": 2} + expected = 'PolygonField can only accept a valid GeoJson dictionary or lists of (x, y)' + self._test_for_expected_error(Location, invalid_coords, expected) + + invalid_coords = {"type": "MadeUp", "coordinates": [[]]} + expected = 'PolygonField type must be "Polygon"' + self._test_for_expected_error(Location, invalid_coords, expected) + + invalid_coords = {"type": "Polygon", "coordinates": [[[1, 2, 3]]]} + expected = "Invalid Polygon:\nValue ([1, 2, 3]) must be a two-dimensional point" + self._test_for_expected_error(Location, invalid_coords, expected) + + invalid_coords = [[[5, "a"]]] + expected = "Invalid Polygon:\nBoth values ([5, 'a']) in point must be float or int" + self._test_for_expected_error(Location, invalid_coords, expected) + + invalid_coords = [[[]]] + expected = "Invalid Polygon must contain at least one valid linestring" + self._test_for_expected_error(Location, invalid_coords, expected) + + invalid_coords = [[[1, 2, 3]]] + expected = "Invalid Polygon:\nValue ([1, 2, 3]) must be a two-dimensional point" + self._test_for_expected_error(Location, invalid_coords, expected) + + invalid_coords = [[[{}, {}]], [("a", "b")]] + expected = "Invalid Polygon:\nBoth values ([{}, {}]) in point must be float or int, Both values (('a', 'b')) in point must be float or int" + self._test_for_expected_error(Location, invalid_coords, expected) + + invalid_coords = [[[1, 2], [3, 4]]] + expected = "Invalid Polygon:\nLineStrings must start and end at the same point" + self._test_for_expected_error(Location, invalid_coords, expected) + + Location(loc=[[[1, 2], [3, 4], [5, 6], [1, 2]]]).validate() + + def test_indexes_geopoint(self): + """Ensure that indexes are created automatically for GeoPointFields. + """ + class Event(Document): + title = StringField() + location = GeoPointField() + + geo_indicies = Event._geo_indices() + self.assertEqual(geo_indicies, [{'fields': [('location', '2d')]}]) + + def test_geopoint_embedded_indexes(self): + """Ensure that indexes are created automatically for GeoPointFields on + embedded documents. + """ + class Venue(EmbeddedDocument): + location = GeoPointField() + name = StringField() + + class Event(Document): + title = StringField() + venue = EmbeddedDocumentField(Venue) + + geo_indicies = Event._geo_indices() + self.assertEqual(geo_indicies, [{'fields': [('venue.location', '2d')]}]) + + def test_indexes_2dsphere(self): + """Ensure that indexes are created automatically for GeoPointFields. + """ + class Event(Document): + title = StringField() + point = PointField() + line = LineStringField() + polygon = PolygonField() + + geo_indicies = Event._geo_indices() + self.assertEqual(geo_indicies, [{'fields': [('line', '2dsphere')]}, + {'fields': [('polygon', '2dsphere')]}, + {'fields': [('point', '2dsphere')]}]) + + def test_indexes_2dsphere_embedded(self): + """Ensure that indexes are created automatically for GeoPointFields. + """ + class Venue(EmbeddedDocument): + name = StringField() + point = PointField() + line = LineStringField() + polygon = PolygonField() + + class Event(Document): + title = StringField() + venue = EmbeddedDocumentField(Venue) + + geo_indicies = Event._geo_indices() + self.assertTrue({'fields': [('venue.line', '2dsphere')]} in geo_indicies) + self.assertTrue({'fields': [('venue.polygon', '2dsphere')]} in geo_indicies) + self.assertTrue({'fields': [('venue.point', '2dsphere')]} in geo_indicies) + + def test_geo_indexes_recursion(self): + + class Location(Document): + name = StringField() + location = GeoPointField() + + class Parent(Document): + name = StringField() + location = ReferenceField(Location) + + Location.drop_collection() + Parent.drop_collection() + + list(Parent.objects) + + collection = Parent._get_collection() + info = collection.index_information() + + self.assertFalse('location_2d' in info) + + self.assertEqual(len(Parent._geo_indices()), 0) + self.assertEqual(len(Location._geo_indices()), 1) + + def test_geo_indexes_auto_index(self): + + # Test just listing the fields + class Log(Document): + location = PointField(auto_index=False) + datetime = DateTimeField() + + meta = { + 'indexes': [[("location", "2dsphere"), ("datetime", 1)]] + } + + self.assertEqual([], Log._geo_indices()) + + Log.drop_collection() + Log.ensure_indexes() + + info = Log._get_collection().index_information() + self.assertEqual(info["location_2dsphere_datetime_1"]["key"], + [('location', '2dsphere'), ('datetime', 1)]) + + # Test listing explicitly + class Log(Document): + location = PointField(auto_index=False) + datetime = DateTimeField() + + meta = { + 'indexes': [ + {'fields': [("location", "2dsphere"), ("datetime", 1)]} + ] + } + + self.assertEqual([], Log._geo_indices()) + + Log.drop_collection() + Log.ensure_indexes() + + info = Log._get_collection().index_information() + self.assertEqual(info["location_2dsphere_datetime_1"]["key"], + [('location', '2dsphere'), ('datetime', 1)]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/queryset/__init__.py b/tests/queryset/__init__.py index 93cb8c2..8a93c19 100644 --- a/tests/queryset/__init__.py +++ b/tests/queryset/__init__.py @@ -1,5 +1,5 @@ - from transform import * from field_list import * from queryset import * -from visitor import * \ No newline at end of file +from visitor import * +from geo import * diff --git a/tests/queryset/geo.py b/tests/queryset/geo.py new file mode 100644 index 0000000..f564896 --- /dev/null +++ b/tests/queryset/geo.py @@ -0,0 +1,418 @@ +import sys +sys.path[0:0] = [""] + +import unittest +from datetime import datetime, timedelta +from mongoengine import * + +__all__ = ("GeoQueriesTest",) + + +class GeoQueriesTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + + def test_geospatial_operators(self): + """Ensure that geospatial queries are working. + """ + class Event(Document): + title = StringField() + date = DateTimeField() + location = GeoPointField() + + def __unicode__(self): + return self.title + + Event.drop_collection() + + event1 = Event(title="Coltrane Motion @ Double Door", + date=datetime.now() - timedelta(days=1), + location=[-87.677137, 41.909889]).save() + event2 = Event(title="Coltrane Motion @ Bottom of the Hill", + date=datetime.now() - timedelta(days=10), + location=[-122.4194155, 37.7749295]).save() + event3 = Event(title="Coltrane Motion @ Empty Bottle", + date=datetime.now(), + location=[-87.686638, 41.900474]).save() + + # find all events "near" pitchfork office, chicago. + # note that "near" will show the san francisco event, too, + # although it sorts to last. + events = Event.objects(location__near=[-87.67892, 41.9120459]) + self.assertEqual(events.count(), 3) + self.assertEqual(list(events), [event1, event3, event2]) + + # find events within 5 degrees of pitchfork office, chicago + point_and_distance = [[-87.67892, 41.9120459], 5] + events = Event.objects(location__within_distance=point_and_distance) + self.assertEqual(events.count(), 2) + events = list(events) + self.assertTrue(event2 not in events) + self.assertTrue(event1 in events) + self.assertTrue(event3 in events) + + # ensure ordering is respected by "near" + events = Event.objects(location__near=[-87.67892, 41.9120459]) + events = events.order_by("-date") + self.assertEqual(events.count(), 3) + self.assertEqual(list(events), [event3, event1, event2]) + + # find events within 10 degrees of san francisco + point = [-122.415579, 37.7566023] + events = Event.objects(location__near=point, location__max_distance=10) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0], event2) + + # find events within 10 degrees of san francisco + point_and_distance = [[-122.415579, 37.7566023], 10] + events = Event.objects(location__within_distance=point_and_distance) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0], event2) + + # find events within 1 degree of greenpoint, broolyn, nyc, ny + point_and_distance = [[-73.9509714, 40.7237134], 1] + events = Event.objects(location__within_distance=point_and_distance) + self.assertEqual(events.count(), 0) + + # ensure ordering is respected by "within_distance" + point_and_distance = [[-87.67892, 41.9120459], 10] + events = Event.objects(location__within_distance=point_and_distance) + events = events.order_by("-date") + self.assertEqual(events.count(), 2) + self.assertEqual(events[0], event3) + + # check that within_box works + box = [(-125.0, 35.0), (-100.0, 40.0)] + events = Event.objects(location__within_box=box) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0].id, event2.id) + + polygon = [ + (-87.694445, 41.912114), + (-87.69084, 41.919395), + (-87.681742, 41.927186), + (-87.654276, 41.911731), + (-87.656164, 41.898061), + ] + events = Event.objects(location__within_polygon=polygon) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0].id, event1.id) + + polygon2 = [ + (-1.742249, 54.033586), + (-1.225891, 52.792797), + (-4.40094, 53.389881) + ] + events = Event.objects(location__within_polygon=polygon2) + self.assertEqual(events.count(), 0) + + def test_geo_spatial_embedded(self): + + class Venue(EmbeddedDocument): + location = GeoPointField() + name = StringField() + + class Event(Document): + title = StringField() + venue = EmbeddedDocumentField(Venue) + + Event.drop_collection() + + venue1 = Venue(name="The Rock", location=[-87.677137, 41.909889]) + venue2 = Venue(name="The Bridge", location=[-122.4194155, 37.7749295]) + + event1 = Event(title="Coltrane Motion @ Double Door", + venue=venue1).save() + event2 = Event(title="Coltrane Motion @ Bottom of the Hill", + venue=venue2).save() + event3 = Event(title="Coltrane Motion @ Empty Bottle", + venue=venue1).save() + + # find all events "near" pitchfork office, chicago. + # note that "near" will show the san francisco event, too, + # although it sorts to last. + events = Event.objects(venue__location__near=[-87.67892, 41.9120459]) + self.assertEqual(events.count(), 3) + self.assertEqual(list(events), [event1, event3, event2]) + + def test_spherical_geospatial_operators(self): + """Ensure that spherical geospatial queries are working + """ + class Point(Document): + location = GeoPointField() + + Point.drop_collection() + + # These points are one degree apart, which (according to Google Maps) + # is about 110 km apart at this place on the Earth. + north_point = Point(location=[-122, 38]).save() # Near Concord, CA + south_point = Point(location=[-122, 37]).save() # Near Santa Cruz, CA + + earth_radius = 6378.009 # in km (needs to be a float for dividing by) + + # Finds both points because they are within 60 km of the reference + # point equidistant between them. + points = Point.objects(location__near_sphere=[-122, 37.5]) + self.assertEqual(points.count(), 2) + + # Same behavior for _within_spherical_distance + points = Point.objects( + location__within_spherical_distance=[[-122, 37.5], 60/earth_radius] + ) + self.assertEqual(points.count(), 2) + + points = Point.objects(location__near_sphere=[-122, 37.5], + location__max_distance=60 / earth_radius) + self.assertEqual(points.count(), 2) + + # Finds both points, but orders the north point first because it's + # closer to the reference point to the north. + points = Point.objects(location__near_sphere=[-122, 38.5]) + self.assertEqual(points.count(), 2) + self.assertEqual(points[0].id, north_point.id) + self.assertEqual(points[1].id, south_point.id) + + # Finds both points, but orders the south point first because it's + # closer to the reference point to the south. + points = Point.objects(location__near_sphere=[-122, 36.5]) + self.assertEqual(points.count(), 2) + self.assertEqual(points[0].id, south_point.id) + self.assertEqual(points[1].id, north_point.id) + + # Finds only one point because only the first point is within 60km of + # the reference point to the south. + points = Point.objects( + location__within_spherical_distance=[[-122, 36.5], 60/earth_radius]) + self.assertEqual(points.count(), 1) + self.assertEqual(points[0].id, south_point.id) + + def test_2dsphere_point(self): + + class Event(Document): + title = StringField() + date = DateTimeField() + location = PointField() + + def __unicode__(self): + return self.title + + Event.drop_collection() + + event1 = Event(title="Coltrane Motion @ Double Door", + date=datetime.now() - timedelta(days=1), + location=[-87.677137, 41.909889]) + event1.save() + event2 = Event(title="Coltrane Motion @ Bottom of the Hill", + date=datetime.now() - timedelta(days=10), + location=[-122.4194155, 37.7749295]).save() + event3 = Event(title="Coltrane Motion @ Empty Bottle", + date=datetime.now(), + location=[-87.686638, 41.900474]).save() + + # find all events "near" pitchfork office, chicago. + # note that "near" will show the san francisco event, too, + # although it sorts to last. + events = Event.objects(location__near=[-87.67892, 41.9120459]) + self.assertEqual(events.count(), 3) + self.assertEqual(list(events), [event1, event3, event2]) + + # find events within 5 degrees of pitchfork office, chicago + point_and_distance = [[-87.67892, 41.9120459], 2] + events = Event.objects(location__geo_within_center=point_and_distance) + self.assertEqual(events.count(), 2) + events = list(events) + self.assertTrue(event2 not in events) + self.assertTrue(event1 in events) + self.assertTrue(event3 in events) + + # ensure ordering is respected by "near" + events = Event.objects(location__near=[-87.67892, 41.9120459]) + events = events.order_by("-date") + self.assertEqual(events.count(), 3) + self.assertEqual(list(events), [event3, event1, event2]) + + # find events within 10km of san francisco + point = [-122.415579, 37.7566023] + events = Event.objects(location__near=point, location__max_distance=10000) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0], event2) + + # find events within 1km of greenpoint, broolyn, nyc, ny + events = Event.objects(location__near=[-73.9509714, 40.7237134], location__max_distance=1000) + self.assertEqual(events.count(), 0) + + # ensure ordering is respected by "near" + events = Event.objects(location__near=[-87.67892, 41.9120459], + location__max_distance=10000).order_by("-date") + self.assertEqual(events.count(), 2) + self.assertEqual(events[0], event3) + + # check that within_box works + box = [(-125.0, 35.0), (-100.0, 40.0)] + events = Event.objects(location__geo_within_box=box) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0].id, event2.id) + + polygon = [ + (-87.694445, 41.912114), + (-87.69084, 41.919395), + (-87.681742, 41.927186), + (-87.654276, 41.911731), + (-87.656164, 41.898061), + ] + events = Event.objects(location__geo_within_polygon=polygon) + self.assertEqual(events.count(), 1) + self.assertEqual(events[0].id, event1.id) + + polygon2 = [ + (-1.742249, 54.033586), + (-1.225891, 52.792797), + (-4.40094, 53.389881) + ] + events = Event.objects(location__geo_within_polygon=polygon2) + self.assertEqual(events.count(), 0) + + def test_2dsphere_point_embedded(self): + + class Venue(EmbeddedDocument): + location = GeoPointField() + name = StringField() + + class Event(Document): + title = StringField() + venue = EmbeddedDocumentField(Venue) + + Event.drop_collection() + + venue1 = Venue(name="The Rock", location=[-87.677137, 41.909889]) + venue2 = Venue(name="The Bridge", location=[-122.4194155, 37.7749295]) + + event1 = Event(title="Coltrane Motion @ Double Door", + venue=venue1).save() + event2 = Event(title="Coltrane Motion @ Bottom of the Hill", + venue=venue2).save() + event3 = Event(title="Coltrane Motion @ Empty Bottle", + venue=venue1).save() + + # find all events "near" pitchfork office, chicago. + # note that "near" will show the san francisco event, too, + # although it sorts to last. + events = Event.objects(venue__location__near=[-87.67892, 41.9120459]) + self.assertEqual(events.count(), 3) + self.assertEqual(list(events), [event1, event3, event2]) + + def test_linestring(self): + + class Road(Document): + name = StringField() + line = LineStringField() + + Road.drop_collection() + + Road(name="66", line=[[40, 5], [41, 6]]).save() + + # near + point = {"type": "Point", "coordinates": [40, 5]} + roads = Road.objects.filter(line__near=point["coordinates"]).count() + self.assertEqual(1, roads) + + roads = Road.objects.filter(line__near=point).count() + self.assertEqual(1, roads) + + roads = Road.objects.filter(line__near={"$geometry": point}).count() + self.assertEqual(1, roads) + + # Within + polygon = {"type": "Polygon", + "coordinates": [[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]]} + roads = Road.objects.filter(line__geo_within=polygon["coordinates"]).count() + self.assertEqual(1, roads) + + roads = Road.objects.filter(line__geo_within=polygon).count() + self.assertEqual(1, roads) + + roads = Road.objects.filter(line__geo_within={"$geometry": polygon}).count() + self.assertEqual(1, roads) + + # Intersects + line = {"type": "LineString", + "coordinates": [[40, 5], [40, 6]]} + roads = Road.objects.filter(line__geo_intersects=line["coordinates"]).count() + self.assertEqual(1, roads) + + roads = Road.objects.filter(line__geo_intersects=line).count() + self.assertEqual(1, roads) + + roads = Road.objects.filter(line__geo_intersects={"$geometry": line}).count() + self.assertEqual(1, roads) + + polygon = {"type": "Polygon", + "coordinates": [[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]]} + roads = Road.objects.filter(line__geo_intersects=polygon["coordinates"]).count() + self.assertEqual(1, roads) + + roads = Road.objects.filter(line__geo_intersects=polygon).count() + self.assertEqual(1, roads) + + roads = Road.objects.filter(line__geo_intersects={"$geometry": polygon}).count() + self.assertEqual(1, roads) + + def test_polygon(self): + + class Road(Document): + name = StringField() + poly = PolygonField() + + Road.drop_collection() + + Road(name="66", poly=[[[40, 5], [40, 6], [41, 6], [40, 5]]]).save() + + # near + point = {"type": "Point", "coordinates": [40, 5]} + roads = Road.objects.filter(poly__near=point["coordinates"]).count() + self.assertEqual(1, roads) + + roads = Road.objects.filter(poly__near=point).count() + self.assertEqual(1, roads) + + roads = Road.objects.filter(poly__near={"$geometry": point}).count() + self.assertEqual(1, roads) + + # Within + polygon = {"type": "Polygon", + "coordinates": [[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]]} + roads = Road.objects.filter(poly__geo_within=polygon["coordinates"]).count() + self.assertEqual(1, roads) + + roads = Road.objects.filter(poly__geo_within=polygon).count() + self.assertEqual(1, roads) + + roads = Road.objects.filter(poly__geo_within={"$geometry": polygon}).count() + self.assertEqual(1, roads) + + # Intersects + line = {"type": "LineString", + "coordinates": [[40, 5], [41, 6]]} + roads = Road.objects.filter(poly__geo_intersects=line["coordinates"]).count() + self.assertEqual(1, roads) + + roads = Road.objects.filter(poly__geo_intersects=line).count() + self.assertEqual(1, roads) + + roads = Road.objects.filter(poly__geo_intersects={"$geometry": line}).count() + self.assertEqual(1, roads) + + polygon = {"type": "Polygon", + "coordinates": [[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]]} + roads = Road.objects.filter(poly__geo_intersects=polygon["coordinates"]).count() + self.assertEqual(1, roads) + + roads = Road.objects.filter(poly__geo_intersects=polygon).count() + self.assertEqual(1, roads) + + roads = Road.objects.filter(poly__geo_intersects={"$geometry": polygon}).count() + self.assertEqual(1, roads) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 5bf8183..40aef7e 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -2380,167 +2380,6 @@ class QuerySetTest(unittest.TestCase): def tearDown(self): self.Person.drop_collection() - def test_geospatial_operators(self): - """Ensure that geospatial queries are working. - """ - class Event(Document): - title = StringField() - date = DateTimeField() - location = GeoPointField() - - def __unicode__(self): - return self.title - - Event.drop_collection() - - event1 = Event(title="Coltrane Motion @ Double Door", - date=datetime.now() - timedelta(days=1), - location=[41.909889, -87.677137]) - event2 = Event(title="Coltrane Motion @ Bottom of the Hill", - date=datetime.now() - timedelta(days=10), - location=[37.7749295, -122.4194155]) - event3 = Event(title="Coltrane Motion @ Empty Bottle", - date=datetime.now(), - location=[41.900474, -87.686638]) - - event1.save() - event2.save() - event3.save() - - # find all events "near" pitchfork office, chicago. - # note that "near" will show the san francisco event, too, - # although it sorts to last. - events = Event.objects(location__near=[41.9120459, -87.67892]) - self.assertEqual(events.count(), 3) - self.assertEqual(list(events), [event1, event3, event2]) - - # find events within 5 degrees of pitchfork office, chicago - point_and_distance = [[41.9120459, -87.67892], 5] - events = Event.objects(location__within_distance=point_and_distance) - self.assertEqual(events.count(), 2) - events = list(events) - self.assertTrue(event2 not in events) - self.assertTrue(event1 in events) - self.assertTrue(event3 in events) - - # ensure ordering is respected by "near" - events = Event.objects(location__near=[41.9120459, -87.67892]) - events = events.order_by("-date") - self.assertEqual(events.count(), 3) - self.assertEqual(list(events), [event3, event1, event2]) - - # find events within 10 degrees of san francisco - point = [37.7566023, -122.415579] - events = Event.objects(location__near=point, location__max_distance=10) - self.assertEqual(events.count(), 1) - self.assertEqual(events[0], event2) - - # find events within 10 degrees of san francisco - point_and_distance = [[37.7566023, -122.415579], 10] - events = Event.objects(location__within_distance=point_and_distance) - self.assertEqual(events.count(), 1) - self.assertEqual(events[0], event2) - - # find events within 1 degree of greenpoint, broolyn, nyc, ny - point_and_distance = [[40.7237134, -73.9509714], 1] - events = Event.objects(location__within_distance=point_and_distance) - self.assertEqual(events.count(), 0) - - # ensure ordering is respected by "within_distance" - point_and_distance = [[41.9120459, -87.67892], 10] - events = Event.objects(location__within_distance=point_and_distance) - events = events.order_by("-date") - self.assertEqual(events.count(), 2) - self.assertEqual(events[0], event3) - - # check that within_box works - box = [(35.0, -125.0), (40.0, -100.0)] - events = Event.objects(location__within_box=box) - self.assertEqual(events.count(), 1) - self.assertEqual(events[0].id, event2.id) - - # check that polygon works for users who have a server >= 1.9 - server_version = tuple( - get_connection().server_info()['version'].split('.') - ) - required_version = tuple("1.9.0".split(".")) - if server_version >= required_version: - polygon = [ - (41.912114,-87.694445), - (41.919395,-87.69084), - (41.927186,-87.681742), - (41.911731,-87.654276), - (41.898061,-87.656164), - ] - events = Event.objects(location__within_polygon=polygon) - self.assertEqual(events.count(), 1) - self.assertEqual(events[0].id, event1.id) - - polygon2 = [ - (54.033586,-1.742249), - (52.792797,-1.225891), - (53.389881,-4.40094) - ] - events = Event.objects(location__within_polygon=polygon2) - self.assertEqual(events.count(), 0) - - Event.drop_collection() - - def test_spherical_geospatial_operators(self): - """Ensure that spherical geospatial queries are working - """ - class Point(Document): - location = GeoPointField() - - Point.drop_collection() - - # These points are one degree apart, which (according to Google Maps) - # is about 110 km apart at this place on the Earth. - north_point = Point(location=[-122, 38]) # Near Concord, CA - south_point = Point(location=[-122, 37]) # Near Santa Cruz, CA - north_point.save() - south_point.save() - - earth_radius = 6378.009; # in km (needs to be a float for dividing by) - - # Finds both points because they are within 60 km of the reference - # point equidistant between them. - points = Point.objects(location__near_sphere=[-122, 37.5]) - self.assertEqual(points.count(), 2) - - # Same behavior for _within_spherical_distance - points = Point.objects( - location__within_spherical_distance=[[-122, 37.5], 60/earth_radius] - ); - self.assertEqual(points.count(), 2) - - points = Point.objects(location__near_sphere=[-122, 37.5], - location__max_distance=60 / earth_radius); - self.assertEqual(points.count(), 2) - - # Finds both points, but orders the north point first because it's - # closer to the reference point to the north. - points = Point.objects(location__near_sphere=[-122, 38.5]) - self.assertEqual(points.count(), 2) - self.assertEqual(points[0].id, north_point.id) - self.assertEqual(points[1].id, south_point.id) - - # Finds both points, but orders the south point first because it's - # closer to the reference point to the south. - points = Point.objects(location__near_sphere=[-122, 36.5]) - self.assertEqual(points.count(), 2) - self.assertEqual(points[0].id, south_point.id) - self.assertEqual(points[1].id, north_point.id) - - # Finds only one point because only the first point is within 60km of - # the reference point to the south. - points = Point.objects( - location__within_spherical_distance=[[-122, 36.5], 60/earth_radius]) - self.assertEqual(points.count(), 1) - self.assertEqual(points[0].id, south_point.id) - - Point.drop_collection() - def test_custom_querysets(self): """Ensure that custom QuerySet classes may be used. """ From 68f760b56375255173bb31b04af485f5987c96da Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 30 Apr 2013 15:05:41 +0000 Subject: [PATCH 1070/1279] get_db() only assigns the db after authentication (#257) --- mongoengine/connection.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 3c53ea3..abab269 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -137,11 +137,12 @@ def get_db(alias=DEFAULT_CONNECTION_NAME, reconnect=False): if alias not in _dbs: conn = get_connection(alias) conn_settings = _connection_settings[alias] - _dbs[alias] = conn[conn_settings['name']] + db = conn[conn_settings['name']] # Authenticate if necessary if conn_settings['username'] and conn_settings['password']: - _dbs[alias].authenticate(conn_settings['username'], - conn_settings['password']) + db.authenticate(conn_settings['username'], + conn_settings['password']) + _dbs[alias] = db return _dbs[alias] From 473d5ead7bf2b61e3b7dbab2845e496994ef5aed Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 30 Apr 2013 16:42:38 +0000 Subject: [PATCH 1071/1279] Geo errors fix and test update --- mongoengine/base/fields.py | 4 ++-- tests/fields/geo.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index fa0b134..72a9e8e 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -456,7 +456,7 @@ class GeoJsonBaseField(BaseField): if error and error not in errors: errors.append(error) if errors: - return "Invalid Polygon:\n%s" % ", ".join(set(errors)) + return "Invalid Polygon:\n%s" % ", ".join(errors) def _validate_linestring(self, value, top_level=True): """Validates a linestring""" @@ -478,7 +478,7 @@ class GeoJsonBaseField(BaseField): if top_level: return "Invalid LineString:\n%s" % ", ".join(errors) else: - return "%s" % ", ".join(set(errors)) + return "%s" % ", ".join(errors) def _validate_point(self, value): """Validate each set of coords""" diff --git a/tests/fields/geo.py b/tests/fields/geo.py index 2936f72..31ded26 100644 --- a/tests/fields/geo.py +++ b/tests/fields/geo.py @@ -184,9 +184,9 @@ class GeoFieldTest(unittest.TestCase): polygon = PolygonField() geo_indicies = Event._geo_indices() - self.assertEqual(geo_indicies, [{'fields': [('line', '2dsphere')]}, - {'fields': [('polygon', '2dsphere')]}, - {'fields': [('point', '2dsphere')]}]) + self.assertTrue({'fields': [('line', '2dsphere')]} in geo_indicies) + self.assertTrue({'fields': [('polygon', '2dsphere')]} in geo_indicies) + self.assertTrue({'fields': [('point', '2dsphere')]} in geo_indicies) def test_indexes_2dsphere_embedded(self): """Ensure that indexes are created automatically for GeoPointFields. From 7aa1f473785ed17cff280835285a69e401fd9b86 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 30 Apr 2013 16:46:08 +0000 Subject: [PATCH 1072/1279] Updated minimum requirements --- .travis.yml | 1 - docs/changelog.rst | 1 + docs/upgrade.rst | 6 +++--- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index e78bda5..b7c56a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,6 @@ env: - PYMONGO=dev DJANGO=1.4.2 - PYMONGO=2.5 DJANGO=1.5.1 - PYMONGO=2.5 DJANGO=1.4.2 - - PYMONGO=2.4.2 DJANGO=1.4.2 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi diff --git a/docs/changelog.rst b/docs/changelog.rst index 207f0dd..6aa6214 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Updated minimum requirement for pymongo to 2.5 - Add support for new geojson fields, indexes and queries (#299) - If values cant be compared mark as changed (#287) - Ensure as_pymongo() and to_json honour only() and exclude() (#293) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index bb5705c..c633c28 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -15,10 +15,10 @@ possible for the whole of the release. live. There maybe multiple manual steps in migrating and these are best honed on a staging / test system. -Python -======= +Python and PyMongo +================== -Support for python 2.5 has been dropped. +MongoEngine requires python 2.6 (or above) and pymongo 2.5 (or above) Data Model ========== diff --git a/setup.py b/setup.py index bdd0182..10a6dbc 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,7 @@ setup(name='mongoengine', long_description=LONG_DESCRIPTION, platforms=['any'], classifiers=CLASSIFIERS, - install_requires=['pymongo'], + install_requires=['pymongo>=2.5'], test_suite='nose.collector', **extra_opts ) From 1c345edc49b9b5e382fdb8b64ab6bf5058d48288 Mon Sep 17 00:00:00 2001 From: Alex Kelly Date: Tue, 30 Apr 2013 21:36:43 +0100 Subject: [PATCH 1073/1279] Updated tests for passing write_concern to update and update_one to check return. --- tests/queryset/queryset.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 40aef7e..7ca0596 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -287,15 +287,19 @@ class QuerySetTest(unittest.TestCase): name='Test User', write_concern=write_concern) author.save(write_concern=write_concern) - self.Person.objects.update(set__name='Ross', - write_concern=write_concern) + result = self.Person.objects.update( + set__name='Ross',write_concern={"w": 1}) + self.assertEqual(result, 1) + result = self.Person.objects.update( + set__name='Ross',write_concern={"w": 0}) + self.assertEqual(result, None) - author = self.Person.objects.first() - self.assertEqual(author.name, 'Ross') - - self.Person.objects.update_one(set__name='Test User', write_concern=write_concern) - author = self.Person.objects.first() - self.assertEqual(author.name, 'Test User') + result = self.Person.objects.update_one( + set__name='Test User', write_concern={"w": 1}) + self.assertEqual(result, 1) + result = self.Person.objects.update_one( + set__name='Test User', write_concern={"w": 0}) + self.assertEqual(result, None) def test_update_update_has_a_value(self): """Test to ensure that update is passed a value to update to""" From 00a57f6cea8ba679281deba4fd4362ca23fb06c8 Mon Sep 17 00:00:00 2001 From: Alex Kelly Date: Tue, 30 Apr 2013 21:13:49 +0100 Subject: [PATCH 1074/1279] Pass write_concern parameter from update_one --- mongoengine/queryset/queryset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index bfb5a48..1739f05 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -469,7 +469,8 @@ class QuerySet(object): .. versionadded:: 0.2 """ - return self.update(upsert=upsert, multi=False, write_concern=None, **update) + return self.update( + upsert=upsert, multi=False, write_concern=write_concern, **update) def with_id(self, object_id): """Retrieve the object matching the id provided. Uses `object_id` only From e58b3390aa8855659c006a8758fa23c075cdcb68 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 1 May 2013 08:48:14 +0000 Subject: [PATCH 1075/1279] Removed import with from future --- AUTHORS | 1 + docs/changelog.rst | 1 + mongoengine/document.py | 1 - tests/document/class_methods.py | 1 - tests/document/indexes.py | 1 - tests/document/instance.py | 1 - tests/fields/fields.py | 1 - tests/fields/file_tests.py | 1 - tests/queryset/queryset.py | 1 - tests/queryset/transform.py | 1 - tests/queryset/visitor.py | 1 - tests/test_connection.py | 1 - tests/test_context_managers.py | 1 - tests/test_dereference.py | 1 - tests/test_django.py | 1 - 15 files changed, 2 insertions(+), 13 deletions(-) diff --git a/AUTHORS b/AUTHORS index 44e19bf..181ad5a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -157,3 +157,4 @@ that much better: * Kenneth Falck * Lukasz Balcerzak * Nicolas Cortot + * Alex (https://github.com/kelsta) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6aa6214..314967e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.X ================ +- Fixed update_one write concern (#302) - Updated minimum requirement for pymongo to 2.5 - Add support for new geojson fields, indexes and queries (#299) - If values cant be compared mark as changed (#287) diff --git a/mongoengine/document.py b/mongoengine/document.py index 143802c..0e9be56 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,4 +1,3 @@ -from __future__ import with_statement import warnings import pymongo diff --git a/tests/document/class_methods.py b/tests/document/class_methods.py index 83e68ff..231dd8f 100644 --- a/tests/document/class_methods.py +++ b/tests/document/class_methods.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import with_statement import sys sys.path[0:0] = [""] import unittest diff --git a/tests/document/indexes.py b/tests/document/indexes.py index ddc147b..04d5632 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import with_statement import unittest import sys sys.path[0:0] = [""] diff --git a/tests/document/instance.py b/tests/document/instance.py index 06744ab..d8df0b2 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import with_statement import sys sys.path[0:0] = [""] diff --git a/tests/fields/fields.py b/tests/fields/fields.py index f7ab63e..3047156 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import with_statement import sys sys.path[0:0] = [""] diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py index c5842d8..52bd88a 100644 --- a/tests/fields/file_tests.py +++ b/tests/fields/file_tests.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import with_statement import sys sys.path[0:0] = [""] diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 40aef7e..dfaae85 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -1,4 +1,3 @@ -from __future__ import with_statement import sys sys.path[0:0] = [""] diff --git a/tests/queryset/transform.py b/tests/queryset/transform.py index bde4b6f..7886965 100644 --- a/tests/queryset/transform.py +++ b/tests/queryset/transform.py @@ -1,4 +1,3 @@ -from __future__ import with_statement import sys sys.path[0:0] = [""] diff --git a/tests/queryset/visitor.py b/tests/queryset/visitor.py index bd81a65..2e9195e 100644 --- a/tests/queryset/visitor.py +++ b/tests/queryset/visitor.py @@ -1,4 +1,3 @@ -from __future__ import with_statement import sys sys.path[0:0] = [""] diff --git a/tests/test_connection.py b/tests/test_connection.py index 4b8a3d1..d792648 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -1,4 +1,3 @@ -from __future__ import with_statement import sys sys.path[0:0] = [""] import unittest diff --git a/tests/test_context_managers.py b/tests/test_context_managers.py index eef63be..f87d638 100644 --- a/tests/test_context_managers.py +++ b/tests/test_context_managers.py @@ -1,4 +1,3 @@ -from __future__ import with_statement import sys sys.path[0:0] = [""] import unittest diff --git a/tests/test_dereference.py b/tests/test_dereference.py index ef5a10d..e146963 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import with_statement import sys sys.path[0:0] = [""] import unittest diff --git a/tests/test_django.py b/tests/test_django.py index 573c072..e30fe3c 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -1,4 +1,3 @@ -from __future__ import with_statement import sys sys.path[0:0] = [""] import unittest From 9654fe0d8d4657ed24b98bb1396faea77813b290 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 1 May 2013 09:30:20 +0000 Subject: [PATCH 1076/1279] 0.8.0RC1 is a go! --- docs/changelog.rst | 2 +- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 314967e..6140925 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,7 +2,7 @@ Changelog ========= -Changes in 0.8.X +Changes in 0.8.0 ================ - Fixed update_one write concern (#302) - Updated minimum requirement for pymongo to 2.5 diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 6fe6d08..0a3ca24 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -15,7 +15,7 @@ import django __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + list(queryset.__all__) + signals.__all__ + list(errors.__all__)) -VERSION = (0, 8, 0, '+') +VERSION = (0, 8, 0, 'RC1') def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index eaf478d..33ea48c 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.7.10 +Version: 0.8.0.RC1 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From cd73654683a2c68d951e7a9c33310fc9fc1ab211 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 1 May 2013 09:48:58 +0000 Subject: [PATCH 1077/1279] Update readme --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5eab502..ea4b505 100644 --- a/README.rst +++ b/README.rst @@ -26,7 +26,7 @@ setup.py install``. Dependencies ============ -- pymongo 2.1.1+ +- pymongo 2.5+ - sphinx (optional - for documentation generation) Examples From 8c9afbd278eac13afdb9e12e23ec0e324d56d539 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 1 May 2013 19:40:49 +0000 Subject: [PATCH 1078/1279] Fix cloning of sliced querysets (#303) --- mongoengine/queryset/queryset.py | 14 +++----- tests/test_django.py | 60 +++++++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 1739f05..c1c9378 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -72,7 +72,6 @@ class QuerySet(object): self._cursor_obj = None self._limit = None self._skip = None - self._slice = None self._hint = -1 # Using -1 as None is a valid value for hint def __call__(self, q_obj=None, class_check=True, slave_okay=False, @@ -127,8 +126,10 @@ class QuerySet(object): if isinstance(key, slice): try: queryset._cursor_obj = queryset._cursor[key] - queryset._slice = key queryset._skip, queryset._limit = key.start, key.stop + queryset._limit + if key.start and key.stop: + queryset._limit = key.stop - key.start except IndexError, err: # PyMongo raises an error if key.start == key.stop, catch it, # bin it, kill it. @@ -537,15 +538,9 @@ class QuerySet(object): val = getattr(self, prop) setattr(c, prop, copy.copy(val)) - if self._slice: - c._slice = self._slice - if self._cursor_obj: c._cursor_obj = self._cursor_obj.clone() - if self._slice: - c._cursor[self._slice] - return c def select_related(self, max_depth=1): @@ -571,7 +566,6 @@ class QuerySet(object): else: queryset._cursor.limit(n) queryset._limit = n - # Return self to allow chaining return queryset @@ -1155,7 +1149,7 @@ class QuerySet(object): self._cursor_obj.sort(order) if self._limit is not None: - self._cursor_obj.limit(self._limit - (self._skip or 0)) + self._cursor_obj.limit(self._limit) if self._skip is not None: self._cursor_obj.skip(self._skip) diff --git a/tests/test_django.py b/tests/test_django.py index e30fe3c..f81213c 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -150,22 +150,74 @@ class QuerySetTest(unittest.TestCase): # Try iterating the same queryset twice, nested, in a Django template. names = ['A', 'B', 'C', 'D'] - class User(Document): + class CustomUser(Document): name = StringField() def __unicode__(self): return self.name - User.drop_collection() + CustomUser.drop_collection() for name in names: - User(name=name).save() + CustomUser(name=name).save() - users = User.objects.all().order_by('name') + users = CustomUser.objects.all().order_by('name') template = Template("{% for user in users %}{{ user.name }}{% ifequal forloop.counter 2 %} {% for inner_user in users %}{{ inner_user.name }}{% endfor %} {% endifequal %}{% endfor %}") rendered = template.render(Context({'users': users})) self.assertEqual(rendered, 'AB ABCD CD') + def test_filter(self): + """Ensure that a queryset and filters work as expected + """ + + class Note(Document): + text = StringField() + + for i in xrange(1, 101): + Note(name="Note: %s" % i).save() + + # Check the count + self.assertEqual(Note.objects.count(), 100) + + # Get the first 10 and confirm + notes = Note.objects[:10] + self.assertEqual(notes.count(), 10) + + # Test djangos template filters + # self.assertEqual(length(notes), 10) + t = Template("{{ notes.count }}") + c = Context({"notes": notes}) + self.assertEqual(t.render(c), "10") + + # Test with skip + notes = Note.objects.skip(90) + self.assertEqual(notes.count(), 10) + + # Test djangos template filters + self.assertEqual(notes.count(), 10) + t = Template("{{ notes.count }}") + c = Context({"notes": notes}) + self.assertEqual(t.render(c), "10") + + # Test with limit + notes = Note.objects.skip(90) + self.assertEqual(notes.count(), 10) + + # Test djangos template filters + self.assertEqual(notes.count(), 10) + t = Template("{{ notes.count }}") + c = Context({"notes": notes}) + self.assertEqual(t.render(c), "10") + + # Test with skip and limit + notes = Note.objects.skip(10).limit(10) + + # Test djangos template filters + self.assertEqual(notes.count(), 10) + t = Template("{{ notes.count }}") + c = Context({"notes": notes}) + self.assertEqual(t.render(c), "10") + class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): backend = SessionStore From 1cdbade7619f887c017905d42cec374ee012d8ff Mon Sep 17 00:00:00 2001 From: Jin Date: Wed, 1 May 2013 16:54:48 -0700 Subject: [PATCH 1079/1279] fixed typo in defining-documents.rst --- docs/guide/defining-documents.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 2c744b7..0ee5ad3 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -493,7 +493,7 @@ Compound Indexes and Indexing sub documents Compound indexes can be created by adding the Embedded field or dictionary field name to the index definition. -Sometimes its more efficient to index parts of Embeedded / dictionary fields, +Sometimes its more efficient to index parts of Embedded / dictionary fields, in this case use 'dot' notation to identify the value to index eg: `rank.title` Geospatial indexes From 3002e79c9844973d545a1f9560b88bff0cee4b71 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 2 May 2013 07:35:33 +0000 Subject: [PATCH 1080/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6140925..edadbd7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.0 ================ +- Fix cloning of sliced querysets (#303) - Fixed update_one write concern (#302) - Updated minimum requirement for pymongo to 2.5 - Add support for new geojson fields, indexes and queries (#299) From 268dd80cd09ddcc4be8df8547d0ccc6eae8a5618 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 2 May 2013 07:35:44 +0000 Subject: [PATCH 1081/1279] Added Jin Zhang to authors (#304) --- AUTHORS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 181ad5a..0ff48e8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -158,3 +158,5 @@ that much better: * Lukasz Balcerzak * Nicolas Cortot * Alex (https://github.com/kelsta) + * Jin Zhang + From 4a71c5b4249610a16752046ae9268207c3d272a9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 2 May 2013 10:47:37 +0000 Subject: [PATCH 1082/1279] Updates to CONTRIBUTING.rst --- CONTRIBUTING.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 9688339..8754896 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -20,7 +20,7 @@ post to the `user group ` Supported Interpreters ---------------------- -PyMongo supports CPython 2.5 and newer. Language +MongoEngine supports CPython 2.6 and newer. Language features not supported by all interpreters can not be used. Please also ensure that your code is properly converted by `2to3 `_ for Python 3 support. @@ -46,7 +46,7 @@ General Guidelines - Write tests and make sure they pass (make sure you have a mongod running on the default port, then execute ``python setup.py test`` from the cmd line to run the test suite). -- Add yourself to AUTHORS.rst :) +- Add yourself to AUTHORS :) Documentation ------------- From a2c429a4a5029358ec9d38d64d53a19f532906ed Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 2 May 2013 10:48:09 +0000 Subject: [PATCH 1083/1279] Queryset cursor regeneration testcase --- tests/queryset/queryset.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 13039f2..bbb28bd 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -115,6 +115,15 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(len(people), 1) self.assertEqual(people[0].name, 'User B') + # Test slice limit and skip cursor reset + qs = self.Person.objects[1:2] + # fetch then delete the cursor + qs._cursor + qs._cursor_obj = None + people = list(qs) + self.assertEqual(len(people), 1) + self.assertEqual(people[0].name, 'User B') + people = list(self.Person.objects[1:1]) self.assertEqual(len(people), 0) From f2c16452c66c064b466301fa0da1df7cd10c3770 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 2 May 2013 10:48:30 +0000 Subject: [PATCH 1084/1279] Help with backwards compatibility --- mongoengine/base/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongoengine/base/__init__.py b/mongoengine/base/__init__.py index ce119b3..e8d4b6a 100644 --- a/mongoengine/base/__init__.py +++ b/mongoengine/base/__init__.py @@ -3,3 +3,6 @@ from mongoengine.base.datastructures import * from mongoengine.base.document import * from mongoengine.base.fields import * from mongoengine.base.metaclasses import * + +# Help with backwards compatibility +from mongoengine.errors import * From 0eda7a5a3c4a82f5160d50a0e4694f96c54c300d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 2 May 2013 10:51:04 +0000 Subject: [PATCH 1085/1279] 0.8.0RC2 is a go --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 0a3ca24..b6adcb4 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -15,7 +15,7 @@ import django __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + list(queryset.__all__) + signals.__all__ + list(errors.__all__)) -VERSION = (0, 8, 0, 'RC1') +VERSION = (0, 8, 0, 'RC2') def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 33ea48c..62ec8f8 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.8.0.RC1 +Version: 0.8.0.RC2 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 3ccc495c758aaa2112405663fe9c9f6607dcc24d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 3 May 2013 12:56:53 +0000 Subject: [PATCH 1086/1279] Fixed register_delete_rule inheritance issue --- mongoengine/base/metaclasses.py | 49 +++++++++++++++++---------------- tests/document/class_methods.py | 21 +++++++++++++- 2 files changed, 45 insertions(+), 25 deletions(-) diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index def8a05..444d9a2 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -140,8 +140,31 @@ class DocumentMetaclass(type): base._subclasses += (_cls,) base._types = base._subclasses # TODO depreciate _types - # Handle delete rules Document, EmbeddedDocument, DictField = cls._import_classes() + + if issubclass(new_class, Document): + new_class._collection = None + + # Add class to the _document_registry + _document_registry[new_class._class_name] = new_class + + # In Python 2, User-defined methods objects have special read-only + # attributes 'im_func' and 'im_self' which contain the function obj + # and class instance object respectively. With Python 3 these special + # attributes have been replaced by __func__ and __self__. The Blinker + # module continues to use im_func and im_self, so the code below + # copies __func__ into im_func and __self__ into im_self for + # classmethod objects in Document derived classes. + if PY3: + for key, val in new_class.__dict__.items(): + if isinstance(val, classmethod): + f = val.__get__(new_class) + if hasattr(f, '__func__') and not hasattr(f, 'im_func'): + f.__dict__.update({'im_func': getattr(f, '__func__')}) + if hasattr(f, '__self__') and not hasattr(f, 'im_self'): + f.__dict__.update({'im_self': getattr(f, '__self__')}) + + # Handle delete rules for field in new_class._fields.itervalues(): f = field f.owner_document = new_class @@ -167,33 +190,11 @@ class DocumentMetaclass(type): field.name, delete_rule) if (field.name and hasattr(Document, field.name) and - EmbeddedDocument not in new_class.mro()): + EmbeddedDocument not in new_class.mro()): msg = ("%s is a document method and not a valid " "field name" % field.name) raise InvalidDocumentError(msg) - if issubclass(new_class, Document): - new_class._collection = None - - # Add class to the _document_registry - _document_registry[new_class._class_name] = new_class - - # In Python 2, User-defined methods objects have special read-only - # attributes 'im_func' and 'im_self' which contain the function obj - # and class instance object respectively. With Python 3 these special - # attributes have been replaced by __func__ and __self__. The Blinker - # module continues to use im_func and im_self, so the code below - # copies __func__ into im_func and __self__ into im_self for - # classmethod objects in Document derived classes. - if PY3: - for key, val in new_class.__dict__.items(): - if isinstance(val, classmethod): - f = val.__get__(new_class) - if hasattr(f, '__func__') and not hasattr(f, 'im_func'): - f.__dict__.update({'im_func': getattr(f, '__func__')}) - if hasattr(f, '__self__') and not hasattr(f, 'im_self'): - f.__dict__.update({'im_self': getattr(f, '__self__')}) - return new_class def add_to_class(self, name, value): diff --git a/tests/document/class_methods.py b/tests/document/class_methods.py index 231dd8f..b2c7283 100644 --- a/tests/document/class_methods.py +++ b/tests/document/class_methods.py @@ -5,7 +5,7 @@ import unittest from mongoengine import * -from mongoengine.queryset import NULLIFY +from mongoengine.queryset import NULLIFY, PULL from mongoengine.connection import get_db __all__ = ("ClassMethodsTest", ) @@ -85,6 +85,25 @@ class ClassMethodsTest(unittest.TestCase): self.assertEqual(self.Person._meta['delete_rules'], {(Job, 'employee'): NULLIFY}) + def test_register_delete_rule_inherited(self): + + class Vaccine(Document): + name = StringField(required=True) + + meta = {"indexes": ["name"]} + + class Animal(Document): + family = StringField(required=True) + vaccine_made = ListField(ReferenceField("Vaccine", reverse_delete_rule=PULL)) + + meta = {"allow_inheritance": True, "indexes": ["family"]} + + class Cat(Animal): + name = StringField(required=True) + + self.assertEqual(Vaccine._meta['delete_rules'][(Animal, 'vaccine_made')], PULL) + self.assertEqual(Vaccine._meta['delete_rules'][(Cat, 'vaccine_made')], PULL) + def test_collection_naming(self): """Ensure that a collection with a specified name may be used. """ From ebd15616827149f7c7d1d2a710056fbc652d4da5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 3 May 2013 14:21:36 +0000 Subject: [PATCH 1087/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index edadbd7..bfa809c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.0 ================ +- Fixed register_delete_rule inheritance issue - Fix cloning of sliced querysets (#303) - Fixed update_one write concern (#302) - Updated minimum requirement for pymongo to 2.5 From 2c119dea472a92e3ac9b3e5be35cc90b260ad6fe Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 7 May 2013 10:34:13 +0000 Subject: [PATCH 1088/1279] Upserting is the only way to ensure docs are saved correctly (#306) --- docs/changelog.rst | 1 + mongoengine/document.py | 3 +-- tests/document/instance.py | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index bfa809c..205df4e 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.0 ================ +- Upserting is the only way to ensure docs are saved correctly (#306) - Fixed register_delete_rule inheritance issue - Fix cloning of sliced querysets (#303) - Fixed update_one write concern (#302) diff --git a/mongoengine/document.py b/mongoengine/document.py index 0e9be56..6c1045b 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -231,7 +231,6 @@ class Document(BaseDocument): return not updated return created - upsert = self._created update_query = {} if updates: @@ -240,7 +239,7 @@ class Document(BaseDocument): update_query["$unset"] = removals if updates or removals: last_error = collection.update(select_dict, update_query, - upsert=upsert, **write_concern) + upsert=True, **write_concern) created = is_new_object(last_error) cascade = (self._meta.get('cascade', True) diff --git a/tests/document/instance.py b/tests/document/instance.py index d8df0b2..d84d65c 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -852,6 +852,14 @@ class InstanceTest(unittest.TestCase): self.assertEqual(person.name, None) self.assertEqual(person.age, None) + def test_inserts_if_you_set_the_pk(self): + p1 = self.Person(name='p1', id=bson.ObjectId()).save() + p2 = self.Person(name='p2') + p2.id = bson.ObjectId() + p2.save() + + self.assertEqual(2, self.Person.objects.count()) + def test_can_save_if_not_included(self): class EmbeddedDoc(EmbeddedDocument): From ddd11c7ed21d1be4392c67fa9e6ef137caecff82 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 7 May 2013 10:57:52 +0000 Subject: [PATCH 1089/1279] Added offline docs links --- docs/index.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 6358a31..77f965c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -55,6 +55,14 @@ See the :doc:`changelog` for a full list of changes to MongoEngine and .. note:: Always read and test the `upgrade `_ documentation before putting updates live in production **;)** +Offline Reading +--------------- + +Download the docs in `pdf `_ +or `epub `_ +formats for offline reading. + + .. toctree:: :maxdepth: 1 :numbered: From 52c162a478ad4796d9a9621a7a1eb68529d992d7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 7 May 2013 11:01:23 +0000 Subject: [PATCH 1090/1279] Pep8 --- mongoengine/fields.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 274ad3c..de9b44f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -107,11 +107,11 @@ class URLField(StringField): """ _URL_REGEX = re.compile( - r'^(?:http|ftp)s?://' # http:// or https:// - r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain... - r'localhost|' #localhost... - r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip - r'(?::\d+)?' # optional port + r'^(?:http|ftp)s?://' # http:// or https:// + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain... + r'localhost|' # localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip + r'(?::\d+)?' # optional port r'(?:/?|[/?]\S+)$', re.IGNORECASE) def __init__(self, verify_exists=False, url_regex=None, **kwargs): @@ -128,8 +128,7 @@ class URLField(StringField): warnings.warn( "The URLField verify_exists argument has intractable security " "and performance issues. Accordingly, it has been deprecated.", - DeprecationWarning - ) + DeprecationWarning) try: request = urllib2.Request(value) urllib2.urlopen(request) @@ -469,7 +468,7 @@ class ComplexDateTimeField(StringField): def __get__(self, instance, owner): data = super(ComplexDateTimeField, self).__get__(instance, owner) - if data == None: + if data is None: return datetime.datetime.now() if isinstance(data, datetime.datetime): return data @@ -658,15 +657,15 @@ class ListField(ComplexBaseField): """Make sure that a list of valid fields is being used. """ if (not isinstance(value, (list, tuple, QuerySet)) or - isinstance(value, basestring)): + isinstance(value, basestring)): self.error('Only lists and tuples may be used in a list field') super(ListField, self).validate(value) def prepare_query_value(self, op, value): if self.field: if op in ('set', 'unset') and (not isinstance(value, basestring) - and not isinstance(value, BaseDocument) - and hasattr(value, '__iter__')): + and not isinstance(value, BaseDocument) + and hasattr(value, '__iter__')): return [self.field.prepare_query_value(op, v) for v in value] return self.field.prepare_query_value(op, value) return super(ListField, self).prepare_query_value(op, value) @@ -701,7 +700,7 @@ class SortedListField(ListField): value = super(SortedListField, self).to_mongo(value) if self._ordering is not None: return sorted(value, key=itemgetter(self._ordering), - reverse=self._order_reverse) + reverse=self._order_reverse) return sorted(value, reverse=self._order_reverse) @@ -1001,7 +1000,7 @@ class BinaryField(BaseField): if not isinstance(value, (bin_type, txt_type, Binary)): self.error("BinaryField only accepts instances of " "(%s, %s, Binary)" % ( - bin_type.__name__, txt_type.__name__)) + bin_type.__name__, txt_type.__name__)) if self.max_bytes is not None and len(value) > self.max_bytes: self.error('Binary value is too long') @@ -1235,8 +1234,6 @@ class ImageGridFsProxy(GridFSProxy): Insert a image in database applying field properties (size, thumbnail_size) """ - if not self.instance: - import ipdb; ipdb.set_trace(); field = self.instance._fields[self.key] try: @@ -1308,6 +1305,7 @@ class ImageGridFsProxy(GridFSProxy): height=h, format=format, **kwargs) + @property def size(self): """ From 870ff1d4d986077f9e306b7a9098d2b339b4c246 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 7 May 2013 11:11:55 +0000 Subject: [PATCH 1091/1279] Added $setOnInsert support for upserts (#308) Upserts now possible with just query parameters (#309) --- docs/changelog.rst | 2 ++ mongoengine/queryset/queryset.py | 2 +- mongoengine/queryset/transform.py | 7 +++++-- tests/queryset/queryset.py | 22 ++++++++++++++++++++-- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 205df4e..ad5f615 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,8 @@ Changelog Changes in 0.8.0 ================ +- Added $setOnInsert support for upserts (#308) +- Upserts now possible with just query parameters (#309) - Upserting is the only way to ensure docs are saved correctly (#306) - Fixed register_delete_rule inheritance issue - Fix cloning of sliced querysets (#303) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index c1c9378..85b683d 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -427,7 +427,7 @@ class QuerySet(object): .. versionadded:: 0.2 """ - if not update: + if not update and not upsert: raise OperationError("No update parameters, would remove data") if not write_concern: diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 96d9904..4062fc1 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -24,7 +24,8 @@ MATCH_OPERATORS = (COMPARISON_OPERATORS + GEO_OPERATORS + STRING_OPERATORS + CUSTOM_OPERATORS) UPDATE_OPERATORS = ('set', 'unset', 'inc', 'dec', 'pop', 'push', - 'push_all', 'pull', 'pull_all', 'add_to_set') + 'push_all', 'pull', 'pull_all', 'add_to_set', + 'set_on_insert') def query(_doc_cls=None, _field_operation=False, **query): @@ -163,7 +164,9 @@ def update(_doc_cls=None, **update): if value > 0: value = -value elif op == 'add_to_set': - op = op.replace('_to_set', 'ToSet') + op = 'addToSet' + elif op == 'set_on_insert': + op = "setOnInsert" match = None if parts[-1] in COMPARISON_OPERATORS: diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index bbb28bd..ffb5378 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -296,10 +296,10 @@ class QuerySetTest(unittest.TestCase): author.save(write_concern=write_concern) result = self.Person.objects.update( - set__name='Ross',write_concern={"w": 1}) + set__name='Ross', write_concern={"w": 1}) self.assertEqual(result, 1) result = self.Person.objects.update( - set__name='Ross',write_concern={"w": 0}) + set__name='Ross', write_concern={"w": 0}) self.assertEqual(result, None) result = self.Person.objects.update_one( @@ -536,6 +536,24 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(club.members['John']['gender'], "F") self.assertEqual(club.members['John']['age'], 14) + def test_upsert(self): + self.Person.drop_collection() + + self.Person.objects(pk=ObjectId(), name="Bob", age=30).update(upsert=True) + + bob = self.Person.objects.first() + self.assertEqual("Bob", bob.name) + self.assertEqual(30, bob.age) + + def test_set_on_insert(self): + self.Person.drop_collection() + + self.Person.objects(pk=ObjectId()).update(set__name='Bob', set_on_insert__age=30, upsert=True) + + bob = self.Person.objects.first() + self.assertEqual("Bob", bob.name) + self.assertEqual(30, bob.age) + def test_get_or_create(self): """Ensure that ``get_or_create`` returns one result or creates a new document. From 7cde97973696eb28e513b522e227d65786378739 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 7 May 2013 11:39:16 +0000 Subject: [PATCH 1092/1279] Updated fields --- mongoengine/fields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index de9b44f..4995998 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -879,7 +879,7 @@ class ReferenceField(BaseField): """Convert a MongoDB-compatible type to a Python type. """ if (not self.dbref and - not isinstance(value, (DBRef, Document, EmbeddedDocument))): + not isinstance(value, (DBRef, Document, EmbeddedDocument))): collection = self.document_type._get_collection_name() value = DBRef(collection, self.document_type.id.to_python(value)) return value From 9dfee83e687a9aef625fbc38bf5dd10b16e463dc Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 7 May 2013 11:54:47 +0000 Subject: [PATCH 1093/1279] Fixed querying string versions of ObjectIds issue with ReferenceField (#307) --- docs/changelog.rst | 1 + mongoengine/fields.py | 2 +- mongoengine/queryset/queryset.py | 5 ++-- tests/queryset/queryset.py | 45 +++++++++++++++++++++++++++++++- 4 files changed, 49 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ad5f615..842bc7d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.0 ================ +- Fixed querying string versions of ObjectIds issue with ReferenceField (#307) - Added $setOnInsert support for upserts (#308) - Upserts now possible with just query parameters (#309) - Upserting is the only way to ensure docs are saved correctly (#306) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 4995998..573d9a0 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -854,7 +854,7 @@ class ReferenceField(BaseField): return document.id return document elif not self.dbref and isinstance(document, basestring): - return document + return ObjectId(document) id_field_name = self.document_type._meta['id_field'] id_field = self.document_type._fields[id_field_name] diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 85b683d..191afdd 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -544,8 +544,9 @@ class QuerySet(object): return c def select_related(self, max_depth=1): - """Handles dereferencing of :class:`~bson.dbref.DBRef` objects to - a maximum depth in order to cut down the number queries to mongodb. + """Handles dereferencing of :class:`~bson.dbref.DBRef` objects or + :class:`~bson.object_id.ObjectId` a maximum depth in order to cut down + the number queries to mongodb. .. versionadded:: 0.5 """ diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index ffb5378..b9db297 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -1263,7 +1263,7 @@ class QuerySetTest(unittest.TestCase): class BlogPost(Document): content = StringField() authors = ListField(ReferenceField(self.Person, - reverse_delete_rule=PULL)) + reverse_delete_rule=PULL)) BlogPost.drop_collection() self.Person.drop_collection() @@ -1321,6 +1321,49 @@ class QuerySetTest(unittest.TestCase): self.Person.objects()[:1].delete() self.assertEqual(1, BlogPost.objects.count()) + + def test_reference_field_find(self): + """Ensure cascading deletion of referring documents from the database. + """ + class BlogPost(Document): + content = StringField() + author = ReferenceField(self.Person) + + BlogPost.drop_collection() + self.Person.drop_collection() + + me = self.Person(name='Test User').save() + BlogPost(content="test 123", author=me).save() + + self.assertEqual(1, BlogPost.objects(author=me).count()) + self.assertEqual(1, BlogPost.objects(author=me.pk).count()) + self.assertEqual(1, BlogPost.objects(author="%s" % me.pk).count()) + + self.assertEqual(1, BlogPost.objects(author__in=[me]).count()) + self.assertEqual(1, BlogPost.objects(author__in=[me.pk]).count()) + self.assertEqual(1, BlogPost.objects(author__in=["%s" % me.pk]).count()) + + def test_reference_field_find_dbref(self): + """Ensure cascading deletion of referring documents from the database. + """ + class BlogPost(Document): + content = StringField() + author = ReferenceField(self.Person, dbref=True) + + BlogPost.drop_collection() + self.Person.drop_collection() + + me = self.Person(name='Test User').save() + BlogPost(content="test 123", author=me).save() + + self.assertEqual(1, BlogPost.objects(author=me).count()) + self.assertEqual(1, BlogPost.objects(author=me.pk).count()) + self.assertEqual(1, BlogPost.objects(author="%s" % me.pk).count()) + + self.assertEqual(1, BlogPost.objects(author__in=[me]).count()) + self.assertEqual(1, BlogPost.objects(author__in=[me.pk]).count()) + self.assertEqual(1, BlogPost.objects(author__in=["%s" % me.pk]).count()) + def test_update(self): """Ensure that atomic updates work properly. """ From 9e513e08aeafec19399677e9bd813fedd3d596ed Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 7 May 2013 11:55:56 +0000 Subject: [PATCH 1094/1279] Updated RC version --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index b6adcb4..3a4d7c9 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -15,7 +15,7 @@ import django __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + list(queryset.__all__) + signals.__all__ + list(errors.__all__)) -VERSION = (0, 8, 0, 'RC2') +VERSION = (0, 8, 0, 'RC3') def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 62ec8f8..68cb72c 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.8.0.RC2 +Version: 0.8.0.RC3 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 96a964a18332b13e5ba4ecd11ee246cf80a8a8da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Thu, 9 May 2013 13:18:58 -0300 Subject: [PATCH 1095/1279] added .disable_inheritance method for the simple fetch exclusives classes --- mongoengine/queryset/queryset.py | 9 +++++++++ tests/queryset/queryset.py | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 191afdd..407bf2f 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -520,6 +520,15 @@ class QuerySet(object): queryset._none = True return queryset + def disable_inheritance(self): + """ + Disable inheritance query, fetch only objects for the query class + """ + if self._document._meta.get('allow_inheritance') is True: + self._initial_query = {"_cls": self._document._class_name} + + return self + def clone(self): """Creates a copy of the current :class:`~mongoengine.queryset.QuerySet` diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index b9db297..49ed36c 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3272,5 +3272,25 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(outer_count, 7) # outer loop should be executed seven times total self.assertEqual(inner_total_count, 7 * 7) # inner loop should be executed fourtynine times total + def test_disable_inheritance_queryset(self): + class A(Document): + x = IntField() + y = IntField() + + meta = {'allow_inheritance': True} + + class B(A): + z = IntField() + + A.drop_collection() + + A(x=10, y=20).save() + A(x=15, y=30).save() + B(x=20, y=40).save() + B(x=30, y=50).save() + + for obj in A.objects.disable_inheritance(): + self.assertEqual(obj.__class__, A) + if __name__ == '__main__': unittest.main() From 9251ce312bc0545d3b86224e35a913029a86695e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 10 May 2013 13:57:32 +0000 Subject: [PATCH 1096/1279] Querysets now utilises a local cache Changed __len__ behavour in the queryset (#247, #311) --- docs/changelog.rst | 3 +- docs/upgrade.rst | 13 ++-- mongoengine/queryset/queryset.py | 111 ++++++++++++++++++++++--------- setup.py | 4 +- tests/queryset/queryset.py | 82 ++++++++++++++++++----- tests/test_jinja.py | 47 +++++++++++++ 6 files changed, 204 insertions(+), 56 deletions(-) create mode 100644 tests/test_jinja.py diff --git a/docs/changelog.rst b/docs/changelog.rst index 842bc7d..3b6813a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,8 @@ Changelog Changes in 0.8.0 ================ +- Querysets now utilises a local cache +- Changed __len__ behavour in the queryset (#247, #311) - Fixed querying string versions of ObjectIds issue with ReferenceField (#307) - Added $setOnInsert support for upserts (#308) - Upserts now possible with just query parameters (#309) @@ -25,7 +27,6 @@ Changes in 0.8.0 - Added SequenceField.set_next_value(value) helper (#159) - Updated .only() behaviour - now like exclude it is chainable (#202) - Added with_limit_and_skip support to count() (#235) -- Removed __len__ from queryset (#247) - Objects queryset manager now inherited (#256) - Updated connection to use MongoClient (#262, #274) - Fixed db_alias and inherited Documents (#143) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index c633c28..fe9e4fa 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -235,12 +235,15 @@ update your code like so: :: mammals = Animal.objects(type="mammal").filter(order="Carnivora") # The final queryset is assgined to mammals [m for m in mammals] # This will return all carnivores -No more len ------------ +Len iterates the queryset +-------------------------- -If you ever did len(queryset) it previously did a count() under the covers, this -caused some unusual issues - so now it has been removed in favour of the -explicit `queryset.count()` to update:: +If you ever did `len(queryset)` it previously did a `count()` under the covers, +this caused some unusual issues. As `len(queryset)` is most often used by +`list(queryset)` we now cache the queryset results and use that for the length. + +This isn't as performant as a `count()` and if you aren't iterating the +queryset you should upgrade to use count:: # Old code len(Animal.objects(type="mammal")) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 191afdd..2d63183 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -26,6 +26,7 @@ __all__ = ('QuerySet', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL') # The maximum number of items to display in a QuerySet.__repr__ REPR_OUTPUT_SIZE = 20 +ITER_CHUNK_SIZE = 100 # Delete rules DO_NOTHING = 0 @@ -63,6 +64,9 @@ class QuerySet(object): self._none = False self._as_pymongo = False self._as_pymongo_coerce = False + self._result_cache = [] + self._has_more = True + self._len = None # If inheritance is allowed, only return instances and instances of # subclasses of the class being used @@ -109,13 +113,60 @@ class QuerySet(object): queryset._class_check = class_check return queryset + def __len__(self): + """Since __len__ is called quite frequently (for example, as part of + list(qs) we populate the result cache and cache the length. + """ + if self._len is not None: + return self._len + if self._has_more: + # populate the cache + list(self._iter_results()) + + self._len = len(self._result_cache) + return self._len + def __iter__(self): - """Support iterator protocol""" - queryset = self - if queryset._iter: - queryset = self.clone() - queryset.rewind() - return queryset + """Iteration utilises a results cache which iterates the cursor + in batches of ``ITER_CHUNK_SIZE``. + + If ``self._has_more`` the cursor hasn't been exhausted so cache then + batch. Otherwise iterate the result_cache. + """ + self._iter = True + if self._has_more: + return self._iter_results() + + # iterating over the cache. + return iter(self._result_cache) + + def _iter_results(self): + """A generator for iterating over the result cache. + + Also populates the cache if there are more possible results to yield. + Raises StopIteration when there are no more results""" + pos = 0 + while True: + upper = len(self._result_cache) + while pos < upper: + yield self._result_cache[pos] + pos = pos + 1 + if not self._has_more: + raise StopIteration + if len(self._result_cache) <= pos: + self._populate_cache() + + def _populate_cache(self): + """ + Populates the result cache with ``ITER_CHUNK_SIZE`` more entries + (until the cursor is exhausted). + """ + if self._has_more: + try: + for i in xrange(ITER_CHUNK_SIZE): + self._result_cache.append(self.next()) + except StopIteration: + self._has_more = False def __getitem__(self, key): """Support skip and limit using getitem and slicing syntax. @@ -157,22 +208,15 @@ class QuerySet(object): def __repr__(self): """Provides the string representation of the QuerySet - - .. versionchanged:: 0.6.13 Now doesnt modify the cursor """ + if self._iter: return '.. queryset mid-iteration ..' - data = [] - for i in xrange(REPR_OUTPUT_SIZE + 1): - try: - data.append(self.next()) - except StopIteration: - break + self._populate_cache() + data = self._result_cache[:REPR_OUTPUT_SIZE + 1] if len(data) > REPR_OUTPUT_SIZE: data[-1] = "...(remaining elements truncated)..." - - self.rewind() return repr(data) # Core functions @@ -201,7 +245,7 @@ class QuerySet(object): result = queryset.next() except StopIteration: msg = ("%s matching query does not exist." - % queryset._document._class_name) + % queryset._document._class_name) raise queryset._document.DoesNotExist(msg) try: queryset.next() @@ -352,7 +396,12 @@ class QuerySet(object): """ if self._limit == 0: return 0 - return self._cursor.count(with_limit_and_skip=with_limit_and_skip) + if with_limit_and_skip and self._len is not None: + return self._len + count = self._cursor.count(with_limit_and_skip=with_limit_and_skip) + if with_limit_and_skip: + self._len = count + return count def delete(self, write_concern=None): """Delete the documents matched by the query. @@ -910,7 +959,7 @@ class QuerySet(object): mr_args['out'] = output results = getattr(queryset._collection, map_reduce_function)( - map_f, reduce_f, **mr_args) + map_f, reduce_f, **mr_args) if map_reduce_function == 'map_reduce': results = results.find() @@ -1084,20 +1133,18 @@ class QuerySet(object): def next(self): """Wrap the result in a :class:`~mongoengine.Document` object. """ - self._iter = True - try: - if self._limit == 0 or self._none: - raise StopIteration - if self._scalar: - return self._get_scalar(self._document._from_son( - self._cursor.next())) - if self._as_pymongo: - return self._get_as_pymongo(self._cursor.next()) + if self._limit == 0 or self._none: + raise StopIteration - return self._document._from_son(self._cursor.next()) - except StopIteration, e: - self.rewind() - raise e + raw_doc = self._cursor.next() + if self._as_pymongo: + return self._get_as_pymongo(raw_doc) + + doc = self._document._from_son(raw_doc) + if self._scalar: + return self._get_scalar(doc) + + return doc def rewind(self): """Rewind the cursor to its unevaluated state. diff --git a/setup.py b/setup.py index 10a6dbc..594f7f8 100644 --- a/setup.py +++ b/setup.py @@ -51,13 +51,13 @@ CLASSIFIERS = [ extra_opts = {} if sys.version_info[0] == 3: extra_opts['use_2to3'] = True - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2'] extra_opts['packages'] = find_packages(exclude=('tests',)) if "test" in sys.argv or "nosetests" in sys.argv: extra_opts['packages'].append("tests") extra_opts['package_data'] = {"tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"]} else: - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2'] extra_opts['packages'] = find_packages(exclude=('tests',)) setup(name='mongoengine', diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index b9db297..b9c1396 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -793,7 +793,7 @@ class QuerySetTest(unittest.TestCase): p = p.snapshot(True).slave_okay(True).timeout(True) self.assertEqual(p._cursor_args, - {'snapshot': True, 'slave_okay': True, 'timeout': True}) + {'snapshot': True, 'slave_okay': True, 'timeout': True}) def test_repeated_iteration(self): """Ensure that QuerySet rewinds itself one iteration finishes. @@ -835,6 +835,7 @@ class QuerySetTest(unittest.TestCase): self.assertTrue("Doc: 0" in docs_string) self.assertEqual(docs.count(), 1000) + self.assertTrue('(remaining elements truncated)' in "%s" % docs) # Limit and skip docs = docs[1:4] @@ -3231,6 +3232,51 @@ class QuerySetTest(unittest.TestCase): Organization)) self.assertTrue(isinstance(qs.first().organization, Organization)) + def test_cached_queryset(self): + class Person(Document): + name = StringField() + + Person.drop_collection() + for i in xrange(100): + Person(name="No: %s" % i).save() + + with query_counter() as q: + self.assertEqual(q, 0) + people = Person.objects + + [x for x in people] + self.assertEqual(100, len(people._result_cache)) + self.assertEqual(None, people._len) + self.assertEqual(q, 1) + + list(people) + self.assertEqual(100, people._len) # Caused by list calling len + self.assertEqual(q, 1) + + people.count() # count is cached + self.assertEqual(q, 1) + + def test_cache_not_cloned(self): + + class User(Document): + name = StringField() + + def __unicode__(self): + return self.name + + User.drop_collection() + + User(name="Alice").save() + User(name="Bob").save() + + users = User.objects.all().order_by('name') + self.assertEqual("%s" % users, "[, ]") + self.assertEqual(2, len(users._result_cache)) + + users = users.filter(name="Bob") + self.assertEqual("%s" % users, "[]") + self.assertEqual(1, len(users._result_cache)) + def test_nested_queryset_iterator(self): # Try iterating the same queryset twice, nested. names = ['Alice', 'Bob', 'Chuck', 'David', 'Eric', 'Francis', 'George'] @@ -3247,30 +3293,34 @@ class QuerySetTest(unittest.TestCase): User(name=name).save() users = User.objects.all().order_by('name') - outer_count = 0 inner_count = 0 inner_total_count = 0 - self.assertEqual(users.count(), 7) + with query_counter() as q: + self.assertEqual(q, 0) - for i, outer_user in enumerate(users): - self.assertEqual(outer_user.name, names[i]) - outer_count += 1 - inner_count = 0 - - # Calling len might disrupt the inner loop if there are bugs self.assertEqual(users.count(), 7) - for j, inner_user in enumerate(users): - self.assertEqual(inner_user.name, names[j]) - inner_count += 1 - inner_total_count += 1 + for i, outer_user in enumerate(users): + self.assertEqual(outer_user.name, names[i]) + outer_count += 1 + inner_count = 0 - self.assertEqual(inner_count, 7) # inner loop should always be executed seven times + # Calling len might disrupt the inner loop if there are bugs + self.assertEqual(users.count(), 7) - self.assertEqual(outer_count, 7) # outer loop should be executed seven times total - self.assertEqual(inner_total_count, 7 * 7) # inner loop should be executed fourtynine times total + for j, inner_user in enumerate(users): + self.assertEqual(inner_user.name, names[j]) + inner_count += 1 + inner_total_count += 1 + + self.assertEqual(inner_count, 7) # inner loop should always be executed seven times + + self.assertEqual(outer_count, 7) # outer loop should be executed seven times total + self.assertEqual(inner_total_count, 7 * 7) # inner loop should be executed fourtynine times total + + self.assertEqual(q, 2) if __name__ == '__main__': unittest.main() diff --git a/tests/test_jinja.py b/tests/test_jinja.py new file mode 100644 index 0000000..0449f86 --- /dev/null +++ b/tests/test_jinja.py @@ -0,0 +1,47 @@ +import sys +sys.path[0:0] = [""] + +import unittest + +from mongoengine import * + +import jinja2 + + +class TemplateFilterTest(unittest.TestCase): + + def setUp(self): + connect(db='mongoenginetest') + + def test_jinja2(self): + env = jinja2.Environment() + + class TestData(Document): + title = StringField() + description = StringField() + + TestData.drop_collection() + + examples = [('A', '1'), + ('B', '2'), + ('C', '3')] + + for title, description in examples: + TestData(title=title, description=description).save() + + tmpl = """ +{%- for record in content -%} + {%- if loop.first -%}{ {%- endif -%} + "{{ record.title }}": "{{ record.description }}" + {%- if loop.last -%} }{%- else -%},{% endif -%} +{%- endfor -%} +""" + ctx = {'content': TestData.objects} + template = env.from_string(tmpl) + rendered = template.render(**ctx) + + self.assertEqual('{"A": "1","B": "2","C": "3"}', rendered) + + +if __name__ == '__main__': + unittest.main() From 5b498bd8d6f161e81605a686fe927307b2e28078 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 10 May 2013 15:05:16 +0000 Subject: [PATCH 1097/1279] Added no_sub_classes context manager and queryset helper (#312) --- docs/changelog.rst | 1 + mongoengine/context_managers.py | 36 ++++++++++++++++++++-- mongoengine/queryset/queryset.py | 4 +-- tests/queryset/queryset.py | 23 ++++++++++++-- tests/test_context_managers.py | 51 +++++++++++++++++++++++++++++++- 5 files changed, 108 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3b6813a..c3e50e4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.0 ================ +- Added no_sub_classes context manager and queryset helper (#312) - Querysets now utilises a local cache - Changed __len__ behavour in the queryset (#247, #311) - Fixed querying string versions of ObjectIds issue with ReferenceField (#307) diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index 76d5fbf..1280e11 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -1,8 +1,10 @@ from mongoengine.common import _import_class from mongoengine.connection import DEFAULT_CONNECTION_NAME, get_db -from mongoengine.queryset import OperationError, QuerySet +from mongoengine.queryset import QuerySet -__all__ = ("switch_db", "switch_collection", "no_dereference", "query_counter") + +__all__ = ("switch_db", "switch_collection", "no_dereference", + "no_sub_classes", "query_counter") class switch_db(object): @@ -130,6 +132,36 @@ class no_dereference(object): return self.cls +class no_sub_classes(object): + """ no_sub_classes context manager. + + Only returns instances of this class and no sub (inherited) classes:: + + with no_sub_classes(Group) as Group: + Group.objects.find() + + """ + + def __init__(self, cls): + """ Construct the no_sub_classes context manager. + + :param cls: the class to turn querying sub classes on + """ + self.cls = cls + + def __enter__(self): + """ change the objects default and _auto_dereference values""" + self.cls._all_subclasses = self.cls._subclasses + self.cls._subclasses = (self.cls,) + return self.cls + + def __exit__(self, t, value, traceback): + """ Reset the default and _auto_dereference values""" + self.cls._subclasses = self.cls._all_subclasses + delattr(self.cls, '_all_subclasses') + return self.cls + + class QuerySetNoDeRef(QuerySet): """Special no_dereference QuerySet""" def __dereference(items, max_depth=1, instance=None, name=None): diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 4cf86d1..5da6295 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -569,9 +569,9 @@ class QuerySet(object): queryset._none = True return queryset - def disable_inheritance(self): + def no_sub_classes(self): """ - Disable inheritance query, fetch only objects for the query class + Only return instances of this document and not any inherited documents """ if self._document._meta.get('allow_inheritance') is True: self._initial_query = {"_cls": self._document._class_name} diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 27a418d..bf23761 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3322,7 +3322,7 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(q, 2) - def test_disable_inheritance_queryset(self): + def test_no_sub_classes(self): class A(Document): x = IntField() y = IntField() @@ -3332,15 +3332,34 @@ class QuerySetTest(unittest.TestCase): class B(A): z = IntField() + class C(B): + zz = IntField() + A.drop_collection() A(x=10, y=20).save() A(x=15, y=30).save() B(x=20, y=40).save() B(x=30, y=50).save() + C(x=40, y=60).save() - for obj in A.objects.disable_inheritance(): + self.assertEqual(A.objects.no_sub_classes().count(), 2) + self.assertEqual(A.objects.count(), 5) + + self.assertEqual(B.objects.no_sub_classes().count(), 2) + self.assertEqual(B.objects.count(), 3) + + self.assertEqual(C.objects.no_sub_classes().count(), 1) + self.assertEqual(C.objects.count(), 1) + + for obj in A.objects.no_sub_classes(): self.assertEqual(obj.__class__, A) + for obj in B.objects.no_sub_classes(): + self.assertEqual(obj.__class__, B) + + for obj in C.objects.no_sub_classes(): + self.assertEqual(obj.__class__, C) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_context_managers.py b/tests/test_context_managers.py index f87d638..c201a5f 100644 --- a/tests/test_context_managers.py +++ b/tests/test_context_managers.py @@ -5,7 +5,8 @@ import unittest from mongoengine import * from mongoengine.connection import get_db from mongoengine.context_managers import (switch_db, switch_collection, - no_dereference, query_counter) + no_sub_classes, no_dereference, + query_counter) class ContextManagersTest(unittest.TestCase): @@ -138,6 +139,54 @@ class ContextManagersTest(unittest.TestCase): self.assertTrue(isinstance(group.ref, User)) self.assertTrue(isinstance(group.generic, User)) + def test_no_sub_classes(self): + class A(Document): + x = IntField() + y = IntField() + + meta = {'allow_inheritance': True} + + class B(A): + z = IntField() + + class C(B): + zz = IntField() + + A.drop_collection() + + A(x=10, y=20).save() + A(x=15, y=30).save() + B(x=20, y=40).save() + B(x=30, y=50).save() + C(x=40, y=60).save() + + self.assertEqual(A.objects.count(), 5) + self.assertEqual(B.objects.count(), 3) + self.assertEqual(C.objects.count(), 1) + + with no_sub_classes(A) as A: + self.assertEqual(A.objects.count(), 2) + + for obj in A.objects: + self.assertEqual(obj.__class__, A) + + with no_sub_classes(B) as B: + self.assertEqual(B.objects.count(), 2) + + for obj in B.objects: + self.assertEqual(obj.__class__, B) + + with no_sub_classes(C) as C: + self.assertEqual(C.objects.count(), 1) + + for obj in C.objects: + self.assertEqual(obj.__class__, C) + + # Confirm context manager exit correctly + self.assertEqual(A.objects.count(), 5) + self.assertEqual(B.objects.count(), 3) + self.assertEqual(C.objects.count(), 1) + def test_query_counter(self): connect('mongoenginetest') db = get_db() From f8350409ad57c95c1b89c47c0c331b58bee26be6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 10 May 2013 15:08:01 +0000 Subject: [PATCH 1098/1279] assertEquals is bad --- tests/document/instance.py | 20 ++++++++++---------- tests/queryset/queryset.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index d84d65c..dcb0de3 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -319,8 +319,8 @@ class InstanceTest(unittest.TestCase): Location.drop_collection() - self.assertEquals(Area, get_document("Area")) - self.assertEquals(Area, get_document("Location.Area")) + self.assertEqual(Area, get_document("Area")) + self.assertEqual(Area, get_document("Location.Area")) def test_creation(self): """Ensure that document may be created using keyword arguments. @@ -508,12 +508,12 @@ class InstanceTest(unittest.TestCase): t = TestDocument(status="published") t.save(clean=False) - self.assertEquals(t.pub_date, None) + self.assertEqual(t.pub_date, None) t = TestDocument(status="published") t.save(clean=True) - self.assertEquals(type(t.pub_date), datetime) + self.assertEqual(type(t.pub_date), datetime) def test_document_embedded_clean(self): class TestEmbeddedDocument(EmbeddedDocument): @@ -545,7 +545,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(e.to_dict(), {'doc': {'__all__': expect_msg}}) t = TestDocument(doc=TestEmbeddedDocument(x=10, y=25)).save() - self.assertEquals(t.doc.z, 35) + self.assertEqual(t.doc.z, 35) # Asserts not raises t = TestDocument(doc=TestEmbeddedDocument(x=15, y=35, z=5)) @@ -1903,11 +1903,11 @@ class InstanceTest(unittest.TestCase): A.objects.all() - self.assertEquals('testdb-2', B._meta.get('db_alias')) - self.assertEquals('mongoenginetest', - A._get_collection().database.name) - self.assertEquals('mongoenginetest2', - B._get_collection().database.name) + self.assertEqual('testdb-2', B._meta.get('db_alias')) + self.assertEqual('mongoenginetest', + A._get_collection().database.name) + self.assertEqual('mongoenginetest2', + B._get_collection().database.name) def test_db_alias_propagates(self): """db_alias propagates? diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index bf23761..9e1fda2 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -282,7 +282,7 @@ class QuerySetTest(unittest.TestCase): a_objects = A.objects(s='test1') query = B.objects(ref__in=a_objects) query = query.filter(boolfield=True) - self.assertEquals(query.count(), 1) + self.assertEqual(query.count(), 1) def test_update_write_concern(self): """Test that passing write_concern works""" From b16eabd2b6350fdc5a05036034d7d0175c33d6a7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 10 May 2013 15:09:08 +0000 Subject: [PATCH 1099/1279] Updated version --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 3a4d7c9..0f8913a 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -15,7 +15,7 @@ import django __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + list(queryset.__all__) + signals.__all__ + list(errors.__all__)) -VERSION = (0, 8, 0, 'RC3') +VERSION = (0, 8, 0, 'RC4') def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 68cb72c..be9c67b 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.8.0.RC3 +Version: 0.8.0.RC4 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 0efb90deb6daf1f47a324be2b295a59600226d02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20S?= Date: Mon, 13 May 2013 13:14:15 +0200 Subject: [PATCH 1100/1279] Added a failing test when using pickle with signal hooks --- tests/document/instance.py | 8 +++++++- tests/fixtures.py | 20 ++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index dcb0de3..d972ae5 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -9,7 +9,7 @@ import unittest import uuid from datetime import datetime -from tests.fixtures import PickleEmbedded, PickleTest +from tests.fixtures import PickleEmbedded, PickleTest, PickleSignalsTest from mongoengine import * from mongoengine.errors import (NotRegistered, InvalidDocumentError, @@ -1730,6 +1730,12 @@ class InstanceTest(unittest.TestCase): self.assertEqual(pickle_doc.string, "Two") self.assertEqual(pickle_doc.lists, ["1", "2", "3"]) + def test_picklable_on_signals(self): + pickle_doc = PickleSignalsTest(number=1, string="One", lists=['1', '2']) + pickle_doc.embedded = PickleEmbedded() + pickle_doc.save() + pickle_doc.delete() + def test_throw_invalid_document_error(self): # test handles people trying to upsert diff --git a/tests/fixtures.py b/tests/fixtures.py index fd9062e..a35f144 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -1,6 +1,8 @@ +import pickle from datetime import datetime from mongoengine import * +from mongoengine import signals class PickleEmbedded(EmbeddedDocument): @@ -15,6 +17,24 @@ class PickleTest(Document): photo = FileField() +class PickleSignalsTest(Document): + number = IntField() + string = StringField(choices=(('One', '1'), ('Two', '2'))) + embedded = EmbeddedDocumentField(PickleEmbedded) + lists = ListField(StringField()) + + @classmethod + def post_save(self, sender, document, created, **kwargs): + pickled = pickle.dumps(document) + + @classmethod + def post_delete(self, sender, document, **kwargs): + pickled = pickle.dumps(document) + +signals.post_save.connect(PickleSignalsTest.post_save, sender=PickleSignalsTest) +signals.post_delete.connect(PickleSignalsTest.post_delete, sender=PickleSignalsTest) + + class Mixin(object): name = StringField() From f6d0b53ae57e37cc2dab7782b7077df1b9536b35 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Mon, 13 May 2013 21:42:20 -0700 Subject: [PATCH 1101/1279] test reference to a custom pk doc --- tests/queryset/queryset.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 9e1fda2..01c53d0 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3361,5 +3361,25 @@ class QuerySetTest(unittest.TestCase): for obj in C.objects.no_sub_classes(): self.assertEqual(obj.__class__, C) + def test_query_reference_to_custom_pk_doc(self): + + class A(Document): + id = StringField(unique=True, primary_key=True) + + class B(Document): + a = ReferenceField(A) + + A.drop_collection() + B.drop_collection() + + a = A.objects.create(id='custom_id') + + b = B.objects.create(a=a) + + self.assertEqual(B.objects.count(), 1) + self.assertEqual(B.objects.get(a=a).a, a) + self.assertEqual(B.objects.get(a=a.id).a, a) + + if __name__ == '__main__': unittest.main() From 731d8fc6bed91bb12598c70d29205985f2b9f7fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Thu, 16 May 2013 12:50:34 -0300 Subject: [PATCH 1102/1279] added get_next_value to SequenceField --- mongoengine/fields.py | 11 +++++++++++ tests/fields/fields.py | 17 +++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 2e14933..b2f5488 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1465,6 +1465,17 @@ class SequenceField(BaseField): upsert=True) return self.value_decorator(counter['next']) + def get_next_value(self): + sequence_name = self.get_sequence_name() + sequence_id = "%s.%s" % (sequence_name, self.name) + collection = get_db(alias=self.db_alias)[self.collection_name] + data = collection.find_one({"_id": sequence_id}) + + if data: + return data['next'] + + return 1 + def get_sequence_name(self): if self.sequence_name: return self.sequence_name diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 4fa6989..444b71a 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -1917,6 +1917,23 @@ class FieldTest(unittest.TestCase): c = self.db['mongoengine.counters'].find_one({'_id': 'person.id'}) self.assertEqual(c['next'], 1000) + + def test_sequence_field_get_next_value(self): + class Person(Document): + id = SequenceField(primary_key=True) + name = StringField() + + self.db['mongoengine.counters'].drop() + Person.drop_collection() + + for x in xrange(10): + Person(name="Person %s" % x).save() + + self.assertEqual(Person.id.get_next_value(), 10) + self.db['mongoengine.counters'].drop() + + self.assertEqual(Person.id.get_next_value(), 1) + def test_sequence_field_sequence_name(self): class Person(Document): id = SequenceField(primary_key=True, sequence_name='jelly') From 0b1e11ba1fd351fd78d33f03090b5f20e21f5085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Thu, 16 May 2013 12:55:16 -0300 Subject: [PATCH 1103/1279] added my github profile --- AUTHORS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 0ff48e8..fbe697a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -25,7 +25,7 @@ that much better: * flosch * Deepak Thukral * Colin Howe - * Wilson Júnior + * Wilson Júnior (https://github.com/wpjunior) * Alistair Roche * Dan Crosta * Viktor Kerkez From f7e22d2b8bc8acb2e4ab6d33c8814d8f6d49d63c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Thu, 16 May 2013 13:05:07 -0300 Subject: [PATCH 1104/1279] fixes for get_next_value --- mongoengine/fields.py | 2 +- tests/fields/fields.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 0b6486a..a56bad8 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1449,7 +1449,7 @@ class SequenceField(BaseField): data = collection.find_one({"_id": sequence_id}) if data: - return data['next'] + return data['next']+1 return 1 diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 527baa9..a9fed3c 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -1910,7 +1910,7 @@ class FieldTest(unittest.TestCase): for x in xrange(10): Person(name="Person %s" % x).save() - self.assertEqual(Person.id.get_next_value(), 10) + self.assertEqual(Person.id.get_next_value(), 11) self.db['mongoengine.counters'].drop() self.assertEqual(Person.id.get_next_value(), 1) From bc92f78afb3694f1eb78104f6dc09902516541b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Thu, 16 May 2013 13:12:49 -0300 Subject: [PATCH 1105/1279] fixes for value_decorator --- mongoengine/fields.py | 4 ++-- tests/fields/fields.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index a56bad8..b192961 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1449,9 +1449,9 @@ class SequenceField(BaseField): data = collection.find_one({"_id": sequence_id}) if data: - return data['next']+1 + return self.value_decorator(data['next']+1) - return 1 + return self.value_decorator(1) def get_sequence_name(self): if self.sequence_name: diff --git a/tests/fields/fields.py b/tests/fields/fields.py index a9fed3c..e803af8 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -1914,6 +1914,21 @@ class FieldTest(unittest.TestCase): self.db['mongoengine.counters'].drop() self.assertEqual(Person.id.get_next_value(), 1) + + class Person(Document): + id = SequenceField(primary_key=True, value_decorator=str) + name = StringField() + + self.db['mongoengine.counters'].drop() + Person.drop_collection() + + for x in xrange(10): + Person(name="Person %s" % x).save() + + self.assertEqual(Person.id.get_next_value(), '11') + self.db['mongoengine.counters'].drop() + + self.assertEqual(Person.id.get_next_value(), '1') def test_sequence_field_sequence_name(self): class Person(Document): From 36a3770673b34e912b894043f4c3d7ce8771c594 Mon Sep 17 00:00:00 2001 From: Daniel Axtens Date: Mon, 20 May 2013 15:49:01 +1000 Subject: [PATCH 1106/1279] If you need to read from another database, use switch_db not switch_collection. --- mongoengine/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 6c1045b..89627dc 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -388,7 +388,7 @@ class Document(BaseDocument): user.save() If you need to read from another database see - :class:`~mongoengine.context_managers.switch_collection` + :class:`~mongoengine.context_managers.switch_db` :param collection_name: The database alias to use for saving the document From 89f1c21f20bdbe5ab635f67b3f9f41c19108b54d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 20 May 2013 08:08:52 +0000 Subject: [PATCH 1107/1279] Updated AUTHORS (#325) --- AUTHORS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 0ff48e8..b3756e8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -159,4 +159,4 @@ that much better: * Nicolas Cortot * Alex (https://github.com/kelsta) * Jin Zhang - + * Daniel Axtens From 8165131419641205b4cba45110df1849ccb3009d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 20 May 2013 08:12:09 +0000 Subject: [PATCH 1108/1279] Doc updated --- mongoengine/fields.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index b192961..a2ba202 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1443,6 +1443,11 @@ class SequenceField(BaseField): return self.value_decorator(counter['next']) def get_next_value(self): + """Helper method to get the next value for previewing. + + .. warning:: There is no guarantee this will be the next value + as it is only fixed on set. + """ sequence_name = self.get_sequence_name() sequence_id = "%s.%s" % (sequence_name, self.name) collection = get_db(alias=self.db_alias)[self.collection_name] From 367f49ce1c6831d202b2ef511ce03f131456490e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 20 May 2013 08:12:50 +0000 Subject: [PATCH 1109/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index c3e50e4..07145b2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.0 ================ +- Added `get_next_value` preview for SequenceFields (#319) - Added no_sub_classes context manager and queryset helper (#312) - Querysets now utilises a local cache - Changed __len__ behavour in the queryset (#247, #311) From 6299015039895cadf518fcee7941267161f9ef8f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 20 May 2013 10:04:17 +0000 Subject: [PATCH 1110/1279] Updated pickling (#316) --- mongoengine/base/document.py | 18 +++++++++--------- tests/fixtures.py | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index c2ccc48..e3202b9 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -141,16 +141,16 @@ class BaseDocument(object): super(BaseDocument, self).__setattr__(name, value) def __getstate__(self): - removals = ("get_%s_display" % k - for k, v in self._fields.items() if v.choices) - for k in removals: - if hasattr(self, k): - delattr(self, k) - return self.__dict__ + data = {} + for k in ('_changed_fields', '_initialised', '_created'): + data[k] = getattr(self, k) + data['_data'] = self.to_mongo() + return data - def __setstate__(self, __dict__): - self.__dict__ = __dict__ - self.__set_field_display() + def __setstate__(self, data): + for k in ('_changed_fields', '_initialised', '_created'): + setattr(self, k, data[k]) + self._data = self.__class__._from_son(data["_data"])._data def __iter__(self): if 'id' in self._fields and 'id' not in self._fields_ordered: diff --git a/tests/fixtures.py b/tests/fixtures.py index a35f144..e207044 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -25,11 +25,11 @@ class PickleSignalsTest(Document): @classmethod def post_save(self, sender, document, created, **kwargs): - pickled = pickle.dumps(document) + pickled = pickle.dumps(document) @classmethod def post_delete(self, sender, document, **kwargs): - pickled = pickle.dumps(document) + pickled = pickle.dumps(document) signals.post_save.connect(PickleSignalsTest.post_save, sender=PickleSignalsTest) signals.post_delete.connect(PickleSignalsTest.post_delete, sender=PickleSignalsTest) From 56cd73823e7ed4b216bb740c612c21eea59fd1a7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 20 May 2013 10:09:16 +0000 Subject: [PATCH 1111/1279] Add backwards compat for pickle --- mongoengine/base/document.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index e3202b9..018adbf 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -148,9 +148,10 @@ class BaseDocument(object): return data def __setstate__(self, data): - for k in ('_changed_fields', '_initialised', '_created'): + if isinstance(data["_data"], SON): + data["_data"] = self.__class__._from_son(data["_data"])._data + for k in ('_changed_fields', '_initialised', '_created', '_data'): setattr(self, k, data[k]) - self._data = self.__class__._from_son(data["_data"])._data def __iter__(self): if 'id' in self._fields and 'id' not in self._fields_ordered: From a6bc870815d78021adeb57119b68376a44864f82 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 20 May 2013 10:10:53 +0000 Subject: [PATCH 1112/1279] Fixed pickle issues with collections (#316) --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index f014a9f..40ba450 100644 --- a/AUTHORS +++ b/AUTHORS @@ -160,3 +160,4 @@ that much better: * Alex (https://github.com/kelsta) * Jin Zhang * Daniel Axtens + * Leo-Naeka diff --git a/docs/changelog.rst b/docs/changelog.rst index 07145b2..46b641b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.0 ================ +- Fixed pickle issues with collections (#316) - Added `get_next_value` preview for SequenceFields (#319) - Added no_sub_classes context manager and queryset helper (#312) - Querysets now utilises a local cache From ebdd2d730cb2bafe2d62eb5935a77a86a6affc03 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 20 May 2013 10:20:43 +0000 Subject: [PATCH 1113/1279] Fixed querying ReferenceField custom_id (#317) --- docs/changelog.rst | 1 + mongoengine/fields.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 46b641b..383f9af 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.0 ================ +- Fixed querying ReferenceField custom_id (#317) - Fixed pickle issues with collections (#316) - Added `get_next_value` preview for SequenceFields (#319) - Added no_sub_classes context manager and queryset helper (#312) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index a2ba202..df2c19e 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -853,8 +853,6 @@ class ReferenceField(BaseField): if not self.dbref: return document.id return document - elif not self.dbref and isinstance(document, basestring): - return ObjectId(document) id_field_name = self.document_type._meta['id_field'] id_field = self.document_type._fields[id_field_name] From 5ef56116820f60e1761e37685ade6c1623373f65 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 20 May 2013 12:34:47 +0000 Subject: [PATCH 1114/1279] 0.8.0 is a go --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 0f8913a..7c8407b 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -15,7 +15,7 @@ import django __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + list(queryset.__all__) + signals.__all__ + list(errors.__all__)) -VERSION = (0, 8, 0, 'RC4') +VERSION = (0, 8, 0) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index be9c67b..1a26f47 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.8.0.RC4 +Version: 0.8.0 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 306f9c5ffd046a5702b98de9aa9ed47be6d88622 Mon Sep 17 00:00:00 2001 From: Mitar Date: Mon, 20 May 2013 17:30:41 -0700 Subject: [PATCH 1115/1279] importlib does not exist on Python 2.6. Use Django version. --- mongoengine/django/mongo_auth/models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mongoengine/django/mongo_auth/models.py b/mongoengine/django/mongo_auth/models.py index 9629e64..3529d8e 100644 --- a/mongoengine/django/mongo_auth/models.py +++ b/mongoengine/django/mongo_auth/models.py @@ -1,9 +1,8 @@ -from importlib import import_module - from django.conf import settings from django.contrib.auth.models import UserManager from django.core.exceptions import ImproperlyConfigured from django.db import models +from django.utils.importlib import import_module from django.utils.translation import ugettext_lazy as _ From d060da094f5415288fa2c27d5f4c887a04905f8b Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Mon, 20 May 2013 17:40:56 -0700 Subject: [PATCH 1116/1279] update pickling test case to show the error --- tests/document/instance.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/document/instance.py b/tests/document/instance.py index d972ae5..cdc6fe0 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1709,6 +1709,7 @@ class InstanceTest(unittest.TestCase): pickle_doc = PickleTest(number=1, string="One", lists=['1', '2']) pickle_doc.embedded = PickleEmbedded() + pickled_doc = pickle.dumps(pickle_doc) # make sure pickling works even before the doc is saved pickle_doc.save() pickled_doc = pickle.dumps(pickle_doc) From 9aa77bb3c967f3ceb5e14047791a7b8cc4176503 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 21 May 2013 07:07:17 +0000 Subject: [PATCH 1117/1279] Fixed pickle unsaved document regression (#327) --- docs/changelog.rst | 4 ++++ mongoengine/base/document.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 383f9af..6954cfd 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.8.1 +================ +- Fixed pickle unsaved document regression (#327) + Changes in 0.8.0 ================ - Fixed querying ReferenceField custom_id (#317) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 018adbf..719d886 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -143,7 +143,8 @@ class BaseDocument(object): def __getstate__(self): data = {} for k in ('_changed_fields', '_initialised', '_created'): - data[k] = getattr(self, k) + if hasattr(self, k): + data[k] = getattr(self, k) data['_data'] = self.to_mongo() return data From 50f1ca91d478136d1d39969dbc1c132b5b84a21a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 21 May 2013 09:05:55 +0000 Subject: [PATCH 1118/1279] Updated Changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6954cfd..c016676 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.1 ================ +- Fixed Python 2.6 django auth importlib issue (#326) - Fixed pickle unsaved document regression (#327) Changes in 0.8.0 From a7470360d2cb33d3d5c82b4c065511133fd1ea12 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 21 May 2013 09:12:09 +0000 Subject: [PATCH 1119/1279] Version bump --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 7c8407b..8c167f0 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -15,7 +15,7 @@ import django __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + list(queryset.__all__) + signals.__all__ + list(errors.__all__)) -VERSION = (0, 8, 0) +VERSION = (0, 8, 1) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 1a26f47..7c87b1c 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.8.0 +Version: 0.8.1 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 3ffc9dffc22ab326d22db02a84c5d90e514dc321 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 21 May 2013 09:37:22 +0000 Subject: [PATCH 1120/1279] Updated requirements for test suite --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 594f7f8..365791f 100644 --- a/setup.py +++ b/setup.py @@ -51,13 +51,13 @@ CLASSIFIERS = [ extra_opts = {} if sys.version_info[0] == 3: extra_opts['use_2to3'] = True - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6'] extra_opts['packages'] = find_packages(exclude=('tests',)) if "test" in sys.argv or "nosetests" in sys.argv: extra_opts['packages'].append("tests") extra_opts['package_data'] = {"tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"]} else: - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2==2.6'] extra_opts['packages'] = find_packages(exclude=('tests',)) setup(name='mongoengine', From a84e1f17bb209e294cf437f3c51a08207eb9bc9b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 21 May 2013 09:42:22 +0000 Subject: [PATCH 1121/1279] Fixing django tests for py 2.6 --- tests/test_django.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_django.py b/tests/test_django.py index f81213c..63e3245 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -275,7 +275,7 @@ class MongoAuthTest(unittest.TestCase): def test_user_manager(self): manager = get_user_model()._default_manager - self.assertIsInstance(manager, MongoUserManager) + self.assertTrue(isinstance(manager, MongoUserManager)) def test_user_manager_exception(self): manager = get_user_model()._default_manager @@ -285,14 +285,14 @@ class MongoAuthTest(unittest.TestCase): def test_create_user(self): manager = get_user_model()._default_manager user = manager.create_user(**self.user_data) - self.assertIsInstance(user, User) + self.assertTrue(isinstance(user, User)) db_user = User.objects.get(username='user') self.assertEqual(user.id, db_user.id) def test_authenticate(self): get_user_model()._default_manager.create_user(**self.user_data) user = authenticate(username='user', password='fail') - self.assertIsNone(user) + self.assertEqual(None, user) user = authenticate(username='user', password='test') db_user = User.objects.get(username='user') self.assertEqual(user.id, db_user.id) From 1eb643668244c696b1f9f2320503de39fbe0617b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 22 May 2013 10:29:45 +0000 Subject: [PATCH 1122/1279] Added get image by grid_id example --- tests/fields/file_tests.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py index 52bd88a..fa76175 100644 --- a/tests/fields/file_tests.py +++ b/tests/fields/file_tests.py @@ -407,6 +407,25 @@ class FileTest(unittest.TestCase): self.assertEqual(putfile, copy.copy(putfile)) self.assertEqual(putfile, copy.deepcopy(putfile)) + def test_get_image_by_grid_id(self): + + class TestImage(Document): + + image1 = ImageField() + image2 = ImageField() + + TestImage.drop_collection() + + t = TestImage() + t.image1.put(open(TEST_IMAGE_PATH, 'rb')) + t.image2.put(open(TEST_IMAGE2_PATH, 'rb')) + t.save() + + test = TestImage.objects.first() + grid_id = test.image1.grid_id + + self.assertEqual(1, TestImage.objects(Q(image1=grid_id) + or Q(image2=grid_id)).count()) if __name__ == '__main__': unittest.main() From c96a1b00cf86b0be61cdade0df3e48440a35c287 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 23 May 2013 19:09:05 +0000 Subject: [PATCH 1123/1279] Documentation cleanup (#328) --- mongoengine/queryset/queryset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 5da6295..4222459 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1479,7 +1479,7 @@ class QuerySet(object): # Deprecated def ensure_index(self, **kwargs): - """Deprecated use :func:`~Document.ensure_index`""" + """Deprecated use :func:`Document.ensure_index`""" msg = ("Doc.objects()._ensure_index() is deprecated. " "Use Doc.ensure_index() instead.") warnings.warn(msg, DeprecationWarning) From 5f0d86f509ff02b3f9c14405bde1a15c8ecda9b1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 23 May 2013 19:12:13 +0000 Subject: [PATCH 1124/1279] Upgrade doc fix (#330) --- docs/upgrade.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index fe9e4fa..6d9f529 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -116,8 +116,8 @@ eg:: # Mark all ReferenceFields as dirty and save for p in Person.objects: - p._mark_as_dirty('parent') - p._mark_as_dirty('friends') + p._mark_as_changed('parent') + p._mark_as_changed('friends') p.save() `An example test migration for ReferenceFields is available on github @@ -145,7 +145,7 @@ eg:: # Mark all ReferenceFields as dirty and save for a in Animal.objects: - a._mark_as_dirty('uuid') + a._mark_as_changed('uuid') a.save() `An example test migration for UUIDFields is available on github @@ -174,7 +174,7 @@ eg:: # Mark all ReferenceFields as dirty and save for p in Person.objects: - p._mark_as_dirty('balance') + p._mark_as_changed('balance') p.save() .. note:: DecimalField's have also been improved with the addition of precision From b4a98a40001348a1dd657a80c2f6c33dcf59901d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 23 May 2013 19:30:57 +0000 Subject: [PATCH 1125/1279] More upgrade clarifications #331 --- docs/upgrade.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 6d9f529..b5f3304 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -123,6 +123,10 @@ eg:: `An example test migration for ReferenceFields is available on github `_. +.. Note:: Internally mongoengine handles ReferenceFields the same, so they are + converted to DBRef on loading and ObjectIds or DBRefs depending on settings + on storage. + UUIDField --------- @@ -143,7 +147,7 @@ eg:: class Animal(Document): uuid = UUIDField() - # Mark all ReferenceFields as dirty and save + # Mark all UUIDFields as dirty and save for a in Animal.objects: a._mark_as_changed('uuid') a.save() @@ -172,7 +176,7 @@ eg:: class Person(Document): balance = DecimalField() - # Mark all ReferenceFields as dirty and save + # Mark all DecimalField's as dirty and save for p in Person.objects: p._mark_as_changed('balance') p.save() From c5ce96c391bf6d45c0395392bec28051727e6db4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 23 May 2013 19:44:05 +0000 Subject: [PATCH 1126/1279] Fix py3 test --- tests/fields/file_tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py index fa76175..b3b6108 100644 --- a/tests/fields/file_tests.py +++ b/tests/fields/file_tests.py @@ -409,6 +409,9 @@ class FileTest(unittest.TestCase): def test_get_image_by_grid_id(self): + if PY3: + raise SkipTest('PIL does not have Python 3 support') + class TestImage(Document): image1 = ImageField() From 774895ec8c43616b9e6f1ba0788dfe47f6cec4e1 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Thu, 23 May 2013 17:49:28 -0700 Subject: [PATCH 1127/1279] dont simplify queries with duplicate conditions --- mongoengine/queryset/visitor.py | 11 ++++++++--- tests/queryset/visitor.py | 6 ++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py index 95d11e8..024f454 100644 --- a/mongoengine/queryset/visitor.py +++ b/mongoengine/queryset/visitor.py @@ -23,6 +23,9 @@ class QNodeVisitor(object): return query +class DuplicateQueryConditionsError(InvalidQueryError): + pass + class SimplificationVisitor(QNodeVisitor): """Simplifies query trees by combinging unnecessary 'and' connection nodes into a single Q-object. @@ -33,7 +36,10 @@ class SimplificationVisitor(QNodeVisitor): # The simplification only applies to 'simple' queries if all(isinstance(node, Q) for node in combination.children): queries = [n.query for n in combination.children] - return Q(**self._query_conjunction(queries)) + try: + return Q(**self._query_conjunction(queries)) + except DuplicateQueryConditionsError: + pass return combination def _query_conjunction(self, queries): @@ -47,8 +53,7 @@ class SimplificationVisitor(QNodeVisitor): # to a single field intersection = ops.intersection(query_ops) if intersection: - msg = 'Duplicate query conditions: ' - raise InvalidQueryError(msg + ', '.join(intersection)) + raise DuplicateQueryConditionsError() query_ops.update(ops) combined_query.update(copy.deepcopy(query)) diff --git a/tests/queryset/visitor.py b/tests/queryset/visitor.py index 2e9195e..8443621 100644 --- a/tests/queryset/visitor.py +++ b/tests/queryset/visitor.py @@ -69,10 +69,8 @@ class QTest(unittest.TestCase): y = StringField() # Check than an error is raised when conflicting queries are anded - def invalid_combination(): - query = Q(x__lt=7) & Q(x__lt=3) - query.to_query(TestDoc) - self.assertRaises(InvalidQueryError, invalid_combination) + query = (Q(x__lt=7) & Q(x__lt=3)).to_query(TestDoc) + self.assertEqual(query, {'$and': [ {'x': {'$lt': 7}}, {'x': {'$lt': 3}} ]}) # Check normal cases work without an error query = Q(x__lt=7) & Q(x__gt=3) From ab4ff99105d3ed946fae2de0bb36ddcfa9cbc522 Mon Sep 17 00:00:00 2001 From: Ryan Witt Date: Fri, 24 May 2013 11:24:40 -0300 Subject: [PATCH 1128/1279] fix guide link --- docs/tutorial.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index c2f481b..0c592a0 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -298,5 +298,5 @@ Learning more about mongoengine ------------------------------- If you got this far you've made a great start, so well done! The next step on -your mongoengine journey is the `full user guide `_, where you -can learn indepth about how to use mongoengine and mongodb. \ No newline at end of file +your mongoengine journey is the `full user guide `_, where you +can learn indepth about how to use mongoengine and mongodb. From 2b6c42a56c3e5de144eceae663688bb4e69a7992 Mon Sep 17 00:00:00 2001 From: Ryan Witt Date: Fri, 24 May 2013 11:34:15 -0300 Subject: [PATCH 1129/1279] minor typos --- docs/guide/connecting.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index 8674b5e..854e2c3 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -36,7 +36,7 @@ MongoEngine supports :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetCl to use them please use a URI style connection and provide the `replicaSet` name in the connection kwargs. -Read preferences are supported throught the connection or via individual +Read preferences are supported through the connection or via individual queries by passing the read_preference :: Bar.objects().read_preference(ReadPreference.PRIMARY) @@ -83,7 +83,7 @@ reasons. The :class:`~mongoengine.context_managers.switch_db` context manager allows you to change the database alias for a given class allowing quick and easy -access to the same User document across databases.eg :: +access to the same User document across databases:: from mongoengine.context_managers import switch_db From 7a760f5640b77e1a17a783ca3a606818e523a384 Mon Sep 17 00:00:00 2001 From: Jin Zhang Date: Sat, 25 May 2013 06:46:23 -0600 Subject: [PATCH 1130/1279] Update django.rst --- docs/django.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/django.rst b/docs/django.rst index 09c91e7..da15188 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -27,9 +27,9 @@ MongoEngine includes a Django authentication backend, which uses MongoDB. The :class:`~mongoengine.Document`, but implements most of the methods and attributes that the standard Django :class:`User` model does - so the two are moderately compatible. Using this backend will allow you to store users in -MongoDB but still use many of the Django authentication infrastucture (such as +MongoDB but still use many of the Django authentication infrastructure (such as the :func:`login_required` decorator and the :func:`authenticate` function). To -enable the MongoEngine auth backend, add the following to you **settings.py** +enable the MongoEngine auth backend, add the following to your **settings.py** file:: AUTHENTICATION_BACKENDS = ( @@ -46,7 +46,7 @@ Custom User model ================= Django 1.5 introduced `Custom user Models ` -which can be used as an alternative the Mongoengine authentication backend. +which can be used as an alternative to the MongoEngine authentication backend. The main advantage of this option is that other components relying on :mod:`django.contrib.auth` and supporting the new swappable user model are more @@ -82,16 +82,16 @@ Sessions ======== Django allows the use of different backend stores for its sessions. MongoEngine provides a MongoDB-based session backend for Django, which allows you to use -sessions in you Django application with just MongoDB. To enable the MongoEngine +sessions in your Django application with just MongoDB. To enable the MongoEngine session backend, ensure that your settings module has ``'django.contrib.sessions.middleware.SessionMiddleware'`` in the ``MIDDLEWARE_CLASSES`` field and ``'django.contrib.sessions'`` in your ``INSTALLED_APPS``. From there, all you need to do is add the following line -into you settings module:: +into your settings module:: SESSION_ENGINE = 'mongoengine.django.sessions' -Django provides session cookie, which expires after ```SESSION_COOKIE_AGE``` seconds, but doesnt delete cookie at sessions backend, so ``'mongoengine.django.sessions'`` supports `mongodb TTL +Django provides session cookie, which expires after ```SESSION_COOKIE_AGE``` seconds, but doesn't delete cookie at sessions backend, so ``'mongoengine.django.sessions'`` supports `mongodb TTL `_. .. versionadded:: 0.2.1 From 159ef12ed78fcded1a6ccc1fee6dde1752dc870b Mon Sep 17 00:00:00 2001 From: ichuang Date: Mon, 27 May 2013 11:19:34 -0400 Subject: [PATCH 1131/1279] FileField should pass db_alias to GridFSProxy in __set__ call --- mongoengine/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index df2c19e..b588eaa 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1194,6 +1194,7 @@ class FileField(BaseField): # Create a new proxy object as we don't already have one instance._data[key] = self.proxy_class(key=key, instance=instance, + db_alias=self.db_alias, collection_name=self.collection_name) instance._data[key].put(value) else: From 4670f09a6720f523938355376849f7e54f08b0d5 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Mon, 27 May 2013 13:48:02 -0700 Subject: [PATCH 1132/1279] fix __set_state__ --- mongoengine/base/document.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 719d886..2ffcbc5 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -152,7 +152,8 @@ class BaseDocument(object): if isinstance(data["_data"], SON): data["_data"] = self.__class__._from_son(data["_data"])._data for k in ('_changed_fields', '_initialised', '_created', '_data'): - setattr(self, k, data[k]) + if k in data: + setattr(self, k, data[k]) def __iter__(self): if 'id' in self._fields and 'id' not in self._fields_ordered: From 18d8008b895d0f0a1f94bf23b0e93dba666ef4e7 Mon Sep 17 00:00:00 2001 From: Paul Swartz Date: Tue, 28 May 2013 15:59:32 -0400 Subject: [PATCH 1133/1279] if `dateutil` is available, use it to parse datetimes In particular, this picks up the default `datetime.isoformat()` output, with a "T" as the separator. --- mongoengine/fields.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index df2c19e..8ea48c2 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -7,6 +7,12 @@ import urllib2 import uuid import warnings from operator import itemgetter +try: + import dateutil +except ImportError: + dateutil = None +else: + import dateutil.parser import pymongo import gridfs @@ -371,6 +377,8 @@ class DateTimeField(BaseField): return value() # Attempt to parse a datetime: + if dateutil: + return dateutil.parser.parse(value) # value = smart_str(value) # split usecs, because they are not recognized by strptime. if '.' in value: From 1302316eb0ebd2c40c20402f5013c1b977f78cbc Mon Sep 17 00:00:00 2001 From: Paul Swartz Date: Tue, 28 May 2013 16:08:33 -0400 Subject: [PATCH 1134/1279] add some tests --- tests/fields/fields.py | 64 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tests/fields/fields.py b/tests/fields/fields.py index e803af8..6c3f49f 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -6,6 +6,11 @@ import datetime import unittest import uuid +try: + import dateutil +except ImportError: + dateutil = None + from decimal import Decimal from bson import Binary, DBRef, ObjectId @@ -482,6 +487,65 @@ class FieldTest(unittest.TestCase): LogEntry.drop_collection() + def test_datetime_usage(self): + """Tests for regular datetime fields""" + class LogEntry(Document): + date = DateTimeField() + + LogEntry.drop_collection() + + d1 = datetime.datetime(1970, 01, 01, 00, 00, 01) + log = LogEntry() + log.date = d1 + log.save() + + for query in (d1, d1.isoformat(' ')): + log1 = LogEntry.objects.get(date=query) + self.assertEqual(log, log1) + + if dateutil: + log1 = LogEntry.objects.get(date=d1.isoformat('T')) + self.assertEqual(log, log1) + + LogEntry.drop_collection() + + # create 60 log entries + for i in xrange(1950, 2010): + d = datetime.datetime(i, 01, 01, 00, 00, 01) + LogEntry(date=d).save() + + self.assertEqual(LogEntry.objects.count(), 60) + + # Test ordering + logs = LogEntry.objects.order_by("date") + count = logs.count() + i = 0 + while i == count - 1: + self.assertTrue(logs[i].date <= logs[i + 1].date) + i += 1 + + logs = LogEntry.objects.order_by("-date") + count = logs.count() + i = 0 + while i == count - 1: + self.assertTrue(logs[i].date >= logs[i + 1].date) + i += 1 + + # Test searching + logs = LogEntry.objects.filter(date__gte=datetime.datetime(1980, 1, 1)) + self.assertEqual(logs.count(), 30) + + logs = LogEntry.objects.filter(date__lte=datetime.datetime(1980, 1, 1)) + self.assertEqual(logs.count(), 30) + + logs = LogEntry.objects.filter( + date__lte=datetime.datetime(2011, 1, 1), + date__gte=datetime.datetime(2000, 1, 1), + ) + self.assertEqual(logs.count(), 10) + + LogEntry.drop_collection() + def test_complexdatetime_storage(self): """Tests for complex datetime fields - which can handle microseconds without rounding. From c0571beec82cedc5bd4f52463deb449b6226d89c Mon Sep 17 00:00:00 2001 From: Paul Swartz Date: Tue, 28 May 2013 17:18:54 -0400 Subject: [PATCH 1135/1279] fix change tracking for ComplexBaseFields --- mongoengine/base/fields.py | 6 ------ tests/fields/fields.py | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 72a9e8e..9f08c09 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -205,12 +205,6 @@ class ComplexBaseField(BaseField): return value - def __set__(self, instance, value): - """Descriptor for assigning a value to a field in a document. - """ - instance._data[self.name] = value - instance._mark_as_changed(self.name) - def to_python(self, value): """Convert a MongoDB-compatible type to a Python type. """ diff --git a/tests/fields/fields.py b/tests/fields/fields.py index e803af8..c9b3313 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -808,6 +808,27 @@ class FieldTest(unittest.TestCase): self.assertRaises(ValidationError, e.save) + def test_complex_field_same_value_not_changed(self): + """ + If a complex field is set to the same value, it should not be marked as + changed. + """ + class Simple(Document): + mapping = ListField() + + Simple.drop_collection() + e = Simple().save() + e.mapping = [] + self.assertEqual([], e._changed_fields) + + class Simple(Document): + mapping = DictField() + + Simple.drop_collection() + e = Simple().save() + e.mapping = {} + self.assertEqual([], e._changed_fields) + def test_list_field_complex(self): """Ensure that the list fields can handle the complex types.""" From 04592c876b6ff6fb0c11338499ed4e0bf6934330 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 29 May 2013 12:04:53 -0400 Subject: [PATCH 1136/1279] Moved pre_save after validation and determination of creation state; added pre_save_validation where pre_save had been. --- mongoengine/document.py | 6 ++++-- mongoengine/signals.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 89627dc..9946ffa 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -195,7 +195,7 @@ class Document(BaseDocument): the cascade save using cascade_kwargs which overwrites the existing kwargs with custom values """ - signals.pre_save.send(self.__class__, document=self) + signals.pre_save_validation.send(self.__class__, document=self) if validate: self.validate(clean=clean) @@ -206,7 +206,9 @@ class Document(BaseDocument): doc = self.to_mongo() created = ('_id' not in doc or self._created or force_insert) - + + signals.pre_save.send(self.__class__, document=self, created=created) + try: collection = self._get_collection() if created: diff --git a/mongoengine/signals.py b/mongoengine/signals.py index 52ef312..50f8e94 100644 --- a/mongoengine/signals.py +++ b/mongoengine/signals.py @@ -38,6 +38,7 @@ _signals = Namespace() pre_init = _signals.signal('pre_init') post_init = _signals.signal('post_init') +pre_save_validation = _signals.signal('pre_save_validation') pre_save = _signals.signal('pre_save') post_save = _signals.signal('post_save') pre_delete = _signals.signal('pre_delete') From 5d44e1d6ca8eb530ddc00c16c0d1cbf5452b90c8 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 29 May 2013 12:12:51 -0400 Subject: [PATCH 1137/1279] Added missing reference in __all__. --- mongoengine/signals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/signals.py b/mongoengine/signals.py index 50f8e94..f12ab1b 100644 --- a/mongoengine/signals.py +++ b/mongoengine/signals.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -__all__ = ['pre_init', 'post_init', 'pre_save', 'post_save', - 'pre_delete', 'post_delete'] +__all__ = ['pre_init', 'post_init', 'pre_save_validation', 'pre_save', + 'post_save', 'pre_delete', 'post_delete'] signals_available = False try: From 12f6a3f5a3b0e791614bc5fa9f2ab63c0e8adf69 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 29 May 2013 12:22:15 -0400 Subject: [PATCH 1138/1279] Added tests for pre_save_validation and updated tests for pre_save to encompass created flag. --- tests/test_signals.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_signals.py b/tests/test_signals.py index 32517dd..f1ce3c9 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -39,9 +39,18 @@ class SignalTests(unittest.TestCase): def post_init(cls, sender, document, **kwargs): signal_output.append('post_init signal, %s' % document) + @classmethod + def pre_save_validation(cls, sender, document, **kwargs): + signal_output.append('pre_save_validation signal, %s' % document) + @classmethod def pre_save(cls, sender, document, **kwargs): signal_output.append('pre_save signal, %s' % document) + if 'created' in kwargs: + if kwargs['created']: + signal_output.append('Is created') + else: + signal_output.append('Is updated') @classmethod def post_save(cls, sender, document, **kwargs): @@ -89,9 +98,18 @@ class SignalTests(unittest.TestCase): def post_init(cls, sender, document, **kwargs): signal_output.append('post_init Another signal, %s' % document) + @classmethod + def pre_save_validation(cls, sender, document, **kwargs): + signal_output.append('pre_save_validation Another signal, %s' % document) + @classmethod def pre_save(cls, sender, document, **kwargs): signal_output.append('pre_save Another signal, %s' % document) + if 'created' in kwargs: + if kwargs['created']: + signal_output.append('Is created') + else: + signal_output.append('Is updated') @classmethod def post_save(cls, sender, document, **kwargs): @@ -132,6 +150,7 @@ class SignalTests(unittest.TestCase): self.pre_signals = ( len(signals.pre_init.receivers), len(signals.post_init.receivers), + len(signals.pre_save_validation.receivers), len(signals.pre_save.receivers), len(signals.post_save.receivers), len(signals.pre_delete.receivers), @@ -142,6 +161,7 @@ class SignalTests(unittest.TestCase): signals.pre_init.connect(Author.pre_init, sender=Author) signals.post_init.connect(Author.post_init, sender=Author) + signals.pre_save_validation.connect(Author.pre_save_validation, sender=Author) signals.pre_save.connect(Author.pre_save, sender=Author) signals.post_save.connect(Author.post_save, sender=Author) signals.pre_delete.connect(Author.pre_delete, sender=Author) @@ -151,6 +171,7 @@ class SignalTests(unittest.TestCase): signals.pre_init.connect(Another.pre_init, sender=Another) signals.post_init.connect(Another.post_init, sender=Another) + signals.pre_save_validation.connect(Another.pre_save_validation, sender=Another) signals.pre_save.connect(Another.pre_save, sender=Another) signals.post_save.connect(Another.post_save, sender=Another) signals.pre_delete.connect(Another.pre_delete, sender=Another) @@ -165,6 +186,7 @@ class SignalTests(unittest.TestCase): signals.pre_delete.disconnect(self.Author.pre_delete) signals.post_save.disconnect(self.Author.post_save) signals.pre_save.disconnect(self.Author.pre_save) + signals.pre_save_validation.disconnect(self.Author.pre_save_validation) signals.pre_bulk_insert.disconnect(self.Author.pre_bulk_insert) signals.post_bulk_insert.disconnect(self.Author.post_bulk_insert) @@ -174,6 +196,7 @@ class SignalTests(unittest.TestCase): signals.pre_delete.disconnect(self.Another.pre_delete) signals.post_save.disconnect(self.Another.post_save) signals.pre_save.disconnect(self.Another.pre_save) + signals.pre_save_validation.disconnect(self.Another.pre_save_validation) signals.post_save.disconnect(self.ExplicitId.post_save) @@ -181,6 +204,7 @@ class SignalTests(unittest.TestCase): post_signals = ( len(signals.pre_init.receivers), len(signals.post_init.receivers), + len(signals.pre_save_validation.receivers), len(signals.pre_save.receivers), len(signals.post_save.receivers), len(signals.pre_delete.receivers), @@ -215,7 +239,9 @@ class SignalTests(unittest.TestCase): a1 = self.Author(name='Bill Shakespeare') self.assertEqual(self.get_signal_output(a1.save), [ + "pre_save_validation signal, Bill Shakespeare", "pre_save signal, Bill Shakespeare", + "Is created", "post_save signal, Bill Shakespeare", "Is created" ]) @@ -223,7 +249,9 @@ class SignalTests(unittest.TestCase): a1.reload() a1.name = 'William Shakespeare' self.assertEqual(self.get_signal_output(a1.save), [ + "pre_save_validation signal, William Shakespeare", "pre_save signal, William Shakespeare", + "Is updated", "post_save signal, William Shakespeare", "Is updated" ]) From 122d75f677724e66260561b34f5b86d9b32794c8 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 29 May 2013 12:23:32 -0400 Subject: [PATCH 1139/1279] Added pre_save_validation to signal list in documentation. --- docs/guide/signals.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/guide/signals.rst b/docs/guide/signals.rst index 75f81e2..bc31fbd 100644 --- a/docs/guide/signals.rst +++ b/docs/guide/signals.rst @@ -15,6 +15,7 @@ The following document signals exist in MongoEngine and are pretty self-explanat * `mongoengine.signals.pre_init` * `mongoengine.signals.post_init` + * `mongoengine.signals.pre_save_validation` * `mongoengine.signals.pre_save` * `mongoengine.signals.post_save` * `mongoengine.signals.pre_delete` From f28f336026c8ea20f607a4324a5f17ea6b581d5b Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 29 May 2013 13:17:08 -0400 Subject: [PATCH 1140/1279] Improved signals documentation and some typo fixes. --- docs/guide/defining-documents.rst | 2 +- docs/guide/signals.rst | 131 +++++++++++++++++++++++++----- 2 files changed, 111 insertions(+), 22 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 0ee5ad3..b5ba2bf 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -403,7 +403,7 @@ either a single field name, or a list or tuple of field names:: Skipping Document validation on save ------------------------------------ You can also skip the whole document validation process by setting -``validate=False`` when caling the :meth:`~mongoengine.document.Document.save` +``validate=False`` when calling the :meth:`~mongoengine.document.Document.save` method:: class Recipient(Document): diff --git a/docs/guide/signals.rst b/docs/guide/signals.rst index 75f81e2..3fef757 100644 --- a/docs/guide/signals.rst +++ b/docs/guide/signals.rst @@ -1,5 +1,6 @@ .. _signals: +======= Signals ======= @@ -7,36 +8,96 @@ Signals .. note:: - Signal support is provided by the excellent `blinker`_ library and - will gracefully fall back if it is not available. + Signal support is provided by the excellent `blinker`_ library. If you wish + to enable signal support this library must be installed, though it is not + required for MongoEngine to function. + +Overview +-------- + +Signals are found within the :module:`~mongoengine.signals` module. Unless +specified signals receive no additional arguments beyond the `sender` class and +`document` instance. Post-signals are only called if there were no exceptions +raised during the processing of their related function. + +Available signals include: + +`pre_init` + Called during the creation of a new :class:`~mongoengine.Document` or + :class:`~mongoengine.EmbeddedDocument` instance, after the constructor + arguments have been collected but before any additional processing has been + done to them. (I.e. assignment of default values.) Handlers for this signal + are passed the dictionary of arguments using the `values` keyword argument + and may modify this dictionary prior to returning. + +`post_init` + Called after all processing of a new :class:`~mongoengine.Document` or + :class:`~mongoengine.EmbeddedDocument` instance has been completed. + +`pre_save` + Called within :meth:`~mongoengine.document.Document.save` prior to performing + any actions. + +`post_save` + Called within :meth:`~mongoengine.document.Document.save` after all actions + (validation, insert/update, cascades, clearing dirty flags) have completed + successfully. Passed the additional boolean keyword argument `created` to + indicate if the save was an insert or an update. + +`pre_delete` + Called within :meth:`~mongoengine.document.Document.delete` prior to + attempting the delete operation. + +`post_delete` + Called within :meth:`~mongoengine.document.Document.delete` upon successful + deletion of the record. + +`pre_bulk_insert` + Called after validation of the documents to insert, but prior to any data + being written. In this case, the `document` argument is replaced by a + `documents` argument representing the list of documents being inserted. + +`post_bulk_insert` + Called after a successful bulk insert operation. As per `pre_bulk_insert`, + the `document` argument is omitted and replaced with a `documents` argument. + An additional boolean argument, `loaded`, identifies the contents of + `documents` as either :class:`~mongoengine.Document` instances when `True` or + simply a list of primary key values for the inserted records if `False`. -The following document signals exist in MongoEngine and are pretty self-explanatory: +Attaching Events +---------------- - * `mongoengine.signals.pre_init` - * `mongoengine.signals.post_init` - * `mongoengine.signals.pre_save` - * `mongoengine.signals.post_save` - * `mongoengine.signals.pre_delete` - * `mongoengine.signals.post_delete` - * `mongoengine.signals.pre_bulk_insert` - * `mongoengine.signals.post_bulk_insert` - -Example usage:: +After writing a handler function like the following:: + import logging + from datetime import datetime + from mongoengine import * from mongoengine import signals + + def update_modified(sender, document): + document.modified = datetime.utcnow() + +You attach the event handler to your :class:`~mongoengine.Document` or +:class:`~mongoengine.EmbeddedDocument` subclass:: + + class Record(Document): + modified = DateTimeField() + + signals.pre_save.connect(update_modified) + +While this is not the most elaborate document model, it does demonstrate the +concepts involved. As a more complete demonstration you can also define your +handlers within your subclass:: class Author(Document): name = StringField() - - def __unicode__(self): - return self.name - + @classmethod def pre_save(cls, sender, document, **kwargs): logging.debug("Pre Save: %s" % document.name) - + @classmethod def post_save(cls, sender, document, **kwargs): logging.debug("Post Save: %s" % document.name) @@ -45,16 +106,44 @@ Example usage:: logging.debug("Created") else: logging.debug("Updated") - + signals.pre_save.connect(Author.pre_save, sender=Author) signals.post_save.connect(Author.post_save, sender=Author) +Finally, you can also use this small decorator to quickly create a number of +signals and attach them to your :class:`~mongoengine.Document` or +:class:`~mongoengine.EmbeddedDocument` subclasses as class decorators:: -ReferenceFields and signals + def handler(event): + """Signal decorator to allow use of callback functions as class decorators.""" + + def decorator(fn): + def apply(cls): + event.connect(fn, sender=cls) + return cls + + fn.apply = apply + return fn + + return decorator + +Using the first example of updating a modification time the code is now much +cleaner looking while still allowing manual execution of the callback:: + + @handler(signals.pre_save) + def update_modified(sender, document): + document.modified = datetime.utcnow() + + @update_modified.apply + class Record(Document): + modified = DateTimeField() + + +ReferenceFields and Signals --------------------------- Currently `reverse_delete_rules` do not trigger signals on the other part of -the relationship. If this is required you must manually handled the +the relationship. If this is required you must manually handle the reverse deletion. .. _blinker: http://pypi.python.org/pypi/blinker From 35f084ba76d0a6aaf2eaf900530c885ac953da19 Mon Sep 17 00:00:00 2001 From: Alice Bevan-McGregor Date: Wed, 29 May 2013 13:23:18 -0400 Subject: [PATCH 1141/1279] Fixed :module: reference in docs and added myself to authors. --- AUTHORS | 1 + docs/guide/signals.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 40ba450..ba69bc6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -161,3 +161,4 @@ that much better: * Jin Zhang * Daniel Axtens * Leo-Naeka + * Alice Bevan-McGregor (https://github.com/amcgregor/) diff --git a/docs/guide/signals.rst b/docs/guide/signals.rst index 3fef757..16c1cd0 100644 --- a/docs/guide/signals.rst +++ b/docs/guide/signals.rst @@ -15,7 +15,7 @@ Signals Overview -------- -Signals are found within the :module:`~mongoengine.signals` module. Unless +Signals are found within the `mongoengine.signals` module. Unless specified signals receive no additional arguments beyond the `sender` class and `document` instance. Post-signals are only called if there were no exceptions raised during the processing of their related function. From 4c9e90732e711dfe3fac8f4f887330673147f51c Mon Sep 17 00:00:00 2001 From: Nigel McNie Date: Thu, 30 May 2013 16:37:40 +1200 Subject: [PATCH 1142/1279] Apply defaults to fields with None value at 'set' time. If a field has a default, and you explicitly set it to None, the behaviour before this patch was very confusing: class Person(Document): created = DateTimeField(default=datetime.datetime.utcnow) >>> p = Person(created=None) >>> p.created datetime.datetime(2013, 5, 30, 0, 18, 20, 242628) >>> p.created datetime.datetime(2013, 5, 30, 0, 18, 20, 995248) >>> p.created datetime.datetime(2013, 5, 30, 0, 18, 21, 370578) It would be stored as None, and then at 'get' time, the default would be applied. As you can see, if the default is a generator, this leads to some crazy behaviour. There's an argument that if I asked it to be set to None, why not respect that? But I don't think that's how the rest of mongoengine seems to work (for example, setting a field to None seems to mean it doesn't even get set in mongo - as opposed to being set but with a 'null' value). Besides, as the code shows above, you'd expect p.created to return None. So clearly, mongoengine is already expecting None to mean 'default' where a default is available. This bug also interacts nastily with required=True - if you're forcibly setting the field to None, then at validation time, the None will fail validation despite a perfectly valid default being available. With this patch, when the field is set, the default is immediately applied. This means any generation happens once, the getter always returns the same value, and 'required' validation always respects the default. Note: this breakage seems to be new since mongoengine 0.8. --- mongoengine/base/fields.py | 6 ++++++ tests/fields/fields.py | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 72a9e8e..9454023 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -82,6 +82,12 @@ class BaseField(object): def __set__(self, instance, value): """Descriptor for assigning a value to a field in a document. """ + if value is None: + value = self.default + # Allow callable default values + if callable(value): + value = value() + if instance._initialised: try: if (self.name not in instance._data or diff --git a/tests/fields/fields.py b/tests/fields/fields.py index e803af8..32c33f7 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -44,6 +44,19 @@ class FieldTest(unittest.TestCase): self.assertEqual(person._fields['age'].help_text, "Your real age") self.assertEqual(person._fields['userid'].verbose_name, "User Identity") + class Person2(Document): + created = DateTimeField(default=datetime.datetime.utcnow) + + person = Person2() + date1 = person.created + date2 = person.created + self.assertEqual(date1, date2) + + person = Person2(created=None) + date1 = person.created + date2 = person.created + self.assertEqual(date1, date2) + def test_required_values(self): """Ensure that required field constraints are enforced. """ From 0493bbbc76457b12dfaaf2a5558a84bc36a1b62a Mon Sep 17 00:00:00 2001 From: Jiequan Date: Sun, 2 Jun 2013 20:46:51 +0800 Subject: [PATCH 1143/1279] Update upgrade.rst Added docs for the new function: clean() --- docs/upgrade.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index b5f3304..c3d3182 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -91,6 +91,13 @@ the case and the data is set only in the ``document._data`` dictionary: :: File "", line 1, in AttributeError: 'Animal' object has no attribute 'size' +The Document class has introduced a reserved function `clean()`, which will be +called before saving the document. If your document class happen to have a method +with the same name, please try rename it. + + def clean(self): + pass + ReferenceField -------------- From 0fb976a80a3a9a90e7acea5eca07c8eec1c2941c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Jun 2013 13:01:48 +0000 Subject: [PATCH 1144/1279] Added Ryan to AUTHORS #334 --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 40ba450..39621e9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -161,3 +161,4 @@ that much better: * Jin Zhang * Daniel Axtens * Leo-Naeka + * Ryan Witt (https://github.com/ryanwitt) From 2fe1c20475443cadcf8a38a826485bffdb0b617a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Jun 2013 13:03:07 +0000 Subject: [PATCH 1145/1279] Added Jiequan to AUTHORS #354 --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 39621e9..c3d6ff9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -162,3 +162,4 @@ that much better: * Daniel Axtens * Leo-Naeka * Ryan Witt (https://github.com/ryanwitt) + * Jiequan (https://github.com/Jiequan) \ No newline at end of file From b2f78fadd92823120ea73b3ed27ebac7585e1fcb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Jun 2013 13:05:52 +0000 Subject: [PATCH 1146/1279] Added test for upsert & update_one #336 --- tests/queryset/queryset.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 01c53d0..5425741 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -545,6 +545,15 @@ class QuerySetTest(unittest.TestCase): self.assertEqual("Bob", bob.name) self.assertEqual(30, bob.age) + def test_upsert_one(self): + self.Person.drop_collection() + + self.Person.objects(name="Bob", age=30).update_one(upsert=True) + + bob = self.Person.objects.first() + self.assertEqual("Bob", bob.name) + self.assertEqual(30, bob.age) + def test_set_on_insert(self): self.Person.drop_collection() From 8d2e7b43726c44eb1bff5ffb3e012d9aa6ec2d6a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Jun 2013 13:31:35 +0000 Subject: [PATCH 1147/1279] Django session ttl index expiry fixed (#329) --- mongoengine/django/sessions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index 29583f5..c90807e 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -39,7 +39,7 @@ class MongoSession(Document): 'indexes': [ { 'fields': ['expire_date'], - 'expireAfterSeconds': settings.SESSION_COOKIE_AGE + 'expireAfterSeconds': 0 } ] } From fbc46a52af132dd82a207350e43072ba956d5b21 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Jun 2013 13:31:42 +0000 Subject: [PATCH 1148/1279] Updated changelog --- docs/changelog.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index c016676..02fb824 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,13 @@ Changelog ========= + +Changes in 0.8.2 +================ +- Django session ttl index expiry fixed (#329) +- Fixed pickle.loads (#342) +- Documentation fixes + Changes in 0.8.1 ================ - Fixed Python 2.6 django auth importlib issue (#326) From 7e6b035ca21282cb57762311fe876ec912be31e1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Jun 2013 13:32:30 +0000 Subject: [PATCH 1149/1279] Added hensom to AUTHORS #329 --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index c3d6ff9..11f2fa7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -162,4 +162,5 @@ that much better: * Daniel Axtens * Leo-Naeka * Ryan Witt (https://github.com/ryanwitt) - * Jiequan (https://github.com/Jiequan) \ No newline at end of file + * Jiequan (https://github.com/Jiequan) + * hensom (https://github.com/hensom) From ceece5a7e214b4bfae9902e03dcd6130c7783b22 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Jun 2013 13:38:58 +0000 Subject: [PATCH 1150/1279] Improved PIL detection for tests --- tests/fields/file_tests.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py index b3b6108..d9dec6f 100644 --- a/tests/fields/file_tests.py +++ b/tests/fields/file_tests.py @@ -14,6 +14,12 @@ from mongoengine import * from mongoengine.connection import get_db from mongoengine.python_support import PY3, b, StringIO +try: + from PIL import Image + HAS_PIL = True +except ImportError: + HAS_PIL = False + TEST_IMAGE_PATH = os.path.join(os.path.dirname(__file__), 'mongoengine.png') TEST_IMAGE2_PATH = os.path.join(os.path.dirname(__file__), 'mongodb_leaf.png') @@ -255,8 +261,8 @@ class FileTest(unittest.TestCase): self.assertFalse(test_file.the_file in [{"test": 1}]) def test_image_field(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') + if not HAS_PIL: + raise SkipTest('PIL not installed') class TestImage(Document): image = ImageField() @@ -278,8 +284,8 @@ class FileTest(unittest.TestCase): t.image.delete() def test_image_field_reassigning(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') + if not HAS_PIL: + raise SkipTest('PIL not installed') class TestFile(Document): the_file = ImageField() @@ -294,8 +300,8 @@ class FileTest(unittest.TestCase): self.assertEqual(test_file.the_file.size, (45, 101)) def test_image_field_resize(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') + if not HAS_PIL: + raise SkipTest('PIL not installed') class TestImage(Document): image = ImageField(size=(185, 37)) @@ -317,8 +323,8 @@ class FileTest(unittest.TestCase): t.image.delete() def test_image_field_resize_force(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') + if not HAS_PIL: + raise SkipTest('PIL not installed') class TestImage(Document): image = ImageField(size=(185, 37, True)) @@ -340,8 +346,8 @@ class FileTest(unittest.TestCase): t.image.delete() def test_image_field_thumbnail(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') + if not HAS_PIL: + raise SkipTest('PIL not installed') class TestImage(Document): image = ImageField(thumbnail_size=(92, 18)) @@ -409,8 +415,8 @@ class FileTest(unittest.TestCase): def test_get_image_by_grid_id(self): - if PY3: - raise SkipTest('PIL does not have Python 3 support') + if not HAS_PIL: + raise SkipTest('PIL not installed') class TestImage(Document): From 4c8dfc3fc25e03c8e58209d48cb524194934b3ff Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 3 Jun 2013 15:40:54 +0000 Subject: [PATCH 1151/1279] Fixed Doc.objects(read_preference=X) not setting read preference (#352) --- docs/changelog.rst | 1 + mongoengine/queryset/queryset.py | 10 +++++++--- tests/queryset/queryset.py | 5 ++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 02fb824..6a4dab6 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.8.2 ================ +- Fixed Doc.objects(read_preference=X) not setting read preference (#352) - Django session ttl index expiry fixed (#329) - Fixed pickle.loads (#342) - Documentation fixes diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 4222459..00a0abc 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -104,13 +104,17 @@ class QuerySet(object): raise InvalidQueryError(msg) query &= q_obj - queryset = self.clone() + if read_preference is None: + queryset = self.clone() + else: + # Use the clone provided when setting read_preference + queryset = self.read_preference(read_preference) + queryset._query_obj &= query queryset._mongo_query = None queryset._cursor_obj = None - if read_preference is not None: - queryset.read_preference(read_preference) queryset._class_check = class_check + return queryset def __len__(self): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 5425741..507408d 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3098,7 +3098,10 @@ class QuerySetTest(unittest.TestCase): self.assertEqual([], bars) self.assertRaises(ConfigurationError, Bar.objects, - read_preference='Primary') + read_preference='Primary') + + bars = Bar.objects(read_preference=ReadPreference.SECONDARY_PREFERRED) + self.assertEqual(bars._read_preference, ReadPreference.SECONDARY_PREFERRED) def test_json_simple(self): From 5447c6e947fb6cf16c3995cd24fe7618e0707855 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Jun 2013 09:08:13 +0000 Subject: [PATCH 1152/1279] DateTimeField now auto converts valid datetime isostrings into dates (#343) --- docs/changelog.rst | 2 ++ mongoengine/fields.py | 25 +++++++++++++++++++------ setup.py | 2 +- tests/fields/fields.py | 12 ++++++++++-- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6a4dab6..6b666aa 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,8 @@ Changelog Changes in 0.8.2 ================ +- DateTimeField now auto converts valid datetime isostrings into dates (#343) +- DateTimeField now uses dateutil for parsing if available (#343) - Fixed Doc.objects(read_preference=X) not setting read preference (#352) - Django session ttl index expiry fixed (#329) - Fixed pickle.loads (#342) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 8ea48c2..2b0e395 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -7,6 +7,7 @@ import urllib2 import uuid import warnings from operator import itemgetter + try: import dateutil except ImportError: @@ -353,6 +354,11 @@ class BooleanField(BaseField): class DateTimeField(BaseField): """A datetime field. + Uses the python-dateutil library if available alternatively use time.strptime + to parse the dates. Note: python-dateutil's parser is fully featured and when + installed you can utilise it to convert varing types of date formats into valid + python datetime objects. + Note: Microseconds are rounded to the nearest millisecond. Pre UTC microsecond support is effecively broken. Use :class:`~mongoengine.fields.ComplexDateTimeField` if you @@ -360,13 +366,11 @@ class DateTimeField(BaseField): """ def validate(self, value): - if not isinstance(value, (datetime.datetime, datetime.date)): + new_value = self.to_mongo(value) + if not isinstance(new_value, (datetime.datetime, datetime.date)): self.error(u'cannot parse date "%s"' % value) def to_mongo(self, value): - return self.prepare_query_value(None, value) - - def prepare_query_value(self, op, value): if value is None: return value if isinstance(value, datetime.datetime): @@ -376,10 +380,16 @@ class DateTimeField(BaseField): if callable(value): return value() + if not isinstance(value, basestring): + return None + # Attempt to parse a datetime: if dateutil: - return dateutil.parser.parse(value) - # value = smart_str(value) + try: + return dateutil.parser.parse(value) + except ValueError: + return None + # split usecs, because they are not recognized by strptime. if '.' in value: try: @@ -404,6 +414,9 @@ class DateTimeField(BaseField): except ValueError: return None + def prepare_query_value(self, op, value): + return self.to_mongo(value) + class ComplexDateTimeField(StringField): """ diff --git a/setup.py b/setup.py index 365791f..1888828 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ if sys.version_info[0] == 3: extra_opts['packages'].append("tests") extra_opts['package_data'] = {"tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"]} else: - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2==2.6'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2==2.6', 'python-dateutil==1.5'] extra_opts['packages'] = find_packages(exclude=('tests',)) setup(name='mongoengine', diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 6c3f49f..00a4bd7 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -408,9 +408,16 @@ class FieldTest(unittest.TestCase): log.time = datetime.date.today() log.validate() + log.time = datetime.datetime.now().isoformat(' ') + log.validate() + + if dateutil: + log.time = datetime.datetime.now().isoformat('T') + log.validate() + log.time = -1 self.assertRaises(ValidationError, log.validate) - log.time = '1pm' + log.time = 'ABC' self.assertRaises(ValidationError, log.validate) def test_datetime_tz_aware_mark_as_changed(self): @@ -497,6 +504,7 @@ class FieldTest(unittest.TestCase): d1 = datetime.datetime(1970, 01, 01, 00, 00, 01) log = LogEntry() log.date = d1 + log.validate() log.save() for query in (d1, d1.isoformat(' ')): @@ -1993,7 +2001,7 @@ class FieldTest(unittest.TestCase): self.db['mongoengine.counters'].drop() self.assertEqual(Person.id.get_next_value(), '1') - + def test_sequence_field_sequence_name(self): class Person(Document): id = SequenceField(primary_key=True, sequence_name='jelly') From 4244e7569b7b7d1c131fb4aa842810d976e3d655 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Jun 2013 09:35:44 +0000 Subject: [PATCH 1153/1279] Added pre_save_post_validation signal (#345) --- docs/changelog.rst | 1 + mongoengine/document.py | 8 ++++---- mongoengine/signals.py | 4 ++-- setup.py | 2 +- tests/test_signals.py | 36 ++++++++++++++++++------------------ 5 files changed, 26 insertions(+), 25 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6b666aa..006dfc7 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.8.2 ================ +- Added pre_save_post_validation signal (#345) - DateTimeField now auto converts valid datetime isostrings into dates (#343) - DateTimeField now uses dateutil for parsing if available (#343) - Fixed Doc.objects(read_preference=X) not setting read preference (#352) diff --git a/mongoengine/document.py b/mongoengine/document.py index 9946ffa..92d0631 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -195,7 +195,7 @@ class Document(BaseDocument): the cascade save using cascade_kwargs which overwrites the existing kwargs with custom values """ - signals.pre_save_validation.send(self.__class__, document=self) + signals.pre_save.send(self.__class__, document=self) if validate: self.validate(clean=clean) @@ -206,9 +206,9 @@ class Document(BaseDocument): doc = self.to_mongo() created = ('_id' not in doc or self._created or force_insert) - - signals.pre_save.send(self.__class__, document=self, created=created) - + + signals.pre_save_post_validation.send(self.__class__, document=self, created=created) + try: collection = self._get_collection() if created: diff --git a/mongoengine/signals.py b/mongoengine/signals.py index f12ab1b..06fb8b4 100644 --- a/mongoengine/signals.py +++ b/mongoengine/signals.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__all__ = ['pre_init', 'post_init', 'pre_save_validation', 'pre_save', +__all__ = ['pre_init', 'post_init', 'pre_save', 'pre_save_post_validation', 'post_save', 'pre_delete', 'post_delete'] signals_available = False @@ -38,8 +38,8 @@ _signals = Namespace() pre_init = _signals.signal('pre_init') post_init = _signals.signal('post_init') -pre_save_validation = _signals.signal('pre_save_validation') pre_save = _signals.signal('pre_save') +pre_save_post_validation = _signals.signal('pre_save_post_validation') post_save = _signals.signal('post_save') pre_delete = _signals.signal('pre_delete') post_delete = _signals.signal('post_delete') diff --git a/setup.py b/setup.py index 1888828..effb6f1 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ if sys.version_info[0] == 3: extra_opts['packages'].append("tests") extra_opts['package_data'] = {"tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"]} else: - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2==2.6', 'python-dateutil==1.5'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2==2.6', 'python-dateutil'] extra_opts['packages'] = find_packages(exclude=('tests',)) setup(name='mongoengine', diff --git a/tests/test_signals.py b/tests/test_signals.py index f1ce3c9..65289c2 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -40,12 +40,12 @@ class SignalTests(unittest.TestCase): signal_output.append('post_init signal, %s' % document) @classmethod - def pre_save_validation(cls, sender, document, **kwargs): - signal_output.append('pre_save_validation signal, %s' % document) + def pre_save(cls, sender, document, **kwargs): + signal_output.append('pre_save signal,, %s' % document) @classmethod - def pre_save(cls, sender, document, **kwargs): - signal_output.append('pre_save signal, %s' % document) + def pre_save_post_validation(cls, sender, document, **kwargs): + signal_output.append('pre_save_post_validation signal, %s' % document) if 'created' in kwargs: if kwargs['created']: signal_output.append('Is created') @@ -98,13 +98,13 @@ class SignalTests(unittest.TestCase): def post_init(cls, sender, document, **kwargs): signal_output.append('post_init Another signal, %s' % document) - @classmethod - def pre_save_validation(cls, sender, document, **kwargs): - signal_output.append('pre_save_validation Another signal, %s' % document) - @classmethod def pre_save(cls, sender, document, **kwargs): signal_output.append('pre_save Another signal, %s' % document) + + @classmethod + def pre_save_post_validation(cls, sender, document, **kwargs): + signal_output.append('pre_save_post_validation Another signal, %s' % document) if 'created' in kwargs: if kwargs['created']: signal_output.append('Is created') @@ -150,8 +150,8 @@ class SignalTests(unittest.TestCase): self.pre_signals = ( len(signals.pre_init.receivers), len(signals.post_init.receivers), - len(signals.pre_save_validation.receivers), len(signals.pre_save.receivers), + len(signals.pre_save_post_validation.receivers), len(signals.post_save.receivers), len(signals.pre_delete.receivers), len(signals.post_delete.receivers), @@ -161,8 +161,8 @@ class SignalTests(unittest.TestCase): signals.pre_init.connect(Author.pre_init, sender=Author) signals.post_init.connect(Author.post_init, sender=Author) - signals.pre_save_validation.connect(Author.pre_save_validation, sender=Author) signals.pre_save.connect(Author.pre_save, sender=Author) + signals.pre_save_post_validation.connect(Author.pre_save_post_validation, sender=Author) signals.post_save.connect(Author.post_save, sender=Author) signals.pre_delete.connect(Author.pre_delete, sender=Author) signals.post_delete.connect(Author.post_delete, sender=Author) @@ -171,8 +171,8 @@ class SignalTests(unittest.TestCase): signals.pre_init.connect(Another.pre_init, sender=Another) signals.post_init.connect(Another.post_init, sender=Another) - signals.pre_save_validation.connect(Another.pre_save_validation, sender=Another) signals.pre_save.connect(Another.pre_save, sender=Another) + signals.pre_save_post_validation.connect(Another.pre_save_post_validation, sender=Another) signals.post_save.connect(Another.post_save, sender=Another) signals.pre_delete.connect(Another.pre_delete, sender=Another) signals.post_delete.connect(Another.post_delete, sender=Another) @@ -185,8 +185,8 @@ class SignalTests(unittest.TestCase): signals.post_delete.disconnect(self.Author.post_delete) signals.pre_delete.disconnect(self.Author.pre_delete) signals.post_save.disconnect(self.Author.post_save) + signals.pre_save_post_validation.disconnect(self.Author.pre_save_post_validation) signals.pre_save.disconnect(self.Author.pre_save) - signals.pre_save_validation.disconnect(self.Author.pre_save_validation) signals.pre_bulk_insert.disconnect(self.Author.pre_bulk_insert) signals.post_bulk_insert.disconnect(self.Author.post_bulk_insert) @@ -195,8 +195,8 @@ class SignalTests(unittest.TestCase): signals.post_delete.disconnect(self.Another.post_delete) signals.pre_delete.disconnect(self.Another.pre_delete) signals.post_save.disconnect(self.Another.post_save) + signals.pre_save_post_validation.disconnect(self.Another.pre_save_post_validation) signals.pre_save.disconnect(self.Another.pre_save) - signals.pre_save_validation.disconnect(self.Another.pre_save_validation) signals.post_save.disconnect(self.ExplicitId.post_save) @@ -204,8 +204,8 @@ class SignalTests(unittest.TestCase): post_signals = ( len(signals.pre_init.receivers), len(signals.post_init.receivers), - len(signals.pre_save_validation.receivers), len(signals.pre_save.receivers), + len(signals.pre_save_post_validation.receivers), len(signals.post_save.receivers), len(signals.pre_delete.receivers), len(signals.post_delete.receivers), @@ -239,8 +239,8 @@ class SignalTests(unittest.TestCase): a1 = self.Author(name='Bill Shakespeare') self.assertEqual(self.get_signal_output(a1.save), [ - "pre_save_validation signal, Bill Shakespeare", - "pre_save signal, Bill Shakespeare", + "pre_save signal,, Bill Shakespeare", + "pre_save_post_validation signal, Bill Shakespeare", "Is created", "post_save signal, Bill Shakespeare", "Is created" @@ -249,8 +249,8 @@ class SignalTests(unittest.TestCase): a1.reload() a1.name = 'William Shakespeare' self.assertEqual(self.get_signal_output(a1.save), [ - "pre_save_validation signal, William Shakespeare", - "pre_save signal, William Shakespeare", + "pre_save signal,, William Shakespeare", + "pre_save_post_validation signal, William Shakespeare", "Is updated", "post_save signal, William Shakespeare", "Is updated" From 626a3369b522de7fc0f1af268d833e0207290237 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Jun 2013 09:51:58 +0000 Subject: [PATCH 1154/1279] Removed unused var in _get_changed_fields (#347) --- docs/changelog.rst | 1 + mongoengine/base/document.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 006dfc7..bc6f283 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.8.2 ================ +- Removed unused var in _get_changed_fields (#347) - Added pre_save_post_validation signal (#345) - DateTimeField now auto converts valid datetime isostrings into dates (#343) - DateTimeField now uses dateutil for parsing if available (#343) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 2ffcbc5..e2944fb 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -392,7 +392,7 @@ class BaseDocument(object): if field_value: field_value._clear_changed_fields() - def _get_changed_fields(self, key='', inspected=None): + def _get_changed_fields(self, inspected=None): """Returns a list of all fields that have explicitly been changed. """ EmbeddedDocument = _import_class("EmbeddedDocument") @@ -423,7 +423,7 @@ class BaseDocument(object): if (isinstance(field, (EmbeddedDocument, DynamicEmbeddedDocument)) and db_field_name not in _changed_fields): # Find all embedded fields that have been changed - changed = field._get_changed_fields(key, inspected) + changed = field._get_changed_fields(inspected) _changed_fields += ["%s%s" % (key, k) for k in changed if k] elif (isinstance(field, (list, tuple, dict)) and db_field_name not in _changed_fields): @@ -437,7 +437,7 @@ class BaseDocument(object): if not hasattr(value, '_get_changed_fields'): continue list_key = "%s%s." % (key, index) - changed = value._get_changed_fields(list_key, inspected) + changed = value._get_changed_fields(inspected) _changed_fields += ["%s%s" % (list_key, k) for k in changed if k] return _changed_fields From f27a53653b0eac351af283757f01ba9bf94323b4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Jun 2013 09:56:38 +0000 Subject: [PATCH 1155/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index bc6f283..640ac39 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.8.2 ================ +- Removed customised __set__ change tracking in ComplexBaseField (#344) - Removed unused var in _get_changed_fields (#347) - Added pre_save_post_validation signal (#345) - DateTimeField now auto converts valid datetime isostrings into dates (#343) From d94a191656b0761bf554ea15ec49e683e3085ef0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Jun 2013 10:20:24 +0000 Subject: [PATCH 1156/1279] Updated Changelog added test for #341 --- docs/changelog.rst | 1 + tests/fields/file_tests.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 640ac39..0113a72 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.8.2 ================ +- FileField now honouring db_alias (#341) - Removed customised __set__ change tracking in ComplexBaseField (#344) - Removed unused var in _get_changed_fields (#347) - Added pre_save_post_validation signal (#345) diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py index d9dec6f..5bcc3a2 100644 --- a/tests/fields/file_tests.py +++ b/tests/fields/file_tests.py @@ -394,6 +394,14 @@ class FileTest(unittest.TestCase): self.assertEqual(test_file.the_file.read(), b('Hello, World!')) + test_file = TestFile.objects.first() + test_file.the_file = b('HELLO, WORLD!') + test_file.save() + + test_file = TestFile.objects.first() + self.assertEqual(test_file.the_file.read(), + b('HELLO, WORLD!')) + def test_copyable(self): class PutFile(Document): the_file = FileField() From 0d35e3a3e91b98ede77921d93b3c5a76132ff15f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Jun 2013 10:20:49 +0000 Subject: [PATCH 1157/1279] Added debugging for query counter --- mongoengine/context_managers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index 1280e11..a5e2524 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -189,7 +189,10 @@ class query_counter(object): def __eq__(self, value): """ == Compare querycounter. """ - return value == self._get_count() + counter = self._get_count() + if value != counter: + print [x for x in self.db.system.profile.find()] + return value == counter def __ne__(self, value): """ != Compare querycounter. """ From ee725354db066ea11f25dd01387f8d3dcb721c6c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Jun 2013 10:46:38 +0000 Subject: [PATCH 1158/1279] Querysets are now lest restrictive when querying duplicate fields (#332, #333) --- docs/changelog.rst | 1 + mongoengine/queryset/visitor.py | 5 +++-- tests/queryset/visitor.py | 30 ++++++++++++++++++++++++------ 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0113a72..3e86988 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.8.2 ================ +- Querysets are now lest restrictive when querying duplicate fields (#332, #333) - FileField now honouring db_alias (#341) - Removed customised __set__ change tracking in ComplexBaseField (#344) - Removed unused var in _get_changed_fields (#347) diff --git a/mongoengine/queryset/visitor.py b/mongoengine/queryset/visitor.py index 024f454..41f4ebf 100644 --- a/mongoengine/queryset/visitor.py +++ b/mongoengine/queryset/visitor.py @@ -26,6 +26,7 @@ class QNodeVisitor(object): class DuplicateQueryConditionsError(InvalidQueryError): pass + class SimplificationVisitor(QNodeVisitor): """Simplifies query trees by combinging unnecessary 'and' connection nodes into a single Q-object. @@ -39,6 +40,7 @@ class SimplificationVisitor(QNodeVisitor): try: return Q(**self._query_conjunction(queries)) except DuplicateQueryConditionsError: + # Cannot be simplified pass return combination @@ -127,8 +129,7 @@ class QCombination(QNode): # If the child is a combination of the same type, we can merge its # children directly into this combinations children if isinstance(node, QCombination) and node.operation == operation: - # self.children += node.children - self.children.append(node) + self.children += node.children else: self.children.append(node) diff --git a/tests/queryset/visitor.py b/tests/queryset/visitor.py index 8443621..0bb6f69 100644 --- a/tests/queryset/visitor.py +++ b/tests/queryset/visitor.py @@ -68,9 +68,11 @@ class QTest(unittest.TestCase): x = IntField() y = StringField() - # Check than an error is raised when conflicting queries are anded query = (Q(x__lt=7) & Q(x__lt=3)).to_query(TestDoc) - self.assertEqual(query, {'$and': [ {'x': {'$lt': 7}}, {'x': {'$lt': 3}} ]}) + self.assertEqual(query, {'$and': [{'x': {'$lt': 7}}, {'x': {'$lt': 3}}]}) + + query = (Q(y="a") & Q(x__lt=7) & Q(x__lt=3)).to_query(TestDoc) + self.assertEqual(query, {'$and': [{'y': "a"}, {'x': {'$lt': 7}}, {'x': {'$lt': 3}}]}) # Check normal cases work without an error query = Q(x__lt=7) & Q(x__gt=3) @@ -323,10 +325,26 @@ class QTest(unittest.TestCase): pk = ObjectId() User(email='example@example.com', pk=pk).save() - self.assertEqual(1, User.objects.filter( - Q(email='example@example.com') | - Q(name='John Doe') - ).limit(2).filter(pk=pk).count()) + self.assertEqual(1, User.objects.filter(Q(email='example@example.com') | + Q(name='John Doe')).limit(2).filter(pk=pk).count()) + + def test_chained_q_or_filtering(self): + + class Post(EmbeddedDocument): + name = StringField(required=True) + + class Item(Document): + postables = ListField(EmbeddedDocumentField(Post)) + + Item.drop_collection() + + Item(postables=[Post(name="a"), Post(name="b")]).save() + Item(postables=[Post(name="a"), Post(name="c")]).save() + Item(postables=[Post(name="a"), Post(name="b"), Post(name="c")]).save() + + self.assertEqual(Item.objects(Q(postables__name="a") & Q(postables__name="b")).count(), 2) + self.assertEqual(Item.objects.filter(postables__name="a").filter(postables__name="b").count(), 2) + if __name__ == '__main__': unittest.main() From d47134bbf13a73e0a4b2168f709305a4bca93430 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Jun 2013 11:03:50 +0000 Subject: [PATCH 1159/1279] Reload forces read preference to be PRIMARY (#355) --- docs/changelog.rst | 1 + mongoengine/document.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3e86988..20e2046 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.8.2 ================ +- Reload forces read preference to be PRIMARY (#355) - Querysets are now lest restrictive when querying duplicate fields (#332, #333) - FileField now honouring db_alias (#341) - Removed customised __set__ change tracking in ComplexBaseField (#344) diff --git a/mongoengine/document.py b/mongoengine/document.py index 92d0631..e04e2bc 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -3,6 +3,7 @@ import warnings import pymongo import re +from pymongo.read_preferences import ReadPreference from bson.dbref import DBRef from mongoengine import signals from mongoengine.base import (DocumentMetaclass, TopLevelDocumentMetaclass, @@ -421,8 +422,9 @@ class Document(BaseDocument): .. versionchanged:: 0.6 Now chainable """ id_field = self._meta['id_field'] - obj = self._qs.filter(**{id_field: self[id_field]} - ).limit(1).select_related(max_depth=max_depth) + obj = self._qs.read_preference(ReadPreference.PRIMARY).filter( + **{id_field: self[id_field]}).limit(1).select_related(max_depth=max_depth) + if obj: obj = obj[0] else: From eeb5a83e98c598ad18b6183495a89cd0c7d1cf32 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Jun 2013 16:35:25 +0000 Subject: [PATCH 1160/1279] Added lock when calling doc.Delete() for when signals have no sender (#350) --- docs/changelog.rst | 1 + mongoengine/document.py | 3 +- mongoengine/queryset/queryset.py | 17 +++++--- tests/test_signals.py | 70 ++------------------------------ 4 files changed, 16 insertions(+), 75 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 20e2046..df24a5f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.8.2 ================ +- Added lock when calling doc.Delete() for when signals have no sender (#350) - Reload forces read preference to be PRIMARY (#355) - Querysets are now lest restrictive when querying duplicate fields (#332, #333) - FileField now honouring db_alias (#341) diff --git a/mongoengine/document.py b/mongoengine/document.py index e04e2bc..5edfc81 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -347,11 +347,10 @@ class Document(BaseDocument): signals.pre_delete.send(self.__class__, document=self) try: - self._qs.filter(**self._object_key).delete(write_concern=write_concern) + self._qs.filter(**self._object_key).delete(write_concern=write_concern, _from_doc_delete=True) except pymongo.errors.OperationFailure, err: message = u'Could not delete document (%s)' % err.message raise OperationError(message) - signals.post_delete.send(self.__class__, document=self) def switch_db(self, db_alias): diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 00a0abc..5077f89 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -407,7 +407,7 @@ class QuerySet(object): self._len = count return count - def delete(self, write_concern=None): + def delete(self, write_concern=None, _from_doc_delete=False): """Delete the documents matched by the query. :param write_concern: Extra keyword arguments are passed down which @@ -416,20 +416,25 @@ class QuerySet(object): ``save(..., write_concern={w: 2, fsync: True}, ...)`` will wait until at least two servers have recorded the write and will force an fsync on the primary server. + :param _from_doc_delete: True when called from document delete therefore + signals will have been triggered so don't loop. """ queryset = self.clone() doc = queryset._document + if not write_concern: + write_concern = {} + + # Handle deletes where skips or limits have been applied or + # there is an untriggered delete signal has_delete_signal = signals.signals_available and ( signals.pre_delete.has_receivers_for(self._document) or signals.post_delete.has_receivers_for(self._document)) - if not write_concern: - write_concern = {} + call_document_delete = (queryset._skip or queryset._limit or + has_delete_signal) and not _from_doc_delete - # Handle deletes where skips or limits have been applied or has a - # delete signal - if queryset._skip or queryset._limit or has_delete_signal: + if call_document_delete: for doc in queryset: doc.delete(write_concern=write_concern) return diff --git a/tests/test_signals.py b/tests/test_signals.py index 65289c2..27614bd 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -41,7 +41,7 @@ class SignalTests(unittest.TestCase): @classmethod def pre_save(cls, sender, document, **kwargs): - signal_output.append('pre_save signal,, %s' % document) + signal_output.append('pre_save signal, %s' % document) @classmethod def pre_save_post_validation(cls, sender, document, **kwargs): @@ -83,54 +83,6 @@ class SignalTests(unittest.TestCase): self.Author = Author Author.drop_collection() - class Another(Document): - name = StringField() - - def __unicode__(self): - return self.name - - @classmethod - def pre_init(cls, sender, document, **kwargs): - signal_output.append('pre_init Another signal, %s' % cls.__name__) - signal_output.append(str(kwargs['values'])) - - @classmethod - def post_init(cls, sender, document, **kwargs): - signal_output.append('post_init Another signal, %s' % document) - - @classmethod - def pre_save(cls, sender, document, **kwargs): - signal_output.append('pre_save Another signal, %s' % document) - - @classmethod - def pre_save_post_validation(cls, sender, document, **kwargs): - signal_output.append('pre_save_post_validation Another signal, %s' % document) - if 'created' in kwargs: - if kwargs['created']: - signal_output.append('Is created') - else: - signal_output.append('Is updated') - - @classmethod - def post_save(cls, sender, document, **kwargs): - signal_output.append('post_save Another signal, %s' % document) - if 'created' in kwargs: - if kwargs['created']: - signal_output.append('Is created') - else: - signal_output.append('Is updated') - - @classmethod - def pre_delete(cls, sender, document, **kwargs): - signal_output.append('pre_delete Another signal, %s' % document) - - @classmethod - def post_delete(cls, sender, document, **kwargs): - signal_output.append('post_delete Another signal, %s' % document) - - self.Another = Another - Another.drop_collection() - class ExplicitId(Document): id = IntField(primary_key=True) @@ -169,14 +121,6 @@ class SignalTests(unittest.TestCase): signals.pre_bulk_insert.connect(Author.pre_bulk_insert, sender=Author) signals.post_bulk_insert.connect(Author.post_bulk_insert, sender=Author) - signals.pre_init.connect(Another.pre_init, sender=Another) - signals.post_init.connect(Another.post_init, sender=Another) - signals.pre_save.connect(Another.pre_save, sender=Another) - signals.pre_save_post_validation.connect(Another.pre_save_post_validation, sender=Another) - signals.post_save.connect(Another.post_save, sender=Another) - signals.pre_delete.connect(Another.pre_delete, sender=Another) - signals.post_delete.connect(Another.post_delete, sender=Another) - signals.post_save.connect(ExplicitId.post_save, sender=ExplicitId) def tearDown(self): @@ -190,14 +134,6 @@ class SignalTests(unittest.TestCase): signals.pre_bulk_insert.disconnect(self.Author.pre_bulk_insert) signals.post_bulk_insert.disconnect(self.Author.post_bulk_insert) - signals.pre_init.disconnect(self.Another.pre_init) - signals.post_init.disconnect(self.Another.post_init) - signals.post_delete.disconnect(self.Another.post_delete) - signals.pre_delete.disconnect(self.Another.pre_delete) - signals.post_save.disconnect(self.Another.post_save) - signals.pre_save_post_validation.disconnect(self.Another.pre_save_post_validation) - signals.pre_save.disconnect(self.Another.pre_save) - signals.post_save.disconnect(self.ExplicitId.post_save) # Check that all our signals got disconnected properly. @@ -239,7 +175,7 @@ class SignalTests(unittest.TestCase): a1 = self.Author(name='Bill Shakespeare') self.assertEqual(self.get_signal_output(a1.save), [ - "pre_save signal,, Bill Shakespeare", + "pre_save signal, Bill Shakespeare", "pre_save_post_validation signal, Bill Shakespeare", "Is created", "post_save signal, Bill Shakespeare", @@ -249,7 +185,7 @@ class SignalTests(unittest.TestCase): a1.reload() a1.name = 'William Shakespeare' self.assertEqual(self.get_signal_output(a1.save), [ - "pre_save signal,, William Shakespeare", + "pre_save signal, William Shakespeare", "pre_save_post_validation signal, William Shakespeare", "Is updated", "post_save signal, William Shakespeare", From 74a3fd7596c66da36aa3e7fd77ed05665d0712de Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 4 Jun 2013 16:59:25 +0000 Subject: [PATCH 1161/1279] Added queryset delete tests for signals --- tests/test_signals.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/tests/test_signals.py b/tests/test_signals.py index 27614bd..50e5e6b 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -83,6 +83,24 @@ class SignalTests(unittest.TestCase): self.Author = Author Author.drop_collection() + class Another(Document): + + name = StringField() + + def __unicode__(self): + return self.name + + @classmethod + def pre_delete(cls, sender, document, **kwargs): + signal_output.append('pre_delete signal, %s' % document) + + @classmethod + def post_delete(cls, sender, document, **kwargs): + signal_output.append('post_delete signal, %s' % document) + + self.Another = Another + Another.drop_collection() + class ExplicitId(Document): id = IntField(primary_key=True) @@ -121,6 +139,9 @@ class SignalTests(unittest.TestCase): signals.pre_bulk_insert.connect(Author.pre_bulk_insert, sender=Author) signals.post_bulk_insert.connect(Author.post_bulk_insert, sender=Author) + signals.pre_delete.connect(Another.pre_delete, sender=Another) + signals.post_delete.connect(Another.post_delete, sender=Another) + signals.post_save.connect(ExplicitId.post_save, sender=ExplicitId) def tearDown(self): @@ -134,6 +155,9 @@ class SignalTests(unittest.TestCase): signals.pre_bulk_insert.disconnect(self.Author.pre_bulk_insert) signals.post_bulk_insert.disconnect(self.Author.post_bulk_insert) + signals.post_delete.disconnect(self.Another.post_delete) + signals.pre_delete.disconnect(self.Another.pre_delete) + signals.post_save.disconnect(self.ExplicitId.post_save) # Check that all our signals got disconnected properly. @@ -216,7 +240,14 @@ class SignalTests(unittest.TestCase): "Not loaded", ]) - self.Author.objects.delete() + def test_queryset_delete_signals(self): + """ Queryset delete should throw some signals. """ + + self.Another(name='Bill Shakespeare').save() + self.assertEqual(self.get_signal_output(self.Another.objects.delete), [ + 'pre_delete signal, Bill Shakespeare', + 'post_delete signal, Bill Shakespeare', + ]) def test_signals_with_explicit_doc_ids(self): """ Model saves must have a created flag the first time.""" From eba81e368b8ca2377b85f2ac2ca0f6bd51a5e4a9 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Tue, 4 Jun 2013 15:32:23 -0700 Subject: [PATCH 1162/1279] dont use $in for _cls queries with a single subclass --- mongoengine/queryset/queryset.py | 5 ++++- tests/queryset/queryset.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 5077f89..dc5fab4 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -71,7 +71,10 @@ class QuerySet(object): # If inheritance is allowed, only return instances and instances of # subclasses of the class being used if document._meta.get('allow_inheritance') is True: - self._initial_query = {"_cls": {"$in": self._document._subclasses}} + if len(self._document._subclasses) == 1: + self._initial_query = {"_cls": self._document._subclasses[0]} + else: + self._initial_query = {"_cls": {"$in": self._document._subclasses}} self._loaded_fields = QueryFieldList(always_include=['_cls']) self._cursor_obj = None self._limit = None diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 507408d..07ddf2d 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3392,6 +3392,34 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(B.objects.get(a=a).a, a) self.assertEqual(B.objects.get(a=a.id).a, a) + def test_cls_query_in_subclassed_docs(self): + + class Animal(Document): + name = StringField() + + meta = { + 'allow_inheritance': True + } + + class Dog(Animal): + pass + + class Cat(Animal): + pass + + self.assertEqual(Animal.objects(name='Charlie')._query, { + 'name': 'Charlie', + '_cls': { '$in': ('Animal', 'Animal.Dog', 'Animal.Cat') } + }) + self.assertEqual(Dog.objects(name='Charlie')._query, { + 'name': 'Charlie', + '_cls': 'Animal.Dog' + }) + self.assertEqual(Cat.objects(name='Charlie')._query, { + 'name': 'Charlie', + '_cls': 'Animal.Cat' + }) + if __name__ == '__main__': unittest.main() From 27e8aa9c6815d1e514e003a7f7b32e98425d0d03 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 5 Jun 2013 09:30:01 +0000 Subject: [PATCH 1163/1279] Added comment about why temp debugging exists --- mongoengine/context_managers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index a5e2524..db0830d 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -190,6 +190,7 @@ class query_counter(object): def __eq__(self, value): """ == Compare querycounter. """ counter = self._get_count() + # Temp debugging to try and understand intermittent travis-ci failures if value != counter: print [x for x in self.db.system.profile.find()] return value == counter From 940dfff625774aeb443434e23628279f5fbbc52e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 5 Jun 2013 09:49:26 +0000 Subject: [PATCH 1164/1279] Code cleanup --- mongoengine/queryset/queryset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index dc5fab4..7adfa65 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -185,7 +185,6 @@ class QuerySet(object): try: queryset._cursor_obj = queryset._cursor[key] queryset._skip, queryset._limit = key.start, key.stop - queryset._limit if key.start and key.stop: queryset._limit = key.stop - key.start except IndexError, err: From 1a54dad643b562539cc539f84801802831b5634b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 5 Jun 2013 10:42:41 +0000 Subject: [PATCH 1165/1279] Filter out index scan for pymongo cache --- mongoengine/context_managers.py | 6 ++---- tests/queryset/queryset.py | 7 +++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index db0830d..13ed100 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -190,9 +190,6 @@ class query_counter(object): def __eq__(self, value): """ == Compare querycounter. """ counter = self._get_count() - # Temp debugging to try and understand intermittent travis-ci failures - if value != counter: - print [x for x in self.db.system.profile.find()] return value == counter def __ne__(self, value): @@ -225,6 +222,7 @@ class query_counter(object): def _get_count(self): """ Get the number of queries. """ - count = self.db.system.profile.find().count() - self.counter + ignore_query = {"ns": {"$ne": "%s.system.indexes" % self.db.name}} + count = self.db.system.profile.find(ignore_query).count() - self.counter self.counter += 1 return count diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 07ddf2d..21df22c 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -631,14 +631,13 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(q, 1) # 1 for the insert Blog.drop_collection() + Blog.ensure_indexes() + with query_counter() as q: self.assertEqual(q, 0) - Blog.ensure_indexes() - self.assertEqual(q, 1) - Blog.objects.insert(blogs) - self.assertEqual(q, 3) # 1 for insert, and 1 for in bulk fetch (3 in total) + self.assertEqual(q, 2) # 1 for insert, and 1 for in bulk fetch Blog.drop_collection() From ce44843e27605a096d22ad6fd086ba616e7aab5f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 5 Jun 2013 11:11:02 +0000 Subject: [PATCH 1166/1279] Doc fix for #340 --- docs/guide/querying.rst | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index f1b6470..1350130 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -15,11 +15,8 @@ fetch documents from the database:: .. note:: - Once the iteration finishes (when :class:`StopIteration` is raised), - :meth:`~mongoengine.queryset.QuerySet.rewind` will be called so that the - :class:`~mongoengine.queryset.QuerySet` may be iterated over again. The - results of the first iteration are *not* cached, so the database will be hit - each time the :class:`~mongoengine.queryset.QuerySet` is iterated over. + As of MongoEngine 0.8 the querysets utilise a local cache. So iterating + it multiple times will only cause a single query. Filtering queries ================= From a246154961b4625d140ea4b9cc619302abfaee6c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 5 Jun 2013 11:31:13 +0000 Subject: [PATCH 1167/1279] Fixed hashing of EmbeddedDocuments (#348) --- docs/changelog.rst | 1 + mongoengine/base/document.py | 2 +- tests/document/instance.py | 8 ++++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index df24a5f..b61a06d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.8.2 ================ +- Fixed hashing of EmbeddedDocuments (#348) - Added lock when calling doc.Delete() for when signals have no sender (#350) - Reload forces read preference to be PRIMARY (#355) - Querysets are now lest restrictive when querying duplicate fields (#332, #333) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index e2944fb..ca154a2 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -215,7 +215,7 @@ class BaseDocument(object): return not self.__eq__(other) def __hash__(self): - if self.pk is None: + if getattr(self, 'pk', None) is None: # For new object return super(BaseDocument, self).__hash__() else: diff --git a/tests/document/instance.py b/tests/document/instance.py index cdc6fe0..f29cec2 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1705,6 +1705,14 @@ class InstanceTest(unittest.TestCase): self.assertTrue(u1 in all_user_set) + def test_embedded_document_hash(self): + """Test embedded document can be hashed + """ + class User(EmbeddedDocument): + pass + + hash(User()) + def test_picklable(self): pickle_doc = PickleTest(number=1, string="One", lists=['1', '2']) From e5648a4af96d0681e3fe6b0a7657b86d3ab8ee34 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 5 Jun 2013 11:45:08 +0000 Subject: [PATCH 1168/1279] ImageFields now include PIL error messages if invalid error (#353) --- docs/changelog.rst | 1 + mongoengine/fields.py | 4 ++-- tests/fields/file_tests.py | 11 +++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b61a06d..55cae98 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.8.2 ================ +- ImageFields now include PIL error messages if invalid error (#353) - Fixed hashing of EmbeddedDocuments (#348) - Added lock when calling doc.Delete() for when signals have no sender (#350) - Reload forces read preference to be PRIMARY (#355) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 9bc18e0..451f7ac 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1259,8 +1259,8 @@ class ImageGridFsProxy(GridFSProxy): try: img = Image.open(file_obj) img_format = img.format - except: - raise ValidationError('Invalid image') + except Exception, e: + raise ValidationError('Invalid image: %s' % e) if (field.size and (img.size[0] > field.size['width'] or img.size[1] > field.size['height'])): diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py index 5bcc3a2..dfef9ee 100644 --- a/tests/fields/file_tests.py +++ b/tests/fields/file_tests.py @@ -269,6 +269,17 @@ class FileTest(unittest.TestCase): TestImage.drop_collection() + with tempfile.TemporaryFile() as f: + f.write(b("Hello World!")) + f.flush() + + t = TestImage() + try: + t.image.put(f) + self.fail("Should have raised an invalidation error") + except ValidationError, e: + self.assertEquals("%s" % e, "Invalid image: cannot identify image file") + t = TestImage() t.image.put(open(TEST_IMAGE_PATH, 'rb')) t.save() From eb1df23e68ae88c10b44b6af45c77d38e61bee44 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 5 Jun 2013 11:50:26 +0000 Subject: [PATCH 1169/1279] Updated AUTHORS (#340, #348, #353) --- AUTHORS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AUTHORS b/AUTHORS index 3176238..4977f73 100644 --- a/AUTHORS +++ b/AUTHORS @@ -164,3 +164,6 @@ that much better: * Ryan Witt (https://github.com/ryanwitt) * Jiequan (https://github.com/Jiequan) * hensom (https://github.com/hensom) + * zhy0216 (https://github.com/zhy0216) + * istinspring (https://github.com/istinspring) + * Massimo Santini (https://github.com/mapio) From f8904a5504c74eb20b9b54f55eae8db569ae3e34 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 5 Jun 2013 12:14:22 +0000 Subject: [PATCH 1170/1279] Explicitly set w:1 if None in save --- mongoengine/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 5edfc81..563f57a 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -202,7 +202,7 @@ class Document(BaseDocument): self.validate(clean=clean) if not write_concern: - write_concern = {} + write_concern = {"w": 1} doc = self.to_mongo() From 5cb281223149a34d9446dedfa0d051ce1c7eb1a7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 5 Jun 2013 13:03:15 +0000 Subject: [PATCH 1171/1279] Reverting Fixed hashing of EmbeddedDocuments (#348) --- docs/changelog.rst | 1 - mongoengine/document.py | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 55cae98..8ccd395 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -6,7 +6,6 @@ Changelog Changes in 0.8.2 ================ - ImageFields now include PIL error messages if invalid error (#353) -- Fixed hashing of EmbeddedDocuments (#348) - Added lock when calling doc.Delete() for when signals have no sender (#350) - Reload forces read preference to be PRIMARY (#355) - Querysets are now lest restrictive when querying duplicate fields (#332, #333) diff --git a/mongoengine/document.py b/mongoengine/document.py index 563f57a..585fcf7 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -1,9 +1,11 @@ import warnings +import hashlib import pymongo import re from pymongo.read_preferences import ReadPreference +from bson import ObjectId from bson.dbref import DBRef from mongoengine import signals from mongoengine.base import (DocumentMetaclass, TopLevelDocumentMetaclass, @@ -53,6 +55,9 @@ class EmbeddedDocument(BaseDocument): return self._data == other._data return False + def __ne__(self, other): + return not self.__eq__(other) + class Document(BaseDocument): """The base class used for defining the structure and properties of From c3a065dd3322d0642b325ae812dda79325e37f29 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 5 Jun 2013 13:44:21 +0000 Subject: [PATCH 1172/1279] Removing old test re: #348 --- tests/document/instance.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index f29cec2..cdc6fe0 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -1705,14 +1705,6 @@ class InstanceTest(unittest.TestCase): self.assertTrue(u1 in all_user_set) - def test_embedded_document_hash(self): - """Test embedded document can be hashed - """ - class User(EmbeddedDocument): - pass - - hash(User()) - def test_picklable(self): pickle_doc = PickleTest(number=1, string="One", lists=['1', '2']) From ad15781d8f1b06562ab4fdcf255b60805952387e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 6 Jun 2013 13:31:52 +0000 Subject: [PATCH 1173/1279] Fixed amibiguity and differing behaviour regarding field defaults (#349) Now field defaults are king, unsetting or setting to None on a field with a default means the default is reapplied. --- AUTHORS | 1 + docs/apireference.rst | 1 + docs/changelog.rst | 1 + mongoengine/base/fields.py | 38 ++++++-- mongoengine/document.py | 2 +- mongoengine/queryset/queryset.py | 6 +- tests/fields/fields.py | 144 ++++++++++++++++++++++++++----- 7 files changed, 160 insertions(+), 33 deletions(-) diff --git a/AUTHORS b/AUTHORS index 4977f73..4caed40 100644 --- a/AUTHORS +++ b/AUTHORS @@ -167,3 +167,4 @@ that much better: * zhy0216 (https://github.com/zhy0216) * istinspring (https://github.com/istinspring) * Massimo Santini (https://github.com/mapio) + * Nigel McNie (https://github.com/nigelmcnie) diff --git a/docs/apireference.rst b/docs/apireference.rst index 37370e2..0fa410e 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -54,6 +54,7 @@ Querying Fields ====== +.. autoclass:: mongoengine.base.fields.BaseField .. autoclass:: mongoengine.fields.StringField .. autoclass:: mongoengine.fields.URLField .. autoclass:: mongoengine.fields.EmailField diff --git a/docs/changelog.rst b/docs/changelog.rst index 8ccd395..bce26c5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.8.2 ================ +- Fixed amibiguity and differing behaviour regarding field defaults (#349) - ImageFields now include PIL error messages if invalid error (#353) - Added lock when calling doc.Delete() for when signals have no sender (#350) - Reload forces read preference to be PRIMARY (#355) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index 35075ec..e4c88a7 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -36,6 +36,29 @@ class BaseField(object): unique=False, unique_with=None, primary_key=False, validation=None, choices=None, verbose_name=None, help_text=None): + """ + :param db_field: The database field to store this field in + (defaults to the name of the field) + :param name: Depreciated - use db_field + :param required: If the field is required. Whether it has to have a + value or not. Defaults to False. + :param default: (optional) The default value for this field if no value + has been set (or if the value has been unset). It Can be a + callable. + :param unique: Is the field value unique or not. Defaults to False. + :param unique_with: (optional) The other field this field should be + unique with. + :param primary_key: Mark this field as the primary key. Defaults to False. + :param validation: (optional) A callable to validate the value of the + field. Generally this is deprecated in favour of the + `FIELD.validate` method + :param choices: (optional) The valid choices + :param verbose_name: (optional) The verbose name for the field. + Designed to be human readable and is often used when generating + model forms from the document model. + :param help_text: (optional) The help text for this field and is often + used when generating model forms from the document model. + """ self.db_field = (db_field or name) if not primary_key else '_id' if name: msg = "Fields' 'name' attribute deprecated in favour of 'db_field'" @@ -65,14 +88,9 @@ class BaseField(object): if instance is None: # Document class being used rather than a document object return self - # Get value from document instance if available, if not use default - value = instance._data.get(self.name) - if value is None: - value = self.default - # Allow callable default values - if callable(value): - value = value() + # Get value from document instance if available + value = instance._data.get(self.name) EmbeddedDocument = _import_class('EmbeddedDocument') if isinstance(value, EmbeddedDocument) and value._instance is None: @@ -82,9 +100,11 @@ class BaseField(object): def __set__(self, instance, value): """Descriptor for assigning a value to a field in a document. """ - if value is None: + + # If setting to None and theres a default + # Then set the value to the default value + if value is None and self.default is not None: value = self.default - # Allow callable default values if callable(value): value = value() diff --git a/mongoengine/document.py b/mongoengine/document.py index 585fcf7..8e7ccc2 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -206,7 +206,7 @@ class Document(BaseDocument): if validate: self.validate(clean=clean) - if not write_concern: + if write_concern is None: write_concern = {"w": 1} doc = self.to_mongo() diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 7adfa65..d58a13b 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -348,7 +348,7 @@ class QuerySet(object): """ Document = _import_class('Document') - if not write_concern: + if write_concern is None: write_concern = {} docs = doc_or_docs @@ -424,7 +424,7 @@ class QuerySet(object): queryset = self.clone() doc = queryset._document - if not write_concern: + if write_concern is None: write_concern = {} # Handle deletes where skips or limits have been applied or @@ -490,7 +490,7 @@ class QuerySet(object): if not update and not upsert: raise OperationError("No update parameters, would remove data") - if not write_concern: + if write_concern is None: write_concern = {} queryset = self.clone() diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 5118437..3e48a21 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -34,33 +34,137 @@ class FieldTest(unittest.TestCase): self.db.drop_collection('fs.files') self.db.drop_collection('fs.chunks') - def test_default_values(self): + def test_default_values_nothing_set(self): """Ensure that default field values are used when creating a document. """ class Person(Document): name = StringField() - age = IntField(default=30, help_text="Your real age") - userid = StringField(default=lambda: 'test', verbose_name="User Identity") - - person = Person(name='Test Person') - self.assertEqual(person._data['age'], 30) - self.assertEqual(person._data['userid'], 'test') - self.assertEqual(person._fields['name'].help_text, None) - self.assertEqual(person._fields['age'].help_text, "Your real age") - self.assertEqual(person._fields['userid'].verbose_name, "User Identity") - - class Person2(Document): + age = IntField(default=30, required=False) + userid = StringField(default=lambda: 'test', required=True) created = DateTimeField(default=datetime.datetime.utcnow) - person = Person2() - date1 = person.created - date2 = person.created - self.assertEqual(date1, date2) + person = Person(name="Ross") - person = Person2(created=None) - date1 = person.created - date2 = person.created - self.assertEqual(date1, date2) + # Confirm saving now would store values + data_to_be_saved = sorted(person.to_mongo().keys()) + self.assertEqual(data_to_be_saved, ['age', 'created', 'name', 'userid']) + + self.assertTrue(person.validate() is None) + + self.assertEqual(person.name, person.name) + self.assertEqual(person.age, person.age) + self.assertEqual(person.userid, person.userid) + self.assertEqual(person.created, person.created) + + self.assertEqual(person._data['name'], person.name) + self.assertEqual(person._data['age'], person.age) + self.assertEqual(person._data['userid'], person.userid) + self.assertEqual(person._data['created'], person.created) + + # Confirm introspection changes nothing + data_to_be_saved = sorted(person.to_mongo().keys()) + self.assertEqual(data_to_be_saved, ['age', 'created', 'name', 'userid']) + + def test_default_values_set_to_None(self): + """Ensure that default field values are used when creating a document. + """ + class Person(Document): + name = StringField() + age = IntField(default=30, required=False) + userid = StringField(default=lambda: 'test', required=True) + created = DateTimeField(default=datetime.datetime.utcnow) + + # Trying setting values to None + person = Person(name=None, age=None, userid=None, created=None) + + # Confirm saving now would store values + data_to_be_saved = sorted(person.to_mongo().keys()) + self.assertEqual(data_to_be_saved, ['age', 'created', 'userid']) + + self.assertTrue(person.validate() is None) + + self.assertEqual(person.name, person.name) + self.assertEqual(person.age, person.age) + self.assertEqual(person.userid, person.userid) + self.assertEqual(person.created, person.created) + + self.assertEqual(person._data['name'], person.name) + self.assertEqual(person._data['age'], person.age) + self.assertEqual(person._data['userid'], person.userid) + self.assertEqual(person._data['created'], person.created) + + # Confirm introspection changes nothing + data_to_be_saved = sorted(person.to_mongo().keys()) + self.assertEqual(data_to_be_saved, ['age', 'created', 'userid']) + + def test_default_values_when_setting_to_None(self): + """Ensure that default field values are used when creating a document. + """ + class Person(Document): + name = StringField() + age = IntField(default=30, required=False) + userid = StringField(default=lambda: 'test', required=True) + created = DateTimeField(default=datetime.datetime.utcnow) + + person = Person() + person.name = None + person.age = None + person.userid = None + person.created = None + + # Confirm saving now would store values + data_to_be_saved = sorted(person.to_mongo().keys()) + self.assertEqual(data_to_be_saved, ['age', 'created', 'userid']) + + self.assertTrue(person.validate() is None) + + self.assertEqual(person.name, person.name) + self.assertEqual(person.age, person.age) + self.assertEqual(person.userid, person.userid) + self.assertEqual(person.created, person.created) + + self.assertEqual(person._data['name'], person.name) + self.assertEqual(person._data['age'], person.age) + self.assertEqual(person._data['userid'], person.userid) + self.assertEqual(person._data['created'], person.created) + + # Confirm introspection changes nothing + data_to_be_saved = sorted(person.to_mongo().keys()) + self.assertEqual(data_to_be_saved, ['age', 'created', 'userid']) + + def test_default_values_when_deleting_value(self): + """Ensure that default field values are used when creating a document. + """ + class Person(Document): + name = StringField() + age = IntField(default=30, required=False) + userid = StringField(default=lambda: 'test', required=True) + created = DateTimeField(default=datetime.datetime.utcnow) + + person = Person(name="Ross") + del person.name + del person.age + del person.userid + del person.created + + data_to_be_saved = sorted(person.to_mongo().keys()) + self.assertEqual(data_to_be_saved, ['age', 'created', 'userid']) + + self.assertTrue(person.validate() is None) + + self.assertEqual(person.name, person.name) + self.assertEqual(person.age, person.age) + self.assertEqual(person.userid, person.userid) + self.assertEqual(person.created, person.created) + + self.assertEqual(person._data['name'], person.name) + self.assertEqual(person._data['age'], person.age) + self.assertEqual(person._data['userid'], person.userid) + self.assertEqual(person._data['created'], person.created) + + # Confirm introspection changes nothing + data_to_be_saved = sorted(person.to_mongo().keys()) + self.assertEqual(data_to_be_saved, ['age', 'created', 'userid']) def test_required_values(self): """Ensure that required field constraints are enforced. From dc3b09c21879332b35bfc0d809226fe966b99016 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 6 Jun 2013 16:36:17 +0000 Subject: [PATCH 1174/1279] Improved cascading saves write performance (#361) --- docs/changelog.rst | 1 + mongoengine/base/metaclasses.py | 8 +++ mongoengine/document.py | 11 ++-- tests/document/instance.py | 106 ++++++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index bce26c5..c9cdda5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.8.2 ================ +- Improved cascading saves write performance (#361) - Fixed amibiguity and differing behaviour regarding field defaults (#349) - ImageFields now include PIL error messages if invalid error (#353) - Added lock when calling doc.Delete() for when signals have no sender (#350) diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index 444d9a2..651228d 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -97,6 +97,14 @@ class DocumentMetaclass(type): attrs['_reverse_db_field_map'] = dict( (v, k) for k, v in attrs['_db_field_map'].iteritems()) + # Set cascade flag if not set + if 'cascade' not in attrs['_meta']: + ReferenceField = _import_class('ReferenceField') + GenericReferenceField = _import_class('GenericReferenceField') + cascade = any([isinstance(x, (ReferenceField, GenericReferenceField)) + for x in doc_fields.values()]) + attrs['_meta']['cascade'] = cascade + # # Set document hierarchy # diff --git a/mongoengine/document.py b/mongoengine/document.py index 8e7ccc2..2bdecb7 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -8,6 +8,7 @@ from pymongo.read_preferences import ReadPreference from bson import ObjectId from bson.dbref import DBRef from mongoengine import signals +from mongoengine.common import _import_class from mongoengine.base import (DocumentMetaclass, TopLevelDocumentMetaclass, BaseDocument, BaseDict, BaseList, ALLOW_INHERITANCE, get_document) @@ -284,15 +285,17 @@ class Document(BaseDocument): def cascade_save(self, *args, **kwargs): """Recursively saves any references / generic references on an objects""" - import fields _refs = kwargs.get('_refs', []) or [] + ReferenceField = _import_class('ReferenceField') + GenericReferenceField = _import_class('GenericReferenceField') + for name, cls in self._fields.items(): - if not isinstance(cls, (fields.ReferenceField, - fields.GenericReferenceField)): + if not isinstance(cls, (ReferenceField, + GenericReferenceField)): continue - ref = getattr(self, name) + ref = self._data.get(name) if not ref or isinstance(ref, DBRef): continue diff --git a/tests/document/instance.py b/tests/document/instance.py index cdc6fe0..f4abd02 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -646,6 +646,22 @@ class InstanceTest(unittest.TestCase): self.assertEqual(b.picture, b.bar.picture, b.bar.bar.picture) + def test_setting_cascade(self): + + class ForcedCascade(Document): + meta = {'cascade': True} + + class Feed(Document): + name = StringField() + + class Subscription(Document): + name = StringField() + feed = ReferenceField(Feed) + + self.assertTrue(ForcedCascade._meta['cascade']) + self.assertTrue(Subscription._meta['cascade']) + self.assertFalse(Feed._meta['cascade']) + def test_save_cascades(self): class Person(Document): @@ -1018,6 +1034,96 @@ class InstanceTest(unittest.TestCase): self.assertEqual(person.age, 21) self.assertEqual(person.active, False) + def test_query_count_when_saving(self): + """Ensure references don't cause extra fetches when saving""" + class Organization(Document): + name = StringField() + + class User(Document): + name = StringField() + orgs = ListField(ReferenceField('Organization')) + + class Feed(Document): + name = StringField() + + class UserSubscription(Document): + name = StringField() + user = ReferenceField(User) + feed = ReferenceField(Feed) + + Organization.drop_collection() + User.drop_collection() + Feed.drop_collection() + UserSubscription.drop_collection() + + self.assertTrue(UserSubscription._meta['cascade']) + + o1 = Organization(name="o1").save() + o2 = Organization(name="o2").save() + + u1 = User(name="Ross", orgs=[o1, o2]).save() + f1 = Feed(name="MongoEngine").save() + + sub = UserSubscription(user=u1, feed=f1).save() + + user = User.objects.first() + # Even if stored as ObjectId's internally mongoengine uses DBRefs + # As ObjectId's aren't automatically derefenced + self.assertTrue(isinstance(user._data['orgs'][0], DBRef)) + self.assertTrue(isinstance(user.orgs[0], Organization)) + self.assertTrue(isinstance(user._data['orgs'][0], Organization)) + + # Changing a value + with query_counter() as q: + self.assertEqual(q, 0) + sub = UserSubscription.objects.first() + self.assertEqual(q, 1) + sub.name = "Test Sub" + sub.save() + self.assertEqual(q, 2) + + # Changing a value that will cascade + with query_counter() as q: + self.assertEqual(q, 0) + sub = UserSubscription.objects.first() + self.assertEqual(q, 1) + sub.user.name = "Test" + self.assertEqual(q, 2) + sub.save() + self.assertEqual(q, 3) + + # Changing a value and one that will cascade + with query_counter() as q: + self.assertEqual(q, 0) + sub = UserSubscription.objects.first() + sub.name = "Test Sub 2" + self.assertEqual(q, 1) + sub.user.name = "Test 2" + self.assertEqual(q, 2) + sub.save() + self.assertEqual(q, 4) # One for the UserSub and one for the User + + # Saving with just the refs + with query_counter() as q: + self.assertEqual(q, 0) + sub = UserSubscription(user=u1.pk, feed=f1.pk) + sub.validate() + self.assertEqual(q, 0) # Check no change + sub.save() + self.assertEqual(q, 1) + + # Saving new objects + with query_counter() as q: + self.assertEqual(q, 0) + user = User.objects.first() + self.assertEqual(q, 1) + feed = Feed.objects.first() + self.assertEqual(q, 2) + sub = UserSubscription(user=user, feed=feed) + self.assertEqual(q, 2) # Check no change + sub.save() + self.assertEqual(q, 3) + def test_set_unset_one_operation(self): """Ensure that $set and $unset actions are performed in the same operation. From 06f5dc6ad74c7ffc708c2da73df6267b451eba0f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 6 Jun 2013 16:44:43 +0000 Subject: [PATCH 1175/1279] Docs update --- docs/guide/defining-documents.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index b5ba2bf..ed9c142 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -100,9 +100,6 @@ arguments can be set on all fields: :attr:`db_field` (Default: None) The MongoDB field name. -:attr:`name` (Default: None) - The mongoengine field name. - :attr:`required` (Default: False) If set to True and the field is not set on the document instance, a :class:`~mongoengine.ValidationError` will be raised when the document is @@ -129,6 +126,7 @@ arguments can be set on all fields: # instead to just an object values = ListField(IntField(), default=[1,2,3]) + .. note:: Unsetting a field with a default value will revert back to the default. :attr:`unique` (Default: False) When True, no documents in the collection will have the same value for this From 9f3394dc6d65c0eab8179dc78350f3f48d004b35 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 6 Jun 2013 17:19:19 +0000 Subject: [PATCH 1176/1279] Added testcase for ListFields with just pks (#361) --- tests/document/instance.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/document/instance.py b/tests/document/instance.py index f4abd02..35338ab 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -9,6 +9,7 @@ import unittest import uuid from datetime import datetime +from bson import DBRef from tests.fixtures import PickleEmbedded, PickleTest, PickleSignalsTest from mongoengine import * @@ -1107,11 +1108,16 @@ class InstanceTest(unittest.TestCase): with query_counter() as q: self.assertEqual(q, 0) sub = UserSubscription(user=u1.pk, feed=f1.pk) - sub.validate() - self.assertEqual(q, 0) # Check no change + self.assertEqual(q, 0) sub.save() self.assertEqual(q, 1) + # Saving with just the refs on a ListField + with query_counter() as q: + self.assertEqual(q, 0) + User(name="Bob", orgs=[o1.pk, o2.pk]).save() + self.assertEqual(q, 1) + # Saving new objects with query_counter() as q: self.assertEqual(q, 0) From 542049f2526d393605d58a688698f04d2d70a46c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 6 Jun 2013 17:31:50 +0000 Subject: [PATCH 1177/1279] Trying to fix annoying python-dateutil bug --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b7c56a0..f6870a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install django==$DJANGO --use-mirrors ; true; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install $TRAVIS_PYTHON_VERSION-dateutil ; true; fi - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - python setup.py install From 8aae4f0ed085c3c4d90d8df90c8040aa9d33fefb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 6 Jun 2013 17:34:34 +0000 Subject: [PATCH 1178/1279] Trying to stabalise the build --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f6870a7..173f739 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install django==$DJANGO --use-mirrors ; true; fi - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install $TRAVIS_PYTHON_VERSION-dateutil ; true; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install python-dateutil ; true; fi - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - python setup.py install From a7631223a38879f4e59fd4050acf446dda0a4916 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 6 Jun 2013 17:58:10 +0000 Subject: [PATCH 1179/1279] Fixed Datastructures so instances are a Document or EmbeddedDocument (#363) --- docs/changelog.rst | 1 + mongoengine/base/datastructures.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c9cdda5..c35cd9c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.8.2 ================ +- Fixed Datastructures so instances are a Document or EmbeddedDocument (#363) - Improved cascading saves write performance (#361) - Fixed amibiguity and differing behaviour regarding field defaults (#349) - ImageFields now include PIL error messages if invalid error (#353) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index c750b5b..adcd8d0 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -13,7 +13,11 @@ class BaseDict(dict): _name = None def __init__(self, dict_items, instance, name): - self._instance = weakref.proxy(instance) + Document = _import_class('Document') + EmbeddedDocument = _import_class('EmbeddedDocument') + + if isinstance(instance, (Document, EmbeddedDocument)): + self._instance = weakref.proxy(instance) self._name = name return super(BaseDict, self).__init__(dict_items) @@ -80,7 +84,11 @@ class BaseList(list): _name = None def __init__(self, list_items, instance, name): - self._instance = weakref.proxy(instance) + Document = _import_class('Document') + EmbeddedDocument = _import_class('EmbeddedDocument') + + if isinstance(instance, (Document, EmbeddedDocument)): + self._instance = weakref.proxy(instance) self._name = name return super(BaseList, self).__init__(list_items) From f3af76e38cb760566cdaf488defe55876a0b8507 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 6 Jun 2013 17:59:07 +0000 Subject: [PATCH 1180/1279] Added ygbourhis to AUTHORS (#363) --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 4caed40..72b1124 100644 --- a/AUTHORS +++ b/AUTHORS @@ -168,3 +168,4 @@ that much better: * istinspring (https://github.com/istinspring) * Massimo Santini (https://github.com/mapio) * Nigel McNie (https://github.com/nigelmcnie) + * ygbourhis (https://github.com/ygbourhis) \ No newline at end of file From d935b5764a6c93e78750bbefee56f115e287591e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 6 Jun 2013 18:02:06 +0000 Subject: [PATCH 1181/1279] apt only had an ancient version of python-dateutil *sigh* --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 173f739..b7c56a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,6 @@ install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install django==$DJANGO --use-mirrors ; true; fi - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install python-dateutil ; true; fi - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - python setup.py install From 7451244cd27f83a82c4a3e65a767679f15c4af5e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 6 Jun 2013 21:04:54 +0000 Subject: [PATCH 1182/1279] Fixed cascading saves which weren't turned off as planned (#291) --- docs/changelog.rst | 1 + mongoengine/base/metaclasses.py | 8 -------- mongoengine/document.py | 18 ++++++++++------- tests/document/instance.py | 36 +++++++++------------------------ 4 files changed, 22 insertions(+), 41 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index c35cd9c..a046847 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.8.2 ================ +- Fixed cascading saves which weren't turned off as planned (#291) - Fixed Datastructures so instances are a Document or EmbeddedDocument (#363) - Improved cascading saves write performance (#361) - Fixed amibiguity and differing behaviour regarding field defaults (#349) diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index 651228d..444d9a2 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -97,14 +97,6 @@ class DocumentMetaclass(type): attrs['_reverse_db_field_map'] = dict( (v, k) for k, v in attrs['_db_field_map'].iteritems()) - # Set cascade flag if not set - if 'cascade' not in attrs['_meta']: - ReferenceField = _import_class('ReferenceField') - GenericReferenceField = _import_class('GenericReferenceField') - cascade = any([isinstance(x, (ReferenceField, GenericReferenceField)) - for x in doc_fields.values()]) - attrs['_meta']['cascade'] = cascade - # # Set document hierarchy # diff --git a/mongoengine/document.py b/mongoengine/document.py index 2bdecb7..8b152d5 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -186,8 +186,8 @@ class Document(BaseDocument): will force an fsync on the primary server. :param cascade: Sets the flag for cascading saves. You can set a default by setting "cascade" in the document __meta__ - :param cascade_kwargs: optional kwargs dictionary to be passed throw - to cascading saves + :param cascade_kwargs: (optional) kwargs dictionary to be passed throw + to cascading saves. Implies ``cascade=True``. :param _refs: A list of processed references used in cascading saves .. versionchanged:: 0.5 @@ -196,11 +196,13 @@ class Document(BaseDocument): :class:`~bson.dbref.DBRef` objects that have changes are saved as well. .. versionchanged:: 0.6 - Cascade saves are optional = defaults to True, if you want + Added cascading saves + .. versionchanged:: 0.8 + Cascade saves are optional and default to False. If you want fine grain control then you can turn off using document - meta['cascade'] = False Also you can pass different kwargs to + meta['cascade'] = True. Also you can pass different kwargs to the cascade save using cascade_kwargs which overwrites the - existing kwargs with custom values + existing kwargs with custom values. """ signals.pre_save.send(self.__class__, document=self) @@ -251,8 +253,10 @@ class Document(BaseDocument): upsert=True, **write_concern) created = is_new_object(last_error) - cascade = (self._meta.get('cascade', True) - if cascade is None else cascade) + + if cascade is None: + cascade = self._meta.get('cascade', False) or cascade_kwargs is not None + if cascade: kwargs = { "force_insert": force_insert, diff --git a/tests/document/instance.py b/tests/document/instance.py index 35338ab..81734aa 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -647,22 +647,6 @@ class InstanceTest(unittest.TestCase): self.assertEqual(b.picture, b.bar.picture, b.bar.bar.picture) - def test_setting_cascade(self): - - class ForcedCascade(Document): - meta = {'cascade': True} - - class Feed(Document): - name = StringField() - - class Subscription(Document): - name = StringField() - feed = ReferenceField(Feed) - - self.assertTrue(ForcedCascade._meta['cascade']) - self.assertTrue(Subscription._meta['cascade']) - self.assertFalse(Feed._meta['cascade']) - def test_save_cascades(self): class Person(Document): @@ -681,7 +665,7 @@ class InstanceTest(unittest.TestCase): p = Person.objects(name="Wilson Jr").get() p.parent.name = "Daddy Wilson" - p.save() + p.save(cascade=True) p1.reload() self.assertEqual(p1.name, p.parent.name) @@ -700,14 +684,12 @@ class InstanceTest(unittest.TestCase): p2 = Person(name="Wilson Jr") p2.parent = p1 + p1.name = "Daddy Wilson" p2.save(force_insert=True, cascade_kwargs={"force_insert": False}) - p = Person.objects(name="Wilson Jr").get() - p.parent.name = "Daddy Wilson" - p.save() - p1.reload() - self.assertEqual(p1.name, p.parent.name) + p2.reload() + self.assertEqual(p1.name, p2.parent.name) def test_save_cascade_meta_false(self): @@ -782,6 +764,10 @@ class InstanceTest(unittest.TestCase): p.parent.name = "Daddy Wilson" p.save() + p1.reload() + self.assertNotEqual(p1.name, p.parent.name) + + p.save(cascade=True) p1.reload() self.assertEqual(p1.name, p.parent.name) @@ -1057,8 +1043,6 @@ class InstanceTest(unittest.TestCase): Feed.drop_collection() UserSubscription.drop_collection() - self.assertTrue(UserSubscription._meta['cascade']) - o1 = Organization(name="o1").save() o2 = Organization(name="o2").save() @@ -1090,7 +1074,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(q, 1) sub.user.name = "Test" self.assertEqual(q, 2) - sub.save() + sub.save(cascade=True) self.assertEqual(q, 3) # Changing a value and one that will cascade @@ -1101,7 +1085,7 @@ class InstanceTest(unittest.TestCase): self.assertEqual(q, 1) sub.user.name = "Test 2" self.assertEqual(q, 2) - sub.save() + sub.save(cascade=True) self.assertEqual(q, 4) # One for the UserSub and one for the User # Saving with just the refs From c2928d8a57460e60d4b26d5f25f965d40eb4e1a6 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Thu, 6 Jun 2013 17:16:03 -0700 Subject: [PATCH 1183/1279] list_indexes and compare_indexes class methods + unit tests --- mongoengine/document.py | 88 ++++++++++++++++++++++--- tests/document/class_methods.py | 111 ++++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+), 8 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 585fcf7..83f60ee 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -20,6 +20,19 @@ __all__ = ('Document', 'EmbeddedDocument', 'DynamicDocument', 'InvalidCollectionError', 'NotUniqueError', 'MapReduceDocument') +def includes_cls(fields): + """ Helper function used for ensuring and comparing indexes + """ + + first_field = None + if len(fields): + if isinstance(fields[0], basestring): + first_field = fields[0] + elif isinstance(fields[0], (list, tuple)) and len(fields[0]): + first_field = fields[0][0] + return first_field == '_cls' + + class InvalidCollectionError(Exception): pass @@ -529,14 +542,6 @@ class Document(BaseDocument): # an extra index on _cls, as mongodb will use the existing # index to service queries against _cls cls_indexed = False - def includes_cls(fields): - first_field = None - if len(fields): - if isinstance(fields[0], basestring): - first_field = fields[0] - elif isinstance(fields[0], (list, tuple)) and len(fields[0]): - first_field = fields[0][0] - return first_field == '_cls' # Ensure document-defined indexes are created if cls._meta['index_specs']: @@ -557,6 +562,73 @@ class Document(BaseDocument): collection.ensure_index('_cls', background=background, **index_opts) + @classmethod + def list_indexes(cls, go_up=True, go_down=True): + """ Lists all of the indexes that should be created for given + collection. It includes all the indexes from super- and sub-classes. + """ + + if cls._meta.get('abstract'): + return [] + + indexes = [] + index_cls = cls._meta.get('index_cls', True) + + # Ensure document-defined indexes are created + if cls._meta['index_specs']: + index_spec = cls._meta['index_specs'] + for spec in index_spec: + spec = spec.copy() + fields = spec.pop('fields') + indexes.append(fields) + + # add all of the indexes from the base classes + if go_up: + for base_cls in cls.__bases__: + for index in base_cls.list_indexes(go_up=True, go_down=False): + if index not in indexes: + indexes.append(index) + + # add all of the indexes from subclasses + if go_down: + for subclass in cls.__subclasses__(): + for index in subclass.list_indexes(go_up=False, go_down=True): + if index not in indexes: + indexes.append(index) + + # finish up by appending _id, if needed + if go_up and go_down: + if [(u'_id', 1)] not in indexes: + indexes.append([(u'_id', 1)]) + if (index_cls and + cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) is True): + indexes.append([(u'_cls', 1)]) + + return indexes + + @classmethod + def compare_indexes(cls): + """ Compares the indexes defined in MongoEngine with the ones existing + in the database. Returns any missing/extra indexes. + """ + + required = cls.list_indexes() + existing = [info['key'] for info in cls._get_collection().index_information().values()] + missing = [index for index in required if index not in existing] + extra = [index for index in existing if index not in required] + + # if { _cls: 1 } is missing, make sure it's *really* necessary + if [(u'_cls', 1)] in missing: + cls_obsolete = False + for index in existing: + if includes_cls(index) and index not in extra: + cls_obsolete = True + break + if cls_obsolete: + missing.remove([(u'_cls', 1)]) + + return {'missing': missing, 'extra': extra} + class DynamicDocument(Document): """A Dynamic Document class allowing flexible, expandable and uncontrolled diff --git a/tests/document/class_methods.py b/tests/document/class_methods.py index b2c7283..6bd2e3c 100644 --- a/tests/document/class_methods.py +++ b/tests/document/class_methods.py @@ -85,6 +85,117 @@ class ClassMethodsTest(unittest.TestCase): self.assertEqual(self.Person._meta['delete_rules'], {(Job, 'employee'): NULLIFY}) + def test_compare_indexes(self): + """ Ensure that the indexes are properly created and that + compare_indexes identifies the missing/extra indexes + """ + + class BlogPost(Document): + author = StringField() + title = StringField() + description = StringField() + tags = StringField() + + meta = { + 'indexes': [('author', 'title')] + } + + BlogPost.drop_collection() + + BlogPost.ensure_indexes() + self.assertEqual(BlogPost.compare_indexes(), { 'missing': [], 'extra': [] }) + + BlogPost.ensure_index(['author', 'description']) + self.assertEqual(BlogPost.compare_indexes(), { 'missing': [], 'extra': [[('author', 1), ('description', 1)]] }) + + BlogPost._get_collection().drop_index('author_1_description_1') + self.assertEqual(BlogPost.compare_indexes(), { 'missing': [], 'extra': [] }) + + BlogPost._get_collection().drop_index('author_1_title_1') + self.assertEqual(BlogPost.compare_indexes(), { 'missing': [[('author', 1), ('title', 1)]], 'extra': [] }) + + def test_compare_indexes_inheritance(self): + """ Ensure that the indexes are properly created and that + compare_indexes identifies the missing/extra indexes for subclassed + documents (_cls included) + """ + + class BlogPost(Document): + author = StringField() + title = StringField() + description = StringField() + + meta = { + 'allow_inheritance': True + } + + class BlogPostWithTags(BlogPost): + tags = StringField() + tag_list = ListField(StringField()) + + meta = { + 'indexes': [('author', 'tags')] + } + + BlogPost.drop_collection() + + BlogPost.ensure_indexes() + BlogPostWithTags.ensure_indexes() + self.assertEqual(BlogPost.compare_indexes(), { 'missing': [], 'extra': [] }) + + BlogPostWithTags.ensure_index(['author', 'tag_list']) + self.assertEqual(BlogPost.compare_indexes(), { 'missing': [], 'extra': [[('_cls', 1), ('author', 1), ('tag_list', 1)]] }) + + BlogPostWithTags._get_collection().drop_index('_cls_1_author_1_tag_list_1') + self.assertEqual(BlogPost.compare_indexes(), { 'missing': [], 'extra': [] }) + + BlogPostWithTags._get_collection().drop_index('_cls_1_author_1_tags_1') + self.assertEqual(BlogPost.compare_indexes(), { 'missing': [[('_cls', 1), ('author', 1), ('tags', 1)]], 'extra': [] }) + + def test_list_indexes_inheritance(self): + """ ensure that all of the indexes are listed regardless of the super- + or sub-class that we call it from + """ + + class BlogPost(Document): + author = StringField() + title = StringField() + description = StringField() + + meta = { + 'allow_inheritance': True + } + + class BlogPostWithTags(BlogPost): + tags = StringField() + + meta = { + 'indexes': [('author', 'tags')] + } + + class BlogPostWithTagsAndExtraText(BlogPostWithTags): + extra_text = StringField() + + meta = { + 'indexes': [('author', 'tags', 'extra_text')] + } + + BlogPost.drop_collection() + + BlogPost.ensure_indexes() + BlogPostWithTags.ensure_indexes() + BlogPostWithTagsAndExtraText.ensure_indexes() + + self.assertEqual(BlogPost.list_indexes(), + BlogPostWithTags.list_indexes()) + self.assertEqual(BlogPost.list_indexes(), + BlogPostWithTagsAndExtraText.list_indexes()) + print BlogPost.list_indexes() + self.assertEqual(BlogPost.list_indexes(), + [[('_cls', 1), ('author', 1), ('tags', 1)], + [('_cls', 1), ('author', 1), ('tags', 1), ('extra_text', 1)], + [(u'_id', 1)], [('_cls', 1)]]) + def test_register_delete_rule_inherited(self): class Vaccine(Document): From 305540f0fd1731e9dd232cb995ab457f9a0fc6f4 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Thu, 6 Jun 2013 17:21:27 -0700 Subject: [PATCH 1184/1279] better comment --- mongoengine/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 83f60ee..95ad0dc 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -596,7 +596,7 @@ class Document(BaseDocument): if index not in indexes: indexes.append(index) - # finish up by appending _id, if needed + # finish up by appending { '_id': 1 } and { '_cls': 1 }, if needed if go_up and go_down: if [(u'_id', 1)] not in indexes: indexes.append([(u'_id', 1)]) From a2457df45e0fcc0077dde4f7fe09891e44ad0635 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Thu, 6 Jun 2013 19:14:21 -0700 Subject: [PATCH 1185/1279] make sure to only search for indexes in base classes inheriting from TopLevelDocumentMetaclass --- mongoengine/document.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 95ad0dc..3b6df4f 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -585,9 +585,10 @@ class Document(BaseDocument): # add all of the indexes from the base classes if go_up: for base_cls in cls.__bases__: - for index in base_cls.list_indexes(go_up=True, go_down=False): - if index not in indexes: - indexes.append(index) + if isinstance(base_cls, TopLevelDocumentMetaclass): + for index in base_cls.list_indexes(go_up=True, go_down=False): + if index not in indexes: + indexes.append(index) # add all of the indexes from subclasses if go_down: From ba7101ff92588185c4fa3355351d28c57ede4f78 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Thu, 6 Jun 2013 22:22:43 -0700 Subject: [PATCH 1186/1279] list_indexes support for multiple inheritance --- mongoengine/document.py | 70 ++++++++++++++++++++------------- tests/document/class_methods.py | 38 +++++++++++++++++- tests/document/inheritance.py | 35 +++++++++++++++++ 3 files changed, 115 insertions(+), 28 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 3b6df4f..6345e6d 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -571,39 +571,55 @@ class Document(BaseDocument): if cls._meta.get('abstract'): return [] - indexes = [] - index_cls = cls._meta.get('index_cls', True) + # get all the base classes, subclasses and sieblings + classes = [] + def get_classes(cls): - # Ensure document-defined indexes are created - if cls._meta['index_specs']: - index_spec = cls._meta['index_specs'] - for spec in index_spec: - spec = spec.copy() - fields = spec.pop('fields') - indexes.append(fields) + if (cls not in classes and + isinstance(cls, TopLevelDocumentMetaclass)): + classes.append(cls) - # add all of the indexes from the base classes - if go_up: for base_cls in cls.__bases__: - if isinstance(base_cls, TopLevelDocumentMetaclass): - for index in base_cls.list_indexes(go_up=True, go_down=False): - if index not in indexes: - indexes.append(index) - - # add all of the indexes from subclasses - if go_down: + if (isinstance(base_cls, TopLevelDocumentMetaclass) and + base_cls != Document and + not base_cls._meta.get('abstract') and + base_cls._get_collection().full_name == cls._get_collection().full_name and + base_cls not in classes): + classes.append(base_cls) + get_classes(base_cls) for subclass in cls.__subclasses__(): - for index in subclass.list_indexes(go_up=False, go_down=True): - if index not in indexes: - indexes.append(index) + if (isinstance(base_cls, TopLevelDocumentMetaclass) and + subclass._get_collection().full_name == cls._get_collection().full_name and + subclass not in classes): + classes.append(subclass) + get_classes(subclass) + + get_classes(cls) + + # get the indexes spec for all of the gathered classes + def get_indexes_spec(cls): + indexes = [] + + if cls._meta['index_specs']: + index_spec = cls._meta['index_specs'] + for spec in index_spec: + spec = spec.copy() + fields = spec.pop('fields') + indexes.append(fields) + return indexes + + indexes = [] + for cls in classes: + for index in get_indexes_spec(cls): + if index not in indexes: + indexes.append(index) # finish up by appending { '_id': 1 } and { '_cls': 1 }, if needed - if go_up and go_down: - if [(u'_id', 1)] not in indexes: - indexes.append([(u'_id', 1)]) - if (index_cls and - cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) is True): - indexes.append([(u'_cls', 1)]) + if [(u'_id', 1)] not in indexes: + indexes.append([(u'_id', 1)]) + if (cls._meta.get('index_cls', True) and + cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) is True): + indexes.append([(u'_cls', 1)]) return indexes diff --git a/tests/document/class_methods.py b/tests/document/class_methods.py index 6bd2e3c..52e3794 100644 --- a/tests/document/class_methods.py +++ b/tests/document/class_methods.py @@ -152,6 +152,43 @@ class ClassMethodsTest(unittest.TestCase): BlogPostWithTags._get_collection().drop_index('_cls_1_author_1_tags_1') self.assertEqual(BlogPost.compare_indexes(), { 'missing': [[('_cls', 1), ('author', 1), ('tags', 1)]], 'extra': [] }) + def test_compare_indexes_multiple_subclasses(self): + """ Ensure that compare_indexes behaves correctly if called from a + class, which base class has multiple subclasses + """ + + class BlogPost(Document): + author = StringField() + title = StringField() + description = StringField() + + meta = { + 'allow_inheritance': True + } + + class BlogPostWithTags(BlogPost): + tags = StringField() + tag_list = ListField(StringField()) + + meta = { + 'indexes': [('author', 'tags')] + } + + class BlogPostWithCustomField(BlogPost): + custom = DictField() + + meta = { + 'indexes': [('author', 'custom')] + } + + BlogPost.ensure_indexes() + BlogPostWithTags.ensure_indexes() + BlogPostWithCustomField.ensure_indexes() + + self.assertEqual(BlogPost.compare_indexes(), { 'missing': [], 'extra': [] }) + self.assertEqual(BlogPostWithTags.compare_indexes(), { 'missing': [], 'extra': [] }) + self.assertEqual(BlogPostWithCustomField.compare_indexes(), { 'missing': [], 'extra': [] }) + def test_list_indexes_inheritance(self): """ ensure that all of the indexes are listed regardless of the super- or sub-class that we call it from @@ -190,7 +227,6 @@ class ClassMethodsTest(unittest.TestCase): BlogPostWithTags.list_indexes()) self.assertEqual(BlogPost.list_indexes(), BlogPostWithTagsAndExtraText.list_indexes()) - print BlogPost.list_indexes() self.assertEqual(BlogPost.list_indexes(), [[('_cls', 1), ('author', 1), ('tags', 1)], [('_cls', 1), ('author', 1), ('tags', 1), ('extra_text', 1)], diff --git a/tests/document/inheritance.py b/tests/document/inheritance.py index f011631..28490c9 100644 --- a/tests/document/inheritance.py +++ b/tests/document/inheritance.py @@ -189,6 +189,41 @@ class InheritanceTest(unittest.TestCase): self.assertEqual(Employee._get_collection_name(), Person._get_collection_name()) + def test_indexes_and_multiple_inheritance(self): + """ Ensure that all of the indexes are created for a document with + multiple inheritance. + """ + + class A(Document): + a = StringField() + + meta = { + 'allow_inheritance': True, + 'indexes': ['a'] + } + + class B(Document): + b = StringField() + + meta = { + 'allow_inheritance': True, + 'indexes': ['b'] + } + + class C(A, B): + pass + + A.drop_collection() + B.drop_collection() + C.drop_collection() + + C.ensure_indexes() + + self.assertEqual( + [idx['key'] for idx in C._get_collection().index_information().values()], + [[(u'_cls', 1), (u'b', 1)], [(u'_id', 1)], [(u'_cls', 1), (u'a', 1)]] + ) + def test_polymorphic_queries(self): """Ensure that the correct subclasses are returned from a query """ From f0d4e76418ff3f02b3401ee41467356a939c39a0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 7 Jun 2013 08:21:15 +0000 Subject: [PATCH 1187/1279] Documentation updates --- docs/apireference.rst | 5 +++++ mongoengine/base/fields.py | 3 +-- mongoengine/common.py | 14 +++++++++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 0fa410e..d062727 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -88,3 +88,8 @@ Fields .. autoclass:: mongoengine.fields.GridFSProxy .. autoclass:: mongoengine.fields.ImageGridFsProxy .. autoclass:: mongoengine.fields.ImproperlyConfigured + +Misc +==== + +.. autofunction:: mongoengine.common._import_class diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index e4c88a7..eda9b3c 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -82,8 +82,7 @@ class BaseField(object): BaseField.creation_counter += 1 def __get__(self, instance, owner): - """Descriptor for retrieving a value from a field in a document. Do - any necessary conversion between Python and MongoDB types. + """Descriptor for retrieving a value from a field in a document. """ if instance is None: # Document class being used rather than a document object diff --git a/mongoengine/common.py b/mongoengine/common.py index bff55ac..20d5138 100644 --- a/mongoengine/common.py +++ b/mongoengine/common.py @@ -2,7 +2,19 @@ _class_registry_cache = {} def _import_class(cls_name): - """Cached mechanism for imports""" + """Cache mechanism for imports. + + Due to complications of circular imports mongoengine needs to do lots of + inline imports in functions. This is inefficient as classes are + imported repeated throughout the mongoengine code. This is + compounded by some recursive functions requiring inline imports. + + :mod:`mongoengine.common` provides a single point to import all these + classes. Circular imports aren't an issue as it dynamically imports the + class when first needed. Subsequent calls to the + :func:`~mongoengine.common._import_class` can then directly retrieve the + class from the :data:`mongoengine.common._class_registry_cache`. + """ if cls_name in _class_registry_cache: return _class_registry_cache.get(cls_name) From 000eff73ccdad3f2b5a8dbb283ccc035707c46b8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 7 Jun 2013 08:33:34 +0000 Subject: [PATCH 1188/1279] Make test_indexes_and_multiple_inheritance place nice with py3.3 (#364) --- tests/document/inheritance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/document/inheritance.py b/tests/document/inheritance.py index 28490c9..5a48f75 100644 --- a/tests/document/inheritance.py +++ b/tests/document/inheritance.py @@ -220,8 +220,8 @@ class InheritanceTest(unittest.TestCase): C.ensure_indexes() self.assertEqual( - [idx['key'] for idx in C._get_collection().index_information().values()], - [[(u'_cls', 1), (u'b', 1)], [(u'_id', 1)], [(u'_cls', 1), (u'a', 1)]] + sorted([idx['key'] for idx in C._get_collection().index_information().values()]), + sorted([[(u'_cls', 1), (u'b', 1)], [(u'_id', 1)], [(u'_cls', 1), (u'a', 1)]]) ) def test_polymorphic_queries(self): From 025c16c95d920865e1f60c3f38aa70f07fae0f3a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 7 Jun 2013 08:34:57 +0000 Subject: [PATCH 1189/1279] Add BobDickinson to authors (#361) --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 72b1124..7788139 100644 --- a/AUTHORS +++ b/AUTHORS @@ -168,4 +168,5 @@ that much better: * istinspring (https://github.com/istinspring) * Massimo Santini (https://github.com/mapio) * Nigel McNie (https://github.com/nigelmcnie) - * ygbourhis (https://github.com/ygbourhis) \ No newline at end of file + * ygbourhis (https://github.com/ygbourhis) + * Bob Dickinson (https://github.com/BobDickinson) \ No newline at end of file From e2b32b4bb378bedf6fa962b02ebe893e7b8e1e4b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 7 Jun 2013 08:43:05 +0000 Subject: [PATCH 1190/1279] Added more docs about compare_indexes (#364) --- docs/guide/defining-documents.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index ed9c142..b3b1e59 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -497,7 +497,6 @@ in this case use 'dot' notation to identify the value to index eg: `rank.title` Geospatial indexes ------------------ - The best geo index for mongodb is the new "2dsphere", which has an improved spherical model and provides better performance and more options when querying. The following fields will explicitly add a "2dsphere" index: @@ -559,6 +558,14 @@ documentation for more information. A common usecase might be session data:: ] } +Comparing Indexes +----------------- + +Use :func:`mongoengine.Document.compare_indexes` to compare actual indexes in +the database to those that your document definitions define. This is useful +for maintenance purposes and ensuring you have the correct indexes for your +schema. + Ordering ======== A default ordering can be specified for your From a3d43b77ca59facb5a8cde618bfd33c9cb14eb07 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 7 Jun 2013 08:44:33 +0000 Subject: [PATCH 1191/1279] Updated changelog --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index a046847..0f86a62 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,6 +5,7 @@ Changelog Changes in 0.8.2 ================ +- Added compare_indexes helper (#361) - Fixed cascading saves which weren't turned off as planned (#291) - Fixed Datastructures so instances are a Document or EmbeddedDocument (#363) - Improved cascading saves write performance (#361) From ede9fcfb0021f8ff924bad894af77c6ee6a8ed35 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 7 Jun 2013 08:45:40 +0000 Subject: [PATCH 1192/1279] Version bump 0.8.2 --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 8c167f0..5bd1201 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -15,7 +15,7 @@ import django __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + list(queryset.__all__) + signals.__all__ + list(errors.__all__)) -VERSION = (0, 8, 1) +VERSION = (0, 8, 2) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 7c87b1c..4eaba4d 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.8.1 +Version: 0.8.2 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From 44a2a164c0fa46bb3839a08d9b240243b9ab71fe Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 13 Jun 2013 10:54:39 +0000 Subject: [PATCH 1193/1279] Doc updates --- docs/changelog.rst | 2 +- docs/guide/defining-documents.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 0f86a62..3a39752 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,7 +9,7 @@ Changes in 0.8.2 - Fixed cascading saves which weren't turned off as planned (#291) - Fixed Datastructures so instances are a Document or EmbeddedDocument (#363) - Improved cascading saves write performance (#361) -- Fixed amibiguity and differing behaviour regarding field defaults (#349) +- Fixed ambiguity and differing behaviour regarding field defaults (#349) - ImageFields now include PIL error messages if invalid error (#353) - Added lock when calling doc.Delete() for when signals have no sender (#350) - Reload forces read preference to be PRIMARY (#355) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index b3b1e59..a61d8fe 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -450,8 +450,8 @@ by creating a list of index specifications called :attr:`indexes` in the :attr:`~mongoengine.Document.meta` dictionary, where an index specification may either be a single field name, a tuple containing multiple field names, or a dictionary containing a full index definition. A direction may be specified on -fields by prefixing the field name with a **+** or a **-** sign. Note that -direction only matters on multi-field indexes. :: +fields by prefixing the field name with a **+** (for ascending) or a **-** sign +(for descending). Note that direction only matters on multi-field indexes. :: class Page(Document): title = StringField() From c31d6a68985e9807795aaf60bc796ac86a227e84 Mon Sep 17 00:00:00 2001 From: kelvinhammond Date: Wed, 19 Jun 2013 10:34:33 -0400 Subject: [PATCH 1194/1279] Fixed sum and average mapreduce function for issue #375 --- AUTHORS | 3 ++- mongoengine/queryset/queryset.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index 7788139..780c9e6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -16,6 +16,7 @@ Dervived from the git logs, inevitably incomplete but all of whom and others have submitted patches, reported bugs and generally helped make MongoEngine that much better: + * Kelvin Hammond (https://github.com/kelvinhammond) * Harry Marr * Ross Lawley * blackbrrr @@ -169,4 +170,4 @@ that much better: * Massimo Santini (https://github.com/mapio) * Nigel McNie (https://github.com/nigelmcnie) * ygbourhis (https://github.com/ygbourhis) - * Bob Dickinson (https://github.com/BobDickinson) \ No newline at end of file + * Bob Dickinson (https://github.com/BobDickinson) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index d58a13b..e2ff43f 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1062,7 +1062,7 @@ class QuerySet(object): """ map_func = Code(""" function() { - emit(1, this[field] || 0); + emit(1, eval("this." + field) || 0); } """, scope={'field': field}) @@ -1093,7 +1093,7 @@ class QuerySet(object): map_func = Code(""" function() { if (this.hasOwnProperty(field)) - emit(1, {t: this[field] || 0, c: 1}); + emit(1, {t: eval("this." + field) || 0, c: 1}); } """, scope={'field': field}) From 574f3c23d3ce633d830f55918a009c30305e3dbd Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Jun 2013 09:35:22 +0000 Subject: [PATCH 1195/1279] get should clone before calling --- mongoengine/queryset/queryset.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index d58a13b..9b53df2 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -245,8 +245,10 @@ class QuerySet(object): .. versionadded:: 0.3 """ - queryset = self.__call__(*q_objs, **query) + queryset = self.clone() queryset = queryset.limit(2) + queryset = queryset.filter(*q_objs, **query) + try: result = queryset.next() except StopIteration: From f1a1aa54d8a31336d2a76a21f2e4b166d776f526 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Jun 2013 10:19:40 +0000 Subject: [PATCH 1196/1279] Added full_result kwarg to update (#380) --- docs/changelog.rst | 3 +++ mongoengine/document.py | 8 +++++++- mongoengine/queryset/queryset.py | 16 ++++++++++------ tests/queryset/queryset.py | 17 +++++++++++++++++ 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3a39752..8fa5af0 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,9 @@ Changelog ========= +Changes in 0.8.3 +================ +- Added full_result kwarg to update (#380) Changes in 0.8.2 ================ diff --git a/mongoengine/document.py b/mongoengine/document.py index a61ed07..a1bac19 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -353,7 +353,13 @@ class Document(BaseDocument): been saved. """ if not self.pk: - raise OperationError('attempt to update a document not yet saved') + if kwargs.get('upsert', False): + query = self.to_mongo() + if "_cls" in query: + del(query["_cls"]) + return self._qs.filter(**query).update_one(**kwargs) + else: + raise OperationError('attempt to update a document not yet saved') # Need to add shard key to query, or you get an error return self._qs.filter(**self._object_key).update_one(**kwargs) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 9b53df2..4b32ab1 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -474,7 +474,8 @@ class QuerySet(object): queryset._collection.remove(queryset._query, write_concern=write_concern) - def update(self, upsert=False, multi=True, write_concern=None, **update): + def update(self, upsert=False, multi=True, write_concern=None, + full_result=False, **update): """Perform an atomic update on the fields matched by the query. :param upsert: Any existing document with that "_id" is overwritten. @@ -485,6 +486,8 @@ class QuerySet(object): ``save(..., write_concern={w: 2, fsync: True}, ...)`` will wait until at least two servers have recorded the write and will force an fsync on the primary server. + :param full_result: Return the full result rather than just the number + updated. :param update: Django-style update keyword arguments .. versionadded:: 0.2 @@ -506,12 +509,13 @@ class QuerySet(object): update["$set"]["_cls"] = queryset._document._class_name else: update["$set"] = {"_cls": queryset._document._class_name} - try: - ret = queryset._collection.update(query, update, multi=multi, - upsert=upsert, **write_concern) - if ret is not None and 'n' in ret: - return ret['n'] + result = queryset._collection.update(query, update, multi=multi, + upsert=upsert, **write_concern) + if full_result: + return result + elif result: + return result['n'] except pymongo.errors.OperationFailure, err: if unicode(err) == u'multi not coded yet': message = u'update() method requires MongoDB 1.1.3+' diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 21df22c..bd231e3 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -536,6 +536,23 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(club.members['John']['gender'], "F") self.assertEqual(club.members['John']['age'], 14) + def test_update_results(self): + self.Person.drop_collection() + + result = self.Person(name="Bob", age=25).update(upsert=True, full_result=True) + self.assertIsInstance(result, dict) + self.assertTrue("upserted" in result) + self.assertFalse(result["updatedExisting"]) + + bob = self.Person.objects.first() + result = bob.update(set__age=30, full_result=True) + self.assertIsInstance(result, dict) + self.assertTrue(result["updatedExisting"]) + + self.Person(name="Bob", age=20).save() + result = self.Person.objects(name="Bob").update(set__name="bobby", multi=True) + self.assertEqual(result, 2) + def test_upsert(self): self.Person.drop_collection() From e116bb92272246528df242c587f1b8e5e6fb6f48 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Jun 2013 10:39:10 +0000 Subject: [PATCH 1197/1279] Fixed queryset.get() respecting no_dereference (#373) --- docs/changelog.rst | 1 + mongoengine/queryset/queryset.py | 4 ++-- tests/queryset/queryset.py | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8fa5af0..b04bba5 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.3 ================ +- Fixed queryset.get() respecting no_dereference (#373) - Added full_result kwarg to update (#380) Changes in 0.8.2 diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 4b32ab1..ded8d5e 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1165,8 +1165,8 @@ class QuerySet(object): raw_doc = self._cursor.next() if self._as_pymongo: return self._get_as_pymongo(raw_doc) - - doc = self._document._from_son(raw_doc) + doc = self._document._from_son(raw_doc, + _auto_dereference=self._auto_dereference) if self._scalar: return self._get_scalar(doc) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index bd231e3..7c47360 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3258,6 +3258,8 @@ class QuerySetTest(unittest.TestCase): self.assertTrue(isinstance(qs.first().organization, Organization)) self.assertFalse(isinstance(qs.no_dereference().first().organization, Organization)) + self.assertFalse(isinstance(qs.no_dereference().get().organization, + Organization)) self.assertTrue(isinstance(qs.first().organization, Organization)) def test_cached_queryset(self): From e6374ab425d45c3d4007a1773b364bd29a40bd24 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Jun 2013 10:40:15 +0000 Subject: [PATCH 1198/1279] Added Michael Bartnett to Authors (#373) --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 7788139..a50eb57 100644 --- a/AUTHORS +++ b/AUTHORS @@ -169,4 +169,5 @@ that much better: * Massimo Santini (https://github.com/mapio) * Nigel McNie (https://github.com/nigelmcnie) * ygbourhis (https://github.com/ygbourhis) - * Bob Dickinson (https://github.com/BobDickinson) \ No newline at end of file + * Bob Dickinson (https://github.com/BobDickinson) + * Michael Bartnett (https://github.com/michaelbartnett) \ No newline at end of file From 9867e918fa58f99cceca8a8917c9673251cc309d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Jun 2013 11:04:29 +0000 Subject: [PATCH 1199/1279] Fixed weakref being valid after reload (#374) --- docs/changelog.rst | 1 + mongoengine/document.py | 1 + tests/queryset/queryset.py | 26 ++++++++++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index b04bba5..265ad13 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.3 ================ +- Fixed weakref being valid after reload (#374) - Fixed queryset.get() respecting no_dereference (#373) - Added full_result kwarg to update (#380) diff --git a/mongoengine/document.py b/mongoengine/document.py index a1bac19..ab8fa2a 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -480,6 +480,7 @@ class Document(BaseDocument): value = [self._reload(key, v) for v in value] value = BaseList(value, self, key) elif isinstance(value, (EmbeddedDocument, DynamicEmbeddedDocument)): + value._instance = None value._changed_fields = [] return value diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 7c47360..6dcbd9f 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -1613,6 +1613,32 @@ class QuerySetTest(unittest.TestCase): self.assertEqual(message.authors[1].name, "Ross") self.assertEqual(message.authors[2].name, "Adam") + def test_reload_embedded_docs_instance(self): + + class SubDoc(EmbeddedDocument): + val = IntField() + + class Doc(Document): + embedded = EmbeddedDocumentField(SubDoc) + + doc = Doc(embedded=SubDoc(val=0)).save() + doc.reload() + + self.assertEqual(doc.pk, doc.embedded._instance.pk) + + def test_reload_list_embedded_docs_instance(self): + + class SubDoc(EmbeddedDocument): + val = IntField() + + class Doc(Document): + embedded = ListField(EmbeddedDocumentField(SubDoc)) + + doc = Doc(embedded=[SubDoc(val=0)]).save() + doc.reload() + + self.assertEqual(doc.pk, doc.embedded[0]._instance.pk) + def test_order_by(self): """Ensure that QuerySets may be ordered. """ From d6edef98c6b08383cff7384f0548cc8991adc7bb Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 21 Jun 2013 11:29:23 +0000 Subject: [PATCH 1200/1279] Added match ($elemMatch) support for EmbeddedDocuments (#379) --- docs/changelog.rst | 1 + mongoengine/queryset/transform.py | 1 + tests/queryset/queryset.py | 13 ++++++++----- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 265ad13..1927bee 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.3 ================ +- Added match ($elemMatch) support for EmbeddedDocuments (#379) - Fixed weakref being valid after reload (#374) - Fixed queryset.get() respecting no_dereference (#373) - Added full_result kwarg to update (#380) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 4062fc1..352774f 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -95,6 +95,7 @@ def query(_doc_cls=None, _field_operation=False, **query): value = _geo_operator(field, op, value) elif op in CUSTOM_OPERATORS: if op == 'match': + value = field.prepare_query_value(op, value) value = {"$elemMatch": value} else: NotImplementedError("Custom method '%s' has not " diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 6dcbd9f..eabb3c5 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3091,7 +3091,7 @@ class QuerySetTest(unittest.TestCase): class Foo(EmbeddedDocument): shape = StringField() color = StringField() - trick = BooleanField() + thick = BooleanField() meta = {'allow_inheritance': False} class Bar(Document): @@ -3100,17 +3100,20 @@ class QuerySetTest(unittest.TestCase): Bar.drop_collection() - b1 = Bar(foo=[Foo(shape= "square", color ="purple", thick = False), - Foo(shape= "circle", color ="red", thick = True)]) + b1 = Bar(foo=[Foo(shape="square", color="purple", thick=False), + Foo(shape="circle", color="red", thick=True)]) b1.save() - b2 = Bar(foo=[Foo(shape= "square", color ="red", thick = True), - Foo(shape= "circle", color ="purple", thick = False)]) + b2 = Bar(foo=[Foo(shape="square", color="red", thick=True), + Foo(shape="circle", color="purple", thick=False)]) b2.save() ak = list(Bar.objects(foo__match={'shape': "square", "color": "purple"})) self.assertEqual([b1], ak) + ak = list(Bar.objects(foo__match=Foo(shape="square", color="purple"))) + self.assertEqual([b1], ak) + def test_upsert_includes_cls(self): """Upserts should include _cls information for inheritable classes """ From caff44c663295322ba62c1dc26ff488b1d13b651 Mon Sep 17 00:00:00 2001 From: kelvinhammond Date: Fri, 21 Jun 2013 09:39:11 -0400 Subject: [PATCH 1201/1279] Fixed sum and average queryset function * Fixed sum and average map reduce functions for sum and average so that it works with mongo dot notation. * Added unittest cases / updated them for the new changes --- mongoengine/queryset/queryset.py | 37 +++++++++++++++++++++++++++++--- tests/queryset/queryset.py | 26 ++++++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index bf80c69..e5026fd 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1068,7 +1068,22 @@ class QuerySet(object): """ map_func = Code(""" function() { - emit(1, eval("this." + field) || 0); + function deepFind(obj, path) { + var paths = path.split('.') + , current = obj + , i; + + for (i = 0; i < paths.length; ++i) { + if (current[paths[i]] == undefined) { + return undefined; + } else { + current = current[paths[i]]; + } + } + return current; + } + + emit(1, deepFind(this, field) || 0); } """, scope={'field': field}) @@ -1098,8 +1113,24 @@ class QuerySet(object): """ map_func = Code(""" function() { - if (this.hasOwnProperty(field)) - emit(1, {t: eval("this." + field) || 0, c: 1}); + function deepFind(obj, path) { + var paths = path.split('.') + , current = obj + , i; + + for (i = 0; i < paths.length; ++i) { + if (current[paths[i]] == undefined) { + return undefined; + } else { + current = current[paths[i]]; + } + } + return current; + } + + val = deepFind(this, field) + if (val !== undefined) + emit(1, {t: val || 0, c: 1}); } """, scope={'field': field}) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 6dcbd9f..de66ddc 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -30,12 +30,17 @@ class QuerySetTest(unittest.TestCase): def setUp(self): connect(db='mongoenginetest') + class PersonMeta(EmbeddedDocument): + weight = IntField() + class Person(Document): name = StringField() age = IntField() + person_meta = EmbeddedDocumentField(PersonMeta) meta = {'allow_inheritance': True} Person.drop_collection() + self.PersonMeta = PersonMeta self.Person = Person def test_initialisation(self): @@ -2208,6 +2213,19 @@ class QuerySetTest(unittest.TestCase): self.Person(name='ageless person').save() self.assertEqual(int(self.Person.objects.average('age')), avg) + # dot notation + self.Person(name='person meta', person_meta=self.PersonMeta(weight=0)).save() + self.assertAlmostEqual(int(self.Person.objects.average('person_meta.weight')), 0) + + for i, weight in enumerate(ages): + self.Person(name='test meta%i', person_meta=self.PersonMeta(weight=weight)).save() + + self.assertAlmostEqual(int(self.Person.objects.average('person_meta.weight')), avg) + + self.Person(name='test meta none').save() + self.assertEqual(int(self.Person.objects.average('person_meta.weight')), avg) + + def test_sum(self): """Ensure that field can be summed over correctly. """ @@ -2220,6 +2238,14 @@ class QuerySetTest(unittest.TestCase): self.Person(name='ageless person').save() self.assertEqual(int(self.Person.objects.sum('age')), sum(ages)) + for i, age in enumerate(ages): + self.Person(name='test meta%s' % i, person_meta=self.PersonMeta(weight=age)).save() + + self.assertEqual(int(self.Person.objects.sum('person_meta.weight')), sum(ages)) + + self.Person(name='weightless person').save() + self.assertEqual(int(self.Person.objects.sum('age')), sum(ages)) + def test_distinct(self): """Ensure that the QuerySet.distinct method works. """ From fbe5df84c0f5d49396093f0549b9b3fab28c454d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 25 Jun 2013 09:30:28 +0000 Subject: [PATCH 1202/1279] Remove users post uri test --- tests/test_connection.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_connection.py b/tests/test_connection.py index d792648..d27a66d 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -56,6 +56,9 @@ class ConnectionTest(unittest.TestCase): self.assertTrue(isinstance(db, pymongo.database.Database)) self.assertEqual(db.name, 'mongoenginetest') + c.admin.system.users.remove({}) + c.mongoenginetest.system.users.remove({}) + def test_register_connection(self): """Ensure that connections with different aliases may be registered. """ From 8d21e5f3c10a0341db2687aef56385ec245f3bae Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 2 Jul 2013 09:47:54 +0000 Subject: [PATCH 1203/1279] Fix tests for py2.6 --- tests/queryset/queryset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index eabb3c5..4d91b55 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -540,13 +540,13 @@ class QuerySetTest(unittest.TestCase): self.Person.drop_collection() result = self.Person(name="Bob", age=25).update(upsert=True, full_result=True) - self.assertIsInstance(result, dict) + self.assertTrue(isinstance(result, dict)) self.assertTrue("upserted" in result) self.assertFalse(result["updatedExisting"]) bob = self.Person.objects.first() result = bob.update(set__age=30, full_result=True) - self.assertIsInstance(result, dict) + self.assertTrue(isinstance(result, dict)) self.assertTrue(result["updatedExisting"]) self.Person(name="Bob", age=20).save() From 43d6e64cfa959e31ff6e983c74534c06ac5b3108 Mon Sep 17 00:00:00 2001 From: Jan Schrewe Date: Tue, 2 Jul 2013 17:04:15 +0200 Subject: [PATCH 1204/1279] Added a get_proxy_obj method to FileField and handle FileFields in container fields properly in ImageGridFsProxy. --- mongoengine/fields.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 451f7ac..727803f 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1190,9 +1190,7 @@ class FileField(BaseField): # Check if a file already exists for this model grid_file = instance._data.get(self.name) if not isinstance(grid_file, self.proxy_class): - grid_file = self.proxy_class(key=self.name, instance=instance, - db_alias=self.db_alias, - collection_name=self.collection_name) + grid_file = self.get_proxy_obj(key=key, instance=instance) instance._data[self.name] = grid_file if not grid_file.key: @@ -1214,14 +1212,22 @@ class FileField(BaseField): pass # Create a new proxy object as we don't already have one - instance._data[key] = self.proxy_class(key=key, instance=instance, - db_alias=self.db_alias, - collection_name=self.collection_name) + instance._data[key] = self.get_proxy_obj(key=key, instance=instance) instance._data[key].put(value) else: instance._data[key] = value instance._mark_as_changed(key) + + def get_proxy_obj(self, key, instance, db_alias=None, collection_name=None): + if db_alias is None: + db_alias = self.db_alias + if collection_name is None: + collection_name = self.collection_name + + return self.proxy_class(key=key, instance=instance, + db_alias=db_alias, + collection_name=collection_name) def to_mongo(self, value): # Store the GridFS file id in MongoDB @@ -1255,6 +1261,11 @@ class ImageGridFsProxy(GridFSProxy): applying field properties (size, thumbnail_size) """ field = self.instance._fields[self.key] + # if the field from the instance has an attribute field + # we use that one and hope for the best. Usually only container + # fields have a field attribute. + if hasattr(field, 'field'): + field = field.field try: img = Image.open(file_obj) From 5021b1053599a53a9b50d02236fa701230065a71 Mon Sep 17 00:00:00 2001 From: Serge Matveenko Date: Wed, 3 Jul 2013 01:17:40 +0400 Subject: [PATCH 1205/1279] Fix crash on Python 3.x and Django >= 1.5 --- mongoengine/django/sessions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mongoengine/django/sessions.py b/mongoengine/django/sessions.py index c90807e..7e4e182 100644 --- a/mongoengine/django/sessions.py +++ b/mongoengine/django/sessions.py @@ -1,7 +1,10 @@ from django.conf import settings from django.contrib.sessions.backends.base import SessionBase, CreateError from django.core.exceptions import SuspiciousOperation -from django.utils.encoding import force_unicode +try: + from django.utils.encoding import force_unicode +except ImportError: + from django.utils.encoding import force_text as force_unicode from mongoengine.document import Document from mongoengine import fields From 592c654916c38eb3b01bf98db7160f4c871ab936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Fri, 5 Jul 2013 10:36:11 -0300 Subject: [PATCH 1206/1279] extending support for queryset.sum and queryset.average methods --- mongoengine/queryset/queryset.py | 51 ++++++++--- tests/queryset/queryset.py | 140 +++++++++++++++++++++++++++++++ 2 files changed, 181 insertions(+), 10 deletions(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index ded8d5e..86a14b5 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1066,11 +1066,27 @@ class QuerySet(object): .. versionchanged:: 0.5 - updated to map_reduce as db.eval doesnt work with sharding. """ - map_func = Code(""" + map_func = """ function() { - emit(1, this[field] || 0); + var path = '{{~%(field)s}}'.split('.'), + field = this; + + for (p in path) { + if (typeof field != 'undefined') + field = field[path[p]]; + else + break; + } + + if (field && field.constructor == Array) { + field.forEach(function(item) { + emit(1, item||0); + }); + } else if (typeof field != 'undefined') { + emit(1, field||0); + } } - """, scope={'field': field}) + """ % dict(field=field) reduce_func = Code(""" function(key, values) { @@ -1096,13 +1112,28 @@ class QuerySet(object): .. versionchanged:: 0.5 - updated to map_reduce as db.eval doesnt work with sharding. """ - map_func = Code(""" + map_func = """ function() { - if (this.hasOwnProperty(field)) - emit(1, {t: this[field] || 0, c: 1}); - } - """, scope={'field': field}) + var path = '{{~%(field)s}}'.split('.'), + field = this; + for (p in path) { + if (typeof field != 'undefined') + field = field[path[p]]; + else + break; + } + + if (field && field.constructor == Array) { + field.forEach(function(item) { + emit(1, {t: item||0, c: 1}); + }); + } else if (typeof field != 'undefined') { + emit(1, {t: field||0, c: 1}); + } + } + """ % dict(field=field) + reduce_func = Code(""" function(key, values) { var out = {t: 0, c: 0}; @@ -1263,8 +1294,8 @@ class QuerySet(object): def _item_frequencies_map_reduce(self, field, normalize=False): map_func = """ function() { - var path = '{{~%(field)s}}'.split('.'); - var field = this; + var path = '{{~%(field)s}}'.split('.'), + field = this; for (p in path) { if (typeof field != 'undefined') diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 4d91b55..3f9bd23 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -2208,6 +2208,75 @@ class QuerySetTest(unittest.TestCase): self.Person(name='ageless person').save() self.assertEqual(int(self.Person.objects.average('age')), avg) + def test_embedded_average(self): + class Pay(EmbeddedDocument): + value = DecimalField() + + class Doc(Document): + name = StringField() + pay = EmbeddedDocumentField( + Pay) + + Doc.drop_collection() + + Doc(name=u"Wilson Junior", + pay=Pay(value=150)).save() + + Doc(name=u"Isabella Luanna", + pay=Pay(value=530)).save() + + Doc(name=u"Tayza mariana", + pay=Pay(value=165)).save() + + Doc(name=u"Eliana Costa", + pay=Pay(value=115)).save() + + self.assertEqual( + Doc.objects.average('pay.value'), + 240) + + def test_embedded_array_average(self): + class Pay(EmbeddedDocument): + values = ListField(DecimalField()) + + class Doc(Document): + name = StringField() + pay = EmbeddedDocumentField( + Pay) + + Doc.drop_collection() + + Doc(name=u"Wilson Junior", + pay=Pay(values=[150, 100])).save() + + Doc(name=u"Isabella Luanna", + pay=Pay(values=[530, 100])).save() + + Doc(name=u"Tayza mariana", + pay=Pay(values=[165, 100])).save() + + Doc(name=u"Eliana Costa", + pay=Pay(values=[115, 100])).save() + + self.assertEqual( + Doc.objects.average('pay.values'), + 170) + + def test_array_average(self): + class Doc(Document): + values = ListField(DecimalField()) + + Doc.drop_collection() + + Doc(values=[150, 100]).save() + Doc(values=[530, 100]).save() + Doc(values=[165, 100]).save() + Doc(values=[115, 100]).save() + + self.assertEqual( + Doc.objects.average('values'), + 170) + def test_sum(self): """Ensure that field can be summed over correctly. """ @@ -2220,6 +2289,77 @@ class QuerySetTest(unittest.TestCase): self.Person(name='ageless person').save() self.assertEqual(int(self.Person.objects.sum('age')), sum(ages)) + def test_embedded_sum(self): + class Pay(EmbeddedDocument): + value = DecimalField() + + class Doc(Document): + name = StringField() + pay = EmbeddedDocumentField( + Pay) + + Doc.drop_collection() + + Doc(name=u"Wilson Junior", + pay=Pay(value=150)).save() + + Doc(name=u"Isabella Luanna", + pay=Pay(value=530)).save() + + Doc(name=u"Tayza mariana", + pay=Pay(value=165)).save() + + Doc(name=u"Eliana Costa", + pay=Pay(value=115)).save() + + self.assertEqual( + Doc.objects.sum('pay.value'), + 960) + + + def test_embedded_array_sum(self): + class Pay(EmbeddedDocument): + values = ListField(DecimalField()) + + class Doc(Document): + name = StringField() + pay = EmbeddedDocumentField( + Pay) + + Doc.drop_collection() + + Doc(name=u"Wilson Junior", + pay=Pay(values=[150, 100])).save() + + Doc(name=u"Isabella Luanna", + pay=Pay(values=[530, 100])).save() + + Doc(name=u"Tayza mariana", + pay=Pay(values=[165, 100])).save() + + Doc(name=u"Eliana Costa", + pay=Pay(values=[115, 100])).save() + + self.assertEqual( + Doc.objects.sum('pay.values'), + 1360) + + def test_array_sum(self): + class Doc(Document): + values = ListField(DecimalField()) + + Doc.drop_collection() + + Doc(values=[150, 100]).save() + Doc(values=[530, 100]).save() + Doc(values=[165, 100]).save() + Doc(values=[115, 100]).save() + + self.assertEqual( + Doc.objects.sum('values'), + 1360) + + def test_distinct(self): """Ensure that the QuerySet.distinct method works. """ From a1d142d3a4bcfb6dd2d9df5d3adf5eec2c51edb5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 10 Jul 2013 08:38:13 +0000 Subject: [PATCH 1207/1279] Prep for django and py3 support --- .travis.yml | 2 ++ setup.py | 2 +- tests/test_django.py | 60 ++++++++++++++++++-------------------------- 3 files changed, 27 insertions(+), 37 deletions(-) diff --git a/.travis.yml b/.travis.yml index b7c56a0..2bb5863 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,8 @@ env: - PYMONGO=dev DJANGO=1.4.2 - PYMONGO=2.5 DJANGO=1.5.1 - PYMONGO=2.5 DJANGO=1.4.2 + - PYMONGO=3.2 DJANGO=1.5.1 + - PYMONGO=3.3 DJANGO=1.5.1 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi diff --git a/setup.py b/setup.py index effb6f1..f6b3c1b 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ CLASSIFIERS = [ extra_opts = {} if sys.version_info[0] == 3: extra_opts['use_2to3'] = True - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6', 'django>=1.5.1'] extra_opts['packages'] = find_packages(exclude=('tests',)) if "test" in sys.argv or "nosetests" in sys.argv: extra_opts['packages'].append("tests") diff --git a/tests/test_django.py b/tests/test_django.py index 63e3245..d67b126 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -2,48 +2,42 @@ import sys sys.path[0:0] = [""] import unittest from nose.plugins.skip import SkipTest -from mongoengine.python_support import PY3 from mongoengine import * + +from mongoengine.django.shortcuts import get_document_or_404 + +from django.http import Http404 +from django.template import Context, Template +from django.conf import settings +from django.core.paginator import Paginator + +settings.configure( + USE_TZ=True, + INSTALLED_APPS=('django.contrib.auth', 'mongoengine.django.mongo_auth'), + AUTH_USER_MODEL=('mongo_auth.MongoUser'), +) + try: - from mongoengine.django.shortcuts import get_document_or_404 - - from django.http import Http404 - from django.template import Context, Template - from django.conf import settings - from django.core.paginator import Paginator - - settings.configure( - USE_TZ=True, - INSTALLED_APPS=('django.contrib.auth', 'mongoengine.django.mongo_auth'), - AUTH_USER_MODEL=('mongo_auth.MongoUser'), - ) - - try: - from django.contrib.auth import authenticate, get_user_model - from mongoengine.django.auth import User - from mongoengine.django.mongo_auth.models import MongoUser, MongoUserManager - DJ15 = True - except Exception: - DJ15 = False - from django.contrib.sessions.tests import SessionTestsMixin - from mongoengine.django.sessions import SessionStore, MongoSession -except Exception, err: - if PY3: - SessionTestsMixin = type # dummy value so no error - SessionStore = None # dummy value so no error - else: - raise err + from django.contrib.auth import authenticate, get_user_model + from mongoengine.django.auth import User + from mongoengine.django.mongo_auth.models import MongoUser, MongoUserManager + DJ15 = True +except Exception: + DJ15 = False +from django.contrib.sessions.tests import SessionTestsMixin +from mongoengine.django.sessions import SessionStore, MongoSession from datetime import tzinfo, timedelta ZERO = timedelta(0) + class FixedOffset(tzinfo): """Fixed offset in minutes east from UTC.""" def __init__(self, offset, name): - self.__offset = timedelta(minutes = offset) + self.__offset = timedelta(minutes=offset) self.__name = name def utcoffset(self, dt): @@ -70,8 +64,6 @@ def activate_timezone(tz): class QuerySetTest(unittest.TestCase): def setUp(self): - if PY3: - raise SkipTest('django does not have Python 3 support') connect(db='mongoenginetest') class Person(Document): @@ -223,8 +215,6 @@ class MongoDBSessionTest(SessionTestsMixin, unittest.TestCase): backend = SessionStore def setUp(self): - if PY3: - raise SkipTest('django does not have Python 3 support') connect(db='mongoenginetest') MongoSession.drop_collection() super(MongoDBSessionTest, self).setUp() @@ -262,8 +252,6 @@ class MongoAuthTest(unittest.TestCase): } def setUp(self): - if PY3: - raise SkipTest('django does not have Python 3 support') if not DJ15: raise SkipTest('mongo_auth requires Django 1.5') connect(db='mongoenginetest') From 0cb40703641d94f6334b92ae23148a9b7331bfa4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 10 Jul 2013 08:53:56 +0000 Subject: [PATCH 1208/1279] Added Django 1.5 PY3 support (#392) --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 1927bee..f433f21 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.3 ================ +- Added Django 1.5 PY3 support (#392) - Added match ($elemMatch) support for EmbeddedDocuments (#379) - Fixed weakref being valid after reload (#374) - Fixed queryset.get() respecting no_dereference (#373) From 7cb46d0761e6b058d55f8ad93f584fa6bb6ade15 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 10 Jul 2013 09:11:50 +0000 Subject: [PATCH 1209/1279] Fixed ListField setslice and delslice dirty tracking (#390) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + mongoengine/base/datastructures.py | 8 ++++++++ tests/fields/fields.py | 26 ++++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index a50eb57..e88de8c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -170,4 +170,5 @@ that much better: * Nigel McNie (https://github.com/nigelmcnie) * ygbourhis (https://github.com/ygbourhis) * Bob Dickinson (https://github.com/BobDickinson) - * Michael Bartnett (https://github.com/michaelbartnett) \ No newline at end of file + * Michael Bartnett (https://github.com/michaelbartnett) + * Alon Horev (https://github.com/alonho) \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index f433f21..27d51a4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.3 ================ +- Fixed ListField setslice and delslice dirty tracking (#390) - Added Django 1.5 PY3 support (#392) - Added match ($elemMatch) support for EmbeddedDocuments (#379) - Fixed weakref being valid after reload (#374) diff --git a/mongoengine/base/datastructures.py b/mongoengine/base/datastructures.py index adcd8d0..4652fb5 100644 --- a/mongoengine/base/datastructures.py +++ b/mongoengine/base/datastructures.py @@ -108,6 +108,14 @@ class BaseList(list): self._mark_as_changed() return super(BaseList, self).__delitem__(*args, **kwargs) + def __setslice__(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).__setslice__(*args, **kwargs) + + def __delslice__(self, *args, **kwargs): + self._mark_as_changed() + return super(BaseList, self).__delslice__(*args, **kwargs) + def __getstate__(self): self.instance = None self._dereferenced = False diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 3e48a21..8f02499 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -1018,6 +1018,32 @@ class FieldTest(unittest.TestCase): e.mapping = {} self.assertEqual([], e._changed_fields) + def test_slice_marks_field_as_changed(self): + + class Simple(Document): + widgets = ListField() + + simple = Simple(widgets=[1, 2, 3, 4]).save() + simple.widgets[:3] = [] + self.assertEqual(['widgets'], simple._changed_fields) + simple.save() + + simple = simple.reload() + self.assertEqual(simple.widgets, [4]) + + def test_del_slice_marks_field_as_changed(self): + + class Simple(Document): + widgets = ListField() + + simple = Simple(widgets=[1, 2, 3, 4]).save() + del simple.widgets[:3] + self.assertEqual(['widgets'], simple._changed_fields) + simple.save() + + simple = simple.reload() + self.assertEqual(simple.widgets, [4]) + def test_list_field_complex(self): """Ensure that the list fields can handle the complex types.""" From af86aee9700504458ae945914a2b9412c5da8ea6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 10 Jul 2013 10:57:24 +0000 Subject: [PATCH 1210/1279] _dynamic field updates - fixed pickling and creation order Dynamic fields are ordered based on creation and stored in _fields_ordered (#396) Fixed pickling dynamic documents `_dynamic_fields` (#387) --- docs/changelog.rst | 2 ++ docs/guide/defining-documents.rst | 2 +- docs/upgrade.rst | 10 +++++++ mongoengine/base/document.py | 44 ++++++++++++------------------- mongoengine/base/metaclasses.py | 11 ++++++-- mongoengine/document.py | 5 +--- tests/document/delta.py | 13 ++++----- tests/document/instance.py | 36 ++++++++++++++++++++++++- tests/fixtures.py | 8 ++++++ 9 files changed, 90 insertions(+), 41 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 27d51a4..78deafb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,8 @@ Changelog Changes in 0.8.3 ================ +- Dynamic fields are ordered based on creation and stored in _fields_ordered (#396) +- Fixed pickling dynamic documents `_dynamic_fields` (#387) - Fixed ListField setslice and delslice dirty tracking (#390) - Added Django 1.5 PY3 support (#392) - Added match ($elemMatch) support for EmbeddedDocuments (#379) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index a61d8fe..a50450e 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -54,7 +54,7 @@ be saved :: There is one caveat on Dynamic Documents: fields cannot start with `_` -Dynamic fields are stored in alphabetical order *after* any declared fields. +Dynamic fields are stored in creation order *after* any declared fields. Fields ====== diff --git a/docs/upgrade.rst b/docs/upgrade.rst index c3d3182..b8864b0 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -2,6 +2,16 @@ Upgrading ######### + +0.8.2 to 0.8.2 +************** + +Minor change that may impact users: + +DynamicDocument fields are now stored in creation order after any declared +fields. Previously they were stored alphabetically. + + 0.7 to 0.8 ********** diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index ca154a2..04b0c05 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -42,6 +42,9 @@ class BaseDocument(object): # Combine positional arguments with named arguments. # We only want named arguments. field = iter(self._fields_ordered) + # If its an automatic id field then skip to the first defined field + if self._auto_id_field: + next(field) for value in args: name = next(field) if name in values: @@ -51,6 +54,7 @@ class BaseDocument(object): signals.pre_init.send(self.__class__, document=self, values=values) self._data = {} + self._dynamic_fields = SON() # Assign default values to instance for key, field in self._fields.iteritems(): @@ -61,7 +65,6 @@ class BaseDocument(object): # Set passed values after initialisation if self._dynamic: - self._dynamic_fields = {} dynamic_data = {} for key, value in values.iteritems(): if key in self._fields or key == '_id': @@ -116,6 +119,7 @@ class BaseDocument(object): field = DynamicField(db_field=name) field.name = name self._dynamic_fields[name] = field + self._fields_ordered += (name,) if not name.startswith('_'): value = self.__expand_dynamic_values(name, value) @@ -142,7 +146,8 @@ class BaseDocument(object): def __getstate__(self): data = {} - for k in ('_changed_fields', '_initialised', '_created'): + for k in ('_changed_fields', '_initialised', '_created', + '_dynamic_fields', '_fields_ordered'): if hasattr(self, k): data[k] = getattr(self, k) data['_data'] = self.to_mongo() @@ -151,21 +156,21 @@ class BaseDocument(object): def __setstate__(self, data): if isinstance(data["_data"], SON): data["_data"] = self.__class__._from_son(data["_data"])._data - for k in ('_changed_fields', '_initialised', '_created', '_data'): + for k in ('_changed_fields', '_initialised', '_created', '_data', + '_fields_ordered', '_dynamic_fields'): if k in data: setattr(self, k, data[k]) + for k in data.get('_dynamic_fields').keys(): + setattr(self, k, data["_data"].get(k)) def __iter__(self): - if 'id' in self._fields and 'id' not in self._fields_ordered: - return iter(('id', ) + self._fields_ordered) - return iter(self._fields_ordered) def __getitem__(self, name): """Dictionary-style field access, return a field's value if present. """ try: - if name in self._fields: + if name in self._fields_ordered: return getattr(self, name) except AttributeError: pass @@ -241,6 +246,8 @@ class BaseDocument(object): for field_name in self: value = self._data.get(field_name, None) field = self._fields.get(field_name) + if field is None and self._dynamic: + field = self._dynamic_fields.get(field_name) if value is not None: value = field.to_mongo(value) @@ -265,15 +272,6 @@ class BaseDocument(object): not self._meta.get('allow_inheritance', ALLOW_INHERITANCE)): data.pop('_cls') - if not self._dynamic: - return data - - # Sort dynamic fields by key - dynamic_fields = sorted(self._dynamic_fields.iteritems(), - key=operator.itemgetter(0)) - for name, field in dynamic_fields: - data[name] = field.to_mongo(self._data.get(name, None)) - return data def validate(self, clean=True): @@ -289,11 +287,8 @@ class BaseDocument(object): errors[NON_FIELD_ERRORS] = error # Get a list of tuples of field names and their current values - fields = [(field, self._data.get(name)) - for name, field in self._fields.items()] - if self._dynamic: - fields += [(field, self._data.get(name)) - for name, field in self._dynamic_fields.items()] + fields = [(self._fields.get(name, self._dynamic_fields.get(name)), + self._data.get(name)) for name in self._fields_ordered] EmbeddedDocumentField = _import_class("EmbeddedDocumentField") GenericEmbeddedDocumentField = _import_class("GenericEmbeddedDocumentField") @@ -406,11 +401,7 @@ class BaseDocument(object): return _changed_fields inspected.add(self.id) - field_list = self._fields.copy() - if self._dynamic: - field_list.update(self._dynamic_fields) - - for field_name in field_list: + for field_name in self._fields_ordered: db_field_name = self._db_field_map.get(field_name, field_name) key = '%s.' % db_field_name @@ -450,7 +441,6 @@ class BaseDocument(object): doc = self.to_mongo() set_fields = self._get_changed_fields() - set_data = {} unset_data = {} parts = [] if hasattr(self, '_changed_fields'): diff --git a/mongoengine/base/metaclasses.py b/mongoengine/base/metaclasses.py index 444d9a2..ff5afdd 100644 --- a/mongoengine/base/metaclasses.py +++ b/mongoengine/base/metaclasses.py @@ -91,11 +91,12 @@ class DocumentMetaclass(type): attrs['_fields'] = doc_fields attrs['_db_field_map'] = dict([(k, getattr(v, 'db_field', k)) for k, v in doc_fields.iteritems()]) + attrs['_reverse_db_field_map'] = dict( + (v, k) for k, v in attrs['_db_field_map'].iteritems()) + attrs['_fields_ordered'] = tuple(i[1] for i in sorted( (v.creation_counter, v.name) for v in doc_fields.itervalues())) - attrs['_reverse_db_field_map'] = dict( - (v, k) for k, v in attrs['_db_field_map'].iteritems()) # # Set document hierarchy @@ -358,12 +359,18 @@ class TopLevelDocumentMetaclass(DocumentMetaclass): new_class.id = field # Set primary key if not defined by the document + new_class._auto_id_field = False if not new_class._meta.get('id_field'): + new_class._auto_id_field = True new_class._meta['id_field'] = 'id' new_class._fields['id'] = ObjectIdField(db_field='_id') new_class._fields['id'].name = 'id' new_class.id = new_class._fields['id'] + # Prepend id field to _fields_ordered + if 'id' in new_class._fields and 'id' not in new_class._fields_ordered: + new_class._fields_ordered = ('id', ) + new_class._fields_ordered + # Merge in exceptions with parent hierarchy exceptions_to_merge = (DoesNotExist, MultipleObjectsReturned) module = attrs.get('__module__') diff --git a/mongoengine/document.py b/mongoengine/document.py index ab8fa2a..d0c9e61 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -460,11 +460,8 @@ class Document(BaseDocument): else: msg = "Reloaded document has been deleted" raise OperationError(msg) - for field in self._fields: + for field in self._fields_ordered: setattr(self, field, self._reload(field, obj[field])) - if self._dynamic: - for name in self._dynamic_fields.keys(): - setattr(self, name, self._reload(name, obj._data[name])) self._changed_fields = obj._changed_fields self._created = False return obj diff --git a/tests/document/delta.py b/tests/document/delta.py index 16ab609..3656d9e 100644 --- a/tests/document/delta.py +++ b/tests/document/delta.py @@ -3,6 +3,7 @@ import sys sys.path[0:0] = [""] import unittest +from bson import SON from mongoengine import * from mongoengine.connection import get_db @@ -613,13 +614,13 @@ class DeltaTest(unittest.TestCase): Person.drop_collection() p = Person(name="James", age=34) - self.assertEqual(p._delta(), ({'age': 34, 'name': 'James', - '_cls': 'Person'}, {})) + self.assertEqual(p._delta(), ( + SON([('_cls', 'Person'), ('name', 'James'), ('age', 34)]), {})) p.doc = 123 del(p.doc) - self.assertEqual(p._delta(), ({'age': 34, 'name': 'James', - '_cls': 'Person'}, {'doc': 1})) + self.assertEqual(p._delta(), ( + SON([('_cls', 'Person'), ('name', 'James'), ('age', 34)]), {})) p = Person() p.name = "Dean" @@ -631,14 +632,14 @@ class DeltaTest(unittest.TestCase): self.assertEqual(p._get_changed_fields(), ['age']) self.assertEqual(p._delta(), ({'age': 24}, {})) - p = self.Person.objects(age=22).get() + p = Person.objects(age=22).get() p.age = 24 self.assertEqual(p.age, 24) self.assertEqual(p._get_changed_fields(), ['age']) self.assertEqual(p._delta(), ({'age': 24}, {})) p.save() - self.assertEqual(1, self.Person.objects(age=24).count()) + self.assertEqual(1, Person.objects(age=24).count()) def test_dynamic_delta(self): diff --git a/tests/document/instance.py b/tests/document/instance.py index 81734aa..e85c9d8 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -10,7 +10,8 @@ import uuid from datetime import datetime from bson import DBRef -from tests.fixtures import PickleEmbedded, PickleTest, PickleSignalsTest +from tests.fixtures import (PickleEmbedded, PickleTest, PickleSignalsTest, + PickleDyanmicEmbedded, PickleDynamicTest) from mongoengine import * from mongoengine.errors import (NotRegistered, InvalidDocumentError, @@ -1827,6 +1828,29 @@ class InstanceTest(unittest.TestCase): self.assertEqual(pickle_doc.string, "Two") self.assertEqual(pickle_doc.lists, ["1", "2", "3"]) + def test_dynamic_document_pickle(self): + + pickle_doc = PickleDynamicTest(name="test", number=1, string="One", lists=['1', '2']) + pickle_doc.embedded = PickleDyanmicEmbedded(foo="Bar") + pickled_doc = pickle.dumps(pickle_doc) # make sure pickling works even before the doc is saved + + pickle_doc.save() + + pickled_doc = pickle.dumps(pickle_doc) + resurrected = pickle.loads(pickled_doc) + + self.assertEqual(resurrected, pickle_doc) + self.assertEqual(resurrected._fields_ordered, + pickle_doc._fields_ordered) + self.assertEqual(resurrected._dynamic_fields.keys(), + pickle_doc._dynamic_fields.keys()) + + self.assertEqual(resurrected.embedded, pickle_doc.embedded) + self.assertEqual(resurrected.embedded._fields_ordered, + pickle_doc.embedded._fields_ordered) + self.assertEqual(resurrected.embedded._dynamic_fields.keys(), + pickle_doc.embedded._dynamic_fields.keys()) + def test_picklable_on_signals(self): pickle_doc = PickleSignalsTest(number=1, string="One", lists=['1', '2']) pickle_doc.embedded = PickleEmbedded() @@ -2289,6 +2313,16 @@ class InstanceTest(unittest.TestCase): self.assertEqual(person.name, "Test User") self.assertEqual(person.age, 42) + def test_mixed_creation_dynamic(self): + """Ensure that document may be created using mixed arguments. + """ + class Person(DynamicDocument): + name = StringField() + + person = Person("Test User", age=42) + self.assertEqual(person.name, "Test User") + self.assertEqual(person.age, 42) + def test_bad_mixed_creation(self): """Ensure that document gives correct error when duplicating arguments """ diff --git a/tests/fixtures.py b/tests/fixtures.py index e207044..f1344d7 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -17,6 +17,14 @@ class PickleTest(Document): photo = FileField() +class PickleDyanmicEmbedded(DynamicEmbeddedDocument): + date = DateTimeField(default=datetime.now) + + +class PickleDynamicTest(DynamicDocument): + number = IntField() + + class PickleSignalsTest(Document): number = IntField() string = StringField(choices=(('One', '1'), ('Two', '2'))) From fa83fba6374d0da5774157012b20b74310c37984 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 10 Jul 2013 11:18:49 +0000 Subject: [PATCH 1211/1279] Reload uses shard_key if applicable (#384) --- docs/changelog.rst | 1 + mongoengine/document.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 78deafb..42fd9bb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.3 ================ +- Reload uses shard_key if applicable (#384) - Dynamic fields are ordered based on creation and stored in _fields_ordered (#396) - Fixed pickling dynamic documents `_dynamic_fields` (#387) - Fixed ListField setslice and delslice dirty tracking (#390) diff --git a/mongoengine/document.py b/mongoengine/document.py index d0c9e61..c9901a2 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -266,7 +266,6 @@ class Document(BaseDocument): upsert=True, **write_concern) created = is_new_object(last_error) - if cascade is None: cascade = self._meta.get('cascade', False) or cascade_kwargs is not None @@ -451,9 +450,8 @@ class Document(BaseDocument): .. versionadded:: 0.1.2 .. versionchanged:: 0.6 Now chainable """ - id_field = self._meta['id_field'] obj = self._qs.read_preference(ReadPreference.PRIMARY).filter( - **{id_field: self[id_field]}).limit(1).select_related(max_depth=max_depth) + **self._object_key).limit(1).select_related(max_depth=max_depth) if obj: obj = obj[0] From 4209d61b1368717047927cee40a9d64768def93a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 10 Jul 2013 12:49:19 +0000 Subject: [PATCH 1212/1279] Document.select_related() now respects `db_alias` (#377) --- docs/changelog.rst | 1 + mongoengine/document.py | 4 ++-- tests/fields/fields.py | 31 +++++++++++++++++++++++++++ tests/test_dereference.py | 45 +++++++++++++++++---------------------- 4 files changed, 54 insertions(+), 27 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 42fd9bb..926c6cb 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.3 ================ +- Document.select_related() now respects `db_alias` (#377) - Reload uses shard_key if applicable (#384) - Dynamic fields are ordered based on creation and stored in _fields_ordered (#396) - Fixed pickling dynamic documents `_dynamic_fields` (#387) diff --git a/mongoengine/document.py b/mongoengine/document.py index c9901a2..e331aa1 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -440,8 +440,8 @@ class Document(BaseDocument): .. versionadded:: 0.5 """ - import dereference - self._data = dereference.DeReference()(self._data, max_depth) + DeReference = _import_class('DeReference') + DeReference()([self], max_depth + 1) return self def reload(self, max_depth=1): diff --git a/tests/fields/fields.py b/tests/fields/fields.py index 8f02499..b3d8d52 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -2474,6 +2474,37 @@ class FieldTest(unittest.TestCase): user = User(email='me@example.com') self.assertTrue(user.validate() is None) + def test_tuples_as_tuples(self): + """ + Ensure that tuples remain tuples when they are + inside a ComplexBaseField + """ + from mongoengine.base import BaseField + + class EnumField(BaseField): + + def __init__(self, **kwargs): + super(EnumField, self).__init__(**kwargs) + + def to_mongo(self, value): + return value + + def to_python(self, value): + return tuple(value) + + class TestDoc(Document): + items = ListField(EnumField()) + + TestDoc.drop_collection() + tuples = [(100, 'Testing')] + doc = TestDoc() + doc.items = tuples + doc.save() + x = TestDoc.objects().get() + self.assertTrue(x is not None) + self.assertTrue(len(x.items) == 1) + self.assertTrue(tuple(x.items[0]) in tuples) + self.assertTrue(x.items[0] in tuples) if __name__ == '__main__': unittest.main() diff --git a/tests/test_dereference.py b/tests/test_dereference.py index e146963..db9868a 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1121,37 +1121,32 @@ class FieldTest(unittest.TestCase): self.assertEqual(q, 2) - def test_tuples_as_tuples(self): - """ - Ensure that tuples remain tuples when they are - inside a ComplexBaseField - """ - from mongoengine.base import BaseField + def test_objectid_reference_across_databases(self): + # mongoenginetest - Is default connection alias from setUp() + # Register Aliases + register_connection('testdb-1', 'mongoenginetest2') - class EnumField(BaseField): + class User(Document): + name = StringField() + meta = {"db_alias": "testdb-1"} - def __init__(self, **kwargs): - super(EnumField, self).__init__(**kwargs) + class Book(Document): + name = StringField() + author = ReferenceField(User) - def to_mongo(self, value): - return value + # Drops + User.drop_collection() + Book.drop_collection() - def to_python(self, value): - return tuple(value) + user = User(name="Ross").save() + Book(name="MongoEngine for pros", author=user).save() - class TestDoc(Document): - items = ListField(EnumField()) + # Can't use query_counter across databases - so test the _data object + book = Book.objects.first() + self.assertFalse(isinstance(book._data['author'], User)) - TestDoc.drop_collection() - tuples = [(100, 'Testing')] - doc = TestDoc() - doc.items = tuples - doc.save() - x = TestDoc.objects().get() - self.assertTrue(x is not None) - self.assertTrue(len(x.items) == 1) - self.assertTrue(tuple(x.items[0]) in tuples) - self.assertTrue(x.items[0] in tuples) + book.select_related() + self.assertTrue(isinstance(book._data['author'], User)) def test_non_ascii_pk(self): """ From f34e8a0ff6ca84d1afd8de0bf78f7696c0c7aed2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 10 Jul 2013 13:38:53 +0000 Subject: [PATCH 1213/1279] Fixed as_pymongo to return the id (#386) --- docs/changelog.rst | 1 + mongoengine/queryset/queryset.py | 3 ++- tests/queryset/queryset.py | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 926c6cb..cbc2c94 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.3 ================ +- Fixed as_pymongo to return the id (#386) - Document.select_related() now respects `db_alias` (#377) - Reload uses shard_key if applicable (#384) - Dynamic fields are ordered based on creation and stored in _fields_ordered (#396) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index ded8d5e..c040e39 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1423,7 +1423,8 @@ class QuerySet(object): # used. If not, handle all fields. if not getattr(self, '__as_pymongo_fields', None): self.__as_pymongo_fields = [] - for field in self._loaded_fields.fields - set(['_cls', '_id']): + + for field in self._loaded_fields.fields - set(['_cls']): self.__as_pymongo_fields.append(field) while '.' in field: field, _ = field.rsplit('.', 1) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 4d91b55..566c14e 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3227,6 +3227,9 @@ class QuerySetTest(unittest.TestCase): User(name="Bob Dole", age=89, price=Decimal('1.11')).save() User(name="Barack Obama", age=51, price=Decimal('2.22')).save() + results = User.objects.only('id', 'name').as_pymongo() + self.assertEqual(results[0].keys(), ['_id', 'name']) + users = User.objects.only('name', 'price').as_pymongo() results = list(users) self.assertTrue(isinstance(results[0], dict)) From 8131f0a752d3e1b48713afe90301d77f224b9ace Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 10 Jul 2013 13:53:18 +0000 Subject: [PATCH 1214/1279] Fixed sum and average mapreduce dot notation support (#375, #376) --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index cbc2c94..8bbc4b4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.3 ================ +- Fixed sum and average mapreduce dot notation support (#375, #376) - Fixed as_pymongo to return the id (#386) - Document.select_related() now respects `db_alias` (#377) - Reload uses shard_key if applicable (#384) From daeecef59e47a5d23f9774f4dab70472b35465f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Wed, 10 Jul 2013 10:59:41 -0300 Subject: [PATCH 1215/1279] Update fields.py Typo in documentation for DecimalField --- mongoengine/fields.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 451f7ac..7f24be2 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -279,14 +279,14 @@ class DecimalField(BaseField): :param precision: Number of decimal places to store. :param rounding: The rounding rule from the python decimal libary: - - decimial.ROUND_CEILING (towards Infinity) - - decimial.ROUND_DOWN (towards zero) - - decimial.ROUND_FLOOR (towards -Infinity) - - decimial.ROUND_HALF_DOWN (to nearest with ties going towards zero) - - decimial.ROUND_HALF_EVEN (to nearest with ties going to nearest even integer) - - decimial.ROUND_HALF_UP (to nearest with ties going away from zero) - - decimial.ROUND_UP (away from zero) - - decimial.ROUND_05UP (away from zero if last digit after rounding towards zero would have been 0 or 5; otherwise towards zero) + - decimal.ROUND_CEILING (towards Infinity) + - decimal.ROUND_DOWN (towards zero) + - decimal.ROUND_FLOOR (towards -Infinity) + - decimal.ROUND_HALF_DOWN (to nearest with ties going towards zero) + - decimal.ROUND_HALF_EVEN (to nearest with ties going to nearest even integer) + - decimal.ROUND_HALF_UP (to nearest with ties going away from zero) + - decimal.ROUND_UP (away from zero) + - decimal.ROUND_05UP (away from zero if last digit after rounding towards zero would have been 0 or 5; otherwise towards zero) Defaults to: ``decimal.ROUND_HALF_UP`` From 634b874c469e9a91f199d74c4f71464ed1d20da1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 10 Jul 2013 16:16:50 +0000 Subject: [PATCH 1216/1279] Added QuerySetNoCache and QuerySet.no_cache() for lower memory consumption (#365) --- docs/apireference.rst | 5 + docs/changelog.rst | 1 + docs/guide/querying.rst | 4 +- mongoengine/queryset/base.py | 1479 ++++++++++++++++++++++++++++ mongoengine/queryset/queryset.py | 1545 ++---------------------------- tests/queryset/queryset.py | 30 +- 6 files changed, 1586 insertions(+), 1478 deletions(-) create mode 100644 mongoengine/queryset/base.py diff --git a/docs/apireference.rst b/docs/apireference.rst index d062727..774d3b8 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -49,6 +49,11 @@ Querying .. automethod:: mongoengine.queryset.QuerySet.__call__ +.. autoclass:: mongoengine.queryset.QuerySetNoCache + :members: + + .. automethod:: mongoengine.queryset.QuerySetNoCache.__call__ + .. autofunction:: mongoengine.queryset.queryset_manager Fields diff --git a/docs/changelog.rst b/docs/changelog.rst index 8bbc4b4..d875040 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.3 ================ +- Added QuerySetNoCache and QuerySet.no_cache() for lower memory consumption (#365) - Fixed sum and average mapreduce dot notation support (#375, #376) - Fixed as_pymongo to return the id (#386) - Document.select_related() now respects `db_alias` (#377) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 1350130..5fd0360 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -16,7 +16,9 @@ fetch documents from the database:: .. note:: As of MongoEngine 0.8 the querysets utilise a local cache. So iterating - it multiple times will only cause a single query. + it multiple times will only cause a single query. If this is not the + desired behavour you can call :class:`~mongoengine.QuerySet.no_cache` to + return a non-caching queryset. Filtering queries ================= diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py new file mode 100644 index 0000000..0b2898f --- /dev/null +++ b/mongoengine/queryset/base.py @@ -0,0 +1,1479 @@ +from __future__ import absolute_import + +import copy +import itertools +import operator +import pprint +import re +import warnings + +from bson.code import Code +from bson import json_util +import pymongo +from pymongo.common import validate_read_preference + +from mongoengine import signals +from mongoengine.common import _import_class +from mongoengine.errors import (OperationError, NotUniqueError, + InvalidQueryError) + +from mongoengine.queryset import transform +from mongoengine.queryset.field_list import QueryFieldList +from mongoengine.queryset.visitor import Q, QNode + + +__all__ = ('BaseQuerySet', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL') + +# Delete rules +DO_NOTHING = 0 +NULLIFY = 1 +CASCADE = 2 +DENY = 3 +PULL = 4 + +RE_TYPE = type(re.compile('')) + + +class BaseQuerySet(object): + """A set of results returned from a query. Wraps a MongoDB cursor, + providing :class:`~mongoengine.Document` objects as the results. + """ + __dereference = False + _auto_dereference = True + + def __init__(self, document, collection): + self._document = document + self._collection_obj = collection + self._mongo_query = None + self._query_obj = Q() + self._initial_query = {} + self._where_clause = None + self._loaded_fields = QueryFieldList() + self._ordering = [] + self._snapshot = False + self._timeout = True + self._class_check = True + self._slave_okay = False + self._read_preference = None + self._iter = False + self._scalar = [] + self._none = False + self._as_pymongo = False + self._as_pymongo_coerce = False + self._len = None + + # If inheritance is allowed, only return instances and instances of + # subclasses of the class being used + if document._meta.get('allow_inheritance') is True: + if len(self._document._subclasses) == 1: + self._initial_query = {"_cls": self._document._subclasses[0]} + else: + self._initial_query = {"_cls": {"$in": self._document._subclasses}} + self._loaded_fields = QueryFieldList(always_include=['_cls']) + self._cursor_obj = None + self._limit = None + self._skip = None + self._hint = -1 # Using -1 as None is a valid value for hint + + def __call__(self, q_obj=None, class_check=True, slave_okay=False, + read_preference=None, **query): + """Filter the selected documents by calling the + :class:`~mongoengine.queryset.QuerySet` with a query. + + :param q_obj: a :class:`~mongoengine.queryset.Q` object to be used in + the query; the :class:`~mongoengine.queryset.QuerySet` is filtered + multiple times with different :class:`~mongoengine.queryset.Q` + objects, only the last one will be used + :param class_check: If set to False bypass class name check when + querying collection + :param slave_okay: if True, allows this query to be run against a + replica secondary. + :params read_preference: if set, overrides connection-level + read_preference from `ReplicaSetConnection`. + :param query: Django-style query keyword arguments + """ + query = Q(**query) + if q_obj: + # make sure proper query object is passed + if not isinstance(q_obj, QNode): + msg = ("Not a query object: %s. " + "Did you intend to use key=value?" % q_obj) + raise InvalidQueryError(msg) + query &= q_obj + + if read_preference is None: + queryset = self.clone() + else: + # Use the clone provided when setting read_preference + queryset = self.read_preference(read_preference) + + queryset._query_obj &= query + queryset._mongo_query = None + queryset._cursor_obj = None + queryset._class_check = class_check + + return queryset + + def __getitem__(self, key): + """Support skip and limit using getitem and slicing syntax. + """ + queryset = self.clone() + + # Slice provided + if isinstance(key, slice): + try: + queryset._cursor_obj = queryset._cursor[key] + queryset._skip, queryset._limit = key.start, key.stop + if key.start and key.stop: + queryset._limit = key.stop - key.start + except IndexError, err: + # PyMongo raises an error if key.start == key.stop, catch it, + # bin it, kill it. + start = key.start or 0 + if start >= 0 and key.stop >= 0 and key.step is None: + if start == key.stop: + queryset.limit(0) + queryset._skip = key.start + queryset._limit = key.stop - start + return queryset + raise err + # Allow further QuerySet modifications to be performed + return queryset + # Integer index provided + elif isinstance(key, int): + if queryset._scalar: + return queryset._get_scalar( + queryset._document._from_son(queryset._cursor[key], + _auto_dereference=self._auto_dereference)) + if queryset._as_pymongo: + return queryset._get_as_pymongo(queryset._cursor.next()) + return queryset._document._from_son(queryset._cursor[key], + _auto_dereference=self._auto_dereference) + raise AttributeError + + def __iter__(self): + raise NotImplementedError + + # Core functions + + def all(self): + """Returns all documents.""" + return self.__call__() + + def filter(self, *q_objs, **query): + """An alias of :meth:`~mongoengine.queryset.QuerySet.__call__` + """ + return self.__call__(*q_objs, **query) + + def get(self, *q_objs, **query): + """Retrieve the the matching object raising + :class:`~mongoengine.queryset.MultipleObjectsReturned` or + `DocumentName.MultipleObjectsReturned` exception if multiple results + and :class:`~mongoengine.queryset.DoesNotExist` or + `DocumentName.DoesNotExist` if no results are found. + + .. versionadded:: 0.3 + """ + queryset = self.clone() + queryset = queryset.limit(2) + queryset = queryset.filter(*q_objs, **query) + + try: + result = queryset.next() + except StopIteration: + msg = ("%s matching query does not exist." + % queryset._document._class_name) + raise queryset._document.DoesNotExist(msg) + try: + queryset.next() + except StopIteration: + return result + + queryset.rewind() + message = u'%d items returned, instead of 1' % queryset.count() + raise queryset._document.MultipleObjectsReturned(message) + + def create(self, **kwargs): + """Create new object. Returns the saved object instance. + + .. versionadded:: 0.4 + """ + return self._document(**kwargs).save() + + def get_or_create(self, write_concern=None, auto_save=True, + *q_objs, **query): + """Retrieve unique object or create, if it doesn't exist. Returns a + tuple of ``(object, created)``, where ``object`` is the retrieved or + created object and ``created`` is a boolean specifying whether a new + object was created. Raises + :class:`~mongoengine.queryset.MultipleObjectsReturned` or + `DocumentName.MultipleObjectsReturned` if multiple results are found. + A new document will be created if the document doesn't exists; a + dictionary of default values for the new document may be provided as a + keyword argument called :attr:`defaults`. + + .. note:: This requires two separate operations and therefore a + race condition exists. Because there are no transactions in + mongoDB other approaches should be investigated, to ensure you + don't accidently duplicate data when using this method. This is + now scheduled to be removed before 1.0 + + :param write_concern: optional extra keyword arguments used if we + have to create a new document. + Passes any write_concern onto :meth:`~mongoengine.Document.save` + + :param auto_save: if the object is to be saved automatically if + not found. + + .. deprecated:: 0.8 + .. versionchanged:: 0.6 - added `auto_save` + .. versionadded:: 0.3 + """ + msg = ("get_or_create is scheduled to be deprecated. The approach is " + "flawed without transactions. Upserts should be preferred.") + warnings.warn(msg, DeprecationWarning) + + defaults = query.get('defaults', {}) + if 'defaults' in query: + del query['defaults'] + + try: + doc = self.get(*q_objs, **query) + return doc, False + except self._document.DoesNotExist: + query.update(defaults) + doc = self._document(**query) + + if auto_save: + doc.save(write_concern=write_concern) + return doc, True + + def first(self): + """Retrieve the first object matching the query. + """ + queryset = self.clone() + try: + result = queryset[0] + except IndexError: + result = None + return result + + def insert(self, doc_or_docs, load_bulk=True, write_concern=None): + """bulk insert documents + + :param docs_or_doc: a document or list of documents to be inserted + :param load_bulk (optional): If True returns the list of document + instances + :param write_concern: Extra keyword arguments are passed down to + :meth:`~pymongo.collection.Collection.insert` + which will be used as options for the resultant + ``getLastError`` command. For example, + ``insert(..., {w: 2, fsync: True})`` will wait until at least + two servers have recorded the write and will force an fsync on + each server being written to. + + By default returns document instances, set ``load_bulk`` to False to + return just ``ObjectIds`` + + .. versionadded:: 0.5 + """ + Document = _import_class('Document') + + if write_concern is None: + write_concern = {} + + docs = doc_or_docs + return_one = False + if isinstance(docs, Document) or issubclass(docs.__class__, Document): + return_one = True + docs = [docs] + + raw = [] + for doc in docs: + if not isinstance(doc, self._document): + msg = ("Some documents inserted aren't instances of %s" + % str(self._document)) + raise OperationError(msg) + if doc.pk and not doc._created: + msg = "Some documents have ObjectIds use doc.update() instead" + raise OperationError(msg) + raw.append(doc.to_mongo()) + + signals.pre_bulk_insert.send(self._document, documents=docs) + try: + ids = self._collection.insert(raw, **write_concern) + except pymongo.errors.OperationFailure, err: + message = 'Could not save document (%s)' + if re.match('^E1100[01] duplicate key', unicode(err)): + # E11000 - duplicate key error index + # E11001 - duplicate key on update + message = u'Tried to save duplicate unique keys (%s)' + raise NotUniqueError(message % unicode(err)) + raise OperationError(message % unicode(err)) + + if not load_bulk: + signals.post_bulk_insert.send( + self._document, documents=docs, loaded=False) + return return_one and ids[0] or ids + + documents = self.in_bulk(ids) + results = [] + for obj_id in ids: + results.append(documents.get(obj_id)) + signals.post_bulk_insert.send( + self._document, documents=results, loaded=True) + return return_one and results[0] or results + + def count(self, with_limit_and_skip=True): + """Count the selected elements in the query. + + :param with_limit_and_skip (optional): take any :meth:`limit` or + :meth:`skip` that has been applied to this cursor into account when + getting the count + """ + if self._limit == 0: + return 0 + if with_limit_and_skip and self._len is not None: + return self._len + count = self._cursor.count(with_limit_and_skip=with_limit_and_skip) + if with_limit_and_skip: + self._len = count + return count + + def delete(self, write_concern=None, _from_doc_delete=False): + """Delete the documents matched by the query. + + :param write_concern: Extra keyword arguments are passed down which + will be used as options for the resultant + ``getLastError`` command. For example, + ``save(..., write_concern={w: 2, fsync: True}, ...)`` will + wait until at least two servers have recorded the write and + will force an fsync on the primary server. + :param _from_doc_delete: True when called from document delete therefore + signals will have been triggered so don't loop. + """ + queryset = self.clone() + doc = queryset._document + + if write_concern is None: + write_concern = {} + + # Handle deletes where skips or limits have been applied or + # there is an untriggered delete signal + has_delete_signal = signals.signals_available and ( + signals.pre_delete.has_receivers_for(self._document) or + signals.post_delete.has_receivers_for(self._document)) + + call_document_delete = (queryset._skip or queryset._limit or + has_delete_signal) and not _from_doc_delete + + if call_document_delete: + for doc in queryset: + doc.delete(write_concern=write_concern) + return + + delete_rules = doc._meta.get('delete_rules') or {} + # Check for DENY rules before actually deleting/nullifying any other + # references + for rule_entry in delete_rules: + document_cls, field_name = rule_entry + rule = doc._meta['delete_rules'][rule_entry] + if rule == DENY and document_cls.objects( + **{field_name + '__in': self}).count() > 0: + msg = ("Could not delete document (%s.%s refers to it)" + % (document_cls.__name__, field_name)) + raise OperationError(msg) + + for rule_entry in delete_rules: + document_cls, field_name = rule_entry + rule = doc._meta['delete_rules'][rule_entry] + if rule == CASCADE: + ref_q = document_cls.objects(**{field_name + '__in': self}) + ref_q_count = ref_q.count() + if (doc != document_cls and ref_q_count > 0 + or (doc == document_cls and ref_q_count > 0)): + ref_q.delete(write_concern=write_concern) + elif rule == NULLIFY: + document_cls.objects(**{field_name + '__in': self}).update( + write_concern=write_concern, **{'unset__%s' % field_name: 1}) + elif rule == PULL: + document_cls.objects(**{field_name + '__in': self}).update( + write_concern=write_concern, + **{'pull_all__%s' % field_name: self}) + + queryset._collection.remove(queryset._query, write_concern=write_concern) + + def update(self, upsert=False, multi=True, write_concern=None, + full_result=False, **update): + """Perform an atomic update on the fields matched by the query. + + :param upsert: Any existing document with that "_id" is overwritten. + :param multi: Update multiple documents. + :param write_concern: Extra keyword arguments are passed down which + will be used as options for the resultant + ``getLastError`` command. For example, + ``save(..., write_concern={w: 2, fsync: True}, ...)`` will + wait until at least two servers have recorded the write and + will force an fsync on the primary server. + :param full_result: Return the full result rather than just the number + updated. + :param update: Django-style update keyword arguments + + .. versionadded:: 0.2 + """ + if not update and not upsert: + raise OperationError("No update parameters, would remove data") + + if write_concern is None: + write_concern = {} + + queryset = self.clone() + query = queryset._query + update = transform.update(queryset._document, **update) + + # If doing an atomic upsert on an inheritable class + # then ensure we add _cls to the update operation + if upsert and '_cls' in query: + if '$set' in update: + update["$set"]["_cls"] = queryset._document._class_name + else: + update["$set"] = {"_cls": queryset._document._class_name} + try: + result = queryset._collection.update(query, update, multi=multi, + upsert=upsert, **write_concern) + if full_result: + return result + elif result: + return result['n'] + except pymongo.errors.OperationFailure, err: + if unicode(err) == u'multi not coded yet': + message = u'update() method requires MongoDB 1.1.3+' + raise OperationError(message) + raise OperationError(u'Update failed (%s)' % unicode(err)) + + def update_one(self, upsert=False, write_concern=None, **update): + """Perform an atomic update on first field matched by the query. + + :param upsert: Any existing document with that "_id" is overwritten. + :param write_concern: Extra keyword arguments are passed down which + will be used as options for the resultant + ``getLastError`` command. For example, + ``save(..., write_concern={w: 2, fsync: True}, ...)`` will + wait until at least two servers have recorded the write and + will force an fsync on the primary server. + :param update: Django-style update keyword arguments + + .. versionadded:: 0.2 + """ + return self.update( + upsert=upsert, multi=False, write_concern=write_concern, **update) + + def with_id(self, object_id): + """Retrieve the object matching the id provided. Uses `object_id` only + and raises InvalidQueryError if a filter has been applied. Returns + `None` if no document exists with that id. + + :param object_id: the value for the id of the document to look up + + .. versionchanged:: 0.6 Raises InvalidQueryError if filter has been set + """ + queryset = self.clone() + if not queryset._query_obj.empty: + msg = "Cannot use a filter whilst using `with_id`" + raise InvalidQueryError(msg) + return queryset.filter(pk=object_id).first() + + def in_bulk(self, object_ids): + """Retrieve a set of documents by their ids. + + :param object_ids: a list or tuple of ``ObjectId``\ s + :rtype: dict of ObjectIds as keys and collection-specific + Document subclasses as values. + + .. versionadded:: 0.3 + """ + doc_map = {} + + docs = self._collection.find({'_id': {'$in': object_ids}}, + **self._cursor_args) + if self._scalar: + for doc in docs: + doc_map[doc['_id']] = self._get_scalar( + self._document._from_son(doc)) + elif self._as_pymongo: + for doc in docs: + doc_map[doc['_id']] = self._get_as_pymongo(doc) + else: + for doc in docs: + doc_map[doc['_id']] = self._document._from_son(doc) + + return doc_map + + def none(self): + """Helper that just returns a list""" + queryset = self.clone() + queryset._none = True + return queryset + + def no_sub_classes(self): + """ + Only return instances of this document and not any inherited documents + """ + if self._document._meta.get('allow_inheritance') is True: + self._initial_query = {"_cls": self._document._class_name} + + return self + + def clone(self): + """Creates a copy of the current + :class:`~mongoengine.queryset.QuerySet` + + .. versionadded:: 0.5 + """ + return self.clone_into(self.__class__(self._document, self._collection_obj)) + + def clone_into(self, cls): + """Creates a copy of the current + :class:`~mongoengine.queryset.base.BaseQuerySet` into another child class + """ + if not isinstance(cls, BaseQuerySet): + raise OperationError('%s is not a subclass of BaseQuerySet' % cls.__name__) + + copy_props = ('_mongo_query', '_initial_query', '_none', '_query_obj', + '_where_clause', '_loaded_fields', '_ordering', '_snapshot', + '_timeout', '_class_check', '_slave_okay', '_read_preference', + '_iter', '_scalar', '_as_pymongo', '_as_pymongo_coerce', + '_limit', '_skip', '_hint', '_auto_dereference') + + for prop in copy_props: + val = getattr(self, prop) + setattr(cls, prop, copy.copy(val)) + + if self._cursor_obj: + cls._cursor_obj = self._cursor_obj.clone() + + return cls + + def select_related(self, max_depth=1): + """Handles dereferencing of :class:`~bson.dbref.DBRef` objects or + :class:`~bson.object_id.ObjectId` a maximum depth in order to cut down + the number queries to mongodb. + + .. versionadded:: 0.5 + """ + # Make select related work the same for querysets + max_depth += 1 + queryset = self.clone() + return queryset._dereference(queryset, max_depth=max_depth) + + def limit(self, n): + """Limit the number of returned documents to `n`. This may also be + achieved using array-slicing syntax (e.g. ``User.objects[:5]``). + + :param n: the maximum number of objects to return + """ + queryset = self.clone() + if n == 0: + queryset._cursor.limit(1) + else: + queryset._cursor.limit(n) + queryset._limit = n + # Return self to allow chaining + return queryset + + def skip(self, n): + """Skip `n` documents before returning the results. This may also be + achieved using array-slicing syntax (e.g. ``User.objects[5:]``). + + :param n: the number of objects to skip before returning results + """ + queryset = self.clone() + queryset._cursor.skip(n) + queryset._skip = n + return queryset + + def hint(self, index=None): + """Added 'hint' support, telling Mongo the proper index to use for the + query. + + Judicious use of hints can greatly improve query performance. When + doing a query on multiple fields (at least one of which is indexed) + pass the indexed field as a hint to the query. + + Hinting will not do anything if the corresponding index does not exist. + The last hint applied to this cursor takes precedence over all others. + + .. versionadded:: 0.5 + """ + queryset = self.clone() + queryset._cursor.hint(index) + queryset._hint = index + return queryset + + def distinct(self, field): + """Return a list of distinct values for a given field. + + :param field: the field to select distinct values from + + .. note:: This is a command and won't take ordering or limit into + account. + + .. versionadded:: 0.4 + .. versionchanged:: 0.5 - Fixed handling references + .. versionchanged:: 0.6 - Improved db_field refrence handling + """ + queryset = self.clone() + try: + field = self._fields_to_dbfields([field]).pop() + finally: + return self._dereference(queryset._cursor.distinct(field), 1, + name=field, instance=self._document) + + def only(self, *fields): + """Load only a subset of this document's fields. :: + + post = BlogPost.objects(...).only("title", "author.name") + + .. note :: `only()` is chainable and will perform a union :: + So with the following it will fetch both: `title` and `author.name`:: + + post = BlogPost.objects.only("title").only("author.name") + + :func:`~mongoengine.queryset.QuerySet.all_fields` will reset any + field filters. + + :param fields: fields to include + + .. versionadded:: 0.3 + .. versionchanged:: 0.5 - Added subfield support + """ + fields = dict([(f, QueryFieldList.ONLY) for f in fields]) + return self.fields(True, **fields) + + def exclude(self, *fields): + """Opposite to .only(), exclude some document's fields. :: + + post = BlogPost.objects(...).exclude("comments") + + .. note :: `exclude()` is chainable and will perform a union :: + So with the following it will exclude both: `title` and `author.name`:: + + post = BlogPost.objects.exclude("title").exclude("author.name") + + :func:`~mongoengine.queryset.QuerySet.all_fields` will reset any + field filters. + + :param fields: fields to exclude + + .. versionadded:: 0.5 + """ + fields = dict([(f, QueryFieldList.EXCLUDE) for f in fields]) + return self.fields(**fields) + + def fields(self, _only_called=False, **kwargs): + """Manipulate how you load this document's fields. Used by `.only()` + and `.exclude()` to manipulate which fields to retrieve. Fields also + allows for a greater level of control for example: + + Retrieving a Subrange of Array Elements: + + You can use the $slice operator to retrieve a subrange of elements in + an array. For example to get the first 5 comments:: + + post = BlogPost.objects(...).fields(slice__comments=5) + + :param kwargs: A dictionary identifying what to include + + .. versionadded:: 0.5 + """ + + # Check for an operator and transform to mongo-style if there is + operators = ["slice"] + cleaned_fields = [] + for key, value in kwargs.items(): + parts = key.split('__') + op = None + if parts[0] in operators: + op = parts.pop(0) + value = {'$' + op: value} + key = '.'.join(parts) + cleaned_fields.append((key, value)) + + fields = sorted(cleaned_fields, key=operator.itemgetter(1)) + queryset = self.clone() + for value, group in itertools.groupby(fields, lambda x: x[1]): + fields = [field for field, value in group] + fields = queryset._fields_to_dbfields(fields) + queryset._loaded_fields += QueryFieldList(fields, value=value, _only_called=_only_called) + + return queryset + + def all_fields(self): + """Include all fields. Reset all previously calls of .only() or + .exclude(). :: + + post = BlogPost.objects.exclude("comments").all_fields() + + .. versionadded:: 0.5 + """ + queryset = self.clone() + queryset._loaded_fields = QueryFieldList( + always_include=queryset._loaded_fields.always_include) + return queryset + + def order_by(self, *keys): + """Order the :class:`~mongoengine.queryset.QuerySet` by the keys. The + order may be specified by prepending each of the keys by a + or a -. + Ascending order is assumed. + + :param keys: fields to order the query results by; keys may be + prefixed with **+** or **-** to determine the ordering direction + """ + queryset = self.clone() + queryset._ordering = queryset._get_order_by(keys) + return queryset + + def explain(self, format=False): + """Return an explain plan record for the + :class:`~mongoengine.queryset.QuerySet`\ 's cursor. + + :param format: format the plan before returning it + """ + plan = self._cursor.explain() + if format: + plan = pprint.pformat(plan) + return plan + + def snapshot(self, enabled): + """Enable or disable snapshot mode when querying. + + :param enabled: whether or not snapshot mode is enabled + + ..versionchanged:: 0.5 - made chainable + """ + queryset = self.clone() + queryset._snapshot = enabled + return queryset + + def timeout(self, enabled): + """Enable or disable the default mongod timeout when querying. + + :param enabled: whether or not the timeout is used + + ..versionchanged:: 0.5 - made chainable + """ + queryset = self.clone() + queryset._timeout = enabled + return queryset + + def slave_okay(self, enabled): + """Enable or disable the slave_okay when querying. + + :param enabled: whether or not the slave_okay is enabled + """ + queryset = self.clone() + queryset._slave_okay = enabled + return queryset + + def read_preference(self, read_preference): + """Change the read_preference when querying. + + :param read_preference: override ReplicaSetConnection-level + preference. + """ + validate_read_preference('read_preference', read_preference) + queryset = self.clone() + queryset._read_preference = read_preference + return queryset + + def scalar(self, *fields): + """Instead of returning Document instances, return either a specific + value or a tuple of values in order. + + Can be used along with + :func:`~mongoengine.queryset.QuerySet.no_dereference` to turn off + dereferencing. + + .. note:: This effects all results and can be unset by calling + ``scalar`` without arguments. Calls ``only`` automatically. + + :param fields: One or more fields to return instead of a Document. + """ + queryset = self.clone() + queryset._scalar = list(fields) + + if fields: + queryset = queryset.only(*fields) + else: + queryset = queryset.all_fields() + + return queryset + + def values_list(self, *fields): + """An alias for scalar""" + return self.scalar(*fields) + + def as_pymongo(self, coerce_types=False): + """Instead of returning Document instances, return raw values from + pymongo. + + :param coerce_type: Field types (if applicable) would be use to + coerce types. + """ + queryset = self.clone() + queryset._as_pymongo = True + queryset._as_pymongo_coerce = coerce_types + return queryset + + # JSON Helpers + + def to_json(self): + """Converts a queryset to JSON""" + return json_util.dumps(self.as_pymongo()) + + def from_json(self, json_data): + """Converts json data to unsaved objects""" + son_data = json_util.loads(json_data) + return [self._document._from_son(data) for data in son_data] + + # JS functionality + + def map_reduce(self, map_f, reduce_f, output, finalize_f=None, limit=None, + scope=None): + """Perform a map/reduce query using the current query spec + and ordering. While ``map_reduce`` respects ``QuerySet`` chaining, + it must be the last call made, as it does not return a maleable + ``QuerySet``. + + See the :meth:`~mongoengine.tests.QuerySetTest.test_map_reduce` + and :meth:`~mongoengine.tests.QuerySetTest.test_map_advanced` + tests in ``tests.queryset.QuerySetTest`` for usage examples. + + :param map_f: map function, as :class:`~bson.code.Code` or string + :param reduce_f: reduce function, as + :class:`~bson.code.Code` or string + :param output: output collection name, if set to 'inline' will try to + use :class:`~pymongo.collection.Collection.inline_map_reduce` + This can also be a dictionary containing output options + see: http://docs.mongodb.org/manual/reference/commands/#mapReduce + :param finalize_f: finalize function, an optional function that + performs any post-reduction processing. + :param scope: values to insert into map/reduce global scope. Optional. + :param limit: number of objects from current query to provide + to map/reduce method + + Returns an iterator yielding + :class:`~mongoengine.document.MapReduceDocument`. + + .. note:: + + Map/Reduce changed in server version **>= 1.7.4**. The PyMongo + :meth:`~pymongo.collection.Collection.map_reduce` helper requires + PyMongo version **>= 1.11**. + + .. versionchanged:: 0.5 + - removed ``keep_temp`` keyword argument, which was only relevant + for MongoDB server versions older than 1.7.4 + + .. versionadded:: 0.3 + """ + queryset = self.clone() + + MapReduceDocument = _import_class('MapReduceDocument') + + if not hasattr(self._collection, "map_reduce"): + raise NotImplementedError("Requires MongoDB >= 1.7.1") + + map_f_scope = {} + if isinstance(map_f, Code): + map_f_scope = map_f.scope + map_f = unicode(map_f) + map_f = Code(queryset._sub_js_fields(map_f), map_f_scope) + + reduce_f_scope = {} + if isinstance(reduce_f, Code): + reduce_f_scope = reduce_f.scope + reduce_f = unicode(reduce_f) + reduce_f_code = queryset._sub_js_fields(reduce_f) + reduce_f = Code(reduce_f_code, reduce_f_scope) + + mr_args = {'query': queryset._query} + + if finalize_f: + finalize_f_scope = {} + if isinstance(finalize_f, Code): + finalize_f_scope = finalize_f.scope + finalize_f = unicode(finalize_f) + finalize_f_code = queryset._sub_js_fields(finalize_f) + finalize_f = Code(finalize_f_code, finalize_f_scope) + mr_args['finalize'] = finalize_f + + if scope: + mr_args['scope'] = scope + + if limit: + mr_args['limit'] = limit + + if output == 'inline' and not queryset._ordering: + map_reduce_function = 'inline_map_reduce' + else: + map_reduce_function = 'map_reduce' + mr_args['out'] = output + + results = getattr(queryset._collection, map_reduce_function)( + map_f, reduce_f, **mr_args) + + if map_reduce_function == 'map_reduce': + results = results.find() + + if queryset._ordering: + results = results.sort(queryset._ordering) + + for doc in results: + yield MapReduceDocument(queryset._document, queryset._collection, + doc['_id'], doc['value']) + + def exec_js(self, code, *fields, **options): + """Execute a Javascript function on the server. A list of fields may be + provided, which will be translated to their correct names and supplied + as the arguments to the function. A few extra variables are added to + the function's scope: ``collection``, which is the name of the + collection in use; ``query``, which is an object representing the + current query; and ``options``, which is an object containing any + options specified as keyword arguments. + + As fields in MongoEngine may use different names in the database (set + using the :attr:`db_field` keyword argument to a :class:`Field` + constructor), a mechanism exists for replacing MongoEngine field names + with the database field names in Javascript code. When accessing a + field, use square-bracket notation, and prefix the MongoEngine field + name with a tilde (~). + + :param code: a string of Javascript code to execute + :param fields: fields that you will be using in your function, which + will be passed in to your function as arguments + :param options: options that you want available to the function + (accessed in Javascript through the ``options`` object) + """ + queryset = self.clone() + + code = queryset._sub_js_fields(code) + + fields = [queryset._document._translate_field_name(f) for f in fields] + collection = queryset._document._get_collection_name() + + scope = { + 'collection': collection, + 'options': options or {}, + } + + query = queryset._query + if queryset._where_clause: + query['$where'] = queryset._where_clause + + scope['query'] = query + code = Code(code, scope=scope) + + db = queryset._document._get_db() + return db.eval(code, *fields) + + def where(self, where_clause): + """Filter ``QuerySet`` results with a ``$where`` clause (a Javascript + expression). Performs automatic field name substitution like + :meth:`mongoengine.queryset.Queryset.exec_js`. + + .. note:: When using this mode of query, the database will call your + function, or evaluate your predicate clause, for each object + in the collection. + + .. versionadded:: 0.5 + """ + queryset = self.clone() + where_clause = queryset._sub_js_fields(where_clause) + queryset._where_clause = where_clause + return queryset + + def sum(self, field): + """Sum over the values of the specified field. + + :param field: the field to sum over; use dot-notation to refer to + embedded document fields + + .. versionchanged:: 0.5 - updated to map_reduce as db.eval doesnt work + with sharding. + """ + map_func = Code(""" + function() { + function deepFind(obj, path) { + var paths = path.split('.') + , current = obj + , i; + + for (i = 0; i < paths.length; ++i) { + if (current[paths[i]] == undefined) { + return undefined; + } else { + current = current[paths[i]]; + } + } + return current; + } + + emit(1, deepFind(this, field) || 0); + } + """, scope={'field': field}) + + reduce_func = Code(""" + function(key, values) { + var sum = 0; + for (var i in values) { + sum += values[i]; + } + return sum; + } + """) + + for result in self.map_reduce(map_func, reduce_func, output='inline'): + return result.value + else: + return 0 + + def average(self, field): + """Average over the values of the specified field. + + :param field: the field to average over; use dot-notation to refer to + embedded document fields + + .. versionchanged:: 0.5 - updated to map_reduce as db.eval doesnt work + with sharding. + """ + map_func = Code(""" + function() { + function deepFind(obj, path) { + var paths = path.split('.') + , current = obj + , i; + + for (i = 0; i < paths.length; ++i) { + if (current[paths[i]] == undefined) { + return undefined; + } else { + current = current[paths[i]]; + } + } + return current; + } + + val = deepFind(this, field) + if (val !== undefined) + emit(1, {t: val || 0, c: 1}); + } + """, scope={'field': field}) + + reduce_func = Code(""" + function(key, values) { + var out = {t: 0, c: 0}; + for (var i in values) { + var value = values[i]; + out.t += value.t; + out.c += value.c; + } + return out; + } + """) + + finalize_func = Code(""" + function(key, value) { + return value.t / value.c; + } + """) + + for result in self.map_reduce(map_func, reduce_func, + finalize_f=finalize_func, output='inline'): + return result.value + else: + return 0 + + def item_frequencies(self, field, normalize=False, map_reduce=True): + """Returns a dictionary of all items present in a field across + the whole queried set of documents, and their corresponding frequency. + This is useful for generating tag clouds, or searching documents. + + .. note:: + + Can only do direct simple mappings and cannot map across + :class:`~mongoengine.fields.ReferenceField` or + :class:`~mongoengine.fields.GenericReferenceField` for more complex + counting a manual map reduce call would is required. + + If the field is a :class:`~mongoengine.fields.ListField`, the items within + each list will be counted individually. + + :param field: the field to use + :param normalize: normalize the results so they add to 1.0 + :param map_reduce: Use map_reduce over exec_js + + .. versionchanged:: 0.5 defaults to map_reduce and can handle embedded + document lookups + """ + if map_reduce: + return self._item_frequencies_map_reduce(field, + normalize=normalize) + return self._item_frequencies_exec_js(field, normalize=normalize) + + # Iterator helpers + + def next(self): + """Wrap the result in a :class:`~mongoengine.Document` object. + """ + if self._limit == 0 or self._none: + raise StopIteration + + raw_doc = self._cursor.next() + if self._as_pymongo: + return self._get_as_pymongo(raw_doc) + doc = self._document._from_son(raw_doc, + _auto_dereference=self._auto_dereference) + if self._scalar: + return self._get_scalar(doc) + + return doc + + def rewind(self): + """Rewind the cursor to its unevaluated state. + + .. versionadded:: 0.3 + """ + self._iter = False + self._cursor.rewind() + + # Properties + + @property + def _collection(self): + """Property that returns the collection object. This allows us to + perform operations only if the collection is accessed. + """ + return self._collection_obj + + @property + def _cursor_args(self): + cursor_args = { + 'snapshot': self._snapshot, + 'timeout': self._timeout + } + if self._read_preference is not None: + cursor_args['read_preference'] = self._read_preference + else: + cursor_args['slave_okay'] = self._slave_okay + if self._loaded_fields: + cursor_args['fields'] = self._loaded_fields.as_dict() + return cursor_args + + @property + def _cursor(self): + if self._cursor_obj is None: + + self._cursor_obj = self._collection.find(self._query, + **self._cursor_args) + # Apply where clauses to cursor + if self._where_clause: + where_clause = self._sub_js_fields(self._where_clause) + self._cursor_obj.where(where_clause) + + if self._ordering: + # Apply query ordering + self._cursor_obj.sort(self._ordering) + elif self._document._meta['ordering']: + # Otherwise, apply the ordering from the document model + order = self._get_order_by(self._document._meta['ordering']) + self._cursor_obj.sort(order) + + if self._limit is not None: + self._cursor_obj.limit(self._limit) + + if self._skip is not None: + self._cursor_obj.skip(self._skip) + + if self._hint != -1: + self._cursor_obj.hint(self._hint) + + return self._cursor_obj + + def __deepcopy__(self, memo): + """Essential for chained queries with ReferenceFields involved""" + return self.clone() + + @property + def _query(self): + if self._mongo_query is None: + self._mongo_query = self._query_obj.to_query(self._document) + if self._class_check: + self._mongo_query.update(self._initial_query) + return self._mongo_query + + @property + def _dereference(self): + if not self.__dereference: + self.__dereference = _import_class('DeReference')() + return self.__dereference + + def no_dereference(self): + """Turn off any dereferencing for the results of this queryset. + """ + queryset = self.clone() + queryset._auto_dereference = False + return queryset + + # Helper Functions + + def _item_frequencies_map_reduce(self, field, normalize=False): + map_func = """ + function() { + var path = '{{~%(field)s}}'.split('.'); + var field = this; + + for (p in path) { + if (typeof field != 'undefined') + field = field[path[p]]; + else + break; + } + if (field && field.constructor == Array) { + field.forEach(function(item) { + emit(item, 1); + }); + } else if (typeof field != 'undefined') { + emit(field, 1); + } else { + emit(null, 1); + } + } + """ % dict(field=field) + reduce_func = """ + function(key, values) { + var total = 0; + var valuesSize = values.length; + for (var i=0; i < valuesSize; i++) { + total += parseInt(values[i], 10); + } + return total; + } + """ + values = self.map_reduce(map_func, reduce_func, 'inline') + frequencies = {} + for f in values: + key = f.key + if isinstance(key, float): + if int(key) == key: + key = int(key) + frequencies[key] = int(f.value) + + if normalize: + count = sum(frequencies.values()) + frequencies = dict([(k, float(v) / count) + for k, v in frequencies.items()]) + + return frequencies + + def _item_frequencies_exec_js(self, field, normalize=False): + """Uses exec_js to execute""" + freq_func = """ + function(path) { + var path = path.split('.'); + + var total = 0.0; + db[collection].find(query).forEach(function(doc) { + var field = doc; + for (p in path) { + if (field) + field = field[path[p]]; + else + break; + } + if (field && field.constructor == Array) { + total += field.length; + } else { + total++; + } + }); + + var frequencies = {}; + var types = {}; + var inc = 1.0; + + db[collection].find(query).forEach(function(doc) { + field = doc; + for (p in path) { + if (field) + field = field[path[p]]; + else + break; + } + if (field && field.constructor == Array) { + field.forEach(function(item) { + frequencies[item] = inc + (isNaN(frequencies[item]) ? 0: frequencies[item]); + }); + } else { + var item = field; + types[item] = item; + frequencies[item] = inc + (isNaN(frequencies[item]) ? 0: frequencies[item]); + } + }); + return [total, frequencies, types]; + } + """ + total, data, types = self.exec_js(freq_func, field) + values = dict([(types.get(k), int(v)) for k, v in data.iteritems()]) + + if normalize: + values = dict([(k, float(v) / total) for k, v in values.items()]) + + frequencies = {} + for k, v in values.iteritems(): + if isinstance(k, float): + if int(k) == k: + k = int(k) + + frequencies[k] = v + + return frequencies + + def _fields_to_dbfields(self, fields): + """Translate fields paths to its db equivalents""" + ret = [] + for field in fields: + field = ".".join(f.db_field for f in + self._document._lookup_field(field.split('.'))) + ret.append(field) + return ret + + def _get_order_by(self, keys): + """Creates a list of order by fields + """ + key_list = [] + for key in keys: + if not key: + continue + direction = pymongo.ASCENDING + if key[0] == '-': + direction = pymongo.DESCENDING + if key[0] in ('-', '+'): + key = key[1:] + key = key.replace('__', '.') + try: + key = self._document._translate_field_name(key) + except: + pass + key_list.append((key, direction)) + + if self._cursor_obj: + self._cursor_obj.sort(key_list) + return key_list + + def _get_scalar(self, doc): + + def lookup(obj, name): + chunks = name.split('__') + for chunk in chunks: + obj = getattr(obj, chunk) + return obj + + data = [lookup(doc, n) for n in self._scalar] + if len(data) == 1: + return data[0] + + return tuple(data) + + def _get_as_pymongo(self, row): + # Extract which fields paths we should follow if .fields(...) was + # used. If not, handle all fields. + if not getattr(self, '__as_pymongo_fields', None): + self.__as_pymongo_fields = [] + + for field in self._loaded_fields.fields - set(['_cls']): + self.__as_pymongo_fields.append(field) + while '.' in field: + field, _ = field.rsplit('.', 1) + self.__as_pymongo_fields.append(field) + + all_fields = not self.__as_pymongo_fields + + def clean(data, path=None): + path = path or '' + + if isinstance(data, dict): + new_data = {} + for key, value in data.iteritems(): + new_path = '%s.%s' % (path, key) if path else key + + if all_fields: + include_field = True + elif self._loaded_fields.value == QueryFieldList.ONLY: + include_field = new_path in self.__as_pymongo_fields + else: + include_field = new_path not in self.__as_pymongo_fields + + if include_field: + new_data[key] = clean(value, path=new_path) + data = new_data + elif isinstance(data, list): + data = [clean(d, path=path) for d in data] + else: + if self._as_pymongo_coerce: + # If we need to coerce types, we need to determine the + # type of this field and use the corresponding + # .to_python(...) + from mongoengine.fields import EmbeddedDocumentField + obj = self._document + for chunk in path.split('.'): + obj = getattr(obj, chunk, None) + if obj is None: + break + elif isinstance(obj, EmbeddedDocumentField): + obj = obj.document_type + if obj and data is not None: + data = obj.to_python(data) + return data + return clean(row) + + def _sub_js_fields(self, code): + """When fields are specified with [~fieldname] syntax, where + *fieldname* is the Python name of a field, *fieldname* will be + substituted for the MongoDB name of the field (specified using the + :attr:`name` keyword argument in a field's constructor). + """ + def field_sub(match): + # Extract just the field name, and look up the field objects + field_name = match.group(1).split('.') + fields = self._document._lookup_field(field_name) + # Substitute the correct name for the field into the javascript + return u'["%s"]' % fields[-1].db_field + + def field_path_sub(match): + # Extract just the field name, and look up the field objects + field_name = match.group(1).split('.') + fields = self._document._lookup_field(field_name) + # Substitute the correct name for the field into the javascript + return ".".join([f.db_field for f in fields]) + + code = re.sub(u'\[\s*~([A-z_][A-z_0-9.]+?)\s*\]', field_sub, code) + code = re.sub(u'\{\{\s*~([A-z_][A-z_0-9.]+?)\s*\}\}', field_path_sub, + code) + return code + + # Deprecated + def ensure_index(self, **kwargs): + """Deprecated use :func:`Document.ensure_index`""" + msg = ("Doc.objects()._ensure_index() is deprecated. " + "Use Doc.ensure_index() instead.") + warnings.warn(msg, DeprecationWarning) + self._document.__class__.ensure_index(**kwargs) + return self + + def _ensure_indexes(self): + """Deprecated use :func:`~Document.ensure_indexes`""" + msg = ("Doc.objects()._ensure_indexes() is deprecated. " + "Use Doc.ensure_indexes() instead.") + warnings.warn(msg, DeprecationWarning) + self._document.__class__.ensure_indexes() \ No newline at end of file diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 690e3f0..9db98a7 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1,137 +1,26 @@ -from __future__ import absolute_import +from mongoengine.errors import OperationError +from mongoengine.queryset.base import (BaseQuerySet, DO_NOTHING, NULLIFY, + CASCADE, DENY, PULL) -import copy -import itertools -import operator -import pprint -import re -import warnings - -from bson.code import Code -from bson import json_util -import pymongo -from pymongo.common import validate_read_preference - -from mongoengine import signals -from mongoengine.common import _import_class -from mongoengine.errors import (OperationError, NotUniqueError, - InvalidQueryError) - -from mongoengine.queryset import transform -from mongoengine.queryset.field_list import QueryFieldList -from mongoengine.queryset.visitor import Q, QNode - - -__all__ = ('QuerySet', 'DO_NOTHING', 'NULLIFY', 'CASCADE', 'DENY', 'PULL') +__all__ = ('QuerySet', 'QuerySetNoCache', 'DO_NOTHING', 'NULLIFY', 'CASCADE', + 'DENY', 'PULL') # The maximum number of items to display in a QuerySet.__repr__ REPR_OUTPUT_SIZE = 20 ITER_CHUNK_SIZE = 100 -# Delete rules -DO_NOTHING = 0 -NULLIFY = 1 -CASCADE = 2 -DENY = 3 -PULL = 4 -RE_TYPE = type(re.compile('')) +class QuerySet(BaseQuerySet): + """The default queryset, that builds queries and handles a set of results + returned from a query. - -class QuerySet(object): - """A set of results returned from a query. Wraps a MongoDB cursor, - providing :class:`~mongoengine.Document` objects as the results. + Wraps a MongoDB cursor, providing :class:`~mongoengine.Document` objects as + the results. """ - __dereference = False - _auto_dereference = True - def __init__(self, document, collection): - self._document = document - self._collection_obj = collection - self._mongo_query = None - self._query_obj = Q() - self._initial_query = {} - self._where_clause = None - self._loaded_fields = QueryFieldList() - self._ordering = [] - self._snapshot = False - self._timeout = True - self._class_check = True - self._slave_okay = False - self._read_preference = None - self._iter = False - self._scalar = [] - self._none = False - self._as_pymongo = False - self._as_pymongo_coerce = False - self._result_cache = [] - self._has_more = True - self._len = None - - # If inheritance is allowed, only return instances and instances of - # subclasses of the class being used - if document._meta.get('allow_inheritance') is True: - if len(self._document._subclasses) == 1: - self._initial_query = {"_cls": self._document._subclasses[0]} - else: - self._initial_query = {"_cls": {"$in": self._document._subclasses}} - self._loaded_fields = QueryFieldList(always_include=['_cls']) - self._cursor_obj = None - self._limit = None - self._skip = None - self._hint = -1 # Using -1 as None is a valid value for hint - - def __call__(self, q_obj=None, class_check=True, slave_okay=False, - read_preference=None, **query): - """Filter the selected documents by calling the - :class:`~mongoengine.queryset.QuerySet` with a query. - - :param q_obj: a :class:`~mongoengine.queryset.Q` object to be used in - the query; the :class:`~mongoengine.queryset.QuerySet` is filtered - multiple times with different :class:`~mongoengine.queryset.Q` - objects, only the last one will be used - :param class_check: If set to False bypass class name check when - querying collection - :param slave_okay: if True, allows this query to be run against a - replica secondary. - :params read_preference: if set, overrides connection-level - read_preference from `ReplicaSetConnection`. - :param query: Django-style query keyword arguments - """ - query = Q(**query) - if q_obj: - # make sure proper query object is passed - if not isinstance(q_obj, QNode): - msg = ("Not a query object: %s. " - "Did you intend to use key=value?" % q_obj) - raise InvalidQueryError(msg) - query &= q_obj - - if read_preference is None: - queryset = self.clone() - else: - # Use the clone provided when setting read_preference - queryset = self.read_preference(read_preference) - - queryset._query_obj &= query - queryset._mongo_query = None - queryset._cursor_obj = None - queryset._class_check = class_check - - return queryset - - def __len__(self): - """Since __len__ is called quite frequently (for example, as part of - list(qs) we populate the result cache and cache the length. - """ - if self._len is not None: - return self._len - if self._has_more: - # populate the cache - list(self._iter_results()) - - self._len = len(self._result_cache) - return self._len + _has_more = True + _len = None + _result_cache = None def __iter__(self): """Iteration utilises a results cache which iterates the cursor @@ -147,11 +36,39 @@ class QuerySet(object): # iterating over the cache. return iter(self._result_cache) + def __len__(self): + """Since __len__ is called quite frequently (for example, as part of + list(qs) we populate the result cache and cache the length. + """ + if self._len is not None: + return self._len + if self._has_more: + # populate the cache + list(self._iter_results()) + + self._len = len(self._result_cache) + return self._len + + def __repr__(self): + """Provides the string representation of the QuerySet + """ + if self._iter: + return '.. queryset mid-iteration ..' + + self._populate_cache() + data = self._result_cache[:REPR_OUTPUT_SIZE + 1] + if len(data) > REPR_OUTPUT_SIZE: + data[-1] = "...(remaining elements truncated)..." + return repr(data) + + def _iter_results(self): """A generator for iterating over the result cache. Also populates the cache if there are more possible results to yield. Raises StopIteration when there are no more results""" + if self._result_cache is None: + self._result_cache = [] pos = 0 while True: upper = len(self._result_cache) @@ -168,6 +85,8 @@ class QuerySet(object): Populates the result cache with ``ITER_CHUNK_SIZE`` more entries (until the cursor is exhausted). """ + if self._result_cache is None: + self._result_cache = [] if self._has_more: try: for i in xrange(ITER_CHUNK_SIZE): @@ -175,1369 +94,43 @@ class QuerySet(object): except StopIteration: self._has_more = False - def __getitem__(self, key): - """Support skip and limit using getitem and slicing syntax. - """ - queryset = self.clone() + def no_cache(self): + """Convert to a non_caching queryset""" + if self._result_cache is not None: + raise OperationError("QuerySet already cached") + return self.clone_into(QuerySetNoCache(self._document, self._collection)) - # Slice provided - if isinstance(key, slice): - try: - queryset._cursor_obj = queryset._cursor[key] - queryset._skip, queryset._limit = key.start, key.stop - if key.start and key.stop: - queryset._limit = key.stop - key.start - except IndexError, err: - # PyMongo raises an error if key.start == key.stop, catch it, - # bin it, kill it. - start = key.start or 0 - if start >= 0 and key.stop >= 0 and key.step is None: - if start == key.stop: - queryset.limit(0) - queryset._skip = key.start - queryset._limit = key.stop - start - return queryset - raise err - # Allow further QuerySet modifications to be performed - return queryset - # Integer index provided - elif isinstance(key, int): - if queryset._scalar: - return queryset._get_scalar( - queryset._document._from_son(queryset._cursor[key], - _auto_dereference=self._auto_dereference)) - if queryset._as_pymongo: - return queryset._get_as_pymongo(queryset._cursor.next()) - return queryset._document._from_son(queryset._cursor[key], - _auto_dereference=self._auto_dereference) - raise AttributeError + +class QuerySetNoCache(BaseQuerySet): + """A non caching QuerySet""" + + def cache(self): + """Convert to a caching queryset""" + return self.clone_into(QuerySet(self._document, self._collection)) def __repr__(self): """Provides the string representation of the QuerySet - """ + .. versionchanged:: 0.6.13 Now doesnt modify the cursor + """ if self._iter: return '.. queryset mid-iteration ..' - self._populate_cache() - data = self._result_cache[:REPR_OUTPUT_SIZE + 1] + data = [] + for i in xrange(REPR_OUTPUT_SIZE + 1): + try: + data.append(self.next()) + except StopIteration: + break if len(data) > REPR_OUTPUT_SIZE: data[-1] = "...(remaining elements truncated)..." + + self.rewind() return repr(data) - # Core functions - - def all(self): - """Returns all documents.""" - return self.__call__() - - def filter(self, *q_objs, **query): - """An alias of :meth:`~mongoengine.queryset.QuerySet.__call__` - """ - return self.__call__(*q_objs, **query) - - def get(self, *q_objs, **query): - """Retrieve the the matching object raising - :class:`~mongoengine.queryset.MultipleObjectsReturned` or - `DocumentName.MultipleObjectsReturned` exception if multiple results - and :class:`~mongoengine.queryset.DoesNotExist` or - `DocumentName.DoesNotExist` if no results are found. - - .. versionadded:: 0.3 - """ - queryset = self.clone() - queryset = queryset.limit(2) - queryset = queryset.filter(*q_objs, **query) - - try: - result = queryset.next() - except StopIteration: - msg = ("%s matching query does not exist." - % queryset._document._class_name) - raise queryset._document.DoesNotExist(msg) - try: - queryset.next() - except StopIteration: - return result - + def __iter__(self): + queryset = self + if queryset._iter: + queryset = self.clone() queryset.rewind() - message = u'%d items returned, instead of 1' % queryset.count() - raise queryset._document.MultipleObjectsReturned(message) - - def create(self, **kwargs): - """Create new object. Returns the saved object instance. - - .. versionadded:: 0.4 - """ - return self._document(**kwargs).save() - - def get_or_create(self, write_concern=None, auto_save=True, - *q_objs, **query): - """Retrieve unique object or create, if it doesn't exist. Returns a - tuple of ``(object, created)``, where ``object`` is the retrieved or - created object and ``created`` is a boolean specifying whether a new - object was created. Raises - :class:`~mongoengine.queryset.MultipleObjectsReturned` or - `DocumentName.MultipleObjectsReturned` if multiple results are found. - A new document will be created if the document doesn't exists; a - dictionary of default values for the new document may be provided as a - keyword argument called :attr:`defaults`. - - .. note:: This requires two separate operations and therefore a - race condition exists. Because there are no transactions in - mongoDB other approaches should be investigated, to ensure you - don't accidently duplicate data when using this method. This is - now scheduled to be removed before 1.0 - - :param write_concern: optional extra keyword arguments used if we - have to create a new document. - Passes any write_concern onto :meth:`~mongoengine.Document.save` - - :param auto_save: if the object is to be saved automatically if - not found. - - .. deprecated:: 0.8 - .. versionchanged:: 0.6 - added `auto_save` - .. versionadded:: 0.3 - """ - msg = ("get_or_create is scheduled to be deprecated. The approach is " - "flawed without transactions. Upserts should be preferred.") - warnings.warn(msg, DeprecationWarning) - - defaults = query.get('defaults', {}) - if 'defaults' in query: - del query['defaults'] - - try: - doc = self.get(*q_objs, **query) - return doc, False - except self._document.DoesNotExist: - query.update(defaults) - doc = self._document(**query) - - if auto_save: - doc.save(write_concern=write_concern) - return doc, True - - def first(self): - """Retrieve the first object matching the query. - """ - queryset = self.clone() - try: - result = queryset[0] - except IndexError: - result = None - return result - - def insert(self, doc_or_docs, load_bulk=True, write_concern=None): - """bulk insert documents - - :param docs_or_doc: a document or list of documents to be inserted - :param load_bulk (optional): If True returns the list of document - instances - :param write_concern: Extra keyword arguments are passed down to - :meth:`~pymongo.collection.Collection.insert` - which will be used as options for the resultant - ``getLastError`` command. For example, - ``insert(..., {w: 2, fsync: True})`` will wait until at least - two servers have recorded the write and will force an fsync on - each server being written to. - - By default returns document instances, set ``load_bulk`` to False to - return just ``ObjectIds`` - - .. versionadded:: 0.5 - """ - Document = _import_class('Document') - - if write_concern is None: - write_concern = {} - - docs = doc_or_docs - return_one = False - if isinstance(docs, Document) or issubclass(docs.__class__, Document): - return_one = True - docs = [docs] - - raw = [] - for doc in docs: - if not isinstance(doc, self._document): - msg = ("Some documents inserted aren't instances of %s" - % str(self._document)) - raise OperationError(msg) - if doc.pk and not doc._created: - msg = "Some documents have ObjectIds use doc.update() instead" - raise OperationError(msg) - raw.append(doc.to_mongo()) - - signals.pre_bulk_insert.send(self._document, documents=docs) - try: - ids = self._collection.insert(raw, **write_concern) - except pymongo.errors.OperationFailure, err: - message = 'Could not save document (%s)' - if re.match('^E1100[01] duplicate key', unicode(err)): - # E11000 - duplicate key error index - # E11001 - duplicate key on update - message = u'Tried to save duplicate unique keys (%s)' - raise NotUniqueError(message % unicode(err)) - raise OperationError(message % unicode(err)) - - if not load_bulk: - signals.post_bulk_insert.send( - self._document, documents=docs, loaded=False) - return return_one and ids[0] or ids - - documents = self.in_bulk(ids) - results = [] - for obj_id in ids: - results.append(documents.get(obj_id)) - signals.post_bulk_insert.send( - self._document, documents=results, loaded=True) - return return_one and results[0] or results - - def count(self, with_limit_and_skip=True): - """Count the selected elements in the query. - - :param with_limit_and_skip (optional): take any :meth:`limit` or - :meth:`skip` that has been applied to this cursor into account when - getting the count - """ - if self._limit == 0: - return 0 - if with_limit_and_skip and self._len is not None: - return self._len - count = self._cursor.count(with_limit_and_skip=with_limit_and_skip) - if with_limit_and_skip: - self._len = count - return count - - def delete(self, write_concern=None, _from_doc_delete=False): - """Delete the documents matched by the query. - - :param write_concern: Extra keyword arguments are passed down which - will be used as options for the resultant - ``getLastError`` command. For example, - ``save(..., write_concern={w: 2, fsync: True}, ...)`` will - wait until at least two servers have recorded the write and - will force an fsync on the primary server. - :param _from_doc_delete: True when called from document delete therefore - signals will have been triggered so don't loop. - """ - queryset = self.clone() - doc = queryset._document - - if write_concern is None: - write_concern = {} - - # Handle deletes where skips or limits have been applied or - # there is an untriggered delete signal - has_delete_signal = signals.signals_available and ( - signals.pre_delete.has_receivers_for(self._document) or - signals.post_delete.has_receivers_for(self._document)) - - call_document_delete = (queryset._skip or queryset._limit or - has_delete_signal) and not _from_doc_delete - - if call_document_delete: - for doc in queryset: - doc.delete(write_concern=write_concern) - return - - delete_rules = doc._meta.get('delete_rules') or {} - # Check for DENY rules before actually deleting/nullifying any other - # references - for rule_entry in delete_rules: - document_cls, field_name = rule_entry - rule = doc._meta['delete_rules'][rule_entry] - if rule == DENY and document_cls.objects( - **{field_name + '__in': self}).count() > 0: - msg = ("Could not delete document (%s.%s refers to it)" - % (document_cls.__name__, field_name)) - raise OperationError(msg) - - for rule_entry in delete_rules: - document_cls, field_name = rule_entry - rule = doc._meta['delete_rules'][rule_entry] - if rule == CASCADE: - ref_q = document_cls.objects(**{field_name + '__in': self}) - ref_q_count = ref_q.count() - if (doc != document_cls and ref_q_count > 0 - or (doc == document_cls and ref_q_count > 0)): - ref_q.delete(write_concern=write_concern) - elif rule == NULLIFY: - document_cls.objects(**{field_name + '__in': self}).update( - write_concern=write_concern, **{'unset__%s' % field_name: 1}) - elif rule == PULL: - document_cls.objects(**{field_name + '__in': self}).update( - write_concern=write_concern, - **{'pull_all__%s' % field_name: self}) - - queryset._collection.remove(queryset._query, write_concern=write_concern) - - def update(self, upsert=False, multi=True, write_concern=None, - full_result=False, **update): - """Perform an atomic update on the fields matched by the query. - - :param upsert: Any existing document with that "_id" is overwritten. - :param multi: Update multiple documents. - :param write_concern: Extra keyword arguments are passed down which - will be used as options for the resultant - ``getLastError`` command. For example, - ``save(..., write_concern={w: 2, fsync: True}, ...)`` will - wait until at least two servers have recorded the write and - will force an fsync on the primary server. - :param full_result: Return the full result rather than just the number - updated. - :param update: Django-style update keyword arguments - - .. versionadded:: 0.2 - """ - if not update and not upsert: - raise OperationError("No update parameters, would remove data") - - if write_concern is None: - write_concern = {} - - queryset = self.clone() - query = queryset._query - update = transform.update(queryset._document, **update) - - # If doing an atomic upsert on an inheritable class - # then ensure we add _cls to the update operation - if upsert and '_cls' in query: - if '$set' in update: - update["$set"]["_cls"] = queryset._document._class_name - else: - update["$set"] = {"_cls": queryset._document._class_name} - try: - result = queryset._collection.update(query, update, multi=multi, - upsert=upsert, **write_concern) - if full_result: - return result - elif result: - return result['n'] - except pymongo.errors.OperationFailure, err: - if unicode(err) == u'multi not coded yet': - message = u'update() method requires MongoDB 1.1.3+' - raise OperationError(message) - raise OperationError(u'Update failed (%s)' % unicode(err)) - - def update_one(self, upsert=False, write_concern=None, **update): - """Perform an atomic update on first field matched by the query. - - :param upsert: Any existing document with that "_id" is overwritten. - :param write_concern: Extra keyword arguments are passed down which - will be used as options for the resultant - ``getLastError`` command. For example, - ``save(..., write_concern={w: 2, fsync: True}, ...)`` will - wait until at least two servers have recorded the write and - will force an fsync on the primary server. - :param update: Django-style update keyword arguments - - .. versionadded:: 0.2 - """ - return self.update( - upsert=upsert, multi=False, write_concern=write_concern, **update) - - def with_id(self, object_id): - """Retrieve the object matching the id provided. Uses `object_id` only - and raises InvalidQueryError if a filter has been applied. Returns - `None` if no document exists with that id. - - :param object_id: the value for the id of the document to look up - - .. versionchanged:: 0.6 Raises InvalidQueryError if filter has been set - """ - queryset = self.clone() - if not queryset._query_obj.empty: - msg = "Cannot use a filter whilst using `with_id`" - raise InvalidQueryError(msg) - return queryset.filter(pk=object_id).first() - - def in_bulk(self, object_ids): - """Retrieve a set of documents by their ids. - - :param object_ids: a list or tuple of ``ObjectId``\ s - :rtype: dict of ObjectIds as keys and collection-specific - Document subclasses as values. - - .. versionadded:: 0.3 - """ - doc_map = {} - - docs = self._collection.find({'_id': {'$in': object_ids}}, - **self._cursor_args) - if self._scalar: - for doc in docs: - doc_map[doc['_id']] = self._get_scalar( - self._document._from_son(doc)) - elif self._as_pymongo: - for doc in docs: - doc_map[doc['_id']] = self._get_as_pymongo(doc) - else: - for doc in docs: - doc_map[doc['_id']] = self._document._from_son(doc) - - return doc_map - - def none(self): - """Helper that just returns a list""" - queryset = self.clone() - queryset._none = True return queryset - - def no_sub_classes(self): - """ - Only return instances of this document and not any inherited documents - """ - if self._document._meta.get('allow_inheritance') is True: - self._initial_query = {"_cls": self._document._class_name} - - return self - - def clone(self): - """Creates a copy of the current - :class:`~mongoengine.queryset.QuerySet` - - .. versionadded:: 0.5 - """ - c = self.__class__(self._document, self._collection_obj) - - copy_props = ('_mongo_query', '_initial_query', '_none', '_query_obj', - '_where_clause', '_loaded_fields', '_ordering', '_snapshot', - '_timeout', '_class_check', '_slave_okay', '_read_preference', - '_iter', '_scalar', '_as_pymongo', '_as_pymongo_coerce', - '_limit', '_skip', '_hint', '_auto_dereference') - - for prop in copy_props: - val = getattr(self, prop) - setattr(c, prop, copy.copy(val)) - - if self._cursor_obj: - c._cursor_obj = self._cursor_obj.clone() - - return c - - def select_related(self, max_depth=1): - """Handles dereferencing of :class:`~bson.dbref.DBRef` objects or - :class:`~bson.object_id.ObjectId` a maximum depth in order to cut down - the number queries to mongodb. - - .. versionadded:: 0.5 - """ - # Make select related work the same for querysets - max_depth += 1 - queryset = self.clone() - return queryset._dereference(queryset, max_depth=max_depth) - - def limit(self, n): - """Limit the number of returned documents to `n`. This may also be - achieved using array-slicing syntax (e.g. ``User.objects[:5]``). - - :param n: the maximum number of objects to return - """ - queryset = self.clone() - if n == 0: - queryset._cursor.limit(1) - else: - queryset._cursor.limit(n) - queryset._limit = n - # Return self to allow chaining - return queryset - - def skip(self, n): - """Skip `n` documents before returning the results. This may also be - achieved using array-slicing syntax (e.g. ``User.objects[5:]``). - - :param n: the number of objects to skip before returning results - """ - queryset = self.clone() - queryset._cursor.skip(n) - queryset._skip = n - return queryset - - def hint(self, index=None): - """Added 'hint' support, telling Mongo the proper index to use for the - query. - - Judicious use of hints can greatly improve query performance. When - doing a query on multiple fields (at least one of which is indexed) - pass the indexed field as a hint to the query. - - Hinting will not do anything if the corresponding index does not exist. - The last hint applied to this cursor takes precedence over all others. - - .. versionadded:: 0.5 - """ - queryset = self.clone() - queryset._cursor.hint(index) - queryset._hint = index - return queryset - - def distinct(self, field): - """Return a list of distinct values for a given field. - - :param field: the field to select distinct values from - - .. note:: This is a command and won't take ordering or limit into - account. - - .. versionadded:: 0.4 - .. versionchanged:: 0.5 - Fixed handling references - .. versionchanged:: 0.6 - Improved db_field refrence handling - """ - queryset = self.clone() - try: - field = self._fields_to_dbfields([field]).pop() - finally: - return self._dereference(queryset._cursor.distinct(field), 1, - name=field, instance=self._document) - - def only(self, *fields): - """Load only a subset of this document's fields. :: - - post = BlogPost.objects(...).only("title", "author.name") - - .. note :: `only()` is chainable and will perform a union :: - So with the following it will fetch both: `title` and `author.name`:: - - post = BlogPost.objects.only("title").only("author.name") - - :func:`~mongoengine.queryset.QuerySet.all_fields` will reset any - field filters. - - :param fields: fields to include - - .. versionadded:: 0.3 - .. versionchanged:: 0.5 - Added subfield support - """ - fields = dict([(f, QueryFieldList.ONLY) for f in fields]) - return self.fields(True, **fields) - - def exclude(self, *fields): - """Opposite to .only(), exclude some document's fields. :: - - post = BlogPost.objects(...).exclude("comments") - - .. note :: `exclude()` is chainable and will perform a union :: - So with the following it will exclude both: `title` and `author.name`:: - - post = BlogPost.objects.exclude("title").exclude("author.name") - - :func:`~mongoengine.queryset.QuerySet.all_fields` will reset any - field filters. - - :param fields: fields to exclude - - .. versionadded:: 0.5 - """ - fields = dict([(f, QueryFieldList.EXCLUDE) for f in fields]) - return self.fields(**fields) - - def fields(self, _only_called=False, **kwargs): - """Manipulate how you load this document's fields. Used by `.only()` - and `.exclude()` to manipulate which fields to retrieve. Fields also - allows for a greater level of control for example: - - Retrieving a Subrange of Array Elements: - - You can use the $slice operator to retrieve a subrange of elements in - an array. For example to get the first 5 comments:: - - post = BlogPost.objects(...).fields(slice__comments=5) - - :param kwargs: A dictionary identifying what to include - - .. versionadded:: 0.5 - """ - - # Check for an operator and transform to mongo-style if there is - operators = ["slice"] - cleaned_fields = [] - for key, value in kwargs.items(): - parts = key.split('__') - op = None - if parts[0] in operators: - op = parts.pop(0) - value = {'$' + op: value} - key = '.'.join(parts) - cleaned_fields.append((key, value)) - - fields = sorted(cleaned_fields, key=operator.itemgetter(1)) - queryset = self.clone() - for value, group in itertools.groupby(fields, lambda x: x[1]): - fields = [field for field, value in group] - fields = queryset._fields_to_dbfields(fields) - queryset._loaded_fields += QueryFieldList(fields, value=value, _only_called=_only_called) - - return queryset - - def all_fields(self): - """Include all fields. Reset all previously calls of .only() or - .exclude(). :: - - post = BlogPost.objects.exclude("comments").all_fields() - - .. versionadded:: 0.5 - """ - queryset = self.clone() - queryset._loaded_fields = QueryFieldList( - always_include=queryset._loaded_fields.always_include) - return queryset - - def order_by(self, *keys): - """Order the :class:`~mongoengine.queryset.QuerySet` by the keys. The - order may be specified by prepending each of the keys by a + or a -. - Ascending order is assumed. - - :param keys: fields to order the query results by; keys may be - prefixed with **+** or **-** to determine the ordering direction - """ - queryset = self.clone() - queryset._ordering = queryset._get_order_by(keys) - return queryset - - def explain(self, format=False): - """Return an explain plan record for the - :class:`~mongoengine.queryset.QuerySet`\ 's cursor. - - :param format: format the plan before returning it - """ - plan = self._cursor.explain() - if format: - plan = pprint.pformat(plan) - return plan - - def snapshot(self, enabled): - """Enable or disable snapshot mode when querying. - - :param enabled: whether or not snapshot mode is enabled - - ..versionchanged:: 0.5 - made chainable - """ - queryset = self.clone() - queryset._snapshot = enabled - return queryset - - def timeout(self, enabled): - """Enable or disable the default mongod timeout when querying. - - :param enabled: whether or not the timeout is used - - ..versionchanged:: 0.5 - made chainable - """ - queryset = self.clone() - queryset._timeout = enabled - return queryset - - def slave_okay(self, enabled): - """Enable or disable the slave_okay when querying. - - :param enabled: whether or not the slave_okay is enabled - """ - queryset = self.clone() - queryset._slave_okay = enabled - return queryset - - def read_preference(self, read_preference): - """Change the read_preference when querying. - - :param read_preference: override ReplicaSetConnection-level - preference. - """ - validate_read_preference('read_preference', read_preference) - queryset = self.clone() - queryset._read_preference = read_preference - return queryset - - def scalar(self, *fields): - """Instead of returning Document instances, return either a specific - value or a tuple of values in order. - - Can be used along with - :func:`~mongoengine.queryset.QuerySet.no_dereference` to turn off - dereferencing. - - .. note:: This effects all results and can be unset by calling - ``scalar`` without arguments. Calls ``only`` automatically. - - :param fields: One or more fields to return instead of a Document. - """ - queryset = self.clone() - queryset._scalar = list(fields) - - if fields: - queryset = queryset.only(*fields) - else: - queryset = queryset.all_fields() - - return queryset - - def values_list(self, *fields): - """An alias for scalar""" - return self.scalar(*fields) - - def as_pymongo(self, coerce_types=False): - """Instead of returning Document instances, return raw values from - pymongo. - - :param coerce_type: Field types (if applicable) would be use to - coerce types. - """ - queryset = self.clone() - queryset._as_pymongo = True - queryset._as_pymongo_coerce = coerce_types - return queryset - - # JSON Helpers - - def to_json(self): - """Converts a queryset to JSON""" - return json_util.dumps(self.as_pymongo()) - - def from_json(self, json_data): - """Converts json data to unsaved objects""" - son_data = json_util.loads(json_data) - return [self._document._from_son(data) for data in son_data] - - # JS functionality - - def map_reduce(self, map_f, reduce_f, output, finalize_f=None, limit=None, - scope=None): - """Perform a map/reduce query using the current query spec - and ordering. While ``map_reduce`` respects ``QuerySet`` chaining, - it must be the last call made, as it does not return a maleable - ``QuerySet``. - - See the :meth:`~mongoengine.tests.QuerySetTest.test_map_reduce` - and :meth:`~mongoengine.tests.QuerySetTest.test_map_advanced` - tests in ``tests.queryset.QuerySetTest`` for usage examples. - - :param map_f: map function, as :class:`~bson.code.Code` or string - :param reduce_f: reduce function, as - :class:`~bson.code.Code` or string - :param output: output collection name, if set to 'inline' will try to - use :class:`~pymongo.collection.Collection.inline_map_reduce` - This can also be a dictionary containing output options - see: http://docs.mongodb.org/manual/reference/commands/#mapReduce - :param finalize_f: finalize function, an optional function that - performs any post-reduction processing. - :param scope: values to insert into map/reduce global scope. Optional. - :param limit: number of objects from current query to provide - to map/reduce method - - Returns an iterator yielding - :class:`~mongoengine.document.MapReduceDocument`. - - .. note:: - - Map/Reduce changed in server version **>= 1.7.4**. The PyMongo - :meth:`~pymongo.collection.Collection.map_reduce` helper requires - PyMongo version **>= 1.11**. - - .. versionchanged:: 0.5 - - removed ``keep_temp`` keyword argument, which was only relevant - for MongoDB server versions older than 1.7.4 - - .. versionadded:: 0.3 - """ - queryset = self.clone() - - MapReduceDocument = _import_class('MapReduceDocument') - - if not hasattr(self._collection, "map_reduce"): - raise NotImplementedError("Requires MongoDB >= 1.7.1") - - map_f_scope = {} - if isinstance(map_f, Code): - map_f_scope = map_f.scope - map_f = unicode(map_f) - map_f = Code(queryset._sub_js_fields(map_f), map_f_scope) - - reduce_f_scope = {} - if isinstance(reduce_f, Code): - reduce_f_scope = reduce_f.scope - reduce_f = unicode(reduce_f) - reduce_f_code = queryset._sub_js_fields(reduce_f) - reduce_f = Code(reduce_f_code, reduce_f_scope) - - mr_args = {'query': queryset._query} - - if finalize_f: - finalize_f_scope = {} - if isinstance(finalize_f, Code): - finalize_f_scope = finalize_f.scope - finalize_f = unicode(finalize_f) - finalize_f_code = queryset._sub_js_fields(finalize_f) - finalize_f = Code(finalize_f_code, finalize_f_scope) - mr_args['finalize'] = finalize_f - - if scope: - mr_args['scope'] = scope - - if limit: - mr_args['limit'] = limit - - if output == 'inline' and not queryset._ordering: - map_reduce_function = 'inline_map_reduce' - else: - map_reduce_function = 'map_reduce' - mr_args['out'] = output - - results = getattr(queryset._collection, map_reduce_function)( - map_f, reduce_f, **mr_args) - - if map_reduce_function == 'map_reduce': - results = results.find() - - if queryset._ordering: - results = results.sort(queryset._ordering) - - for doc in results: - yield MapReduceDocument(queryset._document, queryset._collection, - doc['_id'], doc['value']) - - def exec_js(self, code, *fields, **options): - """Execute a Javascript function on the server. A list of fields may be - provided, which will be translated to their correct names and supplied - as the arguments to the function. A few extra variables are added to - the function's scope: ``collection``, which is the name of the - collection in use; ``query``, which is an object representing the - current query; and ``options``, which is an object containing any - options specified as keyword arguments. - - As fields in MongoEngine may use different names in the database (set - using the :attr:`db_field` keyword argument to a :class:`Field` - constructor), a mechanism exists for replacing MongoEngine field names - with the database field names in Javascript code. When accessing a - field, use square-bracket notation, and prefix the MongoEngine field - name with a tilde (~). - - :param code: a string of Javascript code to execute - :param fields: fields that you will be using in your function, which - will be passed in to your function as arguments - :param options: options that you want available to the function - (accessed in Javascript through the ``options`` object) - """ - queryset = self.clone() - - code = queryset._sub_js_fields(code) - - fields = [queryset._document._translate_field_name(f) for f in fields] - collection = queryset._document._get_collection_name() - - scope = { - 'collection': collection, - 'options': options or {}, - } - - query = queryset._query - if queryset._where_clause: - query['$where'] = queryset._where_clause - - scope['query'] = query - code = Code(code, scope=scope) - - db = queryset._document._get_db() - return db.eval(code, *fields) - - def where(self, where_clause): - """Filter ``QuerySet`` results with a ``$where`` clause (a Javascript - expression). Performs automatic field name substitution like - :meth:`mongoengine.queryset.Queryset.exec_js`. - - .. note:: When using this mode of query, the database will call your - function, or evaluate your predicate clause, for each object - in the collection. - - .. versionadded:: 0.5 - """ - queryset = self.clone() - where_clause = queryset._sub_js_fields(where_clause) - queryset._where_clause = where_clause - return queryset - - def sum(self, field): - """Sum over the values of the specified field. - - :param field: the field to sum over; use dot-notation to refer to - embedded document fields - - .. versionchanged:: 0.5 - updated to map_reduce as db.eval doesnt work - with sharding. - """ - map_func = Code(""" - function() { - function deepFind(obj, path) { - var paths = path.split('.') - , current = obj - , i; - - for (i = 0; i < paths.length; ++i) { - if (current[paths[i]] == undefined) { - return undefined; - } else { - current = current[paths[i]]; - } - } - return current; - } - - emit(1, deepFind(this, field) || 0); - } - """, scope={'field': field}) - - reduce_func = Code(""" - function(key, values) { - var sum = 0; - for (var i in values) { - sum += values[i]; - } - return sum; - } - """) - - for result in self.map_reduce(map_func, reduce_func, output='inline'): - return result.value - else: - return 0 - - def average(self, field): - """Average over the values of the specified field. - - :param field: the field to average over; use dot-notation to refer to - embedded document fields - - .. versionchanged:: 0.5 - updated to map_reduce as db.eval doesnt work - with sharding. - """ - map_func = Code(""" - function() { - function deepFind(obj, path) { - var paths = path.split('.') - , current = obj - , i; - - for (i = 0; i < paths.length; ++i) { - if (current[paths[i]] == undefined) { - return undefined; - } else { - current = current[paths[i]]; - } - } - return current; - } - - val = deepFind(this, field) - if (val !== undefined) - emit(1, {t: val || 0, c: 1}); - } - """, scope={'field': field}) - - reduce_func = Code(""" - function(key, values) { - var out = {t: 0, c: 0}; - for (var i in values) { - var value = values[i]; - out.t += value.t; - out.c += value.c; - } - return out; - } - """) - - finalize_func = Code(""" - function(key, value) { - return value.t / value.c; - } - """) - - for result in self.map_reduce(map_func, reduce_func, - finalize_f=finalize_func, output='inline'): - return result.value - else: - return 0 - - def item_frequencies(self, field, normalize=False, map_reduce=True): - """Returns a dictionary of all items present in a field across - the whole queried set of documents, and their corresponding frequency. - This is useful for generating tag clouds, or searching documents. - - .. note:: - - Can only do direct simple mappings and cannot map across - :class:`~mongoengine.fields.ReferenceField` or - :class:`~mongoengine.fields.GenericReferenceField` for more complex - counting a manual map reduce call would is required. - - If the field is a :class:`~mongoengine.fields.ListField`, the items within - each list will be counted individually. - - :param field: the field to use - :param normalize: normalize the results so they add to 1.0 - :param map_reduce: Use map_reduce over exec_js - - .. versionchanged:: 0.5 defaults to map_reduce and can handle embedded - document lookups - """ - if map_reduce: - return self._item_frequencies_map_reduce(field, - normalize=normalize) - return self._item_frequencies_exec_js(field, normalize=normalize) - - # Iterator helpers - - def next(self): - """Wrap the result in a :class:`~mongoengine.Document` object. - """ - if self._limit == 0 or self._none: - raise StopIteration - - raw_doc = self._cursor.next() - if self._as_pymongo: - return self._get_as_pymongo(raw_doc) - doc = self._document._from_son(raw_doc, - _auto_dereference=self._auto_dereference) - if self._scalar: - return self._get_scalar(doc) - - return doc - - def rewind(self): - """Rewind the cursor to its unevaluated state. - - .. versionadded:: 0.3 - """ - self._iter = False - self._cursor.rewind() - - # Properties - - @property - def _collection(self): - """Property that returns the collection object. This allows us to - perform operations only if the collection is accessed. - """ - return self._collection_obj - - @property - def _cursor_args(self): - cursor_args = { - 'snapshot': self._snapshot, - 'timeout': self._timeout - } - if self._read_preference is not None: - cursor_args['read_preference'] = self._read_preference - else: - cursor_args['slave_okay'] = self._slave_okay - if self._loaded_fields: - cursor_args['fields'] = self._loaded_fields.as_dict() - return cursor_args - - @property - def _cursor(self): - if self._cursor_obj is None: - - self._cursor_obj = self._collection.find(self._query, - **self._cursor_args) - # Apply where clauses to cursor - if self._where_clause: - where_clause = self._sub_js_fields(self._where_clause) - self._cursor_obj.where(where_clause) - - if self._ordering: - # Apply query ordering - self._cursor_obj.sort(self._ordering) - elif self._document._meta['ordering']: - # Otherwise, apply the ordering from the document model - order = self._get_order_by(self._document._meta['ordering']) - self._cursor_obj.sort(order) - - if self._limit is not None: - self._cursor_obj.limit(self._limit) - - if self._skip is not None: - self._cursor_obj.skip(self._skip) - - if self._hint != -1: - self._cursor_obj.hint(self._hint) - - return self._cursor_obj - - def __deepcopy__(self, memo): - """Essential for chained queries with ReferenceFields involved""" - return self.clone() - - @property - def _query(self): - if self._mongo_query is None: - self._mongo_query = self._query_obj.to_query(self._document) - if self._class_check: - self._mongo_query.update(self._initial_query) - return self._mongo_query - - @property - def _dereference(self): - if not self.__dereference: - self.__dereference = _import_class('DeReference')() - return self.__dereference - - def no_dereference(self): - """Turn off any dereferencing for the results of this queryset. - """ - queryset = self.clone() - queryset._auto_dereference = False - return queryset - - # Helper Functions - - def _item_frequencies_map_reduce(self, field, normalize=False): - map_func = """ - function() { - var path = '{{~%(field)s}}'.split('.'); - var field = this; - - for (p in path) { - if (typeof field != 'undefined') - field = field[path[p]]; - else - break; - } - if (field && field.constructor == Array) { - field.forEach(function(item) { - emit(item, 1); - }); - } else if (typeof field != 'undefined') { - emit(field, 1); - } else { - emit(null, 1); - } - } - """ % dict(field=field) - reduce_func = """ - function(key, values) { - var total = 0; - var valuesSize = values.length; - for (var i=0; i < valuesSize; i++) { - total += parseInt(values[i], 10); - } - return total; - } - """ - values = self.map_reduce(map_func, reduce_func, 'inline') - frequencies = {} - for f in values: - key = f.key - if isinstance(key, float): - if int(key) == key: - key = int(key) - frequencies[key] = int(f.value) - - if normalize: - count = sum(frequencies.values()) - frequencies = dict([(k, float(v) / count) - for k, v in frequencies.items()]) - - return frequencies - - def _item_frequencies_exec_js(self, field, normalize=False): - """Uses exec_js to execute""" - freq_func = """ - function(path) { - var path = path.split('.'); - - var total = 0.0; - db[collection].find(query).forEach(function(doc) { - var field = doc; - for (p in path) { - if (field) - field = field[path[p]]; - else - break; - } - if (field && field.constructor == Array) { - total += field.length; - } else { - total++; - } - }); - - var frequencies = {}; - var types = {}; - var inc = 1.0; - - db[collection].find(query).forEach(function(doc) { - field = doc; - for (p in path) { - if (field) - field = field[path[p]]; - else - break; - } - if (field && field.constructor == Array) { - field.forEach(function(item) { - frequencies[item] = inc + (isNaN(frequencies[item]) ? 0: frequencies[item]); - }); - } else { - var item = field; - types[item] = item; - frequencies[item] = inc + (isNaN(frequencies[item]) ? 0: frequencies[item]); - } - }); - return [total, frequencies, types]; - } - """ - total, data, types = self.exec_js(freq_func, field) - values = dict([(types.get(k), int(v)) for k, v in data.iteritems()]) - - if normalize: - values = dict([(k, float(v) / total) for k, v in values.items()]) - - frequencies = {} - for k, v in values.iteritems(): - if isinstance(k, float): - if int(k) == k: - k = int(k) - - frequencies[k] = v - - return frequencies - - def _fields_to_dbfields(self, fields): - """Translate fields paths to its db equivalents""" - ret = [] - for field in fields: - field = ".".join(f.db_field for f in - self._document._lookup_field(field.split('.'))) - ret.append(field) - return ret - - def _get_order_by(self, keys): - """Creates a list of order by fields - """ - key_list = [] - for key in keys: - if not key: - continue - direction = pymongo.ASCENDING - if key[0] == '-': - direction = pymongo.DESCENDING - if key[0] in ('-', '+'): - key = key[1:] - key = key.replace('__', '.') - try: - key = self._document._translate_field_name(key) - except: - pass - key_list.append((key, direction)) - - if self._cursor_obj: - self._cursor_obj.sort(key_list) - return key_list - - def _get_scalar(self, doc): - - def lookup(obj, name): - chunks = name.split('__') - for chunk in chunks: - obj = getattr(obj, chunk) - return obj - - data = [lookup(doc, n) for n in self._scalar] - if len(data) == 1: - return data[0] - - return tuple(data) - - def _get_as_pymongo(self, row): - # Extract which fields paths we should follow if .fields(...) was - # used. If not, handle all fields. - if not getattr(self, '__as_pymongo_fields', None): - self.__as_pymongo_fields = [] - - for field in self._loaded_fields.fields - set(['_cls']): - self.__as_pymongo_fields.append(field) - while '.' in field: - field, _ = field.rsplit('.', 1) - self.__as_pymongo_fields.append(field) - - all_fields = not self.__as_pymongo_fields - - def clean(data, path=None): - path = path or '' - - if isinstance(data, dict): - new_data = {} - for key, value in data.iteritems(): - new_path = '%s.%s' % (path, key) if path else key - - if all_fields: - include_field = True - elif self._loaded_fields.value == QueryFieldList.ONLY: - include_field = new_path in self.__as_pymongo_fields - else: - include_field = new_path not in self.__as_pymongo_fields - - if include_field: - new_data[key] = clean(value, path=new_path) - data = new_data - elif isinstance(data, list): - data = [clean(d, path=path) for d in data] - else: - if self._as_pymongo_coerce: - # If we need to coerce types, we need to determine the - # type of this field and use the corresponding - # .to_python(...) - from mongoengine.fields import EmbeddedDocumentField - obj = self._document - for chunk in path.split('.'): - obj = getattr(obj, chunk, None) - if obj is None: - break - elif isinstance(obj, EmbeddedDocumentField): - obj = obj.document_type - if obj and data is not None: - data = obj.to_python(data) - return data - return clean(row) - - def _sub_js_fields(self, code): - """When fields are specified with [~fieldname] syntax, where - *fieldname* is the Python name of a field, *fieldname* will be - substituted for the MongoDB name of the field (specified using the - :attr:`name` keyword argument in a field's constructor). - """ - def field_sub(match): - # Extract just the field name, and look up the field objects - field_name = match.group(1).split('.') - fields = self._document._lookup_field(field_name) - # Substitute the correct name for the field into the javascript - return u'["%s"]' % fields[-1].db_field - - def field_path_sub(match): - # Extract just the field name, and look up the field objects - field_name = match.group(1).split('.') - fields = self._document._lookup_field(field_name) - # Substitute the correct name for the field into the javascript - return ".".join([f.db_field for f in fields]) - - code = re.sub(u'\[\s*~([A-z_][A-z_0-9.]+?)\s*\]', field_sub, code) - code = re.sub(u'\{\{\s*~([A-z_][A-z_0-9.]+?)\s*\}\}', field_path_sub, - code) - return code - - # Deprecated - def ensure_index(self, **kwargs): - """Deprecated use :func:`Document.ensure_index`""" - msg = ("Doc.objects()._ensure_index() is deprecated. " - "Use Doc.ensure_index() instead.") - warnings.warn(msg, DeprecationWarning) - self._document.__class__.ensure_index(**kwargs) - return self - - def _ensure_indexes(self): - """Deprecated use :func:`~Document.ensure_indexes`""" - msg = ("Doc.objects()._ensure_indexes() is deprecated. " - "Use Doc.ensure_indexes() instead.") - warnings.warn(msg, DeprecationWarning) - self._document.__class__.ensure_indexes() diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 9495a25..6e3eb9b 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3254,7 +3254,7 @@ class QuerySetTest(unittest.TestCase): User(name="Barack Obama", age=51, price=Decimal('2.22')).save() results = User.objects.only('id', 'name').as_pymongo() - self.assertEqual(results[0].keys(), ['_id', 'name']) + self.assertEqual(sorted(results[0].keys()), sorted(['_id', 'name'])) users = User.objects.only('name', 'price').as_pymongo() results = list(users) @@ -3365,6 +3365,34 @@ class QuerySetTest(unittest.TestCase): self.assertEqual("%s" % users, "[]") self.assertEqual(1, len(users._result_cache)) + def test_no_cache(self): + """Ensure you can add meta data to file""" + + class Noddy(Document): + fields = DictField() + + Noddy.drop_collection() + for i in xrange(100): + noddy = Noddy() + for j in range(20): + noddy.fields["key"+str(j)] = "value "+str(j) + noddy.save() + + docs = Noddy.objects.no_cache() + + counter = len([1 for i in docs]) + self.assertEquals(counter, 100) + + self.assertEquals(len(list(docs)), 100) + self.assertRaises(TypeError, lambda: len(docs)) + + with query_counter() as q: + self.assertEqual(q, 0) + list(docs) + self.assertEqual(q, 1) + list(docs) + self.assertEqual(q, 2) + def test_nested_queryset_iterator(self): # Try iterating the same queryset twice, nested. names = ['Alice', 'Bob', 'Chuck', 'David', 'Eric', 'Francis', 'George'] From fb0dd2c1ca6eb4a1a8fef29cce2a39497231d9e3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 10 Jul 2013 19:54:30 +0000 Subject: [PATCH 1217/1279] Updated changelog --- docs/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d875040..76df230 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -5,7 +5,7 @@ Changelog Changes in 0.8.3 ================ - Added QuerySetNoCache and QuerySet.no_cache() for lower memory consumption (#365) -- Fixed sum and average mapreduce dot notation support (#375, #376) +- Fixed sum and average mapreduce dot notation support (#375, #376, #393) - Fixed as_pymongo to return the id (#386) - Document.select_related() now respects `db_alias` (#377) - Reload uses shard_key if applicable (#384) From e155e1fa8621c158b619bcbb1c5b9601f5364634 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 10 Jul 2013 20:10:01 +0000 Subject: [PATCH 1218/1279] Add a default for previously pickled versions --- mongoengine/base/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 04b0c05..0eb63d5 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -160,7 +160,7 @@ class BaseDocument(object): '_fields_ordered', '_dynamic_fields'): if k in data: setattr(self, k, data[k]) - for k in data.get('_dynamic_fields').keys(): + for k in data.get('_dynamic_fields', SON()).keys(): setattr(self, k, data["_data"].get(k)) def __iter__(self): From d9f538170b73f53f43f3f97ae3f6b66d7c40bb90 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 10 Jul 2013 21:19:11 +0000 Subject: [PATCH 1219/1279] Added get_proxy_object helper to filefields (#391) --- docs/changelog.rst | 1 + mongoengine/fields.py | 18 ++++++++---------- tests/fields/file_tests.py | 26 ++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 76df230..e0f47fe 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.3 ================ +- Added get_proxy_object helper to filefields (#391) - Added QuerySetNoCache and QuerySet.no_cache() for lower memory consumption (#365) - Fixed sum and average mapreduce dot notation support (#375, #376, #393) - Fixed as_pymongo to return the id (#386) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 9d3a668..47554e0 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1190,7 +1190,7 @@ class FileField(BaseField): # Check if a file already exists for this model grid_file = instance._data.get(self.name) if not isinstance(grid_file, self.proxy_class): - grid_file = self.get_proxy_obj(key=key, instance=instance) + grid_file = self.get_proxy_obj(key=self.name, instance=instance) instance._data[self.name] = grid_file if not grid_file.key: @@ -1218,16 +1218,16 @@ class FileField(BaseField): instance._data[key] = value instance._mark_as_changed(key) - + def get_proxy_obj(self, key, instance, db_alias=None, collection_name=None): if db_alias is None: db_alias = self.db_alias if collection_name is None: collection_name = self.collection_name - - return self.proxy_class(key=key, instance=instance, - db_alias=db_alias, - collection_name=collection_name) + + return self.proxy_class(key=key, instance=instance, + db_alias=db_alias, + collection_name=collection_name) def to_mongo(self, value): # Store the GridFS file id in MongoDB @@ -1261,10 +1261,8 @@ class ImageGridFsProxy(GridFSProxy): applying field properties (size, thumbnail_size) """ field = self.instance._fields[self.key] - # if the field from the instance has an attribute field - # we use that one and hope for the best. Usually only container - # fields have a field attribute. - if hasattr(field, 'field'): + # Handle nested fields + if hasattr(field, 'field') and isinstance(field.field, FileField): field = field.field try: diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py index dfef9ee..d044500 100644 --- a/tests/fields/file_tests.py +++ b/tests/fields/file_tests.py @@ -455,5 +455,31 @@ class FileTest(unittest.TestCase): self.assertEqual(1, TestImage.objects(Q(image1=grid_id) or Q(image2=grid_id)).count()) + def test_complex_field_filefield(self): + """Ensure you can add meta data to file""" + + class Animal(Document): + genus = StringField() + family = StringField() + photos = ListField(FileField()) + + Animal.drop_collection() + marmot = Animal(genus='Marmota', family='Sciuridae') + + marmot_photo = open(TEST_IMAGE_PATH, 'rb') # Retrieve a photo from disk + + photos_field = marmot._fields['photos'].field + new_proxy = photos_field.get_proxy_obj('photos', marmot) + new_proxy.put(marmot_photo, content_type='image/jpeg', foo='bar') + marmot_photo.close() + + marmot.photos.append(new_proxy) + marmot.save() + + marmot = Animal.objects.get() + self.assertEqual(marmot.photos[0].content_type, 'image/jpeg') + self.assertEqual(marmot.photos[0].foo, 'bar') + self.assertEqual(marmot.photos[0].get().length, 8313) + if __name__ == '__main__': unittest.main() From f48a0b7b7d8cee37c7519bab162123a5493e17f1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 10 Jul 2013 21:30:29 +0000 Subject: [PATCH 1220/1279] Trying to fix travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 2bb5863..8c4d5e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,8 @@ install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install django==$DJANGO --use-mirrors ; true; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install python-dateutils==1.5 --no-allow-external ; true; fi + - if [[ $TRAVIS_PYTHON_VERSION == '3.'* ]]; then pip install python-dateutils --no-allow-external ; true; fi - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - python setup.py install From 6c599ef50678bbef18a49bedddcfe3ea87705c86 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 11 Jul 2013 07:15:34 +0000 Subject: [PATCH 1221/1279] Fix edge case where _dynamic_keys stored as None (#387, #401) --- mongoengine/base/document.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 0eb63d5..cbce4ff 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -160,7 +160,8 @@ class BaseDocument(object): '_fields_ordered', '_dynamic_fields'): if k in data: setattr(self, k, data[k]) - for k in data.get('_dynamic_fields', SON()).keys(): + dynamic_fields = data.get('_dynamic_fields') or SON() + for k in dynamic_fields.keys(): setattr(self, k, data["_data"].get(k)) def __iter__(self): From d593f7e04b6bf26b576b597b2a68dfd72b400269 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 11 Jul 2013 08:11:00 +0000 Subject: [PATCH 1222/1279] Fixed EmbeddedDocuments with `id` also storing `_id` (#402) --- mongoengine/base/document.py | 6 ++++-- tests/document/instance.py | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index cbce4ff..536fc2f 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -262,8 +262,10 @@ class BaseDocument(object): data[field.db_field] = value # If "_id" has not been set, then try and set it - if data["_id"] is None: - data["_id"] = self._data.get("id", None) + Document = _import_class("Document") + if isinstance(self, Document): + if data["_id"] is None: + data["_id"] = self._data.get("id", None) if data['_id'] is None: data.pop('_id') diff --git a/tests/document/instance.py b/tests/document/instance.py index e85c9d8..a61c439 100644 --- a/tests/document/instance.py +++ b/tests/document/instance.py @@ -444,6 +444,13 @@ class InstanceTest(unittest.TestCase): self.assertEqual(Employee(name="Bob", age=35, salary=0).to_mongo().keys(), ['_cls', 'name', 'age', 'salary']) + def test_embedded_document_to_mongo_id(self): + class SubDoc(EmbeddedDocument): + id = StringField(required=True) + + sub_doc = SubDoc(id="abc") + self.assertEqual(sub_doc.to_mongo().keys(), ['id']) + def test_embedded_document(self): """Ensure that embedded documents are set up correctly. """ From 6c2c33cac8010d8658810073ba3c21843f24cb9a Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 11 Jul 2013 08:12:27 +0000 Subject: [PATCH 1223/1279] Add Jatin- to Authors, changelog update --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index 70720f0..b8143a0 100644 --- a/AUTHORS +++ b/AUTHORS @@ -171,3 +171,4 @@ that much better: * Michael Bartnett (https://github.com/michaelbartnett) * Alon Horev (https://github.com/alonho) * Kelvin Hammond (https://github.com/kelvinhammond) + * Jatin- (https://github.com/jatin-) diff --git a/docs/changelog.rst b/docs/changelog.rst index e0f47fe..6ca52b2 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.3 ================ +- Fixed EmbeddedDocuments with `id` also storing `_id` (#402) - Added get_proxy_object helper to filefields (#391) - Added QuerySetNoCache and QuerySet.no_cache() for lower memory consumption (#365) - Fixed sum and average mapreduce dot notation support (#375, #376, #393) From 73026047e9e8be9581a61f561b8cb14cb9613fdf Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 11 Jul 2013 09:29:06 +0000 Subject: [PATCH 1224/1279] Trying to fix dateutil --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8c4d5e1..092b985 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,11 +14,11 @@ env: - PYMONGO=3.2 DJANGO=1.5.1 - PYMONGO=3.3 DJANGO=1.5.1 install: + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install python-dateutil; true; fi + - if [[ $TRAVIS_PYTHON_VERSION == '3.'* ]]; then sudo apt-get install python3-dateutil; true; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install django==$DJANGO --use-mirrors ; true; fi - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install python-dateutils==1.5 --no-allow-external ; true; fi - - if [[ $TRAVIS_PYTHON_VERSION == '3.'* ]]; then pip install python-dateutils --no-allow-external ; true; fi - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - python setup.py install From 1aa2b86df31822ec527460db16422e2177e81eee Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 11 Jul 2013 09:38:59 +0000 Subject: [PATCH 1225/1279] travis install python-dateutil direct --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 092b985..c7e8ea3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,13 +14,12 @@ env: - PYMONGO=3.2 DJANGO=1.5.1 - PYMONGO=3.3 DJANGO=1.5.1 install: - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then sudo apt-get install python-dateutil; true; fi - - if [[ $TRAVIS_PYTHON_VERSION == '3.'* ]]; then sudo apt-get install python3-dateutil; true; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install django==$DJANGO --use-mirrors ; true; fi - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi + - pip install https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.1.tar.gz#md5=1534bb15cf311f07afaa3aacba1c028b - python setup.py install script: - python setup.py test From 48ef176e281bba8bd973a1caa530774540d5ce61 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 12 Jul 2013 08:41:56 +0000 Subject: [PATCH 1226/1279] 0.8.3 is a go --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index 5bd1201..bfa35fb 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -15,7 +15,7 @@ import django __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + list(queryset.__all__) + signals.__all__ + list(errors.__all__)) -VERSION = (0, 8, 2) +VERSION = (0, 8, 3) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 4eaba4d..512c621 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.8.2 +Version: 0.8.3 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From dc5512e4039f81b7773f177eb6824e63af2d513f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 12 Jul 2013 09:01:11 +0000 Subject: [PATCH 1227/1279] Upgrade warning for 0.8.3 --- docs/upgrade.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index b8864b0..0051a62 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -3,7 +3,7 @@ Upgrading ######### -0.8.2 to 0.8.2 +0.8.2 to 0.8.3 ************** Minor change that may impact users: From 35f2781518e7d8d642b54010b9ec6f94ba3e44c9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 12 Jul 2013 09:11:27 +0000 Subject: [PATCH 1228/1279] Update changelog --- docs/_themes/nature/static/nature.css_t | 72 +++++++++++++------------ docs/changelog.rst | 5 ++ 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/docs/_themes/nature/static/nature.css_t b/docs/_themes/nature/static/nature.css_t index 03b0379..337760b 100644 --- a/docs/_themes/nature/static/nature.css_t +++ b/docs/_themes/nature/static/nature.css_t @@ -2,11 +2,15 @@ * Sphinx stylesheet -- default theme * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ - + @import url("basic.css"); - + +#changelog p.first {margin-bottom: 0 !important;} +#changelog p {margin-top: 0 !important; + margin-bottom: 0 !important;} + /* -- page layout ----------------------------------------------------------- */ - + body { font-family: Arial, sans-serif; font-size: 100%; @@ -28,18 +32,18 @@ div.bodywrapper { hr{ border: 1px solid #B1B4B6; } - + div.document { background-color: #eee; } - + div.body { background-color: #ffffff; color: #3E4349; padding: 0 30px 30px 30px; font-size: 0.8em; } - + div.footer { color: #555; width: 100%; @@ -47,12 +51,12 @@ div.footer { text-align: center; font-size: 75%; } - + div.footer a { color: #444; text-decoration: underline; } - + div.related { background-color: #6BA81E; line-height: 32px; @@ -60,11 +64,11 @@ div.related { text-shadow: 0px 1px 0 #444; font-size: 0.80em; } - + div.related a { color: #E2F3CC; } - + div.sphinxsidebar { font-size: 0.75em; line-height: 1.5em; @@ -73,7 +77,7 @@ div.sphinxsidebar { div.sphinxsidebarwrapper{ padding: 20px 0; } - + div.sphinxsidebar h3, div.sphinxsidebar h4 { font-family: Arial, sans-serif; @@ -89,30 +93,30 @@ div.sphinxsidebar h4 { div.sphinxsidebar h4{ font-size: 1.1em; } - + div.sphinxsidebar h3 a { color: #444; } - - + + div.sphinxsidebar p { color: #888; padding: 5px 20px; } - + div.sphinxsidebar p.topless { } - + div.sphinxsidebar ul { margin: 10px 20px; padding: 0; color: #000; } - + div.sphinxsidebar a { color: #444; } - + div.sphinxsidebar input { border: 1px solid #ccc; font-family: sans-serif; @@ -122,19 +126,19 @@ div.sphinxsidebar input { div.sphinxsidebar input[type=text]{ margin-left: 20px; } - + /* -- body styles ----------------------------------------------------------- */ - + a { color: #005B81; text-decoration: none; } - + a:hover { color: #E32E00; text-decoration: underline; } - + div.body h1, div.body h2, div.body h3, @@ -149,30 +153,30 @@ div.body h6 { padding: 5px 0 5px 10px; text-shadow: 0px 1px 0 white } - + div.body h1 { border-top: 20px solid white; margin-top: 0; font-size: 200%; } div.body h2 { font-size: 150%; background-color: #C8D5E3; } div.body h3 { font-size: 120%; background-color: #D8DEE3; } div.body h4 { font-size: 110%; background-color: #D8DEE3; } div.body h5 { font-size: 100%; background-color: #D8DEE3; } div.body h6 { font-size: 100%; background-color: #D8DEE3; } - + a.headerlink { color: #c60f0f; font-size: 0.8em; padding: 0 4px 0 4px; text-decoration: none; } - + a.headerlink:hover { background-color: #c60f0f; color: white; } - + div.body p, div.body dd, div.body li { line-height: 1.5em; } - + div.admonition p.admonition-title + p { display: inline; } @@ -185,29 +189,29 @@ div.note { background-color: #eee; border: 1px solid #ccc; } - + div.seealso { background-color: #ffc; border: 1px solid #ff6; } - + div.topic { background-color: #eee; } - + div.warning { background-color: #ffe4e4; border: 1px solid #f66; } - + p.admonition-title { display: inline; } - + p.admonition-title:after { content: ":"; } - + pre { padding: 10px; background-color: White; @@ -219,7 +223,7 @@ pre { -webkit-box-shadow: 1px 1px 1px #d8d8d8; -moz-box-shadow: 1px 1px 1px #d8d8d8; } - + tt { background-color: #ecf0f3; color: #222; diff --git a/docs/changelog.rst b/docs/changelog.rst index 6ca52b2..ee92d47 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -12,6 +12,9 @@ Changes in 0.8.3 - Document.select_related() now respects `db_alias` (#377) - Reload uses shard_key if applicable (#384) - Dynamic fields are ordered based on creation and stored in _fields_ordered (#396) + + **Potential breaking change:** http://docs.mongoengine.org/en/latest/upgrade.html#to-0-8-3 + - Fixed pickling dynamic documents `_dynamic_fields` (#387) - Fixed ListField setslice and delslice dirty tracking (#390) - Added Django 1.5 PY3 support (#392) @@ -20,6 +23,8 @@ Changes in 0.8.3 - Fixed queryset.get() respecting no_dereference (#373) - Added full_result kwarg to update (#380) + + Changes in 0.8.2 ================ - Added compare_indexes helper (#361) From bcf83ec76190524a5508f17d462b9b360584c07e Mon Sep 17 00:00:00 2001 From: bool-dev Date: Thu, 18 Jul 2013 09:17:28 +0530 Subject: [PATCH 1229/1279] Corrected spelling mistakes, some grammar, and UUID/DecimalField error in upgrade.rst --- docs/upgrade.rst | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/upgrade.rst b/docs/upgrade.rst index 0051a62..a1fccea 100644 --- a/docs/upgrade.rst +++ b/docs/upgrade.rst @@ -16,8 +16,8 @@ fields. Previously they were stored alphabetically. ********** There have been numerous backwards breaking changes in 0.8. The reasons for -these are ensure that MongoEngine has sane defaults going forward and -performs the best it can out the box. Where possible there have been +these are to ensure that MongoEngine has sane defaults going forward and that it +performs the best it can out of the box. Where possible there have been FutureWarnings to help get you ready for the change, but that hasn't been possible for the whole of the release. @@ -71,7 +71,7 @@ inherited classes like so: :: Document Definition ------------------- -The default for inheritance has changed - its now off by default and +The default for inheritance has changed - it is now off by default and :attr:`_cls` will not be stored automatically with the class. So if you extend your :class:`~mongoengine.Document` or :class:`~mongoengine.EmbeddedDocuments` you will need to declare :attr:`allow_inheritance` in the meta data like so: :: @@ -81,7 +81,7 @@ you will need to declare :attr:`allow_inheritance` in the meta data like so: :: meta = {'allow_inheritance': True} -Previously, if you had data the database that wasn't defined in the Document +Previously, if you had data in the database that wasn't defined in the Document definition, it would set it as an attribute on the document. This is no longer the case and the data is set only in the ``document._data`` dictionary: :: @@ -102,8 +102,8 @@ the case and the data is set only in the ``document._data`` dictionary: :: AttributeError: 'Animal' object has no attribute 'size' The Document class has introduced a reserved function `clean()`, which will be -called before saving the document. If your document class happen to have a method -with the same name, please try rename it. +called before saving the document. If your document class happens to have a method +with the same name, please try to rename it. def clean(self): pass @@ -111,7 +111,7 @@ with the same name, please try rename it. ReferenceField -------------- -ReferenceFields now store ObjectId's by default - this is more efficient than +ReferenceFields now store ObjectIds by default - this is more efficient than DBRefs as we already know what Document types they reference:: # Old code @@ -157,7 +157,7 @@ UUIDFields now default to storing binary values:: class Animal(Document): uuid = UUIDField(binary=False) -To migrate all the uuid's you need to touch each object and mark it as dirty +To migrate all the uuids you need to touch each object and mark it as dirty eg:: # Doc definition @@ -175,7 +175,7 @@ eg:: DecimalField ------------ -DecimalField now store floats - previous it was storing strings and that +DecimalFields now store floats - previously it was storing strings and that made it impossible to do comparisons when querying correctly.:: # Old code @@ -186,7 +186,7 @@ made it impossible to do comparisons when querying correctly.:: class Person(Document): balance = DecimalField(force_string=True) -To migrate all the uuid's you need to touch each object and mark it as dirty +To migrate all the DecimalFields you need to touch each object and mark it as dirty eg:: # Doc definition @@ -198,7 +198,7 @@ eg:: p._mark_as_changed('balance') p.save() -.. note:: DecimalField's have also been improved with the addition of precision +.. note:: DecimalFields have also been improved with the addition of precision and rounding. See :class:`~mongoengine.fields.DecimalField` for more information. `An example test migration for DecimalFields is available on github @@ -207,7 +207,7 @@ eg:: Cascading Saves --------------- To improve performance document saves will no longer automatically cascade. -Any changes to a Documents references will either have to be saved manually or +Any changes to a Document's references will either have to be saved manually or you will have to explicitly tell it to cascade on save:: # At the class level: @@ -249,7 +249,7 @@ update your code like so: :: # Update example a) assign queryset after a change: mammals = Animal.objects(type="mammal") - carnivores = mammals.filter(order="Carnivora") # Reassign the new queryset so fitler can be applied + carnivores = mammals.filter(order="Carnivora") # Reassign the new queryset so filter can be applied [m for m in carnivores] # This will return all carnivores # Update example b) chain the queryset: @@ -276,7 +276,7 @@ queryset you should upgrade to use count:: .only() now inline with .exclude() ---------------------------------- -The behaviour of `.only()` was highly ambious, now it works in the mirror fashion +The behaviour of `.only()` was highly ambiguous, now it works in mirror fashion to `.exclude()`. Chaining `.only()` calls will increase the fields required:: # Old code @@ -440,7 +440,7 @@ main areas of changed are: choices in fields, map_reduce and collection names. Choice options: =============== -Are now expected to be an iterable of tuples, with the first element in each +Are now expected to be an iterable of tuples, with the first element in each tuple being the actual value to be stored. The second element is the human-readable name for the option. @@ -462,8 +462,8 @@ such the following have been changed: Default collection naming ========================= -Previously it was just lowercase, its now much more pythonic and readable as -its lowercase and underscores, previously :: +Previously it was just lowercase, it's now much more pythonic and readable as +it's lowercase and underscores, previously :: class MyAceDocument(Document): pass @@ -530,5 +530,5 @@ Alternatively, you can rename your collections eg :: mongodb 1.8 > 2.0 + =================== -Its been reported that indexes may need to be recreated to the newer version of indexes. +It's been reported that indexes may need to be recreated to the newer version of indexes. To do this drop indexes and call ``ensure_indexes`` on each model. From 80b3df89537a50cc9c6f91163306185a0816c386 Mon Sep 17 00:00:00 2001 From: Thom Knowles Date: Mon, 22 Jul 2013 20:07:57 -0400 Subject: [PATCH 1230/1279] dereference instance not thread-safe --- AUTHORS | 1 + mongoengine/base/fields.py | 16 +++++----------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/AUTHORS b/AUTHORS index b8143a0..69a87a2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -172,3 +172,4 @@ that much better: * Alon Horev (https://github.com/alonho) * Kelvin Hammond (https://github.com/kelvinhammond) * Jatin- (https://github.com/jatin-) + * Thom Knowles (https://github.com/fleat) diff --git a/mongoengine/base/fields.py b/mongoengine/base/fields.py index eda9b3c..c6abd02 100644 --- a/mongoengine/base/fields.py +++ b/mongoengine/base/fields.py @@ -186,7 +186,6 @@ class ComplexBaseField(BaseField): """ field = None - __dereference = False def __get__(self, instance, owner): """Descriptor to automatically dereference references. @@ -201,9 +200,11 @@ class ComplexBaseField(BaseField): (self.field is None or isinstance(self.field, (GenericReferenceField, ReferenceField)))) + _dereference = _import_class("DeReference")() + self._auto_dereference = instance._fields[self.name]._auto_dereference - if not self.__dereference and instance._initialised and dereference: - instance._data[self.name] = self._dereference( + if instance._initialised and dereference: + instance._data[self.name] = _dereference( instance._data.get(self.name), max_depth=1, instance=instance, name=self.name ) @@ -222,7 +223,7 @@ class ComplexBaseField(BaseField): if (self._auto_dereference and instance._initialised and isinstance(value, (BaseList, BaseDict)) and not value._dereferenced): - value = self._dereference( + value = _dereference( value, max_depth=1, instance=instance, name=self.name ) value._dereferenced = True @@ -382,13 +383,6 @@ class ComplexBaseField(BaseField): owner_document = property(_get_owner_document, _set_owner_document) - @property - def _dereference(self,): - if not self.__dereference: - DeReference = _import_class("DeReference") - self.__dereference = DeReference() # Cached - return self.__dereference - class ObjectIdField(BaseField): """A field wrapper around MongoDB's ObjectIds. From d92ed04538e7dcf0e94f87e9c3272c0f8a3c1ee0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Jul 2013 08:13:52 +0000 Subject: [PATCH 1231/1279] Docs update #406 --- mongoengine/queryset/queryset.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 9db98a7..9cfb1b6 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -95,7 +95,10 @@ class QuerySet(BaseQuerySet): self._has_more = False def no_cache(self): - """Convert to a non_caching queryset""" + """Convert to a non_caching queryset + + .. versionadded:: 0.8.3 Convert to non caching queryset + """ if self._result_cache is not None: raise OperationError("QuerySet already cached") return self.clone_into(QuerySetNoCache(self._document, self._collection)) @@ -105,7 +108,10 @@ class QuerySetNoCache(BaseQuerySet): """A non caching QuerySet""" def cache(self): - """Convert to a caching queryset""" + """Convert to a caching queryset + + .. versionadded:: 0.8.3 Convert to caching queryset + """ return self.clone_into(QuerySet(self._document, self._collection)) def __repr__(self): From a458d5a176afc645e35ef02f3103c597e728e849 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 23 Jul 2013 08:16:06 +0000 Subject: [PATCH 1232/1279] Docs update #406 --- docs/guide/querying.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 5fd0360..127e479 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -17,8 +17,8 @@ fetch documents from the database:: As of MongoEngine 0.8 the querysets utilise a local cache. So iterating it multiple times will only cause a single query. If this is not the - desired behavour you can call :class:`~mongoengine.QuerySet.no_cache` to - return a non-caching queryset. + desired behavour you can call :class:`~mongoengine.QuerySet.no_cache` + (version **0.8.3+**) to return a non-caching queryset. Filtering queries ================= From dae9e662a57ed6c4e73a1aaa517b6d17a7bb9fcc Mon Sep 17 00:00:00 2001 From: Paul Uithol Date: Thu, 25 Jul 2013 14:30:20 +0200 Subject: [PATCH 1233/1279] Create test case for failing saves (wrong delta) with dbref=False --- tests/document/delta.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/document/delta.py b/tests/document/delta.py index 3656d9e..a5302f1 100644 --- a/tests/document/delta.py +++ b/tests/document/delta.py @@ -313,17 +313,17 @@ class DeltaTest(unittest.TestCase): self.circular_reference_deltas_2(DynamicDocument, Document) self.circular_reference_deltas_2(DynamicDocument, DynamicDocument) - def circular_reference_deltas_2(self, DocClass1, DocClass2): + def circular_reference_deltas_2(self, DocClass1, DocClass2, dbref=True): class Person(DocClass1): name = StringField() - owns = ListField(ReferenceField('Organization')) - employer = ReferenceField('Organization') + owns = ListField(ReferenceField('Organization', dbref=dbref)) + employer = ReferenceField('Organization', dbref=dbref) class Organization(DocClass2): name = StringField() - owner = ReferenceField('Person') - employees = ListField(ReferenceField('Person')) + owner = ReferenceField('Person', dbref=dbref) + employees = ListField(ReferenceField('Person', dbref=dbref)) Person.drop_collection() Organization.drop_collection() @@ -355,6 +355,8 @@ class DeltaTest(unittest.TestCase): self.assertEqual(o.owner, p) self.assertEqual(e.employer, o) + return person, organization, employee + def test_delta_db_field(self): self.delta_db_field(Document) self.delta_db_field(DynamicDocument) @@ -686,6 +688,19 @@ class DeltaTest(unittest.TestCase): self.assertEqual(doc._get_changed_fields(), ['list_field']) self.assertEqual(doc._delta(), ({}, {'list_field': 1})) + def test_delta_with_dbref_true(self): + person, organization, employee = self.circular_reference_deltas_2(Document, Document, True) + employee.name = 'test' + changed = organization._get_changed_fields() + delta = organization._delta() + organization.save() + + def test_delta_with_dbref_false(self): + person, organization, employee = self.circular_reference_deltas_2(Document, Document, False) + employee.name = 'test' + changed = organization._get_changed_fields() + delta = organization._delta() + organization.save() if __name__ == '__main__': unittest.main() From 2ad5ffbda215f71f91261fa1cac3ebfaa978f904 Mon Sep 17 00:00:00 2001 From: Paul Uithol Date: Thu, 25 Jul 2013 14:51:09 +0200 Subject: [PATCH 1234/1279] Add asserts to `test_delta_with_dbref_*`, instead of relying on exceptions --- tests/document/delta.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/document/delta.py b/tests/document/delta.py index a5302f1..fae65c9 100644 --- a/tests/document/delta.py +++ b/tests/document/delta.py @@ -691,15 +691,25 @@ class DeltaTest(unittest.TestCase): def test_delta_with_dbref_true(self): person, organization, employee = self.circular_reference_deltas_2(Document, Document, True) employee.name = 'test' - changed = organization._get_changed_fields() - delta = organization._delta() + + self.assertEqual(organization._get_changed_fields(), ['employees.0.name']) + + updates, removals = organization._delta() + self.assertEqual({}, removals) + self.assertIn('employees.0', updates) + organization.save() def test_delta_with_dbref_false(self): person, organization, employee = self.circular_reference_deltas_2(Document, Document, False) employee.name = 'test' - changed = organization._get_changed_fields() - delta = organization._delta() + + self.assertEqual(organization._get_changed_fields(), ['employees.0.name']) + + updates, removals = organization._delta() + self.assertEqual({}, removals) + self.assertIn('employees.0', updates) + organization.save() if __name__ == '__main__': From e27439be6adf4326177e7ff1530047c22a8e2831 Mon Sep 17 00:00:00 2001 From: Paul Uithol Date: Thu, 25 Jul 2013 14:52:03 +0200 Subject: [PATCH 1235/1279] Fix `BaseDocument._delta` when working with plain ObjectIds instead of DBRefs --- mongoengine/base/document.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 536fc2f..258b3f2 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -4,7 +4,7 @@ import numbers from functools import partial import pymongo -from bson import json_util +from bson import json_util, ObjectId from bson.dbref import DBRef from bson.son import SON @@ -454,7 +454,7 @@ class BaseDocument(object): d = doc new_path = [] for p in parts: - if isinstance(d, DBRef): + if isinstance(d, (ObjectId, DBRef)): break elif isinstance(d, list) and p.isdigit(): d = d[int(p)] From d143e50238c304c6aca9ce9142c59707c24e4572 Mon Sep 17 00:00:00 2001 From: Paul Uithol Date: Thu, 25 Jul 2013 15:34:58 +0200 Subject: [PATCH 1236/1279] Replace `assertIn` with an `assertTrue`; apparently missing in Python 2.6 --- tests/document/delta.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/document/delta.py b/tests/document/delta.py index fae65c9..c6efc02 100644 --- a/tests/document/delta.py +++ b/tests/document/delta.py @@ -696,7 +696,7 @@ class DeltaTest(unittest.TestCase): updates, removals = organization._delta() self.assertEqual({}, removals) - self.assertIn('employees.0', updates) + self.assertTrue('employees.0' in updates) organization.save() @@ -708,7 +708,7 @@ class DeltaTest(unittest.TestCase): updates, removals = organization._delta() self.assertEqual({}, removals) - self.assertIn('employees.0', updates) + self.assertTrue('employees.0' in updates) organization.save() From 67f43b2aad761e7d04908742c14441282f20d0db Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 29 Jul 2013 15:29:48 +0000 Subject: [PATCH 1237/1279] Allow args and kwargs to be passed through to_json (#420) --- docs/changelog.rst | 4 ++++ mongoengine/base/document.py | 4 ++-- mongoengine/queryset/base.py | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index ee92d47..a53d6df 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -2,6 +2,10 @@ Changelog ========= +Changes in 0.8.4 +================ +- Allow args and kwargs to be passed through to_json (#420) + Changes in 0.8.3 ================ - Fixed EmbeddedDocuments with `id` also storing `_id` (#402) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 536fc2f..e8232a0 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -321,9 +321,9 @@ class BaseDocument(object): message = "ValidationError (%s:%s) " % (self._class_name, pk) raise ValidationError(message, errors=errors) - def to_json(self): + def to_json(self, *args, **kwargs): """Converts a document to JSON""" - return json_util.dumps(self.to_mongo()) + return json_util.dumps(self.to_mongo(), *args, **kwargs) @classmethod def from_json(cls, json_data): diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index d3bb4c4..e88feb3 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -827,9 +827,9 @@ class BaseQuerySet(object): # JSON Helpers - def to_json(self): + def to_json(self, *args, **kwargs): """Converts a queryset to JSON""" - return json_util.dumps(self.as_pymongo()) + return json_util.dumps(self.as_pymongo(), *args, **kwargs) def from_json(self, json_data): """Converts json data to unsaved objects""" From 7a97d42338b49aa4253c3d590390a68810ff98cd Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 29 Jul 2013 15:38:08 +0000 Subject: [PATCH 1238/1279] to_json test updates #420 --- tests/document/json_serialisation.py | 4 ++++ tests/queryset/queryset.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/document/json_serialisation.py b/tests/document/json_serialisation.py index dbc09d8..2b5d9a0 100644 --- a/tests/document/json_serialisation.py +++ b/tests/document/json_serialisation.py @@ -31,6 +31,10 @@ class TestJson(unittest.TestCase): doc = Doc(string="Hi", embedded_field=Embedded(string="Hi")) + doc_json = doc.to_json(sort_keys=True, separators=(',', ':')) + expected_json = """{"embedded_field":{"string":"Hi"},"string":"Hi"}""" + self.assertEqual(doc_json, expected_json) + self.assertEqual(doc, Doc.from_json(doc.to_json())) def test_json_complex(self): diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index c56b31e..0ec41fa 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3327,7 +3327,7 @@ class QuerySetTest(unittest.TestCase): Doc(string="Bye", embedded_field=Embedded(string="Bye")).save() Doc().save() - json_data = Doc.objects.to_json() + json_data = Doc.objects.to_json(sort_keys=True, separators=(',', ':')) doc_objects = list(Doc.objects) self.assertEqual(doc_objects, Doc.objects.from_json(json_data)) From 93a2adb3e6b8d4cac791659e0f3154df8a752904 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 29 Jul 2013 15:43:54 +0000 Subject: [PATCH 1239/1279] Updating changelog and authors #417 --- AUTHORS | 1 + docs/changelog.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index b8143a0..e9f6ad9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -172,3 +172,4 @@ that much better: * Alon Horev (https://github.com/alonho) * Kelvin Hammond (https://github.com/kelvinhammond) * Jatin- (https://github.com/jatin-) + * Paul Uithol (https://github.com/PaulUithol) diff --git a/docs/changelog.rst b/docs/changelog.rst index a53d6df..b9c74f8 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.4 ================ +- Fixed _delta including referenced fields when dbref=False (#417) - Allow args and kwargs to be passed through to_json (#420) Changes in 0.8.3 From 1e4d48d371e2920dd3397bb20b2f6f1456ed1566 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Mon, 29 Jul 2013 17:22:24 +0000 Subject: [PATCH 1240/1279] Don't follow references in _get_changed_fields (#422, #417) A better fix so we dont follow down a references rabbit hole. --- docs/changelog.rst | 2 +- mongoengine/base/document.py | 30 ++++++++++++++++++------------ tests/document/delta.py | 30 ++++++++++++++++-------------- 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b9c74f8..9112b2b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,7 @@ Changelog Changes in 0.8.4 ================ -- Fixed _delta including referenced fields when dbref=False (#417) +- Don't follow references in _get_changed_fields (#422, #417) - Allow args and kwargs to be passed through to_json (#420) Changes in 0.8.3 diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index f1c1d55..80111f7 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -395,6 +395,7 @@ class BaseDocument(object): """ EmbeddedDocument = _import_class("EmbeddedDocument") DynamicEmbeddedDocument = _import_class("DynamicEmbeddedDocument") + ReferenceField = _import_class("ReferenceField") _changed_fields = [] _changed_fields += getattr(self, '_changed_fields', []) @@ -405,31 +406,36 @@ class BaseDocument(object): inspected.add(self.id) for field_name in self._fields_ordered: - db_field_name = self._db_field_map.get(field_name, field_name) key = '%s.' % db_field_name - field = self._data.get(field_name, None) - if hasattr(field, 'id'): - if field.id in inspected: - continue - inspected.add(field.id) + data = self._data.get(field_name, None) + field = self._fields.get(field_name) - if (isinstance(field, (EmbeddedDocument, DynamicEmbeddedDocument)) + if hasattr(data, 'id'): + if data.id in inspected: + continue + inspected.add(data.id) + if isinstance(field, ReferenceField): + continue + elif (isinstance(data, (EmbeddedDocument, DynamicEmbeddedDocument)) and db_field_name not in _changed_fields): # Find all embedded fields that have been changed - changed = field._get_changed_fields(inspected) + changed = data._get_changed_fields(inspected) _changed_fields += ["%s%s" % (key, k) for k in changed if k] - elif (isinstance(field, (list, tuple, dict)) and + elif (isinstance(data, (list, tuple, dict)) and db_field_name not in _changed_fields): # Loop list / dict fields as they contain documents # Determine the iterator to use - if not hasattr(field, 'items'): - iterator = enumerate(field) + if not hasattr(data, 'items'): + iterator = enumerate(data) else: - iterator = field.iteritems() + iterator = data.iteritems() for index, value in iterator: if not hasattr(value, '_get_changed_fields'): continue + if (hasattr(field, 'field') and + isinstance(field.field, ReferenceField)): + continue list_key = "%s%s." % (key, index) changed = value._get_changed_fields(inspected) _changed_fields += ["%s%s" % (list_key, k) diff --git a/tests/document/delta.py b/tests/document/delta.py index c6efc02..b4749f3 100644 --- a/tests/document/delta.py +++ b/tests/document/delta.py @@ -328,14 +328,9 @@ class DeltaTest(unittest.TestCase): Person.drop_collection() Organization.drop_collection() - person = Person(name="owner") - person.save() - - employee = Person(name="employee") - employee.save() - - organization = Organization(name="company") - organization.save() + person = Person(name="owner").save() + employee = Person(name="employee").save() + organization = Organization(name="company").save() person.owns.append(organization) organization.owner = person @@ -692,25 +687,32 @@ class DeltaTest(unittest.TestCase): person, organization, employee = self.circular_reference_deltas_2(Document, Document, True) employee.name = 'test' - self.assertEqual(organization._get_changed_fields(), ['employees.0.name']) + self.assertEqual(organization._get_changed_fields(), []) updates, removals = organization._delta() self.assertEqual({}, removals) - self.assertTrue('employees.0' in updates) + self.assertEqual({}, updates) - organization.save() + organization.employees.append(person) + updates, removals = organization._delta() + self.assertEqual({}, removals) + self.assertTrue('employees' in updates) def test_delta_with_dbref_false(self): person, organization, employee = self.circular_reference_deltas_2(Document, Document, False) employee.name = 'test' - self.assertEqual(organization._get_changed_fields(), ['employees.0.name']) + self.assertEqual(organization._get_changed_fields(), []) updates, removals = organization._delta() self.assertEqual({}, removals) - self.assertTrue('employees.0' in updates) + self.assertEqual({}, updates) + + organization.employees.append(person) + updates, removals = organization._delta() + self.assertEqual({}, removals) + self.assertTrue('employees' in updates) - organization.save() if __name__ == '__main__': unittest.main() From 6efd6faa3f5a467012fa4ee128889104ca0ba6f7 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 30 Jul 2013 10:30:16 +0000 Subject: [PATCH 1241/1279] Fixed QuerySetNoCache.count() caching (#410) --- docs/changelog.rst | 1 + mongoengine/queryset/base.py | 10 ++-------- mongoengine/queryset/queryset.py | 15 +++++++++++++++ tests/queryset/queryset.py | 21 +++++++++++++++++++++ 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 9112b2b..e2f07bc 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.4 ================ +- Fixed QuerySetNoCache.count() caching (#410) - Don't follow references in _get_changed_fields (#422, #417) - Allow args and kwargs to be passed through to_json (#420) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index e88feb3..a6ba49b 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -60,7 +60,6 @@ class BaseQuerySet(object): self._none = False self._as_pymongo = False self._as_pymongo_coerce = False - self._len = None # If inheritance is allowed, only return instances and instances of # subclasses of the class being used @@ -331,14 +330,9 @@ class BaseQuerySet(object): :meth:`skip` that has been applied to this cursor into account when getting the count """ - if self._limit == 0: + if self._limit == 0 and with_limit_and_skip: return 0 - if with_limit_and_skip and self._len is not None: - return self._len - count = self._cursor.count(with_limit_and_skip=with_limit_and_skip) - if with_limit_and_skip: - self._len = count - return count + return self._cursor.count(with_limit_and_skip=with_limit_and_skip) def delete(self, write_concern=None, _from_doc_delete=False): """Delete the documents matched by the query. diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 9cfb1b6..1437e76 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -94,6 +94,21 @@ class QuerySet(BaseQuerySet): except StopIteration: self._has_more = False + def count(self, with_limit_and_skip=True): + """Count the selected elements in the query. + + :param with_limit_and_skip (optional): take any :meth:`limit` or + :meth:`skip` that has been applied to this cursor into account when + getting the count + """ + if with_limit_and_skip is False: + return super(QuerySet, self).count(with_limit_and_skip) + + if self._len is None: + self._len = super(QuerySet, self).count(with_limit_and_skip) + + return self._len + def no_cache(self): """Convert to a non_caching queryset diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 0ec41fa..9c04c0b 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3483,6 +3483,27 @@ class QuerySetTest(unittest.TestCase): people.count() # count is cached self.assertEqual(q, 1) + def test_no_cached_queryset(self): + class Person(Document): + name = StringField() + + Person.drop_collection() + for i in xrange(100): + Person(name="No: %s" % i).save() + + with query_counter() as q: + self.assertEqual(q, 0) + people = Person.objects.no_cache() + + [x for x in people] + self.assertEqual(q, 1) + + list(people) + self.assertEqual(q, 2) + + people.count() + self.assertEqual(q, 3) + def test_cache_not_cloned(self): class User(Document): From e98c5e10bc0eefacc11aa6955db9635fabc549de Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 30 Jul 2013 10:49:08 +0000 Subject: [PATCH 1242/1279] Fixed dereference threading issue in ComplexField.__get__ (#412) --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e2f07bc..7cb1456 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.4 ================ +- Fixed dereference threading issue in ComplexField.__get__ (#412) - Fixed QuerySetNoCache.count() caching (#410) - Don't follow references in _get_changed_fields (#422, #417) - Allow args and kwargs to be passed through to_json (#420) From dc310b99f94ddf365369340c074eb2a35d68c685 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 30 Jul 2013 10:54:04 +0000 Subject: [PATCH 1243/1279] Updated docs about TTL indexes and signals (#413) --- docs/guide/defining-documents.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index a50450e..bc78a66 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -558,6 +558,11 @@ documentation for more information. A common usecase might be session data:: ] } +.. warning:: TTL indexes happen on the MongoDB server and not in the application + code, therefore no signals will be fired on document deletion. + If you need signals to be fired on deletion, then you must handle the + deletion of Documents in your application code. + Comparing Indexes ----------------- From 0c437879961008281018e572369a61d58da909d6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 30 Jul 2013 11:43:52 +0000 Subject: [PATCH 1244/1279] Fixed indexing - turn off _cls (#414) --- docs/changelog.rst | 1 + docs/guide/defining-documents.rst | 31 +++++++++++++++++++++++++++++++ mongoengine/base/document.py | 6 ++++-- mongoengine/document.py | 2 ++ tests/document/indexes.py | 19 +++++++++++++++++++ 5 files changed, 57 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 7cb1456..3569132 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.4 ================ +- Fixed indexing - turn off _cls (#414) - Fixed dereference threading issue in ComplexField.__get__ (#412) - Fixed QuerySetNoCache.count() caching (#410) - Don't follow references in _get_changed_fields (#422, #417) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index bc78a66..407fbda 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -442,6 +442,8 @@ The following example shows a :class:`Log` document that will be limited to ip_address = StringField() meta = {'max_documents': 1000, 'max_size': 2000000} +.. defining-indexes_ + Indexes ======= @@ -485,6 +487,35 @@ If a dictionary is passed then the following options are available: Inheritance adds extra fields indices see: :ref:`document-inheritance`. +Global index default options +---------------------------- + +There are a few top level defaults for all indexes that can be set:: + + class Page(Document): + title = StringField() + rating = StringField() + meta = { + 'index_options': {}, + 'index_background': True, + 'index_drop_dups': True, + 'index_cls': False + } + + +:attr:`index_options` (Optional) + Set any default index options - see the `full options list `_ + +:attr:`index_background` (Optional) + Set the default value for if an index should be indexed in the background + +:attr:`index_drop_dups` (Optional) + Set the default value for if an index should drop duplicates + +:attr:`index_cls` (Optional) + A way to turn off a specific index for _cls. + + Compound Indexes and Indexing sub documents ------------------------------------------- diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 80111f7..b9c07cf 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -629,8 +629,10 @@ class BaseDocument(object): # Check to see if we need to include _cls allow_inheritance = cls._meta.get('allow_inheritance', ALLOW_INHERITANCE) - include_cls = allow_inheritance and not spec.get('sparse', False) - + include_cls = (allow_inheritance and not spec.get('sparse', False) and + spec.get('cls', True)) + if "cls" in spec: + spec.pop('cls') for key in spec['fields']: # If inherited spec continue if isinstance(key, (list, tuple)): diff --git a/mongoengine/document.py b/mongoengine/document.py index e331aa1..2f3a92a 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -536,6 +536,8 @@ class Document(BaseDocument): def ensure_indexes(cls): """Checks the document meta data and ensures all the indexes exist. + Global defaults can be set in the meta - see :doc:`guide/defining-documents` + .. note:: You can disable automatic index creation by setting `auto_create_index` to False in the documents meta data """ diff --git a/tests/document/indexes.py b/tests/document/indexes.py index 04d5632..ccf8463 100644 --- a/tests/document/indexes.py +++ b/tests/document/indexes.py @@ -156,6 +156,25 @@ class IndexesTest(unittest.TestCase): self.assertEqual([{'fields': [('_cls', 1), ('title', 1)]}], A._meta['index_specs']) + def test_index_no_cls(self): + """Ensure index specs are inhertited correctly""" + + class A(Document): + title = StringField() + meta = { + 'indexes': [ + {'fields': ('title',), 'cls': False}, + ], + 'allow_inheritance': True, + 'index_cls': False + } + + self.assertEqual([('title', 1)], A._meta['index_specs'][0]['fields']) + A._get_collection().drop_indexes() + A.ensure_indexes() + info = A._get_collection().index_information() + self.assertEqual(len(info.keys()), 2) + def test_build_index_spec_is_not_destructive(self): class MyDoc(Document): From 5e70e1bcb28c60520964e683b60ed1c60ef0f429 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 30 Jul 2013 13:17:38 +0000 Subject: [PATCH 1245/1279] Update transform to handle docs erroneously passed to unset (#416) --- docs/changelog.rst | 1 + mongoengine/queryset/transform.py | 4 +++- tests/queryset/transform.py | 25 +++++++++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 3569132..8199e03 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.4 ================ +- Update transform to handle docs erroneously passed to unset (#416) - Fixed indexing - turn off _cls (#414) - Fixed dereference threading issue in ComplexField.__get__ (#412) - Fixed QuerySetNoCache.count() caching (#410) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 352774f..e0a7d3c 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -203,11 +203,13 @@ def update(_doc_cls=None, **update): value = field.prepare_query_value(op, value) elif op in ('pushAll', 'pullAll'): value = [field.prepare_query_value(op, v) for v in value] - elif op == 'addToSet': + elif op in ('addToSet', 'setOnInsert'): if isinstance(value, (list, tuple, set)): value = [field.prepare_query_value(op, v) for v in value] elif field.required or value is not None: value = field.prepare_query_value(op, value) + elif op == "unset": + value = 1 if match: match = '$' + match diff --git a/tests/queryset/transform.py b/tests/queryset/transform.py index 7886965..d2e8b78 100644 --- a/tests/queryset/transform.py +++ b/tests/queryset/transform.py @@ -31,6 +31,31 @@ class TransformTest(unittest.TestCase): self.assertEqual(transform.query(name__exists=True), {'name': {'$exists': True}}) + def test_transform_update(self): + class DicDoc(Document): + dictField = DictField() + + class Doc(Document): + pass + + DicDoc.drop_collection() + Doc.drop_collection() + + doc = Doc().save() + dic_doc = DicDoc().save() + + for k, v in (("set", "$set"), ("set_on_insert", "$setOnInsert"), ("push", "$push")): + update = transform.update(DicDoc, **{"%s__dictField__test" % k: doc}) + self.assertTrue(isinstance(update[v]["dictField.test"], dict)) + + # Update special cases + update = transform.update(DicDoc, unset__dictField__test=doc) + self.assertEqual(update["$unset"]["dictField.test"], 1) + + update = transform.update(DicDoc, pull__dictField__test=doc) + self.assertTrue(isinstance(update["$pull"]["dictField"]["test"], dict)) + + def test_query_field_name(self): """Ensure that the correct field name is used when querying. """ From a57d9a9303a8487cbc4417dceb5c61012f9a88b9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 30 Jul 2013 13:28:05 +0000 Subject: [PATCH 1246/1279] Added regression test (#418) --- tests/queryset/field_list.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/queryset/field_list.py b/tests/queryset/field_list.py index 2bdfce1..f981c12 100644 --- a/tests/queryset/field_list.py +++ b/tests/queryset/field_list.py @@ -162,6 +162,10 @@ class OnlyExcludeAllTest(unittest.TestCase): self.assertEqual(obj.name, person.name) self.assertEqual(obj.age, person.age) + obj = Person.objects.only(*('id', 'name',)).get() + self.assertEqual(obj.name, person.name) + self.assertEqual(obj.age, None) + # Check polymorphism still works class Employee(self.Person): salary = IntField(db_field='wage') From b4777f7f4f1e2ee5d7cc8e1102421d87540a69f5 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 30 Jul 2013 15:04:52 +0000 Subject: [PATCH 1247/1279] Fix test --- tests/queryset/field_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queryset/field_list.py b/tests/queryset/field_list.py index f981c12..a18e167 100644 --- a/tests/queryset/field_list.py +++ b/tests/queryset/field_list.py @@ -162,7 +162,7 @@ class OnlyExcludeAllTest(unittest.TestCase): self.assertEqual(obj.name, person.name) self.assertEqual(obj.age, person.age) - obj = Person.objects.only(*('id', 'name',)).get() + obj = self.Person.objects.only(*('id', 'name',)).get() self.assertEqual(obj.name, person.name) self.assertEqual(obj.age, None) From c17f94422fd6ac388dbdf1bc60139525bfcbd018 Mon Sep 17 00:00:00 2001 From: Nicolas Cortot Date: Tue, 30 Jul 2013 20:43:21 +0200 Subject: [PATCH 1248/1279] Add get_user_document and improve mongo_auth module * Added a get_user_document() methot to access the actual Document class used for authentication. * Clarified the docstring on MongoUser to prevent its use when the user Document class should be used. * Removed the masking of exceptions when loading the user document class. --- mongoengine/django/mongo_auth/models.py | 42 ++++++++++++++++++------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/mongoengine/django/mongo_auth/models.py b/mongoengine/django/mongo_auth/models.py index 3529d8e..7179718 100644 --- a/mongoengine/django/mongo_auth/models.py +++ b/mongoengine/django/mongo_auth/models.py @@ -6,10 +6,29 @@ from django.utils.importlib import import_module from django.utils.translation import ugettext_lazy as _ +__all__ = ( + 'get_user_document', +) + + MONGOENGINE_USER_DOCUMENT = getattr( settings, 'MONGOENGINE_USER_DOCUMENT', 'mongoengine.django.auth.User') +def get_user_document(self): + """Get the user docuemnt class user for authentcation. + + This is the class defined in settings.MONGOENGINE_USER_DOCUMENT, which + defaults to `mongoengine.django.auth.User`. + + """ + + name = MONGOENGINE_USER_DOCUMENT + dot = name.rindex('.') + module = import_module(name[:dot]) + return getattr(module, name[dot + 1:]) + + class MongoUserManager(UserManager): """A User manager wich allows the use of MongoEngine documents in Django. @@ -44,7 +63,7 @@ class MongoUserManager(UserManager): def contribute_to_class(self, model, name): super(MongoUserManager, self).contribute_to_class(model, name) self.dj_model = self.model - self.model = self._get_user_document() + self.model = get_user_document() self.dj_model.USERNAME_FIELD = self.model.USERNAME_FIELD username = models.CharField(_('username'), max_length=30, unique=True) @@ -55,16 +74,6 @@ class MongoUserManager(UserManager): field = models.CharField(_(name), max_length=30) field.contribute_to_class(self.dj_model, name) - def _get_user_document(self): - try: - name = MONGOENGINE_USER_DOCUMENT - dot = name.rindex('.') - module = import_module(name[:dot]) - return getattr(module, name[dot + 1:]) - except ImportError: - raise ImproperlyConfigured("Error importing %s, please check " - "settings.MONGOENGINE_USER_DOCUMENT" - % name) def get(self, *args, **kwargs): try: @@ -85,5 +94,14 @@ class MongoUserManager(UserManager): class MongoUser(models.Model): - objects = MongoUserManager() + """"Dummy user model for Django. + MongoUser is used to replace Django's UserManager with MongoUserManager. + The actual user document class is mongoengine.django.auth.User or any + other document class specified in MONGOENGINE_USER_DOCUMENT. + + To get the user document class, use `get_user_document()`. + + """ + + objects = MongoUserManager() From a69db231cc7696bb78795caeef2e4d2b3c034148 Mon Sep 17 00:00:00 2001 From: Paul Date: Wed, 31 Jul 2013 11:26:23 +1000 Subject: [PATCH 1249/1279] Pretty-print GridFSProxy objects --- mongoengine/fields.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 47554e0..39a6caa 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1082,6 +1082,10 @@ class GridFSProxy(object): def __repr__(self): return '<%s: %s>' % (self.__class__.__name__, self.grid_id) + + def __unicode__(self): + name = getattr(self.get(), 'filename', self.grid_id) if self.get() else '(no file)' + return '<%s: %s>' % (self.__class__.__name__, name) def __eq__(self, other): if isinstance(other, GridFSProxy): From d8ffa843a9bfbe7c6b645bbab4f0fcd61dd6af43 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 31 Jul 2013 09:29:41 +0000 Subject: [PATCH 1250/1279] Added str representation of GridFSProxy (#424) --- docs/changelog.rst | 1 + mongoengine/fields.py | 4 ++-- tests/fields/file_tests.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 8199e03..49d79fe 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.4 ================ +- Added str representation of GridFSProxy (#424) - Update transform to handle docs erroneously passed to unset (#416) - Fixed indexing - turn off _cls (#414) - Fixed dereference threading issue in ComplexField.__get__ (#412) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 39a6caa..826e125 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -1082,8 +1082,8 @@ class GridFSProxy(object): def __repr__(self): return '<%s: %s>' % (self.__class__.__name__, self.grid_id) - - def __unicode__(self): + + def __str__(self): name = getattr(self.get(), 'filename', self.grid_id) if self.get() else '(no file)' return '<%s: %s>' % (self.__class__.__name__, name) diff --git a/tests/fields/file_tests.py b/tests/fields/file_tests.py index d044500..ba601de 100644 --- a/tests/fields/file_tests.py +++ b/tests/fields/file_tests.py @@ -53,11 +53,12 @@ class FileTest(unittest.TestCase): content_type = 'text/plain' putfile = PutFile() - putfile.the_file.put(text, content_type=content_type) + putfile.the_file.put(text, content_type=content_type, filename="hello") putfile.save() result = PutFile.objects.first() self.assertTrue(putfile == result) + self.assertEqual("%s" % result.the_file, "") self.assertEqual(result.the_file.read(), text) self.assertEqual(result.the_file.content_type, content_type) result.the_file.delete() # Remove file from GridFS From 7431b1f123a84adcc3e7ee9ca86ef18b52d01c5d Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 31 Jul 2013 09:31:04 +0000 Subject: [PATCH 1251/1279] Updated AUTHORS (#424) --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index b779da2..938c5c0 100644 --- a/AUTHORS +++ b/AUTHORS @@ -174,3 +174,4 @@ that much better: * Jatin- (https://github.com/jatin-) * Paul Uithol (https://github.com/PaulUithol) * Thom Knowles (https://github.com/fleat) + * Paul (https://github.com/squamous) \ No newline at end of file From 719bb53c3a01a50c325696f87aaa3fb08256b22f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 31 Jul 2013 09:44:15 +0000 Subject: [PATCH 1252/1279] Updated changelog (#423) --- docs/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index 49d79fe..d93406d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.4 ================ +- Added get_user_document and improve mongo_auth module (#423) - Added str representation of GridFSProxy (#424) - Update transform to handle docs erroneously passed to unset (#416) - Fixed indexing - turn off _cls (#414) From b3f462a39d1f43a7bbc2b887757f340d8fea16cf Mon Sep 17 00:00:00 2001 From: Laurent Payot Date: Thu, 1 Aug 2013 03:51:10 +0200 Subject: [PATCH 1253/1279] updated docs for django shortcuts get_object_or_404 and get_list_or_404 --- docs/django.rst | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/docs/django.rst b/docs/django.rst index da15188..62d4dd4 100644 --- a/docs/django.rst +++ b/docs/django.rst @@ -45,7 +45,7 @@ The :mod:`~mongoengine.django.auth` module also contains a Custom User model ================= Django 1.5 introduced `Custom user Models -` +`_ which can be used as an alternative to the MongoEngine authentication backend. The main advantage of this option is that other components relying on @@ -74,7 +74,7 @@ An additional ``MONGOENGINE_USER_DOCUMENT`` setting enables you to replace the The custom :class:`User` must be a :class:`~mongoengine.Document` class, but otherwise has the same requirements as a standard custom user model, as specified in the `Django Documentation -`. +`_. In particular, the custom class must define :attr:`USERNAME_FIELD` and :attr:`REQUIRED_FIELDS` attributes. @@ -128,7 +128,7 @@ appended to the filename until the generated filename doesn't exist. The >>> fs.listdir() ([], [u'hello.txt']) -All files will be saved and retrieved in GridFS via the :class::`FileDocument` +All files will be saved and retrieved in GridFS via the :class:`FileDocument` document, allowing easy access to the files without the GridFSStorage backend.:: @@ -137,3 +137,36 @@ backend.:: [] .. versionadded:: 0.4 + +Shortcuts +========= +Inspired by the `Django shortcut get_object_or_404 +`_, +the :func:`~mongoengine.django.shortcuts.get_document_or_404` method returns +a document or raises an Http404 exception if the document does not exist:: + + from mongoengine.django.shortcuts import get_document_or_404 + + admin_user = get_document_or_404(User, username='root') + +The first argument may be a Document or QuerySet object. All other passed arguments +and keyword arguments are used in the query:: + + foo_email = get_document_or_404(User.objects.only('email'), username='foo', is_active=True).email + +.. note:: Like with :func:`get`, a MultipleObjectsReturned will be raised if more than one + object is found. + + +Also inspired by the `Django shortcut get_list_or_404 +`_, +the :func:`~mongoengine.django.shortcuts.get_list_or_404` method returns a list of +documents or raises an Http404 exception if the list is empty:: + + from mongoengine.django.shortcuts import get_list_or_404 + + active_users = get_list_or_404(User, is_active=True) + +The first argument may be a Document or QuerySet object. All other passed +arguments and keyword arguments are used to filter the query. + From a448c9aebf573d13c7876fc5380e2519aabc302d Mon Sep 17 00:00:00 2001 From: devoto13 Date: Thu, 1 Aug 2013 17:54:41 +0300 Subject: [PATCH 1254/1279] removed duplicated method --- docs/guide/querying.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 127e479..f50985b 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -497,7 +497,6 @@ that you may use with these methods: * ``unset`` -- delete a particular value (since MongoDB v1.3+) * ``inc`` -- increment a value by a given amount * ``dec`` -- decrement a value by a given amount -* ``pop`` -- remove the last item from a list * ``push`` -- append a value to a list * ``push_all`` -- append several values to a list * ``pop`` -- remove the first or last element of a list From b98b06ff79f54318b8329822bcf081cb8953a1da Mon Sep 17 00:00:00 2001 From: Nicolas Cortot Date: Sun, 4 Aug 2013 11:01:09 +0200 Subject: [PATCH 1255/1279] Fix an error in get_user_document --- mongoengine/django/mongo_auth/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/django/mongo_auth/models.py b/mongoengine/django/mongo_auth/models.py index 7179718..35960c9 100644 --- a/mongoengine/django/mongo_auth/models.py +++ b/mongoengine/django/mongo_auth/models.py @@ -15,7 +15,7 @@ MONGOENGINE_USER_DOCUMENT = getattr( settings, 'MONGOENGINE_USER_DOCUMENT', 'mongoengine.django.auth.User') -def get_user_document(self): +def get_user_document(): """Get the user docuemnt class user for authentcation. This is the class defined in settings.MONGOENGINE_USER_DOCUMENT, which From 40b0a15b350cba12ebe37e595352d10c7628a2d3 Mon Sep 17 00:00:00 2001 From: Nicolas Cortot Date: Sun, 4 Aug 2013 11:03:34 +0200 Subject: [PATCH 1256/1279] Fixing typos --- mongoengine/django/mongo_auth/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/django/mongo_auth/models.py b/mongoengine/django/mongo_auth/models.py index 35960c9..d4947a2 100644 --- a/mongoengine/django/mongo_auth/models.py +++ b/mongoengine/django/mongo_auth/models.py @@ -16,7 +16,7 @@ MONGOENGINE_USER_DOCUMENT = getattr( def get_user_document(): - """Get the user docuemnt class user for authentcation. + """Get the user document class used for authentication. This is the class defined in settings.MONGOENGINE_USER_DOCUMENT, which defaults to `mongoengine.django.auth.User`. From a0d255369aef155fbb0cdd139542cbf5719a4046 Mon Sep 17 00:00:00 2001 From: Nicolas Cortot Date: Sun, 4 Aug 2013 11:08:11 +0200 Subject: [PATCH 1257/1279] Add a test case for get_user_document --- tests/test_django.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_django.py b/tests/test_django.py index 63e3245..8fe0da3 100644 --- a/tests/test_django.py +++ b/tests/test_django.py @@ -22,7 +22,11 @@ try: try: from django.contrib.auth import authenticate, get_user_model from mongoengine.django.auth import User - from mongoengine.django.mongo_auth.models import MongoUser, MongoUserManager + from mongoengine.django.mongo_auth.models import ( + MongoUser, + MongoUserManager, + get_user_document, + ) DJ15 = True except Exception: DJ15 = False @@ -270,9 +274,12 @@ class MongoAuthTest(unittest.TestCase): User.drop_collection() super(MongoAuthTest, self).setUp() - def test_user_model(self): + def test_get_user_model(self): self.assertEqual(get_user_model(), MongoUser) + def test_get_user_document(self): + self.assertEqual(get_user_document(), User) + def test_user_manager(self): manager = get_user_model()._default_manager self.assertTrue(isinstance(manager, MongoUserManager)) From 5bcc4546783e8ee7200ad7669ed14a40ea67376e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 7 Aug 2013 09:07:57 +0000 Subject: [PATCH 1258/1279] Handle dynamic fieldnames that look like digits (#434) --- docs/changelog.rst | 1 + mongoengine/base/document.py | 2 +- tests/queryset/queryset.py | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index d93406d..b877d4d 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.4 ================ +- Handle dynamic fieldnames that look like digits (#434) - Added get_user_document and improve mongo_auth module (#423) - Added str representation of GridFSProxy (#424) - Update transform to handle docs erroneously passed to unset (#416) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index b9c07cf..cea2f09 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -762,7 +762,7 @@ class BaseDocument(object): for field_name in parts: # Handle ListField indexing: - if field_name.isdigit(): + if field_name.isdigit() and hasattr(field, 'field'): new_field = field.field fields.append(field_name) continue diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 9c04c0b..75708ca 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3299,6 +3299,13 @@ class QuerySetTest(unittest.TestCase): Test.objects(test='foo').update_one(upsert=True, set__test='foo') self.assertTrue('_cls' in Test._collection.find_one()) + def test_update_upsert_looks_like_a_digit(self): + class MyDoc(DynamicDocument): + pass + MyDoc.drop_collection() + self.assertEqual(1, MyDoc.objects.update_one(upsert=True, inc__47=1)) + self.assertEqual(MyDoc.objects.get()['47'], 1) + def test_read_preference(self): class Bar(Document): pass From f30208f3453dd7999987f07cf383e6fd9fb546ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cort=C3=A8s?= Date: Mon, 12 Aug 2013 19:12:53 +0200 Subject: [PATCH 1259/1279] Fix the ._get_db() attribute after a Document.switch_db() Without this patch, I've got: ``` myobj._get_db() > ``` I need to `myobj._get_db()()` to get the database. I felt this like a bug. regards, --- mongoengine/document.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index 2f3a92a..1bbd7b7 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -400,7 +400,7 @@ class Document(BaseDocument): """ with switch_db(self.__class__, db_alias) as cls: collection = cls._get_collection() - db = cls._get_db + db = cls._get_db() self._get_collection = lambda: collection self._get_db = lambda: db self._collection = collection From 70b320633ff801ad0885d395e7f6caa64587dbb1 Mon Sep 17 00:00:00 2001 From: crazyzubr Date: Thu, 15 Aug 2013 19:32:13 +0800 Subject: [PATCH 1260/1279] permit the establishment of a field with the name of size or other Example: # model class Example(Document): size = ReferenceField(Size, verbose_name='Size') # query examples = Example.objects(size=instance_size) # caused an error """ File ".../mongoengine/queryset/transform.py", line 50, in query if parts[-1] == 'not': IndexError: list index out of range """ --- mongoengine/queryset/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index e0a7d3c..d82f33d 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -43,11 +43,11 @@ def query(_doc_cls=None, _field_operation=False, **query): parts = [part for part in parts if not part.isdigit()] # Check for an operator and transform to mongo-style if there is op = None - if parts[-1] in MATCH_OPERATORS: + if len(parts) > 1 and parts[-1] in MATCH_OPERATORS: op = parts.pop() negate = False - if parts[-1] == 'not': + if len(parts) > 1 and parts[-1] == 'not': parts.pop() negate = True From d07a9d2ef8b63fea934ef658678823b4d6073338 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 20 Aug 2013 08:30:20 +0000 Subject: [PATCH 1261/1279] Dynamic Fields store and recompose Embedded Documents / Documents correctly (#449) --- docs/changelog.rst | 1 + mongoengine/fields.py | 13 ++++++++++++- tests/fields/fields.py | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b877d4d..fa27d1a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.4 ================ +- Dynamic Fields store and recompose Embedded Documents / Documents correctly (#449) - Handle dynamic fieldnames that look like digits (#434) - Added get_user_document and improve mongo_auth module (#423) - Added str representation of GridFSProxy (#424) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index 826e125..c1fc1a7 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -624,7 +624,9 @@ class DynamicField(BaseField): cls = value.__class__ val = value.to_mongo() # If we its a document thats not inherited add _cls - if (isinstance(value, (Document, EmbeddedDocument))): + if (isinstance(value, Document)): + val = {"_ref": value.to_dbref(), "_cls": cls.__name__} + if (isinstance(value, EmbeddedDocument)): val['_cls'] = cls.__name__ return val @@ -645,6 +647,15 @@ class DynamicField(BaseField): value = [v for k, v in sorted(data.iteritems(), key=itemgetter(0))] return value + def to_python(self, value): + if isinstance(value, dict) and '_cls' in value: + doc_cls = get_document(value['_cls']) + if '_ref' in value: + value = doc_cls._get_db().dereference(value['_ref']) + return doc_cls._from_son(value) + + return super(DynamicField, self).to_python(value) + def lookup_member(self, member_name): return member_name diff --git a/tests/fields/fields.py b/tests/fields/fields.py index b3d8d52..8791781 100644 --- a/tests/fields/fields.py +++ b/tests/fields/fields.py @@ -2506,5 +2506,46 @@ class FieldTest(unittest.TestCase): self.assertTrue(tuple(x.items[0]) in tuples) self.assertTrue(x.items[0] in tuples) + def test_dynamic_fields_class(self): + + class Doc2(Document): + field_1 = StringField(db_field='f') + + class Doc(Document): + my_id = IntField(required=True, unique=True, primary_key=True) + embed_me = DynamicField(db_field='e') + field_x = StringField(db_field='x') + + Doc.drop_collection() + Doc2.drop_collection() + + doc2 = Doc2(field_1="hello") + doc = Doc(my_id=1, embed_me=doc2, field_x="x") + self.assertRaises(OperationError, doc.save) + + doc2.save() + doc.save() + + doc = Doc.objects.get() + self.assertEqual(doc.embed_me.field_1, "hello") + + def test_dynamic_fields_embedded_class(self): + + class Embed(EmbeddedDocument): + field_1 = StringField(db_field='f') + + class Doc(Document): + my_id = IntField(required=True, unique=True, primary_key=True) + embed_me = DynamicField(db_field='e') + field_x = StringField(db_field='x') + + Doc.drop_collection() + + Doc(my_id=1, embed_me=Embed(field_1="hello"), field_x="x").save() + + doc = Doc.objects.get() + self.assertEqual(doc.embed_me.field_1, "hello") + + if __name__ == '__main__': unittest.main() From ee7666ddea20b7a261c7dec021bbe595f015a970 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 20 Aug 2013 08:31:56 +0000 Subject: [PATCH 1262/1279] Update AUTHORS and Changelog (#441) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 938c5c0..2a2eb36 100644 --- a/AUTHORS +++ b/AUTHORS @@ -174,4 +174,5 @@ that much better: * Jatin- (https://github.com/jatin-) * Paul Uithol (https://github.com/PaulUithol) * Thom Knowles (https://github.com/fleat) - * Paul (https://github.com/squamous) \ No newline at end of file + * Paul (https://github.com/squamous) + * Olivier Cortès (https://github.com/Karmak23) \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index fa27d1a..b027562 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.4 ================ +- Fixed ._get_db() attribute after a Document.switch_db() (#441) - Dynamic Fields store and recompose Embedded Documents / Documents correctly (#449) - Handle dynamic fieldnames that look like digits (#434) - Added get_user_document and improve mongo_auth module (#423) From 67baf465f4ebd1024d4730dc206900bb1bd3a1a2 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 20 Aug 2013 09:14:58 +0000 Subject: [PATCH 1263/1279] Fixed slice when using inheritance causing fields to be excluded (#437) --- docs/changelog.rst | 1 + mongoengine/queryset/field_list.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index b027562..26a0716 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.4 ================ +- Fixed slice when using inheritance causing fields to be excluded (#437) - Fixed ._get_db() attribute after a Document.switch_db() (#441) - Dynamic Fields store and recompose Embedded Documents / Documents correctly (#449) - Handle dynamic fieldnames that look like digits (#434) diff --git a/mongoengine/queryset/field_list.py b/mongoengine/queryset/field_list.py index 73d3cc2..140a71e 100644 --- a/mongoengine/queryset/field_list.py +++ b/mongoengine/queryset/field_list.py @@ -55,7 +55,8 @@ class QueryFieldList(object): if self.always_include: if self.value is self.ONLY and self.fields: - self.fields = self.fields.union(self.always_include) + if sorted(self.slice.keys()) != sorted(self.fields): + self.fields = self.fields.union(self.always_include) else: self.fields -= self.always_include From 49f5b4fa5cf9a86ce548123478bcdec94a0698c6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 20 Aug 2013 09:45:00 +0000 Subject: [PATCH 1264/1279] Fix Queryset docs (#448) --- docs/apireference.rst | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/apireference.rst b/docs/apireference.rst index 774d3b8..9057de5 100644 --- a/docs/apireference.rst +++ b/docs/apireference.rst @@ -44,17 +44,21 @@ Context Managers Querying ======== -.. autoclass:: mongoengine.queryset.QuerySet - :members: +.. automodule:: mongoengine.queryset + :synopsis: Queryset level operations - .. automethod:: mongoengine.queryset.QuerySet.__call__ + .. autoclass:: mongoengine.queryset.QuerySet + :members: + :inherited-members: -.. autoclass:: mongoengine.queryset.QuerySetNoCache - :members: + .. automethod:: QuerySet.__call__ - .. automethod:: mongoengine.queryset.QuerySetNoCache.__call__ + .. autoclass:: mongoengine.queryset.QuerySetNoCache + :members: -.. autofunction:: mongoengine.queryset.queryset_manager + .. automethod:: mongoengine.queryset.QuerySetNoCache.__call__ + + .. autofunction:: mongoengine.queryset.queryset_manager Fields ====== From 2cd722d751438141d1bcfff5824f49496f2ffddd Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 20 Aug 2013 10:20:05 +0000 Subject: [PATCH 1265/1279] Updated setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f6b3c1b..6bd778b 100644 --- a/setup.py +++ b/setup.py @@ -51,13 +51,13 @@ CLASSIFIERS = [ extra_opts = {} if sys.version_info[0] == 3: extra_opts['use_2to3'] = True - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6', 'django>=1.5.1'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2>=2.6', 'django>=1.5.1'] extra_opts['packages'] = find_packages(exclude=('tests',)) if "test" in sys.argv or "nosetests" in sys.argv: extra_opts['packages'].append("tests") extra_opts['package_data'] = {"tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"]} else: - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2==2.6', 'python-dateutil'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2>=2.6', 'python-dateutil'] extra_opts['packages'] = find_packages(exclude=('tests',)) setup(name='mongoengine', From 661398d8914bda0821091b7734c75b7b74c2566f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 20 Aug 2013 10:22:06 +0000 Subject: [PATCH 1266/1279] Fixed dereference issue with embedded listfield referencefields (#439) --- docs/changelog.rst | 1 + mongoengine/dereference.py | 5 +++-- tests/test_dereference.py | 24 ++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 26a0716..74e2e50 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.4 ================ +- Fixed dereference issue with embedded listfield referencefields (#439) - Fixed slice when using inheritance causing fields to be excluded (#437) - Fixed ._get_db() attribute after a Document.switch_db() (#441) - Dynamic Fields store and recompose Embedded Documents / Documents correctly (#449) diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py index e5e8886..ceda403 100644 --- a/mongoengine/dereference.py +++ b/mongoengine/dereference.py @@ -4,7 +4,7 @@ from base import (BaseDict, BaseList, TopLevelDocumentMetaclass, get_document) from fields import (ReferenceField, ListField, DictField, MapField) from connection import get_db from queryset import QuerySet -from document import Document +from document import Document, EmbeddedDocument class DeReference(object): @@ -33,7 +33,8 @@ class DeReference(object): self.max_depth = max_depth doc_type = None - if instance and isinstance(instance, (Document, TopLevelDocumentMetaclass)): + if instance and isinstance(instance, (Document, EmbeddedDocument, + TopLevelDocumentMetaclass)): doc_type = instance._fields.get(name) if hasattr(doc_type, 'field'): doc_type = doc_type.field diff --git a/tests/test_dereference.py b/tests/test_dereference.py index db9868a..6f2664a 100644 --- a/tests/test_dereference.py +++ b/tests/test_dereference.py @@ -1171,6 +1171,30 @@ class FieldTest(unittest.TestCase): self.assertEqual(2, len([brand for bg in brand_groups for brand in bg.brands])) + def test_dereferencing_embedded_listfield_referencefield(self): + class Tag(Document): + meta = {'collection': 'tags'} + name = StringField() + + class Post(EmbeddedDocument): + body = StringField() + tags = ListField(ReferenceField("Tag", dbref=True)) + + class Page(Document): + meta = {'collection': 'pages'} + tags = ListField(ReferenceField("Tag", dbref=True)) + posts = ListField(EmbeddedDocumentField(Post)) + + Tag.drop_collection() + Page.drop_collection() + + tag = Tag(name='test').save() + post = Post(body='test body', tags=[tag]) + Page(tags=[tag], posts=[post]).save() + + page = Page.objects.first() + self.assertEqual(page.tags[0], page.posts[0].tags[0]) + if __name__ == '__main__': unittest.main() From 29c887f30b0f7db13d30c920d29d2b4f2f490047 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 20 Aug 2013 12:21:20 +0000 Subject: [PATCH 1267/1279] Updated field filter logic - can now exclude subclass fields (#443) --- docs/changelog.rst | 1 + mongoengine/queryset/base.py | 31 ++++++++++++++++++++++++++----- tests/queryset/field_list.py | 23 +++++++++++++++++++++++ 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 74e2e50..489f2ff 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.4 ================ +- Fixed can now exclude subclass fields (#443) - Fixed dereference issue with embedded listfield referencefields (#439) - Fixed slice when using inheritance causing fields to be excluded (#437) - Fixed ._get_db() attribute after a Document.switch_db() (#441) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index a6ba49b..7af9daa 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -14,8 +14,9 @@ from pymongo.common import validate_read_preference from mongoengine import signals from mongoengine.common import _import_class +from mongoengine.base.common import get_document from mongoengine.errors import (OperationError, NotUniqueError, - InvalidQueryError) + InvalidQueryError, LookUpError) from mongoengine.queryset import transform from mongoengine.queryset.field_list import QueryFieldList @@ -1333,13 +1334,33 @@ class BaseQuerySet(object): return frequencies - def _fields_to_dbfields(self, fields): + def _fields_to_dbfields(self, fields, subdoc=False): """Translate fields paths to its db equivalents""" ret = [] + subclasses = [] + document = self._document + if document._meta['allow_inheritance']: + subclasses = [get_document(x) + for x in document._subclasses][1:] for field in fields: - field = ".".join(f.db_field for f in - self._document._lookup_field(field.split('.'))) - ret.append(field) + try: + field = ".".join(f.db_field for f in + document._lookup_field(field.split('.'))) + ret.append(field) + except LookUpError, e: + found = False + for subdoc in subclasses: + try: + subfield = ".".join(f.db_field for f in + subdoc._lookup_field(field.split('.'))) + ret.append(subfield) + found = True + break + except LookUpError, e: + pass + + if not found: + raise e return ret def _get_order_by(self, keys): diff --git a/tests/queryset/field_list.py b/tests/queryset/field_list.py index a18e167..7d66d26 100644 --- a/tests/queryset/field_list.py +++ b/tests/queryset/field_list.py @@ -399,5 +399,28 @@ class OnlyExcludeAllTest(unittest.TestCase): numbers = Numbers.objects.fields(embedded__n={"$slice": [-5, 10]}).get() self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2, -1]) + + def test_exclude_from_subclasses_docs(self): + + class Base(Document): + username = StringField() + + meta = {'allow_inheritance': True} + + class Anon(Base): + anon = BooleanField() + + class User(Base): + password = StringField() + wibble = StringField() + + Base.drop_collection() + User(username="mongodb", password="secret").save() + + user = Base.objects().exclude("password", "wibble").first() + self.assertEqual(user.password, None) + + self.assertRaises(LookUpError, Base.objects.exclude, "made_up") + if __name__ == '__main__': unittest.main() From a707598042fb0312eb01c15b57b4796dd63ad9a1 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 20 Aug 2013 13:13:17 +0000 Subject: [PATCH 1268/1279] Allow fields to be named the same as query operators (#445) --- AUTHORS | 3 ++- docs/changelog.rst | 3 ++- tests/queryset/queryset.py | 17 +++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 2a2eb36..a5b73c7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -175,4 +175,5 @@ that much better: * Paul Uithol (https://github.com/PaulUithol) * Thom Knowles (https://github.com/fleat) * Paul (https://github.com/squamous) - * Olivier Cortès (https://github.com/Karmak23) \ No newline at end of file + * Olivier Cortès (https://github.com/Karmak23) + * crazyzubr (https://github.com/crazyzubr) \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 489f2ff..2775429 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,7 +4,8 @@ Changelog Changes in 0.8.4 ================ -- Fixed can now exclude subclass fields (#443) +- Allow fields to be named the same as query operators (#445) +- Updated field filter logic - can now exclude subclass fields (#443) - Fixed dereference issue with embedded listfield referencefields (#439) - Fixed slice when using inheritance causing fields to be excluded (#437) - Fixed ._get_db() attribute after a Document.switch_db() (#441) diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 75708ca..7f64135 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -3691,6 +3691,23 @@ class QuerySetTest(unittest.TestCase): '_cls': 'Animal.Cat' }) + def test_can_have_field_same_name_as_query_operator(self): + + class Size(Document): + name = StringField() + + class Example(Document): + size = ReferenceField(Size) + + Size.drop_collection() + Example.drop_collection() + + instance_size = Size(name="Large").save() + Example(size=instance_size).save() + + self.assertEqual(Example.objects(size=instance_size).count(), 1) + self.assertEqual(Example.objects(size__in=[instance_size]).count(), 1) + if __name__ == '__main__': unittest.main() From 0dd01bda016e44aca102d4998bf7c1a0a89739e9 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 20 Aug 2013 15:54:42 +0000 Subject: [PATCH 1269/1279] Fixed "$pull" semantics for nested ListFields (#447) --- AUTHORS | 3 +- docs/changelog.rst | 1 + mongoengine/common.py | 5 ++- mongoengine/fields.py | 4 ++ mongoengine/queryset/transform.py | 25 ++++++++++- tests/queryset/queryset.py | 71 ++++++++++++++++++++++++++++--- 6 files changed, 99 insertions(+), 10 deletions(-) diff --git a/AUTHORS b/AUTHORS index a5b73c7..452ba37 100644 --- a/AUTHORS +++ b/AUTHORS @@ -176,4 +176,5 @@ that much better: * Thom Knowles (https://github.com/fleat) * Paul (https://github.com/squamous) * Olivier Cortès (https://github.com/Karmak23) - * crazyzubr (https://github.com/crazyzubr) \ No newline at end of file + * crazyzubr (https://github.com/crazyzubr) + * FrankSomething (https://github.com/FrankSomething) \ No newline at end of file diff --git a/docs/changelog.rst b/docs/changelog.rst index 2775429..6a0258c 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.4 ================ +- Fixed "$pull" semantics for nested ListFields (#447) - Allow fields to be named the same as query operators (#445) - Updated field filter logic - can now exclude subclass fields (#443) - Fixed dereference issue with embedded listfield referencefields (#439) diff --git a/mongoengine/common.py b/mongoengine/common.py index 20d5138..6303231 100644 --- a/mongoengine/common.py +++ b/mongoengine/common.py @@ -23,8 +23,9 @@ def _import_class(cls_name): field_classes = ('DictField', 'DynamicField', 'EmbeddedDocumentField', 'FileField', 'GenericReferenceField', 'GenericEmbeddedDocumentField', 'GeoPointField', - 'PointField', 'LineStringField', 'PolygonField', - 'ReferenceField', 'StringField', 'ComplexBaseField') + 'PointField', 'LineStringField', 'ListField', + 'PolygonField', 'ReferenceField', 'StringField', + 'ComplexBaseField') queryset_classes = ('OperationError',) deref_classes = ('DeReference',) diff --git a/mongoengine/fields.py b/mongoengine/fields.py index c1fc1a7..419f2ef 100644 --- a/mongoengine/fields.py +++ b/mongoengine/fields.py @@ -780,6 +780,10 @@ class DictField(ComplexBaseField): if op in match_operators and isinstance(value, basestring): return StringField().prepare_query_value(op, value) + + if hasattr(self.field, 'field'): + return self.field.prepare_query_value(op, value) + return super(DictField, self).prepare_query_value(op, value) diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index d82f33d..2ee7e38 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -182,6 +182,7 @@ def update(_doc_cls=None, **update): parts = [] cleaned_fields = [] + appended_sub_field = False for field in fields: append_field = True if isinstance(field, basestring): @@ -193,10 +194,17 @@ def update(_doc_cls=None, **update): else: parts.append(field.db_field) if append_field: + appended_sub_field = False cleaned_fields.append(field) + if hasattr(field, 'field'): + cleaned_fields.append(field.field) + appended_sub_field = True # Convert value to proper value - field = cleaned_fields[-1] + if appended_sub_field: + field = cleaned_fields[-2] + else: + field = cleaned_fields[-1] if op in (None, 'set', 'push', 'pull'): if field.required or value is not None: @@ -223,11 +231,24 @@ def update(_doc_cls=None, **update): if 'pull' in op and '.' in key: # Dot operators don't work on pull operations - # it uses nested dict syntax + # unless they point to a list field + # Otherwise it uses nested dict syntax if op == 'pullAll': raise InvalidQueryError("pullAll operations only support " "a single field depth") + # Look for the last list field and use dot notation until there + field_classes = [c.__class__ for c in cleaned_fields] + field_classes.reverse() + ListField = _import_class('ListField') + if ListField in field_classes: + # Join all fields via dot notation to the last ListField + # Then process as normal + last_listField = len(cleaned_fields) - field_classes.index(ListField) + key = ".".join(parts[:last_listField]) + parts = parts[last_listField:] + parts.insert(0, key) + parts.reverse() for key in parts: value = {key: value} diff --git a/tests/queryset/queryset.py b/tests/queryset/queryset.py index 7f64135..b4bcf2a 100644 --- a/tests/queryset/queryset.py +++ b/tests/queryset/queryset.py @@ -1497,9 +1497,6 @@ class QuerySetTest(unittest.TestCase): def test_pull_nested(self): - class User(Document): - name = StringField() - class Collaborator(EmbeddedDocument): user = StringField() @@ -1514,8 +1511,7 @@ class QuerySetTest(unittest.TestCase): Site.drop_collection() c = Collaborator(user='Esteban') - s = Site(name="test", collaborators=[c]) - s.save() + s = Site(name="test", collaborators=[c]).save() Site.objects(id=s.id).update_one(pull__collaborators__user='Esteban') self.assertEqual(Site.objects.first().collaborators, []) @@ -1525,6 +1521,71 @@ class QuerySetTest(unittest.TestCase): self.assertRaises(InvalidQueryError, pull_all) + def test_pull_from_nested_embedded(self): + + class User(EmbeddedDocument): + name = StringField() + + def __unicode__(self): + return '%s' % self.name + + class Collaborator(EmbeddedDocument): + helpful = ListField(EmbeddedDocumentField(User)) + unhelpful = ListField(EmbeddedDocumentField(User)) + + class Site(Document): + name = StringField(max_length=75, unique=True, required=True) + collaborators = EmbeddedDocumentField(Collaborator) + + + Site.drop_collection() + + c = User(name='Esteban') + f = User(name='Frank') + s = Site(name="test", collaborators=Collaborator(helpful=[c], unhelpful=[f])).save() + + Site.objects(id=s.id).update_one(pull__collaborators__helpful=c) + self.assertEqual(Site.objects.first().collaborators['helpful'], []) + + Site.objects(id=s.id).update_one(pull__collaborators__unhelpful={'name': 'Frank'}) + self.assertEqual(Site.objects.first().collaborators['unhelpful'], []) + + def pull_all(): + Site.objects(id=s.id).update_one(pull_all__collaborators__helpful__name=['Ross']) + + self.assertRaises(InvalidQueryError, pull_all) + + def test_pull_from_nested_mapfield(self): + + class Collaborator(EmbeddedDocument): + user = StringField() + + def __unicode__(self): + return '%s' % self.user + + class Site(Document): + name = StringField(max_length=75, unique=True, required=True) + collaborators = MapField(ListField(EmbeddedDocumentField(Collaborator))) + + + Site.drop_collection() + + c = Collaborator(user='Esteban') + f = Collaborator(user='Frank') + s = Site(name="test", collaborators={'helpful':[c],'unhelpful':[f]}) + s.save() + + Site.objects(id=s.id).update_one(pull__collaborators__helpful__user='Esteban') + self.assertEqual(Site.objects.first().collaborators['helpful'], []) + + Site.objects(id=s.id).update_one(pull__collaborators__unhelpful={'user':'Frank'}) + self.assertEqual(Site.objects.first().collaborators['unhelpful'], []) + + def pull_all(): + Site.objects(id=s.id).update_one(pull_all__collaborators__helpful__user=['Ross']) + + self.assertRaises(InvalidQueryError, pull_all) + def test_update_one_pop_generic_reference(self): class BlogTag(Document): From a0ef649dd8777c874927686ef02b533590420390 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 20 Aug 2013 18:31:33 +0000 Subject: [PATCH 1270/1279] Update travis.yml --- .travis.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c7e8ea3..609d898 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,10 +16,10 @@ env: install: - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install django==$DJANGO --use-mirrors ; true; fi - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - pip install https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.1.tar.gz#md5=1534bb15cf311f07afaa3aacba1c028b + - pip install django==$DJANGO --use-mirrors - python setup.py install script: - python setup.py test diff --git a/setup.py b/setup.py index 6bd778b..f5498f1 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ CLASSIFIERS = [ extra_opts = {} if sys.version_info[0] == 3: extra_opts['use_2to3'] = True - extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2>=2.6', 'django>=1.5.1'] + extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6', 'django>=1.5.1'] extra_opts['packages'] = find_packages(exclude=('tests',)) if "test" in sys.argv or "nosetests" in sys.argv: extra_opts['packages'].append("tests") From 200e52bab50906761fa09b8a34f0cac5ddec335e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 20 Aug 2013 18:44:12 +0000 Subject: [PATCH 1271/1279] Added documentation about abstract meta Refs #438 --- docs/guide/defining-documents.rst | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/guide/defining-documents.rst b/docs/guide/defining-documents.rst index 407fbda..ba1af33 100644 --- a/docs/guide/defining-documents.rst +++ b/docs/guide/defining-documents.rst @@ -689,7 +689,6 @@ document.:: .. note:: From 0.8 onwards you must declare :attr:`allow_inheritance` defaults to False, meaning you must set it to True to use inheritance. - Working with existing data -------------------------- As MongoEngine no longer defaults to needing :attr:`_cls` you can quickly and @@ -709,3 +708,25 @@ defining all possible field types. If you use :class:`~mongoengine.Document` and the database contains data that isn't defined then that data will be stored in the `document._data` dictionary. + +Abstract classes +================ + +If you want to add some extra functionality to a group of Document classes but +you don't need or want the overhead of inheritance you can use the +:attr:`abstract` attribute of :attr:`-mongoengine.Document.meta`. +This won't turn on :ref:`document-inheritance` but will allow you to keep your +code DRY:: + + class BaseDocument(Document): + meta = { + 'abstract': True, + } + def check_permissions(self): + ... + + class User(BaseDocument): + ... + +Now the User class will have access to the inherited `check_permissions` method +and won't store any of the extra `_cls` information. From fffd0e899019c33e08cb53bdf7ce9492884a4e3e Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Tue, 20 Aug 2013 18:54:14 +0000 Subject: [PATCH 1272/1279] Fixed error raise --- mongoengine/queryset/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 7af9daa..b4dad0c 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1347,7 +1347,7 @@ class BaseQuerySet(object): field = ".".join(f.db_field for f in document._lookup_field(field.split('.'))) ret.append(field) - except LookUpError, e: + except LookUpError, err: found = False for subdoc in subclasses: try: @@ -1360,7 +1360,7 @@ class BaseQuerySet(object): pass if not found: - raise e + raise err return ret def _get_order_by(self, keys): From f57569f553ab2ffd1947db0a4c014e5026cd0f0d Mon Sep 17 00:00:00 2001 From: Alexandr Morozov Date: Wed, 21 Aug 2013 13:52:24 +0400 Subject: [PATCH 1273/1279] Remove database name necessity in uri connection schema --- docs/guide/connecting.rst | 7 +++++-- mongoengine/connection.py | 5 +---- tests/test_connection.py | 26 ++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/docs/guide/connecting.rst b/docs/guide/connecting.rst index 854e2c3..f681aad 100644 --- a/docs/guide/connecting.rst +++ b/docs/guide/connecting.rst @@ -23,12 +23,15 @@ arguments should be provided:: connect('project1', username='webapp', password='pwd123') -Uri style connections are also supported as long as you include the database -name - just supply the uri as the :attr:`host` to +Uri style connections are also supported - just supply the uri as +the :attr:`host` to :func:`~mongoengine.connect`:: connect('project1', host='mongodb://localhost/database_name') +Note that database name from uri has priority over name +in ::func:`~mongoengine.connect` + ReplicaSets =========== diff --git a/mongoengine/connection.py b/mongoengine/connection.py index abab269..4275da5 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -55,12 +55,9 @@ def register_connection(alias, name, host='localhost', port=27017, # Handle uri style connections if "://" in host: uri_dict = uri_parser.parse_uri(host) - if uri_dict.get('database') is None: - raise ConnectionError("If using URI style connection include "\ - "database name in string") conn_settings.update({ 'host': host, - 'name': uri_dict.get('database'), + 'name': uri_dict.get('database') or name, 'username': uri_dict.get('username'), 'password': uri_dict.get('password'), 'read_preference': read_preference, diff --git a/tests/test_connection.py b/tests/test_connection.py index d27a66d..62d795c 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -59,6 +59,32 @@ class ConnectionTest(unittest.TestCase): c.admin.system.users.remove({}) c.mongoenginetest.system.users.remove({}) + def test_connect_uri_without_db(self): + """Ensure that the connect() method works properly with uri's + without database_name + """ + c = connect(db='mongoenginetest', alias='admin') + c.admin.system.users.remove({}) + c.mongoenginetest.system.users.remove({}) + + c.admin.add_user("admin", "password") + c.admin.authenticate("admin", "password") + c.mongoenginetest.add_user("username", "password") + + self.assertRaises(ConnectionError, connect, "testdb_uri_bad", host='mongodb://test:password@localhost') + + connect("mongoenginetest", host='mongodb://localhost/') + + conn = get_connection() + self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient)) + + db = get_db() + self.assertTrue(isinstance(db, pymongo.database.Database)) + self.assertEqual(db.name, 'mongoenginetest') + + c.admin.system.users.remove({}) + c.mongoenginetest.system.users.remove({}) + def test_register_connection(self): """Ensure that connections with different aliases may be registered. """ From f4db0da58581dc4956add7278c625e9fd37a4f7c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 23 Aug 2013 09:03:51 +0000 Subject: [PATCH 1274/1279] Update changelog add LK4D4 to authors (#452) --- AUTHORS | 3 ++- docs/changelog.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 452ba37..f043207 100644 --- a/AUTHORS +++ b/AUTHORS @@ -177,4 +177,5 @@ that much better: * Paul (https://github.com/squamous) * Olivier Cortès (https://github.com/Karmak23) * crazyzubr (https://github.com/crazyzubr) - * FrankSomething (https://github.com/FrankSomething) \ No newline at end of file + * FrankSomething (https://github.com/FrankSomething) + * Alexandr Morozov (https://github.com/LK4D4) diff --git a/docs/changelog.rst b/docs/changelog.rst index 6a0258c..926fb8a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,7 @@ Changelog Changes in 0.8.4 ================ +- Remove database name necessity in uri connection schema (#452) - Fixed "$pull" semantics for nested ListFields (#447) - Allow fields to be named the same as query operators (#445) - Updated field filter logic - can now exclude subclass fields (#443) From 23843ec86e4bdced0996de4a68d01bbf61e86a31 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 23 Aug 2013 09:06:57 +0000 Subject: [PATCH 1275/1279] Updated travis config --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 609d898..26b502c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ install: - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - pip install https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.1.tar.gz#md5=1534bb15cf311f07afaa3aacba1c028b - - pip install django==$DJANGO --use-mirrors + - pip install django==$DJANGO -U --use-mirrors - python setup.py install script: - python setup.py test From 6738a9433b86fa2e7b4fef6bf124d5c166f5652b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 23 Aug 2013 09:36:33 +0000 Subject: [PATCH 1276/1279] Updated travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 26b502c..4395107 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,6 @@ install: - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi - pip install https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.1.tar.gz#md5=1534bb15cf311f07afaa3aacba1c028b - - pip install django==$DJANGO -U --use-mirrors - python setup.py install script: - python setup.py test From 86c8929d774483ed992557210b7a83a7bc7579cf Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 23 Aug 2013 10:03:10 +0000 Subject: [PATCH 1277/1279] 0.8.4 is a go --- mongoengine/__init__.py | 2 +- python-mongoengine.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongoengine/__init__.py b/mongoengine/__init__.py index bfa35fb..2b68b3c 100644 --- a/mongoengine/__init__.py +++ b/mongoengine/__init__.py @@ -15,7 +15,7 @@ import django __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ + list(queryset.__all__) + signals.__all__ + list(errors.__all__)) -VERSION = (0, 8, 3) +VERSION = (0, 8, 4) def get_version(): diff --git a/python-mongoengine.spec b/python-mongoengine.spec index 512c621..b9c45ef 100644 --- a/python-mongoengine.spec +++ b/python-mongoengine.spec @@ -5,7 +5,7 @@ %define srcname mongoengine Name: python-%{srcname} -Version: 0.8.3 +Version: 0.8.4 Release: 1%{?dist} Summary: A Python Document-Object Mapper for working with MongoDB From bcbe740598747c97d1911ecad8c2865887363df8 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Fri, 23 Aug 2013 13:41:15 +0000 Subject: [PATCH 1278/1279] Updated setup.py --- setup.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index f5498f1..85707d0 100644 --- a/setup.py +++ b/setup.py @@ -48,17 +48,15 @@ CLASSIFIERS = [ 'Topic :: Software Development :: Libraries :: Python Modules', ] -extra_opts = {} +extra_opts = {"packages": find_packages(exclude=["tests", "tests.*"])} if sys.version_info[0] == 3: extra_opts['use_2to3'] = True extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6', 'django>=1.5.1'] - extra_opts['packages'] = find_packages(exclude=('tests',)) if "test" in sys.argv or "nosetests" in sys.argv: - extra_opts['packages'].append("tests") + extra_opts['packages'] = find_packages() extra_opts['package_data'] = {"tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"]} else: extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2>=2.6', 'python-dateutil'] - extra_opts['packages'] = find_packages(exclude=('tests',)) setup(name='mongoengine', version=VERSION, From 654cca82a9ee44527b22418c4f751e98efd9427a Mon Sep 17 00:00:00 2001 From: Joey Payne Date: Wed, 18 Sep 2013 11:38:38 -0600 Subject: [PATCH 1279/1279] Fixes AttributeError when using storage.exists() on a non-existing file. --- mongoengine/django/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/django/storage.py b/mongoengine/django/storage.py index 341455c..9df6f9e 100644 --- a/mongoengine/django/storage.py +++ b/mongoengine/django/storage.py @@ -76,7 +76,7 @@ class GridFSStorage(Storage): """Find the documents in the store with the given name """ docs = self.document.objects - doc = [d for d in docs if getattr(d, self.field).name == name] + doc = [d for d in docs if hasattr(getattr(d, self.field), 'name') and getattr(d, self.field).name == name] if doc: return doc[0] else: