Fixes AttributeError when using storage.exists() on a non-existing file.

Remove database name necessity in uri connection schema

.travis.yml

@@ -16,7 +16,6 @@ env:
 install:
     - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cp /usr/lib/*/libz.so $VIRTUAL_ENV/lib/; fi
     - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install pil --use-mirrors ; true; fi
-    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then pip install django==$DJANGO --use-mirrors ; true; fi
     - if [[ $PYMONGO == 'dev' ]]; then pip install https://github.com/mongodb/mongo-python-driver/tarball/master; true; fi
     - if [[ $PYMONGO != 'dev' ]]; then pip install pymongo==$PYMONGO --use-mirrors; true; fi
     - pip install https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.1.tar.gz#md5=1534bb15cf311f07afaa3aacba1c028b

AUTHORS

@@ -172,3 +172,10 @@ that much better:
  * Alon Horev (https://github.com/alonho)
  * Kelvin Hammond (https://github.com/kelvinhammond)
  * Jatin- (https://github.com/jatin-)
+ * Paul Uithol (https://github.com/PaulUithol)
+ * Thom Knowles (https://github.com/fleat)
+ * Paul (https://github.com/squamous)
+ * Olivier Cortès (https://github.com/Karmak23)
+ * crazyzubr (https://github.com/crazyzubr)
+ * FrankSomething (https://github.com/FrankSomething)
+ * Alexandr Morozov (https://github.com/LK4D4)

docs/_themes/nature/static/nature.css_t

@@ -2,11 +2,15 @@
  * Sphinx stylesheet -- default theme
  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  */
- 
+
 @import url("basic.css");
- 
+
+#changelog p.first {margin-bottom: 0 !important;}
+#changelog p {margin-top: 0 !important;
+              margin-bottom: 0 !important;}
+
 /* -- page layout ----------------------------------------------------------- */
- 
+
 body {
     font-family: Arial, sans-serif;
     font-size: 100%;
@@ -28,18 +32,18 @@ div.bodywrapper {
 hr{
     border: 1px solid #B1B4B6;
 }
- 
+
 div.document {
     background-color: #eee;
 }
- 
+
 div.body {
     background-color: #ffffff;
     color: #3E4349;
     padding: 0 30px 30px 30px;
     font-size: 0.8em;
 }
- 
+
 div.footer {
     color: #555;
     width: 100%;
@@ -47,12 +51,12 @@ div.footer {
     text-align: center;
     font-size: 75%;
 }
- 
+
 div.footer a {
     color: #444;
     text-decoration: underline;
 }
- 
+
 div.related {
     background-color: #6BA81E;
     line-height: 32px;
@@ -60,11 +64,11 @@ div.related {
     text-shadow: 0px 1px 0 #444;
     font-size: 0.80em;
 }
- 
+
 div.related a {
     color: #E2F3CC;
 }
- 
+
 div.sphinxsidebar {
     font-size: 0.75em;
     line-height: 1.5em;
@@ -73,7 +77,7 @@ div.sphinxsidebar {
 div.sphinxsidebarwrapper{
     padding: 20px 0;
 }
- 
+
 div.sphinxsidebar h3,
 div.sphinxsidebar h4 {
     font-family: Arial, sans-serif;
@@ -89,30 +93,30 @@ div.sphinxsidebar h4 {
 div.sphinxsidebar h4{
     font-size: 1.1em;
 }
- 
+
 div.sphinxsidebar h3 a {
     color: #444;
 }
- 
- 
+
+
 div.sphinxsidebar p {
     color: #888;
     padding: 5px 20px;
 }
- 
+
 div.sphinxsidebar p.topless {
 }
- 
+
 div.sphinxsidebar ul {
     margin: 10px 20px;
     padding: 0;
     color: #000;
 }
- 
+
 div.sphinxsidebar a {
     color: #444;
 }
- 
+
 div.sphinxsidebar input {
     border: 1px solid #ccc;
     font-family: sans-serif;
@@ -122,19 +126,19 @@ div.sphinxsidebar input {
 div.sphinxsidebar input[type=text]{
     margin-left: 20px;
 }
- 
+
 /* -- body styles ----------------------------------------------------------- */
- 
+
 a {
     color: #005B81;
     text-decoration: none;
 }
- 
+
 a:hover {
     color: #E32E00;
     text-decoration: underline;
 }
- 
+
 div.body h1,
 div.body h2,
 div.body h3,
@@ -149,30 +153,30 @@ div.body h6 {
     padding: 5px 0 5px 10px;
     text-shadow: 0px 1px 0 white
 }
- 
+
 div.body h1 { border-top: 20px solid white; margin-top: 0; font-size: 200%; }
 div.body h2 { font-size: 150%; background-color: #C8D5E3; }
 div.body h3 { font-size: 120%; background-color: #D8DEE3; }
 div.body h4 { font-size: 110%; background-color: #D8DEE3; }
 div.body h5 { font-size: 100%; background-color: #D8DEE3; }
 div.body h6 { font-size: 100%; background-color: #D8DEE3; }
- 
+
 a.headerlink {
     color: #c60f0f;
     font-size: 0.8em;
     padding: 0 4px 0 4px;
     text-decoration: none;
 }
- 
+
 a.headerlink:hover {
     background-color: #c60f0f;
     color: white;
 }
- 
+
 div.body p, div.body dd, div.body li {
     line-height: 1.5em;
 }
- 
+
 div.admonition p.admonition-title + p {
     display: inline;
 }
@@ -185,29 +189,29 @@ div.note {
     background-color: #eee;
     border: 1px solid #ccc;
 }
- 
+
 div.seealso {
     background-color: #ffc;
     border: 1px solid #ff6;
 }
- 
+
 div.topic {
     background-color: #eee;
 }
- 
+
 div.warning {
     background-color: #ffe4e4;
     border: 1px solid #f66;
 }
- 
+
 p.admonition-title {
     display: inline;
 }
- 
+
 p.admonition-title:after {
     content: ":";
 }
- 
+
 pre {
     padding: 10px;
     background-color: White;
@@ -219,7 +223,7 @@ pre {
     -webkit-box-shadow: 1px 1px 1px #d8d8d8;
     -moz-box-shadow: 1px 1px 1px #d8d8d8;
 }
- 
+
 tt {
     background-color: #ecf0f3;
     color: #222;

docs/apireference.rst

@@ -44,17 +44,21 @@ Context Managers
 Querying
 ========
 
-.. autoclass:: mongoengine.queryset.QuerySet
-   :members:
+.. automodule:: mongoengine.queryset
+    :synopsis: Queryset level operations
 
-   .. automethod:: mongoengine.queryset.QuerySet.__call__
+    .. autoclass:: mongoengine.queryset.QuerySet
+      :members:
+      :inherited-members:
 
-.. autoclass:: mongoengine.queryset.QuerySetNoCache
-   :members:
+      .. automethod:: QuerySet.__call__
 
-   .. automethod:: mongoengine.queryset.QuerySetNoCache.__call__
+    .. autoclass:: mongoengine.queryset.QuerySetNoCache
+      :members:
 
-.. autofunction:: mongoengine.queryset.queryset_manager
+       .. automethod:: mongoengine.queryset.QuerySetNoCache.__call__
+
+    .. autofunction:: mongoengine.queryset.queryset_manager
 
 Fields
 ======

docs/changelog.rst

@@ -2,6 +2,26 @@
 Changelog
 =========
 
+Changes in 0.8.4
+================
+- Remove database name necessity in uri connection schema (#452)
+- Fixed "$pull" semantics for nested ListFields (#447)
+- Allow fields to be named the same as query operators (#445)
+- Updated field filter logic - can now exclude subclass fields (#443)
+- Fixed dereference issue with embedded listfield referencefields (#439)
+- Fixed slice when using inheritance causing fields to be excluded (#437)
+- Fixed ._get_db() attribute after a Document.switch_db() (#441)
+- Dynamic Fields store and recompose Embedded Documents / Documents correctly (#449)
+- Handle dynamic fieldnames that look like digits (#434)
+- Added get_user_document and improve mongo_auth module (#423)
+- Added str representation of GridFSProxy (#424)
+- Update transform to handle docs erroneously passed to unset (#416)
+- Fixed indexing - turn off _cls (#414)
+- Fixed dereference threading issue in ComplexField.__get__ (#412)
+- Fixed QuerySetNoCache.count() caching (#410)
+- Don't follow references in _get_changed_fields (#422, #417)
+- Allow args and kwargs to be passed through to_json (#420)
+
 Changes in 0.8.3
 ================
 - Fixed EmbeddedDocuments with `id` also storing `_id` (#402)
@@ -12,6 +32,9 @@ Changes in 0.8.3
 - Document.select_related() now respects `db_alias` (#377)
 - Reload uses shard_key if applicable (#384)
 - Dynamic fields are ordered based on creation and stored in _fields_ordered (#396)
+
+  **Potential breaking change:** http://docs.mongoengine.org/en/latest/upgrade.html#to-0-8-3
+
 - Fixed pickling dynamic documents `_dynamic_fields` (#387)
 - Fixed ListField setslice and delslice dirty tracking (#390)
 - Added Django 1.5 PY3 support (#392)
@@ -20,6 +43,8 @@ Changes in 0.8.3
 - Fixed queryset.get() respecting no_dereference (#373)
 - Added full_result kwarg to update (#380)
 
+
+
 Changes in 0.8.2
 ================
 - Added compare_indexes helper (#361)

docs/django.rst

@@ -45,7 +45,7 @@ The :mod:`~mongoengine.django.auth` module also contains a
 Custom User model
 =================
 Django 1.5 introduced `Custom user Models
-<https://docs.djangoproject.com/en/dev/topics/auth/customizing/#auth-custom-user>`
+<https://docs.djangoproject.com/en/dev/topics/auth/customizing/#auth-custom-user>`_
 which can be used as an alternative to the MongoEngine authentication backend.
 
 The main advantage of this option is that other components relying on
@@ -74,7 +74,7 @@ An additional ``MONGOENGINE_USER_DOCUMENT`` setting enables you to replace the
 The custom :class:`User` must be a :class:`~mongoengine.Document` class, but
 otherwise has the same requirements as a standard custom user model,
 as specified in the `Django Documentation
-<https://docs.djangoproject.com/en/dev/topics/auth/customizing/>`.
+<https://docs.djangoproject.com/en/dev/topics/auth/customizing/>`_.
 In particular, the custom class must define :attr:`USERNAME_FIELD` and
 :attr:`REQUIRED_FIELDS` attributes.
 
@@ -128,7 +128,7 @@ appended to the filename until the generated filename doesn't exist. The
     >>> fs.listdir()
     ([], [u'hello.txt'])
 
-All files will be saved and retrieved in GridFS via the :class::`FileDocument`
+All files will be saved and retrieved in GridFS via the :class:`FileDocument`
 document, allowing easy access to the files without the GridFSStorage
 backend.::
 
@@ -137,3 +137,36 @@ backend.::
     [<FileDocument: FileDocument object>]
 
 .. versionadded:: 0.4
+
+Shortcuts
+=========
+Inspired by the `Django shortcut get_object_or_404
+<https://docs.djangoproject.com/en/dev/topics/http/shortcuts/#get-object-or-404>`_,
+the :func:`~mongoengine.django.shortcuts.get_document_or_404` method returns 
+a document or raises an Http404 exception if the document does not exist::
+
+    from mongoengine.django.shortcuts import get_document_or_404
+    
+    admin_user = get_document_or_404(User, username='root')
+
+The first argument may be a Document or QuerySet object. All other passed arguments
+and keyword arguments are used in the query::
+
+    foo_email = get_document_or_404(User.objects.only('email'), username='foo', is_active=True).email
+
+.. note:: Like with :func:`get`, a MultipleObjectsReturned will be raised if more than one
+    object is found.
+
+
+Also inspired by the `Django shortcut get_list_or_404
+<https://docs.djangoproject.com/en/dev/topics/http/shortcuts/#get-list-or-404>`_,
+the :func:`~mongoengine.django.shortcuts.get_list_or_404` method returns a list of
+documents or raises an Http404 exception if the list is empty::
+
+    from mongoengine.django.shortcuts import get_list_or_404
+    
+    active_users = get_list_or_404(User, is_active=True)
+
+The first argument may be a Document or QuerySet object. All other passed
+arguments and keyword arguments are used to filter the query.
+

docs/guide/connecting.rst

@@ -23,12 +23,15 @@ arguments should be provided::
 
     connect('project1', username='webapp', password='pwd123')
 
-Uri style connections are also supported as long as you include the database
-name - just supply the uri as the :attr:`host` to
+Uri style connections are also supported - just supply the uri as
+the :attr:`host` to
 :func:`~mongoengine.connect`::
 
     connect('project1', host='mongodb://localhost/database_name')
 
+Note that database name from uri has priority over name
+in ::func:`~mongoengine.connect`
+
 ReplicaSets
 ===========
 

docs/guide/defining-documents.rst

@@ -442,6 +442,8 @@ The following example shows a :class:`Log` document that will be limited to
         ip_address = StringField()
         meta = {'max_documents': 1000, 'max_size': 2000000}
 
+.. defining-indexes_
+
 Indexes
 =======
 
@@ -485,6 +487,35 @@ If a dictionary is passed then the following options are available:
 
     Inheritance adds extra fields indices see: :ref:`document-inheritance`.
 
+Global index default options
+----------------------------
+
+There are a few top level defaults for all indexes that can be set::
+
+    class Page(Document):
+        title = StringField()
+        rating = StringField()
+        meta = {
+            'index_options': {},
+            'index_background': True,
+            'index_drop_dups': True,
+            'index_cls': False
+        }
+
+
+:attr:`index_options` (Optional)
+    Set any default index options - see the `full options list <http://docs.mongodb.org/manual/reference/method/db.collection.ensureIndex/#db.collection.ensureIndex>`_
+
+:attr:`index_background` (Optional)
+    Set the default value for if an index should be indexed in the background
+
+:attr:`index_drop_dups` (Optional)
+    Set the default value for if an index should drop duplicates
+
+:attr:`index_cls` (Optional)
+    A way to turn off a specific index for _cls.
+
+
 Compound Indexes and Indexing sub documents
 -------------------------------------------
 
@@ -558,6 +589,11 @@ documentation for more information.  A common usecase might be session data::
             ]
         }
 
+.. warning:: TTL indexes happen on the MongoDB server and not in the application
+    code, therefore no signals will be fired on document deletion.
+    If you need signals to be fired on deletion, then you must handle the
+    deletion of Documents in your application code.
+
 Comparing Indexes
 -----------------
 
@@ -653,7 +689,6 @@ document.::
 .. note:: From 0.8 onwards you must declare :attr:`allow_inheritance` defaults
           to False, meaning you must set it to True to use inheritance.
 
-
 Working with existing data
 --------------------------
 As MongoEngine no longer defaults to needing :attr:`_cls` you can quickly and
@@ -673,3 +708,25 @@ defining all possible field types.
 
 If you use :class:`~mongoengine.Document` and the database contains data that
 isn't defined then that data will be stored in the `document._data` dictionary.
+
+Abstract classes
+================
+
+If you want to add some extra functionality to a group of Document classes but
+you don't need or want the overhead of inheritance you can use the
+:attr:`abstract` attribute of :attr:`-mongoengine.Document.meta`.
+This won't turn on :ref:`document-inheritance` but will allow you to keep your
+code DRY::
+
+        class BaseDocument(Document):
+            meta = {
+                'abstract': True,
+            }
+            def check_permissions(self):
+                ...
+
+        class User(BaseDocument):
+           ...
+
+Now the User class will have access to the inherited `check_permissions` method
+and won't store any of the extra `_cls` information.

docs/guide/querying.rst

@@ -17,8 +17,8 @@ fetch documents from the database::
 
     As of MongoEngine 0.8 the querysets utilise a local cache.  So iterating
     it multiple times will only cause a single query.  If this is not the
-    desired behavour you can call :class:`~mongoengine.QuerySet.no_cache` to
-    return a non-caching queryset.
+    desired behavour you can call :class:`~mongoengine.QuerySet.no_cache`
+    (version **0.8.3+**) to return a non-caching queryset.
 
 Filtering queries
 =================
@@ -497,7 +497,6 @@ that you may use with these methods:
 * ``unset`` -- delete a particular value (since MongoDB v1.3+)
 * ``inc`` -- increment a value by a given amount
 * ``dec`` -- decrement a value by a given amount
-* ``pop`` -- remove the last item from a list
 * ``push`` -- append a value to a list
 * ``push_all`` -- append several values to a list
 * ``pop`` -- remove the first or last element of a list

docs/upgrade.rst

@@ -3,7 +3,7 @@ Upgrading
 #########
 
 
-0.8.2 to 0.8.2
+0.8.2 to 0.8.3
 **************
 
 Minor change that may impact users:
@@ -16,8 +16,8 @@ fields.  Previously they were stored alphabetically.
 **********
 
 There have been numerous backwards breaking changes in 0.8.  The reasons for
-these are ensure that MongoEngine has sane defaults going forward and
-performs the best it can out the box.  Where possible there have been
+these are to ensure that MongoEngine has sane defaults going forward and that it
+performs the best it can out of the box.  Where possible there have been
 FutureWarnings to help get you ready for the change, but that hasn't been
 possible for the whole of the release.
 
@@ -71,7 +71,7 @@ inherited classes like so: ::
 Document Definition
 -------------------
 
-The default for inheritance has changed - its now off by default and
+The default for inheritance has changed - it is now off by default and
 :attr:`_cls` will not be stored automatically with the class.  So if you extend
 your :class:`~mongoengine.Document` or :class:`~mongoengine.EmbeddedDocuments`
 you will need to declare :attr:`allow_inheritance` in the meta data like so: ::
@@ -81,7 +81,7 @@ you will need to declare :attr:`allow_inheritance` in the meta data like so: ::
 
         meta = {'allow_inheritance': True}
 
-Previously, if you had data the database that wasn't defined in the Document
+Previously, if you had data in the database that wasn't defined in the Document
 definition, it would set it as an attribute on the document.  This is no longer
 the case and the data is set only in the ``document._data`` dictionary: ::
 
@@ -102,8 +102,8 @@ the case and the data is set only in the ``document._data`` dictionary: ::
     AttributeError: 'Animal' object has no attribute 'size'
 
 The Document class has introduced a reserved function `clean()`, which will be
-called before saving the document. If your document class happen to have a method
-with the same name, please try rename it.
+called before saving the document. If your document class happens to have a method
+with the same name, please try to rename it.
 
     def clean(self):
         pass
@@ -111,7 +111,7 @@ with the same name, please try rename it.
 ReferenceField
 --------------
 
-ReferenceFields now store ObjectId's by default - this is more efficient than
+ReferenceFields now store ObjectIds by default - this is more efficient than
 DBRefs as we already know what Document types they reference::
 
     # Old code
@@ -157,7 +157,7 @@ UUIDFields now default to storing binary values::
     class Animal(Document):
         uuid = UUIDField(binary=False)
 
-To migrate all the uuid's you need to touch each object and mark it as dirty
+To migrate all the uuids you need to touch each object and mark it as dirty
 eg::
 
     # Doc definition
@@ -175,7 +175,7 @@ eg::
 DecimalField
 ------------
 
-DecimalField now store floats - previous it was storing strings and that
+DecimalFields now store floats - previously it was storing strings and that
 made it impossible to do comparisons when querying correctly.::
 
     # Old code
@@ -186,7 +186,7 @@ made it impossible to do comparisons when querying correctly.::
     class Person(Document):
         balance = DecimalField(force_string=True)
 
-To migrate all the uuid's you need to touch each object and mark it as dirty
+To migrate all the DecimalFields you need to touch each object and mark it as dirty
 eg::
 
     # Doc definition
@@ -198,7 +198,7 @@ eg::
         p._mark_as_changed('balance')
         p.save()
 
-.. note:: DecimalField's have also been improved with the addition of precision
+.. note:: DecimalFields have also been improved with the addition of precision
     and rounding.  See :class:`~mongoengine.fields.DecimalField` for more information.
 
 `An example test migration for DecimalFields is available on github
@@ -207,7 +207,7 @@ eg::
 Cascading Saves
 ---------------
 To improve performance document saves will no longer automatically cascade.
-Any changes to a Documents references will either have to be saved manually or
+Any changes to a Document's references will either have to be saved manually or
 you will have to explicitly tell it to cascade on save::
 
     # At the class level:
@@ -249,7 +249,7 @@ update your code like so: ::
 
     # Update example a) assign queryset after a change:
     mammals = Animal.objects(type="mammal")
-    carnivores = mammals.filter(order="Carnivora") # Reassign the new queryset so fitler can be applied
+    carnivores = mammals.filter(order="Carnivora") # Reassign the new queryset so filter can be applied
     [m for m in carnivores]                        # This will return all carnivores
 
     # Update example b) chain the queryset:
@@ -276,7 +276,7 @@ queryset you should upgrade to use count::
 .only() now inline with .exclude()
 ----------------------------------
 
-The behaviour of `.only()` was highly ambious, now it works in the mirror fashion
+The behaviour of `.only()` was highly ambiguous, now it works in mirror fashion
 to `.exclude()`.  Chaining `.only()` calls will increase the fields required::
 
     # Old code
@@ -440,7 +440,7 @@ main areas of changed are: choices in fields, map_reduce and collection names.
 Choice options:
 ===============
 
-Are now expected to be an iterable of tuples, with  the first element in each
+Are now expected to be an iterable of tuples, with the first element in each
 tuple being the actual value to be stored. The second element is the
 human-readable name for the option.
 
@@ -462,8 +462,8 @@ such the following have been changed:
 Default collection naming
 =========================
 
-Previously it was just lowercase, its now much more pythonic and readable as
-its lowercase and underscores, previously ::
+Previously it was just lowercase, it's now much more pythonic and readable as
+it's lowercase and underscores, previously ::
 
     class MyAceDocument(Document):
         pass
@@ -530,5 +530,5 @@ Alternatively, you can rename your collections eg ::
 mongodb 1.8 > 2.0 +
 ===================
 
-Its been reported that indexes may need to be recreated to the newer version of indexes.
+It's been reported that indexes may need to be recreated to the newer version of indexes.
 To do this drop indexes and call ``ensure_indexes`` on each model.

mongoengine/__init__.py

@@ -15,7 +15,7 @@ import django
 __all__ = (list(document.__all__) + fields.__all__ + connection.__all__ +
            list(queryset.__all__) + signals.__all__ + list(errors.__all__))
 
-VERSION = (0, 8, 3)
+VERSION = (0, 8, 4)
 
 
 def get_version():

mongoengine/base/document.py

@@ -4,7 +4,7 @@ import numbers
 from functools import partial
 
 import pymongo
-from bson import json_util
+from bson import json_util, ObjectId
 from bson.dbref import DBRef
 from bson.son import SON
 
@@ -321,9 +321,9 @@ class BaseDocument(object):
             message = "ValidationError (%s:%s) " % (self._class_name, pk)
             raise ValidationError(message, errors=errors)
 
-    def to_json(self):
+    def to_json(self, *args, **kwargs):
         """Converts a document to JSON"""
-        return json_util.dumps(self.to_mongo())
+        return json_util.dumps(self.to_mongo(),  *args, **kwargs)
 
     @classmethod
     def from_json(cls, json_data):
@@ -395,6 +395,7 @@ class BaseDocument(object):
         """
         EmbeddedDocument = _import_class("EmbeddedDocument")
         DynamicEmbeddedDocument = _import_class("DynamicEmbeddedDocument")
+        ReferenceField = _import_class("ReferenceField")
         _changed_fields = []
         _changed_fields += getattr(self, '_changed_fields', [])
 
@@ -405,31 +406,36 @@ class BaseDocument(object):
             inspected.add(self.id)
 
         for field_name in self._fields_ordered:
-
             db_field_name = self._db_field_map.get(field_name, field_name)
             key = '%s.' % db_field_name
-            field = self._data.get(field_name, None)
-            if hasattr(field, 'id'):
-                if field.id in inspected:
-                    continue
-                inspected.add(field.id)
+            data = self._data.get(field_name, None)
+            field = self._fields.get(field_name)
 
-            if (isinstance(field, (EmbeddedDocument, DynamicEmbeddedDocument))
+            if hasattr(data, 'id'):
+                if data.id in inspected:
+                    continue
+                inspected.add(data.id)
+            if isinstance(field, ReferenceField):
+                continue
+            elif (isinstance(data, (EmbeddedDocument, DynamicEmbeddedDocument))
                and db_field_name not in _changed_fields):
                  # Find all embedded fields that have been changed
-                changed = field._get_changed_fields(inspected)
+                changed = data._get_changed_fields(inspected)
                 _changed_fields += ["%s%s" % (key, k) for k in changed if k]
-            elif (isinstance(field, (list, tuple, dict)) and
+            elif (isinstance(data, (list, tuple, dict)) and
                     db_field_name not in _changed_fields):
                 # Loop list / dict fields as they contain documents
                 # Determine the iterator to use
-                if not hasattr(field, 'items'):
-                    iterator = enumerate(field)
+                if not hasattr(data, 'items'):
+                    iterator = enumerate(data)
                 else:
-                    iterator = field.iteritems()
+                    iterator = data.iteritems()
                 for index, value in iterator:
                     if not hasattr(value, '_get_changed_fields'):
                         continue
+                    if (hasattr(field, 'field') and
+                        isinstance(field.field, ReferenceField)):
+                        continue
                     list_key = "%s%s." % (key, index)
                     changed = value._get_changed_fields(inspected)
                     _changed_fields += ["%s%s" % (list_key, k)
@@ -454,7 +460,7 @@ class BaseDocument(object):
                 d = doc
                 new_path = []
                 for p in parts:
-                    if isinstance(d, DBRef):
+                    if isinstance(d, (ObjectId, DBRef)):
                         break
                     elif isinstance(d, list) and p.isdigit():
                         d = d[int(p)]
@@ -623,8 +629,10 @@ class BaseDocument(object):
         # Check to see if we need to include _cls
         allow_inheritance = cls._meta.get('allow_inheritance',
                                           ALLOW_INHERITANCE)
-        include_cls = allow_inheritance and not spec.get('sparse', False)
-
+        include_cls = (allow_inheritance and not spec.get('sparse', False) and
+                       spec.get('cls',  True))
+        if "cls" in spec:
+            spec.pop('cls')
         for key in spec['fields']:
             # If inherited spec continue
             if isinstance(key, (list, tuple)):
@@ -754,7 +762,7 @@ class BaseDocument(object):
 
         for field_name in parts:
             # Handle ListField indexing:
-            if field_name.isdigit():
+            if field_name.isdigit() and hasattr(field, 'field'):
                 new_field = field.field
                 fields.append(field_name)
                 continue

mongoengine/base/fields.py

@@ -186,7 +186,6 @@ class ComplexBaseField(BaseField):
     """
 
     field = None
-    __dereference = False
 
     def __get__(self, instance, owner):
         """Descriptor to automatically dereference references.
@@ -201,9 +200,11 @@ class ComplexBaseField(BaseField):
                        (self.field is None or isinstance(self.field,
                         (GenericReferenceField, ReferenceField))))
 
+        _dereference = _import_class("DeReference")()
+
         self._auto_dereference = instance._fields[self.name]._auto_dereference
-        if not self.__dereference and instance._initialised and dereference:
-            instance._data[self.name] = self._dereference(
+        if instance._initialised and dereference:
+            instance._data[self.name] = _dereference(
                 instance._data.get(self.name), max_depth=1, instance=instance,
                 name=self.name
             )
@@ -222,7 +223,7 @@ class ComplexBaseField(BaseField):
         if (self._auto_dereference and instance._initialised and
            isinstance(value, (BaseList, BaseDict))
            and not value._dereferenced):
-            value = self._dereference(
+            value = _dereference(
                 value, max_depth=1, instance=instance, name=self.name
             )
             value._dereferenced = True
@@ -382,13 +383,6 @@ class ComplexBaseField(BaseField):
 
     owner_document = property(_get_owner_document, _set_owner_document)
 
-    @property
-    def _dereference(self,):
-        if not self.__dereference:
-            DeReference = _import_class("DeReference")
-            self.__dereference = DeReference()  # Cached
-        return self.__dereference
-
 
 class ObjectIdField(BaseField):
     """A field wrapper around MongoDB's ObjectIds.

mongoengine/common.py

@@ -23,8 +23,9 @@ def _import_class(cls_name):
     field_classes = ('DictField', 'DynamicField', 'EmbeddedDocumentField',
                      'FileField', 'GenericReferenceField',
                      'GenericEmbeddedDocumentField', 'GeoPointField',
-                     'PointField', 'LineStringField', 'PolygonField',
-                     'ReferenceField', 'StringField', 'ComplexBaseField')
+                     'PointField', 'LineStringField', 'ListField',
+                     'PolygonField', 'ReferenceField', 'StringField',
+                     'ComplexBaseField')
     queryset_classes = ('OperationError',)
     deref_classes = ('DeReference',)
 

mongoengine/connection.py

@@ -55,12 +55,9 @@ def register_connection(alias, name, host='localhost', port=27017,
     # Handle uri style connections
     if "://" in host:
         uri_dict = uri_parser.parse_uri(host)
-        if uri_dict.get('database') is None:
-            raise ConnectionError("If using URI style connection include "\
-                                  "database name in string")
         conn_settings.update({
             'host': host,
-            'name': uri_dict.get('database'),
+            'name': uri_dict.get('database') or name,
             'username': uri_dict.get('username'),
             'password': uri_dict.get('password'),
             'read_preference': read_preference,

mongoengine/dereference.py

@@ -4,7 +4,7 @@ from base import (BaseDict, BaseList, TopLevelDocumentMetaclass, get_document)
 from fields import (ReferenceField, ListField, DictField, MapField)
 from connection import get_db
 from queryset import QuerySet
-from document import Document
+from document import Document, EmbeddedDocument
 
 
 class DeReference(object):
@@ -33,7 +33,8 @@ class DeReference(object):
         self.max_depth = max_depth
         doc_type = None
 
-        if instance and isinstance(instance, (Document, TopLevelDocumentMetaclass)):
+        if instance and isinstance(instance, (Document, EmbeddedDocument,
+                                              TopLevelDocumentMetaclass)):
             doc_type = instance._fields.get(name)
             if hasattr(doc_type, 'field'):
                 doc_type = doc_type.field

mongoengine/django/mongo_auth/models.py

@@ -6,10 +6,29 @@ from django.utils.importlib import import_module
 from django.utils.translation import ugettext_lazy as _
 
 
+__all__ = (
+    'get_user_document',
+)
+
+
 MONGOENGINE_USER_DOCUMENT = getattr(
     settings, 'MONGOENGINE_USER_DOCUMENT', 'mongoengine.django.auth.User')
 
 
+def get_user_document():
+    """Get the user document class used for authentication.
+
+    This is the class defined in settings.MONGOENGINE_USER_DOCUMENT, which
+    defaults to `mongoengine.django.auth.User`.
+
+    """
+
+    name = MONGOENGINE_USER_DOCUMENT
+    dot = name.rindex('.')
+    module = import_module(name[:dot])
+    return getattr(module, name[dot + 1:])
+
+
 class MongoUserManager(UserManager):
     """A User manager wich allows the use of MongoEngine documents in Django.
 
@@ -44,7 +63,7 @@ class MongoUserManager(UserManager):
     def contribute_to_class(self, model, name):
         super(MongoUserManager, self).contribute_to_class(model, name)
         self.dj_model = self.model
-        self.model = self._get_user_document()
+        self.model = get_user_document()
 
         self.dj_model.USERNAME_FIELD = self.model.USERNAME_FIELD
         username = models.CharField(_('username'), max_length=30, unique=True)
@@ -55,16 +74,6 @@ class MongoUserManager(UserManager):
             field = models.CharField(_(name), max_length=30)
             field.contribute_to_class(self.dj_model, name)
 
-    def _get_user_document(self):
-        try:
-            name = MONGOENGINE_USER_DOCUMENT
-            dot = name.rindex('.')
-            module = import_module(name[:dot])
-            return getattr(module, name[dot + 1:])
-        except ImportError:
-            raise ImproperlyConfigured("Error importing %s, please check "
-                                       "settings.MONGOENGINE_USER_DOCUMENT"
-                                       % name)
 
     def get(self, *args, **kwargs):
         try:
@@ -85,5 +94,14 @@ class MongoUserManager(UserManager):
 
 
 class MongoUser(models.Model):
-    objects = MongoUserManager()
+    """"Dummy user model for Django.
 
+    MongoUser is used to replace Django's UserManager with MongoUserManager.
+    The actual user document class is mongoengine.django.auth.User or any
+    other document class specified in MONGOENGINE_USER_DOCUMENT.
+
+    To get the user document class, use `get_user_document()`.
+
+    """
+
+    objects = MongoUserManager()

mongoengine/django/storage.py

@@ -76,7 +76,7 @@ class GridFSStorage(Storage):
         """Find the documents in the store with the given name
         """
         docs = self.document.objects
-        doc = [d for d in docs if getattr(d, self.field).name == name]
+        doc = [d for d in docs if hasattr(getattr(d, self.field), 'name') and getattr(d, self.field).name == name]
         if doc:
             return doc[0]
         else:

mongoengine/document.py

@@ -400,7 +400,7 @@ class Document(BaseDocument):
         """
         with switch_db(self.__class__, db_alias) as cls:
             collection = cls._get_collection()
-            db = cls._get_db
+            db = cls._get_db()
         self._get_collection = lambda: collection
         self._get_db = lambda: db
         self._collection = collection
@@ -536,6 +536,8 @@ class Document(BaseDocument):
     def ensure_indexes(cls):
         """Checks the document meta data and ensures all the indexes exist.
 
+        Global defaults can be set in the meta - see :doc:`guide/defining-documents`
+
         .. note:: You can disable automatic index creation by setting
                   `auto_create_index` to False in the documents meta data
         """

mongoengine/fields.py

@@ -624,7 +624,9 @@ class DynamicField(BaseField):
             cls = value.__class__
             val = value.to_mongo()
             # If we its a document thats not inherited add _cls
-            if (isinstance(value, (Document, EmbeddedDocument))):
+            if (isinstance(value, Document)):
+                val = {"_ref": value.to_dbref(), "_cls": cls.__name__}
+            if (isinstance(value, EmbeddedDocument)):
                 val['_cls'] = cls.__name__
             return val
 
@@ -645,6 +647,15 @@ class DynamicField(BaseField):
             value = [v for k, v in sorted(data.iteritems(), key=itemgetter(0))]
         return value
 
+    def to_python(self, value):
+        if isinstance(value, dict) and '_cls' in value:
+            doc_cls = get_document(value['_cls'])
+            if '_ref' in value:
+                value = doc_cls._get_db().dereference(value['_ref'])
+            return doc_cls._from_son(value)
+
+        return super(DynamicField, self).to_python(value)
+
     def lookup_member(self, member_name):
         return member_name
 
@@ -769,6 +780,10 @@ class DictField(ComplexBaseField):
 
         if op in match_operators and isinstance(value, basestring):
             return StringField().prepare_query_value(op, value)
+
+        if hasattr(self.field, 'field'):
+            return self.field.prepare_query_value(op, value)
+
         return super(DictField, self).prepare_query_value(op, value)
 
 
@@ -1083,6 +1098,10 @@ class GridFSProxy(object):
     def __repr__(self):
         return '<%s: %s>' % (self.__class__.__name__, self.grid_id)
 
+    def __str__(self):
+        name = getattr(self.get(), 'filename', self.grid_id) if self.get() else '(no file)'
+        return '<%s: %s>' % (self.__class__.__name__, name)
+
     def __eq__(self, other):
         if isinstance(other, GridFSProxy):
             return ((self.grid_id == other.grid_id) and

mongoengine/queryset/base.py

@@ -14,8 +14,9 @@ from pymongo.common import validate_read_preference
 
 from mongoengine import signals
 from mongoengine.common import _import_class
+from mongoengine.base.common import get_document
 from mongoengine.errors import (OperationError, NotUniqueError,
-                                InvalidQueryError)
+                                InvalidQueryError, LookUpError)
 
 from mongoengine.queryset import transform
 from mongoengine.queryset.field_list import QueryFieldList
@@ -60,7 +61,6 @@ class BaseQuerySet(object):
         self._none = False
         self._as_pymongo = False
         self._as_pymongo_coerce = False
-        self._len = None
 
         # If inheritance is allowed, only return instances and instances of
         # subclasses of the class being used
@@ -331,14 +331,9 @@ class BaseQuerySet(object):
             :meth:`skip` that has been applied to this cursor into account when
             getting the count
         """
-        if self._limit == 0:
+        if self._limit == 0 and with_limit_and_skip:
             return 0
-        if with_limit_and_skip and self._len is not None:
-            return self._len
-        count = self._cursor.count(with_limit_and_skip=with_limit_and_skip)
-        if with_limit_and_skip:
-            self._len = count
-        return count
+        return self._cursor.count(with_limit_and_skip=with_limit_and_skip)
 
     def delete(self, write_concern=None, _from_doc_delete=False):
         """Delete the documents matched by the query.
@@ -827,9 +822,9 @@ class BaseQuerySet(object):
 
     # JSON Helpers
 
-    def to_json(self):
+    def to_json(self, *args, **kwargs):
         """Converts a queryset to JSON"""
-        return json_util.dumps(self.as_pymongo())
+        return json_util.dumps(self.as_pymongo(), *args, **kwargs)
 
     def from_json(self, json_data):
         """Converts json data to unsaved objects"""
@@ -1339,13 +1334,33 @@ class BaseQuerySet(object):
 
         return frequencies
 
-    def _fields_to_dbfields(self, fields):
+    def _fields_to_dbfields(self, fields, subdoc=False):
         """Translate fields paths to its db equivalents"""
         ret = []
+        subclasses = []
+        document = self._document
+        if document._meta['allow_inheritance']:
+            subclasses = [get_document(x)
+                          for x in document._subclasses][1:]
         for field in fields:
-            field = ".".join(f.db_field for f in
-                             self._document._lookup_field(field.split('.')))
-            ret.append(field)
+            try:
+                field = ".".join(f.db_field for f in
+                                 document._lookup_field(field.split('.')))
+                ret.append(field)
+            except LookUpError, err:
+                found = False
+                for subdoc in subclasses:
+                    try:
+                        subfield = ".".join(f.db_field for f in
+                                        subdoc._lookup_field(field.split('.')))
+                        ret.append(subfield)
+                        found = True
+                        break
+                    except LookUpError, e:
+                        pass
+
+                if not found:
+                    raise err
         return ret
 
     def _get_order_by(self, keys):

mongoengine/queryset/field_list.py

@@ -55,7 +55,8 @@ class QueryFieldList(object):
 
         if self.always_include:
             if self.value is self.ONLY and self.fields:
-                self.fields = self.fields.union(self.always_include)
+                if sorted(self.slice.keys()) != sorted(self.fields):
+                    self.fields = self.fields.union(self.always_include)
             else:
                 self.fields -= self.always_include
 

mongoengine/queryset/queryset.py

@@ -94,8 +94,26 @@ class QuerySet(BaseQuerySet):
             except StopIteration:
                 self._has_more = False
 
+    def count(self, with_limit_and_skip=True):
+        """Count the selected elements in the query.
+
+        :param with_limit_and_skip (optional): take any :meth:`limit` or
+            :meth:`skip` that has been applied to this cursor into account when
+            getting the count
+        """
+        if with_limit_and_skip is False:
+            return super(QuerySet, self).count(with_limit_and_skip)
+
+        if self._len is None:
+            self._len = super(QuerySet, self).count(with_limit_and_skip)
+
+        return self._len
+
     def no_cache(self):
-        """Convert to a non_caching queryset"""
+        """Convert to a non_caching queryset
+
+        .. versionadded:: 0.8.3 Convert to non caching queryset
+        """
         if self._result_cache is not None:
             raise OperationError("QuerySet already cached")
         return self.clone_into(QuerySetNoCache(self._document, self._collection))
@@ -105,7 +123,10 @@ class QuerySetNoCache(BaseQuerySet):
     """A non caching QuerySet"""
 
     def cache(self):
-        """Convert to a caching queryset"""
+        """Convert to a caching queryset
+
+        .. versionadded:: 0.8.3 Convert to caching queryset
+        """
         return self.clone_into(QuerySet(self._document, self._collection))
 
     def __repr__(self):

mongoengine/queryset/transform.py

@@ -43,11 +43,11 @@ def query(_doc_cls=None, _field_operation=False, **query):
         parts = [part for part in parts if not part.isdigit()]
         # Check for an operator and transform to mongo-style if there is
         op = None
-        if parts[-1] in MATCH_OPERATORS:
+        if len(parts) > 1 and parts[-1] in MATCH_OPERATORS:
             op = parts.pop()
 
         negate = False
-        if parts[-1] == 'not':
+        if len(parts) > 1 and parts[-1] == 'not':
             parts.pop()
             negate = True
 
@@ -182,6 +182,7 @@ def update(_doc_cls=None, **update):
             parts = []
 
             cleaned_fields = []
+            appended_sub_field = False
             for field in fields:
                 append_field = True
                 if isinstance(field, basestring):
@@ -193,21 +194,30 @@ def update(_doc_cls=None, **update):
                 else:
                     parts.append(field.db_field)
                 if append_field:
+                    appended_sub_field = False
                     cleaned_fields.append(field)
+                    if hasattr(field, 'field'):
+                        cleaned_fields.append(field.field)
+                        appended_sub_field = True
 
             # Convert value to proper value
-            field = cleaned_fields[-1]
+            if appended_sub_field:
+                field = cleaned_fields[-2]
+            else:
+                field = cleaned_fields[-1]
 
             if op in (None, 'set', 'push', 'pull'):
                 if field.required or value is not None:
                     value = field.prepare_query_value(op, value)
             elif op in ('pushAll', 'pullAll'):
                 value = [field.prepare_query_value(op, v) for v in value]
-            elif op == 'addToSet':
+            elif op in ('addToSet', 'setOnInsert'):
                 if isinstance(value, (list, tuple, set)):
                     value = [field.prepare_query_value(op, v) for v in value]
                 elif field.required or value is not None:
                     value = field.prepare_query_value(op, value)
+            elif op == "unset":
+                value = 1
 
         if match:
             match = '$' + match
@@ -221,11 +231,24 @@ def update(_doc_cls=None, **update):
 
         if 'pull' in op and '.' in key:
             # Dot operators don't work on pull operations
-            # it uses nested dict syntax
+            # unless they point to a list field
+            # Otherwise it uses nested dict syntax
             if op == 'pullAll':
                 raise InvalidQueryError("pullAll operations only support "
                                         "a single field depth")
 
+            # Look for the last list field and use dot notation until there
+            field_classes = [c.__class__ for c in cleaned_fields]
+            field_classes.reverse()
+            ListField = _import_class('ListField')
+            if ListField in field_classes:
+                # Join all fields via dot notation to the last ListField
+                # Then process as normal
+                last_listField = len(cleaned_fields) - field_classes.index(ListField)
+                key = ".".join(parts[:last_listField])
+                parts = parts[last_listField:]
+                parts.insert(0, key)
+
             parts.reverse()
             for key in parts:
                 value = {key: value}

python-mongoengine.spec

@@ -5,7 +5,7 @@
 %define srcname mongoengine
 
 Name:           python-%{srcname}
-Version:        0.8.3
+Version:        0.8.4
 Release:        1%{?dist}
 Summary:        A Python Document-Object Mapper for working with MongoDB
 

setup.py

@@ -48,17 +48,15 @@ CLASSIFIERS = [
     'Topic :: Software Development :: Libraries :: Python Modules',
 ]
 
-extra_opts = {}
+extra_opts = {"packages": find_packages(exclude=["tests", "tests.*"])}
 if sys.version_info[0] == 3:
     extra_opts['use_2to3'] = True
     extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'jinja2==2.6', 'django>=1.5.1']
-    extra_opts['packages'] = find_packages(exclude=('tests',))
     if "test" in sys.argv or "nosetests" in sys.argv:
-        extra_opts['packages'].append("tests")
+        extra_opts['packages'] = find_packages()
         extra_opts['package_data'] = {"tests": ["fields/mongoengine.png", "fields/mongodb_leaf.png"]}
 else:
-    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2==2.6', 'python-dateutil']
-    extra_opts['packages'] = find_packages(exclude=('tests',))
+    extra_opts['tests_require'] = ['nose', 'coverage', 'blinker', 'django>=1.4.2', 'PIL', 'jinja2>=2.6', 'python-dateutil']
 
 setup(name='mongoengine',
       version=VERSION,

tests/document/delta.py

@@ -313,29 +313,24 @@ class DeltaTest(unittest.TestCase):
         self.circular_reference_deltas_2(DynamicDocument, Document)
         self.circular_reference_deltas_2(DynamicDocument, DynamicDocument)
 
-    def circular_reference_deltas_2(self, DocClass1, DocClass2):
+    def circular_reference_deltas_2(self, DocClass1, DocClass2, dbref=True):
 
         class Person(DocClass1):
             name = StringField()
-            owns = ListField(ReferenceField('Organization'))
-            employer = ReferenceField('Organization')
+            owns = ListField(ReferenceField('Organization', dbref=dbref))
+            employer = ReferenceField('Organization', dbref=dbref)
 
         class Organization(DocClass2):
             name = StringField()
-            owner = ReferenceField('Person')
-            employees = ListField(ReferenceField('Person'))
+            owner = ReferenceField('Person', dbref=dbref)
+            employees = ListField(ReferenceField('Person', dbref=dbref))
 
         Person.drop_collection()
         Organization.drop_collection()
 
-        person = Person(name="owner")
-        person.save()
-
-        employee = Person(name="employee")
-        employee.save()
-
-        organization = Organization(name="company")
-        organization.save()
+        person = Person(name="owner").save()
+        employee = Person(name="employee").save()
+        organization = Organization(name="company").save()
 
         person.owns.append(organization)
         organization.owner = person
@@ -355,6 +350,8 @@ class DeltaTest(unittest.TestCase):
         self.assertEqual(o.owner, p)
         self.assertEqual(e.employer, o)
 
+        return person, organization, employee
+
     def test_delta_db_field(self):
         self.delta_db_field(Document)
         self.delta_db_field(DynamicDocument)
@@ -686,6 +683,36 @@ class DeltaTest(unittest.TestCase):
         self.assertEqual(doc._get_changed_fields(), ['list_field'])
         self.assertEqual(doc._delta(), ({}, {'list_field': 1}))
 
+    def test_delta_with_dbref_true(self):
+        person, organization, employee = self.circular_reference_deltas_2(Document, Document, True)
+        employee.name = 'test'
+
+        self.assertEqual(organization._get_changed_fields(), [])
+
+        updates, removals = organization._delta()
+        self.assertEqual({}, removals)
+        self.assertEqual({}, updates)
+
+        organization.employees.append(person)
+        updates, removals = organization._delta()
+        self.assertEqual({}, removals)
+        self.assertTrue('employees' in updates)
+
+    def test_delta_with_dbref_false(self):
+        person, organization, employee = self.circular_reference_deltas_2(Document, Document, False)
+        employee.name = 'test'
+
+        self.assertEqual(organization._get_changed_fields(), [])
+
+        updates, removals = organization._delta()
+        self.assertEqual({}, removals)
+        self.assertEqual({}, updates)
+
+        organization.employees.append(person)
+        updates, removals = organization._delta()
+        self.assertEqual({}, removals)
+        self.assertTrue('employees' in updates)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/document/indexes.py

@@ -156,6 +156,25 @@ class IndexesTest(unittest.TestCase):
         self.assertEqual([{'fields': [('_cls', 1), ('title', 1)]}],
                          A._meta['index_specs'])
 
+    def test_index_no_cls(self):
+        """Ensure index specs are inhertited correctly"""
+
+        class A(Document):
+            title = StringField()
+            meta = {
+                'indexes': [
+                        {'fields': ('title',), 'cls': False},
+                ],
+                'allow_inheritance': True,
+                'index_cls': False
+                }
+
+        self.assertEqual([('title', 1)], A._meta['index_specs'][0]['fields'])
+        A._get_collection().drop_indexes()
+        A.ensure_indexes()
+        info = A._get_collection().index_information()
+        self.assertEqual(len(info.keys()), 2)
+
     def test_build_index_spec_is_not_destructive(self):
 
         class MyDoc(Document):

tests/document/json_serialisation.py

@@ -31,6 +31,10 @@ class TestJson(unittest.TestCase):
 
         doc = Doc(string="Hi", embedded_field=Embedded(string="Hi"))
 
+        doc_json = doc.to_json(sort_keys=True, separators=(',', ':'))
+        expected_json = """{"embedded_field":{"string":"Hi"},"string":"Hi"}"""
+        self.assertEqual(doc_json, expected_json)
+
         self.assertEqual(doc, Doc.from_json(doc.to_json()))
 
     def test_json_complex(self):

tests/fields/fields.py

@@ -2506,5 +2506,46 @@ class FieldTest(unittest.TestCase):
         self.assertTrue(tuple(x.items[0]) in tuples)
         self.assertTrue(x.items[0] in tuples)
 
+    def test_dynamic_fields_class(self):
+
+        class Doc2(Document):
+            field_1 = StringField(db_field='f')
+
+        class Doc(Document):
+            my_id = IntField(required=True, unique=True, primary_key=True)
+            embed_me = DynamicField(db_field='e')
+            field_x = StringField(db_field='x')
+
+        Doc.drop_collection()
+        Doc2.drop_collection()
+
+        doc2 = Doc2(field_1="hello")
+        doc = Doc(my_id=1, embed_me=doc2, field_x="x")
+        self.assertRaises(OperationError, doc.save)
+
+        doc2.save()
+        doc.save()
+
+        doc = Doc.objects.get()
+        self.assertEqual(doc.embed_me.field_1, "hello")
+
+    def test_dynamic_fields_embedded_class(self):
+
+        class Embed(EmbeddedDocument):
+            field_1 = StringField(db_field='f')
+
+        class Doc(Document):
+            my_id = IntField(required=True, unique=True, primary_key=True)
+            embed_me = DynamicField(db_field='e')
+            field_x = StringField(db_field='x')
+
+        Doc.drop_collection()
+
+        Doc(my_id=1, embed_me=Embed(field_1="hello"), field_x="x").save()
+
+        doc = Doc.objects.get()
+        self.assertEqual(doc.embed_me.field_1, "hello")
+
+
 if __name__ == '__main__':
     unittest.main()

tests/fields/file_tests.py

@@ -53,11 +53,12 @@ class FileTest(unittest.TestCase):
         content_type = 'text/plain'
 
         putfile = PutFile()
-        putfile.the_file.put(text, content_type=content_type)
+        putfile.the_file.put(text, content_type=content_type, filename="hello")
         putfile.save()
 
         result = PutFile.objects.first()
         self.assertTrue(putfile == result)
+        self.assertEqual("%s" % result.the_file, "<GridFSProxy: hello>")
         self.assertEqual(result.the_file.read(), text)
         self.assertEqual(result.the_file.content_type, content_type)
         result.the_file.delete()  # Remove file from GridFS

tests/queryset/field_list.py

@@ -162,6 +162,10 @@ class OnlyExcludeAllTest(unittest.TestCase):
         self.assertEqual(obj.name, person.name)
         self.assertEqual(obj.age, person.age)
 
+        obj = self.Person.objects.only(*('id', 'name',)).get()
+        self.assertEqual(obj.name, person.name)
+        self.assertEqual(obj.age, None)
+
         # Check polymorphism still works
         class Employee(self.Person):
             salary = IntField(db_field='wage')
@@ -395,5 +399,28 @@ class OnlyExcludeAllTest(unittest.TestCase):
         numbers = Numbers.objects.fields(embedded__n={"$slice": [-5, 10]}).get()
         self.assertEqual(numbers.embedded.n, [-5, -4, -3, -2, -1])
 
+
+    def test_exclude_from_subclasses_docs(self):
+
+        class Base(Document):
+            username = StringField()
+
+            meta = {'allow_inheritance': True}
+
+        class Anon(Base):
+            anon = BooleanField()
+
+        class User(Base):
+            password = StringField()
+            wibble = StringField()
+
+        Base.drop_collection()
+        User(username="mongodb", password="secret").save()
+
+        user = Base.objects().exclude("password", "wibble").first()
+        self.assertEqual(user.password, None)
+
+        self.assertRaises(LookUpError, Base.objects.exclude, "made_up")
+
 if __name__ == '__main__':
     unittest.main()

tests/queryset/queryset.py

@@ -1497,9 +1497,6 @@ class QuerySetTest(unittest.TestCase):
 
     def test_pull_nested(self):
 
-        class User(Document):
-            name = StringField()
-
         class Collaborator(EmbeddedDocument):
             user = StringField()
 
@@ -1514,8 +1511,7 @@ class QuerySetTest(unittest.TestCase):
         Site.drop_collection()
 
         c = Collaborator(user='Esteban')
-        s = Site(name="test", collaborators=[c])
-        s.save()
+        s = Site(name="test", collaborators=[c]).save()
 
         Site.objects(id=s.id).update_one(pull__collaborators__user='Esteban')
         self.assertEqual(Site.objects.first().collaborators, [])
@@ -1525,6 +1521,71 @@ class QuerySetTest(unittest.TestCase):
 
         self.assertRaises(InvalidQueryError, pull_all)
 
+    def test_pull_from_nested_embedded(self):
+
+        class User(EmbeddedDocument):
+            name = StringField()
+
+            def __unicode__(self):
+                return '%s' % self.name
+
+        class Collaborator(EmbeddedDocument):
+            helpful = ListField(EmbeddedDocumentField(User))
+            unhelpful = ListField(EmbeddedDocumentField(User))
+
+        class Site(Document):
+            name = StringField(max_length=75, unique=True, required=True)
+            collaborators = EmbeddedDocumentField(Collaborator)
+
+
+        Site.drop_collection()
+
+        c = User(name='Esteban')
+        f = User(name='Frank')
+        s = Site(name="test", collaborators=Collaborator(helpful=[c], unhelpful=[f])).save()
+
+        Site.objects(id=s.id).update_one(pull__collaborators__helpful=c)
+        self.assertEqual(Site.objects.first().collaborators['helpful'], [])
+
+        Site.objects(id=s.id).update_one(pull__collaborators__unhelpful={'name': 'Frank'})
+        self.assertEqual(Site.objects.first().collaborators['unhelpful'], [])
+
+        def pull_all():
+            Site.objects(id=s.id).update_one(pull_all__collaborators__helpful__name=['Ross'])
+
+        self.assertRaises(InvalidQueryError, pull_all)
+
+    def test_pull_from_nested_mapfield(self):
+
+        class Collaborator(EmbeddedDocument):
+            user = StringField()
+
+            def __unicode__(self):
+                return '%s' % self.user
+
+        class Site(Document):
+            name = StringField(max_length=75, unique=True, required=True)
+            collaborators = MapField(ListField(EmbeddedDocumentField(Collaborator)))
+
+
+        Site.drop_collection()
+
+        c = Collaborator(user='Esteban')
+        f = Collaborator(user='Frank')
+        s = Site(name="test", collaborators={'helpful':[c],'unhelpful':[f]})
+        s.save()
+
+        Site.objects(id=s.id).update_one(pull__collaborators__helpful__user='Esteban')
+        self.assertEqual(Site.objects.first().collaborators['helpful'], [])
+
+        Site.objects(id=s.id).update_one(pull__collaborators__unhelpful={'user':'Frank'})
+        self.assertEqual(Site.objects.first().collaborators['unhelpful'], [])
+
+        def pull_all():
+            Site.objects(id=s.id).update_one(pull_all__collaborators__helpful__user=['Ross'])
+
+        self.assertRaises(InvalidQueryError, pull_all)
+
     def test_update_one_pop_generic_reference(self):
 
         class BlogTag(Document):
@@ -3299,6 +3360,13 @@ class QuerySetTest(unittest.TestCase):
         Test.objects(test='foo').update_one(upsert=True, set__test='foo')
         self.assertTrue('_cls' in Test._collection.find_one())
 
+    def test_update_upsert_looks_like_a_digit(self):
+        class MyDoc(DynamicDocument):
+            pass
+        MyDoc.drop_collection()
+        self.assertEqual(1, MyDoc.objects.update_one(upsert=True, inc__47=1))
+        self.assertEqual(MyDoc.objects.get()['47'], 1)
+
     def test_read_preference(self):
         class Bar(Document):
             pass
@@ -3327,7 +3395,7 @@ class QuerySetTest(unittest.TestCase):
         Doc(string="Bye", embedded_field=Embedded(string="Bye")).save()
 
         Doc().save()
-        json_data = Doc.objects.to_json()
+        json_data = Doc.objects.to_json(sort_keys=True, separators=(',', ':'))
         doc_objects = list(Doc.objects)
 
         self.assertEqual(doc_objects, Doc.objects.from_json(json_data))
@@ -3483,6 +3551,27 @@ class QuerySetTest(unittest.TestCase):
             people.count()  # count is cached
             self.assertEqual(q, 1)
 
+    def test_no_cached_queryset(self):
+        class Person(Document):
+            name = StringField()
+
+        Person.drop_collection()
+        for i in xrange(100):
+            Person(name="No: %s" % i).save()
+
+        with query_counter() as q:
+            self.assertEqual(q, 0)
+            people = Person.objects.no_cache()
+
+            [x for x in people]
+            self.assertEqual(q, 1)
+
+            list(people)
+            self.assertEqual(q, 2)
+
+            people.count()
+            self.assertEqual(q, 3)
+
     def test_cache_not_cloned(self):
 
         class User(Document):
@@ -3663,6 +3752,23 @@ class QuerySetTest(unittest.TestCase):
             '_cls': 'Animal.Cat'
         })
 
+    def test_can_have_field_same_name_as_query_operator(self):
+
+        class Size(Document):
+            name = StringField()
+
+        class Example(Document):
+            size = ReferenceField(Size)
+
+        Size.drop_collection()
+        Example.drop_collection()
+
+        instance_size = Size(name="Large").save()
+        Example(size=instance_size).save()
+
+        self.assertEqual(Example.objects(size=instance_size).count(), 1)
+        self.assertEqual(Example.objects(size__in=[instance_size]).count(), 1)
+
 
 if __name__ == '__main__':
     unittest.main()

tests/queryset/transform.py

@@ -31,6 +31,31 @@ class TransformTest(unittest.TestCase):
         self.assertEqual(transform.query(name__exists=True),
                          {'name': {'$exists': True}})
 
+    def test_transform_update(self):
+        class DicDoc(Document):
+            dictField = DictField()
+
+        class Doc(Document):
+            pass
+
+        DicDoc.drop_collection()
+        Doc.drop_collection()
+
+        doc = Doc().save()
+        dic_doc = DicDoc().save()
+
+        for k, v in (("set", "$set"), ("set_on_insert", "$setOnInsert"), ("push", "$push")):
+            update = transform.update(DicDoc, **{"%s__dictField__test" % k: doc})
+            self.assertTrue(isinstance(update[v]["dictField.test"], dict))
+
+        # Update special cases
+        update = transform.update(DicDoc, unset__dictField__test=doc)
+        self.assertEqual(update["$unset"]["dictField.test"], 1)
+
+        update = transform.update(DicDoc, pull__dictField__test=doc)
+        self.assertTrue(isinstance(update["$pull"]["dictField"]["test"], dict))
+
+
     def test_query_field_name(self):
         """Ensure that the correct field name is used when querying.
         """

tests/test_connection.py

@@ -59,6 +59,32 @@ class ConnectionTest(unittest.TestCase):
         c.admin.system.users.remove({})
         c.mongoenginetest.system.users.remove({})
 
+    def test_connect_uri_without_db(self):
+        """Ensure that the connect() method works properly with uri's
+        without database_name
+        """
+        c = connect(db='mongoenginetest', alias='admin')
+        c.admin.system.users.remove({})
+        c.mongoenginetest.system.users.remove({})
+
+        c.admin.add_user("admin", "password")
+        c.admin.authenticate("admin", "password")
+        c.mongoenginetest.add_user("username", "password")
+
+        self.assertRaises(ConnectionError, connect, "testdb_uri_bad", host='mongodb://test:password@localhost')
+
+        connect("mongoenginetest", host='mongodb://localhost/')
+
+        conn = get_connection()
+        self.assertTrue(isinstance(conn, pymongo.mongo_client.MongoClient))
+
+        db = get_db()
+        self.assertTrue(isinstance(db, pymongo.database.Database))
+        self.assertEqual(db.name, 'mongoenginetest')
+
+        c.admin.system.users.remove({})
+        c.mongoenginetest.system.users.remove({})
+
     def test_register_connection(self):
         """Ensure that connections with different aliases may be registered.
         """

tests/test_dereference.py

@@ -1171,6 +1171,30 @@ class FieldTest(unittest.TestCase):
 
         self.assertEqual(2, len([brand for bg in brand_groups for brand in bg.brands]))
 
+    def test_dereferencing_embedded_listfield_referencefield(self):
+        class Tag(Document):
+            meta = {'collection': 'tags'}
+            name = StringField()
+
+        class Post(EmbeddedDocument):
+            body = StringField()
+            tags = ListField(ReferenceField("Tag", dbref=True))
+
+        class Page(Document):
+            meta = {'collection': 'pages'}
+            tags = ListField(ReferenceField("Tag", dbref=True))
+            posts = ListField(EmbeddedDocumentField(Post))
+
+        Tag.drop_collection()
+        Page.drop_collection()
+
+        tag = Tag(name='test').save()
+        post = Post(body='test body', tags=[tag])
+        Page(tags=[tag], posts=[post]).save()
+
+        page = Page.objects.first()
+        self.assertEqual(page.tags[0], page.posts[0].tags[0])
+
 if __name__ == '__main__':
     unittest.main()
 

tests/test_django.py

@@ -21,14 +21,16 @@ settings.configure(
 try:
     from django.contrib.auth import authenticate, get_user_model
     from mongoengine.django.auth import User
-    from mongoengine.django.mongo_auth.models import MongoUser, MongoUserManager
+    from mongoengine.django.mongo_auth.models import (
+        MongoUser,
+        MongoUserManager,
+        get_user_document,
+    )
     DJ15 = True
 except Exception:
     DJ15 = False
 from django.contrib.sessions.tests import SessionTestsMixin
 from mongoengine.django.sessions import SessionStore, MongoSession
-
-
 from datetime import tzinfo, timedelta
 ZERO = timedelta(0)
 
@@ -165,6 +167,8 @@ class QuerySetTest(unittest.TestCase):
         class Note(Document):
             text = StringField()
 
+        Note.drop_collection()
+
         for i in xrange(1, 101):
             Note(name="Note: %s" % i).save()
 
@@ -258,9 +262,12 @@ class MongoAuthTest(unittest.TestCase):
         User.drop_collection()
         super(MongoAuthTest, self).setUp()
 
-    def test_user_model(self):
+    def test_get_user_model(self):
         self.assertEqual(get_user_model(), MongoUser)
 
+    def test_get_user_document(self):
+        self.assertEqual(get_user_document(), User)
+
     def test_user_manager(self):
         manager = get_user_model()._default_manager
         self.assertTrue(isinstance(manager, MongoUserManager))