NBL-67: Python: Add examples on complex data types. Fix serialization defaults

This closes #502

Author: dmelnichuk

docs/datatypes/parsers.rst

@@ -122,17 +122,17 @@ with your data, in to some API function as a *type conversion hint*.
 +-----------+--------------------+-------------------------------+------------------------------------------------------------------+
 |*Object collections, special types, and complex object*                                                                            |
 +-----------+--------------------+-------------------------------+------------------------------------------------------------------+
-|0x17       |`Object array`_     |iterable/list                  |:class:`~pygridgain.datatypes.complex.ObjectArrayObject`          |
+|0x17       |`Object array`_     |tuple[int, iterable/list]      |:class:`~pygridgain.datatypes.complex.ObjectArrayObject`          |
 +-----------+--------------------+-------------------------------+------------------------------------------------------------------+
-|0x18       |`Collection`_       |tuple                          |:class:`~pygridgain.datatypes.complex.CollectionObject`           |
+|0x18       |`Collection`_       |tuple[int, iterable/list]      |:class:`~pygridgain.datatypes.complex.CollectionObject`           |
 +-----------+--------------------+-------------------------------+------------------------------------------------------------------+
-|0x19       |`Map`_              |dict, collections.OrderedDict  |:class:`~pygridgain.datatypes.complex.MapObject`                  |
+|0x19       |`Map`_              |tuple[int, dict/OrderedDict]   |:class:`~pygridgain.datatypes.complex.MapObject`                  |
 +-----------+--------------------+-------------------------------+------------------------------------------------------------------+
 |0x1d       |`Enum array`_       |iterable/list                  |:class:`~pygridgain.datatypes.standard.EnumArrayObject`           |
 +-----------+--------------------+-------------------------------+------------------------------------------------------------------+
 |0x67       |`Complex object`_   |object                         |:class:`~pygridgain.datatypes.complex.BinaryObject`               |
 +-----------+--------------------+-------------------------------+------------------------------------------------------------------+
-|0x1b       |`Wrapped data`_     |tuple                          |:class:`~pygridgain.datatypes.complex.WrappedDataObject`          |
+|0x1b       |`Wrapped data`_     |tuple[int, bytes]              |:class:`~pygridgain.datatypes.complex.WrappedDataObject`          |
 +-----------+--------------------+-------------------------------+------------------------------------------------------------------+
 
 .. _Byte: https://apacheignite.readme.io/docs/binary-client-protocol-data-format#section-byte

docs/examples.rst

@@ -120,6 +120,62 @@ Destroy created cache and close connection.
 
 .. _sql_examples:
 
+Object collections
+------------------
+
+File: `get_and_put_complex.py`_.
+
+GridGain collection types are represented in `pygridgain` as two-tuples.
+First comes collection type ID or deserialization hint, which is specific for
+each of the collection type. Second comes the data value.
+
+.. literalinclude:: ../examples/get_and_put_complex.py
+  :language: python
+  :lines: 19-21
+
+Map
+===
+
+For Python prior to 3.6, it might be important to distinguish between ordered
+(`collections.OrderedDict`) and unordered (`dict`) dictionary types, so you
+could use :py:attr:`~pygridgain.datatypes.complex.Map.LINKED_HASH_MAP`
+for the former and :py:attr:`~pygridgain.datatypes.complex.Map.HASH_MAP`
+for the latter.
+
+Since CPython 3.6 all dictionaries became de facto ordered. You can always use
+`LINKED_HASH_MAP` as a safe default.
+
+.. literalinclude:: ../examples/get_and_put_complex.py
+  :language: python
+  :lines: 29-41
+
+Collection
+==========
+
+See :class:`~pygridgain.datatypes.complex.CollectionObject` and GridGain
+documentation on `Collection`_ type for the description of various Java
+collection types. Note that not all of them have a direct Python
+representative. For example, Python do not have ordered sets (it is indeed
+recommended to use `OrderedDict`'s keys and disregard its values).
+
+As for the `pygridgain`, the rules are simple: pass any iterable as a data,
+and you always get `list` back.
+
+.. literalinclude:: ../examples/get_and_put_complex.py
+  :language: python
+  :lines: 43-57
+
+Object array
+============
+
+:class:`~pygridgain.datatypes.complex.ObjectArrayObject` has a very limited
+functionality in `pygridgain`, since no type checks can be enforced on its
+contents. But it still can be used for interoperability with Java.
+
+.. literalinclude:: ../examples/get_and_put_complex.py
+  :language: python
+  :lines: 59-68
+
 SQL
 ---
 File: `sql.py`_.
@@ -611,4 +667,6 @@ with the following message:
 .. _openssl: https://www.openssl.org/docs/manmaster/man1/openssl.html
 .. _Authentication: https://apacheignite.readme.io/docs/advanced-security#section-authentication
 .. _attrs: https://pypi.org/project/attrs/
-.. _simple class names: https://apacheignite.readme.io/docs/binary-marshaller#binary-name-mapper-and-binary-id-mapper
+.. _get_and_put_complex.py: https://github.com/gridgain/gridgain/tree/master/modules/platforms/python/examples/get_and_put_complex.py
+.. _Collection: https://apacheignite.readme.io/docs/binary-client-protocol-data-format#section-collection
+.. _simple class names: https://apacheignite.readme.io/docs/binary-marshaller#binary-name-mapper-and-binary-id-mapper

examples/get_and_put_complex.py

@@ -0,0 +1,68 @@
+#
+# Copyright 2019 GridGain Systems, Inc. and Contributors.
+#
+# Licensed under the GridGain Community Edition License (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.gridgain.com/products/software/community-edition/gridgain-community-edition-license
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from collections import OrderedDict
+
+from pygridgain import Client
+from pygridgain.datatypes import (
+    CollectionObject, MapObject, ObjectArrayObject,
+)
+
+
+client = Client()
+client.connect('127.0.0.1', 10800)
+
+my_cache = client.get_or_create_cache('my cache')
+
+value = OrderedDict([(1, 'test'), ('key', 2.0)])
+
+# saving ordered dictionary
+type_id = MapObject.LINKED_HASH_MAP
+my_cache.put('my dict', (type_id, value))
+result = my_cache.get('my dict')
+print(result)  # (2, OrderedDict([(1, 'test'), ('key', 2.0)]))
+
+# saving unordered dictionary
+type_id = MapObject.HASH_MAP
+my_cache.put('my dict', (type_id, value))
+result = my_cache.get('my dict')
+print(result)  # (1, {'key': 2.0, 1: 'test'})
+
+type_id = CollectionObject.LINKED_LIST
+value = [1, '2', 3.0]
+
+my_cache.put('my list', (type_id, value))
+
+result = my_cache.get('my list')
+print(result)  # (2, [1, '2', 3.0])
+
+type_id = CollectionObject.HASH_SET
+value = [4, 4, 'test', 5.6]
+
+my_cache.put('my set', (type_id, value))
+
+result = my_cache.get('my set')
+print(result)  # (3, [5.6, 4, 'test'])
+
+type_id = ObjectArrayObject.OBJECT
+value = [7, '8', 9.0]
+
+my_cache.put(
+    'my array of objects',
+    (type_id, value),
+    value_hint=ObjectArrayObject  # this hint is mandatory!
+)
+result = my_cache.get('my array of objects')
+print(result)  # (-1, [7, '8', 9.0])

pygridgain/datatypes/complex.py

@@ -35,13 +35,16 @@
 
 class ObjectArrayObject(GridGainDataType):
     """
-    Array of objects of any type. Its Python representation is
-    tuple(type_id, iterable of any type).
+    Array of Ignite objects of any consistent type. Its Python representation
+    is tuple(type_id, iterable of any type). The only type ID that makes sense
+    in Python client is :py:attr:`~OBJECT`, that corresponds directly to
+    the root object type in Java type hierarchy (`java.lang.Object`).
     """
+    OBJECT = -1
+
     _type_name = NAME_OBJ_ARR
     _type_id = TYPE_OBJ_ARR
     type_code = TC_OBJECT_ARRAY
-    type_or_id_name = 'type_id'
 
     @staticmethod
     def hashcode(value: Iterable) -> int:
@@ -95,7 +98,7 @@ def to_python(cls, ctype_object, *args, **kwargs):
                     *args, **kwargs
                 )
             )
-        return getattr(ctype_object, cls.type_or_id_name), result
+        return ctype_object.type_id, result
 
     @classmethod
     def from_python(cls, value):
@@ -112,7 +115,7 @@ def from_python(cls, value):
             value = [value]
             length = 1
         header.length = length
-        setattr(header, cls.type_or_id_name, type_or_id)
+        header.type_id = type_or_id
         buffer = bytearray(header)
 
         for x in value:
@@ -176,18 +179,45 @@ def from_python(cls, value):
         raise ParseError('Send unwrapped data.')
 
 
-class CollectionObject(ObjectArrayObject):
+class CollectionObject(GridGainDataType):
     """
-    Just like object array, but contains deserialization type hint instead of
-    type id. This hint is also useless in Python, because the list type along
-    covers all the use cases.
-
-    Also represented as tuple(type_id, iterable of any type) in Python.
+    Similar to object array, but contains platform-agnostic deserialization
+    type hint instead of type ID.
+
+    Represented as tuple(hint, iterable of any type) in Python. Hints are:
+
+    * :py:attr:`~pygridgain.datatypes.complex.CollectionObject.USER_SET` −
+      a set of unique Ignite thin data objects. The exact Java type of a set
+      is undefined,
+    * :py:attr:`~pygridgain.datatypes.complex.CollectionObject.USER_COL` −
+      a collection of Ignite thin data objects. The exact Java type
+      of a collection is undefined,
+    * :py:attr:`~pygridgain.datatypes.complex.CollectionObject.ARR_LIST` −
+      represents the `java.util.ArrayList` type,
+    * :py:attr:`~pygridgain.datatypes.complex.CollectionObject.LINKED_LIST` −
+      represents the `java.util.LinkedList` type,
+    * :py:attr:`~pygridgain.datatypes.complex.CollectionObject.HASH_SET`−
+      represents the `java.util.HashSet` type,
+    * :py:attr:`~pygridgain.datatypes.complex.CollectionObject.LINKED_HASH_SET` −
+      represents the `java.util.LinkedHashSet` type,
+    * :py:attr:`~pygridgain.datatypes.complex.CollectionObject.SINGLETON_LIST` −
+      represents the return type of the `java.util.Collection.singletonList`
+      method.
+
+    It is safe to say that `USER_SET` (`set` in Python) and `USER_COL` (`list`)
+    can cover all the imaginable use cases from Python perspective.
     """
+    USER_SET = -1
+    USER_COL = 0
+    ARR_LIST = 1
+    LINKED_LIST = 2
+    HASH_SET = 3
+    LINKED_HASH_SET = 4
+    SINGLETON_LIST = 5
+
     _type_name = NAME_COL
     _type_id = TYPE_COL
     type_code = TC_COLLECTION
-    type_or_id_name = 'type'
     pythonic = list
     default = []
 
@@ -211,13 +241,69 @@ def build_header(cls):
             }
         )
 
+    @classmethod
+    def parse(cls, client: 'Client'):
+        header_class = cls.build_header()
+        buffer = client.recv(ctypes.sizeof(header_class))
+        header = header_class.from_buffer_copy(buffer)
+        fields = []
+
+        for i in range(header.length):
+            c_type, buffer_fragment = AnyDataObject.parse(client)
+            buffer += buffer_fragment
+            fields.append(('element_{}'.format(i), c_type))
+
+        final_class = type(
+            cls.__name__,
+            (header_class,),
+            {
+                '_pack_': 1,
+                '_fields_': fields,
+            }
+        )
+        return final_class, buffer
+
+    @classmethod
+    def to_python(cls, ctype_object, *args, **kwargs):
+        result = []
+        for i in range(ctype_object.length):
+            result.append(
+                AnyDataObject.to_python(
+                    getattr(ctype_object, 'element_{}'.format(i)),
+                    *args, **kwargs
+                )
+            )
+        return ctype_object.type, result
+
+    @classmethod
+    def from_python(cls, value):
+        type_or_id, value = value
+        header_class = cls.build_header()
+        header = header_class()
+        header.type_code = int.from_bytes(
+            cls.type_code,
+            byteorder=PROTOCOL_BYTE_ORDER
+        )
+        try:
+            length = len(value)
+        except TypeError:
+            value = [value]
+            length = 1
+        header.length = length
+        header.type = type_or_id
+        buffer = bytearray(header)
+
+        for x in value:
+            buffer += infer_from_python(x)
+        return bytes(buffer)
+
 
 class Map(GridGainDataType):
     """
     Dictionary type, payload-only.
 
-    GridGain does not track the order of key-value pairs in its caches, hence
-    the ordinary Python dict type, not the collections.OrderedDict.
+    Keys and values in map are independent data objects, but `count`
+    counts pairs. Very annoying.
     """
     _type_name = NAME_MAP
     _type_id = TYPE_MAP
@@ -304,11 +390,12 @@ def from_python(cls, value, type_id=None):
 
 class MapObject(Map):
     """
-    This is a dictionary type. Type conversion hint can be a `HASH_MAP`
-    (ordinary dict) or `LINKED_HASH_MAP` (collections.OrderedDict).
+    This is a dictionary type.
 
-    Keys and values in map are independent data objects, but `count`
-    counts pairs. Very annoying.
+    Represented as tuple(type_id, value).
+
+    Type ID can be a :py:attr:`~HASH_MAP` (corresponds to an ordinary `dict`
+    in Python) or a :py:attr:`~LINKED_HASH_MAP` (`collections.OrderedDict`).
     """
     _type_name = NAME_MAP
     _type_id = TYPE_MAP

pygridgain/datatypes/internal.py

@@ -348,7 +348,7 @@ def _init_python_array_map(cls):
     @classmethod
     def map_python_type(cls, value):
         from pygridgain.datatypes import (
-            MapObject, ObjectArrayObject, BinaryObject,
+            MapObject, CollectionObject, BinaryObject,
         )
 
         if cls._python_map is None:
@@ -362,7 +362,7 @@ def map_python_type(cls, value):
             if value_subtype in cls._python_array_map:
                 return cls._python_array_map[value_subtype]
 
-            # a little heuristics (order may be important)
+            # a little heuristics (order is important)
             if all([
                 value_subtype is None,
                 len(value) == 2,
@@ -377,7 +377,9 @@ def map_python_type(cls, value):
                 isinstance(value[0], int),
                 is_iterable(value[1]),
             ]):
-                return ObjectArrayObject
+                return CollectionObject
+
+            # no default for ObjectArrayObject, sorry
 
             raise TypeError(
                 'Type `array of {}` is invalid'.format(value_subtype)

tests/test_datatypes.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+from collections import OrderedDict
 import ctypes
 from datetime import datetime, timedelta
 import decimal
@@ -118,14 +119,14 @@
         ((-1, [(6001, 1), (6002, 2), (6003, 3)]), BinaryEnumArrayObject),
 
         # object array
-        ((-1, [1, 2, decimal.Decimal('3')]), None),
+        ((ObjectArrayObject.OBJECT, [1, 2, decimal.Decimal('3')]), ObjectArrayObject),
 
         # collection
-        ((3, [1, 2, 3]), CollectionObject),
+        ((CollectionObject.LINKED_LIST, [1, 2, 3]), None),
 
         # map
-        ((1, {'key': 4, 5: 6.0}), None),
-        ((2, {'key': 4, 5: 6.0}), None),
+        ((MapObject.HASH_MAP, {'key': 4, 5: 6.0}), None),
+        ((MapObject.LINKED_HASH_MAP, OrderedDict([('key', 4), (5, 6.0)])), None),
     ]
 )
 def test_put_get_data(client, cache, value, value_hint):