Merge pull request #776 from haileyajohnson/test-data

[5.x] update zarr test data

Author: lesserwhirls

cdm/zarr/src/test/data/scripts/make_zarr_dtype_test_data.py

@@ -6,50 +6,50 @@
 
 import numpy as np
 # create data
-be_short_data = np.arange(100, dtype='>i2').reshape(10, 10) 
-le_short_data = np.arange(100, dtype='<i2').reshape(10, 10)
-be_int_data = np.arange(100, dtype='>i4').reshape(10, 10) 
-le_int_data = np.arange(100, dtype='<i4').reshape(10, 10)
+be_short_data = np.arange(20, dtype='>i2').reshape(4,5)
+le_short_data = np.arange(20, dtype='<i2').reshape(4,5)
+be_int_data = np.arange(20, dtype='>i4').reshape(4,5)
+le_int_data = np.arange(20, dtype='<i4').reshape(4,5)
 
 
 # In[ ]:
 
 
-be_long_data = np.arange(100, dtype='>i8').reshape(10, 10)
-le_long_data  = np.arange(100, dtype='<i8').reshape(10, 10)
+be_long_data = np.arange(20, dtype='>i8').reshape(5,4)
+le_long_data  = np.arange(20, dtype='<i8').reshape(5,4)
 
 
 # In[ ]:
 
 
-be_float_data = np.arange(200, dtype='>f4').reshape(20, 10)
-le_float_data = np.arange(200, dtype='<f4').reshape(20, 10)
+be_float_data = np.arange(20, dtype='>f4').reshape(4,5)
+le_float_data = np.arange(20, dtype='<f4').reshape(4,5)
 
 
 # In[ ]:
 
 
-be_double_data = np.arange(100, dtype='>f8').reshape(10, 10)
-le_double_data = np.arange(100, dtype='<f8').reshape(10, 10)
+be_double_data = np.arange(20, dtype='>f8').reshape(5,4)
+le_double_data = np.arange(20, dtype='<f8').reshape(5,4)
 
 
 # In[ ]:
 
 
-boolean_data = np.full((10, 4), [0, 1, 0, 1], dtype='|b1')
+boolean_data = np.full((4,5), [0, 1, 0, 1,0], dtype='|b1')
 
 
 # In[ ]:
 
 
 byte_data = be_int_data.tobytes()
-bdata = np.frombuffer(byte_data, dtype='|i1').reshape(20, 20);
+bdata = np.frombuffer(byte_data, dtype='|i1').reshape(10,8);
 
 
 # In[ ]:
 
 
-charar = np.chararray((10, 12), itemsize=4)
+charar = np.chararray((10,12), itemsize=4)
 charar[:] = 'abcd'
 
 
@@ -65,83 +65,83 @@
 
 root_grp = zarr.group(store, overwrite=True)
 # create a group for byte-order independent data types
-unordered_group = root_grp.create_group('unordered_group')
+unordered_group = root_grp.create_group('unordered_group', overwrite=True)
 
 
 # create a group for byte-ordered data types
-byte_ordered_group = root_grp.create_group('byte_ordered_group')
+byte_ordered_group = root_grp.create_group('byte_ordered_group', overwrite=True)
 # add groups for big and little endian
-big_endian = byte_ordered_group.create_group('big_endian')
-little_endian = byte_ordered_group.create_group('little_endian')
+big_endian = byte_ordered_group.create_group('big_endian', overwrite=True)
+little_endian = byte_ordered_group.create_group('little_endian', overwrite=True)
 
 # create group for string types
-string_group = root_grp.create_group('string_types')
+string_group = root_grp.create_group('string_types', overwrite=True)
 
 
 # In[ ]:
 
 
 # add data to unordered group
-b = unordered_group.create_dataset('boolean_data', shape=(10, 4), chunks=(2, 2), dtype='|b1', overwrite=True)
+b = unordered_group.create_dataset('boolean_data', shape=(4,5), chunks=(2,5), dtype='|b1', overwrite=True, compressor=None)
 b[:] = boolean_data
-byte = unordered_group.create_dataset('byte_data', shape=(20, 20), chunks=(3, 3), dtype='|i1', overwrite=True)
+byte = unordered_group.create_dataset('byte_data', shape=(10,8), chunks=(5,4), dtype='|i1', overwrite=True, compressor=None)
 byte[:] = bdata
-ubyte = unordered_group.create_dataset('ubyte_data', shape=(20, 20), chunks=(4, 5), dtype='|u1', overwrite=True)
+ubyte = unordered_group.create_dataset('ubyte_data', shape=(10,8), chunks=(5,4), dtype='|u1', overwrite=True, compressor=None)
 ubyte[:] = bdata
 
 
 # In[ ]:
 
 
 # add data to big endian group
-shorts = big_endian.create_dataset('short_data', shape = (10, 10), chunks=(10, 5), dtype='>i2', overwrite=True)
+shorts = big_endian.create_dataset('short_data', shape=(4,5), chunks=(2,5), dtype='>i2', overwrite=True, compressor=None)
 shorts[:] = be_short_data
-ushorts = big_endian.create_dataset('ushort_data', shape = (10, 10), chunks=(10, 5), dtype='>u2', overwrite=True)
+ushorts = big_endian.create_dataset('ushort_data', shape=(4,5), chunks=(2,5), dtype='>u2', overwrite=True, compressor=None)
 ushorts[:] = be_short_data
-ints = big_endian.create_dataset('int_data', shape = (10, 10), chunks=(10, 5), dtype='>i4', overwrite=True)
+ints = big_endian.create_dataset('int_data', shape=(4,5), chunks=(2,5), dtype='>i4', overwrite=True, compressor=None)
 ints[:] = be_int_data
-uints = big_endian.create_dataset('uint_data', shape = (10, 10), chunks=(10, 5), dtype='>u4', overwrite=True)
+uints = big_endian.create_dataset('uint_data', shape=(4,5), chunks=(2,5), dtype='>u4', overwrite=True, compressor=None)
 uints[:] = be_int_data
-longs = big_endian.create_dataset('long_data', shape = (10, 10), chunks=(5, 10), dtype='>i8', overwrite=True)
+longs = big_endian.create_dataset('long_data', shape=(5,4), chunks=(5,2), dtype='>i8', overwrite=True, compressor=None)
 longs[:] = be_long_data
-ulongs = big_endian.create_dataset('ulong_data', shape = (10, 10), chunks=(5, 10), dtype='>u8', overwrite=True)
+ulongs = big_endian.create_dataset('ulong_data', shape=(5,4), chunks=(5,2), dtype='>u8', overwrite=True, compressor=None)
 ulongs[:] = be_long_data
-floats = big_endian.create_dataset('float_data', shape = (20, 10), chunks=(5, 5), dtype='>f4', overwrite=True)
+floats = big_endian.create_dataset('float_data', shape=(4,5), chunks=(2,5), dtype='>f4', overwrite=True, compressor=None)
 floats[:] = be_float_data
-doubles = big_endian.create_dataset('double_data', shape = (10, 10), chunks=(10, 10), dtype='>f8', overwrite=True)
+doubles = big_endian.create_dataset('double_data', shape=(5,4), chunks=(5,2), dtype='>f8', overwrite=True, compressor=None)
 doubles[:] = be_double_data
 
 
 # In[ ]:
 
 
 # add data to little endian group
-shorts = little_endian.create_dataset('short_data', shape = (10, 10), chunks=(10, 5), dtype='<i2', overwrite=True)
+shorts = little_endian.create_dataset('short_data', shape=(4,5), chunks=(2,5), dtype='<i2', overwrite=True, compressor=None)
 shorts[:] = le_short_data
-ushorts = little_endian.create_dataset('ushort_data', shape = (10, 10), chunks=(10, 5), dtype='<u2', overwrite=True)
+ushorts = little_endian.create_dataset('ushort_data', shape=(4,5), chunks=(2,5), dtype='<u2', overwrite=True, compressor=None)
 ushorts[:] = le_short_data
-ints = little_endian.create_dataset('int_data', shape = (10, 10), chunks=(10, 5), dtype='<i4', overwrite=True)
+ints = little_endian.create_dataset('int_data', shape=(4,5), chunks=(2,5), dtype='<i4', overwrite=True, compressor=None)
 ints[:] = le_int_data
-uints = little_endian.create_dataset('uint_data', shape = (10, 10), chunks=(10, 5), dtype='<u4', overwrite=True)
+uints = little_endian.create_dataset('uint_data', shape=(4,5), chunks=(2,5), dtype='<u4', overwrite=True, compressor=None)
 uints[:] = le_int_data
-longs = little_endian.create_dataset('long_data', shape = (10, 10), chunks=(5, 10), dtype='<i8', overwrite=True)
+longs = little_endian.create_dataset('long_data', shape=(5,4), chunks=(5,2), dtype='<i8', overwrite=True, compressor=None)
 longs[:] = le_long_data
-ulongs = little_endian.create_dataset('ulong_data', shape = (10, 10), chunks=(5, 10), dtype='<u8', overwrite=True)
+ulongs = little_endian.create_dataset('ulong_data', shape=(5,4), chunks=(5,2), dtype='<u8', overwrite=True, compressor=None)
 ulongs[:] = le_long_data
-floats = little_endian.create_dataset('float_data', shape = (20, 10), chunks=(5, 5), dtype='<f4', overwrite=True)
+floats = little_endian.create_dataset('float_data', shape=(4,5), chunks=(2,5), dtype='<f4', overwrite=True, compressor=None)
 floats[:] = le_float_data
-doubles = little_endian.create_dataset('double_data', shape = (10, 10), chunks=(10, 10), dtype='<f8', overwrite=True)
+doubles = little_endian.create_dataset('double_data', shape=(5,4), chunks=(5,2), dtype='<f8', overwrite=True, compressor=None)
 doubles[:] = le_double_data
 
 
 # In[ ]:
 
 
 # add string data
-chars = string_group.create_dataset('char_data', shape=(10,12), chunks=(2,4), dtype='S1', overwrite=True)
+chars = string_group.create_dataset('char_data', shape=(10,12), chunks=(5,3), dtype='S1', overwrite=True, compressor=None)
 chars[:] = charar
-strs = string_group.create_dataset('str_data', shape=(10,12), chunks=(2,2), dtype='S4', overwrite=True)
+strs = string_group.create_dataset('str_data', shape=(10,12), chunks=(5,6), dtype='S4', overwrite=True, compressor=None)
 strs[:] = charar
-unicode = string_group.create_dataset('unicode_data', shape=(10,12), chunks=(2,2), dtype='U4', overwrite=True)
+unicode = string_group.create_dataset('unicode_data', shape=(10,12), chunks=(5,6), dtype='U4', overwrite=True, compressor=None)
 unicode[:] = charar
 

cdm/zarr/src/test/data/scripts/make_zarr_test_data.py

@@ -7,21 +7,13 @@
 import zarr
 ## toggle comment to switch between directory store and zip store
 store = zarr.DirectoryStore('../zarr_test_data.zarr')
-# store = zarr.ZipStore('../zarr_test_data.zip')
+# store = zarr.ZipStore('../zarr_test_data.zip', mode='w')
 root_grp = zarr.group(store, overwrite=True)
 
 
 # In[ ]:
 
 
-# make array without group and uninitialized data, set fill_vaue
-a = zarr.create(shape=(20, 20), chunks=(10, 10), dtype='f4', fill_value=999.0, store=store, overwrite=True)
-a[:] = 0
-
-
-# In[ ]:
-
-
 # make group with '/' separator and 'F' order
 attrs_grp = root_grp.create_group('group_with_attrs', overwrite=True)
 
@@ -37,7 +29,7 @@
 
 
 # add array to group with 'F' order
-a = attrs_grp.create_dataset('F_order_array', shape=(20, 20), chunks=(4, 5), dtype='i4', order='F', overwrite=True)
+a = attrs_grp.create_dataset('F_order_array', shape=(20, 20), chunks=(4, 5), dtype='i4', order='F', overwrite=True, compressor=None)
 
 
 # In[ ]:
@@ -61,6 +53,32 @@
 # In[ ]:
 
 
+# create uninitialized array
+a = attrs_grp.create_dataset('uninitialized', shape=(20, 20), chunks=(10, 10), dtype='f4', fill_value=999.0, overwrite=True, compressor=None)
+
+
+# In[ ]:
+
+
+# create partially initialized arrays
+a = attrs_grp.create_dataset('partial_fill1', shape=(20, 20), chunks=(10, 10), dtype='f4', fill_value=999.0, overwrite=True, compressor=None)
+a[0:10,0:10]=0
+a = attrs_grp.create_dataset('partial_fill2', shape=(20, 20), chunks=(10, 10), dtype='f4', fill_value=999.0, overwrite=True, compressor=None)
+a[15:20,10:20]=0
+
+
+# In[ ]:
+
+
+# create nested arrays
+# dimension_separator keyword does not work for now, data is manually edited
+a = attrs_grp.create_dataset('nested', shape=(20, 20), chunks=(10, 10), dtype='i2', overwrite=True, compressor=None) #, dimension_separator='/')
+a[:]=0
+
+
+# In[ ]:
+
+
 # make group for multidimensonal data
 dims_grp = root_grp.create_group('group_with_dims', overwrite=True)
 
@@ -69,7 +87,7 @@
 
 
 # add 1D array
-a1 = dims_grp.create_dataset('var1D', shape=(20,), chunks=(4,), dtype='i4', overwrite=True)
+a1 = dims_grp.create_dataset('var1D', shape=(20,), chunks=(5,), dtype='i4', overwrite=True, compressor=None)
 data = np.arange(20)
 a1[:] = data
 
@@ -78,23 +96,29 @@
 
 
 # add 2D array
-a2 = dims_grp.create_dataset('var2D', shape=(20,20), chunks=(4,4), dtype='i4', overwrite=True)
+a2 = dims_grp.create_dataset('var2D', shape=(20,20), chunks=(5,5), dtype='i4', overwrite=True, compressor=None)
 a2[:] = np.tile(data, (20,1))
 
 
 # In[ ]:
 
 
 # add 3D array
-a3 = dims_grp.create_dataset('var3D', shape=(20,20,20), chunks=(4,4,4), dtype='i4', overwrite=True)
+a3 = dims_grp.create_dataset('var3D', shape=(20,20,20), chunks=(5,5,5), dtype='i4', overwrite=True, compressor=None)
 a3[:] = np.tile(data, (20,20,1))
 
 
 # In[ ]:
 
 
 # add 4D array
-a4 = dims_grp.create_dataset('var4D', shape=(20,20,20,20), chunks=(4,4,4,4), dtype='i4', overwrite=True)
+a4 = dims_grp.create_dataset('var4D', shape=(20,20,20,20), chunks=(5,5,5,5), dtype='i4', overwrite=True, compressor=None)
 a4[:] = np.tile(data, (20,20,20,1))
-a[:]
+
+
+# In[ ]:
+
+
+## uncomment for zip store
+# store.close()
 

cdm/zarr/src/test/data/test_dtypes.zarr/byte_ordered_group/big_endian/double_data/.zarray

@@ -1,22 +1,16 @@
 {
     "chunks": [
-        10,
-        10
+        5,
+        2
     ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
+    "compressor": null,
     "dtype": ">f8",
     "fill_value": 0.0,
     "filters": null,
     "order": "C",
     "shape": [
-        10,
-        10
+        5,
+        4
     ],
     "zarr_format": 2
 }

cdm/zarr/src/test/data/test_dtypes.zarr/byte_ordered_group/big_endian/double_data/0.0

@@ -1,22 +1,16 @@
 {
     "chunks": [
-        5,
+        2,
         5
     ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
+    "compressor": null,
     "dtype": ">f4",
     "fill_value": 0.0,
     "filters": null,
     "order": "C",
     "shape": [
-        20,
-        10
+        4,
+        5
     ],
     "zarr_format": 2
 }