f order array support

Author: haileyajohnson

cdm/core/src/main/java/ucar/nc2/iosp/IospHelper.java

@@ -301,7 +301,7 @@ public static Object readData(LayoutBB layout, DataType dataType, Object arr) {
     if (showLayoutTypes)
       System.out.println("***BB LayoutType=" + layout.getClass().getName());
 
-    if (dataType.getPrimitiveClassType() == byte.class || (dataType == DataType.CHAR)) {
+    if (dataType.getPrimitiveClassType() == byte.class || (dataType == DataType.CHAR) || dataType == DataType.BOOLEAN) {
       byte[] pa = (byte[]) arr;
       while (layout.hasNext()) {
         LayoutBB.Chunk chunk = layout.next();
@@ -312,8 +312,12 @@ public static Object readData(LayoutBB layout, DataType dataType, Object arr) {
           pa[pos++] = bb.get();
       }
       // return (dataType == DataType.CHAR) ? convertByteToChar(pa) : pa;
-      if (dataType == DataType.CHAR)
+      if (dataType == DataType.CHAR) {
         return convertByteToChar(pa);
+      }
+      else if (dataType == DataType.BOOLEAN) {
+        return convertByteToBoolean(pa);
+      }
       else
         return pa;
 
@@ -389,6 +393,23 @@ public static Object readData(LayoutBB layout, DataType dataType, Object arr) {
           pa[pos++] = bb.get();
       }
       return pa;
+    } else if (dataType == DataType.STRING) {
+      String[] pa = (String[]) arr;
+      int recsize = layout.getElemSize();
+      while (layout.hasNext()) {
+        LayoutBB.Chunk chunk = layout.next();
+        ByteBuffer bb = chunk.getByteBuffer();
+        bb.position(chunk.getSrcElem() * recsize);
+        int pos = (int) chunk.getDestElem();
+        for (int i = 0; i < chunk.getNelems(); i++) {
+          char[] ch = new char[dataType.getSize()];
+          for ( int j = 0; j < ch.length; j++) {
+            ch[j] = (char)bb.get();
+          }
+          pa[pos++] = new String(ch);
+        }
+      }
+      return pa;
     }
 
     throw new IllegalStateException();
@@ -666,9 +687,15 @@ public static Object makePrimitiveArray(int size, DataType dataType) {
    */
   public static Object makePrimitiveArray(int size, DataType dataType, Object fillValue) {
 
-    if (dataType.getPrimitiveClassType() == byte.class || (dataType == DataType.CHAR)) {
+    if (dataType.getPrimitiveClassType() == byte.class || (dataType == DataType.CHAR) || dataType == DataType.BOOLEAN) {
       byte[] pa = new byte[size];
-      byte val = ((Number) fillValue).byteValue();
+      byte val;
+      if (dataType == DataType.CHAR) {
+        byte[] bytes = ((String) fillValue).getBytes();
+        val = bytes.length > 0 ? ((String) fillValue).getBytes()[0] : 0;
+      } else {
+        val = ((Number) fillValue).byteValue();
+      }
       if (val != 0)
         for (int i = 0; i < size; i++)
           pa[i] = val;
@@ -777,6 +804,17 @@ public static byte[] convertCharToByte(char[] from) {
     return to;
   }
 
+  public static boolean[] convertByteToBoolean(byte[] from) {
+    boolean[] to = null;
+    if (from != null) {
+      int size = from.length;
+      to = new boolean[size];
+      for (int i = 0; i < size; i++)
+        to[i] = from[i] != 0;
+    }
+    return to;
+  }
+
   public static long transferData(Array result, WritableByteChannel channel) throws java.io.IOException {
 
     // LOOK should we buffer ??

cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZArray.java

@@ -60,7 +60,7 @@ public enum Order {
   // .zarray fields
   private final int[] shape;
   private final int[] chunks;
-  private final Number fillValue;
+  private final Object fillValue;
   private final DataType datatype;
   private final String dtype;
   private final ZarrFilter compressor;
@@ -69,7 +69,7 @@ public enum Order {
   private final List<ZarrFilter> filters;
   private final String separator;
 
-  public ZArray(int[] shape, int[] chunks, Number fill_value, String dtype, ZarrFilter compressor, String order,
+  public ZArray(int[] shape, int[] chunks, Object fill_value, String dtype, ZarrFilter compressor, String order,
       List<ZarrFilter> filters, String separator) throws ZarrFormatException {
     this.shape = shape;
     this.chunks = chunks;
@@ -99,7 +99,7 @@ public List<ZarrFilter> getFilters() {
     return this.filters;
   }
 
-  public Number getFillValue() {
+  public Object getFillValue() {
     return fillValue;
   }
 
@@ -176,13 +176,15 @@ public ZArray deserialize(JsonParser p, DeserializationContext ctxt) throws IOEx
           .mapToInt(JsonNode::asInt).toArray();
       String dtype = ((JsonNode) root.path(ZarrKeys.DTYPE)).asText();
       JsonNode fillValueNode = (JsonNode) root.path(ZarrKeys.FILL_VALUE);
-      final Number fill;
+      final Object fill;
       if (fillValueNode.isLong()) {
         fill = fillValueNode.longValue();
       } else if (fillValueNode.isFloat()) {
         fill = fillValueNode.floatValue();
-      } else {
+      }  else if (fillValueNode.isNumber()){
         fill = fillValueNode.asDouble();
+      } else {
+        fill = fillValueNode.asText("");
       }
 
       String order = ((JsonNode) root.path(ZarrKeys.ORDER)).asText();

cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrHeader.java

@@ -99,7 +99,7 @@ private void makeGroup(RandomAccessDirectoryItem item) throws ZarrFormatExceptio
     // make new Group
     Group.Builder group = Group.builder();
     String location = ZarrPathUtils.trimLocation(item.getLocation());
-    if (location.equals(this.rootLocation + ZarrKeys.ZGROUP)) {
+    if (location.equals(this.rootLocation + '/' + ZarrKeys.ZGROUP)) {
       group = this.rootGroup;
     }
     // set Group name
@@ -216,15 +216,15 @@ private Group.Builder findGroup(String location) throws ZarrFormatException {
    */
   class VInfo {
     private final int[] chunks;
-    private final Number fillValue;
+    private final Object fillValue;
     private final ZarrFilter compressor;
     private final ByteOrder byteOrder;
     private final ZArray.Order order;
     private final String separator;
     private final List<ZarrFilter> filters;
     private final long offset;
 
-    VInfo(int[] chunks, Number fillValue, ZarrFilter compressor, ByteOrder byteOrder, ZArray.Order order,
+    VInfo(int[] chunks, Object fillValue, ZarrFilter compressor, ByteOrder byteOrder, ZArray.Order order,
         String separator, List<ZarrFilter> filters, long offset) {
       this.chunks = chunks;
       this.fillValue = fillValue;
@@ -240,7 +240,7 @@ public int[] getChunks() {
       return this.chunks;
     }
 
-    public Number getFillValue() {
+    public Object getFillValue() {
       return this.fillValue;
     }
 

cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java

@@ -74,13 +74,13 @@ public Array readData(Variable v2, Section section) throws IOException, InvalidR
     long offset = vinfo.getOffset();
     DataType dataType = v2.getDataType();
 
-    // if data is uninitialized, return array with fill value
-    if (!checkIsDataFile(offset)) {
-      Object pa = IospHelper.makePrimitiveArray((int) section.computeSize(), dataType, vinfo.getFillValue());
-      if (dataType == DataType.CHAR)
-        pa = IospHelper.convertByteToChar((byte[]) pa);
-      return Array.factory(dataType, section.getShape(), pa);
-    }
+//    // if data is uninitialized, return array with fill value
+//    if (!checkIsDataFile(offset)) {
+//      Object pa = IospHelper.makePrimitiveArray((int) section.computeSize(), dataType, vinfo.getFillValue());
+//      if (dataType == DataType.CHAR)
+//        pa = IospHelper.convertByteToChar((byte[]) pa);
+//      return Array.factory(dataType, section.getShape(), pa);
+//    }
 
     // create layout object
     Layout layout = new ZarrLayoutBB(v2, section, this.raf);
@@ -96,19 +96,4 @@ public Array readData(Variable v2, Section section) throws IOException, InvalidR
 
     return array;
   }
-
-  /**
-   * Checks whether the files containing @pos is a datafile based on filename.
-   * Any file that is not a known metadata file is assumed to be a datafile.
-   */
-  private boolean checkIsDataFile(long pos) throws IOException {
-    try {
-      this.raf.seek(pos);
-    } catch (EOFException eof) {
-      return false;
-    }
-    String filename = ZarrPathUtils.trimLocation(((RandomAccessDirectory) this.raf).getCurrentFile().getLocation());
-    return !(filename.endsWith(ZarrKeys.ZGROUP) || filename.endsWith(ZarrKeys.ZARRAY)
-        || filename.endsWith(ZarrKeys.ZATTRS));
-  }
 }

cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrLayoutBB.java

@@ -32,6 +32,7 @@ public class ZarrLayoutBB implements LayoutBB {
   private int nBytes; // number of bytes per chunk
   private int totalNChunks; // total number of chunks
   private int totalChunkSize; // total number of elements per chunk
+  private boolean F_order = false;
 
   // bytes representing compressing
   private static final int ZARR_COMPRESSOR_OFFSET = 16;
@@ -52,6 +53,7 @@ public ZarrLayoutBB(Variable v2, Section wantSection, RandomAccessFile raf) thro
 
     // transpose Section and chunk if F order
     if (vinfo.getOrder() == ZArray.Order.F) {
+      this.F_order = true;
       List<Range> ranges = wantSection.getRanges();
       List<Range> transpose = new ArrayList<>();
       int[] temp = new int[ndims];
@@ -119,19 +121,39 @@ public boolean hasNext() {
 
     public LayoutBBTiled.DataChunk next() {
       // TODO: handle uninitialized chunks
-      DataChunk chunk = new ZarrLayoutBB.DataChunk(this.currChunk, totalChunkSize, this.chunkNum);
+      DataChunk chunk = new ZarrLayoutBB.DataChunk(this.currChunk, this.chunkNum);
       incrementChunk();
       return chunk;
     }
 
     private void incrementChunk() {
-      this.chunkNum++;
-      int i = this.currChunk.length - 1;
-      while (this.currChunk[i] + 1 >= nChunks[i] && i > 0) {
-        this.currChunk[i] = 0;
-        i--;
+      //this.chunkNum++;
+      int i;
+      if (F_order) {
+        i = 0;
+        while (this.currChunk[i] + 1 >= nChunks[i] && i < this.currChunk.length-1) {
+          this.currChunk[i] = 0;
+          i++;
+        }
+      } else {
+        i = this.currChunk.length - 1;
+        while (this.currChunk[i] + 1 >= nChunks[i] && i > 0) {
+          this.currChunk[i] = 0;
+          i--;
+        }
       }
       this.currChunk[i]++;
+      this.chunkNum = indicesToChunkNum();
+    }
+
+    private int indicesToChunkNum() {
+      int num = 0;
+      int sz = 1;
+      for (int i = chunkSize.length-1; i >= 0; i--) {
+        num += sz * this.currChunk[i];
+        sz *= nChunks[i];
+      }
+      return num;
     }
   }
 
@@ -140,7 +162,7 @@ private class DataChunk implements LayoutBBTiled.DataChunk {
     private int[] offset; // start indices of chunk in elements
     private long rafOffset; // start position of chunk in bytes
 
-    DataChunk(int[] index, int totalChunkSize, int chunkNum) {
+    DataChunk(int[] index, int chunkNum) {
       this.offset = new int[index.length];
       this.rafOffset = varOffset + (chunkNum * (nBytes + data_bytes_offset)) + data_bytes_offset;
       for (int i = 0; i < index.length; i++) {

cdm/zarr/src/main/java/ucar/unidata/io/zarr/RandomAccessDirectory.java

@@ -81,10 +81,20 @@ private static List<MFile> sortIterator(Iterator<MFile> mfiles) {
   }
 
   /**
-   * @return RandomAccessFile containing the current file pointer
+   * Return the directory item containing the specified position
+   * @param pos
+   * @return Directory item containing bytes at `pos`
    */
-  public RandomAccessFile getCurrentFile() {
-    return this.currentFile;
+  public RandomAccessDirectoryItem getFileAtPos(int pos) {
+    long tempPos = 0;
+    for (RandomAccessDirectoryItem item : this.children) {
+      long rafLength = item.length();
+      if (tempPos + rafLength > pos) {
+        return item;
+      }
+      tempPos += rafLength;
+    }
+    return null;
   }
 
   /**