Skip to content

Commit 9641bc1

Browse files
lesserwhirlslesserwhirls
authored
Merge pull request #802 from haileyajohnson/zarr-reads
[5.x] Zarr reads
2 parents 68bf984 + 8c43855 commit 9641bc1

393 files changed

Lines changed: 1029 additions & 248 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎cdm/core/src/main/java/ucar/nc2/iosp/IospHelper.java‎

Lines changed: 74 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -301,27 +301,37 @@ public static Object readData(LayoutBB layout, DataType dataType, Object arr) {
301301
if (showLayoutTypes)
302302
System.out.println("***BB LayoutType=" + layout.getClass().getName());
303303

304-
if (dataType.getPrimitiveClassType() == byte.class || (dataType == DataType.CHAR)) {
304+
if (dataType.getPrimitiveClassType() == byte.class || (dataType == DataType.CHAR) || dataType == DataType.BOOLEAN) {
305305
byte[] pa = (byte[]) arr;
306306
while (layout.hasNext()) {
307307
LayoutBB.Chunk chunk = layout.next();
308308
ByteBuffer bb = chunk.getByteBuffer();
309+
// if chunk is empty, use fill value
310+
if (!bb.hasRemaining()) {
311+
continue;
312+
}
309313
bb.position(chunk.getSrcElem());
310314
int pos = (int) chunk.getDestElem();
311315
for (int i = 0; i < chunk.getNelems(); i++)
312316
pa[pos++] = bb.get();
313317
}
314318
// return (dataType == DataType.CHAR) ? convertByteToChar(pa) : pa;
315-
if (dataType == DataType.CHAR)
319+
if (dataType == DataType.CHAR) {
316320
return convertByteToChar(pa);
317-
else
321+
} else if (dataType == DataType.BOOLEAN) {
322+
return convertByteToBoolean(pa);
323+
} else
318324
return pa;
319325

320326
} else if (dataType.getPrimitiveClassType() == short.class) {
321327
short[] pa = (short[]) arr;
322328
while (layout.hasNext()) {
323329
LayoutBB.Chunk chunk = layout.next();
324330
ShortBuffer buff = chunk.getShortBuffer();
331+
// if chunk is empty, use fill value
332+
if (!buff.hasRemaining()) {
333+
continue;
334+
}
325335
buff.position(chunk.getSrcElem());
326336
int pos = (int) chunk.getDestElem();
327337
for (int i = 0; i < chunk.getNelems(); i++)
@@ -334,6 +344,10 @@ public static Object readData(LayoutBB layout, DataType dataType, Object arr) {
334344
while (layout.hasNext()) {
335345
LayoutBB.Chunk chunk = layout.next();
336346
IntBuffer buff = chunk.getIntBuffer();
347+
// if chunk is empty, use fill value
348+
if (!buff.hasRemaining()) {
349+
continue;
350+
}
337351
buff.position(chunk.getSrcElem());
338352
int pos = (int) chunk.getDestElem();
339353
for (int i = 0; i < chunk.getNelems(); i++)
@@ -346,6 +360,10 @@ public static Object readData(LayoutBB layout, DataType dataType, Object arr) {
346360
while (layout.hasNext()) {
347361
LayoutBB.Chunk chunk = layout.next();
348362
FloatBuffer buff = chunk.getFloatBuffer();
363+
// if chunk is empty, use fill value
364+
if (!buff.hasRemaining()) {
365+
continue;
366+
}
349367
buff.position(chunk.getSrcElem());
350368
int pos = (int) chunk.getDestElem();
351369
for (int i = 0; i < chunk.getNelems(); i++)
@@ -358,6 +376,10 @@ public static Object readData(LayoutBB layout, DataType dataType, Object arr) {
358376
while (layout.hasNext()) {
359377
LayoutBB.Chunk chunk = layout.next();
360378
DoubleBuffer buff = chunk.getDoubleBuffer();
379+
// if chunk is empty, use fill value
380+
if (!buff.hasRemaining()) {
381+
continue;
382+
}
361383
buff.position(chunk.getSrcElem());
362384
int pos = (int) chunk.getDestElem();
363385
for (int i = 0; i < chunk.getNelems(); i++)
@@ -370,6 +392,10 @@ public static Object readData(LayoutBB layout, DataType dataType, Object arr) {
370392
while (layout.hasNext()) {
371393
LayoutBB.Chunk chunk = layout.next();
372394
LongBuffer buff = chunk.getLongBuffer();
395+
// if chunk is empty, use fill value
396+
if (!buff.hasRemaining()) {
397+
continue;
398+
}
373399
buff.position(chunk.getSrcElem());
374400
int pos = (int) chunk.getDestElem();
375401
for (int i = 0; i < chunk.getNelems(); i++)
@@ -383,12 +409,37 @@ public static Object readData(LayoutBB layout, DataType dataType, Object arr) {
383409
while (layout.hasNext()) {
384410
LayoutBB.Chunk chunk = layout.next();
385411
ByteBuffer bb = chunk.getByteBuffer();
412+
// if chunk is empty, use fill value
413+
if (!bb.hasRemaining()) {
414+
continue;
415+
}
386416
bb.position(chunk.getSrcElem() * recsize);
387417
int pos = (int) chunk.getDestElem() * recsize;
388418
for (int i = 0; i < chunk.getNelems() * recsize; i++)
389419
pa[pos++] = bb.get();
390420
}
391421
return pa;
422+
} else if (dataType == DataType.STRING) {
423+
String[] pa = (String[]) arr;
424+
int recsize = layout.getElemSize();
425+
while (layout.hasNext()) {
426+
LayoutBB.Chunk chunk = layout.next();
427+
ByteBuffer bb = chunk.getByteBuffer();
428+
// if chunk is empty, use fill value
429+
if (!bb.hasRemaining()) {
430+
continue;
431+
}
432+
bb.position(chunk.getSrcElem() * recsize);
433+
int pos = (int) chunk.getDestElem();
434+
for (int i = 0; i < chunk.getNelems(); i++) {
435+
char[] ch = new char[dataType.getSize()];
436+
for (int j = 0; j < ch.length; j++) {
437+
ch[j] = (char) bb.get();
438+
}
439+
pa[pos++] = new String(ch);
440+
}
441+
}
442+
return pa;
392443
}
393444

394445
throw new IllegalStateException();
@@ -666,9 +717,15 @@ public static Object makePrimitiveArray(int size, DataType dataType) {
666717
*/
667718
public static Object makePrimitiveArray(int size, DataType dataType, Object fillValue) {
668719

669-
if (dataType.getPrimitiveClassType() == byte.class || (dataType == DataType.CHAR)) {
720+
if (dataType.getPrimitiveClassType() == byte.class || (dataType == DataType.CHAR) || dataType == DataType.BOOLEAN) {
670721
byte[] pa = new byte[size];
671-
byte val = ((Number) fillValue).byteValue();
722+
byte val;
723+
if (fillValue instanceof String) {
724+
byte[] bytes = ((String) fillValue).getBytes();
725+
val = bytes.length > 0 ? ((String) fillValue).getBytes()[0] : 0;
726+
} else {
727+
val = ((Number) fillValue).byteValue();
728+
}
672729
if (val != 0)
673730
for (int i = 0; i < size; i++)
674731
pa[i] = val;
@@ -755,7 +812,6 @@ public static byte[] convertCharToByteUTF(char[] from) {
755812
}
756813

757814
// convert byte array to char array
758-
759815
public static char[] convertByteToChar(byte[] byteArray) {
760816
int size = byteArray.length;
761817
char[] cbuff = new char[size];
@@ -765,7 +821,6 @@ public static char[] convertByteToChar(byte[] byteArray) {
765821
}
766822

767823
// convert char array to byte array
768-
769824
public static byte[] convertCharToByte(char[] from) {
770825
byte[] to = null;
771826
if (from != null) {
@@ -777,6 +832,18 @@ public static byte[] convertCharToByte(char[] from) {
777832
return to;
778833
}
779834

835+
// convert byte array to boolean array
836+
public static boolean[] convertByteToBoolean(byte[] from) {
837+
boolean[] to = null;
838+
if (from != null) {
839+
int size = from.length;
840+
to = new boolean[size];
841+
for (int i = 0; i < size; i++)
842+
to[i] = from[i] != 0;
843+
}
844+
return to;
845+
}
846+
780847
public static long transferData(Array result, WritableByteChannel channel) throws java.io.IOException {
781848

782849
// LOOK should we buffer ??

‎cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZArray.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public enum Order {
6060
// .zarray fields
6161
private final int[] shape;
6262
private final int[] chunks;
63-
private final Number fillValue;
63+
private final Object fillValue;
6464
private final DataType datatype;
6565
private final String dtype;
6666
private final ZarrFilter compressor;
@@ -69,7 +69,7 @@ public enum Order {
6969
private final List<ZarrFilter> filters;
7070
private final String separator;
7171

72-
public ZArray(int[] shape, int[] chunks, Number fill_value, String dtype, ZarrFilter compressor, String order,
72+
public ZArray(int[] shape, int[] chunks, Object fill_value, String dtype, ZarrFilter compressor, String order,
7373
List<ZarrFilter> filters, String separator) throws ZarrFormatException {
7474
this.shape = shape;
7575
this.chunks = chunks;
@@ -99,7 +99,7 @@ public List<ZarrFilter> getFilters() {
9999
return this.filters;
100100
}
101101

102-
public Number getFillValue() {
102+
public Object getFillValue() {
103103
return fillValue;
104104
}
105105

@@ -176,13 +176,15 @@ public ZArray deserialize(JsonParser p, DeserializationContext ctxt) throws IOEx
176176
.mapToInt(JsonNode::asInt).toArray();
177177
String dtype = ((JsonNode) root.path(ZarrKeys.DTYPE)).asText();
178178
JsonNode fillValueNode = (JsonNode) root.path(ZarrKeys.FILL_VALUE);
179-
final Number fill;
179+
final Object fill;
180180
if (fillValueNode.isLong()) {
181181
fill = fillValueNode.longValue();
182182
} else if (fillValueNode.isFloat()) {
183183
fill = fillValueNode.floatValue();
184-
} else {
184+
} else if (fillValueNode.isNumber()) {
185185
fill = fillValueNode.asDouble();
186+
} else {
187+
fill = fillValueNode.asText("");
186188
}
187189

188190
String order = ((JsonNode) root.path(ZarrKeys.ORDER)).asText();

‎cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrFormatException.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,7 @@ public ZarrFormatException(String invalidField, String value) {
1919
super(String.format(customExceptionMessageFormat, invalidField, value));
2020
}
2121

22+
public ZarrFormatException(String msg) {
23+
super(msg);
24+
}
2225
}

0 commit comments

Comments
 (0)