Skip to content

Commit b5ef7fa

Browse files
tdrwenskitdrwenski
authored
Zarr unit tests and minor cleanups (#1352)
* Add unit tests for ZArray * Remove commented out code * Fix raw use of parametrized class warning * Remove unneeded null check * Remove empty doc strings * Remove unused imports * Make variable final * Remove unneeded explicit type * Copy zip file to temp folder in tests to avoid creating an unzipped file not cleaned up by tests * Remove unused imports
1 parent 16972e4 commit b5ef7fa

6 files changed

Lines changed: 114 additions & 55 deletions

File tree

‎cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrHeader.java‎

Lines changed: 4 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,10 @@
77

88
import com.fasterxml.jackson.databind.ObjectMapper;
99

10-
import ucar.ma2.Array;
1110
import ucar.ma2.ArrayObject;
12-
import ucar.ma2.ArrayString;
1311
import ucar.nc2.Attribute;
1412
import ucar.nc2.Dimension;
1513
import ucar.nc2.Group;
16-
import ucar.ma2.Index;
1714
import ucar.nc2.Variable;
1815
import ucar.nc2.filter.Filter;
1916
import ucar.unidata.io.RandomAccessFile;
@@ -37,11 +34,8 @@ public class ZarrHeader {
3734
private final RandomAccessDirectory rootRaf;
3835
private final Group.Builder rootGroup;
3936
private final String rootLocation;
40-
private static ObjectMapper objectMapper = new ObjectMapper();
37+
private static final ObjectMapper objectMapper = new ObjectMapper();
4138

42-
/*
43-
*
44-
*/
4539
public ZarrHeader(RandomAccessDirectory raf, Group.Builder rootGroup) {
4640
this.rootRaf = raf;
4741
this.rootGroup = rootGroup;
@@ -60,16 +54,10 @@ private class DelayedVarMaker {
6054
private List<Attribute> attrs; // list of variable attributes
6155
private long dataOffset; // byte position where data starts
6256

63-
/*
64-
*
65-
*/
6657
void setAttrs(List<Attribute> attrs) {
6758
this.attrs = attrs;
6859
}
6960

70-
/*
71-
*
72-
*/
7361
void setVar(RandomAccessDirectoryItem var) {
7462
this.var = var;
7563
this.attrs = null;
@@ -102,9 +90,6 @@ boolean myAttrs(RandomAccessDirectoryItem attrs) {
10290
return ZarrUtils.getObjectNameFromPath(attrPath).equals(ZarrUtils.getObjectNameFromPath(varPath));
10391
}
10492

105-
/*
106-
*
107-
*/
10893
void processItem(RandomAccessDirectoryItem item) {
10994
if (var == null) {
11095
return;
@@ -122,9 +107,6 @@ void processItem(RandomAccessDirectoryItem item) {
122107
}
123108
}
124109

125-
/*
126-
*
127-
*/
128110
void makeVar() {
129111
if (var == null) {
130112
return; // do nothing if no variable is in progress
@@ -187,9 +169,6 @@ public void read() throws IOException {
187169
delayedVarMaker.makeVar();
188170
}
189171

190-
/*
191-
*
192-
*/
193172
private void makeGroup(RandomAccessDirectoryItem item, List<Attribute> attrs) {
194173
// make new Group
195174
Group.Builder group = Group.builder();
@@ -217,13 +196,10 @@ private void makeGroup(RandomAccessDirectoryItem item, List<Attribute> attrs) {
217196
}
218197
}
219198

220-
/*
221-
*
222-
*/
223199
private void makeVariable(RandomAccessDirectoryItem item, long dataOffset, ZArray zarray,
224200
Map<Integer, Long> initializedChunks, List<Attribute> attrs) throws ZarrFormatException {
225201
// make new Variable
226-
Variable.Builder var = Variable.builder();
202+
Variable.Builder<?> var = Variable.builder();
227203
String location = ZarrUtils.trimLocation(item.getLocation());
228204

229205
// set var name
@@ -255,22 +231,9 @@ private void makeVariable(RandomAccessDirectoryItem item, long dataOffset, ZArra
255231
dimNames[i] = (String) aod1.get(i);
256232
}
257233
hasNamedDimensions = true;
258-
// logger.trace(" found _ARRAY_DIMENSIONS array {}", aod1);
259234
} catch (final Exception exc) {
260235
logger.debug(" Could not extract _ARRAY_DIMENSIONS for {}, {}", vname, exc.getMessage());
261236
}
262-
263-
//// Informational logging
264-
// } else if ("coordinates".equals(attrName) || "standard_name".equals(attrName) || "units".equals(attrName))
265-
//// {
266-
// try {
267-
// ArrayObject.D1 aod1 = (ArrayObject.D1) attr.getValues();
268-
// String coordsStr = (String) aod1.get(0);
269-
// logger.trace(" var {} has {} attr '{}'", vname, attrName, coordsStr);
270-
// } catch (final Exception exc) {
271-
// logger.debug(" Exception extracting {} attr value, {}", attrName, exc.getMessage());
272-
// }
273-
274237
}
275238
}
276239
}
@@ -334,7 +297,7 @@ private void makeVariable(RandomAccessDirectoryItem item, long dataOffset, ZArra
334297
// Include some info from .zarray file in attributes for display when showing variable detail.
335298
// Possibly add to this fill_value if in .zarray but not .zattrs?
336299
if (attrs == null) {
337-
attrs = new ArrayList<Attribute>();
300+
attrs = new ArrayList<>();
338301
}
339302
final Filter compressor = zarray.getCompressor();
340303
if (compressor == null) {
@@ -344,17 +307,12 @@ private void makeVariable(RandomAccessDirectoryItem item, long dataOffset, ZArra
344307
}
345308

346309
// add current attributes, if any exist
347-
if (attrs != null) {
348-
var.addAttributes(attrs);
349-
}
310+
var.addAttributes(attrs);
350311

351312
// Add var to parent.
352313
parentGroup.addVariable(var);
353314
}
354315

355-
/*
356-
*
357-
*/
358316
private List<Attribute> makeAttributes(RandomAccessDirectoryItem item) {
359317
// get RandomAccessFile for JSON parsing
360318
try {

‎cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrLayoutBB.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import java.util.ArrayList;
1515
import java.util.List;
1616
import java.util.Map;
17-
import java.util.Set;
1817

1918
/**
2019
* A tiled layout for Zarr formats that accommodates uncompressing and filtering data before returning

‎cdm/zarr/src/main/java/ucar/unidata/io/zarr/VirtualRandomAccessFile.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,6 @@
99
import ucar.unidata.io.RandomAccessFile;
1010

1111
import java.io.IOException;
12-
import java.util.ArrayList;
13-
import java.util.List;
14-
import java.util.OptionalLong;
1512

1613
/**
1714
* A wrapper for a RandomAccessFile that allows lazy loading

‎cdm/zarr/src/test/java/ucar/nc2/iosp/zarr/TestZArray.java‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
package ucar.nc2.iosp.zarr;
2+
3+
import static com.google.common.truth.Truth.assertThat;
4+
import static org.junit.Assert.assertThrows;
5+
6+
import java.nio.ByteOrder;
7+
import org.junit.Test;
8+
import ucar.ma2.DataType;
9+
import ucar.nc2.iosp.zarr.ZArray.Order;
10+
11+
public class TestZArray {
12+
private static final String DTYPE = ">f8";
13+
private static final String ORDER = "C";
14+
private static final String DELIMITER = ".";
15+
16+
@Test
17+
public void shouldParseDataType() throws ZarrFormatException {
18+
final String dtype = ">f8";
19+
final ZArray zArray = new ZArray(null, null, null, dtype, null, ORDER, null, DELIMITER);
20+
assertThat(zArray.getDataType()).isEqualTo(DataType.DOUBLE);
21+
assertThat(zArray.getDtype()).isEqualTo(dtype);
22+
}
23+
24+
@Test
25+
public void shouldRefuseInvalidDataType() {
26+
final String dtype = ">a5";
27+
assertThrows(ZarrFormatException.class, () -> new ZArray(null, null, null, dtype, null, ORDER, null, DELIMITER));
28+
}
29+
30+
@Test
31+
public void shouldParseBigEndianByteOrder() throws ZarrFormatException {
32+
final String dtype = ">f4";
33+
final ZArray zArray = new ZArray(null, null, null, dtype, null, ORDER, null, DELIMITER);
34+
assertThat(zArray.getByteOrder()).isEqualTo(ByteOrder.BIG_ENDIAN);
35+
}
36+
37+
@Test
38+
public void shouldParseLittleEndianByteOrder() throws ZarrFormatException {
39+
final String dtype = "<f4";
40+
final ZArray zArray = new ZArray(null, null, null, dtype, null, ORDER, null, DELIMITER);
41+
assertThat(zArray.getByteOrder()).isEqualTo(ByteOrder.LITTLE_ENDIAN);
42+
}
43+
44+
@Test
45+
public void shouldParseNativeByteOrder() throws ZarrFormatException {
46+
final String dtype = "|f4";
47+
final ZArray zArray = new ZArray(null, null, null, dtype, null, ORDER, null, DELIMITER);
48+
assertThat(zArray.getByteOrder()).isEqualTo(ByteOrder.nativeOrder());
49+
}
50+
51+
@Test
52+
public void shouldRefuseInvalidByteOrder() {
53+
final String dtype = "f4";
54+
assertThrows(ZarrFormatException.class, () -> new ZArray(null, null, null, dtype, null, ORDER, null, DELIMITER));
55+
}
56+
57+
@Test
58+
public void shouldParseRowMajorOrder() throws ZarrFormatException {
59+
final String order = "C";
60+
final ZArray zArray = new ZArray(null, null, null, DTYPE, null, order, null, DELIMITER);
61+
assertThat(zArray.getOrder()).isEqualTo(Order.C);
62+
}
63+
64+
@Test
65+
public void shouldParseColumnMajorOrder() throws ZarrFormatException {
66+
final String order = "F";
67+
final ZArray zArray = new ZArray(null, null, null, DTYPE, null, order, null, DELIMITER);
68+
assertThat(zArray.getOrder()).isEqualTo(Order.F);
69+
}
70+
71+
@Test
72+
public void shouldRefuseInvalidOrder() {
73+
final String order = "A";
74+
assertThrows(ZarrFormatException.class, () -> new ZArray(null, null, null, DTYPE, null, order, null, DELIMITER));
75+
}
76+
77+
@Test
78+
public void shouldParseDotSeparator() throws ZarrFormatException {
79+
final String separator = ".";
80+
final ZArray zArray = new ZArray(null, null, null, DTYPE, null, ORDER, null, separator);
81+
assertThat(zArray.getSeparator()).isEqualTo(separator);
82+
}
83+
84+
@Test
85+
public void shouldParseSlashSeparator() throws ZarrFormatException {
86+
final String separator = "/";
87+
final ZArray zArray = new ZArray(null, null, null, DTYPE, null, ORDER, null, separator);
88+
assertThat(zArray.getSeparator()).isEqualTo(separator);
89+
}
90+
91+
@Test
92+
public void shouldRefuseInvalidSeparator() {
93+
final String separator = "-";
94+
assertThrows(ZarrFormatException.class, () -> new ZArray(null, null, null, DTYPE, null, ORDER, null, separator));
95+
}
96+
}

‎cdm/zarr/src/test/java/ucar/nc2/iosp/zarr/TestZarrDataTypes.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
import org.junit.Test;
1111
import org.slf4j.Logger;
1212
import org.slf4j.LoggerFactory;
13-
import ucar.ma2.Array;
1413
import ucar.ma2.DataType;
1514
import ucar.ma2.InvalidRangeException;
1615
import ucar.nc2.NetcdfFile;

‎cdm/zarr/src/test/java/ucar/nc2/iosp/zarr/TestZarrIosp.java‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,11 @@
55

66
package ucar.nc2.iosp.zarr;
77

8+
import java.nio.file.*;
89
import org.junit.BeforeClass;
10+
import org.junit.ClassRule;
911
import org.junit.Test;
12+
import org.junit.rules.TemporaryFolder;
1013
import ucar.ma2.Array;
1114
import ucar.ma2.DataType;
1215
import ucar.ma2.InvalidRangeException;
@@ -15,7 +18,6 @@
1518
import ucar.nc2.NetcdfFile;
1619
import ucar.nc2.NetcdfFiles;
1720
import ucar.nc2.Variable;
18-
import ucar.nc2.filter.Filters;
1921

2022
import java.io.IOException;
2123
import java.nio.ByteOrder;
@@ -28,6 +30,9 @@
2830
*/
2931
public class TestZarrIosp {
3032

33+
@ClassRule
34+
public static TemporaryFolder tempFolder = new TemporaryFolder();
35+
3136
// file names
3237
private static final String ZARR_FILENAME = "zarr_test_data.zarr/";
3338
private static final String ZARR_ZIP_NAME = "zarr_test_data.zip";
@@ -46,18 +51,23 @@ public class TestZarrIosp {
4651

4752
// Non zarr zipped file
4853
private static final String NON_ZARR_DATA = ZarrTestsCommon.LOCAL_TEST_DATA_PATH + NON_ZARR_FILENAME;
54+
// Copy to temp folder so unzipped file created when opened will get cleaned up after test
55+
private static String TEMP_NON_ZARR_DATA;
4956

5057
// fill values file
5158
private static final String FILL_VALUES_DATA = ZarrTestsCommon.LOCAL_TEST_DATA_PATH + FILL_VALUES_FILENAME;
5259

5360
private static List<String> stores;
5461

5562
@BeforeClass
56-
public static void setUpTests() {
63+
public static void setUpTests() throws IOException {
5764
stores = new ArrayList<>();
5865
stores.add(DIRECTORY_STORE_URI);
5966
stores.add(ZIP_STORE_URI);
6067
stores.add(OBJECT_STORE_ZARR_URI);
68+
69+
TEMP_NON_ZARR_DATA =
70+
Files.copy(Paths.get(NON_ZARR_DATA), Paths.get(tempFolder.getRoot().toString(), NON_ZARR_FILENAME)).toString();
6171
}
6272

6373
@Test
@@ -72,7 +82,7 @@ public void testIsValidFile() throws IOException {
7282
@Test
7383
public void testNonZarrZipIsNotValid() throws IOException {
7484
ZarrIosp iosp = new ZarrIosp();
75-
assertThat(iosp.isValidFile(NetcdfFiles.getRaf(NON_ZARR_DATA, -1))).isFalse();
85+
assertThat(iosp.isValidFile(NetcdfFiles.getRaf(TEMP_NON_ZARR_DATA, -1))).isFalse();
7686
}
7787

7888
//////////////////////////////////////////////////////
@@ -254,7 +264,7 @@ private void _testUninitialized(String location) throws IOException, InvalidRang
254264

255265
@Test
256266
public void testReadNonZarrZipFile() throws IOException {
257-
try (NetcdfFile ncfile = NetcdfFiles.open(NON_ZARR_DATA)) {
267+
try (NetcdfFile ncfile = NetcdfFiles.open(TEMP_NON_ZARR_DATA)) {
258268
assertThat(ncfile).isNotNull();
259269
assertThat(ncfile.findDimension("x")).isNotNull();
260270
}

0 commit comments

Comments
 (0)