Skip to content

Commit 71654bf

Browse files
author
haileyajohnson
authored
Merge pull request #775 from lesserwhirls/dtCoverageEnhance
[5.x]: DtCoverageDataset bug and failing Jenkins tests
2 parents 3d240f5 + 0ae2861 commit 71654bf

5 files changed

Lines changed: 54 additions & 34 deletions

File tree

‎cdm-test/src/test/java/ucar/nc2/ft/coverage/TestRemoteCoverage.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
2+
* Copyright (c) 1998-2021 John Caron and University Corporation for Atmospheric Research/Unidata
33
* See LICENSE for license information.
44
*/
55
package ucar.nc2.ft.coverage;
@@ -58,7 +58,7 @@ public void testCdmRemoteCoverage() throws Exception {
5858

5959
int[] shape = geoCoordsys.getShape();
6060
logger.debug("grid_section.getShape = {}", new Section(shape));
61-
int[] expectShape = new int[] {1, 31, 241, 480};
61+
int[] expectShape = new int[] {1, 41, 241, 480};
6262
Assert.assertArrayEquals("subset shape", expectShape, shape);
6363
}
6464
}

‎cdm-test/src/test/java/ucar/nc2/grib/TestGribCdmIndexUpdating.java‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
2+
* Copyright (c) 1998-2021 University Corporation for Atmospheric Research/Unidata
33
* See LICENSE for license information.
44
*/
55

@@ -27,6 +27,7 @@
2727
import java.io.IOException;
2828
import java.lang.invoke.MethodHandles;
2929
import java.util.ArrayList;
30+
import java.util.Arrays;
3031
import java.util.List;
3132

3233
/**
@@ -48,9 +49,8 @@ public static List<Object[]> getTestParameters() {
4849
String dataDir = TestDir.cdmUnitTestDir + "gribCollections/changing/filePartition/";
4950
FeatureCollectionConfig config = new FeatureCollectionConfig("TestGribCdmIndex", "changing/filePartition",
5051
FeatureCollectionType.GRIB1, dataDir + "GFS_CONUS_80km_#yyyyMMdd_HHmm#.grib1", null, null, null, "file", null);
51-
// String dataDir, String newModel, FeatureCollectionConfig config, String indexFile, String varIdValue, int orgLen
52-
result.add(new Object[] {dataDir, "GFS_CONUS_80km_20141024_1200.grib1", config, "TestGribCdmIndex.ncx4",
53-
"Relative_humidity_isobaric", 4, 3});
52+
result
53+
.add(new Object[] {dataDir, "GFS_CONUS_80km_20141024_1200.grib1", config, "Relative_humidity_isobaric", 4, 3});
5454

5555
/*
5656
* directory partition
@@ -73,6 +73,12 @@ public static List<Object[]> getTestParameters() {
7373
return result;
7474
}
7575

76+
@Parameterized.AfterParam
77+
public static void after(String dataDir, String newModel, FeatureCollectionConfig config, String varName, int orgLen,
78+
int remLen) {
79+
// cleanup index files created during test for a given parameter input
80+
cleanUpIndexFiles(GribCdmIndex.getTopIndexFileFromConfig(config).getParent());
81+
}
7682

7783
///////////////////////////////////////
7884

@@ -83,15 +89,16 @@ public static List<Object[]> getTestParameters() {
8389
String varName;
8490
int orgLen, remLen;
8591

86-
public TestGribCdmIndexUpdating(String dataDir, String newModel, FeatureCollectionConfig config, String indexFile,
87-
String varName, int orgLen, int remLen) {
92+
public TestGribCdmIndexUpdating(String dataDir, String newModel, FeatureCollectionConfig config, String varName,
93+
int orgLen, int remLen) throws IOException {
8894
this.dataDir = dataDir;
8995
this.newModel = newModel;
9096
this.config = config;
91-
this.indexFile = indexFile;
9297
this.varName = varName;
9398
this.orgLen = orgLen;
9499
this.remLen = remLen;
100+
GribCdmIndex.updateGribCollection(config, CollectionUpdateType.always, logger);
101+
this.indexFile = GribCdmIndex.getTopIndexFileFromConfig(config).getAbsolutePath();
95102
}
96103

97104
@Test
@@ -140,7 +147,7 @@ private void testRemoveFileFromCollection(CollectionUpdateType updateType, int o
140147
System.out.printf("changed = %s%n", changed);
141148

142149
// open the resulting index
143-
try (NetcdfFile ncfile = NetcdfFiles.open(dataDir + indexFile)) {
150+
try (NetcdfFile ncfile = NetcdfFiles.open(indexFile)) {
144151
System.out.printf("opened = %s%n", ncfile.getLocation());
145152
Group g = ncfile.findGroup("TwoD");
146153
assert g != null;
@@ -160,7 +167,7 @@ private void testRemoveFileFromCollection(CollectionUpdateType updateType, int o
160167
System.out.printf("changed2 = %s%n", changed2);
161168

162169
// open the resulting index
163-
try (NetcdfFile ncfile = NetcdfFiles.open(dataDir + indexFile)) {
170+
try (NetcdfFile ncfile = NetcdfFiles.open(indexFile)) {
164171
System.out.printf("opened = %s%n", ncfile.getLocation());
165172
Group g = ncfile.findGroup("TwoD");
166173
assert g != null;
@@ -174,8 +181,11 @@ private void testRemoveFileFromCollection(CollectionUpdateType updateType, int o
174181
} finally { // leave it it the way we found it
175182
boolean ok = newModelFileSave.renameTo(newModelFile);
176183
}
177-
178184
}
179185

180-
186+
private static void cleanUpIndexFiles(String indexFile) {
187+
File indexDir = new File(indexFile).getParentFile();
188+
Arrays.stream(indexDir.listFiles()).filter(f -> f.getName().endsWith(GribCdmIndex.NCX_SUFFIX))
189+
.forEach(File::delete);
190+
}
181191
}

‎cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionCoordinates.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -90,12 +90,13 @@ public void testExtraCoordinates() throws IOException {
9090
config.gribConfig.setExcludeZero(true); // no longer the default
9191

9292
boolean changed = GribCdmIndex.updateGribCollection(config, CollectionUpdateType.always, logger);
93+
String topLevelIndex = GribCdmIndex.getTopIndexFileFromConfig(config).getAbsolutePath();
94+
9395
System.out.printf("changed = %s%n", changed);
9496

9597
boolean ok = true;
9698

97-
try (NetcdfDataset ds =
98-
NetcdfDatasets.openDataset(TestDir.cdmUnitTestDir + "gribCollections/namAlaska22/namAlaska22.ncx4")) {
99+
try (NetcdfDataset ds = NetcdfDatasets.openDataset(topLevelIndex)) {
99100
for (Variable vds : ds.getVariables()) {
100101
String stdname = vds.findAttributeString("standard_name", "no");
101102
if (!stdname.equalsIgnoreCase("time"))
@@ -147,11 +148,10 @@ public void testBestReftimeMonotonic() throws IOException {
147148

148149
boolean changed = GribCdmIndex.updateGribCollection(config, updateMode, logger);
149150
System.out.printf("changed = %s%n", changed);
150-
151+
String topLevelIndex = GribCdmIndex.getTopIndexFileFromConfig(config).getAbsolutePath();
151152
boolean ok = true;
152153

153-
try (NetcdfDataset ds =
154-
NetcdfDatasets.openDataset(TestDir.cdmUnitTestDir + "gribCollections/gfs_2p5deg/gfs_2p5deg.ncx4")) {
154+
try (NetcdfDataset ds = NetcdfDatasets.openDataset(topLevelIndex)) {
155155
for (Variable vds : ds.getVariables()) {
156156
String stdname = vds.findAttributeString("standard_name", "no");
157157
if (!stdname.equalsIgnoreCase("forecast_reference_time"))

‎cdm/core/src/main/java/ucar/nc2/ft2/coverage/adapter/DtCoverageDataset.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ public DtCoverageDataset(NetcdfDataset ncd, Formatter parseInfo) throws IOExcept
118118
Set<NetcdfDataset.Enhance> enhance = ncd.getEnhanceMode();
119119
if (enhance == null || enhance.isEmpty())
120120
enhance = NetcdfDataset.getDefaultEnhanceMode();
121-
NetcdfDatasets.enhance(ncd, enhance, null);
121+
ncd = NetcdfDatasets.enhance(ncd, enhance, null);
122122

123123
DtCoverageCSBuilder facc = DtCoverageCSBuilder.classify(ncd, parseInfo);
124124
if (facc != null)

‎cdm/s3/src/test/java/thredds/filesystem/s3/TestControllerS3.java‎

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -68,55 +68,55 @@ public static void setup() {
6868
@Test
6969
public void testGetInventoryTopBucketNoDelimiterAws() throws URISyntaxException {
7070
CdmS3Uri uri = new CdmS3Uri(S3TestsCommon.TOP_LEVEL_AWS_BUCKET);
71-
checkInventoryTopCount(uri, LIMIT_COUNT_MAX);
71+
checkInventoryTopCountExact(uri, LIMIT_COUNT_MAX);
7272
}
7373

7474
@Test
7575
public void testGetInventoryTopBucketNoDelimiterGcs() throws URISyntaxException {
7676
CdmS3Uri uri = new CdmS3Uri(S3TestsCommon.TOP_LEVEL_GCS_BUCKET);
77-
checkInventoryTopCount(uri, LIMIT_COUNT_MAX);
77+
checkInventoryTopCountExact(uri, LIMIT_COUNT_MAX);
7878
}
7979

8080
@Test
8181
public void testGetInventoryTopBucketGcsNoDelimiterOsdc() throws URISyntaxException {
8282
CdmS3Uri uri = new CdmS3Uri(S3TestsCommon.TOP_LEVEL_OSDC_BUCKET);
83-
checkInventoryTopCount(uri, LIMIT_COUNT_MAX);
83+
checkInventoryTopCountExact(uri, LIMIT_COUNT_MAX);
8484
}
8585

8686
@Test
8787
public void testGetInventoryTopBucketDelimiterAws() throws URISyntaxException {
8888
CdmS3Uri uri = new CdmS3Uri(S3TestsCommon.TOP_LEVEL_AWS_BUCKET + DELIMITER_FRAGMENT);
8989
// contains a single object at "/" (/index.html)
90-
checkInventoryTopCount(uri, 1);
90+
checkInventoryTopCountAtMost(uri, 3);
9191
}
9292

9393
@Test
9494
public void testGetInventoryTopBucketDelimiterGcs() throws URISyntaxException {
9595
CdmS3Uri uri = new CdmS3Uri(S3TestsCommon.TOP_LEVEL_GCS_BUCKET + DELIMITER_FRAGMENT);
9696
// does not contain anything at "/"
97-
checkInventoryTopCount(uri, 0);
97+
checkInventoryTopCountExact(uri, 0);
9898
}
9999

100100
@Test
101101
public void testGetInventoryTopBucketDelimiterOsdc() throws URISyntaxException {
102102
CdmS3Uri uri = new CdmS3Uri(S3TestsCommon.TOP_LEVEL_OSDC_BUCKET + DELIMITER_FRAGMENT);
103103
// does not contain anything at "/"
104-
checkInventoryTopCount(uri, 0);
104+
checkInventoryTopCountExact(uri, 0);
105105
}
106106

107107
@Test
108108
public void testGetInventoryTopBucketAndPrefixSingleMatchAws() throws URISyntaxException {
109109
for (String delimiter : DELIMITER_FRAGMENTS) {
110110
CdmS3Uri uri = new CdmS3Uri(S3TestsCommon.TOP_LEVEL_AWS_BUCKET + "?" + G16_KEY_PREFIX_SINGLE_MATCH + delimiter);
111-
checkInventoryTopCount(uri, 1);
111+
checkInventoryTopCountExact(uri, 1);
112112
}
113113
}
114114

115115
@Test
116116
public void testGetInventoryTopBucketAndPrefixSingleMatchGcs() throws URISyntaxException {
117117
for (String delimiter : DELIMITER_FRAGMENTS) {
118118
CdmS3Uri uri = new CdmS3Uri(S3TestsCommon.TOP_LEVEL_GCS_BUCKET + "?" + G16_KEY_PREFIX_SINGLE_MATCH + delimiter);
119-
checkInventoryTopCount(uri, 1);
119+
checkInventoryTopCountExact(uri, 1);
120120
}
121121
}
122122

@@ -125,7 +125,7 @@ public void testGetInventoryTopBucketAndPrefixSingleMatchOsdc() throws URISyntax
125125
for (String delimiter : DELIMITER_FRAGMENTS) {
126126
CdmS3Uri uri =
127127
new CdmS3Uri(S3TestsCommon.TOP_LEVEL_OSDC_BUCKET + "?" + getOsdcKey(G16_KEY_PREFIX_SINGLE_MATCH) + delimiter);
128-
checkInventoryTopCount(uri, 1);
128+
checkInventoryTopCountExact(uri, 1);
129129
}
130130
}
131131

@@ -134,7 +134,7 @@ public void testGetInventoryTopBucketAndPrefixMultiMatchAws() throws URISyntaxEx
134134
for (String delimiter : DELIMITER_FRAGMENTS) {
135135
CdmS3Uri uri =
136136
new CdmS3Uri(S3TestsCommon.TOP_LEVEL_AWS_BUCKET + "?" + G16_KEY_PREFIX_SINGLE_DAY_PARTIAL_FILE + delimiter);
137-
checkInventoryTopCount(uri, 12);
137+
checkInventoryTopCountExact(uri, 12);
138138
}
139139
}
140140

@@ -143,7 +143,7 @@ public void testGetInventoryTopBucketAndPrefixMultiMatchGcs() throws URISyntaxEx
143143
for (String delimiter : DELIMITER_FRAGMENTS) {
144144
CdmS3Uri uri =
145145
new CdmS3Uri(S3TestsCommon.TOP_LEVEL_GCS_BUCKET + "?" + G16_KEY_PREFIX_SINGLE_DAY_PARTIAL_FILE + delimiter);
146-
checkInventoryTopCount(uri, 12);
146+
checkInventoryTopCountExact(uri, 12);
147147
}
148148
}
149149

@@ -152,7 +152,7 @@ public void testGetInventoryTopBucketAndPrefixMultiMatchOsdc() throws URISyntaxE
152152
for (String delimiter : DELIMITER_FRAGMENTS) {
153153
CdmS3Uri uri = new CdmS3Uri(
154154
S3TestsCommon.TOP_LEVEL_OSDC_BUCKET + "?" + getOsdcKey(G16_KEY_PREFIX_SINGLE_DAY_PARTIAL_FILE) + delimiter);
155-
checkInventoryTopCount(uri, 12);
155+
checkInventoryTopCountExact(uri, 12);
156156
}
157157
}
158158

@@ -286,12 +286,22 @@ private CollectionConfig getCollectionConfig(CdmS3Uri uri) {
286286
return new CollectionConfig(uri.getBucket(), uri.toString(), true, null, null);
287287
}
288288

289-
private void checkInventoryTopCount(CdmS3Uri uri, int expectedCount) {
289+
private void checkInventoryTopCountExact(CdmS3Uri uri, int expectedCount) {
290+
int actualCount = topInventoryCount(uri);
291+
assertThat(actualCount).isEqualTo(expectedCount);
292+
}
293+
294+
private void checkInventoryTopCountAtMost(CdmS3Uri uri, int expectedMaximumCount) {
295+
int actualCount = topInventoryCount(uri);
296+
assertThat(actualCount).isAtMost(expectedMaximumCount);
297+
}
298+
299+
private int topInventoryCount(CdmS3Uri uri) {
290300
logger.debug("getInventoryTop: {}", uri);
291301
ControllerS3 controller = new ControllerS3();
292302
controller.limit = true;
293303
Iterator<MFile> it = controller.getInventoryTop(getCollectionConfig(uri), false);
294-
assertThat(countObjects(it)).isEqualTo(expectedCount);
304+
return countObjects(it);
295305
}
296306

297307
private void checkInventoryAllCount(CdmS3Uri uri, int expectedCount) {
@@ -303,7 +313,7 @@ private void checkInventoryAllCount(CdmS3Uri uri, int expectedCount) {
303313
}
304314

305315
private void checkSubdirsCount(CdmS3Uri uri, int expectedCount) {
306-
logger.debug("getInventoryAll: {}", uri);
316+
logger.debug("getSubdirs: {}", uri);
307317
ControllerS3 controller = new ControllerS3();
308318
controller.limit = true;
309319
Iterator<MFile> it = controller.getSubdirs(getCollectionConfig(uri), false);

0 commit comments

Comments
 (0)