Skip to content

Commit 10c421d

Browse files
lesserwhirlslesserwhirls
authored
Merge pull request #1439 from lesserwhirls/numtime
Enable uduints based time coordinates on scan aggregations
2 parents 5134f31 + e0ceef6 commit 10c421d

16 files changed

Lines changed: 261 additions & 76 deletions

File tree

‎cdm/core/src/main/java/ucar/nc2/internal/ncml/AggDatasetOuter.java‎

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
1-
/* Copyright Unidata */
1+
/*
2+
* Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
3+
* See LICENSE.txt for license information.
4+
*/
5+
26
package ucar.nc2.internal.ncml;
37

48
import java.io.IOException;
@@ -11,6 +15,7 @@
1115
import javax.annotation.Nullable;
1216
import thredds.inventory.MFile;
1317
import ucar.ma2.Array;
18+
import ucar.ma2.DataType;
1419
import ucar.ma2.InvalidRangeException;
1520
import ucar.ma2.Range;
1621
import ucar.ma2.Section;
@@ -32,7 +37,8 @@ class AggDatasetOuter extends AggDataset {
3237
@Nullable
3338
final String coordValue; // if theres a coordValue on the netcdf element - may be multiple, blank separated
3439
final Date coordValueDate; // if its a date
35-
final boolean isStringValued; // if coordinat is a String
40+
final DataType coordDataType; // coordinate data type
41+
final String coordUdunit; // coordinate udunit string, if numeric
3642

3743
// not final because of deffered read
3844
int ncoord; // number of coordinates in outer dimension
@@ -68,18 +74,18 @@ class AggDatasetOuter extends AggDataset {
6874
}
6975
}
7076

71-
boolean isString = false;
77+
DataType aggCoordDataType = DataType.DOUBLE;
7278
if ((aggregationOuter.type == Type.joinNew) || (aggregationOuter.type == Type.joinExistingOne)
7379
|| (aggregationOuter.type == Type.forecastModelRunCollection)) {
7480
if (coordValueS == null) {
7581
coordValueS = extractCoordNameFromFilename(this.getLocation());
76-
isString = true;
82+
aggCoordDataType = DataType.STRING;
7783
} else {
7884
// we just need to know if its string valued
7985
try {
8086
Double.parseDouble(coordValueS);
8187
} catch (NumberFormatException e) {
82-
isString = true;
88+
aggCoordDataType = DataType.STRING;
8389
}
8490
}
8591
}
@@ -90,8 +96,10 @@ class AggDatasetOuter extends AggDataset {
9096
this.ncoord = stoker.countTokens();
9197
}
9298

93-
this.isStringValued = isString; // LOOK ??
99+
coordDataType = aggCoordDataType;
94100
this.coordValue = coordValueS;
101+
// not dealing with scan
102+
this.coordUdunit = "";
95103
this.coordValueDate = null; // LOOK why isnt this set?
96104
}
97105

@@ -115,13 +123,21 @@ private String extractCoordNameFromFilename(String loc) {
115123
// default is that the coordinates are just the filenames
116124
// this can be overriden by an explicit declaration, which will replace the variable after ther agg is processed in
117125
// NcMLReader
126+
DataType aggCoordDataType = DataType.DOUBLE;
127+
String coordUdunit = "";
118128
if ((aggregationOuter.type == Type.joinNew) || (aggregationOuter.type == Type.joinExistingOne)
119129
|| (aggregationOuter.type == Type.forecastModelRunCollection)) {
120130
coordValueS = extractCoordNameFromFilename(this.getLocation());
121-
this.isStringValued = true;
122-
} else {
123-
this.isStringValued = false; // LOOK ??
131+
if (aggregationOuter.numericTimeSettings != null) {
132+
String[] settings = aggregationOuter.numericTimeSettings.split(" ", 2);
133+
aggCoordDataType = DataType.getType(settings[0]);
134+
coordUdunit = settings[1];
135+
} else {
136+
aggCoordDataType = DataType.STRING;
137+
}
124138
}
139+
this.coordDataType = aggCoordDataType;
140+
this.coordUdunit = coordUdunit;
125141

126142
if (null != aggregationOuter.dateFormatMark) {
127143
String filename = cd.getName(); // LOOK operates on name, not path

‎cdm/core/src/main/java/ucar/nc2/internal/ncml/Aggregation.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
/*
2-
* Copyright (c) 1998-2020 John Caron and University Corporation for Atmospheric Research/Unidata
2+
* Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
3+
* See LICENSE.txt for license information.
34
*/
5+
46
package ucar.nc2.internal.ncml;
57

68
import org.jdom2.Element;
@@ -95,6 +97,7 @@ else if (mode.equalsIgnoreCase("first"))
9597

9698
// experimental
9799
protected String dateFormatMark;
100+
protected String numericTimeSettings;
98101
// protected EnumSet<NetcdfDataset.Enhance> enhance = null; // default no enhancement
99102
protected boolean isDate;
100103
protected DateFormatter dateFormatter = new DateFormatter();
@@ -153,13 +156,17 @@ public void addDataset(AggDataset nested) {
153156
* @param subdirs equals "false" if should not descend into subdirectories
154157
* @param olderThan files must be older than this time (now - lastModified >= olderThan); must be a time unit, may ne
155158
* bull
159+
* @param numericTimeSettings numeric time settings (data type and udunits compatible string,
160+
* e.g. "float seconds since 1985-10-18T12:31:00", may be null)
156161
*/
157162
public void addDatasetScan(Element crawlableDatasetElement, String dirName, String suffix, String regexpPatternString,
158-
String dateFormatMark, Set<NetcdfDataset.Enhance> enhanceMode, String subdirs, String olderThan) {
163+
String dateFormatMark, Set<NetcdfDataset.Enhance> enhanceMode, String subdirs, String olderThan,
164+
String numericTimeSettings) {
159165

160166
datasetManager.addDirectoryScan(dirName, suffix, regexpPatternString, subdirs, olderThan, enhanceMode);
161167

162168
this.dateFormatMark = dateFormatMark;
169+
this.numericTimeSettings = numericTimeSettings;
163170
if (dateFormatMark != null) {
164171
isDate = true;
165172
if (type == Type.joinExisting)

‎cdm/core/src/main/java/ucar/nc2/internal/ncml/AggregationExisting.java‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
2+
* Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
33
* See LICENSE for license information.
44
*/
55

@@ -76,7 +76,13 @@ protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
7676
}
7777

7878
} else {
79-
coordCacheVar = new CoordValueVar(dimName, DataType.STRING, "");
79+
DataType coordDataType = DataType.STRING;
80+
String coordUnits = "";
81+
if (typicalDataset instanceof AggDatasetOuter) {
82+
coordDataType = ((AggDatasetOuter) typicalDataset).coordDataType;
83+
coordUnits = ((AggDatasetOuter) typicalDataset).coordUdunit;
84+
}
85+
coordCacheVar = new CoordValueVar(dimName, coordDataType, coordUnits);
8086
}
8187
if (coordCacheVar != null) {
8288
cacheList.add(coordCacheVar); // coordinate variable is always cached
@@ -134,8 +140,14 @@ protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
134140
if (type == Type.joinExistingOne) {
135141
// replace aggregation coordinate variable
136142
joinAggCoordOpt.ifPresent(joinAgg -> rootGroup.removeVariable(joinAgg.shortName));
143+
DataType coordDataType = DataType.STRING;
144+
String coordUnits = "";
145+
if (typicalDataset instanceof AggDatasetOuter) {
146+
coordDataType = ((AggDatasetOuter) typicalDataset).coordDataType;
147+
coordUnits = ((AggDatasetOuter) typicalDataset).coordUdunit;
148+
}
137149

138-
joinAggCoord = VariableDS.builder().setName(dimName).setDataType(DataType.STRING).setParentGroupBuilder(rootGroup)
150+
joinAggCoord = VariableDS.builder().setName(dimName).setDataType(coordDataType).setParentGroupBuilder(rootGroup)
139151
.setDimensionsByName(dimName);
140152
joinAggCoord.setProxyReader(this);
141153
rootGroup.addVariable(joinAggCoord);
@@ -144,6 +156,9 @@ protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
144156
joinAggCoord.addAttribute(new Attribute(_Coordinate.AxisType, "Time"));
145157
joinAggCoord.addAttribute(new Attribute(CDM.LONG_NAME, "time coordinate"));
146158
joinAggCoord.addAttribute(new Attribute(CF.STANDARD_NAME, "time"));
159+
if (coordUnits != null && !coordUnits.equals("")) {
160+
joinAggCoord.addAttribute(new Attribute(CF.UNITS, coordUnits));
161+
}
147162
}
148163

149164
if (timeUnitsChange && joinAggCoord != null) {

‎cdm/core/src/main/java/ucar/nc2/internal/ncml/AggregationNew.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
2+
* Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
33
* See LICENSE for license information.
44
*/
55

@@ -149,7 +149,7 @@ protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
149149
private DataType getCoordinateType() {
150150
List<AggDataset> nestedDatasets = getDatasets();
151151
AggDatasetOuter first = (AggDatasetOuter) nestedDatasets.get(0);
152-
return first.isStringValued ? DataType.STRING : DataType.DOUBLE;
152+
return first.coordDataType;
153153
}
154154

155155
}

‎cdm/core/src/main/java/ucar/nc2/internal/ncml/NcmlReader.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
/*
2-
* Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
2+
* Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
33
* See LICENSE for license information.
44
*/
5+
56
package ucar.nc2.internal.ncml;
67

78
import java.io.FileNotFoundException;
@@ -1526,6 +1527,7 @@ private Aggregation readAgg(Element aggElem, String ncmlLocation, NetcdfDataset.
15261527
String olderS = scanElem.getAttributeValue("olderThan");
15271528

15281529
String dateFormatMark = scanElem.getAttributeValue("dateFormatMark");
1530+
String numericTimeSettings = scanElem.getAttributeValue("numericTimeSettings");
15291531
Set<NetcdfDataset.Enhance> enhanceMode = NetcdfDataset.parseEnhanceMode(scanElem.getAttributeValue("enhance"));
15301532

15311533
// possible relative location
@@ -1534,7 +1536,7 @@ private Aggregation readAgg(Element aggElem, String ncmlLocation, NetcdfDataset.
15341536
// can embed a full-blown crawlableDatasetImpl element
15351537
Element cdElement = scanElem.getChild("crawlableDatasetImpl", ncNS); // ok if null
15361538
agg.addDatasetScan(cdElement, dirLocation, suffix, regexpPatternString, dateFormatMark, enhanceMode, subdirs,
1537-
olderS);
1539+
olderS, numericTimeSettings);
15381540

15391541
if ((cancelTask != null) && cancelTask.isCancel()) {
15401542
return agg;

‎cdm/core/src/main/java/ucar/nc2/ncml/Aggregation.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
* Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
33
* See LICENSE.txt for license information.
44
*/
5+
56
package ucar.nc2.ncml;
67

78
import org.jdom2.Element;
@@ -153,6 +154,7 @@ else if (mode.equalsIgnoreCase("first"))
153154

154155
// experimental
155156
protected String dateFormatMark;
157+
protected String numericTimeSettings;
156158
// protected EnumSet<NetcdfDataset.Enhance> enhance = null; // default no enhancement
157159
protected boolean isDate;
158160
protected DateFormatter dateFormatter = new DateFormatter();
@@ -199,6 +201,7 @@ public void addDataset(Dataset nested) {
199201
explicitDatasets.add(nested);
200202
}
201203

204+
202205
/**
203206
* Add a dataset scan
204207
*
@@ -214,10 +217,33 @@ public void addDataset(Dataset nested) {
214217
*/
215218
public void addDatasetScan(Element crawlableDatasetElement, String dirName, String suffix, String regexpPatternString,
216219
String dateFormatMark, Set<NetcdfDataset.Enhance> enhanceMode, String subdirs, String olderThan) {
220+
addDatasetScan(crawlableDatasetElement, dirName, suffix, regexpPatternString, dateFormatMark, enhanceMode, subdirs,
221+
olderThan, null);
222+
}
223+
224+
/**
225+
* Add a dataset scan
226+
*
227+
* @param crawlableDatasetElement defines a CrawlableDataset, or null
228+
* @param dirName scan this directory
229+
* @param suffix filter on this suffix (may be null)
230+
* @param regexpPatternString include if full name matches this regular expression (may be null)
231+
* @param dateFormatMark create dates from the filename (may be null)
232+
* @param enhanceMode how should files be enhanced
233+
* @param subdirs equals "false" if should not descend into subdirectories
234+
* @param olderThan files must be older than this time (now - lastModified >= olderThan); must be a time unit, may ne
235+
* bull
236+
* @param numericTimeSettings numeric time settings (data type and udunits compatible string,
237+
* e.g. "float seconds since 1985-10-18T12:31:00", may be null)
238+
*/
239+
public void addDatasetScan(Element crawlableDatasetElement, String dirName, String suffix, String regexpPatternString,
240+
String dateFormatMark, Set<NetcdfDataset.Enhance> enhanceMode, String subdirs, String olderThan,
241+
String numericTimeSettings) {
217242

218243
datasetManager.addDirectoryScan(dirName, suffix, regexpPatternString, subdirs, olderThan, enhanceMode);
219244

220245
this.dateFormatMark = dateFormatMark;
246+
this.numericTimeSettings = numericTimeSettings;
221247
if (dateFormatMark != null) {
222248
isDate = true;
223249
if (type == Type.joinExisting)

‎cdm/core/src/main/java/ucar/nc2/ncml/AggregationExisting.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
2+
* Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
33
* See LICENSE for license information.
44
*/
55

@@ -71,7 +71,13 @@ protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
7171
}
7272

7373
} else {
74-
coordCacheVar = new CoordValueVar(dimName, DataType.STRING, "");
74+
DataType coordDataType = DataType.STRING;
75+
String coordUnits = null;
76+
if (typicalDataset instanceof AggregationOuterDimension.DatasetOuterDimension) {
77+
coordDataType = ((AggregationOuterDimension.DatasetOuterDimension) typicalDataset).coordDataType;
78+
coordUnits = ((AggregationOuterDimension.DatasetOuterDimension) typicalDataset).coordUdunit;
79+
}
80+
coordCacheVar = new CoordValueVar(dimName, coordDataType, coordUnits);
7581
}
7682
if (coordCacheVar != null) {
7783
cacheList.add(coordCacheVar); // coordinate variable is always cached
@@ -128,15 +134,24 @@ protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
128134
// replace aggregation coordinate variable
129135
ncDataset.getRootGroup().removeVariable(joinAggCoord.getShortName());
130136
}
131-
132-
joinAggCoord = new VariableDS(ncDataset, null, null, dimName, DataType.STRING, dimName, null, null);
137+
DataType coordDataType = DataType.STRING;
138+
String coordUnits = null;
139+
if (typicalDataset instanceof AggregationOuterDimension.DatasetOuterDimension) {
140+
coordDataType = ((AggregationOuterDimension.DatasetOuterDimension) typicalDataset).coordDataType;
141+
coordUnits = ((AggregationOuterDimension.DatasetOuterDimension) typicalDataset).coordUdunit;
142+
}
143+
coordCacheVar = new CoordValueVar(dimName, coordDataType, coordUnits);
144+
joinAggCoord = new VariableDS(ncDataset, null, null, dimName, coordDataType, dimName, coordUnits, null);
133145
joinAggCoord.setProxyReader(this);
134146
ncDataset.getRootGroup().addVariable(joinAggCoord);
135147
aggVars.add(joinAggCoord);
136148

137149
joinAggCoord.addAttribute(new ucar.nc2.Attribute(_Coordinate.AxisType, "Time"));
138150
joinAggCoord.addAttribute(new Attribute(CDM.LONG_NAME, "time coordinate"));
139151
joinAggCoord.addAttribute(new ucar.nc2.Attribute(CF.STANDARD_NAME, "time"));
152+
if (coordUnits != null && !coordUnits.equals("")) {
153+
joinAggCoord.addAttribute(new Attribute(CF.UNITS, coordUnits));
154+
}
140155
}
141156

142157
if (timeUnitsChange && joinAggCoord != null) {

‎cdm/core/src/main/java/ucar/nc2/ncml/AggregationNew.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
2+
* Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
33
* See LICENSE for license information.
44
*/
55

@@ -134,7 +134,7 @@ protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
134134
private DataType getCoordinateType() {
135135
List<Dataset> nestedDatasets = getDatasets();
136136
DatasetOuterDimension first = (DatasetOuterDimension) nestedDatasets.get(0);
137-
return first.isStringValued ? DataType.STRING : DataType.DOUBLE;
137+
return first.coordDataType;
138138
}
139139

140140
}

0 commit comments

Comments
 (0)