Skip to content

Commit 466ff2d

Browse files
lesserwhirlslesserwhirls
committed
Enable uduints based time coordinates on scan aggregations
Scan-based aggregations with a dateFormatMark produce a String time variable with ISO time string formatted values. This commit enables users to configure the aggregation to produce numeric, udunits based time variables. This improves compatibility with popular clients, such as xarray. Addresses #1327
1 parent 4f9c257 commit 466ff2d

13 files changed

Lines changed: 211 additions & 34 deletions

File tree

‎cdm/core/src/main/java/ucar/nc2/internal/ncml/AggDatasetOuter.java‎

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
1-
/* Copyright Unidata */
1+
/*
2+
* Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
3+
* See LICENSE.txt for license information.
4+
*/
5+
26
package ucar.nc2.internal.ncml;
37

48
import java.io.IOException;
@@ -11,6 +15,7 @@
1115
import javax.annotation.Nullable;
1216
import thredds.inventory.MFile;
1317
import ucar.ma2.Array;
18+
import ucar.ma2.DataType;
1419
import ucar.ma2.InvalidRangeException;
1520
import ucar.ma2.Range;
1621
import ucar.ma2.Section;
@@ -32,7 +37,8 @@ class AggDatasetOuter extends AggDataset {
3237
@Nullable
3338
final String coordValue; // if theres a coordValue on the netcdf element - may be multiple, blank separated
3439
final Date coordValueDate; // if its a date
35-
final boolean isStringValued; // if coordinat is a String
40+
final DataType coordDataType; // coordinate data type
41+
final String coordUdunit; // coordinate udunit string, if numeric
3642

3743
// not final because of deffered read
3844
int ncoord; // number of coordinates in outer dimension
@@ -68,18 +74,18 @@ class AggDatasetOuter extends AggDataset {
6874
}
6975
}
7076

71-
boolean isString = false;
77+
DataType aggCoordDataType = DataType.DOUBLE;
7278
if ((aggregationOuter.type == Type.joinNew) || (aggregationOuter.type == Type.joinExistingOne)
7379
|| (aggregationOuter.type == Type.forecastModelRunCollection)) {
7480
if (coordValueS == null) {
7581
coordValueS = extractCoordNameFromFilename(this.getLocation());
76-
isString = true;
82+
aggCoordDataType = DataType.STRING;
7783
} else {
7884
// we just need to know if its string valued
7985
try {
8086
Double.parseDouble(coordValueS);
8187
} catch (NumberFormatException e) {
82-
isString = true;
88+
aggCoordDataType = DataType.STRING;
8389
}
8490
}
8591
}
@@ -90,8 +96,10 @@ class AggDatasetOuter extends AggDataset {
9096
this.ncoord = stoker.countTokens();
9197
}
9298

93-
this.isStringValued = isString; // LOOK ??
99+
coordDataType = aggCoordDataType;
94100
this.coordValue = coordValueS;
101+
// not dealing with scan
102+
this.coordUdunit = "";
95103
this.coordValueDate = null; // LOOK why isnt this set?
96104
}
97105

@@ -115,13 +123,21 @@ private String extractCoordNameFromFilename(String loc) {
115123
// default is that the coordinates are just the filenames
116124
// this can be overriden by an explicit declaration, which will replace the variable after ther agg is processed in
117125
// NcMLReader
126+
DataType aggCoordDataType = DataType.DOUBLE;
127+
String coordUdunit = "";
118128
if ((aggregationOuter.type == Type.joinNew) || (aggregationOuter.type == Type.joinExistingOne)
119129
|| (aggregationOuter.type == Type.forecastModelRunCollection)) {
120130
coordValueS = extractCoordNameFromFilename(this.getLocation());
121-
this.isStringValued = true;
122-
} else {
123-
this.isStringValued = false; // LOOK ??
131+
if (aggregationOuter.numericTimeSettings != null) {
132+
String[] settings = aggregationOuter.numericTimeSettings.split(" ", 2);
133+
aggCoordDataType = DataType.getType(settings[0]);
134+
coordUdunit = settings[1];
135+
} else {
136+
aggCoordDataType = DataType.STRING;
137+
}
124138
}
139+
this.coordDataType = aggCoordDataType;
140+
this.coordUdunit = coordUdunit;
125141

126142
if (null != aggregationOuter.dateFormatMark) {
127143
String filename = cd.getName(); // LOOK operates on name, not path

‎cdm/core/src/main/java/ucar/nc2/internal/ncml/Aggregation.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
/*
2-
* Copyright (c) 1998-2020 John Caron and University Corporation for Atmospheric Research/Unidata
2+
* Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
3+
* See LICENSE.txt for license information.
34
*/
5+
46
package ucar.nc2.internal.ncml;
57

68
import org.jdom2.Element;
@@ -95,6 +97,7 @@ else if (mode.equalsIgnoreCase("first"))
9597

9698
// experimental
9799
protected String dateFormatMark;
100+
protected String numericTimeSettings;
98101
// protected EnumSet<NetcdfDataset.Enhance> enhance = null; // default no enhancement
99102
protected boolean isDate;
100103
protected DateFormatter dateFormatter = new DateFormatter();
@@ -153,13 +156,17 @@ public void addDataset(AggDataset nested) {
153156
* @param subdirs equals "false" if should not descend into subdirectories
154157
* @param olderThan files must be older than this time (now - lastModified >= olderThan); must be a time unit, may ne
155158
* bull
159+
* @param numericTimeSettings numeric time settings (data type and udunits compatible string,
160+
* e.g. "float seconds since 1985-10-18T12:31:00", may be null)
156161
*/
157162
public void addDatasetScan(Element crawlableDatasetElement, String dirName, String suffix, String regexpPatternString,
158-
String dateFormatMark, Set<NetcdfDataset.Enhance> enhanceMode, String subdirs, String olderThan) {
163+
String dateFormatMark, Set<NetcdfDataset.Enhance> enhanceMode, String subdirs, String olderThan,
164+
String numericTimeSettings) {
159165

160166
datasetManager.addDirectoryScan(dirName, suffix, regexpPatternString, subdirs, olderThan, enhanceMode);
161167

162168
this.dateFormatMark = dateFormatMark;
169+
this.numericTimeSettings = numericTimeSettings;
163170
if (dateFormatMark != null) {
164171
isDate = true;
165172
if (type == Type.joinExisting)

‎cdm/core/src/main/java/ucar/nc2/internal/ncml/AggregationExisting.java‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
2+
* Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
33
* See LICENSE for license information.
44
*/
55

@@ -76,7 +76,13 @@ protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
7676
}
7777

7878
} else {
79-
coordCacheVar = new CoordValueVar(dimName, DataType.STRING, "");
79+
DataType coordDataType = DataType.STRING;
80+
String coordUnits = "";
81+
if (typicalDataset instanceof AggDatasetOuter) {
82+
coordDataType = ((AggDatasetOuter) typicalDataset).coordDataType;
83+
coordUnits = ((AggDatasetOuter) typicalDataset).coordUdunit;
84+
}
85+
coordCacheVar = new CoordValueVar(dimName, coordDataType, coordUnits);
8086
}
8187
if (coordCacheVar != null) {
8288
cacheList.add(coordCacheVar); // coordinate variable is always cached
@@ -134,8 +140,14 @@ protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
134140
if (type == Type.joinExistingOne) {
135141
// replace aggregation coordinate variable
136142
joinAggCoordOpt.ifPresent(joinAgg -> rootGroup.removeVariable(joinAgg.shortName));
143+
DataType coordDataType = DataType.STRING;
144+
String coordUnits = "";
145+
if (typicalDataset instanceof AggDatasetOuter) {
146+
coordDataType = ((AggDatasetOuter) typicalDataset).coordDataType;
147+
coordUnits = ((AggDatasetOuter) typicalDataset).coordUdunit;
148+
}
137149

138-
joinAggCoord = VariableDS.builder().setName(dimName).setDataType(DataType.STRING).setParentGroupBuilder(rootGroup)
150+
joinAggCoord = VariableDS.builder().setName(dimName).setDataType(coordDataType).setParentGroupBuilder(rootGroup)
139151
.setDimensionsByName(dimName);
140152
joinAggCoord.setProxyReader(this);
141153
rootGroup.addVariable(joinAggCoord);
@@ -144,6 +156,9 @@ protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
144156
joinAggCoord.addAttribute(new Attribute(_Coordinate.AxisType, "Time"));
145157
joinAggCoord.addAttribute(new Attribute(CDM.LONG_NAME, "time coordinate"));
146158
joinAggCoord.addAttribute(new Attribute(CF.STANDARD_NAME, "time"));
159+
if (coordUnits != null && !coordUnits.equals("")) {
160+
joinAggCoord.addAttribute(new Attribute(CF.UNITS, coordUnits));
161+
}
147162
}
148163

149164
if (timeUnitsChange && joinAggCoord != null) {

‎cdm/core/src/main/java/ucar/nc2/internal/ncml/AggregationNew.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
2+
* Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
33
* See LICENSE for license information.
44
*/
55

@@ -149,7 +149,7 @@ protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
149149
private DataType getCoordinateType() {
150150
List<AggDataset> nestedDatasets = getDatasets();
151151
AggDatasetOuter first = (AggDatasetOuter) nestedDatasets.get(0);
152-
return first.isStringValued ? DataType.STRING : DataType.DOUBLE;
152+
return first.coordDataType;
153153
}
154154

155155
}

‎cdm/core/src/main/java/ucar/nc2/internal/ncml/NcmlReader.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
/*
2-
* Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
2+
* Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
33
* See LICENSE for license information.
44
*/
5+
56
package ucar.nc2.internal.ncml;
67

78
import java.io.FileNotFoundException;
@@ -1526,6 +1527,7 @@ private Aggregation readAgg(Element aggElem, String ncmlLocation, NetcdfDataset.
15261527
String olderS = scanElem.getAttributeValue("olderThan");
15271528

15281529
String dateFormatMark = scanElem.getAttributeValue("dateFormatMark");
1530+
String numericTimeSettings = scanElem.getAttributeValue("numericTimeSettings");
15291531
Set<NetcdfDataset.Enhance> enhanceMode = NetcdfDataset.parseEnhanceMode(scanElem.getAttributeValue("enhance"));
15301532

15311533
// possible relative location
@@ -1534,7 +1536,7 @@ private Aggregation readAgg(Element aggElem, String ncmlLocation, NetcdfDataset.
15341536
// can embed a full-blown crawlableDatasetImpl element
15351537
Element cdElement = scanElem.getChild("crawlableDatasetImpl", ncNS); // ok if null
15361538
agg.addDatasetScan(cdElement, dirLocation, suffix, regexpPatternString, dateFormatMark, enhanceMode, subdirs,
1537-
olderS);
1539+
olderS, numericTimeSettings);
15381540

15391541
if ((cancelTask != null) && cancelTask.isCancel()) {
15401542
return agg;

‎cdm/core/src/main/java/ucar/nc2/ncml/Aggregation.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
* Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
33
* See LICENSE.txt for license information.
44
*/
5+
56
package ucar.nc2.ncml;
67

78
import org.jdom2.Element;
@@ -153,6 +154,7 @@ else if (mode.equalsIgnoreCase("first"))
153154

154155
// experimental
155156
protected String dateFormatMark;
157+
protected String numericTimeSettings;
156158
// protected EnumSet<NetcdfDataset.Enhance> enhance = null; // default no enhancement
157159
protected boolean isDate;
158160
protected DateFormatter dateFormatter = new DateFormatter();
@@ -199,6 +201,7 @@ public void addDataset(Dataset nested) {
199201
explicitDatasets.add(nested);
200202
}
201203

204+
202205
/**
203206
* Add a dataset scan
204207
*
@@ -214,10 +217,33 @@ public void addDataset(Dataset nested) {
214217
*/
215218
public void addDatasetScan(Element crawlableDatasetElement, String dirName, String suffix, String regexpPatternString,
216219
String dateFormatMark, Set<NetcdfDataset.Enhance> enhanceMode, String subdirs, String olderThan) {
220+
addDatasetScan(crawlableDatasetElement, dirName, suffix, regexpPatternString, dateFormatMark, enhanceMode, subdirs,
221+
olderThan, null);
222+
}
223+
224+
/**
225+
* Add a dataset scan
226+
*
227+
* @param crawlableDatasetElement defines a CrawlableDataset, or null
228+
* @param dirName scan this directory
229+
* @param suffix filter on this suffix (may be null)
230+
* @param regexpPatternString include if full name matches this regular expression (may be null)
231+
* @param dateFormatMark create dates from the filename (may be null)
232+
* @param enhanceMode how should files be enhanced
233+
* @param subdirs equals "false" if should not descend into subdirectories
234+
* @param olderThan files must be older than this time (now - lastModified >= olderThan); must be a time unit, may ne
235+
* bull
236+
* @param numericTimeSettings numeric time settings (data type and udunits compatible string,
237+
* e.g. "float seconds since 1985-10-18T12:31:00", may be null)
238+
*/
239+
public void addDatasetScan(Element crawlableDatasetElement, String dirName, String suffix, String regexpPatternString,
240+
String dateFormatMark, Set<NetcdfDataset.Enhance> enhanceMode, String subdirs, String olderThan,
241+
String numericTimeSettings) {
217242

218243
datasetManager.addDirectoryScan(dirName, suffix, regexpPatternString, subdirs, olderThan, enhanceMode);
219244

220245
this.dateFormatMark = dateFormatMark;
246+
this.numericTimeSettings = numericTimeSettings;
221247
if (dateFormatMark != null) {
222248
isDate = true;
223249
if (type == Type.joinExisting)

‎cdm/core/src/main/java/ucar/nc2/ncml/AggregationExisting.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
2+
* Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
33
* See LICENSE for license information.
44
*/
55

@@ -71,7 +71,13 @@ protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
7171
}
7272

7373
} else {
74-
coordCacheVar = new CoordValueVar(dimName, DataType.STRING, "");
74+
DataType coordDataType = DataType.STRING;
75+
String coordUnits = null;
76+
if (typicalDataset instanceof AggregationOuterDimension.DatasetOuterDimension) {
77+
coordDataType = ((AggregationOuterDimension.DatasetOuterDimension) typicalDataset).coordDataType;
78+
coordUnits = ((AggregationOuterDimension.DatasetOuterDimension) typicalDataset).coordUdunit;
79+
}
80+
coordCacheVar = new CoordValueVar(dimName, coordDataType, coordUnits);
7581
}
7682
if (coordCacheVar != null) {
7783
cacheList.add(coordCacheVar); // coordinate variable is always cached
@@ -128,15 +134,24 @@ protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
128134
// replace aggregation coordinate variable
129135
ncDataset.getRootGroup().removeVariable(joinAggCoord.getShortName());
130136
}
131-
132-
joinAggCoord = new VariableDS(ncDataset, null, null, dimName, DataType.STRING, dimName, null, null);
137+
DataType coordDataType = DataType.STRING;
138+
String coordUnits = null;
139+
if (typicalDataset instanceof AggregationOuterDimension.DatasetOuterDimension) {
140+
coordDataType = ((AggregationOuterDimension.DatasetOuterDimension) typicalDataset).coordDataType;
141+
coordUnits = ((AggregationOuterDimension.DatasetOuterDimension) typicalDataset).coordUdunit;
142+
}
143+
coordCacheVar = new CoordValueVar(dimName, coordDataType, coordUnits);
144+
joinAggCoord = new VariableDS(ncDataset, null, null, dimName, coordDataType, dimName, coordUnits, null);
133145
joinAggCoord.setProxyReader(this);
134146
ncDataset.getRootGroup().addVariable(joinAggCoord);
135147
aggVars.add(joinAggCoord);
136148

137149
joinAggCoord.addAttribute(new ucar.nc2.Attribute(_Coordinate.AxisType, "Time"));
138150
joinAggCoord.addAttribute(new Attribute(CDM.LONG_NAME, "time coordinate"));
139151
joinAggCoord.addAttribute(new ucar.nc2.Attribute(CF.STANDARD_NAME, "time"));
152+
if (coordUnits != null && !coordUnits.equals("")) {
153+
joinAggCoord.addAttribute(new Attribute(CF.UNITS, coordUnits));
154+
}
140155
}
141156

142157
if (timeUnitsChange && joinAggCoord != null) {

‎cdm/core/src/main/java/ucar/nc2/ncml/AggregationNew.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
2+
* Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
33
* See LICENSE for license information.
44
*/
55

@@ -134,7 +134,7 @@ protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
134134
private DataType getCoordinateType() {
135135
List<Dataset> nestedDatasets = getDatasets();
136136
DatasetOuterDimension first = (DatasetOuterDimension) nestedDatasets.get(0);
137-
return first.isStringValued ? DataType.STRING : DataType.DOUBLE;
137+
return first.coordDataType;
138138
}
139139

140140
}

0 commit comments

Comments
 (0)