Skip to content

Commit 698ef5f

Browse files
DennisHeimbignerDennisHeimbigner
committed
Merge branch 'master' into tdsdap4fixes1.dmh
2 parents 4b285d4 + 3cf8964 commit 698ef5f

File tree

8 files changed

+325
-53
lines changed

8 files changed

+325
-53
lines changed

‎tds/src/integrationTests/java/thredds/server/services/TestEnhancements.java

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package thredds.server.services;
2+
3+
import org.junit.Ignore;
4+
import org.junit.Test;
5+
import thredds.test.util.TestOnLocalServer;
6+
import thredds.util.ContentType;
7+
import ucar.ma2.Array;
8+
import ucar.ma2.MAMath;
9+
import ucar.nc2.NetcdfFile;
10+
import ucar.nc2.NetcdfFiles;
11+
import ucar.nc2.Variable;
12+
13+
import javax.servlet.http.HttpServletResponse;
14+
15+
import java.io.IOException;
16+
17+
import static com.google.common.truth.Truth.assertThat;
18+
19+
public class TestEnhancements {
20+
21+
final static String ENHANCED_FILE = "localContent/testOffset.nc";
22+
final static String NCML_ENHANCED_FILE = "testOffsetWithNcml.nc";
23+
final static String ENHANCED_VAR_NAME = "variableWithOffset";
24+
final static String NOT_ENHANCED_VAR_NAME = "variableWithoutOffset";
25+
26+
@Test
27+
public void testNCSSWithEnhancements() throws IOException {
28+
// scale-offset set as variable attribute
29+
final String endpoint = TestOnLocalServer
30+
.withHttpPath("/ncss/grid/" + ENHANCED_FILE + "?var=" + ENHANCED_VAR_NAME + "&var=" + NOT_ENHANCED_VAR_NAME);
31+
final byte[] content = TestOnLocalServer.getContent(endpoint, HttpServletResponse.SC_OK, ContentType.netcdf);
32+
33+
try (NetcdfFile netcdfFile = NetcdfFiles.openInMemory("test_data.nc", content)) {
34+
final Variable enhancedVar = netcdfFile.findVariable(ENHANCED_VAR_NAME);
35+
final Variable orgVar = netcdfFile.findVariable(NOT_ENHANCED_VAR_NAME);
36+
assertThat(enhancedVar != null).isTrue();
37+
assertThat(orgVar != null).isTrue();
38+
assertThat(enhancedVar.findAttribute("add_offset")).isNull();
39+
final Array values1 = enhancedVar.read();
40+
final Array values2 = orgVar.read();
41+
MAMath.nearlyEquals(values1, values2);
42+
}
43+
}
44+
45+
@Test
46+
public void testNCSSWithEnhancementsNcML() throws IOException {
47+
// scale-offset set as variable attribute
48+
final String endpoint = TestOnLocalServer.withHttpPath(
49+
"/ncss/grid/" + NCML_ENHANCED_FILE + "?var=" + ENHANCED_VAR_NAME + "&var=" + NOT_ENHANCED_VAR_NAME);
50+
final byte[] content = TestOnLocalServer.getContent(endpoint, HttpServletResponse.SC_OK, ContentType.netcdf);
51+
52+
try (NetcdfFile netcdfFile = NetcdfFiles.openInMemory("test_data.nc", content)) {
53+
final Variable enhancedVar = netcdfFile.findVariable(ENHANCED_VAR_NAME);
54+
final Variable orgVar = netcdfFile.findVariable(NOT_ENHANCED_VAR_NAME);
55+
assertThat(enhancedVar != null).isTrue();
56+
assertThat(orgVar != null).isTrue();
57+
assertThat(orgVar.findAttribute("add_offset")).isNull();
58+
final Array values1 = enhancedVar.read();
59+
final Array values2 = orgVar.read();
60+
for (int i = 0; i < values1.getSize(); i++) {
61+
assertThat(values1.getDouble(i)).isEqualTo(values2.getDouble(i) + 100);
62+
}
63+
}
64+
}
65+
}

‎tds/src/integrationTests/java/thredds/tds/TestS3Ncml.java

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package thredds.tds;
2+
3+
import static com.google.common.truth.Truth.assertThat;
4+
5+
import java.nio.charset.StandardCharsets;
6+
import javax.servlet.http.HttpServletResponse;
7+
import org.junit.Test;
8+
import thredds.test.util.TestOnLocalServer;
9+
10+
public class TestS3Ncml {
11+
final private static String S3_NCML_PATH = "dodsC/s3-dataset-scan/ncml/";
12+
13+
@Test
14+
public void shouldOpenNcmlOnS3() {
15+
final String endpoint = TestOnLocalServer.withHttpPath(S3_NCML_PATH + "testStandalone.ncml.ascii?time");
16+
final byte[] content = TestOnLocalServer.getContent(endpoint, HttpServletResponse.SC_OK);
17+
final String stringContent = new String(content, StandardCharsets.UTF_8);
18+
19+
assertThat(stringContent).contains("time[2]");
20+
assertThat(stringContent).contains("6, 18");
21+
}
22+
23+
@Test
24+
public void shouldOpenNcmlWithOtherExtensionOnS3() {
25+
final String endpoint = TestOnLocalServer.withHttpPath(S3_NCML_PATH + "testStandaloneNcml.otherExt.ascii?time");
26+
final byte[] content = TestOnLocalServer.getContent(endpoint, HttpServletResponse.SC_OK);
27+
final String stringContent = new String(content, StandardCharsets.UTF_8);
28+
29+
assertThat(stringContent).contains("time[2]");
30+
assertThat(stringContent).contains("6, 18");
31+
}
32+
33+
@Test
34+
public void shouldOpenAggregationWithRelativePathsOnS3() {
35+
final String endpoint = TestOnLocalServer.withHttpPath(S3_NCML_PATH + "nc/namExtract/test_agg.ncml.ascii?time");
36+
final byte[] content = TestOnLocalServer.getContent(endpoint, HttpServletResponse.SC_OK);
37+
final String stringContent = new String(content, StandardCharsets.UTF_8);
38+
39+
assertThat(stringContent).contains("time[8]");
40+
assertThat(stringContent).contains("3.0, 6.0, 9.0, 12.0, 15.0, 18.0, 21.0, 24.0");
41+
}
42+
43+
@Test
44+
public void shouldOpenAggregationWithAbsolutePathsOnS3() {
45+
final String endpoint =
46+
TestOnLocalServer.withHttpPath(S3_NCML_PATH + "nc/namExtract/test_agg_absolute_paths.ncml.ascii?time");
47+
final byte[] content = TestOnLocalServer.getContent(endpoint, HttpServletResponse.SC_OK);
48+
final String stringContent = new String(content, StandardCharsets.UTF_8);
49+
50+
assertThat(stringContent).contains("time[8]");
51+
assertThat(stringContent).contains("3.0, 6.0, 9.0, 12.0, 15.0, 18.0, 21.0, 24.0");
52+
}
53+
}

‎tds/src/main/java/thredds/core/DatasetManager.java

+67-48
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.springframework.beans.factory.annotation.Autowired;
1313
import org.springframework.beans.factory.annotation.Qualifier;
1414
import org.springframework.stereotype.Component;
15+
import thredds.core.DataRootManager.DataRootMatch;
1516
import thredds.featurecollection.FeatureCollectionCache;
1617
import thredds.featurecollection.InvDatasetFeatureCollection;
1718
import thredds.server.admin.DebugCommands;
@@ -174,7 +175,7 @@ public NetcdfFile openNetcdfFile(HttpServletRequest req, HttpServletResponse res
174175
}
175176

176177
// look for a match
177-
DataRootManager.DataRootMatch match = dataRootManager.findDataRootMatch(reqPath);
178+
DataRootMatch match = dataRootManager.findDataRootMatch(reqPath);
178179

179180
// look for a feature collection dataset
180181
if ((match != null) && (match.dataRoot.getFeatureCollection() != null)) {
@@ -200,49 +201,12 @@ public NetcdfFile openNetcdfFile(HttpServletRequest req, HttpServletResponse res
200201

201202
// common case - its a file
202203
if (match != null) {
203-
org.jdom2.Element netcdfElem = null; // find ncml if it exists
204-
if (match.dataRoot != null) {
205-
DatasetScan dscan = match.dataRoot.getDatasetScan();
206-
// if (dscan == null) dscan = match.dataRoot.getDatasetRootProxy(); // no ncml possible in getDatasetRootProxy
207-
if (dscan != null)
208-
netcdfElem = dscan.getNcmlElement();
209-
}
210-
211204
String location = dataRootManager.getLocationFromRequestPath(reqPath);
212205
if (location == null)
213206
throw new FileNotFoundException(reqPath);
214207

215-
// if there's an ncml element, open it through NcMLReader, supplying the underlying file
216-
// from NetcdfFiles.open(), therefore not being cached.
217-
// This is safer given all the trouble we have with ncml and caching.
218-
if (netcdfElem != null) {
219-
String ncmlLocation = "DatasetScan#" + location; // LOOK some descriptive name
220-
// open with openFile(), not acquireFile, so we skip the caches
221-
NetcdfDataset ncd = null;
222-
223-
// look for addRecords attribute on the netcdf element. The new API in netCDF-Java does not handle this,
224-
// so we will handle it special here.
225-
Attribute addRecordsAttr = netcdfElem.getAttribute("addRecords");
226-
boolean addRecords = false;
227-
if (addRecordsAttr != null) {
228-
addRecords = Boolean.valueOf(addRecordsAttr.getValue());
229-
}
230-
231-
NetcdfFile ncf;
232-
if (addRecords) {
233-
DatasetUrl datasetUrl = DatasetUrl.findDatasetUrl(location);
234-
// work around for presence of addRecords="true" on a netcdf element
235-
ncf = NetcdfDatasets.openFile(datasetUrl, -1, null, NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
236-
} else {
237-
ncf = NetcdfDatasets.openFile(location, null);
238-
}
239-
240-
NetcdfDataset.Builder modifiedDsBuilder = NcmlReader.mergeNcml(ncf, netcdfElem);
241-
242-
// set new location to indicate this is a dataset from a dataset scan wrapped with NcML.
243-
modifiedDsBuilder.setLocation(ncmlLocation);
244-
ncd = modifiedDsBuilder.build();
245-
return ncd;
208+
if (hasDatasetScanNcml(match)) {
209+
return openNcmlDatasetScan(location, match);
246210
}
247211

248212
DatasetUrl durl = DatasetUrl.findDatasetUrl(location);
@@ -258,6 +222,44 @@ public NetcdfFile openNetcdfFile(HttpServletRequest req, HttpServletResponse res
258222
return ncfile;
259223
}
260224

225+
private boolean hasDatasetScanNcml(DataRootMatch match) {
226+
return match != null && match.dataRoot != null && match.dataRoot.getDatasetScan() != null
227+
&& match.dataRoot.getDatasetScan().getNcmlElement() != null;
228+
}
229+
230+
private NetcdfFile openNcmlDatasetScan(String location, DataRootMatch match) throws IOException {
231+
org.jdom2.Element netcdfElem = match.dataRoot.getDatasetScan().getNcmlElement();
232+
// if there's an ncml element, open it through NcMLReader, supplying the underlying file
233+
// from NetcdfFiles.open(), therefore not being cached.
234+
// This is safer given all the trouble we have with ncml and caching.
235+
236+
String ncmlLocation = "DatasetScan#" + location; // LOOK some descriptive name
237+
// open with openFile(), not acquireFile, so we skip the caches
238+
239+
// look for addRecords attribute on the netcdf element. The new API in netCDF-Java does not handle this,
240+
// so we will handle it special here.
241+
Attribute addRecordsAttr = netcdfElem.getAttribute("addRecords");
242+
boolean addRecords = false;
243+
if (addRecordsAttr != null) {
244+
addRecords = Boolean.valueOf(addRecordsAttr.getValue());
245+
}
246+
247+
NetcdfFile ncf;
248+
if (addRecords) {
249+
DatasetUrl datasetUrl = DatasetUrl.findDatasetUrl(location);
250+
// work around for presence of addRecords="true" on a netcdf element
251+
ncf = NetcdfDatasets.openFile(datasetUrl, -1, null, NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
252+
} else {
253+
ncf = NetcdfDatasets.openFile(location, null);
254+
}
255+
256+
NetcdfDataset.Builder<?> modifiedDsBuilder = NcmlReader.mergeNcml(ncf, netcdfElem);
257+
258+
// set new location to indicate this is a dataset from a dataset scan wrapped with NcML.
259+
modifiedDsBuilder.setLocation(ncmlLocation);
260+
return modifiedDsBuilder.build();
261+
}
262+
261263
/**
262264
* Open a file as a GridDataset, using getNetcdfFile(), so that it gets wrapped in NcML if needed.
263265
*/
@@ -355,7 +357,8 @@ public FeatureDatasetPoint openPointDataset(HttpServletRequest req, HttpServletR
355357
if (featureDatasetPoint != null) {
356358
return featureDatasetPoint;
357359
} else {
358-
throw new UnsupportedOperationException("Could not open as a PointDataset: " + errlog);
360+
log.error("Could not open as a PointDataset: " + errlog);
361+
throw new UnsupportedOperationException("Could not open as a point dataset");
359362
}
360363
} catch (Throwable t) {
361364
if (ncd == null)
@@ -420,8 +423,14 @@ public CoverageCollection openCoverageDataset(HttpServletRequest req, HttpServle
420423
}
421424

422425
// otherwise, assume it's a local file with a datasetRoot in the urlPath.
423-
// try to open as a FeatureDatasetCoverage. This allows GRIB to be handled specially
424426
String location = getLocationFromRequestPath(reqPath);
427+
428+
// Ncml in datasetScan
429+
if (location != null && hasDatasetScanNcml(match)) {
430+
return openCoverageFromDatasetScanNcml(location, match, reqPath);
431+
}
432+
433+
// try to open as a FeatureDatasetCoverage. This allows GRIB to be handled specially
425434
if (location != null) {
426435
Optional<FeatureDatasetCoverage> opt = CoverageDatasetFactory.openCoverageDataset(location);
427436
// hack - CoverageDatasetFactory bombs out on an object store location string during the grib check,
@@ -431,12 +440,8 @@ public CoverageCollection openCoverageDataset(HttpServletRequest req, HttpServle
431440
// and pass that to CoverageDataset
432441
DtCoverageDataset gds = new DtCoverageDataset(NetcdfDatasets.openDataset(location));
433442
if (!gds.getGrids().isEmpty()) {
434-
Formatter errlog = new Formatter();
435-
FeatureDatasetCoverage result = DtCoverageAdapter.factory(gds, errlog);
436-
if (result != null)
437-
opt = Optional.of(result);
438-
else
439-
opt = Optional.empty(errlog.toString());
443+
FeatureDatasetCoverage result = DtCoverageAdapter.factory(gds, new Formatter());
444+
opt = Optional.of(result);
440445
}
441446
}
442447

@@ -451,6 +456,20 @@ public CoverageCollection openCoverageDataset(HttpServletRequest req, HttpServle
451456
return null;
452457
}
453458

459+
private CoverageCollection openCoverageFromDatasetScanNcml(String location, DataRootMatch match, String reqPath)
460+
throws IOException {
461+
final NetcdfFile ncf = openNcmlDatasetScan(location, match);
462+
final NetcdfDataset ncd = NetcdfDatasets.enhance(ncf, NetcdfDataset.getDefaultEnhanceMode(), null);
463+
final DtCoverageDataset gds = new DtCoverageDataset(ncd);
464+
465+
if (gds.getGrids().isEmpty()) {
466+
throw new FileNotFoundException("Error opening grid dataset " + reqPath + ". err= no grids found.");
467+
}
468+
469+
final FeatureDatasetCoverage coverage = DtCoverageAdapter.factory(gds, new Formatter());
470+
return coverage.getSingleCoverageCollection();
471+
}
472+
454473
public SimpleGeometryFeatureDataset openSimpleGeometryDataset(HttpServletRequest req, HttpServletResponse res,
455474
String reqPath) throws IOException {
456475
// first look for a feature collection

‎tds/src/main/java/thredds/servlet/ServletUtil.java

+6
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,12 @@ public static void writeMFileToResponse(HttpServletRequest request, HttpServletR
282282
final String ncmlLocation = TdsRequestedDataset.getLocationFromNcml(requestPath);
283283
final String location =
284284
ncmlLocation != null ? ncmlLocation : TdsRequestedDataset.getLocationFromRequestPath(requestPath);
285+
286+
if (location == null) {
287+
response.sendError(HttpServletResponse.SC_NOT_FOUND, "Could not find file with URL path: " + requestPath);
288+
return;
289+
}
290+
285291
final MFile file = MFiles.create(location);
286292

287293
if (file == null) {

‎tds/src/test/content/thredds/catalog.xml

+20-3
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,18 @@
4949

5050
<dataset name="Test Single Dataset" ID="testDataset" serviceName="odap" urlPath="localContent/testData.nc" dataType="Grid"/>
5151
<dataset name="Test Grid as Point" ID="testGAP" serviceName="all" urlPath="localContent/testGridAsPoint.nc" dataType="Grid"/>
52+
<dataset name="Test Enhancements">
53+
<dataset name="With Attribute" ID="withAttribute" serviceName="all" urlPath="localContent/testOffset.nc" dataType="Grid"/>
54+
<dataset name="With NcML" ID="withNcMl" serviceName="all" urlPath="testOffsetWithNcml.nc">
55+
<serviceName>all</serviceName>
56+
<ncml:netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2" enhance="all"
57+
location="content/testdata/testOffset.nc">
58+
<variable name="variableWithoutOffset">
59+
<attribute name="add_offset" type="float" value="-100"/>
60+
</variable>
61+
</ncml:netcdf>
62+
</dataset>
63+
</dataset>
5264

5365
<dataset name="Test Dap4 Dataset" ID="testDap4Dataset" serviceName="dap4"
5466
urlPath="cdmUnitTest/ft/coverage/03061219_ruc.nc" dataType="Grid"/>
@@ -161,9 +173,8 @@
161173

162174
<dataset name="Ncml Tests">
163175

164-
<!-- UNTESTED test NcML in dataset LOOK doesnt work -->
165-
<dataset name="Example NcML Modified - DODS" ID="ExampleNcMLModified" urlPath="ExampleNcML/Modified.nc">
166-
<serviceName>odap</serviceName>
176+
<dataset name="Example NcML Modified" ID="ExampleNcMLModified" urlPath="ExampleNcML/Modified.nc">
177+
<serviceName>all</serviceName>
167178
<dataType>Grid</dataType>
168179
<ncml:netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2" location="${cdmUnitTest}/tds/example1.nc" addRecords="true">
169180
<attribute name="name" value="value"/>
@@ -173,6 +184,9 @@
173184
<attribute name="units" value="percent (%)"/>
174185
<remove type="attribute" name="description"/>
175186
</variable>
187+
<variable name="time">
188+
<attribute name="units" value="hours since 2000-01-01 00:00:00"/>
189+
</variable>
176190
</ncml:netcdf>
177191
</dataset>
178192

@@ -234,6 +248,9 @@
234248
<attribute name="units" value="percent (%)"/>
235249
<remove type="attribute" name="description"/>
236250
</variable>
251+
<variable name="time">
252+
<attribute name="units" value="hours since 2000-01-01 00:00:00"/>
253+
</variable>
237254
</ncml:netcdf>
238255

239256
<filter>

‎tds/src/test/java/thredds/server/config/TdsContextTest.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,11 @@
1414
import ucar.unidata.util.test.category.NeedsContentRoot;
1515
import java.lang.invoke.MethodHandles;
1616
import java.util.Map;
17+
import ucar.unidata.util.test.category.NeedsExternalResource;
1718

1819
@RunWith(SpringJUnit4ClassRunner.class)
1920
@ContextConfiguration(locations = {"/WEB-INF/applicationContext.xml"}, loader = MockTdsContextLoader.class)
20-
@Category(NeedsContentRoot.class)
21+
@Category({NeedsContentRoot.class, NeedsExternalResource.class})
2122
public class TdsContextTest {
2223
private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
2324

0 commit comments

Comments
 (0)