Merge pull request #772 from lesserwhirls/fmrcCache

[5.x]: Enable the use of a persisted FMRC cache

Author: haileyajohnson

cdm/core/src/main/java/thredds/inventory/MControllerProvider.java

@@ -22,7 +22,7 @@ default boolean canScan(String location) {
   /**
    * Creates an instance of
    * 
-   * @return An {@link MController} that scans locations to filter and provide a set of {@linik MFile}s defining to
+   * @return An {@link MController} that scans locations to filter and provide a set of {@link MFile}s defining to
    *         be used to define a collection.
    */
   MController create();

cdm/core/src/main/java/ucar/nc2/ft/fmrc/GridDatasetInv.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2021 John Caron and University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
@@ -20,6 +20,7 @@
 import ucar.nc2.dt.GridCoordSystem;
 import ucar.nc2.dt.grid.GridDataset;
 import ucar.nc2.NetcdfFile;
+import ucar.nc2.internal.dataset.ft.fmrc.InventoryCacheProvider;
 import ucar.nc2.ncml.NcMLReader;
 import ucar.nc2.dataset.CoordinateAxis1D;
 import ucar.nc2.time.CalendarDate;
@@ -53,8 +54,7 @@ public class GridDatasetInv {
   private static final int REQ_VERSION = 2; // minimum required version, else regenerate XML
   private static final int CURR_VERSION = 2; // current version
 
-  // Cache the GridDatasetInv directly, not persisted to disk.
-  // TODO: Add persistence if thats shown to be needed.
+  // Cache the GridDatasetInv directly in memory. Persist to disk if InventoryCacheProvider if found.
   private static Cache<String, GridDatasetInv> cache = CacheBuilder.newBuilder().maximumSize(100).build();
 
   public static GridDatasetInv open(MCollection cm, MFile mfile, Element ncml) throws IOException {
@@ -69,6 +69,18 @@ private static class GenerateInv implements Callable<GridDatasetInv> {
     private final MCollection cm;
     private final MFile mfile;
     private final Element ncml;
+    private static final InventoryCacheProvider persistedCache;
+
+    // Look for InventoryCacheProvider, which implements a persistent cache of GridDatasetInv.
+    // Persistence is needed for the TDS
+    static {
+      InventoryCacheProvider icp = null;
+      for (InventoryCacheProvider provider : ServiceLoader.load(InventoryCacheProvider.class)) {
+        // first one wins
+        icp = provider;
+      }
+      persistedCache = icp;
+    }
 
     GenerateInv(MCollection cm, MFile mfile, Element ncml) {
       this.cm = cm;
@@ -79,23 +91,33 @@ private static class GenerateInv implements Callable<GridDatasetInv> {
     @Override
     public GridDatasetInv call() throws Exception {
       GridDataset gds = null;
-      try {
-        if (ncml == null) {
-          gds = GridDataset.open(mfile.getPath());
-
-        } else {
-          NetcdfFile nc = NetcdfDataset.acquireFile(DatasetUrl.create(null, mfile.getPath()), null);
-          NetcdfDataset ncd = NcMLReader.mergeNcML(nc, ncml); // create new dataset
-          ncd.enhance(); // now that the ncml is added, enhance "in place", ie modify the NetcdfDataset
-          gds = new GridDataset(ncd);
-        }
+      GridDatasetInv inv = null;
+      if (persistedCache != null) {
+        inv = persistedCache.get(mfile);
+      }
 
-        return new GridDatasetInv(gds, cm.extractDate(mfile));
-      } finally {
-        if (gds != null) {
-          gds.close();
+      if (inv == null) {
+        try {
+          if (ncml == null) {
+            gds = GridDataset.open(mfile.getPath());
+          } else {
+            NetcdfFile nc = NetcdfDataset.acquireFile(DatasetUrl.create(null, mfile.getPath()), null);
+            NetcdfDataset ncd = NcMLReader.mergeNcML(nc, ncml); // create new dataset
+            ncd.enhance(); // now that the ncml is added, enhance "in place", ie modify the NetcdfDataset
+            gds = new GridDataset(ncd);
+          }
+          inv = new GridDatasetInv(gds, cm.extractDate(mfile));
+          if (persistedCache != null && inv != null) {
+            // add inventory to persisted cache
+            persistedCache.put(mfile, inv);
+          }
+        } finally {
+          if (gds != null) {
+            gds.close();
+          }
         }
       }
+      return inv;
     }
   }
 
@@ -171,6 +193,15 @@ public String toString() {
     return location;
   }
 
+  /**
+   * Check if the GribDatasetInv xml format can be fully understood by this code.
+   *
+   * @return true if version can be fully understood, otherwise false.
+   */
+  public boolean isXmlVersionCompatible() {
+    return this.version >= REQ_VERSION;
+  }
+
   public String getLocation() {
     return location;
   }
@@ -457,7 +488,7 @@ Document writeDocument(Date lastModified) {
    * @return ForecastModelRun
    * @throws IOException on io error
    */
-  private static GridDatasetInv readXML(byte[] xmlString) throws IOException {
+  public static GridDatasetInv readXML(byte[] xmlString) throws IOException {
     InputStream is = new BufferedInputStream(new ByteArrayInputStream(xmlString));
     org.jdom2.Document doc;
     try {

cdm/core/src/main/java/ucar/nc2/internal/dataset/ft/fmrc/InventoryCacheProvider.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2021 University Corporation for Atmospheric Research/Unidata
+ * See LICENSE for license information.
+ */
+
+package ucar.nc2.internal.dataset.ft.fmrc;
+
+import thredds.inventory.MFile;
+import ucar.nc2.ft.fmrc.GridDatasetInv;
+
+import javax.annotation.Nullable;
+import java.io.*;
+
+/**
+ * Service Provider Interface for providing a persisted cache for {@link GridDatasetInv}.
+ *
+ * For use by the THREDDS Data Server and not intended to be used publicly.
+ *
+ */
+public interface InventoryCacheProvider {
+
+  /**
+   * Get the grid inventory associated with an MFile from the cache.
+   *
+   * @param mfile the mfile containing gridded data
+   * @return grid inventory of the of mfile, null if not found
+   * @throws IOException
+   */
+  @Nullable
+  public GridDatasetInv get(MFile mfile) throws IOException;
+
+  /**
+   * Add the grid inventory associated with an MFile to the cache.
+   *
+   * @param mfile the mfile containing gridded data
+   * @param inventory the grid inventory of the of mfile
+   * @throws IOException
+   */
+  public void put(MFile mfile, GridDatasetInv inventory) throws IOException;
+
+}

cdm/core/src/main/java/ucar/unidata/io/RandomAccessFile.java

@@ -1456,7 +1456,7 @@ public final String readLine() throws IOException {
    * The charset parameter is an extension not implemented in java.io.RandomAccessFile.
    *
    * @param charset - character encoding to use
-   * @return
+   * @return the next line of text
    * @throws IOException
    */
   public String readLine(Charset charset) throws IOException {