Merge pull request #1394 from dopplershift/fix-nids-deflate

BUG: Fix decompressing all zlib data for NIDS

Author: lesserwhirls

cdm/radial/src/main/java/ucar/nc2/iosp/nids/Nidsheader.java

@@ -255,7 +255,6 @@ void read(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile) thro
     int rc; /* function return status */
     int hoff = 0;
     int type;
-    int zlibed;
     boolean isZ = false;
     int encrypt;
     long actualSize;
@@ -295,8 +294,8 @@ void read(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile) thro
       // byte[] b4 = new byte[4];
       System.arraycopy(b, hoff, b2, 0, 2);
 
-      zlibed = isZlibHed(b2);
-      if (zlibed == 0) {
+      boolean zlibed = isZlibHed(b2);
+      if (!zlibed) {
         encrypt = IsEncrypt(b2);
         if (encrypt == 1) {
           log.error("error reading encryted product " + raf.getLocation());
@@ -331,7 +330,7 @@ void read(ucar.unidata.io.RandomAccessFile raf, ucar.nc2.NetcdfFile ncfile) thro
           break;
       }
 
-      if (zlibed == 1) {
+      if (zlibed) {
         isZ = true;
         uncompdata = GetZlibedNexr(b, readLen, hoff);
         // uncompdata = Nidsiosp.readCompData(hoff, 160) ;
@@ -3282,23 +3281,23 @@ public void close() throws IOException {
 
   /**
    * Name: IsZlibed
-   *
+   * <p>
    * Purpose: Check a two-byte sequence to see if it indicates the start of a zlib-compressed
    * buffer
    */
-  int isZlibHed(byte[] buf) {
+  boolean isZlibHed(byte[] buf) {
     short b0 = convertunsignedByte2Short(buf[0]);
     short b1 = convertunsignedByte2Short(buf[1]);
 
     if ((b0 & 0xf) == Z_DEFLATED) {
       if ((b0 >> 4) + 8 <= DEF_WBITS) {
         if ((((b0 << 8) + b1) % 31) == 0) {
-          return 1;
+          return true;
         }
       }
     }
 
-    return 0;
+    return false;
 
   }
 
@@ -3363,56 +3362,52 @@ byte[] GetZlibedNexr(byte[] buf, int buflen, int hoff) throws IOException {
     // decompress the bytes
     int resultLength;
     int result = 0;
-    // byte[] inflateData = null;
-    byte[] tmp;
-    int uncompLen = 24500; /* length of decompress space */
-    byte[] uncomp = new byte[uncompLen];
-    Inflater inflater = new Inflater(false);
+    byte[] chunk;
+    List<byte[]> chunks = new ArrayList<>(1);
+    List<Integer> sizes = new ArrayList<>(1);
 
-    inflater.setInput(buf, hoff, numin - 4);
-    int offset = 0;
-    int limit = 20000;
+    Inflater inf = new Inflater(false);
+    inf.setInput(buf, hoff, numin - 4);
 
-    while (inflater.getRemaining() > 0) {
+    while (inf.getRemaining() > 0) {
       try {
-        resultLength = inflater.inflate(uncomp, offset, 4000);
+        chunk = new byte[4000];
+        resultLength = inf.inflate(chunk, 0, 4000);
+        chunks.add(chunk);
+        sizes.add(resultLength);
       } catch (DataFormatException ex) {
-        log.error("nids Inflater", ex);
-        throw new IOException(ex.getMessage(), ex);
-      }
-      offset = offset + resultLength;
-      result = result + resultLength;
-      if (result > limit) {
-        // when uncomp data larger then limit, the uncomp need to increase size
-        tmp = new byte[result];
-        System.arraycopy(uncomp, 0, tmp, 0, result);
-        uncompLen = uncompLen + 10000;
-        uncomp = new byte[uncompLen];
-        System.arraycopy(tmp, 0, uncomp, 0, result);
+        log.error("ERROR on inflation ", ex);
+        throw new IOException(ex.getMessage());
       }
+
+      result += resultLength;
       if (resultLength == 0) {
-        int tt = inflater.getRemaining();
+        int tt = inf.getRemaining();
         byte[] b2 = new byte[2];
         System.arraycopy(buf, hoff + numin - 4 - tt, b2, 0, 2);
-        if (result + tt > uncompLen) {
-          tmp = new byte[result];
-          System.arraycopy(uncomp, 0, tmp, 0, result);
-          uncompLen = uncompLen + 10000;
-          uncomp = new byte[uncompLen];
-          System.arraycopy(tmp, 0, uncomp, 0, result);
-        }
-        if (isZlibHed(b2) == 0) {
-          System.arraycopy(buf, hoff + numin - 4 - tt, uncomp, result, tt);
-          result = result + tt;
+        if (!isZlibHed(b2)) {
+          chunk = new byte[tt];
+          System.arraycopy(buf, hoff + numin - 4 - tt, chunk, 0, tt);
+          chunks.add(chunk);
+          sizes.add(tt);
+          result += tt;
           break;
+        } else {
+          inf.reset();
+          inf.setInput(buf, hoff + numin - 4 - tt, tt);
         }
-        inflater.reset();
-        inflater.setInput(buf, hoff + numin - 4 - tt, tt);
       }
+    }
+    inf.end();
 
+    // Combine the chunks into one buffer
+    byte[] uncomp = new byte[result];
+    int pos = 0;
+    for (int i = 0; i < chunks.size(); ++i) {
+      System.arraycopy(chunks.get(i), 0, uncomp, pos, sizes.get(i));
+      pos += sizes.get(i);
     }
 
-    inflater.end();
     /*
      ** Find out how long CCB is. This is done by using the lower order
      ** 6 bits from the first uncompressed byte and all 8 bits of the

cdm/radial/src/main/java/ucar/nc2/iosp/nids/Nidsiosp.java

@@ -1490,9 +1490,8 @@ public byte[] readCompData1(byte[] uncomp, long hoff, long doff) {
    */
   public byte[] readCompData(long hoff, long doff) throws IOException {
     int numin; /* # input bytes processed */
-    long pos = 0;
     long len = raf.length();
-    raf.seek(pos);
+    raf.seek(0);
     numin = (int) (len - hoff);
     // Read in the contents of the NEXRAD Level III product header
 
@@ -1506,53 +1505,55 @@ public byte[] readCompData(long hoff, long doff) throws IOException {
     // System.arraycopy( b, (int)hoff, comp, 0, numin -4 );
 
     // decompress the bytes
-    Inflater inf = new Inflater(false);
-
     int resultLength;
     int result = 0;
-    // byte[] inflateData = null;
-    byte[] tmp;
-    int uncompLen = 24500; /* length of decompress space */
-    byte[] uncomp = new byte[uncompLen];
+    byte[] chunk;
+    List<byte[]> chunks = new ArrayList<>(1);
+    List<Integer> sizes = new ArrayList<>(1);
 
+    Inflater inf = new Inflater(false);
     inf.setInput(b, (int) hoff, numin - 4);
-    int limit = 20000;
 
     while (inf.getRemaining() > 0) {
       try {
-        resultLength = inf.inflate(uncomp, result, 4000);
+        chunk = new byte[4000];
+        resultLength = inf.inflate(chunk, 0, 4000);
+        chunks.add(chunk);
+        sizes.add(resultLength);
       } catch (DataFormatException ex) {
         logger.error("ERROR on inflation ", ex);
         throw new IOException(ex.getMessage());
       }
 
-      result = result + resultLength;
-      if (result > limit) {
-        // when uncomp data larger then limit, the uncomp need to increase size
-        tmp = new byte[result];
-        System.arraycopy(uncomp, 0, tmp, 0, result);
-        uncompLen = uncompLen + 10000;
-        uncomp = new byte[uncompLen];
-        System.arraycopy(tmp, 0, uncomp, 0, result);
-      }
+      result += resultLength;
       if (resultLength == 0) {
         int tt = inf.getRemaining();
         byte[] b2 = new byte[2];
         System.arraycopy(b, (int) hoff + numin - 4 - tt, b2, 0, 2);
-        if (headerParser.isZlibHed(b2) == 0) {
-          System.arraycopy(b, (int) hoff + numin - 4 - tt, uncomp, result, tt);
-          result = result + tt;
+        if (!headerParser.isZlibHed(b2)) {
+          chunk = new byte[tt];
+          System.arraycopy(b, (int) hoff + numin - 4 - tt, chunk, 0, tt);
+          chunks.add(chunk);
+          sizes.add(tt);
+          result += tt;
           break;
+        } else {
+          inf.reset();
+          inf.setInput(b, (int) hoff + numin - 4 - tt, tt);
         }
-        inf.reset();
-        inf.setInput(b, (int) hoff + numin - 4 - tt, tt);
       }
-
     }
     inf.end();
 
-    int off;
-    off = 2 * (((uncomp[0] & 0x3F) << 8) | (uncomp[1] & 0xFF));
+    // Combine the chunks into one buffer
+    byte[] uncomp = new byte[result];
+    int pos = 0;
+    for (int i = 0; i < chunks.size(); ++i) {
+      System.arraycopy(chunks.get(i), 0, uncomp, pos, sizes.get(i));
+      pos += sizes.get(i);
+    }
+
+    int off = 2 * (((uncomp[0] & 0x3F) << 8) | (uncomp[1] & 0xFF));
     /* eat WMO and PIL */
     for (int i = 0; i < 2; i++) {
       while ((off < result) && (uncomp[off] != '\n'))

cdm/radial/src/test/data/nids/Level3_CYS_N0B_20241108_1948.nids

@@ -40,6 +40,7 @@ public class TestNids {
   private static String digitPrecipArrayFile = TestDir.localTestDataDir + "nids/DPA_20041123_1709";
   private static String vertIntegLiquidFile = TestDir.localTestDataDir + "nids/NVL_20041130_1946";
   private static String vadWindProfileFile = TestDir.localTestDataDir + "nids/NVW_20041117_1657";
+  private static String compressedBlockFile = TestDir.localTestDataDir + "nids/Level3_CYS_N0B_20241108_1948.nids";
 
   @Test
   public void testNidsReadRadial() throws IOException {
@@ -503,6 +504,21 @@ public void testRadialImageMessagePcode167() throws IOException {
     }
   }
 
+  @Test
+  public void testNidsCompressedBlock() throws IOException {
+    NetcdfFile ncfile = null;
+    try {
+      System.out.println("**** Open " + compressedBlockFile);
+      ncfile = NetcdfFile.open(compressedBlockFile);
+    } catch (java.io.IOException e) {
+      System.out.println(" fail = " + e);
+      e.printStackTrace();
+      assert (false);
+    }
+    ncfile.close();
+  }
+
+
   private void testReadData(Variable v) {
     Array a = null;
     assert (null != v);