Merge pull request #1465 from lesserwhirls/gh-1435-log-403

Do not stop trying to disambiguate http locations when 403s are encountered

Author: lesserwhirls

cdm/core/src/main/java/ucar/nc2/dataset/DatasetUrl.java

@@ -1,27 +1,34 @@
 /*
- * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
+
 package ucar.nc2.dataset;
 
 import static java.net.HttpURLConnection.HTTP_FORBIDDEN;
 import static java.net.HttpURLConnection.HTTP_NOT_ACCEPTABLE;
 import static java.net.HttpURLConnection.HTTP_OK;
 import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED;
+
 import com.google.common.annotations.VisibleForTesting;
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
 import javax.annotation.Nullable;
 
-import com.google.common.collect.Multimap;
 import thredds.client.catalog.ServiceType;
-import thredds.inventory.MFile;
-import thredds.inventory.MFiles;
 import ucar.httpservices.HTTPFactory;
 import ucar.httpservices.HTTPMethod;
 import ucar.nc2.util.EscapeStrings;
 import ucar.unidata.util.StringUtil2;
 import ucar.unidata.util.Urlencoded;
-import java.io.*;
-import java.util.*;
 
 /**
  * Detection of the protocol from a location string.
@@ -31,6 +38,7 @@
  * @since 10/20/2015.
  */
 public class DatasetUrl {
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DatasetUrl.class);
   private static final String alpha = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
   private static final String slashalpha = "\\/" + alpha;
 
@@ -398,15 +406,23 @@ private static ServiceType disambiguateHttp(String location) throws IOException
     return null;
   }
 
+  private static void handleCheckIfResponse(HTTPMethod method, int statusCode, ServiceType serviceChecked)
+      throws IOException {
+    if (statusCode == HTTP_UNAUTHORIZED) {
+      throw new IOException("Unauthorized to open dataset " + method.getURI());
+    } else if (statusCode == HTTP_FORBIDDEN) {
+      logger.warn(String.format(
+          "Forbidden Request to %s - assuming remote server is not a %s server, but could also indicate incorrect credentials.",
+          method.getURI(), serviceChecked));
+    }
+  }
+
   // cdmremote
   private static ServiceType checkIfCdmr(String location) throws IOException {
     try (HTTPMethod method = HTTPFactory.Head(location + "?req=header")) {
       int statusCode = method.execute();
       if (statusCode >= 300) {
-        if (statusCode == HTTP_UNAUTHORIZED || statusCode == HTTP_FORBIDDEN)
-          throw new IOException("Unauthorized to open dataset " + location);
-        else
-          throw new IOException(location + " is not a valid URL, return status=" + statusCode);
+        handleCheckIfResponse(method, statusCode, ServiceType.CdmRemote);
       }
 
       Optional<String> value = method.getResponseHeaderValue("Content-Description");
@@ -442,8 +458,7 @@ private static ServiceType checkIfDods(String location) throws IOException {
             throw new IOException("OPeNDAP Server Error= " + method.getResponseAsString());
         }
       }
-      if (status == HTTP_UNAUTHORIZED || status == HTTP_FORBIDDEN)
-        throw new IOException("Unauthorized to open dataset " + location);
+      handleCheckIfResponse(method, status, ServiceType.OPENDAP);
 
       // not dods
       return null;
@@ -471,6 +486,7 @@ else if (location.endsWith(".html"))
             return ServiceType.DAP4;
         }
       }
+      handleCheckIfResponse(method, status, ServiceType.DAP4);
       // not dap4
       return null;
     }
@@ -497,14 +513,15 @@ private static boolean checkIfRemoteNcml(String location) throws IOException {
         method.setRequestHeader("accept-encoding", "identity");
         int statusCode = method.execute();
         if (statusCode >= 300) {
-          if (statusCode == HTTP_UNAUTHORIZED) {
-            throw new IOException("Unauthorized to open dataset " + location);
-          } else if (statusCode == HTTP_NOT_ACCEPTABLE) {
+          handleCheckIfResponse(method, statusCode, ServiceType.NCML);
+          // additional checks
+          if (statusCode == HTTP_NOT_ACCEPTABLE) {
             String msg = location + " - this server does not support returning content without any encoding.";
             msg = msg + " Please download the file locally. Return status=" + statusCode;
             throw new IOException(msg);
           } else {
-            throw new IOException(location + " is not a valid URL, return status=" + statusCode);
+            throw new IOException(String.format("Error opening %s: %s, (HTTP Status Code %d)", location,
+                method.getResponseAsString(), statusCode));
           }
         }
 

cdm/core/src/main/java/ucar/unidata/io/http/HTTPRandomAccessFile.java

@@ -1,7 +1,8 @@
 /*
- * Copyright (c) 1998-2019 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2025 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
+
 package ucar.unidata.io.http;
 
 import java.io.FileNotFoundException;
@@ -40,6 +41,9 @@ public final class HTTPRandomAccessFile extends RemoteRandomAccessFile {
   private static final long httpMaxCacheSize = Long
       .parseLong(System.getProperty("ucar.unidata.io.http.maxReadCacheSize", String.valueOf(defaultMaxReadCacheSize)));
 
+  private static final String HTTPS = "https";
+  private static final String HTTP = "http";
+
   private static final boolean debug = false, debugDetails = false;
 
   ///////////////////////////////////////////////////////////////////////////////////
@@ -234,7 +238,7 @@ public long getLastModified() {
    */
   public static class Provider implements RandomAccessFileProvider {
 
-    private static final List<String> possibleSchemes = Arrays.asList("http", "https", "nodods", "httpserver");
+    private static final List<String> possibleSchemes = Arrays.asList(HTTP, HTTPS, "nodods", "httpserver");
 
     @Override
     public boolean isOwnerOf(String location) {
@@ -250,10 +254,26 @@ public RandomAccessFile open(String location) throws IOException {
     @Override
     public RandomAccessFile open(String location, int bufferSize) throws IOException {
       String scheme = location.split(":")[0];
-      if (!scheme.equalsIgnoreCase("https") && !scheme.equalsIgnoreCase("http")) {
-        location = location.replace(scheme, "http");
+      boolean fallback = false;
+      if (!scheme.equalsIgnoreCase(HTTPS) && !scheme.equalsIgnoreCase(HTTP)) {
+        location = location.replaceFirst(scheme, HTTPS);
+        fallback = true;
+      }
+      HTTPRandomAccessFile raf;
+      try {
+        // try with https first
+        raf = new HTTPRandomAccessFile(location, bufferSize, httpMaxCacheSize);
+      } catch (IOException e) {
+        if (fallback) {
+          // if we had to guess the scheme, fallback to http, and if it does not work, let the IOError be thrown
+          raf = new HTTPRandomAccessFile(location.replaceFirst(HTTPS, HTTP), bufferSize, httpMaxCacheSize);
+        } else {
+          // didn't try to fallback to http, just throw the original error
+          throw e;
+        }
       }
-      return new HTTPRandomAccessFile(location, bufferSize, httpMaxCacheSize);
+
+      return raf;
     }
   }
 }