Fix tests (#68)

drernie · web-flow · commit c5c89be37ab3 · 2024-12-12T20:28:43.000-08:00
* bump versions

* Create Makefile

* ignore intellij files

* ignore blank workflows

* createNamespaceAtUri

* CHANGELOG

* debug testS3WorkflowPush failure

* passed metadata schema

* updated hash for new config file

* publish 0.1.6

* Trim logging

* Manifest.BuildFromPaths

* intellij linting

* Manifest.BuildFromDir

* update hash

- always create well-formed system metadata
- use filename for logical key (to test BuildFromDir)

---------

Co-authored-by: Dr. Ernie Prabhakar &lt;19791+drernie@users.noreply.github.com&gt;
diff --git a/.gitignore b/.gitignore
@@ -10,3 +10,4 @@ lib/bin
 .vscode
 .DS_Store
 gradle.properties
+/.idea
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,13 @@
 # CHANGELOG.md
 
+## [0.1.6] - UNRELEASED
+
+- Fix failing CI tests
+- Add convenience methods:
+  - Registry.CreateNamespaceAtUri
+  - Manifest.BuildFromPaths
+  - Manifest.BuildFromDir
+
 ## [0.1.5] - 2024-08-27
 
 - Remove debugging
diff --git a/Makefile b/Makefile
@@ -0,0 +1,10 @@
+.PHONY: all verify clean compile
+
+verify:
+	./gradlew check || open lib/build/reports/tests/test/index.html
+
+clean:
+	./gradlew clean
+
+compile: clean
+	./gradlew compileJava
diff --git a/lib/build.gradle b/lib/build.gradle
@@ -26,11 +26,11 @@ dependencies {
     implementation 'software.amazon.awssdk.crt:aws-crt:0.30.9'
 
     // JSON and YAML parsing.
-    implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.2'
-    implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.17.2'
+    implementation 'com.fasterxml.jackson.core:jackson-databind:2.18.2'
+    implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.18.2'
 
     // JSON Schema validator.
-    implementation 'io.vertx:vertx-json-schema:4.5.9'
+    implementation 'io.vertx:vertx-json-schema:4.5.11'
 
     // Workarounds for Java's checked exceptions.
     implementation 'com.pivovarit:throwing-function:1.6.1'
@@ -74,7 +74,7 @@ mavenPublishing {
 
     signAllPublications()
 
-    coordinates('com.quiltdata', 'quiltcore', '0.1.5')
+    coordinates('com.quiltdata', 'quiltcore', '0.1.6')
 
     pom {
         name.set('QuiltCore')
@@ -83,7 +83,7 @@ mavenPublishing {
         licenses {
             license {
                 name.set('Apache License, Version 2.0')
-                url.set('http://www.apache.org/licenses/LICENSE-2.0.txt')
+                url.set('https://www.apache.org/licenses/LICENSE-2.0.txt')
             }
         }
         developers {
diff --git a/lib/src/main/java/com/quiltdata/quiltcore/Entry.java b/lib/src/main/java/com/quiltdata/quiltcore/Entry.java
@@ -6,18 +6,6 @@
 import java.security.NoSuchAlgorithmException;
 
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-/**
- * A Jackson class that represents a JSON object node.
- * 
- * <p>
- * An `ObjectNode` is a container for key-value pairs, where the keys are strings and the values can be any valid JSON node.
- * It provides methods to manipulate and access the key-value pairs within the object.
- * </p>
- * 
- * <p>
- * This class is part of the Jackson JSON library, which provides a fast and flexible way to process JSON data in Java.
- * </p>
- */
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.quiltdata.quiltcore.key.PhysicalKey;
 
@@ -26,19 +14,25 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Represents an entry in the Quilt data repository.
- * An entry contains information about a file or object stored in the repository.
- */
-/**
- * Represents an entry in the Quilt dataset.
+ * Represents an individual entry in the Quilt data repository.
+ *
+ * <p>
+ * The {@code Entry} class represents a row in a Quilt package.
+ * </p>
+ *
+ * <h2>Usage Example:</h2>
+ * <pre>{@code
+ * Entry entry = new Entry(new LocalPhysicalKey("foo), 123, hash, metadata)
+ * }</pre>
+ *
  */
 public class Entry {
     private static final Logger logger = LoggerFactory.getLogger(Entry.class);
 
     /**
      * Enumerates the types of hash algorithms supported by Quilt.
      */
-    public static enum HashType {
+    public enum HashType {
         /**
          * The SHA-256 hash algorithm.
          */
@@ -108,6 +102,21 @@ public Entry(PhysicalKey physicalKey, long size, Hash hash, ObjectNode metadata)
         this.metadata = metadata == null ? JsonNodeFactory.instance.objectNode() : metadata.deepCopy();
     }
 
+    /**
+     * String representation
+     *
+     * @return the Entry details
+     */
+    @Override
+    public String toString() {
+        return "Entry{" +
+                "physicalKey=" + physicalKey +
+                ", size=" + size +
+                ", hash=" + hash +
+                ", meta=" + metadata +
+                '}';
+    }
+
     /**
      * Returns the physical key of the entry.
      *
diff --git a/lib/src/main/java/com/quiltdata/quiltcore/Manifest.java b/lib/src/main/java/com/quiltdata/quiltcore/Manifest.java
@@ -60,6 +60,7 @@ public class Manifest {
      * The version of the manifest.
      */
     public static final String VERSION = "v0";
+    public static final String USER_META = "user_meta";
 
     private static final ObjectMapper TOP_HASH_MAPPER;
 
@@ -77,13 +78,13 @@ public class Manifest {
      * Returns a map for a URI of the form
      * "quilt+s3://bucket#package=package{@literal @}hash{@literal &}path=path"
      * 
-     * @param uri
+     * @param uri: Quilt+ URI
      * @return Map{@literal <}String, String{@literal >}
-     * @throws IllegalArgumentException
+     * @throws IllegalArgumentException if scheme != quilt+s3
      */
 
     public static Map<String, String> ParseQuiltURI(URI uri) throws IllegalArgumentException {
-        Map<String, String> result = new TreeMap<java.lang.String, java.lang.String>();
+        Map<String, String> result = new TreeMap<>();
         String scheme = uri.getScheme();
         if (!scheme.equals("quilt+s3")) {
             throw new IllegalArgumentException("Invalid scheme: " + scheme);
@@ -151,15 +152,117 @@ public static Manifest FromQuiltURI(String quiltURI) throws URISyntaxException,
             String revision = parts.get("revision");
             hash = n.getHash(revision);
         }
-        Manifest m = n.getManifest(hash);
-        return m;
+        return n.getManifest(hash);
     }
-    
+
+    /**
+     * Formats the provided user metadata into a JSON {@link ObjectNode}.
+     *
+     * <p>
+     * This method takes a user-provided metadata object, processes it, and converts it
+     * into a structured JSON object. The resulting {@code ObjectNode} can be integrated
+     * into manifests or other contexts where JSON metadata is required.
+     * </p>
+     *
+     * @param user_meta The user metadata to be formatted. This can be a Map, a String, or null.
+     *
+     * @return A JSON {@link ObjectNode} representing the formatted user metadata.
+     *         Returns an empty {@link ObjectNode} if {@code user_meta} is {@code null}.
+     *
+     * @throws IllegalArgumentException if the provided {@code user_meta} is invalid
+     *                                  and cannot be processed.
+     *
+     * <h2>Usage Example:</h2>
+     * <pre>{@code
+     * Object userMeta = new HashMap<>();
+     * ((Map) userMeta).put("key", "value");
+     *
+     * ObjectNode formattedMeta = FormatUserMeta(userMeta);
+     * System.out.println(formattedMeta.toString()); // Outputs JSON representation
+     * }</pre>
+     */
+    public static ObjectNode FormatUserMeta(Object user_meta) {
+        ObjectNode base = JsonNodeFactory.instance.objectNode().put("version", VERSION);
+
+        if (user_meta == null) {
+            return base.set(USER_META, null);
+        } else if (user_meta instanceof Map) {
+            ObjectMapper mapper = new ObjectMapper();
+            return base.set(USER_META, mapper.valueToTree(user_meta));
+        } else if (user_meta instanceof String) {
+            return base.put(USER_META, (String)user_meta);
+        }
+        throw new IllegalArgumentException("Invalid user_meta[" + user_meta.getClass().getName() + "] " + user_meta);
+    }
+
+    /**
+     * Builds a {@link Manifest} from the provided paths, user metadata, and object metadata.
+     *
+     * <p>
+     * This static method constructs a {@code Manifest} object by taking a mapping of
+     * logical names to physical paths, along with optional user-provided metadata and
+     * object-specific metadata. It validates and processes the input to generate
+     * a structured representation of the manifest.
+     * </p>
+     *
+     * @param paths A map where the keys are logical names, and the values are
+     *              corresponding file paths on the system. Cannot be {@code null}.
+     * @param user_meta Optional user metadata associated with the manifest. Can be {@code null}.
+     * @param object_meta A map where the keys are object names,
+     *                    and the values are JSON object nodes containing metadata for each object. Can be {@code null}.
+     *
+     * @return A constructed {@link Manifest} instance encapsulating the provided data.
+     *
+     * @throws IllegalArgumentException if the {@code paths} map is {@code null} or contains invalid entries.
+     *
+     * <h2>Usage Example:</h2>
+     * <pre>{@code
+     * Map<String, Path> paths = new HashMap<>();
+     * paths.put("exampleKey", Paths.get("/example/path"));
+     *
+     * Manifest manifest = Manifest.BuildFromPaths(paths, null, null);
+     * }</pre>
+     */
+    public static Manifest BuildFromPaths(Map<String, Path> paths, Object user_meta, Map<String, ObjectNode> object_meta) {
+        Manifest.Builder b = Manifest.builder();
+        ObjectNode packageMeta = FormatUserMeta(user_meta);
+        b.setMetadata(packageMeta);
+        for (Map.Entry<String, Path> e : paths.entrySet()) {
+            String key = e.getKey();
+            Path p = e.getValue();
+            ObjectNode obj_meta = (object_meta != null) ? object_meta.get(key) : null;
+            try {
+                long size = Files.size(p);
+                b.addEntry(key, new Entry(new LocalPhysicalKey(p), size, null, obj_meta));
+            } catch (IOException ex) {
+                logger.error("Skipping entry[{}]: failed to get size for path {}", key, p, ex);
+            }
+        }
+        return b.build();
+    }
+
+    public static Manifest BuildFromDir(Path dir, Object user_meta, String regex) {
+        Map<String, Path> map = new TreeMap<>();
+        try {
+            Files.walk(dir)
+                    .filter(Files::isRegularFile) // Filter regular files
+                    .forEach(f -> {
+                        String logicalKey = dir.relativize(f).toString();
+                        if (regex == null || logicalKey.matches(regex)) {
+                            map.put(logicalKey, f); // Add the entry to the map
+                        }
+                    });
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return BuildFromPaths(map, user_meta, null);
+    }
+
     /**
      * Represents a builder for creating a {@link Manifest} object.
      */
     public static class Builder {
-        private SortedMap<String, Entry> entries;
+        private final SortedMap<String, Entry> entries;
         private ObjectNode metadata;
 
         /**
@@ -265,12 +368,12 @@ public static Manifest createFromFile(PhysicalKey path) throws IOException, Ille
                 Entry.HashType hashType = Entry.HashType.enumFor(hashNode.get("type").asText());
                 String hashValue = hashNode.get("value").asText();
                 JsonNode meta = row.get("meta");
-                if (meta == null) {
-                    // leave it as is
-                } else if (meta.isNull()) {
-                    meta = null;
-                } else if (!meta.isObject()) {
-                    throw new IOException("Invalid entry metadata: " + node);
+                if (meta != null) {
+                    if (meta.isNull()) {
+                        meta = null;
+                    } else if (!meta.isObject()) {
+                        throw new IOException("Invalid entry metadata: " + node);
+                    }
                 }
 
                 Entry entry = new Entry(physicalKey, size, new Entry.Hash(hashType, hashValue), (ObjectNode)meta);
@@ -442,7 +545,7 @@ public void install(Path dest) throws IOException {
                 S3TransferManager transferManager =
                     S3TransferManager.builder()
                         .s3Client(s3)
-                        .build();
+                        .build()
             ) {
                 List<CompletableFuture<CompletedFileDownload>> futures = new ArrayList<>(bucketEntries.size());
 
@@ -475,9 +578,10 @@ public void install(Path dest) throws IOException {
     }
 
     private JsonNode validate(Namespace namespace, String message, String workflow) throws ConfigurationException, WorkflowException {
+        logger.info("Validating manifest with {} entries for namespace: {} workflow: {}", entries.size(), namespace.getName(), workflow);
         WorkflowConfig config = namespace.getRegistry().getWorkflowConfig();
         if (config == null) {
-            if (workflow == null) {
+            if (workflow == null || workflow.isBlank()) {
                 return null;
             }
             throw new WorkflowException("Workflow is specified, but no workflows config exists");
@@ -534,12 +638,12 @@ public Manifest push(Namespace namespace, String message, String workflow) throw
         }
         builder.setMetadata(newMetadata);
 
-        logger.debug("Building transfer manager for bucket: {}", destBucket);
+        logger.debug("push: building transfer manager for bucket: {}", destBucket);
         try(
             S3TransferManager transferManager =
                 S3TransferManager.builder()
                     .s3Client(s3)
-                    .build();
+                    .build()
         ) {
             List<Map.Entry<String, CompletableFuture<CompletedFileUpload>>> futures =
                 new ArrayList<>(entriesWithHashes.size());
diff --git a/lib/src/main/java/com/quiltdata/quiltcore/Registry.java b/lib/src/main/java/com/quiltdata/quiltcore/Registry.java
@@ -6,6 +6,9 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.net.URI;
+import java.net.URISyntaxException;
+
 /**
  * The Registry class represents a registry of packages and namespaces in the Quilt Core library.
  * It provides methods to access and manipulate the registry.
@@ -17,6 +20,24 @@ public class Registry {
     private final PhysicalKey versions;
     private final PhysicalKey workflowConfigPath;
 
+    /**
+     * Constructs a new Namespace object for a registry as that @uriString
+     *
+     * @param pkgName: namespace of the package
+     * @param uriString: uri of the physical key hosting the registry
+     * @return Namespace
+     * @throws URISyntaxException: if uriString is invalid
+     */
+    public static Namespace CreateNamespaceAtUri(String pkgName, String uriString) throws URISyntaxException {
+        if (!uriString.endsWith("/")) {
+            uriString += '/';
+        }
+        URI uri = new URI(uriString);
+        PhysicalKey pk = PhysicalKey.fromUri(uri);
+        Registry r = new Registry(pk);
+        return r.getNamespace(pkgName);
+    }
+
     /**
      * Constructs a new Registry object with the specified root physical key.
      *
@@ -27,6 +48,7 @@ public Registry(PhysicalKey root) {
         names = root.resolve(".quilt/named_packages");
         versions = root.resolve(".quilt/packages");
         workflowConfigPath = root.resolve(".quilt/workflows/config.yml");
+        // TODO: Handle config.yaml as well
     }
 
     /**
diff --git a/lib/src/main/java/com/quiltdata/quiltcore/S3ClientStore.java b/lib/src/main/java/com/quiltdata/quiltcore/S3ClientStore.java
@@ -20,9 +20,9 @@ public class S3ClientStore {
     private static final Logger logger = LoggerFactory.getLogger(S3ClientStore.class);
     private static final S3Client LOCATION_CLIENT = createClient(Region.US_EAST_1);
 
-    private static Map<String, Region> regionMap = Collections.synchronizedMap(new HashMap<>());
-    private static Map<Region, S3AsyncClient> asyncClientMap = Collections.synchronizedMap(new HashMap<>());
-    private static Map<Region, S3Client> clientMap = Collections.synchronizedMap(new HashMap<>());
+    private static final Map<String, Region> regionMap = Collections.synchronizedMap(new HashMap<>());
+    private static final Map<Region, S3AsyncClient> asyncClientMap = Collections.synchronizedMap(new HashMap<>());
+    private static final Map<Region, S3Client> clientMap = Collections.synchronizedMap(new HashMap<>());
 
     /**
      * Retrieves an asynchronous S3 client for the specified bucket.
diff --git a/lib/src/main/java/com/quiltdata/quiltcore/workflows/WorkflowConfig.java b/lib/src/main/java/com/quiltdata/quiltcore/workflows/WorkflowConfig.java
diff --git a/lib/src/main/java/com/quiltdata/quiltcore/workflows/WorkflowValidator.java b/lib/src/main/java/com/quiltdata/quiltcore/workflows/WorkflowValidator.java
diff --git a/lib/src/test/java/com/quiltdata/quiltcore/RegistryTest.java b/lib/src/test/java/com/quiltdata/quiltcore/RegistryTest.java
diff --git a/lib/src/test/java/com/quiltdata/quiltcore/WorkflowsTest.java b/lib/src/test/java/com/quiltdata/quiltcore/WorkflowsTest.java
