feat: decompress gzip compressed metadata files

Author: lizardoluis

Cargo.lock

@@ -16,6 +16,7 @@ async-trait = { workspace = true }
 bytes = { workspace = true }
 derive-getters = { workspace = true }
 derive_builder = { workspace = true }
+flate2 = { version = "1.1", features = ["zlib-rs"], default-features = false }
 futures = { workspace = true }
 getrandom = { workspace = true }
 iceberg-rust-spec = { path = "../iceberg-rust-spec", version = "0.7.0" }

iceberg-rust/Cargo.toml

@@ -20,6 +20,9 @@ pub enum Error {
     /// Conversion error
     #[error("Failed to convert {0} to {1}.")]
     Conversion(String, String),
+    /// Failed to decompress gzip data
+    #[error("Failed to decompress gzip data: {0}")]
+    Decompress(String),
     /// Not found
     #[error("{0} not found.")]
     NotFound(String),

iceberg-rust/src/error.rs

@@ -8,6 +8,8 @@ use iceberg_rust_spec::{
 use object_store::{Attributes, ObjectStore, PutOptions, TagSet};
 
 use crate::error::Error;
+use flate2::read::GzDecoder;
+use std::io::Read;
 
 /// Simplify interaction with iceberg files
 #[async_trait]
@@ -32,7 +34,16 @@ impl<T: ObjectStore> IcebergStore for T {
             .await?
             .bytes()
             .await?;
-        serde_json::from_slice(&bytes).map_err(Error::from)
+
+        if location.ends_with(".gz.metadata.json") {
+            let mut decoder = GzDecoder::new(&bytes[..]);
+            let mut decompressed_data = Vec::new();
+            decoder.read_to_end(&mut decompressed_data)
+                .map_err(|e| Error::Decompress(e.to_string()))?;
+            serde_json::from_slice(&decompressed_data).map_err(Error::from)
+        } else {
+            serde_json::from_slice(&bytes).map_err(Error::from)
+        }
     }
 
     async fn put_metadata(