documentation

Author: Jan Kaul

iceberg-rust/src/table/manifest_list.rs

@@ -138,6 +138,35 @@ pub(crate) async fn read_snapshot<'metadata>(
     ManifestListReader::new(bytes, table_metadata)
 }
 
+/// A writer for Iceberg manifest list files that manages the creation and updating of manifest lists.
+///
+/// The ManifestListWriter is responsible for:
+/// - Creating new manifest list files from scratch or updating existing ones
+/// - Managing manifest entries and their metadata
+/// - Optimizing data file organization through splitting and partitioning
+/// - Writing the final manifest list to object storage
+///
+/// This writer can operate in two modes:
+/// 1. **New manifest list**: Creates a completely new manifest list from data files
+/// 2. **Append to existing**: Reuses compatible manifests from an existing manifest list
+///
+/// The writer automatically handles:
+/// - Partition boundary calculations
+/// - Manifest splitting for optimal performance
+/// - Schema compatibility between format versions
+/// - Concurrent manifest writing operations
+///
+/// # Type Parameters
+/// * `'schema` - The lifetime of the Avro schema used for serialization
+/// * `'metadata` - The lifetime of the table metadata reference
+///
+/// # Fields
+/// * `table_metadata` - Reference to the table metadata for schema and configuration
+/// * `writer` - The underlying Avro writer for manifest list serialization
+/// * `selected_manifest` - Optional existing manifest that can be reused for appends
+/// * `bounding_partition_values` - Computed partition boundaries for the data files
+/// * `n_existing_files` - Count of existing files for split calculations
+/// * `branch` - Optional branch name for multi-branch table operations
 pub(crate) struct ManifestListWriter<'schema, 'metadata> {
     table_metadata: &'metadata TableMetadata,
     writer: AvroWriter<'schema, Vec<u8>>,
@@ -148,6 +177,36 @@ pub(crate) struct ManifestListWriter<'schema, 'metadata> {
 }
 
 impl<'schema, 'metadata> ManifestListWriter<'schema, 'metadata> {
+    /// Creates a new ManifestListWriter for building a manifest list from scratch.
+    ///
+    /// This constructor initializes a writer that will create a completely new manifest list
+    /// without reusing any existing manifests. It computes partition boundaries from the
+    /// provided data files and sets up the Avro writer with the appropriate schema.
+    ///
+    /// # Arguments
+    /// * `data_files` - Iterator over data files to compute partition boundaries from
+    /// * `schema` - The Avro schema to use for manifest list serialization
+    /// * `table_metadata` - Reference to the table metadata for partition field information
+    /// * `branch` - Optional branch name for multi-branch table operations
+    ///
+    /// # Returns
+    /// * `Result<Self, Error>` - A new ManifestListWriter instance or an error
+    ///
+    /// # Errors
+    /// Returns an error if:
+    /// * The partition fields cannot be retrieved from table metadata
+    /// * Partition boundary computation fails
+    /// * The Avro writer cannot be initialized
+    ///
+    /// # Example Usage
+    /// ```ignore
+    /// let writer = ManifestListWriter::new(
+    ///     data_files.iter(),
+    ///     &manifest_list_schema,
+    ///     &table_metadata,
+    ///     Some("main"),
+    /// )?;
+    /// ```
     pub(crate) fn new<'datafiles>(
         data_files: impl Iterator<Item = &'datafiles DataFile>,
         schema: &'schema AvroSchema,
@@ -176,6 +235,48 @@ impl<'schema, 'metadata> ManifestListWriter<'schema, 'metadata> {
         })
     }
 
+    /// Creates a new ManifestListWriter from an existing manifest list, optimizing for append operations.
+    ///
+    /// This constructor analyzes an existing manifest list to determine which manifests can be
+    /// reused for the new operation. It selects compatible manifests based on partition boundaries
+    /// and copies other manifests to the new manifest list. This approach optimizes append
+    /// operations by avoiding unnecessary manifest rewrites.
+    ///
+    /// The method:
+    /// 1. Reads the existing manifest list to understand current manifests
+    /// 2. Computes partition boundaries for the new data files
+    /// 3. Selects manifests that can be reused (partitioned vs unpartitioned logic)
+    /// 4. Copies non-selected manifests to the new manifest list
+    /// 5. Prepares to append new data to the selected manifest
+    ///
+    /// # Arguments
+    /// * `bytes` - The raw bytes of the existing manifest list file
+    /// * `data_files` - Iterator over new data files to be appended
+    /// * `schema` - The Avro schema to use for manifest list serialization
+    /// * `table_metadata` - Reference to the table metadata for partition field information
+    /// * `branch` - Optional branch name for multi-branch table operations
+    ///
+    /// # Returns
+    /// * `Result<Self, Error>` - A new ManifestListWriter instance with selected manifest or an error
+    ///
+    /// # Errors
+    /// Returns an error if:
+    /// * The existing manifest list cannot be parsed
+    /// * Partition fields cannot be retrieved from table metadata
+    /// * Partition boundary computation fails
+    /// * Manifest selection logic fails
+    /// * The Avro writer cannot be initialized
+    ///
+    /// # Example Usage
+    /// ```ignore
+    /// let writer = ManifestListWriter::from_existing(
+    ///     &existing_manifest_list_bytes,
+    ///     new_data_files.iter(),
+    ///     &manifest_list_schema,
+    ///     &table_metadata,
+    ///     Some("main"),
+    /// )?;
+    /// ```
     pub(crate) fn from_existing<'datafiles>(
         bytes: &[u8],
         data_files: impl Iterator<Item = &'datafiles DataFile>,
@@ -220,6 +321,27 @@ impl<'schema, 'metadata> ManifestListWriter<'schema, 'metadata> {
         })
     }
 
+    /// Calculates the optimal number of manifest splits for the given number of data files.
+    ///
+    /// This method determines how many manifest files should be created to optimize
+    /// query performance and manage file sizes. The calculation considers:
+    /// - The number of existing files in the table
+    /// - The number of new data files being added
+    /// - The number of files in any selected (reusable) manifest
+    ///
+    /// The splitting strategy helps maintain optimal manifest sizes for efficient
+    /// query planning and metadata operations.
+    ///
+    /// # Arguments
+    /// * `n_data_files` - The number of new data files being added
+    ///
+    /// # Returns
+    /// * `u32` - The recommended number of manifest splits
+    ///
+    /// # Example Usage
+    /// ```ignore
+    /// let splits = writer.n_splits(1000); // Calculate splits for 1000 new files
+    /// ```
     pub(crate) fn n_splits(&self, n_data_files: usize) -> u32 {
         let selected_manifest_file_count = self
             .selected_manifest
@@ -244,6 +366,48 @@ impl<'schema, 'metadata> ManifestListWriter<'schema, 'metadata> {
         )
     }
 
+    /// Appends data files to a single manifest and finalizes the manifest list.
+    ///
+    /// This method creates a single manifest file containing all the provided data files,
+    /// either by appending to an existing reusable manifest or creating a new one.
+    /// It then writes the complete manifest list to object storage.
+    ///
+    /// This approach is optimal for:
+    /// - Small to medium append operations
+    /// - Cases where manifest splitting is not required
+    /// - Simple append operations without complex partitioning needs
+    ///
+    /// The process:
+    /// 1. Determines whether to reuse an existing manifest or create new one
+    /// 2. Creates/updates a manifest writer with the selected manifest
+    /// 3. Appends all provided data files to the manifest
+    /// 4. Finalizes the manifest and writes it to storage
+    /// 5. Adds the manifest entry to the manifest list
+    /// 6. Writes the complete manifest list to storage
+    ///
+    /// # Arguments
+    /// * `data_files` - Iterator over manifest entries to append
+    /// * `snapshot_id` - The snapshot ID for the new manifest
+    /// * `object_store` - The object store for writing files
+    ///
+    /// # Returns
+    /// * `Result<String, Error>` - The location of the new manifest list file or an error
+    ///
+    /// # Errors
+    /// Returns an error if:
+    /// * Manifest schema creation fails
+    /// * Manifest writer creation or operation fails
+    /// * Object storage operations fail
+    /// * Avro serialization fails
+    ///
+    /// # Example Usage
+    /// ```ignore
+    /// let manifest_list_location = writer.append_and_finish(
+    ///     data_files_iter,
+    ///     snapshot_id,
+    ///     object_store,
+    /// ).await?;
+    /// ```
     pub(crate) async fn append_and_finish(
         mut self,
         data_files: impl Iterator<Item = Result<ManifestEntry, Error>>,
@@ -314,6 +478,54 @@ impl<'schema, 'metadata> ManifestListWriter<'schema, 'metadata> {
         Ok(new_manifest_list_location)
     }
 
+    /// Appends data files by splitting them across multiple manifests and finalizes the manifest list.
+    ///
+    /// This method is designed for large append operations where splitting data files across
+    /// multiple manifest files provides better query performance and parallelism. It distributes
+    /// the data files across the specified number of splits based on partition boundaries.
+    ///
+    /// This approach is optimal for:
+    /// - Large append operations with hundreds or thousands of files
+    /// - Partitioned tables where files can be split by partition boundaries
+    /// - Cases requiring high query parallelism and performance
+    ///
+    /// The process:
+    /// 1. Computes optimal partition boundaries for splitting
+    /// 2. Merges new data files with existing files from selected manifest (if any)
+    /// 3. Splits all files across the specified number of manifest files
+    /// 4. Creates and writes multiple manifest files concurrently
+    /// 5. Adds all manifest entries to the manifest list
+    /// 6. Writes the complete manifest list to storage
+    ///
+    /// # Arguments
+    /// * `data_files` - Iterator over manifest entries to append and split
+    /// * `snapshot_id` - The snapshot ID for the new manifests
+    /// * `n_splits` - The number of manifest files to create (should match `n_splits()` result)
+    /// * `object_store` - The object store for writing files
+    ///
+    /// # Returns
+    /// * `Result<String, Error>` - The location of the new manifest list file or an error
+    ///
+    /// # Errors
+    /// Returns an error if:
+    /// * Partition field retrieval fails
+    /// * Manifest schema creation fails
+    /// * File splitting logic fails
+    /// * Manifest writer creation or operation fails
+    /// * Concurrent manifest writing fails
+    /// * Object storage operations fail
+    /// * Avro serialization fails
+    ///
+    /// # Example Usage
+    /// ```ignore
+    /// let n_splits = writer.n_splits(data_files.len());
+    /// let manifest_list_location = writer.append_split_and_finish(
+    ///     data_files_iter,
+    ///     snapshot_id,
+    ///     n_splits,
+    ///     object_store,
+    /// ).await?;
+    /// ```
     pub(crate) async fn append_split_and_finish(
         mut self,
         data_files: impl Iterator<Item = Result<ManifestEntry, Error>>,