Fix: Streaming packfile write/read (#79)

* add new API for FsClient to support streamable writes

* write packfile directly into file, without collecting data into memory first

* optimize code for disk reading for packfile instead of loading everything into memory

* check out files in parallel in batches

* increase batch size on checkout to 10

* improve FileSystemAccessApiFsClient.rename function

Author: alex-titarenko

src/clients/fs/FileSystemAccessApiFsClient.ts

@@ -10,7 +10,8 @@ import {
   EncodingOptions,
   FsClient,
   RmOptions,
-  StatsLike
+  StatsLike,
+  WritableStreamHandle
 } from '../../'
 
 import { BasicStats } from './BasicStats'
@@ -249,9 +250,7 @@ export class FileSystemAccessApiFsClient implements FsClient {
 
     const oldFilepathStat = await this.stat(oldPath)
     if (oldFilepathStat.isFile()) {
-      const data = await this.readFile(oldPath)
-      await this.writeFile(newPath, data)
-      await this.rm(oldPath)
+      await this.renameFile(oldPath, newPath)
     } else if (oldFilepathStat.isDirectory()) {
       await this.mkdir(newPath)
       const sourceFolder = await this.getDirectoryByPath(oldPath)
@@ -263,6 +262,40 @@ export class FileSystemAccessApiFsClient implements FsClient {
     }
   }
 
+  private async renameFile(oldPath: string, newPath: string): Promise<void> {
+    const { folderPath: oldFolder, leafSegment: oldName } = this.getFolderPathAndLeafSegment(oldPath)
+    const { folderPath: newFolder, leafSegment: newName } = this.getFolderPathAndLeafSegment(newPath)
+
+    const oldDir = await this.getDirectoryByPath(oldFolder)
+    const fileHandle = await this.getEntry<'file'>(oldDir, oldName, 'file')
+    if (!fileHandle) {
+      throw new ENOENT(oldPath)
+    }
+
+    // Strategy 1: Native move() — zero-copy rename, supported in Chrome and Safari OPFS.
+    // Always pass (directory, newName) form — Safari doesn't support the move(newName) shorthand.
+    if (typeof fileHandle.move === 'function') {
+      const newDir = oldFolder === newFolder ? oldDir : await this.getDirectoryByPath(newFolder)
+      await fileHandle.move(newDir, newName)
+      return
+    }
+
+    // Strategy 2: Streaming copy — read in chunks, write via stream. Never loads entire file.
+    const CHUNK_SIZE = 1024 * 1024
+    const file = await fileHandle.getFile()
+    const writable = await this.createWritableStream(newPath)
+    let offset = 0
+    while (offset < file.size) {
+      const end = Math.min(offset + CHUNK_SIZE, file.size)
+      const blob = file.slice(offset, end)
+      const chunk = new Uint8Array(await blob.arrayBuffer())
+      await writable.write(chunk)
+      offset = end
+    }
+    await writable.close()
+    await this.rm(oldPath)
+  }
+
   /**
    * Symlinks are not supported in the current implementation.
    * @throws Error: symlinks are not supported.
@@ -279,6 +312,44 @@ export class FileSystemAccessApiFsClient implements FsClient {
     throw new Error('Symlinks are not supported.')
   }
 
+  public async createWritableStream(path: string): Promise<WritableStreamHandle> {
+    const { folderPath, leafSegment } = this.getFolderPathAndLeafSegment(path)
+    const targetDir = await this.getDirectoryByPath(folderPath)
+
+    const fileHandle = await targetDir.getFileHandle(leafSegment, { create: true })
+    const writable = await fileHandle.createWritable()
+
+    return {
+      write: async (data: Uint8Array) => {
+        // FileSystemWritableFileStream.write() may write the entire underlying
+        // ArrayBuffer instead of just the TypedArray view when byteOffset > 0.
+        // This happens with Buffer.slice() which shares the backing memory.
+        // Create a clean copy when the view doesn't cover the full buffer.
+        if (data.byteOffset !== 0 || data.buffer.byteLength !== data.byteLength) {
+          data = new Uint8Array(data)
+        }
+        await writable.write(data)
+      },
+      close: async () => {
+        await writable.close()
+      }
+    }
+  }
+
+  public async readFileSlice(path: string, start: number, end: number): Promise<Uint8Array> {
+    const { folderPath, leafSegment } = this.getFolderPathAndLeafSegment(path)
+    const targetDir = await this.getDirectoryByPath(folderPath)
+
+    const fileHandle = await this.getEntry<'file'>(targetDir, leafSegment, 'file')
+    if (!fileHandle) {
+      throw new ENOENT(path)
+    }
+
+    const file = await fileHandle.getFile()
+    const blob = file.slice(start, end)
+    return new Uint8Array(await blob.arrayBuffer())
+  }
+
   /**
    * Return true if a entry exists, false if it doesn't exist.
    * Rethrows errors that aren't related to entry existance.
@@ -388,13 +459,14 @@ export class FileSystemAccessApiFsClient implements FsClient {
 
       if (this.options.useSyncAccessHandle) {
         const accessHandle = await fileHandle.createSyncAccessHandle()
-        const dataArray = typeof data === 'string' ? this.textEncoder.encode(data) : data
+        const dataArray = typeof data === 'string' ? this.textEncoder.encode(data) : new Uint8Array(data)
         accessHandle.write(dataArray.buffer as ArrayBuffer, { at: 0 })
         await accessHandle.flush()
         await accessHandle.close()
       } else {
         const writable = await fileHandle.createWritable()
-        await writable.write(typeof data === 'string' ? data : data.buffer as ArrayBuffer)
+        const writeData = typeof data === 'string' ? data : new Uint8Array(data)
+        await writable.write(writeData)
         await writable.close()
       }
     }, 'writeFile', name)

src/commands/checkout.ts

@@ -219,71 +219,74 @@ export async function _checkout({
     )
 
     await GitIndexManager.acquire({ fs, gitdir, cache }, async function(index) {
-      await Promise.all(
-        ops
-          .filter(
-            ([method]) =>
-              method === 'create' ||
-              method === 'create-index' ||
-              method === 'update' ||
-              method === 'mkdir-index'
-          )
-          .map(async function([method, fullpath, oid, mode, chmod]) {
-            const modeNum = Number(mode)
-            const filepath = `${dir}/${fullpath}`
-            try {
-              if (method !== 'create-index' && method !== 'mkdir-index') {
-                const { object } = await readObject({ fs, cache, gitdir, oid })
-                if (chmod) {
-                  // Note: the mode option of fs.write only works when creating files,
-                  // not updating them. Since the `fs` plugin doesn't expose `chmod` this
-                  // is our only option.
-                  await fs.rm(filepath)
-                }
-                if (modeNum === 0o100644) {
-                  // regular file
-                  await fs.write(filepath, object)
-                } else if (modeNum === 0o100755) {
-                  // executable file
-                  await fs.write(filepath, object, { mode: 0o777 })
-                } else if (modeNum === 0o120000) {
-                  // symlink
-                  await fs.writelink(filepath, object)
-                } else {
-                  throw new InternalError(
-                    `Invalid mode 0o${modeNum.toString(8)} detected in blob ${oid}`
-                  )
-                }
-              }
-
-              const stats = (await fs.lstat(filepath))!
-              // We can't trust the executable bit returned by lstat on Windows,
-              // so we need to preserve this value from the TREE.
-              // TODO: Figure out how git handles this internally.
-              if (modeNum === 0o100755) {
-                stats.mode = 0o755
+      const writeOps = ops.filter(
+        ([method]) =>
+          method === 'create' ||
+          method === 'create-index' ||
+          method === 'update' ||
+          method === 'mkdir-index'
+      )
+      // Process files in small batches to balance I/O concurrency with memory usage.
+      // Full Promise.all would OOM on large packfiles; purely sequential loses I/O overlap.
+      const BATCH_SIZE = 10
+      for (let i = 0; i < writeOps.length; i += BATCH_SIZE) {
+        const batch = writeOps.slice(i, i + BATCH_SIZE)
+        await Promise.all(batch.map(async ([method, fullpath, oid, mode, chmod]) => {
+          const modeNum = Number(mode)
+          const filepath = `${dir}/${fullpath}`
+          try {
+            if (method !== 'create-index' && method !== 'mkdir-index') {
+              const { object } = await readObject({ fs, cache, gitdir, oid })
+              if (chmod) {
+                // Note: the mode option of fs.write only works when creating files,
+                // not updating them. Since the `fs` plugin doesn't expose `chmod` this
+                // is our only option.
+                await fs.rm(filepath)
               }
-              // Submodules are present in the git index but use a unique mode different from trees
-              if (method === 'mkdir-index') {
-                stats.mode = 0o160000
+              if (modeNum === 0o100644) {
+                // regular file
+                await fs.write(filepath, object)
+              } else if (modeNum === 0o100755) {
+                // executable file
+                await fs.write(filepath, object, { mode: 0o777 })
+              } else if (modeNum === 0o120000) {
+                // symlink
+                await fs.writelink(filepath, object)
+              } else {
+                throw new InternalError(
+                  `Invalid mode 0o${modeNum.toString(8)} detected in blob ${oid}`
+                )
               }
-              index.insert({
-                filepath: fullpath,
-                stats,
-                oid,
+            }
+
+            const stats = (await fs.lstat(filepath))!
+            // We can't trust the executable bit returned by lstat on Windows,
+            // so we need to preserve this value from the TREE.
+            // TODO: Figure out how git handles this internally.
+            if (modeNum === 0o100755) {
+              stats.mode = 0o755
+            }
+            // Submodules are present in the git index but use a unique mode different from trees
+            if (method === 'mkdir-index') {
+              stats.mode = 0o160000
+            }
+            index.insert({
+              filepath: fullpath,
+              stats,
+              oid,
+            })
+            if (onProgress) {
+              await onProgress({
+                phase: 'Updating workdir',
+                loaded: ++count,
+                total,
               })
-              if (onProgress) {
-                await onProgress({
-                  phase: 'Updating workdir',
-                  loaded: ++count,
-                  total,
-                })
-              }
-            } catch (e) {
-              console.log(e)
             }
-          })
-      )
+          } catch (e) {
+            console.log(e)
+          }
+        }))
+      }
     })
   }