From 85301b69f478c34e3ee7488f5b04ae4ac5b3ba1e Mon Sep 17 00:00:00 2001 From: Bernhard Guillon Date: Tue, 7 Jul 2026 12:32:15 +0200 Subject: Chunk manifest transfer: 1000 files per batch Avoids bufio.Scanner token limit by sending manifest in batches. Each batch is a separate JSON line with up to 1000 files. --- cmd_linux.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'cmd_linux.go') diff --git a/cmd_linux.go b/cmd_linux.go index 4fcc891..b80b197 100644 --- a/cmd_linux.go +++ b/cmd_linux.go @@ -88,7 +88,7 @@ func handleClient(conn net.Conn, stack *GitignoreStack) { defer conn.Close() scanner := bufio.NewScanner(conn) - scanner.Buffer(make([]byte, 0, 64*1024*1024), 64*1024*1024) // 64MB buffer for large manifests + scanner.Buffer(make([]byte, 0, 1024*1024), 1024*1024) // 1MB buffer per batch remoteManifest := &Manifest{Files: make(map[string]FileMeta)} for scanner.Scan() { @@ -99,11 +99,19 @@ func handleClient(conn net.Conn, stack *GitignoreStack) { } switch msg.Type { - case "manifest": - remoteManifest = ManifestFromJSON(msg.Files) - logInfo("manifest: received %d files from host", len(remoteManifest.Files)) + case "manifest_batch": + for k, v := range msg.Files { + remoteManifest.Files[k] = FileMeta{ + Size: v.Size, + Mode: os.FileMode(v.Mode), + ModTime: parseTime(v.ModTime), + Symlink: v.Symlink, + } + } + logDebug("manifest: received batch (%d files, total: %d)", len(msg.Files), len(remoteManifest.Files)) case "manifest_done": + logInfo("manifest: received %d files from host", len(remoteManifest.Files)) logInfo("manifest: comparing with local state ...") syncFromManifest(remoteManifest, stack) logInfo("sync: done") @@ -276,3 +284,8 @@ func scanLocalDirectory(root string) *Manifest { return manifest } + +func parseTime(s string) time.Time { + t, _ := time.Parse(time.RFC3339Nano, s) + return t +} -- cgit v1.2.3