From 85301b69f478c34e3ee7488f5b04ae4ac5b3ba1e Mon Sep 17 00:00:00 2001
From: Bernhard Guillon <Bernhard.Guillon@begu.org>
Date: Tue, 7 Jul 2026 12:32:15 +0200
Subject: Chunk manifest transfer: 1000 files per batch

Avoids bufio.Scanner token limit by sending manifest in batches.
Each batch is a separate JSON line with up to 1000 files.
---
 cmd_linux.go   | 21 +++++++++++++++++----
 cmd_windows.go | 42 ++++++++++++++++++++++++++++++++++--------
 2 files changed, 51 insertions(+), 12 deletions(-)

diff --git a/cmd_linux.go b/cmd_linux.go
index 4fcc891..b80b197 100644
--- a/cmd_linux.go
+++ b/cmd_linux.go
@@ -88,7 +88,7 @@ func handleClient(conn net.Conn, stack *GitignoreStack) {
 	defer conn.Close()
 
 	scanner := bufio.NewScanner(conn)
-	scanner.Buffer(make([]byte, 0, 64*1024*1024), 64*1024*1024) // 64MB buffer for large manifests
+	scanner.Buffer(make([]byte, 0, 1024*1024), 1024*1024) // 1MB buffer per batch
 	remoteManifest := &Manifest{Files: make(map[string]FileMeta)}
 
 	for scanner.Scan() {
@@ -99,11 +99,19 @@ func handleClient(conn net.Conn, stack *GitignoreStack) {
 		}
 
 		switch msg.Type {
-		case "manifest":
-			remoteManifest = ManifestFromJSON(msg.Files)
-			logInfo("manifest: received %d files from host", len(remoteManifest.Files))
+		case "manifest_batch":
+			for k, v := range msg.Files {
+				remoteManifest.Files[k] = FileMeta{
+					Size:    v.Size,
+					Mode:    os.FileMode(v.Mode),
+					ModTime: parseTime(v.ModTime),
+					Symlink: v.Symlink,
+				}
+			}
+			logDebug("manifest: received batch (%d files, total: %d)", len(msg.Files), len(remoteManifest.Files))
 
 		case "manifest_done":
+			logInfo("manifest: received %d files from host", len(remoteManifest.Files))
 			logInfo("manifest: comparing with local state ...")
 			syncFromManifest(remoteManifest, stack)
 			logInfo("sync: done")
@@ -276,3 +284,8 @@ func scanLocalDirectory(root string) *Manifest {
 
 	return manifest
 }
+
+func parseTime(s string) time.Time {
+	t, _ := time.Parse(time.RFC3339Nano, s)
+	return t
+}
diff --git a/cmd_windows.go b/cmd_windows.go
index 7df0a9b..dfe137a 100644
--- a/cmd_windows.go
+++ b/cmd_windows.go
@@ -275,16 +275,42 @@ func scanDirectory(root string, stack *GitignoreStack) *Manifest {
 }
 
 func sendManifest(conn net.Conn, manifest *Manifest) {
-	msg := &ProtocolMessage{
-		Type:  "manifest",
-		Files: manifest.ToJSON(),
+	const batchSize = 1000
+	files := make([]FileMetaJSON, 0, batchSize)
+	paths := make([]string, 0, batchSize)
+
+	for path, meta := range manifest.ToJSON() {
+		paths = append(paths, path)
+		files = append(files, meta)
+
+		if len(files) >= batchSize {
+			msg := &ProtocolMessage{
+				Type:  "manifest_batch",
+				Files: make(map[string]FileMetaJSON),
+			}
+			for i, p := range paths {
+				msg.Files[p] = files[i]
+			}
+			data, _ := EncodeMessage(msg)
+			data = append(data, '\n')
+			conn.Write(data)
+			files = files[:0]
+			paths = paths[:0]
+		}
 	}
-	data, err := EncodeMessage(msg)
-	if err != nil {
-		return
+
+	if len(files) > 0 {
+		msg := &ProtocolMessage{
+			Type:  "manifest_batch",
+			Files: make(map[string]FileMetaJSON),
+		}
+		for i, p := range paths {
+			msg.Files[p] = files[i]
+		}
+		data, _ := EncodeMessage(msg)
+		data = append(data, '\n')
+		conn.Write(data)
 	}
-	data = append(data, '\n')
-	conn.Write(data)
 
 	done := &ProtocolMessage{Type: "manifest_done"}
 	doneData, _ := EncodeMessage(done)
-- 
cgit v1.2.3