aboutsummaryrefslogtreecommitdiffstats
path: root/cmd_linux.go
diff options
context:
space:
mode:
authorBernhard Guillon <Bernhard.Guillon@begu.org>2026-07-07 12:32:15 +0200
committerBernhard Guillon <Bernhard.Guillon@begu.org>2026-07-07 12:32:15 +0200
commit85301b69f478c34e3ee7488f5b04ae4ac5b3ba1e (patch)
tree8ce845908ced04235a140c8cb231dc6cb672bd0a /cmd_linux.go
parent1bcc79fc8d6d91a26a744bcab9fd1b661e0466d2 (diff)
downloadsourcewatch-85301b69f478c34e3ee7488f5b04ae4ac5b3ba1e.tar.gz
sourcewatch-85301b69f478c34e3ee7488f5b04ae4ac5b3ba1e.zip
Chunk manifest transfer: 1000 files per batch
Avoids bufio.Scanner token limit by sending manifest in batches. Each batch is a separate JSON line with up to 1000 files.
Diffstat (limited to 'cmd_linux.go')
-rw-r--r--cmd_linux.go21
1 files changed, 17 insertions, 4 deletions
diff --git a/cmd_linux.go b/cmd_linux.go
index 4fcc891..b80b197 100644
--- a/cmd_linux.go
+++ b/cmd_linux.go
@@ -88,7 +88,7 @@ func handleClient(conn net.Conn, stack *GitignoreStack) {
defer conn.Close()
scanner := bufio.NewScanner(conn)
- scanner.Buffer(make([]byte, 0, 64*1024*1024), 64*1024*1024) // 64MB buffer for large manifests
+ scanner.Buffer(make([]byte, 0, 1024*1024), 1024*1024) // 1MB buffer per batch
remoteManifest := &Manifest{Files: make(map[string]FileMeta)}
for scanner.Scan() {
@@ -99,11 +99,19 @@ func handleClient(conn net.Conn, stack *GitignoreStack) {
}
switch msg.Type {
- case "manifest":
- remoteManifest = ManifestFromJSON(msg.Files)
- logInfo("manifest: received %d files from host", len(remoteManifest.Files))
+ case "manifest_batch":
+ for k, v := range msg.Files {
+ remoteManifest.Files[k] = FileMeta{
+ Size: v.Size,
+ Mode: os.FileMode(v.Mode),
+ ModTime: parseTime(v.ModTime),
+ Symlink: v.Symlink,
+ }
+ }
+ logDebug("manifest: received batch (%d files, total: %d)", len(msg.Files), len(remoteManifest.Files))
case "manifest_done":
+ logInfo("manifest: received %d files from host", len(remoteManifest.Files))
logInfo("manifest: comparing with local state ...")
syncFromManifest(remoteManifest, stack)
logInfo("sync: done")
@@ -276,3 +284,8 @@ func scanLocalDirectory(root string) *Manifest {
return manifest
}
+
+func parseTime(s string) time.Time {
+ t, _ := time.Parse(time.RFC3339Nano, s)
+ return t
+}