diff options
| author | Bernhard Guillon <Bernhard.Guillon@begu.org> | 2026-07-07 12:32:15 +0200 |
|---|---|---|
| committer | Bernhard Guillon <Bernhard.Guillon@begu.org> | 2026-07-07 12:32:15 +0200 |
| commit | 85301b69f478c34e3ee7488f5b04ae4ac5b3ba1e (patch) | |
| tree | 8ce845908ced04235a140c8cb231dc6cb672bd0a /cmd_linux.go | |
| parent | 1bcc79fc8d6d91a26a744bcab9fd1b661e0466d2 (diff) | |
| download | sourcewatch-85301b69f478c34e3ee7488f5b04ae4ac5b3ba1e.tar.gz sourcewatch-85301b69f478c34e3ee7488f5b04ae4ac5b3ba1e.zip | |
Chunk manifest transfer: 1000 files per batch
Avoids bufio.Scanner token limit by sending manifest in batches.
Each batch is a separate JSON line with up to 1000 files.
Diffstat (limited to 'cmd_linux.go')
| -rw-r--r-- | cmd_linux.go | 21 |
1 files changed, 17 insertions, 4 deletions
diff --git a/cmd_linux.go b/cmd_linux.go index 4fcc891..b80b197 100644 --- a/cmd_linux.go +++ b/cmd_linux.go @@ -88,7 +88,7 @@ func handleClient(conn net.Conn, stack *GitignoreStack) { defer conn.Close() scanner := bufio.NewScanner(conn) - scanner.Buffer(make([]byte, 0, 64*1024*1024), 64*1024*1024) // 64MB buffer for large manifests + scanner.Buffer(make([]byte, 0, 1024*1024), 1024*1024) // 1MB buffer per batch remoteManifest := &Manifest{Files: make(map[string]FileMeta)} for scanner.Scan() { @@ -99,11 +99,19 @@ func handleClient(conn net.Conn, stack *GitignoreStack) { } switch msg.Type { - case "manifest": - remoteManifest = ManifestFromJSON(msg.Files) - logInfo("manifest: received %d files from host", len(remoteManifest.Files)) + case "manifest_batch": + for k, v := range msg.Files { + remoteManifest.Files[k] = FileMeta{ + Size: v.Size, + Mode: os.FileMode(v.Mode), + ModTime: parseTime(v.ModTime), + Symlink: v.Symlink, + } + } + logDebug("manifest: received batch (%d files, total: %d)", len(msg.Files), len(remoteManifest.Files)) case "manifest_done": + logInfo("manifest: received %d files from host", len(remoteManifest.Files)) logInfo("manifest: comparing with local state ...") syncFromManifest(remoteManifest, stack) logInfo("sync: done") @@ -276,3 +284,8 @@ func scanLocalDirectory(root string) *Manifest { return manifest } + +func parseTime(s string) time.Time { + t, _ := time.Parse(time.RFC3339Nano, s) + return t +} |
