diff options
| -rw-r--r-- | cmd_linux.go | 131 | ||||
| -rw-r--r-- | cmd_windows.go | 250 | ||||
| -rw-r--r-- | main.go | 55 | ||||
| -rw-r--r-- | sync.go | 11 | ||||
| -rw-r--r-- | types.go | 23 | ||||
| -rw-r--r-- | watcher.go | 142 | ||||
| -rw-r--r-- | watcher_test.go | 32 | ||||
| -rw-r--r-- | watcher_test_helper.go | 142 |
8 files changed, 575 insertions, 211 deletions
diff --git a/cmd_linux.go b/cmd_linux.go new file mode 100644 index 0000000..9e7e237 --- /dev/null +++ b/cmd_linux.go @@ -0,0 +1,131 @@ +//go:build !windows + +package main + +import ( + "bufio" + "context" + "encoding/json" + "flag" + "net" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" +) + +var ( + hostDir = flag.String("host", "/host", "Host source directory (read-only)") + dataDir = flag.String("data", "/data", "Data destination directory") + verbose = flag.Bool("verbose", false, "Enable verbose logging (shorthand for -log-level debug)") + logLevel = flag.String("log-level", "info", "Log level: debug, info, warn, error, none") + dryRun = flag.Bool("dry-run", false, "Preview what would be synced without copying") + socketPath = flag.String("socket", "", "Unix socket to receive events from (Windows watcher)") +) + +func run() { + flag.Parse() + + if *verbose { + *logLevel = "debug" + } + + level, err := ParseLogLevel(*logLevel) + if err != nil { + logErrorf("%v", err) + os.Exit(1) + } + SetLogLevel(level) + + stack := NewGitignoreStack(*hostDir) + + manifest, err := initialSync(*hostDir, *dataDir, stack, *dryRun) + if err != nil { + logErrorf("initial sync failed: %v", err) + os.Exit(1) + } + + if *dryRun { + logInfo("dry-run complete, exiting") + return + } + + if *socketPath == "" { + logInfo("no -socket specified, running in standalone mode (no file watching)") + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + <-ctx.Done() + return + } + + logInfo("watcher: connecting to %s ...", *socketPath) + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + if err := listenSocket(ctx, *socketPath, manifest, stack); err != nil { + logErrorf("socket listener failed: %v", err) + os.Exit(1) + } +} + +func listenSocket(ctx context.Context, socketPath string, manifest *Manifest, stack *GitignoreStack) error { + var conn net.Conn + var err error + + for { + if ctx.Err() != nil { + return ctx.Err() + } + conn, err = net.Dial("unix", socketPath) + if err == nil { + break + } + logDebug("connect failed: %v, retrying...", err) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(1 * time.Second): + } + } + defer conn.Close() + + logInfo("watcher: connected, monitoring for changes ...") + + scanner := bufio.NewScanner(conn) + for scanner.Scan() { + var evt FileEvent + if err := json.Unmarshal(scanner.Bytes(), &evt); err != nil { + logDebug("bad event: %v", err) + continue + } + + logDebug("event: %s %s", evt.Op, evt.Path) + + if isGitDir(evt.Path) { + continue + } + + if stack.IsIgnored(evt.Path) { + continue + } + + destPath := filepath.Join(*dataDir, evt.Path) + srcPath := filepath.Join(*hostDir, evt.Path) + + switch evt.Op { + case "create": + syncCreate(srcPath, destPath, evt.Path, manifest, stack, *dryRun, nil) + case "write": + syncWrite(srcPath, destPath, evt.Path, manifest, *dryRun) + case "remove": + syncRemove(destPath, evt.Path, manifest, *dryRun) + } + } + + if err := scanner.Err(); err != nil { + return err + } + + return nil +} diff --git a/cmd_windows.go b/cmd_windows.go new file mode 100644 index 0000000..b68851a --- /dev/null +++ b/cmd_windows.go @@ -0,0 +1,250 @@ +//go:build windows + +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "net" + "os" + "os/signal" + "path/filepath" + "sync" + "syscall" + "time" + + "github.com/fsnotify/fsnotify" +) + +var ( + socketPath = flag.String("socket", "", "Unix socket path to serve events on (required)") + sourceDir = flag.String("dir", "", "Source directory to watch (required)") +) + +func run() { + flag.Parse() + + if *socketPath == "" || *sourceDir == "" { + fmt.Fprintln(os.Stderr, "Usage: sourcewatch.exe -dir <source> -socket <socket-path>") + os.Exit(1) + } + + level, err := ParseLogLevel(*logLevel) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + SetLogLevel(level) + + logInfo("watcher: starting on %s", *sourceDir) + logInfo("watcher: serving on %s", *socketPath) + + stack := NewGitignoreStack(*sourceDir) + + watcher, err := fsnotify.NewWatcher() + if err != nil { + logErrorf("watcher: %v", err) + os.Exit(1) + } + defer watcher.Close() + + if err := addWatchersRecursively(watcher, *sourceDir, stack); err != nil { + logErrorf("watcher: %v", err) + os.Exit(1) + } + + os.Remove(*socketPath) + + ln, err := net.Listen("unix", *socketPath) + if err != nil { + logErrorf("socket: %v", err) + os.Exit(1) + } + defer func() { + ln.Close() + os.Remove(*socketPath) + }() + + var mu sync.Mutex + debounceTimers := make(map[string]*time.Timer) + debounceOps := make(map[string]fsnotify.Op) + + clients := make(map[net.Conn]bool) + var clientsMu sync.Mutex + + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + clientsMu.Lock() + clients[conn] = true + clientsMu.Unlock() + logDebug("client connected: %s", conn.RemoteAddr()) + + go func(c net.Conn) { + defer func() { + clientsMu.Lock() + delete(clients, c) + clientsMu.Unlock() + c.Close() + }() + buf := make([]byte, 1) + for { + if _, err := c.Read(buf); err != nil { + return + } + } + }(conn) + } + }() + + broadcast := func(evt FileEvent) { + data, err := json.Marshal(evt) + if err != nil { + return + } + data = append(data, '\n') + + clientsMu.Lock() + defer clientsMu.Unlock() + for c := range clients { + if _, err := c.Write(data); err != nil { + go func() { + clientsMu.Lock() + delete(clients, c) + clientsMu.Unlock() + c.Close() + }() + } + } + } + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + go func() { + <-ctx.Done() + ln.Close() + }() + + for { + select { + case <-ctx.Done(): + return + + case event, ok := <-watcher.Events: + if !ok { + return + } + + relPath, err := filepath.Rel(*sourceDir, event.Name) + if err != nil { + continue + } + + if isGitDir(relPath) { + continue + } + + if stack.IsIgnored(relPath) { + continue + } + + logDebug("event: %s %s", event.Op, relPath) + + if event.Op&fsnotify.Create != 0 { + srcPath := filepath.Join(*sourceDir, relPath) + info, err := os.Lstat(srcPath) + if err == nil && info.IsDir() { + if hasGitignore(srcPath) { + stack.Push(srcPath) + } + if err := watcher.Add(srcPath); err != nil { + logErrorf("watch %s: %v", relPath, err) + } + filepath.Walk(srcPath, func(path string, info os.FileInfo, err error) error { + if err != nil || !info.IsDir() || path == srcPath { + return nil + } + watcher.Add(path) + return nil + }) + } + } + + mu.Lock() + if existing, exists := debounceOps[relPath]; exists { + debounceTimers[relPath].Stop() + debounceOps[relPath] = existing | event.Op + } else { + debounceOps[relPath] = event.Op + } + path := relPath + debounceTimers[relPath] = time.AfterFunc(200*time.Millisecond, func() { + mu.Lock() + op := debounceOps[path] + delete(debounceOps, path) + delete(debounceTimers, path) + mu.Unlock() + + evt := FileEvent{Path: path} + switch { + case op&fsnotify.Create != 0: + evt.Op = "create" + case op&fsnotify.Write != 0: + evt.Op = "write" + case op&fsnotify.Remove != 0: + evt.Op = "remove" + case op&fsnotify.Rename != 0: + evt.Op = "remove" + default: + return + } + + broadcast(evt) + }) + mu.Unlock() + + case err, ok := <-watcher.Errors: + if !ok { + return + } + logErrorf("watcher: %v", err) + } + } +} + +func addWatchersRecursively(watcher *fsnotify.Watcher, root string, stack *GitignoreStack) error { + return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + relPath, err := filepath.Rel(root, path) + if err != nil { + return err + } + + if isGitDir(relPath) { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + + if info.IsDir() { + if hasGitignore(path) { + stack.Push(path) + } + if !stack.IsIgnored(relPath) { + return watcher.Add(path) + } + } + + return nil + }) +} @@ -1,58 +1,5 @@ package main -import ( - "context" - "flag" - "log" - "os" - "os/signal" - "syscall" -) - -var ( - hostDir = flag.String("host", "/host", "Host source directory (read-only)") - dataDir = flag.String("data", "/data", "Data destination directory") - verbose = flag.Bool("verbose", false, "Enable verbose logging (shorthand for -log-level debug)") - logLevel = flag.String("log-level", "info", "Log level: debug, info, warn, error, none") - dryRun = flag.Bool("dry-run", false, "Preview what would be synced without copying") -) - func main() { - flag.Parse() - - if *verbose { - *logLevel = "debug" - } - - level, err := ParseLogLevel(*logLevel) - if err != nil { - log.Fatal(err) - } - SetLogLevel(level) - - log.SetFlags(0) - log.SetOutput(nil) - - stack := NewGitignoreStack(*hostDir) - - manifest, err := initialSync(*hostDir, *dataDir, stack, *dryRun) - if err != nil { - logErrorf("initial sync failed: %v", err) - os.Exit(1) - } - - if *dryRun { - logInfo("dry-run complete, exiting") - return - } - - logInfo("watcher: ready, monitoring for changes ...") - - ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer cancel() - - if err := watch(ctx, *hostDir, *dataDir, manifest, stack, *dryRun); err != nil { - logErrorf("watcher failed: %v", err) - os.Exit(1) - } + run() } @@ -8,17 +8,6 @@ import ( "time" ) -type FileMeta struct { - Size int64 - Mode os.FileMode - ModTime time.Time - Symlink string -} - -type Manifest struct { - Files map[string]FileMeta -} - type syncStats struct { files int dirs int diff --git a/types.go b/types.go new file mode 100644 index 0000000..69d7702 --- /dev/null +++ b/types.go @@ -0,0 +1,23 @@ +package main + +import ( + "os" + "time" +) + +type FileEvent struct { + Op string `json:"op"` + Path string `json:"path"` + New string `json:"new_path,omitempty"` +} + +type FileMeta struct { + Size int64 + Mode os.FileMode + ModTime time.Time + Symlink string +} + +type Manifest struct { + Files map[string]FileMeta +} @@ -1,102 +1,11 @@ package main import ( - "context" "os" "path/filepath" - "sync" - "time" - - "github.com/fsnotify/fsnotify" ) -func watch(ctx context.Context, hostDir, dataDir string, manifest *Manifest, stack *GitignoreStack, dryRun bool) error { - watcher, err := fsnotify.NewWatcher() - if err != nil { - return err - } - defer watcher.Close() - - var mu sync.Mutex - debounceTimers := make(map[string]*time.Timer) - debounceOps := make(map[string]fsnotify.Op) - - if err := addWatchersRecursively(watcher, hostDir, stack); err != nil { - return err - } - - for { - select { - case <-ctx.Done(): - return nil - - case event, ok := <-watcher.Events: - if !ok { - return nil - } - - relPath, err := filepath.Rel(hostDir, event.Name) - if err != nil { - continue - } - - if isGitDir(relPath) { - continue - } - - if stack.IsIgnored(relPath) { - continue - } - - logDebug("event: %s %s", event.Op, relPath) - - mu.Lock() - if existing, exists := debounceOps[relPath]; exists { - debounceTimers[relPath].Stop() - debounceOps[relPath] = existing | event.Op - } else { - debounceOps[relPath] = event.Op - } - path := relPath - debounceTimers[relPath] = time.AfterFunc(200*time.Millisecond, func() { - mu.Lock() - op := debounceOps[path] - delete(debounceOps, path) - delete(debounceTimers, path) - mu.Unlock() - handleEvent(op, hostDir, dataDir, path, manifest, stack, dryRun, watcher) - }) - mu.Unlock() - - case err, ok := <-watcher.Errors: - if !ok { - return nil - } - logErrorf("watcher: %v", err) - } - } -} - -func handleEvent(op fsnotify.Op, hostDir, dataDir, relPath string, manifest *Manifest, stack *GitignoreStack, dryRun bool, watcher *fsnotify.Watcher) { - destPath := filepath.Join(dataDir, relPath) - srcPath := filepath.Join(hostDir, relPath) - - switch { - case op&fsnotify.Create != 0: - syncCreate(srcPath, destPath, relPath, manifest, stack, dryRun, watcher) - - case op&fsnotify.Write != 0: - syncWrite(srcPath, destPath, relPath, manifest, dryRun) - - case op&fsnotify.Remove != 0: - syncRemove(destPath, relPath, manifest, dryRun) - - case op&fsnotify.Rename != 0: - syncRemove(destPath, relPath, manifest, dryRun) - } -} - -func syncCreate(srcPath, destPath, relPath string, manifest *Manifest, stack *GitignoreStack, dryRun bool, watcher *fsnotify.Watcher) { +func syncCreate(srcPath, destPath, relPath string, manifest *Manifest, stack *GitignoreStack, dryRun bool, watcher Watcher) { info, err := os.Lstat(srcPath) if err != nil { if os.IsNotExist(err) { @@ -110,16 +19,16 @@ func syncCreate(srcPath, destPath, relPath string, manifest *Manifest, stack *Gi if hasGitignore(srcPath) { stack.Push(srcPath) } - if err := watcher.Add(srcPath); err != nil { - logErrorf("watch %s: %v", relPath, err) - } - filepath.Walk(srcPath, func(path string, info os.FileInfo, err error) error { - if err != nil || !info.IsDir() || path == srcPath { + if watcher != nil { + watcher.Add(srcPath) + filepath.Walk(srcPath, func(path string, info os.FileInfo, err error) error { + if err != nil || !info.IsDir() || path == srcPath { + return nil + } + watcher.Add(path) return nil - } - watcher.Add(path) - return nil - }) + }) + } if dryRun { logInfo("[dry-run] would create dir: %s", relPath) return @@ -238,33 +147,6 @@ func syncRemove(destPath, relPath string, manifest *Manifest, dryRun bool) { } } -func addWatchersRecursively(watcher *fsnotify.Watcher, root string, stack *GitignoreStack) error { - return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - relPath, err := filepath.Rel(root, path) - if err != nil { - return err - } - - if isGitDir(relPath) { - if info.IsDir() { - return filepath.SkipDir - } - return nil - } - - if info.IsDir() { - if hasGitignore(path) { - stack.Push(path) - } - if !stack.IsIgnored(relPath) { - return watcher.Add(path) - } - } - - return nil - }) +type Watcher interface { + Add(path string) error } diff --git a/watcher_test.go b/watcher_test.go index df16930..56f9697 100644 --- a/watcher_test.go +++ b/watcher_test.go @@ -20,7 +20,7 @@ func TestWatcher_Create(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go watch(ctx, host, data, manifest, stack, false) + go watchForTests(ctx, host, data, manifest, stack, false) time.Sleep(100 * time.Millisecond) @@ -42,7 +42,7 @@ func TestWatcher_CreateInSubdir(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go watch(ctx, host, data, manifest, stack, false) + go watchForTests(ctx, host, data, manifest, stack, false) time.Sleep(100 * time.Millisecond) @@ -66,7 +66,7 @@ func TestWatcher_Write(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go watch(ctx, host, data, manifest, stack, false) + go watchForTests(ctx, host, data, manifest, stack, false) time.Sleep(100 * time.Millisecond) @@ -90,7 +90,7 @@ func TestWatcher_Remove(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go watch(ctx, host, data, manifest, stack, false) + go watchForTests(ctx, host, data, manifest, stack, false) time.Sleep(100 * time.Millisecond) @@ -114,7 +114,7 @@ func TestWatcher_Rename(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go watch(ctx, host, data, manifest, stack, false) + go watchForTests(ctx, host, data, manifest, stack, false) time.Sleep(100 * time.Millisecond) @@ -139,7 +139,7 @@ func TestWatcher_IgnoredFileEvents(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go watch(ctx, host, data, manifest, stack, false) + go watchForTests(ctx, host, data, manifest, stack, false) time.Sleep(100 * time.Millisecond) @@ -167,7 +167,7 @@ func TestWatcher_Debouncing(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go watch(ctx, host, data, manifest, stack, false) + go watchForTests(ctx, host, data, manifest, stack, false) time.Sleep(100 * time.Millisecond) @@ -199,7 +199,7 @@ func TestWatcher_CreateDirectory(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go watch(ctx, host, data, manifest, stack, false) + go watchForTests(ctx, host, data, manifest, stack, false) time.Sleep(100 * time.Millisecond) @@ -229,7 +229,7 @@ func TestWatcher_RemoveDirectory(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go watch(ctx, host, data, manifest, stack, false) + go watchForTests(ctx, host, data, manifest, stack, false) time.Sleep(100 * time.Millisecond) @@ -255,7 +255,7 @@ func TestWatcher_VerifyManifestUpdated(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go watch(ctx, host, data, manifest, stack, false) + go watchForTests(ctx, host, data, manifest, stack, false) time.Sleep(100 * time.Millisecond) @@ -282,7 +282,7 @@ func TestWatcher_NestedGitignore_CreateIgnored(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go watch(ctx, host, data, manifest, stack, false) + go watchForTests(ctx, host, data, manifest, stack, false) time.Sleep(100 * time.Millisecond) @@ -310,7 +310,7 @@ func TestWatcher_NestedGitignore_CreateUnignore(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go watch(ctx, host, data, manifest, stack, false) + go watchForTests(ctx, host, data, manifest, stack, false) time.Sleep(100 * time.Millisecond) @@ -332,7 +332,7 @@ func TestWatcher_CreateSymlink(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go watch(ctx, host, data, manifest, stack, false) + go watchForTests(ctx, host, data, manifest, stack, false) time.Sleep(100 * time.Millisecond) @@ -382,7 +382,7 @@ func TestWatcher_UpdateSymlinkTarget(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go watch(ctx, host, data, manifest, stack, false) + go watchForTests(ctx, host, data, manifest, stack, false) time.Sleep(100 * time.Millisecond) @@ -421,7 +421,7 @@ func TestWatcher_RemoveSymlink(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go watch(ctx, host, data, manifest, stack, false) + go watchForTests(ctx, host, data, manifest, stack, false) time.Sleep(100 * time.Millisecond) @@ -447,7 +447,7 @@ func TestWatcher_ModifySymlinkTargetFile(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go watch(ctx, host, data, manifest, stack, false) + go watchForTests(ctx, host, data, manifest, stack, false) time.Sleep(100 * time.Millisecond) diff --git a/watcher_test_helper.go b/watcher_test_helper.go new file mode 100644 index 0000000..519d3f6 --- /dev/null +++ b/watcher_test_helper.go @@ -0,0 +1,142 @@ +//go:build !windows + +package main + +import ( + "context" + "os" + "path/filepath" + "sync" + "time" + + "github.com/fsnotify/fsnotify" +) + +// watchForTests uses inotify directly for integration testing. +// This is only used in tests - production uses socket-based events from Windows watcher. +func watchForTests(ctx context.Context, hostDir, dataDir string, manifest *Manifest, stack *GitignoreStack, dryRun bool) error { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return err + } + defer watcher.Close() + + var mu sync.Mutex + debounceTimers := make(map[string]*time.Timer) + debounceOps := make(map[string]fsnotify.Op) + + if err := addWatchersForTests(watcher, hostDir, stack); err != nil { + return err + } + + for { + select { + case <-ctx.Done(): + return nil + + case event, ok := <-watcher.Events: + if !ok { + return nil + } + + relPath, err := filepath.Rel(hostDir, event.Name) + if err != nil { + continue + } + + if isGitDir(relPath) { + continue + } + + if stack.IsIgnored(relPath) { + continue + } + + if event.Op&fsnotify.Create != 0 { + srcPath := filepath.Join(hostDir, relPath) + info, err := os.Lstat(srcPath) + if err == nil && info.IsDir() { + if hasGitignore(srcPath) { + stack.Push(srcPath) + } + watcher.Add(srcPath) + filepath.Walk(srcPath, func(path string, info os.FileInfo, err error) error { + if err != nil || !info.IsDir() || path == srcPath { + return nil + } + watcher.Add(path) + return nil + }) + } + } + + mu.Lock() + if existing, exists := debounceOps[relPath]; exists { + debounceTimers[relPath].Stop() + debounceOps[relPath] = existing | event.Op + } else { + debounceOps[relPath] = event.Op + } + path := relPath + debounceTimers[relPath] = time.AfterFunc(200*time.Millisecond, func() { + mu.Lock() + op := debounceOps[path] + delete(debounceOps, path) + delete(debounceTimers, path) + mu.Unlock() + + destPath := filepath.Join(dataDir, path) + srcPath := filepath.Join(hostDir, path) + + switch { + case op&fsnotify.Create != 0: + syncCreate(srcPath, destPath, path, manifest, stack, dryRun, watcher) + case op&fsnotify.Write != 0: + syncWrite(srcPath, destPath, path, manifest, dryRun) + case op&fsnotify.Remove != 0: + syncRemove(destPath, path, manifest, dryRun) + case op&fsnotify.Rename != 0: + syncRemove(destPath, path, manifest, dryRun) + } + }) + mu.Unlock() + + case err, ok := <-watcher.Errors: + if !ok { + return nil + } + _ = err + } + } +} + +func addWatchersForTests(watcher *fsnotify.Watcher, root string, stack *GitignoreStack) error { + return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + relPath, err := filepath.Rel(root, path) + if err != nil { + return err + } + + if isGitDir(relPath) { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + + if info.IsDir() { + if hasGitignore(path) { + stack.Push(path) + } + if !stack.IsIgnored(relPath) { + return watcher.Add(path) + } + } + + return nil + }) +} |
