From 5e69705f34c8ebded73c47e0e8318d1a1536bbae Mon Sep 17 00:00:00 2001 From: azorab Date: Thu, 13 Jun 2019 16:39:40 +0100 Subject: [PATCH 1/2] implement syncdirection flag that enables one-way syncing --- src/main/java/mirror/Mirror.java | 6 +- src/main/java/mirror/MirrorClient.java | 19 +- src/main/java/mirror/MirrorServer.java | 20 +- src/main/java/mirror/MirrorSession.java | 8 +- src/main/java/mirror/SyncDirection.java | 59 ++ src/main/java/mirror/SyncLogic.java | 6 +- src/main/java/mirror/UpdateTreeDiff.java | 9 +- src/main/proto/mirror.proto | 2 + src/test/java/mirror/MirrorSessionTest.java | 4 +- src/test/java/mirror/SyncLogicTest.java | 3 +- .../mirror/UpdateTreeDiffInboundOnlyTest.java | 555 ++++++++++++++++++ .../UpdateTreeDiffOutboundOnlyTest.java | 521 ++++++++++++++++ src/test/java/mirror/UpdateTreeDiffTest.java | 3 +- 13 files changed, 1201 insertions(+), 14 deletions(-) create mode 100644 src/main/java/mirror/SyncDirection.java create mode 100644 src/test/java/mirror/UpdateTreeDiffInboundOnlyTest.java create mode 100644 src/test/java/mirror/UpdateTreeDiffOutboundOnlyTest.java diff --git a/src/main/java/mirror/Mirror.java b/src/main/java/mirror/Mirror.java index 45748a20..3b9e90cb 100644 --- a/src/main/java/mirror/Mirror.java +++ b/src/main/java/mirror/Mirror.java @@ -169,6 +169,9 @@ public static class MirrorClientCommand extends BaseCommand { @Option(name = { "-li", "--use-internal-patterns" }, description = "use hardcoded include/excludes that generally work well for internal repos") public boolean useInternalPatterns; + @Option(name = { "-sd", "--sync-direction"}, description = "direction to sync files, defaults to \"BOTH\", allowed values: \"INBOUND\", \"OUTBOUND\", \"BOTH\"") + public SyncDirection syncDirection = SyncDirection.BOTH; + @Override protected void runIfChecksOkay() { try { @@ -193,7 +196,8 @@ protected void runIfChecksOkay() { new ConnectionDetector.Impl(channelFactory), watcherFactory, new NativeFileAccess(Paths.get(localRoot).toAbsolutePath()), - channelFactory); + channelFactory, + syncDirection); client.startSession(); // dumb way of waiting until they hit control-c CountDownLatch cl = new CountDownLatch(1); diff --git a/src/main/java/mirror/MirrorClient.java b/src/main/java/mirror/MirrorClient.java index c2f4726e..72d76c2c 100644 --- a/src/main/java/mirror/MirrorClient.java +++ b/src/main/java/mirror/MirrorClient.java @@ -34,6 +34,7 @@ public class MirrorClient { private final FileWatcherFactory watcherFactory; private final FileAccess fileAccess; private final ChannelFactory channelFactory; + private final SyncDirection syncDirection; private volatile TaskLogic sessionStarter; private volatile MirrorSession session; @@ -44,12 +45,24 @@ public MirrorClient( FileWatcherFactory watcherFactory, FileAccess fileAccess, ChannelFactory channelFactory) { + this(paths, taskFactory, detector, watcherFactory, fileAccess, channelFactory, SyncDirection.BOTH); + } + + public MirrorClient( + MirrorPaths paths, + TaskFactory taskFactory, + ConnectionDetector detector, + FileWatcherFactory watcherFactory, + FileAccess fileAccess, + ChannelFactory channelFactory, + SyncDirection syncDirection) { this.paths = paths; this.taskFactory = taskFactory; this.detector = detector; this.watcherFactory = watcherFactory; this.fileAccess = fileAccess; this.channelFactory = channelFactory; + this.syncDirection = syncDirection; } /** Connects to the server and starts a sync session. */ @@ -73,7 +86,7 @@ private void startSession(ChannelFactory channelFactory, CountDownLatch onFailur return; } - session = new MirrorSession(taskFactory, paths, fileAccess, watcherFactory); + session = new MirrorSession(taskFactory, paths, fileAccess, watcherFactory, syncDirection); session.addStoppedCallback(channel::shutdownNow); // Automatically re-connect when we're disconnected session.addStoppedCallback(() -> { @@ -98,7 +111,9 @@ private void startSession(ChannelFactory channelFactory, CountDownLatch onFailur .setRemotePath(paths.remoteRoot.toString()) .setClientId(getClientId()) .setVersion(Mirror.getVersion()) - .addAllState(localState); + .addAllState(localState) + .setAllowInbound(syncDirection.getAllowInbound()) + .setAllowOutbound(syncDirection.getAllowOutbound()); paths.addParameters(req); withTimeout(stub).initialSync(req.build(), new StreamObserver() { @Override diff --git a/src/main/java/mirror/MirrorServer.java b/src/main/java/mirror/MirrorServer.java index 8af3329b..4f4723f2 100644 --- a/src/main/java/mirror/MirrorServer.java +++ b/src/main/java/mirror/MirrorServer.java @@ -86,8 +86,16 @@ public synchronized void initialSync(InitialSyncRequest request, StreamObserver< sessions.get(sessionId).stop(); } + //This is the sync direction from the client's point of view. We need to use the complement to construct the session. + SyncDirection syncDirection = getSyncDirection(request); + log.info("Starting new session " + sessionId); - MirrorSession session = new MirrorSession(taskFactory, paths, fileAccessFactory.newFileAccess(paths.root.toAbsolutePath()), watcherFactory); + MirrorSession session = new MirrorSession( + taskFactory, + paths, + fileAccessFactory.newFileAccess(paths.root.toAbsolutePath()), + watcherFactory, + syncDirection.getComplement()); sessions.put(sessionId, session); session.addStoppedCallback(() -> { @@ -201,4 +209,14 @@ private void sendErrorIfClockDriftExists(TimeCheckRequest request, StreamObserve } responseObserver.onCompleted(); } + + private SyncDirection getSyncDirection(InitialSyncRequest request) { + if (request.getAllowOutbound() && !request.getAllowInbound()) { + return SyncDirection.OUTBOUND; + } + if (!request.getAllowOutbound() && request.getAllowInbound()) { + return SyncDirection.INBOUND; + } + return SyncDirection.BOTH; + } } diff --git a/src/main/java/mirror/MirrorSession.java b/src/main/java/mirror/MirrorSession.java index d74f654e..82efb165 100644 --- a/src/main/java/mirror/MirrorSession.java +++ b/src/main/java/mirror/MirrorSession.java @@ -35,18 +35,20 @@ public class MirrorSession { private final FileWatcher fileWatcher; private final UpdateTree tree; private final SyncLogic syncLogic; + private final SyncDirection syncDirection; private volatile SaveToRemote saveToRemote; private volatile OutgoingConnection outgoingChanges; - public MirrorSession(TaskFactory taskFactory, MirrorPaths paths, FileAccess fileAccess, FileWatcherFactory fileWatcherFactory) { + public MirrorSession(TaskFactory taskFactory, MirrorPaths paths, FileAccess fileAccess, FileWatcherFactory fileWatcherFactory, SyncDirection syncDirection) { this.fileAccess = fileAccess; this.fileWatcher = fileWatcherFactory.newWatcher(paths, queues.incomingQueue); this.tree = UpdateTree.newRoot(paths); + this.syncDirection = syncDirection; // Run all our tasks in a pool so they are terminated together taskPool = taskFactory.newTaskPool(); - syncLogic = new SyncLogic(queues, fileAccess, tree); + syncLogic = new SyncLogic(queues, fileAccess, tree, syncDirection); // started in diffAndStartPolling saveToLocal = new SaveToLocal(queues, fileAccess); @@ -79,6 +81,7 @@ public List calcInitialState() throws Exception { // We've drained the initial state, so we can tell FileWatcher to start polling now. // This will start filling up the queue, but not technically start processing/sending // updates to the remote (see #startPolling). + // TODO: We don't need to watch files if we're inbound only - we do need the initial state list though start(fileWatcher); initialUpdates.forEach(u -> tree.addLocal(u)); @@ -93,6 +96,7 @@ public List calcInitialState() throws Exception { seedRemote.add(n.restorePath(n.getLocal())); } }); + // TODO: maybe interrupt the watcher here? return seedRemote; } diff --git a/src/main/java/mirror/SyncDirection.java b/src/main/java/mirror/SyncDirection.java new file mode 100644 index 00000000..42376c82 --- /dev/null +++ b/src/main/java/mirror/SyncDirection.java @@ -0,0 +1,59 @@ +package mirror; + +public enum SyncDirection { + INBOUND { + @Override + public boolean getAllowInbound() { + return true; + } + + @Override + public boolean getAllowOutbound() { + return false; + } + + @Override + public SyncDirection getComplement() { + return OUTBOUND; + } + }, + OUTBOUND { + @Override + public boolean getAllowInbound() { + return false; + } + + @Override + public boolean getAllowOutbound() { + return true; + } + + @Override + public SyncDirection getComplement() { + return INBOUND; + } + }, + BOTH { + @Override + public boolean getAllowInbound() { + return true; + } + + @Override + public boolean getAllowOutbound() { + return true; + } + + @Override + public SyncDirection getComplement() { + return BOTH; + } + }; + + + public abstract boolean getAllowInbound(); + + public abstract boolean getAllowOutbound(); + + public abstract SyncDirection getComplement(); +} diff --git a/src/main/java/mirror/SyncLogic.java b/src/main/java/mirror/SyncLogic.java index 555b9337..9df48245 100644 --- a/src/main/java/mirror/SyncLogic.java +++ b/src/main/java/mirror/SyncLogic.java @@ -40,11 +40,13 @@ public class SyncLogic implements TaskLogic { private final Queues queues; private final FileAccess fileAccess; private final UpdateTree tree; + private final SyncDirection syncDirection; - public SyncLogic(Queues queues, FileAccess fileAccess, UpdateTree tree) { + public SyncLogic(Queues queues, FileAccess fileAccess, UpdateTree tree, SyncDirection syncDirection) { this.queues = queues; this.fileAccess = fileAccess; this.tree = tree; + this.syncDirection = syncDirection; } @Override @@ -95,7 +97,7 @@ private void handleUpdate(Update u) throws InterruptedException { } private void diff() throws InterruptedException { - DiffResults r = new UpdateTreeDiff(tree).diff(); + DiffResults r = new UpdateTreeDiff(tree, syncDirection).diff(); for (Update u : r.saveLocally) { queues.saveToLocal.put(u); } diff --git a/src/main/java/mirror/UpdateTreeDiff.java b/src/main/java/mirror/UpdateTreeDiff.java index bb62751c..f5fc7fcf 100644 --- a/src/main/java/mirror/UpdateTreeDiff.java +++ b/src/main/java/mirror/UpdateTreeDiff.java @@ -49,9 +49,11 @@ public String toString() { } private final UpdateTree tree; + private final SyncDirection syncDirection; - public UpdateTreeDiff(UpdateTree tree) { + public UpdateTreeDiff(UpdateTree tree, SyncDirection syncDirection) { this.tree = tree; + this.syncDirection = syncDirection; } public DiffResults diff() { @@ -65,7 +67,7 @@ private void diff(DiffResults results, Node node) { Update local = node.getLocal(); Update remote = node.getRemote(); - if (node.isLocalNewer()) { + if (node.isLocalNewer() && syncDirection.getAllowOutbound()) { if (!node.shouldIgnore()) { debugIfEnabled(node, "isLocalNewer"); if (local.getDelete() && node.isParentDeleted()) { @@ -75,7 +77,7 @@ private void diff(DiffResults results, Node node) { } } node.setRemote(local); - } else if (node.isRemoteNewer()) { + } else if (node.isRemoteNewer() && syncDirection.getAllowInbound()) { // if we were a directory, and this is now a file, do an explicit delete first if (local != null && !node.isSameType() && !local.getDelete() && !remote.getDelete()) { Update delete = local.toBuilder().setDelete(true).build(); @@ -99,6 +101,7 @@ private void diff(DiffResults results, Node node) { // should rarely/never happen (although it did happen when a bug existed), but // if the remote side sends over data that exactly matches what we already have, // we won't save but, which is fine, but make sure we free it from memory + // This will also occur if the other side sends a message that our direction forbids (which shouldn't happen) node.clearData(); } } diff --git a/src/main/proto/mirror.proto b/src/main/proto/mirror.proto index 41cf9c9c..ad140f24 100644 --- a/src/main/proto/mirror.proto +++ b/src/main/proto/mirror.proto @@ -30,6 +30,8 @@ message InitialSyncRequest { string remotePath = 1; int64 currentTime = 7 [deprecated=true]; string version = 8; + bool allowInbound = 10; + bool allowOutbound = 11; repeated string includes = 3; repeated string excludes = 4; repeated string debugPrefixes = 5; diff --git a/src/test/java/mirror/MirrorSessionTest.java b/src/test/java/mirror/MirrorSessionTest.java index 634efec4..9945f339 100644 --- a/src/test/java/mirror/MirrorSessionTest.java +++ b/src/test/java/mirror/MirrorSessionTest.java @@ -23,6 +23,7 @@ public class MirrorSessionTest { private final FileWatcherFactory fileWatcherFactory = Mockito.mock(FileWatcherFactory.class); private final FileWatcher fileWatcher = Mockito.mock(FileWatcher.class); private final StubTaskFactory taskFactory = new StubTaskFactory(); + private final SyncDirection syncDirection = SyncDirection.BOTH; private MirrorSession session; @Before @@ -33,7 +34,8 @@ public void before() throws Exception { taskFactory, new MirrorPaths(root, null, new PathRules("*.jar"), new PathRules(), false, new ArrayList<>()), fileAccess, - fileWatcherFactory); + fileWatcherFactory, + syncDirection); } @Test diff --git a/src/test/java/mirror/SyncLogicTest.java b/src/test/java/mirror/SyncLogicTest.java index 79ad67b5..c32a0537 100644 --- a/src/test/java/mirror/SyncLogicTest.java +++ b/src/test/java/mirror/SyncLogicTest.java @@ -22,7 +22,8 @@ public class SyncLogicTest { private final StubObserver outgoing = new StubObserver<>(); private final StubFileAccess fileAccess = new StubFileAccess(); private final UpdateTree tree = UpdateTree.newRoot(); - private final SyncLogic l = new SyncLogic(queues, fileAccess, tree); + private final SyncDirection syncDirection = SyncDirection.BOTH; + private final SyncLogic l = new SyncLogic(queues, fileAccess, tree, syncDirection); @Test public void sendLocalChangeToRemote() throws Exception { diff --git a/src/test/java/mirror/UpdateTreeDiffInboundOnlyTest.java b/src/test/java/mirror/UpdateTreeDiffInboundOnlyTest.java new file mode 100644 index 00000000..0323c9e0 --- /dev/null +++ b/src/test/java/mirror/UpdateTreeDiffInboundOnlyTest.java @@ -0,0 +1,555 @@ +package mirror; + +import com.google.protobuf.ByteString; +import mirror.UpdateTree.Node; +import mirror.UpdateTreeDiff.DiffResults; +import org.jooq.lambda.Seq; +import org.junit.Test; + +import static com.google.common.collect.Lists.newArrayList; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.jooq.lambda.Seq.seq; + +public class UpdateTreeDiffInboundOnlyTest { + + private static final ByteString data = ByteString.copyFrom(new byte[] { 1, 2, 3, 4 }); + private UpdateTree tree = UpdateTree.newRoot(); + private DiffResults results = null; + private SyncDirection syncDirection = SyncDirection.INBOUND; + + @Test + public void noSendLocalNewFileToRemote() { + // given a local file that is new + tree.addLocal(Update.newBuilder().setPath("foo.txt").setModTime(2L).build()); + // we don't send it + diff(); + assertNoResults(); + } + + @Test + public void noSendLocalChangedFileToRemote() { + // given a local file that is newer + tree.addLocal(Update.newBuilder().setPath("foo.txt").setModTime(2L).build()); + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(1L).build()); + diff(); + // then we don't send the file to the remote + assertNoResults(); + } + + @Test + public void skipLocalMissingFileThatIsOnRemote() { + // given a remote file that does not exist locally (and we don't have data for it yet) + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(1L).setData(UpdateTree.initialSyncMarker).build()); + diff(); + // then we don't do anything + assertNoResults(); + // and when we do have data, then we will save it + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(1L).setData(data).build()); + diff(); + assertSaveLocally("foo.txt"); + } + + @Test + public void skipLocalStaleFileThatIsOnRemote() { + // given a remote file that is stale locally (and we don't have data for it yet) + tree.addLocal(Update.newBuilder().setPath("foo.txt").setModTime(1L).build()); + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(2L).setData(UpdateTree.initialSyncMarker).build()); + diff(); + // then we don't do anything + assertNoResults(); + // and when we do have data, then we will save it + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(2L).setData(data).build()); + diff(); + assertSaveLocally("foo.txt"); + } + + @Test + public void noSendLocalNewSymlinkToRemote() { + // given a local symlink that is new + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setSymlink("bar").build()); + diff(); + // then we don't send the file to the remote + assertNoResults(); + } + + @Test + public void noSendLocalChangedSymlinkToRemote() { + // given a local symlink that is chagned + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setSymlink("bar2").build()); + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(1L).setSymlink("bar").build()); + diff(); + // then we don't send the file to the remote + assertNoResults(); + } + + @Test + public void noSendLocalNewDirectoryToRemote() { + // given a local directory that is new + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setDirectory(true).build()); + diff(); + // then we don't send the file to the remote + assertNoResults(); + } + + @Test + public void noSendLocalNewNestedFileToRemote() { + // given a local file that is new + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setDirectory(true).build()); + tree.addLocal(Update.newBuilder().setPath("foo/foo.txt").setModTime(2L).build()); + diff(); + // then we don't send the file to the remote + assertNoResults(); + } + + @Test + public void deleteLocalFileThatIsNowADirectory() { + // given a local file + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).build()); + // that is a newer directory on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).setDirectory(true).build()); + diff(); + // then we delete the file + assertSaveLocally("foo", "foo"); + assertThat(results.saveLocally.get(0).getDelete(), is(true)); + // and create the directory + assertThat(results.saveLocally.get(1).getDirectory(), is(true)); + } + + @Test + public void deleteLocalFileThatIsNowASymlink() { + // given a local file + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).build()); + // that is a newer symlink on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).setSymlink("bar").build()); + diff(); + // then we delete the file + assertSaveLocally("foo", "foo"); + assertThat(results.saveLocally.get(0).getDelete(), is(true)); + assertThat(results.saveLocally.get(1).getSymlink(), is("bar")); + } + + @Test + public void leaveLocalFileThatWasADirectory() { + // given a local file + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).build()); + // that is an older directory on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(1L).setDirectory(true).build()); + diff(); + // then we don't send the file to the remote + leave it alone locally + assertNoResults(); + // + assertNoSaveLocally(); + } + + @Test + public void deleteLocalDirectoryThatIsNowAFile() { + // given a local directory + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setDirectory(true).build()); + // that is now a file on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).build()); + diff(); + // then we delete the directory to create the file + assertSaveLocally("foo", "foo"); + assertThat(results.saveLocally.get(0).getDelete(), is(true)); + assertThat(results.saveLocally.get(1).getDelete(), is(false)); + } + + @Test + public void deleteLocalDirectoryThatIsNowASymlink() { + // given a local directory + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setDirectory(true).build()); + tree.addLocal(Update.newBuilder().setPath("foo/bar.txt").setModTime(2L).build()); + // that is now a symlink on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).setSymlink("bar").build()); + diff(); + // then we delete the directory + assertSaveLocally("foo", "foo"); + assertThat(results.saveLocally.get(0).getDelete(), is(true)); + assertThat(results.saveLocally.get(1).getSymlink(), is("bar")); + assertThat(tree.find("foo").getLocal().getSymlink(), is("bar")); + assertThat(tree.find("foo/bar.txt").getLocal().getDelete(), is(true)); + // and when we diff again + diff(); + // then we don't re-delete it + assertNoResults(); + + // client deletes foo/ + // server sends foo/ + // client sees Update(foo, local=true, delete=true, mod=) echo + // --> should see already deleted, do nothing + // client sees Update(foo, mod=X) from server + + // client deletes foo/ + // server sends foo + // client sees Update(foo, mod=X) from server + // client sees Update(foo, local=true, delete=true, mod=) echo + } + + @Test + public void deleteLocalDirectoryThatIsNowASymlinkDuringSync() { + // given a local directory + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setDirectory(true).build()); + tree.addLocal(Update.newBuilder().setPath("foo/bar.txt").setModTime(2L).build()); + // that is now a symlink on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).setSymlink("bar").build()); + // instead of initialDiff + diff(); + assertSaveLocally("foo", "foo"); + // then we delete the directory + assertThat(results.saveLocally.get(0).getDelete(), is(true)); + // and also save the symlink + assertThat(results.saveLocally.get(1).getSymlink(), is("bar")); + // and when we diff again + diff(); + // then we don't re-delete it + assertNoResults(); + } + + @Test + public void leavelLocalDirectoryThatWasAFile() { + // given a local directory + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setDirectory(true).build()); + // that is an older file file on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(1L).build()); + diff(); + // then we send our directory to the remote, and leave it alone locally + assertNoResults(); + } + + @Test + public void deleteLocalSymlinkThatIsNowAFile() { + // given a local symlink + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setSymlink("bar").build()); + // that is now a file on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).build()); + diff(); + // then we delete the symlink to create the file + assertSaveLocally("foo", "foo"); + assertThat(results.saveLocally.get(0).getDelete(), is(true)); + assertThat(results.saveLocally.get(1).getDelete(), is(false)); + // and when we diff again + diff(); + // then we don't re-delete it + assertNoResults(); + } + + @Test + public void deleteLocalSymlinkThatIsNowADirectory() { + // given a local symlink + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setSymlink("bar").build()); + // that is now a directory on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).setDirectory(true).build()); + diff(); + // then we delete the symlink + assertSaveLocally("foo", "foo"); + assertThat(results.saveLocally.get(0).getDelete(), is(true)); + assertThat(results.saveLocally.get(1).getDirectory(), is(true)); + } + + @Test + public void leaveLocalSymlinkThatWasAFile() { + // given a local symlink + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setSymlink("bar").build()); + // that is an older file on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(1L).build()); + diff(); + // then we don't send our symlink to the remote, and leave it alone locally + assertNoResults(); + } + + @Test + public void skipLocalFileIfParentDirectoryHasBeenRemoved() { + // given a local file + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(1L).setDirectory(true).build()); + tree.addLocal(Update.newBuilder().setPath("foo/foo.txt").setModTime(1L).setSymlink("bar").build()); + // but the directory is now a symlink on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(2L).setSymlink("bar").build()); + diff(); + // then we delete our local foo and don't send anything to the remote + assertSaveLocally("foo", "foo"); + assertThat(results.saveLocally.get(0).getDelete(), is(true)); + assertThat(results.saveLocally.get(1).getSymlink(), is("bar")); + assertsendToRemote(); + } + + @Test + public void skipLocalFileThatIsIgnored() { + // given a local file + tree.addLocal(Update.newBuilder().setPath("foo.txt").setModTime(1L).build()); + // that is locally ignored + tree.addLocal(Update.newBuilder().setPath(".gitignore").setModTime(1L).setIgnoreString("*.txt").build()); + diff(); + // then we don't sync the local foo.txt file, but we do sync .gitignore + assertNoResults(); + } + + @Test + public void skipLocalNewFileThatIsNowIgnored() { + // given a local file + tree.addLocal(Update.newBuilder().setPath("foo.txt").setModTime(1L).build()); + // but the remote has a .gitignore in place + tree.addRemote(Update.newBuilder().setPath(".gitignore").setModTime(1L).setIgnoreString("*.txt").build()); + diff(); + // then we don't sync the local file + assertSaveLocally(".gitignore"); + assertsendToRemote(); + } + + @Test + public void skipLocalFileInAnIgnoredDirectory() { + // given a local file + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(1L).setDirectory(true).build()); + tree.addLocal(Update.newBuilder().setPath("foo/foo.txt").setModTime(1L).build()); + tree.addLocal(Update.newBuilder().setPath(".gitignore").setModTime(1L).setIgnoreString("foo/").build()); + // and the .gitignore exists remotely as well + tree.addRemote(Update.newBuilder().setPath(".gitignore").setModTime(1L).setIgnoreString("foo/").build()); + diff(); + // then we don't sync the local file + assertNoResults(); + } + + @Test + public void skipRemoteFileThatIsIgnored() { + // given a remote file + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(1L).build()); + // that is remotely ignored + tree.addRemote(Update.newBuilder().setPath(".gitignore").setModTime(1L).setIgnoreString("*.txt").setData(data).build()); + diff(); + // then we don't sync the local foo.txt file, but we do sync .gitignore + assertSaveLocally(".gitignore"); + } + + @Test + public void includeLocalFileInAnIgnoredDirectoryThatIsExplicitlyIncluded() { + // given a local file + PathRules e = new PathRules(); + PathRules i = new PathRules("*.txt"); + tree = UpdateTree.newRoot(new MirrorPaths(null, null, i, e, false, newArrayList())); + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(1L).setDirectory(true).build()); + tree.addLocal(Update.newBuilder().setPath("foo/foo.txt").setModTime(1L).build()); + tree.addLocal(Update.newBuilder().setPath(".gitignore").setModTime(1L).setIgnoreString("foo/").build()); + // and the .gitignore exists remotely as well + tree.addRemote(Update.newBuilder().setPath(".gitignore").setModTime(1L).setIgnoreString("foo/").build()); + diff(); + // then we do + assertNoResults(); + } + + @Test + public void saveNewRemoteFileLocally() { + // given a remote file that is new + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(2L).setData(data).build()); + diff(); + // then we save the file to locally + assertSaveLocally("foo.txt"); + // assertThat(nodeCapture.getValue().getUpdate().getData(), is(data)); + // and then clear the data from the tree afterwards + Node foo = tree.getChildren().get(0); + assertThat(foo.getName(), is("foo.txt")); + assertThat(foo.getRemote().getData(), is(UpdateTree.initialSyncMarker)); + // and we don't resave it again on the next diff + diff(); + assertNoResults(); + } + + @Test + public void saveNewRemoteDirectoryLocally() { + // given a remote directory that is new + tree.addRemote(Update.newBuilder().setPath("foo").setDirectory(true).setModTime(2L).build()); + diff(); + // then we save the directory to locally + assertSaveLocally("foo"); + // and we don't resave it again on the next diff + diff(); + assertNoResults(); + } + + @Test + public void saveNewRemoteDirectoryAndThenFileLocally() { + // given a remote directory that is new + tree.addRemote(Update.newBuilder().setPath("foo").setDirectory(true).setModTime(2L).build()); + // and it also has a file in it + tree.addRemote(Update.newBuilder().setPath("foo/bar.txt").setData(data).setModTime(2L).build()); + diff(); + // then we save the directory to locally + assertSaveLocally("foo", "foo/bar.txt"); + // and we don't resave it again on the next diff + diff(); + assertNoResults(); + } + + @Test + public void deleteWhenFileDeletedLocally() { + // given a file that exists on both local and remote + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).build()); + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(2L).build()); + // and it is deleted locally + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(3L).setDelete(true).build()); + diff(); + // then we don't send the delete to the remote + assertNoResults(); + // and we don't resend it again on the next diff + diff(); + assertNoResults(); + } + + @Test + public void deleteWhenFileDeletedRemote() { + // given a file that exists on both local and remote + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).build()); + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(2L).build()); + // and it is deleted on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).setDelete(true).build()); + diff(); + // then we delete it locally + assertSaveLocally("foo"); + assertThat(results.saveLocally.get(0).getDelete(), is(true)); + assertThat(results.saveLocally.get(0).getLocal(), is(false)); + // and we don't resend it again on the next diff + diff(); + assertNoResults(); + } + + @Test + public void recreateWhenFileDeletedAndCreatedLocally() { + // given a file that exists on both local and remote + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).build()); + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(2L).build()); + // and it is deleted locally + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(3L).setDelete(true).build()); + diff(); + // then we don't send the delete to the remote + assertNoResults(); + // when it's re-created locally + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(4L).build()); + diff(); + // then we don't send the delete to the remote + assertNoResults(); + } + + @Test + public void deleteMultipleLevelsLocally() { + // given a tree of foo/bar/zaz.txt that exists on both local and remote + tree.addLocal(Update.newBuilder().setPath("foo").setDirectory(true).setModTime(2L).build()); + tree.addLocal(Update.newBuilder().setPath("foo/bar").setDirectory(true).setModTime(2L).build()); + tree.addLocal(Update.newBuilder().setPath("foo/bar/zaz.txt").setModTime(2L).build()); + + tree.addRemote(Update.newBuilder().setPath("foo").setDirectory(true).setModTime(2L).build()); + tree.addRemote(Update.newBuilder().setPath("foo/bar").setDirectory(true).setModTime(2L).build()); + tree.addRemote(Update.newBuilder().setPath("foo/bar/zaz.txt").setModTime(2L).build()); + + // and it is deleted locally (i verified the inotify events are fired child-first + tree.addLocal(Update.newBuilder().setPath("foo/bar/zaz.txt").setModTime(3L).setDelete(true).build()); + tree.addLocal(Update.newBuilder().setPath("foo/bar").setModTime(3L).setDelete(true).build()); + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(3L).setDelete(true).build()); + + // then we don't send anything + diff(); + assertNoResults(); + + // and we don't send it on the next diff + diff(); + assertNoResults(); + assertThat(tree.find("foo").getChildren().size(), is(1)); + assertThat(tree.find("foo/bar").getChildren().size(), is(1)); + } + + @Test + public void deleteMultipleLevelsRemotely() { + // given a tree of foo/bar/zaz.txt that exists on both local and remote + tree.addLocal(Update.newBuilder().setPath("foo").setDirectory(true).setModTime(2L).build()); + tree.addLocal(Update.newBuilder().setPath("foo/bar").setDirectory(true).setModTime(2L).build()); + tree.addLocal(Update.newBuilder().setPath("foo/bar/zaz.txt").setModTime(2L).build()); + + tree.addRemote(Update.newBuilder().setPath("foo").setDirectory(true).setModTime(2L).build()); + tree.addRemote(Update.newBuilder().setPath("foo/bar").setDirectory(true).setModTime(2L).build()); + tree.addRemote(Update.newBuilder().setPath("foo/bar/zaz.txt").setModTime(2L).build()); + + // and it is deleted remotely (per last test, only a delete foo will come across) + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).setDelete(true).build()); + + // then we only need to delete the local directory + diff(); + assertSaveLocally("foo"); + assertThat(results.saveLocally.get(0).getDelete(), is(true)); + + // and we preemptively consider the local children deleted + assertThat(tree.find("foo/bar").getLocal().getDelete(), is(true)); + assertThat(tree.find("foo/bar/zaz.txt").getLocal().getDelete(), is(true)); + + // and we don't resend it again on the next diff + diff(); + assertNoResults(); + assertThat(tree.find("foo").getChildren().size(), is(1)); + assertThat(tree.find("foo/bar").getChildren().size(), is(1)); + + // and when the deletes are echoed by the file system we don't resend the delete + tree.addLocal(Update.newBuilder().setPath("foo/bar/zaz.txt").setDelete(true).build()); + tree.addLocal(Update.newBuilder().setPath("foo/bar").setDelete(true).build()); + // pretend we haven't seen the foo echo delete + diff(); + assertNoResults(); + + // now the foo echo comes by + tree.addLocal(Update.newBuilder().setPath("foo").setDelete(true).build()); + diff(); + assertNoResults(); + } + +// @Test +// public void clearDataOfStaleRemoteFile() { +// // given a remote file that thought it was newer +// tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(2L).setData(data).build()); +// // and a local file that is actually newer +// tree.addLocal(Update.newBuilder().setPath("foo.txt").setModTime(3L).build()); +// diff(); +// // then we ignore the remove change +// assertNoSaveLocally(); +// // and clear it's data from the UpdateTree +// Node foo = tree.getChildren().get(0); +// assertThat(foo.getName(), is("foo.txt")); +// assertThat(foo.getRemote().getData().size(), is(0)); +// } + + @Test + public void clearDataErronouslySentRemoteFile() { + // given a remote file that thought it was newer + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(2L).setData(data).build()); + // and a local file that was already the same + tree.addLocal(Update.newBuilder().setPath("foo.txt").setModTime(2L).build()); + diff(); + // then we ignore the remove change + assertNoSaveLocally(); + // and clear it's data from the UpdateTree + Node foo = tree.getChildren().get(0); + assertThat(foo.getName(), is("foo.txt")); + assertThat(foo.getRemote().getData(), is(UpdateTree.initialSyncMarker)); + } + + private void diff() { + results = new UpdateTreeDiff(tree, syncDirection).diff(); + } + + private void assertNoResults() { + assertSaveLocally(); + assertSendToRemote(); + } + + private void assertNoSaveLocally() { + assertThat(results.saveLocally.size(), is(0)); + } + + private void assertsendToRemote() { + assertThat(results.sendToRemote.size(), is(0)); + } + + private void assertSendToRemote(String... paths) { + assertThat(seq(results.sendToRemote).map(u -> u.getPath()).toList(), is(Seq.of(paths).toList())); + } + + private void assertSaveLocally(String... paths) { + assertThat(seq(results.saveLocally).map(u -> u.getPath()).toList(), is(Seq.of(paths).toList())); + } + +} diff --git a/src/test/java/mirror/UpdateTreeDiffOutboundOnlyTest.java b/src/test/java/mirror/UpdateTreeDiffOutboundOnlyTest.java new file mode 100644 index 00000000..d921be7e --- /dev/null +++ b/src/test/java/mirror/UpdateTreeDiffOutboundOnlyTest.java @@ -0,0 +1,521 @@ +package mirror; + +import com.google.protobuf.ByteString; +import mirror.UpdateTree.Node; +import mirror.UpdateTreeDiff.DiffResults; +import org.jooq.lambda.Seq; +import org.junit.Test; + +import static com.google.common.collect.Lists.newArrayList; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.jooq.lambda.Seq.seq; + +public class UpdateTreeDiffOutboundOnlyTest { + + private static final ByteString data = ByteString.copyFrom(new byte[] { 1, 2, 3, 4 }); + private UpdateTree tree = UpdateTree.newRoot(); + private DiffResults results = null; + private SyncDirection syncDirection = SyncDirection.OUTBOUND; + + @Test + public void sendLocalNewFileToRemote() { + // given a local file that is new + tree.addLocal(Update.newBuilder().setPath("foo.txt").setModTime(2L).build()); + diff(); + // then we send the file to the remote + assertSendToRemote("foo.txt"); + // and then we don't resend it on the next idff + diff(); + assertNoResults(); + } + + @Test + public void sendLocalChangedFileToRemote() { + // given a local file that is newer + tree.addLocal(Update.newBuilder().setPath("foo.txt").setModTime(2L).build()); + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(1L).build()); + diff(); + // then we send the file to the remote + assertSendToRemote("foo.txt"); + assertNoSaveLocally(); + } + + @Test + public void skipLocalMissingFileThatIsOnRemote() { + // given a remote file that does not exist locally (and we don't have data for it yet) + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(1L).setData(UpdateTree.initialSyncMarker).build()); + diff(); + // then we don't do anything + assertNoResults(); + // and when we do have data, then we still don't do anything + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(1L).setData(data).build()); + diff(); + assertNoResults(); + } + + @Test + public void skipLocalStaleFileThatIsOnRemote() { + // given a remote file that is stale locally (and we don't have data for it yet) + tree.addLocal(Update.newBuilder().setPath("foo.txt").setModTime(1L).build()); + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(2L).setData(UpdateTree.initialSyncMarker).build()); + diff(); + // then we don't do anything + assertNoResults(); + // and when we do have data, then we will save it + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(2L).setData(data).build()); + diff(); + assertNoResults(); + } + + @Test + public void sendLocalNewSymlinkToRemote() { + // given a local symlink that is new + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setSymlink("bar").build()); + diff(); + // then we send the file to the remote + assertSendToRemote("foo"); + } + + @Test + public void sendLocalChangedSymlinkToRemote() { + // given a local symlink that is chagned + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setSymlink("bar2").build()); + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(1L).setSymlink("bar").build()); + diff(); + // then we send the file to the remote + assertSendToRemote("foo"); + } + + @Test + public void sendLocalNewDirectoryToRemote() { + // given a local directory that is new + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setDirectory(true).build()); + diff(); + // then we send the file to the remote + assertSendToRemote("foo"); + } + + @Test + public void sendLocalNewNestedFileToRemote() { + // given a local file that is new + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setDirectory(true).build()); + tree.addLocal(Update.newBuilder().setPath("foo/foo.txt").setModTime(2L).build()); + diff(); + // then we send the file to the remote + assertSendToRemote("foo", "foo/foo.txt"); + } + + @Test + public void deleteLocalFileThatIsNowADirectory() { + // given a local file + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).build()); + // that is a newer directory on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).setDirectory(true).build()); + diff(); + // then we don't delete the file + assertNoResults(); + } + + @Test + public void deleteLocalFileThatIsNowASymlink() { + // given a local file + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).build()); + // that is a newer symlink on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).setSymlink("bar").build()); + diff(); + // then we delete the file + assertNoResults(); + } + + @Test + public void leaveLocalFileThatWasADirectory() { + // given a local file + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).build()); + // that is an older directory on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(1L).setDirectory(true).build()); + diff(); + // then we send our file to the remote, and leave it alone locally + assertSendToRemote("foo"); + assertNoSaveLocally(); + } + + @Test + public void deleteLocalDirectoryThatIsNowAFile() { + // given a local directory + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setDirectory(true).build()); + // that is now a file on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).build()); + diff(); + // then we don't delete the directory to create the file + assertNoResults(); + } + +// @Test +// public void deleteLocalDirectoryThatIsNowASymlink() { +// // given a local directory +// tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setDirectory(true).build()); +// tree.addLocal(Update.newBuilder().setPath("foo/bar.txt").setModTime(2L).build()); +// // that is now a symlink on the remote +// tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).setSymlink("bar").build()); +// diff(); +// +// // then we don't delete it +// assertNoResults(); +// +// // client deletes foo/ +// // server sends foo/ +// // client sees Update(foo, local=true, delete=true, mod=) echo +// // --> should see already deleted, do nothing +// // client sees Update(foo, mod=X) from server +// +// // client deletes foo/ +// // server sends foo +// // client sees Update(foo, mod=X) from server +// // client sees Update(foo, local=true, delete=true, mod=) echo +// } + +// @Test +// public void deleteLocalDirectoryThatIsNowASymlinkDuringSync() { +// // given a local directory +// tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setDirectory(true).build()); +// tree.addLocal(Update.newBuilder().setPath("foo/bar.txt").setModTime(2L).build()); +// // that is now a symlink on the remote +// tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).setSymlink("bar").build()); +// // instead of initialDiff +// diff(); +// assertNoResults(); +// } + + @Test + public void leavelLocalDirectoryThatWasAFile() { + // given a local directory + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setDirectory(true).build()); + // that is an older file file on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(1L).build()); + diff(); + // then we send our directory to the remote, and leave it alone locally + assertSendToRemote("foo"); + assertNoSaveLocally(); + } + + @Test + public void deleteLocalSymlinkThatIsNowAFile() { + // given a local symlink + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setSymlink("bar").build()); + // that is now a file on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).build()); + diff(); + // then we don't delete it + assertNoResults(); + } + + @Test + public void deleteLocalSymlinkThatIsNowADirectory() { + // given a local symlink + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setSymlink("bar").build()); + // that is now a directory on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).setDirectory(true).build()); + diff(); + // then we delete the symlink + assertNoResults(); + } + + @Test + public void leaveLocalSymlinkThatWasAFile() { + // given a local symlink + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).setSymlink("bar").build()); + // that is an older file on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(1L).build()); + diff(); + // then we send our symlink to the remote, and leave it alone locally + assertSendToRemote("foo"); + assertNoSaveLocally(); + } + +// @Test +// public void skipLocalFileIfParentDirectoryHasBeenRemoved() { +// // given a local file +// tree.addLocal(Update.newBuilder().setPath("foo").setModTime(1L).setDirectory(true).build()); +// tree.addLocal(Update.newBuilder().setPath("foo/foo.txt").setModTime(1L).setSymlink("bar").build()); +// // but the directory is now a symlink on the remote +// tree.addRemote(Update.newBuilder().setPath("foo").setModTime(2L).setSymlink("bar").build()); +// diff(); +// // then we delete our local foo and don't send anything to the remote +// assertNoResults(); +// } + + @Test + public void skipLocalFileThatIsIgnored() { + // given a local file + tree.addLocal(Update.newBuilder().setPath("foo.txt").setModTime(1L).build()); + // that is locally ignored + tree.addLocal(Update.newBuilder().setPath(".gitignore").setModTime(1L).setIgnoreString("*.txt").build()); + diff(); + // then we don't sync the local foo.txt file, but we do sync .gitignore + assertNoSaveLocally(); + assertSendToRemote(".gitignore"); + } + + @Test + public void skipLocalNewFileThatIsNowIgnored() { + // given a local file + tree.addLocal(Update.newBuilder().setPath("foo.txt").setModTime(1L).build()); + // but the remote has a .gitignore in place + tree.addRemote(Update.newBuilder().setPath(".gitignore").setModTime(1L).setIgnoreString("*.txt").build()); + diff(); + // then we don't sync the local file + assertNoResults(); + } + + @Test + public void skipLocalFileInAnIgnoredDirectory() { + // given a local file + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(1L).setDirectory(true).build()); + tree.addLocal(Update.newBuilder().setPath("foo/foo.txt").setModTime(1L).build()); + tree.addLocal(Update.newBuilder().setPath(".gitignore").setModTime(1L).setIgnoreString("foo/").build()); + // and the .gitignore exists remotely as well + tree.addRemote(Update.newBuilder().setPath(".gitignore").setModTime(1L).setIgnoreString("foo/").build()); + diff(); + // then we don't sync the local file + assertNoResults(); + } + + @Test + public void skipRemoteFileThatIsIgnored() { + // given a remote file + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(1L).build()); + // that is remotely ignored + tree.addRemote(Update.newBuilder().setPath(".gitignore").setModTime(1L).setIgnoreString("*.txt").setData(data).build()); + diff(); + // then we don't sync the local foo.txt file, but we do sync .gitignore + assertNoResults(); + } + + @Test + public void includeLocalFileInAnIgnoredDirectoryThatIsExplicitlyIncluded() { + // given a local file + PathRules e = new PathRules(); + PathRules i = new PathRules("*.txt"); + tree = UpdateTree.newRoot(new MirrorPaths(null, null, i, e, false, newArrayList())); + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(1L).setDirectory(true).build()); + tree.addLocal(Update.newBuilder().setPath("foo/foo.txt").setModTime(1L).build()); + tree.addLocal(Update.newBuilder().setPath(".gitignore").setModTime(1L).setIgnoreString("foo/").build()); + // and the .gitignore exists remotely as well + tree.addRemote(Update.newBuilder().setPath(".gitignore").setModTime(1L).setIgnoreString("foo/").build()); + diff(); + // then we do + assertSendToRemote("foo/foo.txt"); + } + + @Test + public void saveNewRemoteFileLocally() { + // given a remote file that is new + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(2L).setData(data).build()); + diff(); + // then we save the file to locally + assertNoResults(); + // assertThat(nodeCapture.getValue().getUpdate().getData(), is(data)); + // and then clear the data from the tree afterwards + Node foo = tree.getChildren().get(0); + assertThat(foo.getName(), is("foo.txt")); + assertThat(foo.getRemote().getData(), is(UpdateTree.initialSyncMarker)); + } + + @Test + public void saveNewRemoteDirectoryLocally() { + // given a remote directory that is new + tree.addRemote(Update.newBuilder().setPath("foo").setDirectory(true).setModTime(2L).build()); + diff(); + // then we save the directory to locally + assertNoResults(); + } + + @Test + public void saveNewRemoteDirectoryAndThenFileLocally() { + // given a remote directory that is new + tree.addRemote(Update.newBuilder().setPath("foo").setDirectory(true).setModTime(2L).build()); + // and it also has a file in it + tree.addRemote(Update.newBuilder().setPath("foo/bar.txt").setData(data).setModTime(2L).build()); + diff(); + assertNoResults(); + } + + @Test + public void deleteWhenFileDeletedLocally() { + // given a file that exists on both local and remote + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).build()); + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(2L).build()); + // and it is deleted locally + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(3L).setDelete(true).build()); + diff(); + // then we send the delete to the remote + assertSendToRemote("foo"); + assertThat(results.sendToRemote.get(0).getDelete(), is(true)); + assertThat(results.sendToRemote.get(0).getLocal(), is(false)); + // and we don't resend it again on the next diff + diff(); + assertNoResults(); + } + + @Test + public void deleteWhenFileDeletedRemote() { + // given a file that exists on both local and remote + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).build()); + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(2L).build()); + // and it is deleted on the remote + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).setDelete(true).build()); + diff(); + assertNoResults(); + } + + @Test + public void recreateWhenFileDeletedAndCreatedLocally() { + // given a file that exists on both local and remote + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(2L).build()); + tree.addRemote(Update.newBuilder().setPath("foo").setModTime(2L).build()); + // and it is deleted locally + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(3L).setDelete(true).build()); + diff(); + // then we send the delete to the remote + assertSendToRemote("foo"); + assertThat(results.sendToRemote.get(0).getDelete(), is(true)); + assertThat(results.sendToRemote.get(0).getLocal(), is(false)); + // when it's re-created locally + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(4L).build()); + diff(); + // then we send the delete to the remote + assertThat(results.sendToRemote.get(0).getDelete(), is(false)); + assertThat(results.sendToRemote.get(0).getLocal(), is(false)); + // and we don't resend it again on the next diff + diff(); + assertNoResults(); + } + + @Test + public void deleteMultipleLevelsLocally() { + // given a tree of foo/bar/zaz.txt that exists on both local and remote + tree.addLocal(Update.newBuilder().setPath("foo").setDirectory(true).setModTime(2L).build()); + tree.addLocal(Update.newBuilder().setPath("foo/bar").setDirectory(true).setModTime(2L).build()); + tree.addLocal(Update.newBuilder().setPath("foo/bar/zaz.txt").setModTime(2L).build()); + + tree.addRemote(Update.newBuilder().setPath("foo").setDirectory(true).setModTime(2L).build()); + tree.addRemote(Update.newBuilder().setPath("foo/bar").setDirectory(true).setModTime(2L).build()); + tree.addRemote(Update.newBuilder().setPath("foo/bar/zaz.txt").setModTime(2L).build()); + + // and it is deleted locally (i verified the inotify events are fired child-first + tree.addLocal(Update.newBuilder().setPath("foo/bar/zaz.txt").setModTime(3L).setDelete(true).build()); + tree.addLocal(Update.newBuilder().setPath("foo/bar").setModTime(3L).setDelete(true).build()); + tree.addLocal(Update.newBuilder().setPath("foo").setModTime(3L).setDelete(true).build()); + + // then we only need to send the root delete to the remote + diff(); + assertSendToRemote("foo"); + assertThat(results.sendToRemote.get(0).getDelete(), is(true)); + assertThat(results.sendToRemote.get(0).getLocal(), is(false)); + assertThat(results.sendToRemote.get(0).getDirectory(), is(false)); // i guess it's okay for this to be false? + + // and we don't resend it again on the next diff + diff(); + assertNoResults(); + assertThat(tree.find("foo").getChildren().size(), is(1)); + assertThat(tree.find("foo/bar").getChildren().size(), is(1)); + } + +// @Test +// public void deleteMultipleLevelsRemotely() { +// // given a tree of foo/bar/zaz.txt that exists on both local and remote +// tree.addLocal(Update.newBuilder().setPath("foo").setDirectory(true).setModTime(2L).build()); +// tree.addLocal(Update.newBuilder().setPath("foo/bar").setDirectory(true).setModTime(2L).build()); +// tree.addLocal(Update.newBuilder().setPath("foo/bar/zaz.txt").setModTime(2L).build()); +// +// tree.addRemote(Update.newBuilder().setPath("foo").setDirectory(true).setModTime(2L).build()); +// tree.addRemote(Update.newBuilder().setPath("foo/bar").setDirectory(true).setModTime(2L).build()); +// tree.addRemote(Update.newBuilder().setPath("foo/bar/zaz.txt").setModTime(2L).build()); +// +// // and it is deleted remotely (per last test, only a delete foo will come across) +// tree.addRemote(Update.newBuilder().setPath("foo").setModTime(3L).setDelete(true).build()); +// +// // then we only need to delete the local directory +// diff(); +// assertNoResults(); +// +// // and we preemptively consider the local children deleted +// assertThat(tree.find("foo/bar").getLocal().getDelete(), is(true)); +// assertThat(tree.find("foo/bar/zaz.txt").getLocal().getDelete(), is(true)); +// +// // and we don't resend it again on the next diff +// diff(); +// assertNoResults(); +// assertThat(tree.find("foo").getChildren().size(), is(1)); +// assertThat(tree.find("foo/bar").getChildren().size(), is(1)); +// +// // and when the deletes are echoed by the file system we don't resend the delete +// tree.addLocal(Update.newBuilder().setPath("foo/bar/zaz.txt").setDelete(true).build()); +// tree.addLocal(Update.newBuilder().setPath("foo/bar").setDelete(true).build()); +// // pretend we haven't seen the foo echo delete +// diff(); +// assertNoResults(); +// +// // now the foo echo comes by +// tree.addLocal(Update.newBuilder().setPath("foo").setDelete(true).build()); +// diff(); +// assertNoResults(); +// } + + @Test + public void clearDataOfStaleRemoteFile() { + // given a remote file that thought it was newer + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(2L).setData(data).build()); + // and a local file that is actually newer + tree.addLocal(Update.newBuilder().setPath("foo.txt").setModTime(3L).build()); + diff(); + // then we ignore the remove change + assertNoSaveLocally(); + // and clear it's data from the UpdateTree + Node foo = tree.getChildren().get(0); + assertThat(foo.getName(), is("foo.txt")); + assertThat(foo.getRemote().getData().size(), is(0)); + } + + @Test + public void clearDataErronouslySentRemoteFile() { + // given a remote file that thought it was newer + tree.addRemote(Update.newBuilder().setPath("foo.txt").setModTime(2L).setData(data).build()); + // and a local file that was already the same + tree.addLocal(Update.newBuilder().setPath("foo.txt").setModTime(2L).build()); + diff(); + // then we ignore the remove change + assertNoSaveLocally(); + // and clear it's data from the UpdateTree + Node foo = tree.getChildren().get(0); + assertThat(foo.getName(), is("foo.txt")); + assertThat(foo.getRemote().getData(), is(UpdateTree.initialSyncMarker)); + } + + private void diff() { + results = new UpdateTreeDiff(tree, syncDirection).diff(); + } + + private void assertNoResults() { + assertSaveLocally(); + assertSendToRemote(); + } + + private void assertNoSaveLocally() { + assertThat(results.saveLocally.size(), is(0)); + } + + private void assertNoSendToRemote() { + assertThat(results.sendToRemote.size(), is(0)); + } + + private void assertSendToRemote(String... paths) { + assertThat(seq(results.sendToRemote).map(u -> u.getPath()).toList(), is(Seq.of(paths).toList())); + } + + private void assertSaveLocally(String... paths) { + assertThat(seq(results.saveLocally).map(u -> u.getPath()).toList(), is(Seq.of(paths).toList())); + } + +} diff --git a/src/test/java/mirror/UpdateTreeDiffTest.java b/src/test/java/mirror/UpdateTreeDiffTest.java index 4b28dd9d..5cc60a06 100644 --- a/src/test/java/mirror/UpdateTreeDiffTest.java +++ b/src/test/java/mirror/UpdateTreeDiffTest.java @@ -18,6 +18,7 @@ public class UpdateTreeDiffTest { private static final ByteString data = ByteString.copyFrom(new byte[] { 1, 2, 3, 4 }); private UpdateTree tree = UpdateTree.newRoot(); private DiffResults results = null; + private SyncDirection syncDirection = SyncDirection.BOTH; @Test public void sendLocalNewFileToRemote() { @@ -546,7 +547,7 @@ public void clearDataErronouslySentRemoteFile() { } private void diff() { - results = new UpdateTreeDiff(tree).diff(); + results = new UpdateTreeDiff(tree, syncDirection).diff(); } private void assertNoResults() { From f78ac812dced246049dc879bea5a9d97a3b7940f Mon Sep 17 00:00:00 2001 From: azorab Date: Thu, 13 Jun 2019 16:34:28 +0100 Subject: [PATCH 2/2] gradle seems to produce a jar without an '-all' suffix --- Dockerfile | 2 +- mirror | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7ebb96b6..dd1742b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,7 +41,7 @@ RUN install -d -m 777 /usr/local/var/run/watchman WORKDIR "/opt/mirror" COPY --from=mirror-builder /tmp/mirror/mirror ./ -COPY --from=mirror-builder /tmp/mirror/build/libs/mirror-all.jar ./ +COPY --from=mirror-builder /tmp/mirror/build/libs/mirror.jar ./ RUN chmod a+s /usr/sbin/useradd /usr/sbin/groupadd ADD docker/docker-entrypoint.sh docker-entrypoint.sh ENTRYPOINT ["./docker-entrypoint.sh"] diff --git a/mirror b/mirror index d219d360..0dd036b3 100755 --- a/mirror +++ b/mirror @@ -1,8 +1,7 @@ #!/bin/bash -SCRIPT_DIRECTORY="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MAIN=mirror.Mirror -JAR=mirror-all.jar +JAR=mirror.jar OPTS="-Xmx2G -XX:+HeapDumpOnOutOfMemoryError" if [ -e ${SCRIPT_DIRECTORY}/${JAR} ]; then