This is an automated email from the git hooks/post-receive script. apo-guest pushed a commit to branch master in repository syncany.
commit 0643c60c85d5bd5c44f2acfa42ecd974ffa4d4ff Author: Markus Koschany <[email protected]> Date: Wed Nov 11 22:51:40 2015 +0100 Imported Upstream version 0.4.7~alpha --- .gitignore | 1 + CHANGELOG.md | 11 + build.gradle | 2 +- gradle/arch/syncany/PKGBUILD | 6 +- gradle/daemon/syncanyd.bat.skel | 8 +- gradle/gradle/application.distribution.gradle | 6 +- gradle/innosetup/innoinstall.sh | 5 +- gradle/osx/syncany.rb | 11 +- .../main/java/org/syncany/cli/PluginCommand.java | 2 +- .../operations/down/DatabaseFileReader.java | 160 +++++++++++++ .../org/syncany/operations/down/DownOperation.java | 126 ++-------- .../down/FileSystemActionReconciliator.java | 15 +- .../down/actions/ChangeFileSystemAction.java | 6 +- .../down/actions/FileCreatingFileSystemAction.java | 11 +- .../down/actions/NewFileSystemAction.java | 5 +- .../syncany/operations/init/ConnectOperation.java | 2 +- .../restore/RestoreFileSystemAction.java | 5 +- .../operations/restore/RestoreOperation.java | 6 +- .../org/syncany/operations/up/UpOperation.java | 256 +++++++++++---------- .../plugins/local/LocalTransferManager.java | 2 +- .../TransactionAwareFeatureTransferManager.java | 11 +- .../java/org/syncany/tests/ScenarioTestSuite.java | 4 + .../operations/FileSystemActionComparatorTest.java | 2 +- .../Issue520NoResumeOnCorruptXmlScenarioTest.java | 77 +++++++ 24 files changed, 474 insertions(+), 266 deletions(-) diff --git a/.gitignore b/.gitignore index 7f49107..857f604 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ gradle/arch/syncany/pkg gradle/arch/syncany/src gradle/arch/syncany/syncany*.tar* *.orig +.recommenders diff --git a/CHANGELOG.md b/CHANGELOG.md index 4150ba8..69f76cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,17 @@ Change Log ========== +### Syncany 0.4.7-alpha (Date: 7 Nov 2015) +- Developer/alpha release (**We are now nearing the beta phase. Stay tuned!**) +- Bugfixes and other things: + + Refactoring and simplification of UpOperation + + Refactoring DownOperation (better memory management) + + Refactoring of error handling (don't throw 'Exception') + + Fix GUI crashes in 'Add folder' wizard #497 + + Fix OSX daemon start/stop to new style #281/#530 + + Fix Windows spaces in path issue #522/#529 + + Fix not resuming transactions if transaction files are corrupt #520 + ### Syncany 0.4.6-alpha (Date: 11 July 2015) - Developer/alpha release (**We are now nearing the beta phase. Stay tuned!**) - Features and significant changes: diff --git a/build.gradle b/build.gradle index b7b0a02..0b79219 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ apply from: 'gradle/gradle/helpers.gradle' // Global Settings ///////////////////////////////////////////////////////////// project.ext { - applicationVersion = "0.4.6-alpha" + applicationVersion = "0.4.7-alpha" applicationVersionDebian = "1"; applicationRelease = isApplicationRelease() diff --git a/gradle/arch/syncany/PKGBUILD b/gradle/arch/syncany/PKGBUILD index f8ae744..02af931 100644 --- a/gradle/arch/syncany/PKGBUILD +++ b/gradle/arch/syncany/PKGBUILD @@ -1,7 +1,7 @@ # Maintainer: Pim Otte <otte dot pim at gmail dot com> pkgname=syncany -pkgver=0.4.5_alpha -_realver=0.4.5-alpha +pkgver=0.4.6_alpha +_realver=0.4.6-alpha pkgrel=1 pkgdesc="Cloud storage and filesharing application with a focus on security and abstraction of storage." arch=(any) @@ -10,7 +10,7 @@ license=('GPL3') depends=('java-runtime>=7' 'bash-completion') source=("http://syncany.org/dist/$pkgname-${_realver}.tar.gz"; ) -sha256sums=('7e8fbad0d9bf90369ad21fd1b3d54c7025738f2da9b1a364273f16cb76d59069') +sha256sums=('9aab83cc336b898a48dde5e7799d703ef569255e5c6f24a7182a6983c0846bc8') package() { install -Dm644 "$srcdir/$pkgname-${_realver}/bash/syncany.bash-completion" "${pkgdir}/etc/bash_completion.d/syncany" diff --git a/gradle/daemon/syncanyd.bat.skel b/gradle/daemon/syncanyd.bat.skel index f972758..a3ef48d 100644 --- a/gradle/daemon/syncanyd.bat.skel +++ b/gradle/daemon/syncanyd.bat.skel @@ -1,10 +1,10 @@ rem Embedded Daemon script begins ################################SYNCANY_INCL_3# set APP_NAME=syncanyd -set APP_USER_DIR=%AppData%\Syncany -set APP_DAEMON_CONTROL=%APP_USER_DIR%\daemon.ctrl -set APP_DAEMON_PIDFILE=%APP_USER_DIR%\daemon.pid -set APP_DAEMON_PIDFILE_TMP=%APP_USER_DIR%\daemon.pid.tmp +set APP_USER_DIR="%AppData%\Syncany" +set APP_DAEMON_CONTROL="%APP_USER_DIR%\daemon.ctrl" +set APP_DAEMON_PIDFILE="%APP_USER_DIR%\daemon.pid" +set APP_DAEMON_PIDFILE_TMP="%APP_USER_DIR%\daemon.pid.tmp" if not exist "%APP_USER_DIR%" mkdir "%APP_USER_DIR%" diff --git a/gradle/gradle/application.distribution.gradle b/gradle/gradle/application.distribution.gradle index 3d47eeb..ea70a91 100644 --- a/gradle/gradle/application.distribution.gradle +++ b/gradle/gradle/application.distribution.gradle @@ -22,7 +22,7 @@ startScripts { // - Read max memory (-Xmx) from userconfig.xml String winMaxMemoryCommands = "@rem Read max memory from userconfig.xml #SYNCANY_INCL_1#\r\n" - winMaxMemoryCommands += "set APP_USERCONFIG_FILE=%AppData%\\\\Syncany\\\\userconfig.xml\r\n"; + winMaxMemoryCommands += "set APP_USERCONFIG_FILE=\"%AppData%\\\\Syncany\\\\userconfig.xml\"\r\n"; winMaxMemoryCommands += "\r\n"; winMaxMemoryCommands += "if exist \"%APP_USERCONFIG_FILE%\" (\r\n"; winMaxMemoryCommands += " if \"%OS%\"==\"Windows_NT\" setlocal ENABLEDELAYEDEXPANSION\r\n"; @@ -42,7 +42,7 @@ startScripts { // - Post Java process commands: Delayed plugin JAR file deletion (Windows only) String winPurgeFileDeletionCommands = "@rem Delete plugin JARs #SYNCANY_INCL_2#\r\n" - winPurgeFileDeletionCommands += "SET PURGEFILE=%AppData%\\\\Syncany\\\\purgefile\r\n"; + winPurgeFileDeletionCommands += "SET PURGEFILE=\"%AppData%\\\\Syncany\\\\purgefile\"\r\n"; winPurgeFileDeletionCommands += "if exist %PURGEFILE% (\r\n"; winPurgeFileDeletionCommands += " @for /f %%b in (%PURGEFILE%) do del /q \"%%b\" 2>NUL\r\n"; winPurgeFileDeletionCommands += " del /q %PURGEFILE% 2>NUL\r\n"; @@ -52,7 +52,7 @@ startScripts { // - Post Java process commands: Delayed plugin JAR file install (Windows only) String winUpdateFileCommands = "@rem Reinstall plugins after update removal #SYNCANY_INCL_4#\r\n" - winUpdateFileCommands += "SET UPDATEFILE=%AppData%\\\\Syncany\\\\updatefile\r\n"; + winUpdateFileCommands += "SET UPDATEFILE=\"%AppData%\\\\Syncany\\\\updatefile\"\r\n"; winUpdateFileCommands += "if exist %UPDATEFILE% (\r\n"; winUpdateFileCommands += " @for /f %%b in (%UPDATEFILE%) do (\r\n <nul set /p =Updating %%b... \r\n \"%JAVA_EXE%\" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %SYNCANY_OPTS% -classpath \"%CLASSPATH%\" org.syncany.Syncany plugin install %%b -m 2>NUL)\r\n"; winUpdateFileCommands += " del /q %UPDATEFILE% 2>NUL\r\n"; diff --git a/gradle/innosetup/innoinstall.sh b/gradle/innosetup/innoinstall.sh index 44f4a8d..90a960f 100755 --- a/gradle/innosetup/innoinstall.sh +++ b/gradle/innosetup/innoinstall.sh @@ -4,11 +4,8 @@ rm -rf /tmp/inno mkdir /tmp/inno cd /tmp/inno -wget -O is.exe http://www.jrsoftware.org/download.php/is.exe +wget -O is.exe http://files.jrsoftware.org/is/5/isetup-5.5.5.exe innoextract is.exe mkdir -p ~/".wine/drive_c/inno" cp -a app/* ~/".wine/drive_c/inno" -#mkdir ~/".wine/drive_c/Program Files (x86)/Inno Setup 5" -#cp -a app/* ~/".wine/drive_c/Program Files (x86)/Inno Setup 5" - diff --git a/gradle/osx/syncany.rb b/gradle/osx/syncany.rb index 15a8fb4..e5b2952 100644 --- a/gradle/osx/syncany.rb +++ b/gradle/osx/syncany.rb @@ -2,9 +2,9 @@ require "formula" class Syncany < Formula homepage "https://www.syncany.org"; - url "https://codeload.github.com/syncany/syncany/tar.gz/v0.4.5-alpha"; - sha256 "957537a7177a5234e794d871043e67ac8e06ff3ff1ee059e599dfd477c5bb8e5" - version "0.4.5-alpha" + url "https://codeload.github.com/syncany/syncany/tar.gz/v0.4.6-alpha"; + sha256 "1dfd92e7618297eae6ee0d8acb7103f55c8d18500da409d032a5941147fbcd85" + version "0.4.6-alpha" head "https://github.com/syncany/syncany.git";, :branch => "develop" depends_on :java => "1.7+" @@ -26,7 +26,7 @@ class Syncany < Formula bin.install_symlink Dir["#{libexec}/bin/sy"] end - plist_options :manual => "sy --daemon" + plist_options :manual => "sy daemon start" def plist; <<-EOS.undent <?xml version="1.0" encoding="UTF-8"?> @@ -38,7 +38,8 @@ class Syncany < Formula <key>ProgramArguments</key> <array> <string>#{prefix}/bin/sy</string> - <string>--daemon</string> + <string>daemon</string> + <string>start</string> </array> <key>ProcessType</key> <string>Background</string> diff --git a/syncany-cli/src/main/java/org/syncany/cli/PluginCommand.java b/syncany-cli/src/main/java/org/syncany/cli/PluginCommand.java index 1972c94..86c4d55 100644 --- a/syncany-cli/src/main/java/org/syncany/cli/PluginCommand.java +++ b/syncany-cli/src/main/java/org/syncany/cli/PluginCommand.java @@ -225,7 +225,7 @@ public class PluginCommand extends Command { if (thirdPartyCount > 0) { String pluginPlugins = (thirdPartyCount == 1) ? "plugin" : "plugins"; - out.printf("\nThird party plugins:\nPlease note that the Syncany Team does not take review or maintain the third-party %s\nlisted above. Please report issues to the corresponding plugin site.\n", pluginPlugins); + out.printf("\nThird party plugins:\nPlease note that the Syncany Team does not review or maintain the third-party %s\nlisted above. Please report issues to the corresponding plugin site.\n", pluginPlugins); } } else { diff --git a/syncany-lib/src/main/java/org/syncany/operations/down/DatabaseFileReader.java b/syncany-lib/src/main/java/org/syncany/operations/down/DatabaseFileReader.java new file mode 100644 index 0000000..8d5363c --- /dev/null +++ b/syncany-lib/src/main/java/org/syncany/operations/down/DatabaseFileReader.java @@ -0,0 +1,160 @@ +/* + * Syncany, www.syncany.org + * Copyright (C) 2011-2015 Philipp C. Heckel <[email protected]> + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.syncany.operations.down; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.syncany.database.DatabaseVersion; +import org.syncany.database.DatabaseVersionHeader; +import org.syncany.database.MemoryDatabase; +import org.syncany.database.VectorClock; +import org.syncany.database.dao.DatabaseXmlSerializer; +import org.syncany.database.dao.DatabaseXmlSerializer.DatabaseReadType; + +/** + * The DatabaseFileReader provides a way to read a series of database files + * in a memory-efficient way, by converting them to a series of MemoryDatabases, + * none of which are too large. + * + * @author Pim Otte + */ +public class DatabaseFileReader implements Iterator<MemoryDatabase> { + private static final int MAX_FILES = 9999; + + private DatabaseXmlSerializer databaseSerializer; + private List<DatabaseVersionHeader> winnersApplyBranchList; + private Map<DatabaseVersionHeader, File> databaseVersionLocations; + private int branchIndex = 0; + + public DatabaseFileReader(DatabaseXmlSerializer databaseSerializer, DatabaseBranch winnersApplyBranch, + Map<DatabaseVersionHeader, File> databaseVersionLocations) { + + this.winnersApplyBranchList = winnersApplyBranch.getAll(); + this.databaseVersionLocations = databaseVersionLocations; + this.databaseSerializer = databaseSerializer; + } + + public boolean hasNext() { + return branchIndex < winnersApplyBranchList.size(); + } + + /** + * Loads the winner's database branch into the memory in a {@link MemoryDatabase} object, by using + * the already downloaded list of remote database files. + * + * <p>Because database files can contain multiple {@link DatabaseVersion}s per client, a range for which + * to load the database versions must be determined. + * + * <p><b>Example 1:</b><br /> + * <pre> + * db-A-0001 (A1) Already known Not loaded + * db-A-0005 (A2) Already known Not loaded + * (A3) Already known Not loaded + * (A4) Part of winner's branch Loaded + * (A5) Purge database version Ignored (only DEFAULT) + * db-B-0001 (A5,B1) Part of winner's branch Loaded + * db-A-0006 (A6,B1) Part of winner's branch Loaded + * </pre> + * + * <p>In example 1, only (A4)-(A5) must be loaded from db-A-0005, and not all four database versions. + * + * <p><b>Other example:</b><br /> + * <pre> + * db-A-0005 (A1) Part of winner's branch Loaded + * db-A-0005 (A2) Part of winner's branch Loaded + * db-B-0001 (A2,B1) Part of winner's branch Loaded + * db-A-0005 (A3,B1) Part of winner's branch Loaded + * db-A-0005 (A4,B1) Part of winner's branch Loaded + * db-A-0005 (A5,B1) Purge database version Ignored (only DEFAULT) + * </pre> + * + * <p>In example 2, (A1)-(A5,B1) [except (A2,B1)] are contained in db-A-0005 (after merging!), so + * db-A-0005 must be processed twice; each time loading separate parts of the file. In this case: + * First load (A1)-(A2) from db-A-0005, then load (A2,B1) from db-B-0001, then load (A3,B1)-(A4,B1) + * from db-A-0005, and ignore (A5,B1). + * @param databaseFileList + * @param ignoredMostRecentPurgeVersions + * + * @return Returns a loaded memory database containing all metadata from the winner's branch + */ + @Override + public MemoryDatabase next() { + MemoryDatabase winnerBranchDatabase = new MemoryDatabase(); + String rangeClientName = null; + VectorClock rangeVersionFrom = null; + VectorClock rangeVersionTo = null; + + while (branchIndex < winnersApplyBranchList.size() && winnerBranchDatabase.getFileHistories().size() < MAX_FILES) { + DatabaseVersionHeader currentDatabaseVersionHeader = winnersApplyBranchList.get(branchIndex); + DatabaseVersionHeader nextDatabaseVersionHeader = (branchIndex + 1 < winnersApplyBranchList.size()) ? winnersApplyBranchList + .get(branchIndex + 1) : null; + + // First of range for this client + if (rangeClientName == null) { + rangeClientName = currentDatabaseVersionHeader.getClient(); + rangeVersionFrom = currentDatabaseVersionHeader.getVectorClock(); + rangeVersionTo = currentDatabaseVersionHeader.getVectorClock(); + } + + // Still in range for this client + else { + rangeVersionTo = currentDatabaseVersionHeader.getVectorClock(); + } + + // Now load this stuff from the database file (or not) + // - If the database file exists, load the range and reset it + // - If not, only force a load if this is the range end + + File databaseVersionFile = databaseVersionLocations.get(currentDatabaseVersionHeader); + + if (databaseVersionFile == null) { + throw new RuntimeException("Could not find file corresponding to " + currentDatabaseVersionHeader + + ", while it is in the winners branch."); + } + + boolean lastDatabaseVersionHeader = nextDatabaseVersionHeader == null; + boolean nextDatabaseVersionInSameFile = lastDatabaseVersionHeader + || databaseVersionFile.equals(databaseVersionLocations.get(nextDatabaseVersionHeader)); + boolean rangeEnds = lastDatabaseVersionHeader || !nextDatabaseVersionInSameFile; + + if (rangeEnds) { + try { + databaseSerializer.load(winnerBranchDatabase, databaseVersionFile, rangeVersionFrom, rangeVersionTo, DatabaseReadType.FULL); + } + catch (IOException e) { + throw new RuntimeException(e.getMessage(), e); + } + rangeClientName = null; + } + branchIndex++; + } + + return winnerBranchDatabase; + } + + @Override + public void remove() { + throw new UnsupportedOperationException("Removing a databaseversion is not supported"); + + } + +} diff --git a/syncany-lib/src/main/java/org/syncany/operations/down/DownOperation.java b/syncany-lib/src/main/java/org/syncany/operations/down/DownOperation.java index 2735aac..4849eca 100644 --- a/syncany-lib/src/main/java/org/syncany/operations/down/DownOperation.java +++ b/syncany-lib/src/main/java/org/syncany/operations/down/DownOperation.java @@ -457,119 +457,43 @@ public class DownOperation extends AbstractTransferOperation { } else { logger.log(Level.INFO, "Loading winners database (DEFAULT) ..."); - MemoryDatabase winnersDatabase = readWinnersDatabase(winnersApplyBranch, databaseVersionLocations); + DatabaseFileReader databaseFileReader = new DatabaseFileReader(databaseSerializer, winnersApplyBranch, databaseVersionLocations); - if (options.isApplyChanges()) { - new ApplyChangesOperation(config, localDatabase, transferManager, winnersDatabase, result, cleanupOccurred, - preDeleteFileHistoriesWithLastVersion).execute(); - } + boolean noDatabaseVersions = !databaseFileReader.hasNext(); + + if (noDatabaseVersions) { + applyChangesAndPersistDatabase(new MemoryDatabase(), cleanupOccurred, preDeleteFileHistoriesWithLastVersion); + } else { - logger.log(Level.INFO, "Doing nothing on the file system, because --no-apply switched on"); + while (databaseFileReader.hasNext()) { + MemoryDatabase winnersDatabase = databaseFileReader.next(); + applyChangesAndPersistDatabase(winnersDatabase, cleanupOccurred, preDeleteFileHistoriesWithLastVersion); + } } - persistDatabaseVersions(winnersApplyBranch, winnersDatabase); - result.setResultCode(DownResultCode.OK_WITH_REMOTE_CHANGES); } } - /** - * Loads the winner's database branch into the memory in a {@link MemoryDatabase} object, by using - * the already downloaded list of remote database files. - * - * <p>Because database files can contain multiple {@link DatabaseVersion}s per client, a range for which - * to load the database versions must be determined. - * - * <p><b>Example 1:</b><br /> - * <pre> - * db-A-0001 (A1) Already known Not loaded - * db-A-0005 (A2) Already known Not loaded - * (A3) Already known Not loaded - * (A4) Part of winner's branch Loaded - * (A5) Purge database version Ignored (only DEFAULT) - * db-B-0001 (A5,B1) Part of winner's branch Loaded - * db-A-0006 (A6,B1) Part of winner's branch Loaded - * </pre> - * - * <p>In example 1, only (A4)-(A5) must be loaded from db-A-0005, and not all four database versions. - * - * <p><b>Other example:</b><br /> - * <pre> - * db-A-0005 (A1) Part of winner's branch Loaded - * db-A-0005 (A2) Part of winner's branch Loaded - * db-B-0001 (A2,B1) Part of winner's branch Loaded - * db-A-0005 (A3,B1) Part of winner's branch Loaded - * db-A-0005 (A4,B1) Part of winner's branch Loaded - * db-A-0005 (A5,B1) Purge database version Ignored (only DEFAULT) - * </pre> - * - * <p>In example 2, (A1)-(A5,B1) [except (A2,B1)] are contained in db-A-0005 (after merging!), so - * db-A-0005 must be processed twice; each time loading separate parts of the file. In this case: - * First load (A1)-(A2) from db-A-0005, then load (A2,B1) from db-B-0001, then load (A3,B1)-(A4,B1) - * from db-A-0005, and ignore (A5,B1). - * @param databaseFileList - * @param ignoredMostRecentPurgeVersions - * - * @return Returns a loaded memory database containing all metadata from the winner's branch - */ - private MemoryDatabase readWinnersDatabase(DatabaseBranch winnersApplyBranch, Map<DatabaseVersionHeader, File> databaseVersionLocations) - throws IOException, StorageException { - - MemoryDatabase winnerBranchDatabase = new MemoryDatabase(); - - List<DatabaseVersionHeader> winnersApplyBranchList = winnersApplyBranch.getAll(); - - String rangeClientName = null; - VectorClock rangeVersionFrom = null; - VectorClock rangeVersionTo = null; - - for (int i = 0; i < winnersApplyBranchList.size(); i++) { - DatabaseVersionHeader currentDatabaseVersionHeader = winnersApplyBranchList.get(i); - DatabaseVersionHeader nextDatabaseVersionHeader = (i + 1 < winnersApplyBranchList.size()) ? winnersApplyBranchList.get(i + 1) : null; - - // First of range for this client - if (rangeClientName == null) { - rangeClientName = currentDatabaseVersionHeader.getClient(); - rangeVersionFrom = currentDatabaseVersionHeader.getVectorClock(); - rangeVersionTo = currentDatabaseVersionHeader.getVectorClock(); - } - - // Still in range for this client - else { - rangeVersionTo = currentDatabaseVersionHeader.getVectorClock(); - } - - // Now load this stuff from the database file (or not) - // - If the database file exists, load the range and reset it - // - If not, only force a load if this is the range end - - File databaseVersionFile = databaseVersionLocations.get(currentDatabaseVersionHeader); - - if (databaseVersionFile == null) { - throw new StorageException("Could not find file corresponding to " + currentDatabaseVersionHeader - + ", while it is in the winners branch."); - } - - boolean lastDatabaseVersionHeader = nextDatabaseVersionHeader == null; - boolean nextDatabaseVersionInSameFile = lastDatabaseVersionHeader - || databaseVersionFile.equals(databaseVersionLocations.get(nextDatabaseVersionHeader)); - boolean rangeEnds = lastDatabaseVersionHeader || !nextDatabaseVersionInSameFile; - - if (rangeEnds) { - databaseSerializer.load(winnerBranchDatabase, databaseVersionFile, rangeVersionFrom, rangeVersionTo, DatabaseReadType.FULL); - rangeClientName = null; - } + private void applyChangesAndPersistDatabase(MemoryDatabase winnersDatabase, boolean cleanupOccurred, + List<PartialFileHistory> preDeleteFileHistoriesWithLastVersion) throws Exception { + + if (options.isApplyChanges()) { + new ApplyChangesOperation(config, localDatabase, transferManager, winnersDatabase, result, cleanupOccurred, + preDeleteFileHistoriesWithLastVersion).execute(); + } + else { + logger.log(Level.INFO, "Doing nothing on the file system, because --no-apply switched on"); } - if (logger.isLoggable(Level.FINE)) { - logger.log(Level.FINE, "Winner Database Branch:"); - - for (DatabaseVersion dbv : winnerBranchDatabase.getDatabaseVersions()) { - logger.log(Level.FINE, "- " + dbv.getHeader()); - } + // We only persist the versions that we have already applied. + DatabaseBranch currentApplyBranch = new DatabaseBranch(); + for (DatabaseVersion databaseVersion : winnersDatabase.getDatabaseVersions()) { + currentApplyBranch.add(databaseVersion.getHeader()); } - return winnerBranchDatabase; + persistDatabaseVersions(currentApplyBranch, winnersDatabase); + localDatabase.commit(); } /** diff --git a/syncany-lib/src/main/java/org/syncany/operations/down/FileSystemActionReconciliator.java b/syncany-lib/src/main/java/org/syncany/operations/down/FileSystemActionReconciliator.java index c01d2ef..093dc86 100644 --- a/syncany-lib/src/main/java/org/syncany/operations/down/FileSystemActionReconciliator.java +++ b/syncany-lib/src/main/java/org/syncany/operations/down/FileSystemActionReconciliator.java @@ -36,6 +36,7 @@ import org.syncany.database.MemoryDatabase; import org.syncany.database.PartialFileHistory; import org.syncany.database.PartialFileHistory.FileHistoryId; import org.syncany.database.SqlDatabase; +import org.syncany.operations.Assembler; import org.syncany.operations.ChangeSet; import org.syncany.operations.down.actions.ChangeFileSystemAction; import org.syncany.operations.down.actions.DeleteFileSystemAction; @@ -132,6 +133,7 @@ public class FileSystemActionReconciliator { private ChangeSet changeSet; private SqlDatabase localDatabase; private FileVersionComparator fileVersionComparator; + private Assembler assembler; public FileSystemActionReconciliator(Config config, ChangeSet changeSet) { this.config = config; @@ -147,6 +149,7 @@ public class FileSystemActionReconciliator { public List<FileSystemAction> determineFileSystemActions(MemoryDatabase winnersDatabase, boolean cleanupOccurred, List<PartialFileHistory> localFileHistoriesWithLastVersion) throws Exception { + this.assembler = new Assembler(config, localDatabase, winnersDatabase); List<FileSystemAction> fileSystemActions = new ArrayList<FileSystemAction>(); @@ -230,7 +233,7 @@ public class FileSystemActionReconciliator { logger.log(Level.INFO, " -> (1) Equals: Nothing to do, winning version equals winning file: "+winningLastVersion+" AND "+winningLastFile); } else if (winningFileToVersionComparison.getFileChanges().contains(FileChange.DELETED)) { - FileSystemAction action = new NewFileSystemAction(config, winningLastVersion, winnersDatabase); + FileSystemAction action = new NewFileSystemAction(config, winnersDatabase, assembler, winningLastVersion); outFileSystemActions.add(action); logger.log(Level.INFO, " -> (2) Deleted: Local file does NOT exist, but it should, winning version not known: "+winningLastVersion+" AND "+winningLastFile); @@ -272,7 +275,7 @@ public class FileSystemActionReconciliator { throw new Exception("What happend here?"); } else { // Content changed - FileSystemAction action = new NewFileSystemAction(config, winningLastVersion, winnersDatabase); + FileSystemAction action = new NewFileSystemAction(config, winnersDatabase, assembler, winningLastVersion); outFileSystemActions.add(action); logger.log(Level.INFO, " -> (7) Content changed: Winning file differs from winning version: "+winningLastVersion+" AND "+winningLastFile); @@ -294,7 +297,7 @@ public class FileSystemActionReconciliator { logger.log(Level.INFO, " -> (8) Equals: Nothing to do, local file equals local version equals winning version: local file = "+localLastFile+", local version = "+localLastVersion+", winning version = "+winningLastVersion); } else if (winningVersionToLocalVersionComparison.getFileChanges().contains(FileChange.DELETED)) { - FileSystemAction action = new ChangeFileSystemAction(config, localLastVersion, winningLastVersion, winnersDatabase); + FileSystemAction action = new ChangeFileSystemAction(config, winnersDatabase, assembler, localLastVersion, winningLastVersion); fileSystemActions.add(action); logger.log(Level.INFO, " -> (9) Content changed: Local file does not exist, but it should: local file = "+localLastFile+", local version = "+localLastVersion+", winning version = "+winningLastVersion); @@ -333,7 +336,7 @@ public class FileSystemActionReconciliator { changeSet.getChangedFiles().add(winningLastVersion.getPath()); } else { // Content changed - FileSystemAction action = new ChangeFileSystemAction(config, localLastVersion, winningLastVersion, winnersDatabase); + FileSystemAction action = new ChangeFileSystemAction(config, winnersDatabase, assembler, localLastVersion, winningLastVersion); fileSystemActions.add(action); logger.log(Level.INFO, " -> (13) Content changed: Local file differs from winning version: local file = "+localLastFile+", local version = "+localLastVersion+", winning version = "+winningLastVersion); @@ -351,7 +354,7 @@ public class FileSystemActionReconciliator { boolean winningLastVersionDeleted = winningLastVersion.getStatus() == FileStatus.DELETED; if (!winningLastVersionDeleted) { - FileSystemAction action = new ChangeFileSystemAction(config, localLastVersion, winningLastVersion, winnersDatabase); + FileSystemAction action = new ChangeFileSystemAction(config, winnersDatabase, assembler, localLastVersion, winningLastVersion); fileSystemActions.add(action); logger.log(Level.INFO, " -> (14) Content changed: Local file does NOT exist, and winning version changed: local file = "+localLastFile+", local version = "+localLastVersion+", winning version = "+winningLastVersion); @@ -364,7 +367,7 @@ public class FileSystemActionReconciliator { } } else { - FileSystemAction action = new ChangeFileSystemAction(config, localLastVersion, winningLastVersion, winnersDatabase); + FileSystemAction action = new ChangeFileSystemAction(config, winnersDatabase, assembler, winningLastVersion, localLastVersion); fileSystemActions.add(action); logger.log(Level.INFO, " -> (16) Content changed: Local file differs from last version: local file = "+localLastFile+", local version = "+localLastVersion+", winning version = "+winningLastVersion); diff --git a/syncany-lib/src/main/java/org/syncany/operations/down/actions/ChangeFileSystemAction.java b/syncany-lib/src/main/java/org/syncany/operations/down/actions/ChangeFileSystemAction.java index f367b73..1acb363 100644 --- a/syncany-lib/src/main/java/org/syncany/operations/down/actions/ChangeFileSystemAction.java +++ b/syncany-lib/src/main/java/org/syncany/operations/down/actions/ChangeFileSystemAction.java @@ -23,10 +23,12 @@ import org.syncany.config.Config; import org.syncany.database.FileVersion; import org.syncany.database.FileVersion.FileStatus; import org.syncany.database.MemoryDatabase; +import org.syncany.operations.Assembler; public class ChangeFileSystemAction extends FileCreatingFileSystemAction { - public ChangeFileSystemAction(Config config, FileVersion fromFileVersion, FileVersion toFileVersion, MemoryDatabase winningDatabase) { - super(config, winningDatabase, fromFileVersion, toFileVersion); + public ChangeFileSystemAction(Config config, MemoryDatabase winningDatabase, Assembler assembler, FileVersion fromFileVersion, + FileVersion toFileVersion) { + super(config, winningDatabase, assembler, fromFileVersion, toFileVersion); } @Override diff --git a/syncany-lib/src/main/java/org/syncany/operations/down/actions/FileCreatingFileSystemAction.java b/syncany-lib/src/main/java/org/syncany/operations/down/actions/FileCreatingFileSystemAction.java index 903ec0f..fddd5f9 100644 --- a/syncany-lib/src/main/java/org/syncany/operations/down/actions/FileCreatingFileSystemAction.java +++ b/syncany-lib/src/main/java/org/syncany/operations/down/actions/FileCreatingFileSystemAction.java @@ -24,13 +24,15 @@ import org.syncany.config.Config; import org.syncany.database.FileVersion; import org.syncany.database.FileVersion.FileType; import org.syncany.database.MemoryDatabase; -import org.syncany.database.SqlDatabase; import org.syncany.operations.Assembler; import org.syncany.util.NormalizedPath; public abstract class FileCreatingFileSystemAction extends FileSystemAction { - public FileCreatingFileSystemAction(Config config, MemoryDatabase winningDatabase, FileVersion file1, FileVersion file2) { - super(config, winningDatabase, file1, file2); + protected Assembler assembler; + + public FileCreatingFileSystemAction(Config config, MemoryDatabase winningDatabase, Assembler assembler, FileVersion file1, FileVersion file2) { + super(config, winningDatabase, file1, file2); + this.assembler = assembler; } protected void createFileFolderOrSymlink(FileVersion reconstructedFileVersion) throws Exception { @@ -76,9 +78,6 @@ public abstract class FileCreatingFileSystemAction extends FileSystemAction { } protected File assembleFileToCache(FileVersion reconstructedFileVersion) throws Exception { - SqlDatabase localDatabase = new SqlDatabase(config); - Assembler assembler = new Assembler(config, localDatabase, winningDatabase); - File reconstructedFileInCache = assembler.assembleToCache(reconstructedFileVersion); setFileAttributes(reconstructedFileVersion, reconstructedFileInCache); diff --git a/syncany-lib/src/main/java/org/syncany/operations/down/actions/NewFileSystemAction.java b/syncany-lib/src/main/java/org/syncany/operations/down/actions/NewFileSystemAction.java index 0f9f3d3..ad5fd5e 100644 --- a/syncany-lib/src/main/java/org/syncany/operations/down/actions/NewFileSystemAction.java +++ b/syncany-lib/src/main/java/org/syncany/operations/down/actions/NewFileSystemAction.java @@ -20,11 +20,12 @@ package org.syncany.operations.down.actions; import org.syncany.config.Config; import org.syncany.database.FileVersion; import org.syncany.database.MemoryDatabase; +import org.syncany.operations.Assembler; public class NewFileSystemAction extends FileCreatingFileSystemAction { - public NewFileSystemAction(Config config, FileVersion newFileVersion, MemoryDatabase winningDatabase) { - super(config, winningDatabase, null, newFileVersion); + public NewFileSystemAction(Config config, MemoryDatabase winningDatabase, Assembler assembler, FileVersion newFileVersion) { + super(config, winningDatabase, assembler, null, newFileVersion); } @Override diff --git a/syncany-lib/src/main/java/org/syncany/operations/init/ConnectOperation.java b/syncany-lib/src/main/java/org/syncany/operations/init/ConnectOperation.java index 45f72d6..2d1de3b 100644 --- a/syncany-lib/src/main/java/org/syncany/operations/init/ConnectOperation.java +++ b/syncany-lib/src/main/java/org/syncany/operations/init/ConnectOperation.java @@ -315,7 +315,7 @@ public class ConnectOperation extends AbstractInitOperation { throw new RuntimeException("Repository file is encrypted, but password cannot be queried (no listener)."); } - return listener.onUserPassword(null, "Password: "); + return listener.onUserPassword(null, "Master Password: "); } else { return options.getPassword(); diff --git a/syncany-lib/src/main/java/org/syncany/operations/restore/RestoreFileSystemAction.java b/syncany-lib/src/main/java/org/syncany/operations/restore/RestoreFileSystemAction.java index 0cffdc7..61257c0 100644 --- a/syncany-lib/src/main/java/org/syncany/operations/restore/RestoreFileSystemAction.java +++ b/syncany-lib/src/main/java/org/syncany/operations/restore/RestoreFileSystemAction.java @@ -25,14 +25,15 @@ import org.syncany.database.FileVersion; import org.syncany.database.FileVersion.FileStatus; import org.syncany.database.FileVersion.FileType; import org.syncany.database.MemoryDatabase; +import org.syncany.operations.Assembler; import org.syncany.operations.down.actions.FileCreatingFileSystemAction; import org.syncany.util.NormalizedPath; public class RestoreFileSystemAction extends FileCreatingFileSystemAction { private String relativeTargetPath; - public RestoreFileSystemAction(Config config, FileVersion fileVersion, String relativeTargetPath) { - super(config, new MemoryDatabase(), null, fileVersion); + public RestoreFileSystemAction(Config config, Assembler assembler, FileVersion fileVersion, String relativeTargetPath) { + super(config, new MemoryDatabase(), assembler, null, fileVersion); this.relativeTargetPath = relativeTargetPath; } diff --git a/syncany-lib/src/main/java/org/syncany/operations/restore/RestoreOperation.java b/syncany-lib/src/main/java/org/syncany/operations/restore/RestoreOperation.java index 9536b5c..7d2f505 100644 --- a/syncany-lib/src/main/java/org/syncany/operations/restore/RestoreOperation.java +++ b/syncany-lib/src/main/java/org/syncany/operations/restore/RestoreOperation.java @@ -32,6 +32,7 @@ import org.syncany.database.MultiChunkEntry.MultiChunkId; import org.syncany.database.PartialFileHistory.FileHistoryId; import org.syncany.database.SqlDatabase; import org.syncany.operations.AbstractTransferOperation; +import org.syncany.operations.Assembler; import org.syncany.operations.Downloader; import org.syncany.operations.restore.RestoreOperationResult.RestoreResultCode; import org.syncany.plugins.transfer.StorageException; @@ -45,6 +46,8 @@ public class RestoreOperation extends AbstractTransferOperation { private SqlDatabase localDatabase; private Downloader downloader; + private Assembler assembler; + public RestoreOperation(Config config) { this(config, new RestoreOperationOptions()); } @@ -55,6 +58,7 @@ public class RestoreOperation extends AbstractTransferOperation { this.options = options; this.localDatabase = new SqlDatabase(config); this.downloader = new Downloader(config, transferManager); + this.assembler = new Assembler(config, localDatabase, null); } @Override @@ -88,7 +92,7 @@ public class RestoreOperation extends AbstractTransferOperation { // Restore file logger.log(Level.INFO, "- Restoring: " + restoreFileVersion); - RestoreFileSystemAction restoreAction = new RestoreFileSystemAction(config, restoreFileVersion, options.getRelativeTargetPath()); + RestoreFileSystemAction restoreAction = new RestoreFileSystemAction(config, assembler, restoreFileVersion, options.getRelativeTargetPath()); RestoreFileSystemActionResult restoreResult = restoreAction.execute(); return new RestoreOperationResult(RestoreResultCode.ACK, restoreResult.getTargetFile()); diff --git a/syncany-lib/src/main/java/org/syncany/operations/up/UpOperation.java b/syncany-lib/src/main/java/org/syncany/operations/up/UpOperation.java index 9247ce7..a8147fd 100644 --- a/syncany-lib/src/main/java/org/syncany/operations/up/UpOperation.java +++ b/syncany-lib/src/main/java/org/syncany/operations/up/UpOperation.java @@ -106,6 +106,11 @@ public class UpOperation extends AbstractTransferOperation { private UpOperationResult result; private SqlDatabase localDatabase; + + private boolean resuming; + private TransactionRemoteFile transactionRemoteFileToResume; + private Collection<RemoteTransaction> remoteTransactionsToResume; + private BlockingQueue<DatabaseVersion> databaseVersionQueue; public UpOperation(Config config) { this(config, new UpOperationOptions()); @@ -117,6 +122,11 @@ public class UpOperation extends AbstractTransferOperation { this.options = options; this.result = new UpOperationResult(); this.localDatabase = new SqlDatabase(config); + + this.resuming = false; + this.transactionRemoteFileToResume = null; + this.remoteTransactionsToResume = null; + this.databaseVersionQueue = new LinkedBlockingQueue<>(); } @Override @@ -135,82 +145,28 @@ public class UpOperation extends AbstractTransferOperation { // Upload action file (lock for cleanup) startOperation(); - TransactionRemoteFile transactionRemoteFileToResume = null; - Collection<RemoteTransaction> remoteTransactionsToResume = null; - - BlockingQueue<DatabaseVersion> databaseVersionQueue = new LinkedBlockingQueue<>(); - boolean resuming = false; - - if (options.isResume()) { - Collection<Long> versionsToResume = transferManager.loadPendingTransactionList(); - if (versionsToResume != null && versionsToResume.size() != 0) { - logger.log(Level.INFO, "Found local transaction to resume."); - logger.log(Level.INFO, "Attempting to find transactionRemoteFile"); - - remoteTransactionsToResume = attemptResumeTransactions(versionsToResume); - Collection<DatabaseVersion> remoteDatabaseVersionsToResume = attemptResumeDatabaseVersions(versionsToResume); - - if (remoteDatabaseVersionsToResume != null && remoteTransactionsToResume != null && - remoteDatabaseVersionsToResume.size() == remoteTransactionsToResume.size()) { - databaseVersionQueue.addAll(remoteDatabaseVersionsToResume); - resuming = true; - } - // Add stopping marker - databaseVersionQueue.add(new DatabaseVersion()); - - try { - transactionRemoteFileToResume = attemptResumeTransactionRemoteFile(); - } - catch (BlockingTransfersException e) { - stopBecauseOfBlockingTransactions(); - return result; - } - } - else { - transferManager.clearResumableTransactions(); + try { + if (options.isResume()) { + prepareResume(); } - } - if (!resuming) { - // Get a list of files that have been updated - ChangeSet localChanges = result.getStatusResult().getChangeSet(); - List<File> locallyUpdatedFiles = extractLocallyUpdatedFiles(localChanges); - List<File> locallyDeletedFiles = extractLocallyDeletedFiles(localChanges); - // Iterate over the changes, deduplicate, and feed DatabaseVersions into an iterator - Deduper deduper = new Deduper(config.getChunker(), config.getMultiChunker(), config.getTransformer(), options.getTransactionSizeLimit(), - options.getTransactionFileLimit()); - - AsyncIndexer asyncIndexer = new AsyncIndexer(config, deduper, locallyUpdatedFiles, locallyDeletedFiles, databaseVersionQueue); - new Thread(asyncIndexer).start(); - } - - // If we are not resuming from a remote transaction, we need to clean transactions. - if (transactionRemoteFileToResume == null) { - try { - transferManager.cleanTransactions(); - } - catch (BlockingTransfersException e) { - stopBecauseOfBlockingTransactions(); - return result; + if (!resuming) { + startIndexerThread(databaseVersionQueue); } - } - int numberOfPerformedTransactions = 0; - if (resuming) { - numberOfPerformedTransactions = executeTransactions(databaseVersionQueue, remoteTransactionsToResume.iterator(), transactionRemoteFileToResume); + // If we are not resuming from a remote transaction, we need to clean transactions. + if (transactionRemoteFileToResume == null) { + transferManager.cleanTransactions(); + } } - else { - numberOfPerformedTransactions = executeTransactions(databaseVersionQueue); + catch (BlockingTransfersException e) { + stopBecauseOfBlockingTransactions(); + return result; } - // Check if anything has happened. - if (numberOfPerformedTransactions == 0) { - logger.log(Level.INFO, "Local database is up-to-date. NOTHING TO DO!"); - result.setResultCode(UpResultCode.OK_NO_CHANGES); - } else { - logger.log(Level.INFO, "Sync up done."); - result.setResultCode(UpResultCode.OK_CHANGES_UPLOADED); - } + // Go wild + int numberOfPerformedTransactions = executeTransactions(); + updateResult(numberOfPerformedTransactions); // Close database connection localDatabase.finalize(); @@ -218,17 +174,62 @@ public class UpOperation extends AbstractTransferOperation { // Finish 'up' before 'cleanup' starts finishOperation(); fireEndEvent(); + return result; } - /** - * Transfers the given {@link DatabaseVersion} objects to the remote. - * Each {@link DatabaseVersion} will be transferred in its own {@link RemoteTransaction} object. - * - * @param databaseVersions The {@link DatabaseVersion} objects to send to the remote. - */ - private int executeTransactions(BlockingQueue<DatabaseVersion> databaseVersionQueue) throws Exception { - return executeTransactions(databaseVersionQueue, null, null); + private void updateResult(int numberOfPerformedTransactions) { + if (numberOfPerformedTransactions == 0) { + logger.log(Level.INFO, "Local database is up-to-date. NOTHING TO DO!"); + result.setResultCode(UpResultCode.OK_NO_CHANGES); + } + else { + logger.log(Level.INFO, "Sync up done."); + result.setResultCode(UpResultCode.OK_CHANGES_UPLOADED); + } + } + + private void startIndexerThread(BlockingQueue<DatabaseVersion> databaseVersionQueue) { + // Get a list of files that have been updated + ChangeSet localChanges = result.getStatusResult().getChangeSet(); + List<File> locallyUpdatedFiles = extractLocallyUpdatedFiles(localChanges); + List<File> locallyDeletedFiles = extractLocallyDeletedFiles(localChanges); + + // Iterate over the changes, deduplicate, and feed DatabaseVersions into an iterator + Deduper deduper = new Deduper(config.getChunker(), config.getMultiChunker(), config.getTransformer(), options.getTransactionSizeLimit(), + options.getTransactionFileLimit()); + + AsyncIndexer asyncIndexer = new AsyncIndexer(config, deduper, locallyUpdatedFiles, locallyDeletedFiles, databaseVersionQueue); + new Thread(asyncIndexer, "AsyncI/" + config.getLocalDir().getName()).start(); + } + + private void prepareResume() throws Exception { + Collection<Long> versionsToResume = transferManager.loadPendingTransactionList(); + boolean hasVersionsToResume = versionsToResume != null && versionsToResume.size() > 0; + + if (hasVersionsToResume) { + logger.log(Level.INFO, "Found local transaction to resume."); + logger.log(Level.INFO, "Attempting to find transactionRemoteFile"); + + remoteTransactionsToResume = attemptResumeTransactions(versionsToResume); + Collection<DatabaseVersion> remoteDatabaseVersionsToResume = attemptResumeDatabaseVersions(versionsToResume); + + resuming = remoteDatabaseVersionsToResume != null && remoteTransactionsToResume != null && + remoteDatabaseVersionsToResume.size() == remoteTransactionsToResume.size(); + + if (resuming) { + databaseVersionQueue.addAll(remoteDatabaseVersionsToResume); + databaseVersionQueue.add(new DatabaseVersion()); // Empty database version is the stopping marker + + transactionRemoteFileToResume = attemptResumeTransactionRemoteFile(); + } + else { + transferManager.clearResumableTransactions(); + } + } + else { + transferManager.clearResumableTransactions(); + } } /** @@ -246,16 +247,9 @@ public class UpOperation extends AbstractTransferOperation { * @param remoteTransactionsToResume {@link RemoteTransaction} objects that correspond to the given {@link DatabaseVersion} objects. * @param transactionRemoteFileToResume The file on the remote that was used for the specific transaction that was interrupted. */ - private int executeTransactions(BlockingQueue<DatabaseVersion> databaseVersionQueue, Iterator<RemoteTransaction> remoteTransactionsToResume, - TransactionRemoteFile transactionRemoteFileToResume) - throws Exception { + private int executeTransactions() throws Exception { + Iterator<RemoteTransaction> remoteTransactionsToResumeIterator = (resuming) ? remoteTransactionsToResume.iterator() : null; - boolean resuming = true; - - if (remoteTransactionsToResume == null) { - resuming = false; - } - // At this point, if a failure occurs from which we can resume, new transaction files will be written // Delete any old transaction files transferManager.clearPendingTransactions(); @@ -267,14 +261,17 @@ public class UpOperation extends AbstractTransferOperation { DatabaseVersion databaseVersion = databaseVersionQueue.take(); boolean noDatabaseVersions = databaseVersion.isEmpty(); + // Add dirty data to first database addDirtyData(databaseVersion); // while (!databaseVersion.isEmpty()) { RemoteTransaction remoteTransaction = null; + if (!resuming) { VectorClock newVectorClock = findNewVectorClock(); + databaseVersion.setVectorClock(newVectorClock); databaseVersion.setTimestamp(new Date()); databaseVersion.setClient(config.getMachineName()); @@ -283,11 +280,12 @@ public class UpOperation extends AbstractTransferOperation { // Add multichunks to transaction logger.log(Level.INFO, "Uploading new multichunks ..."); + // This call adds newly changed chunks to a "RemoteTransaction", so they can be uploaded later. addMultiChunksToTransaction(remoteTransaction, databaseVersion.getMultiChunks()); } else { - remoteTransaction = remoteTransactionsToResume.next(); + remoteTransaction = remoteTransactionsToResumeIterator.next(); } logger.log(Level.INFO, "Uploading database: " + databaseVersion); @@ -395,6 +393,7 @@ public class UpOperation extends AbstractTransferOperation { else { transactionRemoteFile = transactions.get(0); } + return transactionRemoteFile; } @@ -674,6 +673,7 @@ public class UpOperation extends AbstractTransferOperation { private void addLocalDatabaseToTransaction(RemoteTransaction remoteTransaction, File localDatabaseFile, DatabaseRemoteFile remoteDatabaseFile) throws InterruptedException, StorageException { + logger.log(Level.INFO, "- Uploading " + localDatabaseFile + " to " + remoteDatabaseFile + " ..."); remoteTransaction.upload(localDatabaseFile, remoteDatabaseFile); } @@ -727,56 +727,70 @@ public class UpOperation extends AbstractTransferOperation { return newVectorClock; } - private Collection<RemoteTransaction> attemptResumeTransactions(Collection<Long> versions) throws Exception { - Collection<RemoteTransaction> remoteTransactions = new ArrayList<>(); - for (Long version : versions) { - File transactionFile = config.getTransactionFile(version); + private Collection<RemoteTransaction> attemptResumeTransactions(Collection<Long> versions) { + try { + Collection<RemoteTransaction> remoteTransactions = new ArrayList<>(); - // If a single transaction file is missing, we should restart - if (!transactionFile.exists()) { - return null; - } + for (Long version : versions) { + File transactionFile = config.getTransactionFile(version); + + // If a single transaction file is missing, we should restart + if (!transactionFile.exists()) { + return null; + } - TransactionTO transactionTO = TransactionTO.load(null, transactionFile); + TransactionTO transactionTO = TransactionTO.load(null, transactionFile); - // Verify if all files needed are in cache. - for (ActionTO action : transactionTO.getActions()) { - if (action.getType() == ActionType.UPLOAD) { - if (action.getStatus() == ActionStatus.UNSTARTED) { - if (!action.getLocalTempLocation().exists()) { - // Unstarted upload has no cached local copy, abort - return null; + // Verify if all files needed are in cache. + for (ActionTO action : transactionTO.getActions()) { + if (action.getType() == ActionType.UPLOAD) { + if (action.getStatus() == ActionStatus.UNSTARTED) { + if (!action.getLocalTempLocation().exists()) { + // Unstarted upload has no cached local copy, abort + return null; + } } } } - } - remoteTransactions.add(new RemoteTransaction(config, transferManager, transactionTO)); + remoteTransactions.add(new RemoteTransaction(config, transferManager, transactionTO)); + } + + return remoteTransactions; + } catch (Exception e) { + logger.log(Level.WARNING, "Invalid transaction file. Cannot resume!"); + return null; } - return remoteTransactions; } private Collection<DatabaseVersion> attemptResumeDatabaseVersions(Collection<Long> versions) throws Exception { - Collection<DatabaseVersion> databaseVersions = new ArrayList<>(); - for (Long version : versions) { - File databaseFile = config.getTransactionDatabaseFile(version); + try { + Collection<DatabaseVersion> databaseVersions = new ArrayList<>(); + + for (Long version : versions) { + File databaseFile = config.getTransactionDatabaseFile(version); - // If a single database file is missing, we should restart - if (!databaseFile.exists()) { - return null; - } + // If a single database file is missing, we should restart + if (!databaseFile.exists()) { + return null; + } - DatabaseXmlSerializer databaseSerializer = new DatabaseXmlSerializer(); - MemoryDatabase memoryDatabase = new MemoryDatabase(); - databaseSerializer.load(memoryDatabase, databaseFile, null, null, DatabaseReadType.FULL); + DatabaseXmlSerializer databaseSerializer = new DatabaseXmlSerializer(); + MemoryDatabase memoryDatabase = new MemoryDatabase(); + databaseSerializer.load(memoryDatabase, databaseFile, null, null, DatabaseReadType.FULL); - if (memoryDatabase.getDatabaseVersions().size() == 0) { - return null; - } + if (memoryDatabase.getDatabaseVersions().size() == 0) { + return null; + } - databaseVersions.add(memoryDatabase.getLastDatabaseVersion()); + databaseVersions.add(memoryDatabase.getLastDatabaseVersion()); + } + + return databaseVersions; + } catch (Exception e) { + logger.log(Level.WARNING, "Cannot load database versions from 'state'. Cannot resume."); + return null; } - return databaseVersions; } /** diff --git a/syncany-lib/src/main/java/org/syncany/plugins/local/LocalTransferManager.java b/syncany-lib/src/main/java/org/syncany/plugins/local/LocalTransferManager.java index 1422059..e7ee603 100644 --- a/syncany-lib/src/main/java/org/syncany/plugins/local/LocalTransferManager.java +++ b/syncany-lib/src/main/java/org/syncany/plugins/local/LocalTransferManager.java @@ -229,7 +229,7 @@ public class LocalTransferManager extends AbstractTransferManager { } catch (StorageException e) { logger.log(Level.INFO, "Cannot create instance of " + remoteFileClass.getSimpleName() + " for file " + path - + "; maybe invalid file name pattern. Ignoring file.", e); + + "; maybe invalid file name pattern. Ignoring file."); } } } diff --git a/syncany-lib/src/main/java/org/syncany/plugins/transfer/features/TransactionAwareFeatureTransferManager.java b/syncany-lib/src/main/java/org/syncany/plugins/transfer/features/TransactionAwareFeatureTransferManager.java index bc5c582..b5be175 100644 --- a/syncany-lib/src/main/java/org/syncany/plugins/transfer/features/TransactionAwareFeatureTransferManager.java +++ b/syncany-lib/src/main/java/org/syncany/plugins/transfer/features/TransactionAwareFeatureTransferManager.java @@ -279,16 +279,25 @@ public class TransactionAwareFeatureTransferManager implements FeatureTransferMa */ public Collection<Long> loadPendingTransactionList() throws IOException { Objects.requireNonNull(config, "Cannot read pending transaction list if config is null."); + Collection<Long> databaseVersionNumbers = new ArrayList<>(); File transactionListFile = config.getTransactionListFile(); + if (!transactionListFile.exists()) { return Collections.emptyList(); } Collection<String> transactionLines = Files.readAllLines(transactionListFile.toPath(), Charset.forName("UTF-8")); + for (String transactionLine : transactionLines) { - databaseVersionNumbers.add(Long.parseLong(transactionLine)); + try { + databaseVersionNumbers.add(Long.parseLong(transactionLine)); + } catch (NumberFormatException e) { + logger.log(Level.WARNING, "Cannot parse line in transaction list: " + transactionLine + ". Cannot resume."); + return Collections.emptyList(); + } } + return databaseVersionNumbers; } diff --git a/syncany-lib/src/test/integration/java/org/syncany/tests/ScenarioTestSuite.java b/syncany-lib/src/test/integration/java/org/syncany/tests/ScenarioTestSuite.java index f24be0d..64d7d04 100644 --- a/syncany-lib/src/test/integration/java/org/syncany/tests/ScenarioTestSuite.java +++ b/syncany-lib/src/test/integration/java/org/syncany/tests/ScenarioTestSuite.java @@ -48,6 +48,8 @@ import org.syncany.tests.integration.scenarios.Issue288ScenarioTest; import org.syncany.tests.integration.scenarios.Issue303ScenarioTest; import org.syncany.tests.integration.scenarios.Issue316ScenarioTest; import org.syncany.tests.integration.scenarios.Issue374Pre1965DateScenarioTest; +import org.syncany.tests.integration.scenarios.Issue429ScenarioTest; +import org.syncany.tests.integration.scenarios.Issue520NoResumeOnCorruptXmlScenarioTest; import org.syncany.tests.integration.scenarios.ManyRenamesScenarioTest; import org.syncany.tests.integration.scenarios.ManySyncUpsAndDatabaseFileCleanupScenarioTest; import org.syncany.tests.integration.scenarios.ManySyncUpsAndOtherClientSyncDownScenarioTest; @@ -94,6 +96,8 @@ import org.syncany.tests.integration.scenarios.SymlinkSyncScenarioTest; Issue303ScenarioTest.class, Issue316ScenarioTest.class, Issue374Pre1965DateScenarioTest.class, + Issue429ScenarioTest.class, + Issue520NoResumeOnCorruptXmlScenarioTest.class, ManyRenamesScenarioTest.class, ManySyncUpsAndDatabaseFileCleanupScenarioTest.class, ManySyncUpsLargeFileScenarioTest.class, diff --git a/syncany-lib/src/test/integration/java/org/syncany/tests/integration/operations/FileSystemActionComparatorTest.java b/syncany-lib/src/test/integration/java/org/syncany/tests/integration/operations/FileSystemActionComparatorTest.java index 1648abc..7267076 100644 --- a/syncany-lib/src/test/integration/java/org/syncany/tests/integration/operations/FileSystemActionComparatorTest.java +++ b/syncany-lib/src/test/integration/java/org/syncany/tests/integration/operations/FileSystemActionComparatorTest.java @@ -83,7 +83,7 @@ public class FileSystemActionComparatorTest { private NewFileSystemAction createNewFileSystemAction(String path, FileType type) throws Exception { FileVersion firstFileVersion = createFileVersion(path, type); - return new NewFileSystemAction(createDummyConfig(), firstFileVersion, null); + return new NewFileSystemAction(createDummyConfig(), null, null, firstFileVersion); } private RenameFileSystemAction createRenameFileSystemAction(String fromPath, String toPath, FileType type) throws Exception { diff --git a/syncany-lib/src/test/integration/java/org/syncany/tests/integration/scenarios/Issue520NoResumeOnCorruptXmlScenarioTest.java b/syncany-lib/src/test/integration/java/org/syncany/tests/integration/scenarios/Issue520NoResumeOnCorruptXmlScenarioTest.java new file mode 100644 index 0000000..ed1737d --- /dev/null +++ b/syncany-lib/src/test/integration/java/org/syncany/tests/integration/scenarios/Issue520NoResumeOnCorruptXmlScenarioTest.java @@ -0,0 +1,77 @@ +/* + * Syncany, www.syncany.org + * Copyright (C) 2011-2015 Philipp C. Heckel <[email protected]> + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.syncany.tests.integration.scenarios; + +import java.io.File; + +import org.junit.Test; +import org.syncany.plugins.local.LocalTransferSettings; +import org.syncany.tests.unit.util.TestFileUtil; +import org.syncany.tests.util.TestAssertUtil; +import org.syncany.tests.util.TestClient; +import org.syncany.tests.util.TestConfigUtil; + +import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class Issue520NoResumeOnCorruptXmlScenarioTest { + @Test + public void testCorruptTransactionListFile() throws Exception { + // Setup + LocalTransferSettings testConnection = (LocalTransferSettings) TestConfigUtil.createTestLocalConnection(); + + TestClient clientA = new TestClient("A", testConnection); + + clientA.createNewFile("file1.txt", 1024); + clientA.upWithForceChecksum(); + + TestFileUtil.createFileWithContent(new File(clientA.getConfig().getAppDir(), "/state/transaction-list.txt"), "INVALID"); + + clientA.createNewFile("file2.txt", 1024); + clientA.upWithForceChecksum(); // This did FAIL due to an XML parsing exception + + assertEquals("There should be exactly two database files", 2, new File(testConnection.getPath() + "/databases").listFiles().length); + assertEquals("There should be exactly two multichunks", 2, new File(testConnection.getPath() + "/multichunks").listFiles().length); + + // Tear down + clientA.deleteTestData(); + } + + @Test + public void testCorruptTransactionFile() throws Exception { + // Setup + LocalTransferSettings testConnection = (LocalTransferSettings) TestConfigUtil.createTestLocalConnection(); + + TestClient clientA = new TestClient("A", testConnection); + + clientA.createNewFile("file1.txt", 1024); + clientA.upWithForceChecksum(); + + TestFileUtil.createFileWithContent(new File(clientA.getConfig().getAppDir(), "/state/transaction-list.txt"), "0000000001"); + TestFileUtil.createFileWithContent(new File(clientA.getConfig().getAppDir(), "/state/transaction-database.0000000001.xml"), "invalid"); + + clientA.createNewFile("file2.txt", 1024); + clientA.upWithForceChecksum(); // This did FAIL due to an XML parsing exception + + assertEquals("There should be exactly two database files", 2, new File(testConnection.getPath() + "/databases").listFiles().length); + assertEquals("There should be exactly two multichunks", 2, new File(testConnection.getPath() + "/multichunks").listFiles().length); + + // Tear down + clientA.deleteTestData(); + } +} -- Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-java/syncany.git _______________________________________________ pkg-java-commits mailing list [email protected] http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-java-commits
