diff --git a/CMakeLists.txt b/CMakeLists.txt index b505b8e22..c8799eeb1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,11 @@ set(CMAKE_CXX_STANDARD 14) project(client) +if(UNIT_TESTING) + include(CTest) + enable_testing() +endif() + set(BIN_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") @@ -52,6 +57,9 @@ include(Warnings) include(${CMAKE_SOURCE_DIR}/VERSION.cmake) # For config.h include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}) +#include_directories(BEFORE +# "C:/Program Files (x86)/Windows Kits/10/Include/10.0.17134.0/um" +# "C:/Program Files (x86)/Windows Kits/10/Include/10.0.17134.0/shared") # Allows includes based on src/ like #include "common/utility.h" or #include "csync/csync.h" include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/src @@ -215,8 +223,9 @@ if( WIN32 ) add_definitions( -D__USE_MINGW_ANSI_STDIO=1 ) add_definitions( -DNOMINMAX ) # Get APIs from from Vista onwards. -add_definitions( -D_WIN32_WINNT=0x0601 ) -add_definitions( -DWINVER=0x0601 ) +add_definitions(-D_WIN32_WINNT=0x0601) +add_definitions(-DWINVER=0x0601) +add_definitions(-DNTDDI_VERSION=0x0A000004) if( MSVC ) # Use automatic overload for suitable CRT safe-functions # See https://docs.microsoft.com/de-de/cpp/c-runtime-library/security-features-in-the-crt?view=vs-2019 @@ -254,8 +263,6 @@ if(BUILD_SHELL_INTEGRATION) endif() if(UNIT_TESTING) - include(CTest) - enable_testing() add_subdirectory(test) endif(UNIT_TESTING) diff --git a/shell_integration/windows/NCContextMenu/NCContextMenuRegHandler.cpp b/shell_integration/windows/NCContextMenu/NCContextMenuRegHandler.cpp index 37865b89a..34835e79b 100644 --- a/shell_integration/windows/NCContextMenu/NCContextMenuRegHandler.cpp +++ b/shell_integration/windows/NCContextMenu/NCContextMenuRegHandler.cpp @@ -31,13 +31,23 @@ HRESULT SetHKCRRegistryKeyAndValue(PCWSTR pszSubKey, PCWSTR pszValueName, PCWSTR if (SUCCEEDED(hr)) { + DWORD cbData; + const BYTE * lpData; + if (pszData) { // Set the specified value of the key. - DWORD cbData = lstrlen(pszData) * sizeof(*pszData); - hr = HRESULT_FROM_WIN32(RegSetValueEx(hKey, pszValueName, 0, - REG_SZ, reinterpret_cast<const BYTE *>(pszData), cbData)); + cbData = lstrlen(pszData) * sizeof(*pszData); + lpData = reinterpret_cast<const BYTE *>(pszData); } + else + { + cbData = 0; + lpData = NULL; + } + + hr = HRESULT_FROM_WIN32(RegSetValueEx(hKey, pszValueName, 0, + REG_SZ, lpData, cbData)); RegCloseKey(hKey); } @@ -88,6 +98,11 @@ HRESULT NCContextMenuRegHandler::RegisterInprocServer(PCWSTR pszModule, const CL { hr = SetHKCRRegistryKeyAndValue(szSubkey, nullptr, pszFriendlyName); + // Create the HKCR\CLSID\{<CLSID>}\ContextMenuOptIn subkey. + if (SUCCEEDED(hr)) { + hr = SetHKCRRegistryKeyAndValue(szSubkey, L"ContextMenuOptIn", NULL); + } + // Create the HKCR\CLSID\{<CLSID>}\InprocServer32 key. if (SUCCEEDED(hr)) { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 69c9cfde3..f035a66be 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -84,3 +84,4 @@ if(KRAZY2_EXECUTABLE) ${PROJECT_SOURCE_DIR}/src/cmd/*.cpp ) endif() + diff --git a/src/common/common.cmake b/src/common/common.cmake index 9d7898e8a..3caf829f7 100644 --- a/src/common/common.cmake +++ b/src/common/common.cmake @@ -9,4 +9,5 @@ set(common_SOURCES ${CMAKE_CURRENT_LIST_DIR}/syncjournalfilerecord.cpp ${CMAKE_CURRENT_LIST_DIR}/utility.cpp ${CMAKE_CURRENT_LIST_DIR}/remotepermissions.cpp + ${CMAKE_CURRENT_LIST_DIR}/vfs.cpp ) diff --git a/src/common/utility.h b/src/common/utility.h index 3985704e3..9e2d458d6 100644 --- a/src/common/utility.h +++ b/src/common/utility.h @@ -243,6 +243,12 @@ namespace Utility { OCSYNC_EXPORT bool registryDeleteKeyValue(HKEY hRootKey, const QString &subKey, const QString &valueName); OCSYNC_EXPORT bool registryWalkSubKeys(HKEY hRootKey, const QString &subKey, const std::function<void(HKEY, const QString &)> &callback); OCSYNC_EXPORT QRect getTaskbarDimensions(); + + // Possibly refactor to share code with UnixTimevalToFileTime in c_time.c + OCSYNC_EXPORT void UnixTimeToFiletime(time_t t, FILETIME *filetime); + OCSYNC_EXPORT void FiletimeToLargeIntegerFiletime(FILETIME *filetime, LARGE_INTEGER *hundredNSecs); + OCSYNC_EXPORT void UnixTimeToLargeIntegerFiletime(time_t t, LARGE_INTEGER *hundredNSecs); + #endif } /** @} */ // \addtogroup diff --git a/src/common/utility_win.cpp b/src/common/utility_win.cpp index d8eae7931..20d2e62a8 100644 --- a/src/common/utility_win.cpp +++ b/src/common/utility_win.cpp @@ -314,4 +314,24 @@ DWORD Utility::convertSizeToDWORD(size_t &convertVar) return static_cast<DWORD>(convertVar); } +void Utility::UnixTimeToFiletime(time_t t, FILETIME *filetime) +{ + LONGLONG ll = Int32x32To64(t, 10000000) + 116444736000000000; + filetime->dwLowDateTime = (DWORD) ll; + filetime->dwHighDateTime = ll >>32; +} + +void Utility::FiletimeToLargeIntegerFiletime(FILETIME *filetime, LARGE_INTEGER *hundredNSecs) +{ + hundredNSecs->LowPart = filetime->dwLowDateTime; + hundredNSecs->HighPart = filetime->dwHighDateTime; +} + +void Utility::UnixTimeToLargeIntegerFiletime(time_t t, LARGE_INTEGER *hundredNSecs) +{ + LONGLONG ll = Int32x32To64(t, 10000000) + 116444736000000000; + hundredNSecs->LowPart = (DWORD) ll; + hundredNSecs->HighPart = ll >>32; +} + } // namespace OCC diff --git a/src/common/vfs.cpp b/src/common/vfs.cpp new file mode 100644 index 000000000..b6f4646eb --- /dev/null +++ b/src/common/vfs.cpp @@ -0,0 +1,58 @@ +/* + * Copyright (C) by Dominik Schmidt <dschmidt@owncloud.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "vfs.h" + +using namespace OCC; + +Vfs::Vfs(QObject* parent) + : QObject(parent) +{ +} + +Vfs::~Vfs() = default; + +QString Vfs::modeToString(Mode mode) +{ + // Note: Strings are used for config and must be stable + switch (mode) { + case Off: + return QStringLiteral("off"); + case WithSuffix: + return QStringLiteral("suffix"); + case WindowsCfApi: + return QStringLiteral("wincfapi"); + } + return QStringLiteral("off"); +} + +bool Vfs::modeFromString(const QString &str, Mode *mode) +{ + // Note: Strings are used for config and must be stable + *mode = Off; + if (str == "off") { + return true; + } else if (str == "suffix") { + *mode = WithSuffix; + return true; + } else if (str == "wincfapi") { + *mode = WindowsCfApi; + return true; + } + return false; +} diff --git a/src/common/vfs.h b/src/common/vfs.h new file mode 100644 index 000000000..511551c87 --- /dev/null +++ b/src/common/vfs.h @@ -0,0 +1,97 @@ +/* + * Copyright (C) by Christian Kamm <mail@ckamm.de> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#pragma once + +#include <QObject> +#include <QScopedPointer> +#include <QSharedPointer> + +#include "ocsynclib.h" + +typedef struct csync_file_stat_s csync_file_stat_t; + +namespace OCC { + +class Account; +typedef QSharedPointer<Account> AccountPtr; +class SyncJournalDb; +class VfsPrivate; +class SyncFileItem; +typedef QSharedPointer<SyncFileItem> SyncFileItemPtr; + +struct OCSYNC_EXPORT VfsSetupParams +{ + QString filesystemPath; + QString remotePath; + + AccountPtr account; + // The journal must live at least until the stop() call + SyncJournalDb *journal; + + QString providerName; + QString providerVersion; +}; + +class OCSYNC_EXPORT Vfs : public QObject +{ + Q_OBJECT + +public: + enum Mode + { + Off, + WithSuffix, + WindowsCfApi, + }; + static QString modeToString(Mode mode); + static bool modeFromString(const QString &str, Mode *mode); + +public: + Vfs(QObject* parent = nullptr); + virtual ~Vfs(); + + virtual Mode mode() const = 0; + + // For WithSuffix modes: what's the suffix (including the dot)? + virtual QString fileSuffix() const = 0; + + virtual void registerFolder(const VfsSetupParams ¶ms) = 0; + virtual void start(const VfsSetupParams ¶ms) = 0; + virtual void stop() = 0; + virtual void unregisterFolder() = 0; + + virtual bool isHydrating() const = 0; + + // Update placeholder metadata during discovery + virtual bool updateMetadata(const QString &filePath, time_t modtime, quint64 size, const QByteArray &fileId, QString *error) = 0; + + // Create and convert placeholders in PropagateDownload + virtual void createPlaceholder(const QString &syncFolder, const SyncFileItemPtr &item) = 0; + virtual void convertToPlaceholder(const QString &filename, const SyncFileItemPtr &item) = 0; + + // Determine whether something is a placeholder + virtual bool isDehydratedPlaceholder(const QString &filePath) = 0; + + // Determine whether something is a placeholder in discovery + // stat has at least 'path' filled + // the stat_data argument has platform specific data + // returning true means that the file_stat->type was set and should be fixed + virtual bool statTypeVirtualFile(csync_file_stat_t *stat, void *stat_data) = 0; + +signals: + void beginHydrating(); + void doneHydrating(); +}; + +} // namespace OCC diff --git a/src/csync/csync.h b/src/csync/csync.h index af2927be0..759bced4a 100644 --- a/src/csync/csync.h +++ b/src/csync/csync.h @@ -137,7 +137,8 @@ enum ItemType { ItemTypeDirectory = 2, ItemTypeSkip = 3, ItemTypeVirtualFile = 4, - ItemTypeVirtualFileDownload = 5 + ItemTypeVirtualFileDownload = 5, + ItemTypeVirtualFileDehydration = 6, }; @@ -146,7 +147,9 @@ enum ItemType { // currently specified at https://github.com/owncloud/core/issues/8322 are 9 to 10 #define REMOTE_PERM_BUF_SIZE 15 -struct OCSYNC_EXPORT csync_file_stat_t { +typedef struct csync_file_stat_s csync_file_stat_t; + +struct OCSYNC_EXPORT csync_file_stat_s { time_t modtime = 0; int64_t size = 0; uint64_t inode = 0; @@ -177,7 +180,7 @@ struct OCSYNC_EXPORT csync_file_stat_t { enum csync_instructions_e instruction = CSYNC_INSTRUCTION_NONE; /* u32 */ - csync_file_stat_t() + csync_file_stat_s() : type(ItemTypeSkip) , child_modified(false) , has_ignored_files(false) diff --git a/src/csync/vio/csync_vio_local.h b/src/csync/vio/csync_vio_local.h index 97ac34d63..5ae05a5b3 100644 --- a/src/csync/vio/csync_vio_local.h +++ b/src/csync/vio/csync_vio_local.h @@ -25,7 +25,7 @@ struct csync_vio_handle_t; csync_vio_handle_t OCSYNC_EXPORT *csync_vio_local_opendir(const QString &name); int OCSYNC_EXPORT csync_vio_local_closedir(csync_vio_handle_t *dhandle); -std::unique_ptr<csync_file_stat_t> OCSYNC_EXPORT csync_vio_local_readdir(csync_vio_handle_t *dhandle); +std::unique_ptr<csync_file_stat_t> OCSYNC_EXPORT csync_vio_local_readdir(CSYNC *ctx, csync_vio_handle_t *dhandle); int OCSYNC_EXPORT csync_vio_local_stat(const char *uri, csync_file_stat_t *buf); diff --git a/src/csync/vio/csync_vio_local_unix.cpp b/src/csync/vio/csync_vio_local_unix.cpp index ea6f925df..9de6092fc 100644 --- a/src/csync/vio/csync_vio_local_unix.cpp +++ b/src/csync/vio/csync_vio_local_unix.cpp @@ -35,6 +35,7 @@ #include "csync_util.h" #include "vio/csync_vio_local.h" +#include "common/vfsplugin.h" #include <QtCore/QLoggingCategory> #include <QtCore/QFile> @@ -73,7 +74,7 @@ int csync_vio_local_closedir(csync_vio_handle_t *dhandle) { return rc; } -std::unique_ptr<csync_file_stat_t> csync_vio_local_readdir(csync_vio_handle_t *handle) { +std::unique_ptr<csync_file_stat_t> csync_vio_local_readdir(CSYNC *ctx, csync_vio_handle_t *handle) { struct _tdirent *dirent = NULL; std::unique_ptr<csync_file_stat_t> file_stat; @@ -120,6 +121,11 @@ std::unique_ptr<csync_file_stat_t> csync_vio_local_readdir(csync_vio_handle_t *h // Will get excluded by _csync_detect_update. file_stat->type = ItemTypeSkip; } + + // Override type for virtual files if desired + if (ctx->vfs) + ctx->vfs->statTypeVirtualFile(file_stat.get(), nullptr); + return file_stat; } diff --git a/src/csync/vio/csync_vio_local_win.cpp b/src/csync/vio/csync_vio_local_win.cpp index 7eeab6827..5f094d280 100644 --- a/src/csync/vio/csync_vio_local_win.cpp +++ b/src/csync/vio/csync_vio_local_win.cpp @@ -38,6 +38,8 @@ #include <QtCore/QLoggingCategory> +#include "common/vfs.h" + Q_LOGGING_CATEGORY(lcCSyncVIOLocal, "nextcloud.sync.csync.vio_local", QtInfoMsg) /* @@ -113,7 +115,7 @@ static time_t FileTimeToUnixTime(FILETIME *filetime, DWORD *remainder) } } -std::unique_ptr<csync_file_stat_t> csync_vio_local_readdir(csync_vio_handle_t *handle) { +std::unique_ptr<csync_file_stat_t> csync_vio_local_readdir(CSYNC *ctx, csync_vio_handle_t *handle) { std::unique_ptr<csync_file_stat_t> file_stat; DWORD rem; @@ -137,12 +139,14 @@ std::unique_ptr<csync_file_stat_t> csync_vio_local_readdir(csync_vio_handle_t *h } auto path = c_utf8_from_locale(handle->ffd.cFileName); if (path == "." || path == "..") - return csync_vio_local_readdir(handle); + return csync_vio_local_readdir(ctx, handle); file_stat = std::make_unique<csync_file_stat_t>(); file_stat->path = path; - if (handle->ffd.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + if (ctx->vfs && ctx->vfs->statTypeVirtualFile(file_stat.get(), &handle->ffd)) { + // all good + } else if (handle->ffd.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { // Detect symlinks, and treat junctions as symlinks too. if (handle->ffd.dwReserved0 == IO_REPARSE_TAG_SYMLINK || handle->ffd.dwReserved0 == IO_REPARSE_TAG_MOUNT_POINT) { diff --git a/src/gui/accountsettings.cpp b/src/gui/accountsettings.cpp index 9f06e2d62..630a47f59 100644 --- a/src/gui/accountsettings.cpp +++ b/src/gui/accountsettings.cpp @@ -535,7 +535,11 @@ void AccountSettings::slotFolderWizardAccepted() folderWizard->field(QLatin1String("sourceFolder")).toString()); definition.targetPath = FolderDefinition::prepareTargetPath( folderWizard->property("targetPath").toString()); - definition.useVirtualFiles = folderWizard->property("useVirtualFiles").toBool(); + + if (folderWizard->property("useVirtualFiles").toBool()) { + // ### Determine which vfs is available? + definition.virtualFilesMode = Vfs::WindowsCfApi; + } { QDir dir(definition.localPath); diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index 6951c9b11..052b37c8e 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -32,7 +32,8 @@ #include "filesystem.h" #include "localdiscoverytracker.h" #include "csync_exclude.h" - +#include "common/vfs.h" +#include "plugin.h" #include "creds/abstractcredentials.h" #include <QTimer> @@ -42,6 +43,7 @@ #include <QMessageBox> #include <QPushButton> +#include <QApplication> static const char versionC[] = "version"; @@ -115,10 +117,53 @@ Folder::Folder(const FolderDefinition &definition, _localDiscoveryTracker.data(), &LocalDiscoveryTracker::slotSyncFinished); connect(_engine.data(), &SyncEngine::itemCompleted, _localDiscoveryTracker.data(), &LocalDiscoveryTracker::slotItemCompleted); + + // TODO cfapi: Move to function. Is this platform-universal? + PluginLoader pluginLoader; + if (_definition.virtualFilesMode == Vfs::WindowsCfApi) { + _vfs = pluginLoader.create<Vfs>("vfs", "win", this); + } + if (_definition.virtualFilesMode == Vfs::WithSuffix) { + _vfs = pluginLoader.create<Vfs>("vfs", "suffix", this); + + // Attempt to switch to winvfs mode? + if (_vfs && _definition.upgradeVfsMode) { + if (auto winvfs = pluginLoader.create<Vfs>("vfs", "win", this)) { + // Set "suffix" vfs options and wipe the existing suffix files + SyncEngine::wipeVirtualFiles(path(), _journal, _vfs); + + // Then switch to winvfs mode + _vfs = winvfs; + _definition.virtualFilesMode = Vfs::WindowsCfApi; + saveToSettings(); + } + } + } + if (!_vfs) { + // ### error handling; possibly earlier than in the ctor + qFatal("Could not load any vfs plugin."); + } + + VfsSetupParams vfsParams; + vfsParams.filesystemPath = path(); + vfsParams.remotePath = remotePath(); + vfsParams.account = _accountState->account(); + vfsParams.journal = &_journal; + vfsParams.providerName = Theme::instance()->appNameGUI(); + vfsParams.providerVersion = Theme::instance()->version(); + + connect(_vfs, &OCC::Vfs::beginHydrating, this, &Folder::slotHydrationStarts); + connect(_vfs, &OCC::Vfs::doneHydrating, this, &Folder::slotHydrationDone); + + _vfs->registerFolder(vfsParams); // Do this always? + _vfs->start(vfsParams); } Folder::~Folder() { + // TODO cfapi: unregister on wipe()? There should probably be a wipeForRemoval() where this cleanup is appropriate + _vfs->stop(); + // Reset then engine first as it will abort and try to access members of the Folder _engine.reset(); } @@ -218,12 +263,12 @@ QString Folder::cleanPath() const bool Folder::isBusy() const { - return _engine->isSyncRunning(); + return isSyncRunning(); } bool Folder::isSyncRunning() const { - return _engine->isSyncRunning(); + return _engine->isSyncRunning() || _vfs->isHydrating(); } QString Folder::remotePath() const @@ -551,7 +596,8 @@ void Folder::downloadVirtualFile(const QString &_relativepath) void Folder::setUseVirtualFiles(bool enabled) { - _definition.useVirtualFiles = enabled; + // ### must wipe virtual files, unload old plugin, load new one? + //_definition.useVirtualFiles = enabled; if (enabled) _saveInFoldersWithPlaceholders = true; saveToSettings(); @@ -571,7 +617,7 @@ void Folder::saveToSettings() const return other != this && other->cleanPath() == this->cleanPath(); }); - if (_definition.useVirtualFiles || _saveInFoldersWithPlaceholders) { + if (useVirtualFiles() || _saveInFoldersWithPlaceholders) { // If virtual files are enabled or even were enabled at some point, // save the folder to a group that will not be read by older (<2.5.0) clients. // The name is from when virtual files were called placeholders. @@ -739,8 +785,7 @@ void Folder::setSyncOptions() opt._newBigFolderSizeLimit = newFolderLimit.first ? newFolderLimit.second * 1000LL * 1000LL : -1; // convert from MB to B opt._confirmExternalStorage = cfgFile.confirmExternalStorage(); opt._moveFilesToTrash = cfgFile.moveToTrash(); - opt._newFilesAreVirtual = _definition.useVirtualFiles; - opt._virtualFileSuffix = QStringLiteral(APPLICATION_DOTVIRTUALFILE_SUFFIX); + opt._vfs = _vfs; QByteArray chunkSizeEnv = qgetenv("OWNCLOUD_CHUNK_SIZE"); if (!chunkSizeEnv.isEmpty()) { @@ -1046,6 +1091,30 @@ void Folder::slotWatcherUnreliable(const QString &message) Logger::instance()->postGuiLog(Theme::instance()->appNameGUI(), fullMessage); } +void Folder::slotHydrationStarts() +{ + // Abort any running full sync run and reschedule + if (_engine->isSyncRunning()) { + slotTerminateSync(); + scheduleThisFolderSoon(); + // TODO: This sets the sync state to AbortRequested on done, we don't want that + } + + // Let everyone know we're syncing + _syncResult.reset(); + _syncResult.setStatus(SyncResult::SyncRunning); + emit syncStarted(); + emit syncStateChange(); +} + +void Folder::slotHydrationDone() +{ + // emit signal to update ui and reschedule normal syncs if necessary + _syncResult.setStatus(SyncResult::Success); + emit syncFinished(_syncResult); + emit syncStateChange(); +} + void Folder::scheduleThisFolderSoon() { if (!_scheduleSelfTimer.isActive()) { @@ -1076,6 +1145,11 @@ void Folder::registerFolderWatcher() _folderWatcher->startNotificatonTest(path() + QLatin1String(".owncloudsync.log")); } +bool Folder::useVirtualFiles() const +{ + return _definition.virtualFilesMode != Vfs::Off; +} + void Folder::slotAboutToRemoveAllFiles(SyncFileItem::Direction dir, bool *cancel) { ConfigFile cfgFile; @@ -1117,9 +1191,11 @@ void FolderDefinition::save(QSettings &settings, const FolderDefinition &folder) settings.setValue(QLatin1String("targetPath"), folder.targetPath); settings.setValue(QLatin1String("paused"), folder.paused); settings.setValue(QLatin1String("ignoreHiddenFiles"), folder.ignoreHiddenFiles); - settings.setValue(QLatin1String("usePlaceholders"), folder.useVirtualFiles); settings.setValue(QLatin1String(versionC), maxSettingsVersion()); + settings.setValue(QStringLiteral("virtualFilesMode"), Vfs::modeToString(folder.virtualFilesMode)); + settings.remove(QLatin1String("usePlaceholders")); // deprecated key + // Happens only on Windows when the explorer integration is enabled. if (!folder.navigationPaneClsid.isNull()) settings.setValue(QLatin1String("navigationPaneClsid"), folder.navigationPaneClsid); @@ -1139,7 +1215,18 @@ bool FolderDefinition::load(QSettings &settings, const QString &alias, folder->paused = settings.value(QLatin1String("paused")).toBool(); folder->ignoreHiddenFiles = settings.value(QLatin1String("ignoreHiddenFiles"), QVariant(true)).toBool(); folder->navigationPaneClsid = settings.value(QLatin1String("navigationPaneClsid")).toUuid(); - folder->useVirtualFiles = settings.value(QLatin1String("usePlaceholders")).toBool(); + + folder->virtualFilesMode = Vfs::Off; + QString vfsModeString = settings.value(QStringLiteral("virtualFilesMode")).toString(); + if (!vfsModeString.isEmpty()) { + if (!Vfs::modeFromString(vfsModeString, &folder->virtualFilesMode)) { + qCWarning(lcFolder) << "Unknown virtualFilesMode:" << vfsModeString << "assuming 'off'"; + } + } else if (settings.value(QLatin1String("usePlaceholders")).toBool()) { + folder->virtualFilesMode = Vfs::WithSuffix; + folder->upgradeVfsMode = true; + } + settings.endGroup(); // Old settings can contain paths with native separators. In the rest of the diff --git a/src/gui/folder.h b/src/gui/folder.h index a5b0641c9..b266ed402 100644 --- a/src/gui/folder.h +++ b/src/gui/folder.h @@ -21,6 +21,7 @@ #include "progressdispatcher.h" #include "common/syncjournaldb.h" #include "networkjobs.h" +#include "syncoptions.h" #include <QObject> #include <QStringList> @@ -33,6 +34,7 @@ class QSettings; namespace OCC { +class Vfs; class SyncEngine; class AccountState; class SyncRunFileLog; @@ -58,11 +60,15 @@ public: bool paused = false; /// whether the folder syncs hidden files bool ignoreHiddenFiles = false; - /// New files are downloaded as virtual files - bool useVirtualFiles = false; + /// Which virtual files setting the folder uses + Vfs::Mode virtualFilesMode = Vfs::Off; /// The CLSID where this folder appears in registry for the Explorer navigation pane entry. QUuid navigationPaneClsid; + /// Whether this suffix-vfs should be migrated to a better + /// vfs plugin if possible + bool upgradeVfsMode = false; + /// Saves the folder definition, creating a new settings group. static void save(QSettings &settings, const FolderDefinition &folder); @@ -173,7 +179,7 @@ public: SyncResult syncResult() const; /** - * This is called if the sync folder definition is removed. Do cleanups here. + * This is called when the sync folder definition is removed. Do cleanups here. */ virtual void wipe(); @@ -240,8 +246,8 @@ public: */ void registerFolderWatcher(); - /** new files are downloaded as virtual files */ - bool useVirtualFiles() { return _definition.useVirtualFiles; } + /** virtual files of some kind are enabled */ + bool useVirtualFiles() const; void setUseVirtualFiles(bool enabled); signals: @@ -336,7 +342,19 @@ private slots: /** Warn users about an unreliable folder watcher */ void slotWatcherUnreliable(const QString &message); + /** Aborts any running sync and blocks it until hydration is finished. + * + * Hydration circumvents the regular SyncEngine and both mustn't be running + * at the same time. + */ + void slotHydrationStarts(); + + /** Unblocks normal sync operation */ + void slotHydrationDone(); + private: + void connectSyncRoot(); + bool reloadExcludes(); void showSyncResultPopup(); @@ -416,6 +434,11 @@ private: * Keeps track of locally dirty files so we can skip local discovery sometimes. */ QScopedPointer<LocalDiscoveryTracker> _localDiscoveryTracker; + + /** + * The vfs mode instance (created by plugin) to use. Null means no vfs. + */ + Vfs *_vfs = nullptr; }; } diff --git a/src/gui/folderman.cpp b/src/gui/folderman.cpp index 63e971c0f..43ceae787 100644 --- a/src/gui/folderman.cpp +++ b/src/gui/folderman.cpp @@ -29,9 +29,6 @@ #ifdef Q_OS_MAC #include <CoreServices/CoreServices.h> #endif -#ifdef Q_OS_WIN -#include <shlobj.h> -#endif #include <QMessageBox> #include <QtCore> @@ -1003,7 +1000,6 @@ Folder *FolderMan::addFolder(AccountState *accountState, const FolderDefinition } _navigationPaneHelper.scheduleUpdateCloudStorageRegistry(); - return folder; } diff --git a/src/gui/owncloudsetupwizard.cpp b/src/gui/owncloudsetupwizard.cpp index 0af0094b5..d57c7e1a1 100644 --- a/src/gui/owncloudsetupwizard.cpp +++ b/src/gui/owncloudsetupwizard.cpp @@ -635,7 +635,10 @@ void OwncloudSetupWizard::slotAssistantFinished(int result) folderDefinition.localPath = localFolder; folderDefinition.targetPath = FolderDefinition::prepareTargetPath(_remoteFolder); folderDefinition.ignoreHiddenFiles = folderMan->ignoreHiddenFiles(); - folderDefinition.useVirtualFiles = _ocWizard->useVirtualFileSync(); + if (_ocWizard->useVirtualFileSync()) { + // ### determine best vfs mode! + folderDefinition.virtualFilesMode = Vfs::WindowsCfApi; + } if (folderMan->navigationPaneHelper().showInExplorerNavigationPane()) folderDefinition.navigationPaneClsid = QUuid::createUuid(); diff --git a/src/gui/socketapi.cpp b/src/gui/socketapi.cpp index 22e88e05e..4952a133f 100644 --- a/src/gui/socketapi.cpp +++ b/src/gui/socketapi.cpp @@ -689,52 +689,51 @@ void SocketApi::command_OPEN_PRIVATE_LINK(const QString &localFile, SocketListen void SocketApi::command_DOWNLOAD_VIRTUAL_FILE(const QString &filesArg, SocketListener *) { QStringList files = filesArg.split(QLatin1Char('\x1e')); // Record Separator - auto suffix = QStringLiteral(APPLICATION_DOTVIRTUALFILE_SUFFIX); for (const auto &file : files) { - if (!file.endsWith(suffix) && !QFileInfo(file).isDir()) + auto data = FileData::get(file); + auto record = data.journalRecord(); + if (!record.isValid()) continue; - auto folder = FolderMan::instance()->folderForPath(file); - if (folder) { - QString relativePath = QDir::cleanPath(file).mid(folder->cleanPath().length() + 1); - folder->downloadVirtualFile(relativePath); - } + if (record._type != ItemTypeVirtualFile && !QFileInfo(file).isDir()) + continue; + if (data.folder) + data.folder->downloadVirtualFile(data.folderRelativePath); } } -/* Go over all the files ans replace them by a virtual file */ +/* Go over all the files and replace them by a virtual file */ void SocketApi::command_REPLACE_VIRTUAL_FILE(const QString &filesArg, SocketListener *) { QStringList files = filesArg.split(QLatin1Char('\x1e')); // Record Separator - auto suffix = QStringLiteral(APPLICATION_DOTVIRTUALFILE_SUFFIX); + QSet<Folder *> toSync; for (const auto &file : files) { - auto folder = FolderMan::instance()->folderForPath(file); - if (!folder) + auto data = FileData::get(file); + if (!data.folder) continue; - if (file.endsWith(suffix)) - continue; - QString relativePath = QDir::cleanPath(file).mid(folder->cleanPath().length() + 1); + auto journal = data.folder->journalDb(); + auto markForDehydration = [&](SyncJournalFileRecord rec) { + if (rec._type != ItemTypeFile) + return; + rec._type = ItemTypeVirtualFileDehydration; + journal->setFileRecord(rec); + toSync.insert(data.folder); + }; + QFileInfo fi(file); if (fi.isDir()) { - folder->journalDb()->getFilesBelowPath(relativePath.toUtf8(), [&](const SyncJournalFileRecord &rec) { - if (rec._type != ItemTypeFile || rec._path.endsWith(APPLICATION_DOTVIRTUALFILE_SUFFIX)) - return; - QString file = folder->path() + '/' + QString::fromUtf8(rec._path); - if (!FileSystem::rename(file, file + suffix)) { - qCWarning(lcSocketApi) << "Unable to rename " << file; - } - }); + journal->getFilesBelowPath(data.folderRelativePath.toUtf8(), markForDehydration); continue; } - SyncJournalFileRecord record; - if (!folder->journalDb()->getFileRecord(relativePath, &record) || !record.isValid()) + auto record = data.journalRecord(); + if (!record.isValid() || record._type != ItemTypeFile) continue; - if (!FileSystem::rename(file, file + suffix)) { - qCWarning(lcSocketApi) << "Unable to rename " << file; - } - FolderMan::instance()->scheduleFolder(folder); + markForDehydration(record); } + + for (const auto folder : toSync) + FolderMan::instance()->scheduleFolder(folder); } void SocketApi::copyUrlToClipboard(const QString &link) @@ -1012,18 +1011,18 @@ void SocketApi::command_GET_MENU_ITEMS(const QString &argument, OCC::SocketListe // Virtual file download action if (syncFolder) { - auto virtualFileSuffix = QStringLiteral(APPLICATION_DOTVIRTUALFILE_SUFFIX); bool hasVirtualFile = false; bool hasNormalFiles = false; bool hasDir = false; for (const auto &file : files) { if (QFileInfo(file).isDir()) { hasDir = true; - } else if (file.endsWith(virtualFileSuffix)) { - hasVirtualFile = true; - } else if (!hasNormalFiles) { - bool isOnTheServer = FileData::get(file).journalRecord().isValid(); - hasNormalFiles = isOnTheServer; + } else if (!hasVirtualFile || !hasNormalFiles) { + auto record = FileData::get(file).journalRecord(); + if (record.isValid()) { + hasVirtualFile |= record._type == ItemTypeVirtualFile; + hasNormalFiles |= record._type == ItemTypeFile; + } } } if (hasVirtualFile || (hasDir && syncFolder->useVirtualFiles())) diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index cf744fe1e..bea212d10 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -33,6 +33,7 @@ set(libsync_SRCS networkjobs.cpp owncloudpropagator.cpp nextcloudtheme.cpp + plugin.cpp progressdispatcher.cpp propagatorjobs.cpp propagatedownload.cpp @@ -140,3 +141,4 @@ else() endif() +add_subdirectory(vfs) diff --git a/src/libsync/discoveryphase.cpp b/src/libsync/discoveryphase.cpp index 17ec8fee2..8d3b7c515 100644 --- a/src/libsync/discoveryphase.cpp +++ b/src/libsync/discoveryphase.cpp @@ -77,7 +77,7 @@ bool DiscoveryPhase::isInSelectiveSyncBlackList(const QString &path) const void DiscoveryPhase::checkSelectiveSyncNewFolder(const QString &path, RemotePermissions remotePerm, std::function<void(bool)> callback) { - if (_syncOptions._confirmExternalStorage && !_syncOptions._newFilesAreVirtual + if (_syncOptions._confirmExternalStorage && !_syncOptions._vfs && remotePerm.hasPermission(RemotePermissions::IsMounted)) { // external storage. @@ -100,7 +100,7 @@ void DiscoveryPhase::checkSelectiveSyncNewFolder(const QString &path, RemotePerm } auto limit = _syncOptions._newBigFolderSizeLimit; - if (limit < 0 || _syncOptions._newFilesAreVirtual) { + if (limit < 0 || !_syncOptions._vfs) { // no limit, everything is allowed; return callback(false); } diff --git a/src/libsync/filesystem.cpp b/src/libsync/filesystem.cpp index c4877bdb6..fcf0cc0d7 100644 --- a/src/libsync/filesystem.cpp +++ b/src/libsync/filesystem.cpp @@ -189,5 +189,15 @@ bool FileSystem::removeRecursively(const QString &path, const std::function<void return allRemoved; } +bool FileSystem::getInode(const QString &filename, quint64 *inode) +{ + csync_file_stat_t fs; + if (csync_vio_local_stat(filename.toUtf8().constData(), &fs) == 0) { + *inode = fs.inode; + return true; + } + return false; +} + } // namespace OCC diff --git a/src/libsync/filesystem.h b/src/libsync/filesystem.h index 70dffef42..461028b15 100644 --- a/src/libsync/filesystem.h +++ b/src/libsync/filesystem.h @@ -63,6 +63,11 @@ namespace FileSystem { */ qint64 OWNCLOUDSYNC_EXPORT getSize(const QString &filename); + /** + * @brief Retrieve a file inode with csync + */ + bool OWNCLOUDSYNC_EXPORT getInode(const QString &filename, quint64 *inode); + /** * @brief Check if \a fileName has changed given previous size and mtime * diff --git a/src/libsync/owncloudpropagator.cpp b/src/libsync/owncloudpropagator.cpp index 57cc4f1e9..951d188ed 100644 --- a/src/libsync/owncloudpropagator.cpp +++ b/src/libsync/owncloudpropagator.cpp @@ -584,11 +584,6 @@ QString OwncloudPropagator::getFilePath(const QString &tmp_file_name) const return _localDir + tmp_file_name; } -QString OwncloudPropagator::addVirtualFileSuffix(const QString &fileName) const -{ - return fileName + _syncOptions._virtualFileSuffix; -} - void OwncloudPropagator::scheduleNextJob() { QTimer::singleShot(0, this, &OwncloudPropagator::scheduleNextJobImpl); diff --git a/src/libsync/owncloudpropagator.h b/src/libsync/owncloudpropagator.h index dae966a53..27a299682 100644 --- a/src/libsync/owncloudpropagator.h +++ b/src/libsync/owncloudpropagator.h @@ -449,7 +449,6 @@ public: /* returns the local file path for the given tmp_file_name */ QString getFilePath(const QString &tmp_file_name) const; - QString addVirtualFileSuffix(const QString &fileName) const; /** Creates the job for an item. */ diff --git a/src/libsync/plugin.cpp b/src/libsync/plugin.cpp new file mode 100644 index 000000000..f3c4c857d --- /dev/null +++ b/src/libsync/plugin.cpp @@ -0,0 +1,68 @@ +/* + * Copyright (C) by Dominik Schmidt <dschmidt@owncloud.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "plugin.h" + +#include "config.h" +#include "logger.h" + +#include <QPluginLoader> +#include <QDir> + +Q_LOGGING_CATEGORY(lcPluginLoader, "pluginLoader", QtInfoMsg) + +namespace OCC { + +PluginFactory::~PluginFactory() = default; + +QObject *PluginLoader::createInternal(const QString& type, const QString &name, QObject* parent) +{ + auto factory = load<PluginFactory>(type, name); + if (!factory) { + return nullptr; + } else { + return factory->create(parent); + } +} + +QString PluginLoader::pluginName(const QString &type, const QString &name) +{ + return QString(QLatin1String("%1sync_%2_%3")) + .arg(APPLICATION_EXECUTABLE) + .arg(type) + .arg(name); +} + +QObject *PluginLoader::loadPluginInternal(const QString& type, const QString &name) +{ + QString fileName = pluginName(type, name); + QPluginLoader pluginLoader(fileName); + auto plugin = pluginLoader.load(); + if(plugin) { + qCInfo(lcPluginLoader) << "Loaded plugin" << fileName; + } else { + qCWarning(lcPluginLoader) << "Could not load plugin" + << fileName <<":" + << pluginLoader.errorString() + << "from" << QDir::currentPath(); + } + + return pluginLoader.instance(); +} + +} diff --git a/src/libsync/plugin.h b/src/libsync/plugin.h new file mode 100644 index 000000000..e8f8aec44 --- /dev/null +++ b/src/libsync/plugin.h @@ -0,0 +1,68 @@ +/* + * Copyright (C) by Dominik Schmidt <dschmidt@owncloud.com> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#pragma once + +#include "owncloudlib.h" +#include <QObject> +#include <QPluginLoader> + +namespace OCC { + +class OWNCLOUDSYNC_EXPORT PluginFactory +{ +public: + ~PluginFactory(); + virtual QObject* create(QObject* parent) = 0; +}; + +template<class PLUGIN_CLASS> +class DefaultPluginFactory : public PluginFactory +{ +public: + QObject* create(QObject* parent) override + { + return new PLUGIN_CLASS(parent); + } +}; + +class OWNCLOUDSYNC_EXPORT PluginLoader +{ +public: + static QString pluginName(const QString &type, const QString &name); + + template<class PLUGIN_CLASS, typename ... Args> + PLUGIN_CLASS *create(Args&& ... args) + { + return qobject_cast<PLUGIN_CLASS*>(createInternal(std::forward<Args>(args)...)); + } + +private: + template<class FACTORY_CLASS, typename ... Args> + FACTORY_CLASS *load(Args&& ... args) + { + return qobject_cast<FACTORY_CLASS*>(loadPluginInternal(std::forward<Args>(args)...)); + } + + QObject *loadPluginInternal(const QString& type, const QString &name); + QObject *createInternal(const QString& type, const QString &name, QObject* parent = nullptr); +}; + +} + +Q_DECLARE_INTERFACE(OCC::PluginFactory, "org.owncloud.PluginFactory") diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp index 05a4c89bf..1573f5e69 100644 --- a/src/libsync/propagatedownload.cpp +++ b/src/libsync/propagatedownload.cpp @@ -26,6 +26,7 @@ #include "common/asserts.h" #include "clientsideencryptionjobs.h" #include "propagatedownloadencrypted.h" +#include "common/vfs.h" #include <QLoggingCategory> #include <QNetworkAccessManager> @@ -68,13 +69,15 @@ QString OWNCLOUDSYNC_EXPORT createDownloadTmpFileName(const QString &previous) } // DOES NOT take ownership of the device. -GETFileJob::GETFileJob(AccountPtr account, const QString &path, QFile *device, +GETFileJob::GETFileJob(AccountPtr account, const QString &path, QIODevice *device, const QMap<QByteArray, QByteArray> &headers, const QByteArray &expectedEtagForResume, quint64 resumeStart, QObject *parent) : AbstractNetworkJob(account, path, parent) , _device(device) , _headers(headers) , _expectedEtagForResume(expectedEtagForResume) + , _expectedContentLength(-1) + , _contentLength(0) , _resumeStart(resumeStart) , _errorStatus(SyncFileItem::NoStatus) , _bandwidthLimited(false) @@ -86,7 +89,7 @@ GETFileJob::GETFileJob(AccountPtr account, const QString &path, QFile *device, { } -GETFileJob::GETFileJob(AccountPtr account, const QUrl &url, QFile *device, +GETFileJob::GETFileJob(AccountPtr account, const QUrl &url, QIODevice *device, const QMap<QByteArray, QByteArray> &headers, const QByteArray &expectedEtagForResume, quint64 resumeStart, QObject *parent) @@ -94,6 +97,8 @@ GETFileJob::GETFileJob(AccountPtr account, const QUrl &url, QFile *device, , _device(device) , _headers(headers) , _expectedEtagForResume(expectedEtagForResume) + , _expectedContentLength(-1) + , _contentLength(0) , _resumeStart(resumeStart) , _errorStatus(SyncFileItem::NoStatus) , _directDownloadUrl(url) @@ -201,6 +206,16 @@ void GETFileJob::slotMetaDataChanged() return; } + _contentLength = reply()->header(QNetworkRequest::ContentLengthHeader).toLongLong(); + if (_expectedContentLength != -1 && _contentLength != _expectedContentLength) { + qCWarning(lcGetJob) << "We received a different content length than expected!" + << _expectedContentLength << "vs" << _contentLength; + _errorString = tr("We received an unexpected download Content-Length."); + _errorStatus = SyncFileItem::NormalError; + reply()->abort(); + return; + } + quint64 start = 0; QByteArray ranges = reply()->rawHeader("Content-Range"); if (!ranges.isEmpty()) { @@ -399,17 +414,30 @@ void PropagateDownloadFile::startAfterIsEncryptedIsChecked() { _stopwatch.start(); + auto &syncOptions = propagator()->syncOptions(); + auto vfs = syncOptions._vfs; + // For virtual files just create the file and be done + if (_item->_type == ItemTypeVirtualFileDehydration) { + _item->_type = ItemTypeVirtualFile; + // TODO: Could dehydrate without wiping the file entirely + auto fn = propagator()->getFilePath(_item->_file); + qCDebug(lcPropagateDownload) << "dehydration: wiping base file" << fn; + QFile::remove(fn); + propagator()->_journal->deleteFileRecord(_item->_file); + + if (vfs && vfs->mode() == Vfs::WithSuffix) { + // Normally new suffix-virtual files already have the suffix included in the path + // but for dehydrations that isn't the case. Adjust it here. + _item->_file.append(vfs->fileSuffix()); + } + } if (_item->_type == ItemTypeVirtualFile) { auto fn = propagator()->getFilePath(_item->_file); qCDebug(lcPropagateDownload) << "creating virtual file" << fn; - // NOTE: Other places might depend on contents of placeholder files (like csync_update) - QFile file(fn); - file.open(QFile::ReadWrite | QFile::Truncate); - file.write(" "); - file.close(); - FileSystem::setModTime(fn, _item->_modtime); + ASSERT(vfs); + vfs->createPlaceholder(propagator()->_localDir, _item); updateMetadata(false); return; } @@ -623,7 +651,7 @@ void PropagateDownloadFile::slotGetFinished() // Don't keep the temporary file if it is empty or we // used a bad range header or the file's not on the server anymore. - if (_tmpFile.size() == 0 || badRangeHeader || fileNotFound) { + if (_tmpFile.exists() && (_tmpFile.size() == 0 || badRangeHeader || fileNotFound)) { _tmpFile.close(); FileSystem::remove(_tmpFile.fileName()); propagator()->_journal->setDownloadInfo(_item->_file, SyncJournalDb::DownloadInfo()); @@ -957,6 +985,12 @@ void PropagateDownloadFile::downloadFinished() done(SyncFileItem::SoftError, error); return; } + + // Make the file a hydrated placeholder if possible + if (auto vfs = propagator()->syncOptions()._vfs) { + vfs->convertToPlaceholder(fn, _item); + } + FileSystem::setFileHidden(fn, false); // Maybe we downloaded a newer version of the file than we thought we would... @@ -968,15 +1002,20 @@ void PropagateDownloadFile::downloadFinished() if (_conflictRecord.isValid()) propagator()->_journal->setConflictRecord(_conflictRecord); - // If we downloaded something that used to be a virtual file, - // wipe the virtual file and its db entry now that we're done. if (_item->_type == ItemTypeVirtualFileDownload) { - auto virtualFile = propagator()->addVirtualFileSuffix(_item->_file); - auto fn = propagator()->getFilePath(virtualFile); - qCDebug(lcPropagateDownload) << "Download of previous virtual file finished" << fn; - QFile::remove(fn); - propagator()->_journal->deleteFileRecord(virtualFile); + // A downloaded virtual file becomes normal _item->_type = ItemTypeFile; + + // If the virtual file used to have a different name and db + // entry, wipe both now. + auto vfs = propagator()->syncOptions()._vfs; + if (vfs && vfs->mode() == Vfs::WithSuffix) { + QString virtualFile = _item->_file + vfs->fileSuffix(); + auto fn = propagator()->getFilePath(virtualFile); + qCDebug(lcPropagateDownload) << "Download of previous virtual file finished" << fn; + QFile::remove(fn); + propagator()->_journal->deleteFileRecord(virtualFile); + } } updateMetadata(isConflict); diff --git a/src/libsync/propagatedownload.h b/src/libsync/propagatedownload.h index 856656ddd..cd76a2a6a 100644 --- a/src/libsync/propagatedownload.h +++ b/src/libsync/propagatedownload.h @@ -30,10 +30,12 @@ class PropagateDownloadEncrypted; class GETFileJob : public AbstractNetworkJob { Q_OBJECT - QFile *_device; + QIODevice *_device; QMap<QByteArray, QByteArray> _headers; QString _errorString; QByteArray _expectedEtagForResume; + qint64 _expectedContentLength; + quint64 _contentLength; quint64 _resumeStart; SyncFileItem::Status _errorStatus; QUrl _directDownloadUrl; @@ -50,11 +52,11 @@ class GETFileJob : public AbstractNetworkJob public: // DOES NOT take ownership of the device. - explicit GETFileJob(AccountPtr account, const QString &path, QFile *device, + explicit GETFileJob(AccountPtr account, const QString &path, QIODevice *device, const QMap<QByteArray, QByteArray> &headers, const QByteArray &expectedEtagForResume, quint64 resumeStart, QObject *parent = nullptr); // For directDownloadUrl: - explicit GETFileJob(AccountPtr account, const QUrl &url, QFile *device, + explicit GETFileJob(AccountPtr account, const QUrl &url, QIODevice *device, const QMap<QByteArray, QByteArray> &headers, const QByteArray &expectedEtagForResume, quint64 resumeStart, QObject *parent = nullptr); virtual ~GETFileJob() @@ -101,6 +103,9 @@ public: quint64 resumeStart() { return _resumeStart; } time_t lastModified() { return _lastModified; } + quint64 contentLength() const { return _contentLength; } + qint64 expectedContentLength() const { return _expectedContentLength; } + void setExpectedContentLength(qint64 size) { _expectedContentLength = size; } signals: void finishedSignal(); diff --git a/src/libsync/propagateremotemove.cpp b/src/libsync/propagateremotemove.cpp index 9da698618..e9c4edc40 100644 --- a/src/libsync/propagateremotemove.cpp +++ b/src/libsync/propagateremotemove.cpp @@ -90,8 +90,10 @@ void PropagateRemoteMove::start() QString source = propagator()->_remoteFolder + _item->_file; QString destination = QDir::cleanPath(propagator()->account()->davUrl().path() + propagator()->_remoteFolder + _item->_renameTarget); - if (_item->_type == ItemTypeVirtualFile || _item->_type == ItemTypeVirtualFileDownload) { - auto suffix = propagator()->syncOptions()._virtualFileSuffix; + auto vfs = propagator()->syncOptions()._vfs; + if (vfs && vfs->mode() == Vfs::WithSuffix + && (_item->_type == ItemTypeVirtualFile || _item->_type == ItemTypeVirtualFileDownload)) { + const auto suffix = vfs->fileSuffix(); ASSERT(source.endsWith(suffix) && destination.endsWith(suffix)); if (source.endsWith(suffix) && destination.endsWith(suffix)) { source.chop(suffix.size()); @@ -162,6 +164,7 @@ void PropagateRemoteMove::finalize() record._path = _item->_renameTarget.toUtf8(); if (oldRecord.isValid()) { record._checksumHeader = oldRecord._checksumHeader; + record._type = oldRecord._type; if (record._fileSize != oldRecord._fileSize) { qCWarning(lcPropagateRemoteMove) << "File sizes differ on server vs sync journal: " << record._fileSize << oldRecord._fileSize; diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index df640af1c..e48235567 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -28,6 +28,7 @@ #include "common/asserts.h" #include "configfile.h" #include "discovery.h" +#include "common/vfs.h" #ifdef Q_OS_WIN #include <windows.h> @@ -332,6 +333,8 @@ void OCC::SyncEngine::slotItemDiscovered(const OCC::SyncFileItemPtr &item) rec._serverHasIgnoredFiles |= prev._serverHasIgnoredFiles; _journal->setFileRecord(rec); + // ### Update vfs metadata with Vfs::updateMetadata() + // This might have changed the shared flag, so we must notify SyncFileStatusTracker for example emit itemCompleted(item); } else { @@ -469,10 +472,12 @@ void SyncEngine::startSync() _lastLocalDiscoveryStyle = _localDiscoveryStyle; - if (_syncOptions._newFilesAreVirtual && _syncOptions._virtualFileSuffix.isEmpty()) { - syncError(tr("Using virtual files but suffix is not set")); - finalize(false); - return; + if (_syncOptions._vfs && _syncOptions._vfs->mode() == Vfs::WithSuffix) { + if (_syncOptions._vfs->fileSuffix().isEmpty()) { + syncError(tr("Using virtual files with suffix, but suffix is not set")); + finalize(false); + return; + } } // If needed, make sure we have up to date E2E information before the @@ -985,6 +990,30 @@ bool SyncEngine::shouldDiscoverLocally(const QString &path) const return false; } +void SyncEngine::wipeVirtualFiles(const QString &localPath, SyncJournalDb &journal, Vfs *vfs) +{ + qCInfo(lcEngine) << "Wiping virtual files inside" << localPath; + journal.getFilesBelowPath(QByteArray(), [&](const SyncJournalFileRecord &rec) { + if (rec._type != ItemTypeVirtualFile && rec._type != ItemTypeVirtualFileDownload) + return; + + qCDebug(lcEngine) << "Removing db record for" << rec._path; + journal.deleteFileRecord(rec._path); + + // If the local file is a dehydrated placeholder, wipe it too. + // Otherwise leave it to allow the next sync to have a new-new conflict. + QString localFile = localPath + rec._path; + if (QFile::exists(localFile) && vfs && vfs->isDehydratedPlaceholder(localFile)) { + qCDebug(lcEngine) << "Removing local dehydrated placeholder" << rec._path; + QFile::remove(localFile); + } + }); + + journal.forceRemoteDiscoveryNextSync(); + + // Postcondition: No ItemTypeVirtualFile / ItemTypeVirtualFileDownload left in the db +} + void SyncEngine::abort() { if (_propagator) diff --git a/src/libsync/syncengine.h b/src/libsync/syncengine.h index 3715d1c0f..5fd961a37 100644 --- a/src/libsync/syncengine.h +++ b/src/libsync/syncengine.h @@ -119,6 +119,13 @@ public: /** Access the last sync run's local discovery style */ LocalDiscoveryStyle lastLocalDiscoveryStyle() const { return _lastLocalDiscoveryStyle; } + /** Removes all virtual file db entries and dehydrated local placeholders. + * + * Particularly useful when switching off vfs mode or switching to a + * different kind of vfs. + */ + static void wipeVirtualFiles(const QString &localPath, SyncJournalDb &journal, Vfs *vfs); + auto getPropagator() { return _propagator; } // for the test signals: diff --git a/src/libsync/syncfileitem.cpp b/src/libsync/syncfileitem.cpp index fab9d72f1..623d61859 100644 --- a/src/libsync/syncfileitem.cpp +++ b/src/libsync/syncfileitem.cpp @@ -15,6 +15,7 @@ #include "syncfileitem.h" #include "common/syncjournalfilerecord.h" #include "common/utility.h" +#include "filesystem.h" #include <QLoggingCategory> #include "csync/vio/csync_vio_local.h" @@ -37,17 +38,15 @@ SyncJournalFileRecord SyncFileItem::toSyncJournalFileRecordWithInode(const QStri rec._checksumHeader = _checksumHeader; rec._e2eMangledName = _encryptedFileName.toUtf8(); - // Go through csync vio just to get the inode. - csync_file_stat_t fs; - if (csync_vio_local_stat(localFileName.toUtf8().constData(), &fs) == 0) { - rec._inode = fs.inode; - qCDebug(lcFileItem) << localFileName << "Retrieved inode " << _inode << "(previous item inode: " << _inode << ")"; + // Update the inode if possible + rec._inode = _inode; + if (FileSystem::getInode(localFileName, &rec._inode)) { + qCDebug(lcFileItem) << localFileName << "Retrieved inode " << rec._inode << "(previous item inode: " << _inode << ")"; } else { // use the "old" inode coming with the item for the case where the // filesystem stat fails. That can happen if the the file was removed // or renamed meanwhile. For the rename case we still need the inode to // detect the rename though. - rec._inode = _inode; qCWarning(lcFileItem) << "Failed to query the 'inode' for file " << localFileName; } return rec; diff --git a/src/libsync/syncoptions.h b/src/libsync/syncoptions.h index 3d74b8e51..7e09526c5 100644 --- a/src/libsync/syncoptions.h +++ b/src/libsync/syncoptions.h @@ -17,7 +17,7 @@ #include "owncloudlib.h" #include <QString> #include <chrono> - +#include "common/vfs.h" namespace OCC { @@ -37,8 +37,7 @@ struct SyncOptions bool _moveFilesToTrash = false; /** Create a virtual file for new files instead of downloading */ - bool _newFilesAreVirtual = false; - QString _virtualFileSuffix = ".owncloud"; + Vfs *_vfs = nullptr; /** The initial un-adjusted chunk size in bytes for chunked uploads, both * for old and new chunking algorithm, which classifies the item to be chunked diff --git a/src/libsync/vfs/CMakeLists.txt b/src/libsync/vfs/CMakeLists.txt new file mode 100644 index 000000000..773000f40 --- /dev/null +++ b/src/libsync/vfs/CMakeLists.txt @@ -0,0 +1,12 @@ +### TODO: Find plugins dynamically +list(APPEND vfsPlugins "suffix") + +foreach(vfsPlugin ${vfsPlugins}) + message(STATUS "Add vfsPlugin in dir: ${vfsPlugin}") + add_subdirectory("${vfsPlugin}") + + if(UNIT_TESTING AND IS_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}/${vfsPlugin}/test") + message(STATUS "Add vfsPlugin tests in dir: ${vfsPlugin}") + add_subdirectory("${vfsPlugin}/test" "${vfsPlugin}_test") + endif() +endforeach() diff --git a/src/libsync/vfs/suffix/CMakeLists.txt b/src/libsync/vfs/suffix/CMakeLists.txt new file mode 100644 index 000000000..f77e0934f --- /dev/null +++ b/src/libsync/vfs/suffix/CMakeLists.txt @@ -0,0 +1,15 @@ +add_library("${synclib_NAME}_vfs_suffix" SHARED + vfs_suffix.cpp +) + +target_link_libraries("${synclib_NAME}_vfs_suffix" + "${synclib_NAME}" +) + +set_target_properties("${synclib_NAME}_vfs_suffix" PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${BIN_OUTPUT_DIRECTORY} + RUNTIME_OUTPUT_DIRECTORY ${BIN_OUTPUT_DIRECTORY} + PREFIX "" + AUTOMOC TRUE +) + diff --git a/src/libsync/vfs/suffix/vfs_suffix.cpp b/src/libsync/vfs/suffix/vfs_suffix.cpp new file mode 100644 index 000000000..416486189 --- /dev/null +++ b/src/libsync/vfs/suffix/vfs_suffix.cpp @@ -0,0 +1,108 @@ +/* + * Copyright (C) by Christian Kamm <mail@ckamm.de> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +#include "vfs_suffix.h" + +#include <QFile> + +#include "syncfileitem.h" +#include "filesystem.h" + +namespace OCC { + +class VfsSuffixPrivate +{ +}; + +VfsSuffix::VfsSuffix(QObject *parent) + : Vfs(parent) + , d_ptr(new VfsSuffixPrivate) +{ +} + +VfsSuffix::~VfsSuffix() +{ +} + +Vfs::Mode VfsSuffix::mode() const +{ + return WithSuffix; +} + +QString VfsSuffix::fileSuffix() const +{ + return QStringLiteral(APPLICATION_DOTVIRTUALFILE_SUFFIX); +} + +void VfsSuffix::registerFolder(const VfsSetupParams &) +{ +} + +void VfsSuffix::start(const VfsSetupParams &) +{ +} + +void VfsSuffix::stop() +{ +} + +void VfsSuffix::unregisterFolder() +{ +} + +bool VfsSuffix::isHydrating() const +{ + return false; +} + +bool VfsSuffix::updateMetadata(const QString &filePath, time_t modtime, quint64, const QByteArray &, QString *) +{ + FileSystem::setModTime(filePath, modtime); + return true; +} + +void VfsSuffix::createPlaceholder(const QString &syncFolder, const SyncFileItemPtr &item) +{ + // NOTE: Other places might depend on contents of placeholder files (like csync_update) + QString fn = syncFolder + item->_file; + QFile file(fn); + file.open(QFile::ReadWrite | QFile::Truncate); + file.write(" "); + file.close(); + FileSystem::setModTime(fn, item->_modtime); +} + +void VfsSuffix::convertToPlaceholder(const QString &, const SyncFileItemPtr &) +{ + // Nothing necessary +} + +bool VfsSuffix::isDehydratedPlaceholder(const QString &filePath) +{ + if (!filePath.endsWith(fileSuffix())) + return false; + QFileInfo fi(filePath); + return fi.exists() && fi.size() == 1; +} + +bool VfsSuffix::statTypeVirtualFile(csync_file_stat_t *stat, void *) +{ + if (stat->path.endsWith(fileSuffix().toUtf8())) { + stat->type = ItemTypeVirtualFile; + return true; + } + return false; +} + +} // namespace OCC diff --git a/src/libsync/vfs/suffix/vfs_suffix.h b/src/libsync/vfs/suffix/vfs_suffix.h new file mode 100644 index 000000000..d66f95a7f --- /dev/null +++ b/src/libsync/vfs/suffix/vfs_suffix.h @@ -0,0 +1,62 @@ +/* + * Copyright (C) by Christian Kamm <mail@ckamm.de> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#pragma once + +#include <QObject> +#include <QScopedPointer> + +#include "common/vfs.h" +#include "plugin.h" + +namespace OCC { + +class VfsSuffixPrivate; + +class VfsSuffix : public Vfs +{ + Q_OBJECT + Q_DECLARE_PRIVATE(VfsSuffix) + const QScopedPointer<VfsSuffixPrivate> d_ptr; + +public: + explicit VfsSuffix(QObject *parent = nullptr); + ~VfsSuffix(); + + Mode mode() const override; + QString fileSuffix() const override; + + void registerFolder(const VfsSetupParams ¶ms) override; + void start(const VfsSetupParams ¶ms) override; + void stop() override; + void unregisterFolder() override; + + bool isHydrating() const override; + + bool updateMetadata(const QString &filePath, time_t modtime, quint64 size, const QByteArray &fileId, QString *error) override; + + void createPlaceholder(const QString &syncFolder, const SyncFileItemPtr &item) override; + void convertToPlaceholder(const QString &filename, const SyncFileItemPtr &item) override; + + bool isDehydratedPlaceholder(const QString &filePath) override; + bool statTypeVirtualFile(csync_file_stat_t *stat, void *stat_data) override; +}; + +class SuffixVfsPluginFactory : public QObject, public DefaultPluginFactory<VfsSuffix> +{ + Q_OBJECT + Q_PLUGIN_METADATA(IID "org.owncloud.PluginFactory") + Q_INTERFACES(OCC::PluginFactory) +}; + +} // namespace OCC diff --git a/test/nextcloud_add_test.cmake b/test/nextcloud_add_test.cmake index 552c28479..24e6a5b5d 100644 --- a/test/nextcloud_add_test.cmake +++ b/test/nextcloud_add_test.cmake @@ -21,7 +21,12 @@ macro(nextcloud_add_test test_class additional_cpp) add_definitions(-DOWNCLOUD_TEST) add_definitions(-DOWNCLOUD_BIN_PATH="${CMAKE_BINARY_DIR}/bin") - add_test(NAME ${OWNCLOUD_TEST_CLASS}Test COMMAND ${OWNCLOUD_TEST_CLASS}Test) + message(STATUS "Add test: ${OWNCLOUD_TEST_CLASS}Test") + add_test(NAME ${OWNCLOUD_TEST_CLASS}Test + COMMAND ${OWNCLOUD_TEST_CLASS}Test + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/bin") + + target_include_directories(${OWNCLOUD_TEST_CLASS}Test PRIVATE "${CMAKE_SOURCE_DIR}/test/") endmacro() macro(nextcloud_add_benchmark test_class additional_cpp) diff --git a/test/syncenginetestutils.h b/test/syncenginetestutils.h index 0230d1259..c7acd83cb 100644 --- a/test/syncenginetestutils.h +++ b/test/syncenginetestutils.h @@ -931,6 +931,7 @@ public: syncOnce(); } + OCC::AccountPtr account() const { return _account; } OCC::SyncEngine &syncEngine() const { return *_syncEngine; } OCC::SyncJournalDb &syncJournal() const { return *_journalDb; } diff --git a/test/testsyncvirtualfiles.cpp b/test/testsyncvirtualfiles.cpp index 7a3964c4d..5a271fd0f 100644 --- a/test/testsyncvirtualfiles.cpp +++ b/test/testsyncvirtualfiles.cpp @@ -7,6 +7,8 @@ #include <QtTest> #include "syncenginetestutils.h" +#include "common/vfs.h" +#include "plugin.h" #include <syncengine.h> using namespace OCC; @@ -38,7 +40,7 @@ void triggerDownload(FakeFolder &folder, const QByteArray &path) { auto &journal = folder.syncJournal(); SyncJournalFileRecord record; - journal.getFileRecord(path + ".owncloud", &record); + journal.getFileRecord(path + ".nextcloud", &record); if (!record.isValid()) return; record._type = ItemTypeVirtualFileDownload; @@ -46,6 +48,25 @@ void triggerDownload(FakeFolder &folder, const QByteArray &path) journal.avoidReadFromDbOnNextSync(record._path); } +void markForDehydration(FakeFolder &folder, const QByteArray &path) +{ + auto &journal = folder.syncJournal(); + SyncJournalFileRecord record; + journal.getFileRecord(path, &record); + if (!record.isValid()) + return; + record._type = ItemTypeVirtualFileDehydration; + journal.setFileRecord(record); + journal.avoidReadFromDbOnNextSync(record._path); +} + +SyncOptions vfsSyncOptions() +{ + SyncOptions options; + options._vfs = PluginLoader().create<Vfs>("vfs", "suffix"); + return options; +} + class TestSyncVirtualFiles : public QObject { Q_OBJECT @@ -64,9 +85,7 @@ private slots: QFETCH(bool, doLocalDiscovery); FakeFolder fakeFolder{ FileInfo() }; - SyncOptions syncOptions; - syncOptions._newFilesAreVirtual = true; - fakeFolder.syncEngine().setSyncOptions(syncOptions); + fakeFolder.syncEngine().setSyncOptions(vfsSyncOptions()); QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); QSignalSpy completeSpy(&fakeFolder.syncEngine(), SIGNAL(itemCompleted(const SyncFileItemPtr &))); @@ -84,20 +103,20 @@ private slots: fakeFolder.remoteModifier().setModTime("A/a1", someDate); QVERIFY(fakeFolder.syncOnce()); QVERIFY(!fakeFolder.currentLocalState().find("A/a1")); - QVERIFY(fakeFolder.currentLocalState().find("A/a1.owncloud")); - QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1.owncloud").lastModified(), someDate); + QVERIFY(fakeFolder.currentLocalState().find("A/a1.nextcloud")); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1.nextcloud").lastModified(), someDate); QVERIFY(fakeFolder.currentRemoteState().find("A/a1")); - QVERIFY(itemInstruction(completeSpy, "A/a1.owncloud", CSYNC_INSTRUCTION_NEW)); - QCOMPARE(dbRecord(fakeFolder, "A/a1.owncloud")._type, ItemTypeVirtualFile); + QVERIFY(itemInstruction(completeSpy, "A/a1.nextcloud", CSYNC_INSTRUCTION_NEW)); + QCOMPARE(dbRecord(fakeFolder, "A/a1.nextcloud")._type, ItemTypeVirtualFile); cleanup(); // Another sync doesn't actually lead to changes QVERIFY(fakeFolder.syncOnce()); QVERIFY(!fakeFolder.currentLocalState().find("A/a1")); - QVERIFY(fakeFolder.currentLocalState().find("A/a1.owncloud")); - QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1.owncloud").lastModified(), someDate); + QVERIFY(fakeFolder.currentLocalState().find("A/a1.nextcloud")); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1.nextcloud").lastModified(), someDate); QVERIFY(fakeFolder.currentRemoteState().find("A/a1")); - QCOMPARE(dbRecord(fakeFolder, "A/a1.owncloud")._type, ItemTypeVirtualFile); + QCOMPARE(dbRecord(fakeFolder, "A/a1.nextcloud")._type, ItemTypeVirtualFile); QVERIFY(completeSpy.isEmpty()); cleanup(); @@ -105,10 +124,10 @@ private slots: fakeFolder.syncJournal().forceRemoteDiscoveryNextSync(); QVERIFY(fakeFolder.syncOnce()); QVERIFY(!fakeFolder.currentLocalState().find("A/a1")); - QVERIFY(fakeFolder.currentLocalState().find("A/a1.owncloud")); - QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1.owncloud").lastModified(), someDate); + QVERIFY(fakeFolder.currentLocalState().find("A/a1.nextcloud")); + QCOMPARE(QFileInfo(fakeFolder.localPath() + "A/a1.nextcloud").lastModified(), someDate); QVERIFY(fakeFolder.currentRemoteState().find("A/a1")); - QCOMPARE(dbRecord(fakeFolder, "A/a1.owncloud")._type, ItemTypeVirtualFile); + QCOMPARE(dbRecord(fakeFolder, "A/a1.nextcloud")._type, ItemTypeVirtualFile); QVERIFY(completeSpy.isEmpty()); cleanup(); @@ -116,24 +135,24 @@ private slots: fakeFolder.remoteModifier().appendByte("A/a1"); QVERIFY(fakeFolder.syncOnce()); QVERIFY(!fakeFolder.currentLocalState().find("A/a1")); - QVERIFY(fakeFolder.currentLocalState().find("A/a1.owncloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a1.nextcloud")); QVERIFY(fakeFolder.currentRemoteState().find("A/a1")); - QVERIFY(itemInstruction(completeSpy, "A/a1.owncloud", CSYNC_INSTRUCTION_UPDATE_METADATA)); - QCOMPARE(dbRecord(fakeFolder, "A/a1.owncloud")._type, ItemTypeVirtualFile); - QCOMPARE(dbRecord(fakeFolder, "A/a1.owncloud")._fileSize, 65); + QVERIFY(itemInstruction(completeSpy, "A/a1.nextcloud", CSYNC_INSTRUCTION_UPDATE_METADATA)); + QCOMPARE(dbRecord(fakeFolder, "A/a1.nextcloud")._type, ItemTypeVirtualFile); + QCOMPARE(dbRecord(fakeFolder, "A/a1.nextcloud")._fileSize, 65); cleanup(); // If the local virtual file file is removed, it'll just be recreated if (!doLocalDiscovery) fakeFolder.syncEngine().setLocalDiscoveryOptions(LocalDiscoveryStyle::DatabaseAndFilesystem, { "A" }); - fakeFolder.localModifier().remove("A/a1.owncloud"); + fakeFolder.localModifier().remove("A/a1.nextcloud"); QVERIFY(fakeFolder.syncOnce()); QVERIFY(!fakeFolder.currentLocalState().find("A/a1")); - QVERIFY(fakeFolder.currentLocalState().find("A/a1.owncloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a1.nextcloud")); QVERIFY(fakeFolder.currentRemoteState().find("A/a1")); - QVERIFY(itemInstruction(completeSpy, "A/a1.owncloud", CSYNC_INSTRUCTION_NEW)); - QCOMPARE(dbRecord(fakeFolder, "A/a1.owncloud")._type, ItemTypeVirtualFile); - QCOMPARE(dbRecord(fakeFolder, "A/a1.owncloud")._fileSize, 65); + QVERIFY(itemInstruction(completeSpy, "A/a1.nextcloud", CSYNC_INSTRUCTION_NEW)); + QCOMPARE(dbRecord(fakeFolder, "A/a1.nextcloud")._type, ItemTypeVirtualFile); + QCOMPARE(dbRecord(fakeFolder, "A/a1.nextcloud")._fileSize, 65); cleanup(); // Remote rename is propagated @@ -141,55 +160,53 @@ private slots: QVERIFY(fakeFolder.syncOnce()); QVERIFY(!fakeFolder.currentLocalState().find("A/a1")); QVERIFY(!fakeFolder.currentLocalState().find("A/a1m")); - QVERIFY(!fakeFolder.currentLocalState().find("A/a1.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("A/a1m.owncloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/a1.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a1m.nextcloud")); QVERIFY(!fakeFolder.currentRemoteState().find("A/a1")); QVERIFY(fakeFolder.currentRemoteState().find("A/a1m")); QVERIFY( - itemInstruction(completeSpy, "A/a1m.owncloud", CSYNC_INSTRUCTION_RENAME) - || (itemInstruction(completeSpy, "A/a1m.owncloud", CSYNC_INSTRUCTION_NEW) - && itemInstruction(completeSpy, "A/a1.owncloud", CSYNC_INSTRUCTION_REMOVE))); - QCOMPARE(dbRecord(fakeFolder, "A/a1m.owncloud")._type, ItemTypeVirtualFile); + itemInstruction(completeSpy, "A/a1m.nextcloud", CSYNC_INSTRUCTION_RENAME) + || (itemInstruction(completeSpy, "A/a1m.nextcloud", CSYNC_INSTRUCTION_NEW) + && itemInstruction(completeSpy, "A/a1.nextcloud", CSYNC_INSTRUCTION_REMOVE))); + QCOMPARE(dbRecord(fakeFolder, "A/a1m.nextcloud")._type, ItemTypeVirtualFile); cleanup(); // Remote remove is propagated fakeFolder.remoteModifier().remove("A/a1m"); QVERIFY(fakeFolder.syncOnce()); - QVERIFY(!fakeFolder.currentLocalState().find("A/a1m.owncloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/a1m.nextcloud")); QVERIFY(!fakeFolder.currentRemoteState().find("A/a1m")); - QVERIFY(itemInstruction(completeSpy, "A/a1m.owncloud", CSYNC_INSTRUCTION_REMOVE)); - QVERIFY(!dbRecord(fakeFolder, "A/a1.owncloud").isValid()); - QVERIFY(!dbRecord(fakeFolder, "A/a1m.owncloud").isValid()); + QVERIFY(itemInstruction(completeSpy, "A/a1m.nextcloud", CSYNC_INSTRUCTION_REMOVE)); + QVERIFY(!dbRecord(fakeFolder, "A/a1.nextcloud").isValid()); + QVERIFY(!dbRecord(fakeFolder, "A/a1m.nextcloud").isValid()); cleanup(); // Edge case: Local virtual file but no db entry for some reason fakeFolder.remoteModifier().insert("A/a2", 64); fakeFolder.remoteModifier().insert("A/a3", 64); QVERIFY(fakeFolder.syncOnce()); - QVERIFY(fakeFolder.currentLocalState().find("A/a2.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("A/a3.owncloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a2.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a3.nextcloud")); cleanup(); - fakeFolder.syncEngine().journal()->deleteFileRecord("A/a2.owncloud"); - fakeFolder.syncEngine().journal()->deleteFileRecord("A/a3.owncloud"); + fakeFolder.syncEngine().journal()->deleteFileRecord("A/a2.nextcloud"); + fakeFolder.syncEngine().journal()->deleteFileRecord("A/a3.nextcloud"); fakeFolder.remoteModifier().remove("A/a3"); fakeFolder.syncEngine().setLocalDiscoveryOptions(LocalDiscoveryStyle::FilesystemOnly); QVERIFY(fakeFolder.syncOnce()); - QVERIFY(fakeFolder.currentLocalState().find("A/a2.owncloud")); - QVERIFY(itemInstruction(completeSpy, "A/a2.owncloud", CSYNC_INSTRUCTION_UPDATE_METADATA)); - QVERIFY(dbRecord(fakeFolder, "A/a2.owncloud").isValid()); - QVERIFY(!fakeFolder.currentLocalState().find("A/a3.owncloud")); - QVERIFY(itemInstruction(completeSpy, "A/a3.owncloud", CSYNC_INSTRUCTION_REMOVE)); - QVERIFY(!dbRecord(fakeFolder, "A/a3.owncloud").isValid()); + QVERIFY(fakeFolder.currentLocalState().find("A/a2.nextcloud")); + QVERIFY(itemInstruction(completeSpy, "A/a2.nextcloud", CSYNC_INSTRUCTION_UPDATE_METADATA)); + QVERIFY(dbRecord(fakeFolder, "A/a2.nextcloud").isValid()); + QVERIFY(!fakeFolder.currentLocalState().find("A/a3.nextcloud")); + QVERIFY(itemInstruction(completeSpy, "A/a3.nextcloud", CSYNC_INSTRUCTION_REMOVE)); + QVERIFY(!dbRecord(fakeFolder, "A/a3.nextcloud").isValid()); cleanup(); } void testVirtualFileConflict() { FakeFolder fakeFolder{ FileInfo() }; - SyncOptions syncOptions; - syncOptions._newFilesAreVirtual = true; - fakeFolder.syncEngine().setSyncOptions(syncOptions); + fakeFolder.syncEngine().setSyncOptions(vfsSyncOptions()); QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); QSignalSpy completeSpy(&fakeFolder.syncEngine(), SIGNAL(itemCompleted(const SyncFileItemPtr &))); @@ -208,8 +225,8 @@ private slots: fakeFolder.remoteModifier().mkdir("C"); fakeFolder.remoteModifier().insert("C/c1", 64); QVERIFY(fakeFolder.syncOnce()); - QVERIFY(fakeFolder.currentLocalState().find("A/a1.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("B/b2.owncloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a1.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("B/b2.nextcloud")); cleanup(); // A: the correct file and a conflicting file are added, virtual files stay @@ -219,8 +236,8 @@ private slots: fakeFolder.localModifier().insert("A/a2", 30); fakeFolder.localModifier().insert("B/b1", 64); fakeFolder.localModifier().insert("B/b2", 30); - fakeFolder.localModifier().remove("B/b1.owncloud"); - fakeFolder.localModifier().remove("B/b2.owncloud"); + fakeFolder.localModifier().remove("B/b1.nextcloud"); + fakeFolder.localModifier().remove("B/b2.nextcloud"); fakeFolder.localModifier().mkdir("C/c1"); fakeFolder.localModifier().insert("C/c1/foo"); QVERIFY(fakeFolder.syncOnce()); @@ -233,11 +250,11 @@ private slots: QVERIFY(itemInstruction(completeSpy, "C/c1", CSYNC_INSTRUCTION_CONFLICT)); // no virtual file files should remain - QVERIFY(!fakeFolder.currentLocalState().find("A/a1.owncloud")); - QVERIFY(!fakeFolder.currentLocalState().find("A/a2.owncloud")); - QVERIFY(!fakeFolder.currentLocalState().find("B/b1.owncloud")); - QVERIFY(!fakeFolder.currentLocalState().find("B/b2.owncloud")); - QVERIFY(!fakeFolder.currentLocalState().find("C/c1.owncloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/a1.nextcloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/a2.nextcloud")); + QVERIFY(!fakeFolder.currentLocalState().find("B/b1.nextcloud")); + QVERIFY(!fakeFolder.currentLocalState().find("B/b2.nextcloud")); + QVERIFY(!fakeFolder.currentLocalState().find("C/c1.nextcloud")); // conflict files should exist QCOMPARE(fakeFolder.syncJournal().conflictRecordPaths().size(), 3); @@ -248,11 +265,11 @@ private slots: QCOMPARE(dbRecord(fakeFolder, "B/b1")._type, ItemTypeFile); QCOMPARE(dbRecord(fakeFolder, "B/b2")._type, ItemTypeFile); QCOMPARE(dbRecord(fakeFolder, "C/c1")._type, ItemTypeFile); - QVERIFY(!dbRecord(fakeFolder, "A/a1.owncloud").isValid()); - QVERIFY(!dbRecord(fakeFolder, "A/a2.owncloud").isValid()); - QVERIFY(!dbRecord(fakeFolder, "B/b1.owncloud").isValid()); - QVERIFY(!dbRecord(fakeFolder, "B/b2.owncloud").isValid()); - QVERIFY(!dbRecord(fakeFolder, "C/c1.owncloud").isValid()); + QVERIFY(!dbRecord(fakeFolder, "A/a1.nextcloud").isValid()); + QVERIFY(!dbRecord(fakeFolder, "A/a2.nextcloud").isValid()); + QVERIFY(!dbRecord(fakeFolder, "B/b1.nextcloud").isValid()); + QVERIFY(!dbRecord(fakeFolder, "B/b2.nextcloud").isValid()); + QVERIFY(!dbRecord(fakeFolder, "C/c1.nextcloud").isValid()); cleanup(); } @@ -260,9 +277,7 @@ private slots: void testWithNormalSync() { FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() }; - SyncOptions syncOptions; - syncOptions._newFilesAreVirtual = true; - fakeFolder.syncEngine().setSyncOptions(syncOptions); + fakeFolder.syncEngine().setSyncOptions(vfsSyncOptions()); QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); QSignalSpy completeSpy(&fakeFolder.syncEngine(), SIGNAL(itemCompleted(const SyncFileItemPtr &))); @@ -288,19 +303,17 @@ private slots: fakeFolder.remoteModifier().insert("A/new"); QVERIFY(fakeFolder.syncOnce()); QVERIFY(!fakeFolder.currentLocalState().find("A/new")); - QVERIFY(fakeFolder.currentLocalState().find("A/new.owncloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/new.nextcloud")); QVERIFY(fakeFolder.currentRemoteState().find("A/new")); - QVERIFY(itemInstruction(completeSpy, "A/new.owncloud", CSYNC_INSTRUCTION_NEW)); - QCOMPARE(dbRecord(fakeFolder, "A/new.owncloud")._type, ItemTypeVirtualFile); + QVERIFY(itemInstruction(completeSpy, "A/new.nextcloud", CSYNC_INSTRUCTION_NEW)); + QCOMPARE(dbRecord(fakeFolder, "A/new.nextcloud")._type, ItemTypeVirtualFile); cleanup(); } void testVirtualFileDownload() { FakeFolder fakeFolder{ FileInfo() }; - SyncOptions syncOptions; - syncOptions._newFilesAreVirtual = true; - fakeFolder.syncEngine().setSyncOptions(syncOptions); + fakeFolder.syncEngine().setSyncOptions(vfsSyncOptions()); QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); QSignalSpy completeSpy(&fakeFolder.syncEngine(), SIGNAL(itemCompleted(const SyncFileItemPtr &))); @@ -318,12 +331,12 @@ private slots: fakeFolder.remoteModifier().insert("A/a5"); fakeFolder.remoteModifier().insert("A/a6"); QVERIFY(fakeFolder.syncOnce()); - QVERIFY(fakeFolder.currentLocalState().find("A/a1.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("A/a2.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("A/a3.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("A/a4.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("A/a5.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("A/a6.owncloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a1.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a2.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a3.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a4.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a5.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a6.nextcloud")); cleanup(); // Download by changing the db entry @@ -338,17 +351,17 @@ private slots: fakeFolder.remoteModifier().rename("A/a4", "A/a4m"); fakeFolder.localModifier().insert("A/a5"); fakeFolder.localModifier().insert("A/a6"); - fakeFolder.localModifier().remove("A/a6.owncloud"); + fakeFolder.localModifier().remove("A/a6.nextcloud"); QVERIFY(fakeFolder.syncOnce()); QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_NEW)); - QVERIFY(itemInstruction(completeSpy, "A/a1.owncloud", CSYNC_INSTRUCTION_NONE)); + QVERIFY(itemInstruction(completeSpy, "A/a1.nextcloud", CSYNC_INSTRUCTION_NONE)); QVERIFY(itemInstruction(completeSpy, "A/a2", CSYNC_INSTRUCTION_NEW)); - QVERIFY(itemInstruction(completeSpy, "A/a2.owncloud", CSYNC_INSTRUCTION_NONE)); - QVERIFY(itemInstruction(completeSpy, "A/a3.owncloud", CSYNC_INSTRUCTION_REMOVE)); + QVERIFY(itemInstruction(completeSpy, "A/a2.nextcloud", CSYNC_INSTRUCTION_NONE)); + QVERIFY(itemInstruction(completeSpy, "A/a3.nextcloud", CSYNC_INSTRUCTION_REMOVE)); QVERIFY(itemInstruction(completeSpy, "A/a4m", CSYNC_INSTRUCTION_NEW)); - QVERIFY(itemInstruction(completeSpy, "A/a4.owncloud", CSYNC_INSTRUCTION_REMOVE)); + QVERIFY(itemInstruction(completeSpy, "A/a4.nextcloud", CSYNC_INSTRUCTION_REMOVE)); QVERIFY(itemInstruction(completeSpy, "A/a5", CSYNC_INSTRUCTION_CONFLICT)); - QVERIFY(itemInstruction(completeSpy, "A/a5.owncloud", CSYNC_INSTRUCTION_NONE)); + QVERIFY(itemInstruction(completeSpy, "A/a5.nextcloud", CSYNC_INSTRUCTION_NONE)); QVERIFY(itemInstruction(completeSpy, "A/a6", CSYNC_INSTRUCTION_CONFLICT)); QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); QCOMPARE(dbRecord(fakeFolder, "A/a1")._type, ItemTypeFile); @@ -357,20 +370,18 @@ private slots: QCOMPARE(dbRecord(fakeFolder, "A/a4m")._type, ItemTypeFile); QCOMPARE(dbRecord(fakeFolder, "A/a5")._type, ItemTypeFile); QCOMPARE(dbRecord(fakeFolder, "A/a6")._type, ItemTypeFile); - QVERIFY(!dbRecord(fakeFolder, "A/a1.owncloud").isValid()); - QVERIFY(!dbRecord(fakeFolder, "A/a2.owncloud").isValid()); - QVERIFY(!dbRecord(fakeFolder, "A/a3.owncloud").isValid()); - QVERIFY(!dbRecord(fakeFolder, "A/a4.owncloud").isValid()); - QVERIFY(!dbRecord(fakeFolder, "A/a5.owncloud").isValid()); - QVERIFY(!dbRecord(fakeFolder, "A/a6.owncloud").isValid()); + QVERIFY(!dbRecord(fakeFolder, "A/a1.nextcloud").isValid()); + QVERIFY(!dbRecord(fakeFolder, "A/a2.nextcloud").isValid()); + QVERIFY(!dbRecord(fakeFolder, "A/a3.nextcloud").isValid()); + QVERIFY(!dbRecord(fakeFolder, "A/a4.nextcloud").isValid()); + QVERIFY(!dbRecord(fakeFolder, "A/a5.nextcloud").isValid()); + QVERIFY(!dbRecord(fakeFolder, "A/a6.nextcloud").isValid()); } void testVirtualFileDownloadResume() { FakeFolder fakeFolder{ FileInfo() }; - SyncOptions syncOptions; - syncOptions._newFilesAreVirtual = true; - fakeFolder.syncEngine().setSyncOptions(syncOptions); + fakeFolder.syncEngine().setSyncOptions(vfsSyncOptions()); QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); QSignalSpy completeSpy(&fakeFolder.syncEngine(), SIGNAL(itemCompleted(const SyncFileItemPtr &))); @@ -384,7 +395,7 @@ private slots: fakeFolder.remoteModifier().mkdir("A"); fakeFolder.remoteModifier().insert("A/a1"); QVERIFY(fakeFolder.syncOnce()); - QVERIFY(fakeFolder.currentLocalState().find("A/a1.owncloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a1.nextcloud")); cleanup(); // Download by changing the db entry @@ -392,29 +403,28 @@ private slots: fakeFolder.serverErrorPaths().append("A/a1", 500); QVERIFY(!fakeFolder.syncOnce()); QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_NEW)); - QVERIFY(itemInstruction(completeSpy, "A/a1.owncloud", CSYNC_INSTRUCTION_NONE)); - QVERIFY(fakeFolder.currentLocalState().find("A/a1.owncloud")); + QVERIFY(itemInstruction(completeSpy, "A/a1.nextcloud", CSYNC_INSTRUCTION_NONE)); + QVERIFY(fakeFolder.currentLocalState().find("A/a1.nextcloud")); QVERIFY(!fakeFolder.currentLocalState().find("A/a1")); - QCOMPARE(dbRecord(fakeFolder, "A/a1.owncloud")._type, ItemTypeVirtualFileDownload); + QCOMPARE(dbRecord(fakeFolder, "A/a1.nextcloud")._type, ItemTypeVirtualFileDownload); QVERIFY(!dbRecord(fakeFolder, "A/a1").isValid()); cleanup(); fakeFolder.serverErrorPaths().clear(); QVERIFY(fakeFolder.syncOnce()); QVERIFY(itemInstruction(completeSpy, "A/a1", CSYNC_INSTRUCTION_NEW)); - QVERIFY(itemInstruction(completeSpy, "A/a1.owncloud", CSYNC_INSTRUCTION_NONE)); + QVERIFY(itemInstruction(completeSpy, "A/a1.nextcloud", CSYNC_INSTRUCTION_NONE)); QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); QCOMPARE(dbRecord(fakeFolder, "A/a1")._type, ItemTypeFile); - QVERIFY(!dbRecord(fakeFolder, "A/a1.owncloud").isValid()); + QVERIFY(!dbRecord(fakeFolder, "A/a1.nextcloud").isValid()); } - // Check what might happen if an older sync client encounters virtual files - void testOldVersion1() + // Check what happens if vfs mode is disabled + void testSwitchOfVfs() { QSKIP("Does not work with the new discovery because the way we simulate the old client does not work"); FakeFolder fakeFolder{ FileInfo() }; - SyncOptions syncOptions; - syncOptions._newFilesAreVirtual = true; + SyncOptions syncOptions = vfsSyncOptions(); fakeFolder.syncEngine().setSyncOptions(syncOptions); QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); @@ -422,39 +432,29 @@ private slots: fakeFolder.remoteModifier().mkdir("A"); fakeFolder.remoteModifier().insert("A/a1"); QVERIFY(fakeFolder.syncOnce()); - QVERIFY(fakeFolder.currentLocalState().find("A/a1.owncloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a1.nextcloud")); - // Simulate an old client by switching the type of all ItemTypeVirtualFile - // entries in the db to an invalid type. - auto &db = fakeFolder.syncJournal(); - SyncJournalFileRecord rec; - db.getFileRecord(QByteArray("A/a1.owncloud"), &rec); - QVERIFY(rec.isValid()); - QCOMPARE(rec._type, ItemTypeVirtualFile); - rec._type = static_cast<ItemType>(-1); - db.setFileRecord(rec); - - // Also switch off new files becoming virtual files - syncOptions._newFilesAreVirtual = false; + // Switch off new files becoming virtual files + syncOptions._vfs = nullptr; fakeFolder.syncEngine().setSyncOptions(syncOptions); - // A sync that doesn't do remote discovery has no effect + // A sync that doesn't do remote discovery will wipe the placeholder, but not redownload QVERIFY(fakeFolder.syncOnce()); - QVERIFY(fakeFolder.currentLocalState().find("A/a1.owncloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/a1.nextcloud")); QVERIFY(!fakeFolder.currentLocalState().find("A/a1")); QVERIFY(fakeFolder.currentRemoteState().find("A/a1")); - QVERIFY(!fakeFolder.currentRemoteState().find("A/a1.owncloud")); + QVERIFY(!fakeFolder.currentRemoteState().find("A/a1.nextcloud")); // But with a remote discovery the virtual files will be removed and // the remote files will be downloaded. - db.forceRemoteDiscoveryNextSync(); + fakeFolder.syncJournal().forceRemoteDiscoveryNextSync(); QVERIFY(fakeFolder.syncOnce()); QVERIFY(fakeFolder.currentLocalState().find("A/a1")); - QVERIFY(!fakeFolder.currentLocalState().find("A/a1.owncloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/a1.nextcloud")); QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); } - // Older versions may leave db entries for foo and foo.owncloud + // Older versions may leave db entries for foo and foo.nextcloud void testOldVersion2() { QSKIP("Does not work with the new discovery because the way we simulate the old client does not work"); @@ -469,31 +469,27 @@ private slots: // Create the virtual file too // In the wild, the new version would create the virtual file and the db entry // while the old version would download the plain file. - fakeFolder.localModifier().insert("A/a1.owncloud"); + fakeFolder.localModifier().insert("A/a1.nextcloud"); auto &db = fakeFolder.syncJournal(); SyncJournalFileRecord rec; db.getFileRecord(QByteArray("A/a1"), &rec); rec._type = ItemTypeVirtualFile; - rec._path = "A/a1.owncloud"; + rec._path = "A/a1.nextcloud"; db.setFileRecord(rec); - SyncOptions syncOptions; - syncOptions._newFilesAreVirtual = true; - fakeFolder.syncEngine().setSyncOptions(syncOptions); + fakeFolder.syncEngine().setSyncOptions(vfsSyncOptions()); // Check that a sync removes the virtual file and its db entry QVERIFY(fakeFolder.syncOnce()); - QVERIFY(!fakeFolder.currentLocalState().find("A/a1.owncloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/a1.nextcloud")); QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); - QVERIFY(!dbRecord(fakeFolder, "A/a1.owncloud").isValid()); + QVERIFY(!dbRecord(fakeFolder, "A/a1.nextcloud").isValid()); } void testDownloadRecursive() { FakeFolder fakeFolder{ FileInfo() }; - SyncOptions syncOptions; - syncOptions._newFilesAreVirtual = true; - fakeFolder.syncEngine().setSyncOptions(syncOptions); + fakeFolder.syncEngine().setSyncOptions(vfsSyncOptions()); QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); // Create a virtual file for remote files @@ -512,14 +508,14 @@ private slots: fakeFolder.remoteModifier().insert("B/b1"); fakeFolder.remoteModifier().insert("B/Sub/b2"); QVERIFY(fakeFolder.syncOnce()); - QVERIFY(fakeFolder.currentLocalState().find("A/a1.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("A/a2.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("A/Sub/a3.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("A/Sub/a4.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("A/Sub/SubSub/a5.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("A/Sub2/a6.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("B/b1.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("B/Sub/b2.owncloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a1.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a2.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/Sub/a3.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/Sub/a4.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/Sub/SubSub/a5.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/Sub2/a6.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("B/b1.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("B/Sub/b2.nextcloud")); QVERIFY(!fakeFolder.currentLocalState().find("A/a1")); QVERIFY(!fakeFolder.currentLocalState().find("A/a2")); QVERIFY(!fakeFolder.currentLocalState().find("A/Sub/a3")); @@ -535,14 +531,14 @@ private slots: fakeFolder.syncJournal().markVirtualFileForDownloadRecursively("A/Sub"); QVERIFY(fakeFolder.syncOnce()); - QVERIFY(fakeFolder.currentLocalState().find("A/a1.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("A/a2.owncloud")); - QVERIFY(!fakeFolder.currentLocalState().find("A/Sub/a3.owncloud")); - QVERIFY(!fakeFolder.currentLocalState().find("A/Sub/a4.owncloud")); - QVERIFY(!fakeFolder.currentLocalState().find("A/Sub/SubSub/a5.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("A/Sub2/a6.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("B/b1.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("B/Sub/b2.owncloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a1.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a2.nextcloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/Sub/a3.nextcloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/Sub/a4.nextcloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/Sub/SubSub/a5.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/Sub2/a6.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("B/b1.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("B/Sub/b2.nextcloud")); QVERIFY(!fakeFolder.currentLocalState().find("A/a1")); QVERIFY(!fakeFolder.currentLocalState().find("A/a2")); QVERIFY(fakeFolder.currentLocalState().find("A/Sub/a3")); @@ -556,21 +552,21 @@ private slots: // Currently, this continue to add it as a virtual file. fakeFolder.remoteModifier().insert("A/Sub/SubSub/a7"); QVERIFY(fakeFolder.syncOnce()); - QVERIFY(fakeFolder.currentLocalState().find("A/Sub/SubSub/a7.owncloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/Sub/SubSub/a7.nextcloud")); QVERIFY(!fakeFolder.currentLocalState().find("A/Sub/SubSub/a7")); // Now download all files in "A" fakeFolder.syncJournal().markVirtualFileForDownloadRecursively("A"); QVERIFY(fakeFolder.syncOnce()); - QVERIFY(!fakeFolder.currentLocalState().find("A/a1.owncloud")); - QVERIFY(!fakeFolder.currentLocalState().find("A/a2.owncloud")); - QVERIFY(!fakeFolder.currentLocalState().find("A/Sub/a3.owncloud")); - QVERIFY(!fakeFolder.currentLocalState().find("A/Sub/a4.owncloud")); - QVERIFY(!fakeFolder.currentLocalState().find("A/Sub/SubSub/a5.owncloud")); - QVERIFY(!fakeFolder.currentLocalState().find("A/Sub2/a6.owncloud")); - QVERIFY(!fakeFolder.currentLocalState().find("A/Sub/SubSub/a7.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("B/b1.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("B/Sub/b2.owncloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/a1.nextcloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/a2.nextcloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/Sub/a3.nextcloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/Sub/a4.nextcloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/Sub/SubSub/a5.nextcloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/Sub2/a6.nextcloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/Sub/SubSub/a7.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("B/b1.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("B/Sub/b2.nextcloud")); QVERIFY(fakeFolder.currentLocalState().find("A/a1")); QVERIFY(fakeFolder.currentLocalState().find("A/a2")); QVERIFY(fakeFolder.currentLocalState().find("A/Sub/a3")); @@ -590,9 +586,7 @@ private slots: void testRenameToVirtual() { FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() }; - SyncOptions syncOptions; - syncOptions._newFilesAreVirtual = true; - fakeFolder.syncEngine().setSyncOptions(syncOptions); + fakeFolder.syncEngine().setSyncOptions(vfsSyncOptions()); QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); QSignalSpy completeSpy(&fakeFolder.syncEngine(), SIGNAL(itemCompleted(const SyncFileItemPtr &))); @@ -601,28 +595,28 @@ private slots: }; cleanup(); - // If a file is renamed to <name>.owncloud, it becomes virtual - fakeFolder.localModifier().rename("A/a1", "A/a1.owncloud"); - // If a file is renamed to <random>.owncloud, the file sticks around (to preserve user data) - fakeFolder.localModifier().rename("A/a2", "A/rand.owncloud"); + // If a file is renamed to <name>.nextcloud, it becomes virtual + fakeFolder.localModifier().rename("A/a1", "A/a1.nextcloud"); + // If a file is renamed to <random>.nextcloud, the file sticks around (to preserve user data) + fakeFolder.localModifier().rename("A/a2", "A/rand.nextcloud"); // dangling virtual files are removed - fakeFolder.localModifier().insert("A/dangling.owncloud", 1, ' '); + fakeFolder.localModifier().insert("A/dangling.nextcloud", 1, ' '); QVERIFY(fakeFolder.syncOnce()); QVERIFY(!fakeFolder.currentLocalState().find("A/a1")); - QVERIFY(fakeFolder.currentLocalState().find("A/a1.owncloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a1.nextcloud")); QVERIFY(fakeFolder.currentRemoteState().find("A/a1")); - QVERIFY(itemInstruction(completeSpy, "A/a1.owncloud", CSYNC_INSTRUCTION_NEW)); - QCOMPARE(dbRecord(fakeFolder, "A/a1.owncloud")._type, ItemTypeVirtualFile); + QVERIFY(itemInstruction(completeSpy, "A/a1.nextcloud", CSYNC_INSTRUCTION_NEW)); + QCOMPARE(dbRecord(fakeFolder, "A/a1.nextcloud")._type, ItemTypeVirtualFile); QVERIFY(!fakeFolder.currentLocalState().find("A/a2")); - QVERIFY(!fakeFolder.currentLocalState().find("A/a2.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("A/rand.owncloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/a2.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/rand.nextcloud")); QVERIFY(!fakeFolder.currentRemoteState().find("A/a2")); QVERIFY(itemInstruction(completeSpy, "A/a2", CSYNC_INSTRUCTION_REMOVE)); - QVERIFY(!dbRecord(fakeFolder, "A/rand.owncloud").isValid()); + QVERIFY(!dbRecord(fakeFolder, "A/rand.nextcloud").isValid()); - QVERIFY(!fakeFolder.currentLocalState().find("A/dangling.owncloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/dangling.nextcloud")); cleanup(); } @@ -630,9 +624,7 @@ private slots: void testRenameVirtual() { FakeFolder fakeFolder{ FileInfo() }; - SyncOptions syncOptions; - syncOptions._newFilesAreVirtual = true; - fakeFolder.syncEngine().setSyncOptions(syncOptions); + fakeFolder.syncEngine().setSyncOptions(vfsSyncOptions()); QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); QSignalSpy completeSpy(&fakeFolder.syncEngine(), SIGNAL(itemCompleted(const SyncFileItemPtr &))); @@ -645,30 +637,153 @@ private slots: fakeFolder.remoteModifier().insert("file2", 256, 'C'); QVERIFY(fakeFolder.syncOnce()); - QVERIFY(fakeFolder.currentLocalState().find("file1.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("file2.owncloud")); + QVERIFY(fakeFolder.currentLocalState().find("file1.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("file2.nextcloud")); cleanup(); - fakeFolder.localModifier().rename("file1.owncloud", "renamed1.owncloud"); - fakeFolder.localModifier().rename("file2.owncloud", "renamed2.owncloud"); + fakeFolder.localModifier().rename("file1.nextcloud", "renamed1.nextcloud"); + fakeFolder.localModifier().rename("file2.nextcloud", "renamed2.nextcloud"); triggerDownload(fakeFolder, "file2"); QVERIFY(fakeFolder.syncOnce()); - QVERIFY(!fakeFolder.currentLocalState().find("file1.owncloud")); - QVERIFY(fakeFolder.currentLocalState().find("renamed1.owncloud")); + QVERIFY(!fakeFolder.currentLocalState().find("file1.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("renamed1.nextcloud")); QVERIFY(!fakeFolder.currentRemoteState().find("file1")); QVERIFY(fakeFolder.currentRemoteState().find("renamed1")); - QVERIFY(itemInstruction(completeSpy, "renamed1.owncloud", CSYNC_INSTRUCTION_RENAME)); - QVERIFY(dbRecord(fakeFolder, "renamed1.owncloud").isValid()); + QVERIFY(itemInstruction(completeSpy, "renamed1.nextcloud", CSYNC_INSTRUCTION_RENAME)); + QVERIFY(dbRecord(fakeFolder, "renamed1.nextcloud").isValid()); // file2 has a conflict between the download request and the rename: // currently the download wins - QVERIFY(!fakeFolder.currentLocalState().find("file2.owncloud")); + QVERIFY(!fakeFolder.currentLocalState().find("file2.nextcloud")); QVERIFY(fakeFolder.currentLocalState().find("file2")); QVERIFY(fakeFolder.currentRemoteState().find("file2")); QVERIFY(itemInstruction(completeSpy, "file2", CSYNC_INSTRUCTION_NEW)); QVERIFY(dbRecord(fakeFolder, "file2").isValid()); } + + // Dehydration via sync works + void testSyncDehydration() + { + FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() }; + fakeFolder.syncEngine().setSyncOptions(vfsSyncOptions()); + + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + QSignalSpy completeSpy(&fakeFolder.syncEngine(), SIGNAL(itemCompleted(const SyncFileItemPtr &))); + auto cleanup = [&]() { + completeSpy.clear(); + }; + cleanup(); + + // + // Mark for dehydration and check + // + + markForDehydration(fakeFolder, "A/a1"); + + markForDehydration(fakeFolder, "A/a2"); + fakeFolder.remoteModifier().appendByte("A/a2"); + // expect: normal dehydration + + markForDehydration(fakeFolder, "B/b1"); + fakeFolder.remoteModifier().remove("B/b1"); + // expect: local removal + + markForDehydration(fakeFolder, "B/b2"); + fakeFolder.remoteModifier().rename("B/b2", "B/b3"); + // expect: B/b2 is gone, B/b3 is NEW placeholder + + markForDehydration(fakeFolder, "C/c1"); + fakeFolder.localModifier().appendByte("C/c1"); + // expect: no dehydration, upload of c1 + + markForDehydration(fakeFolder, "C/c2"); + fakeFolder.localModifier().appendByte("C/c2"); + fakeFolder.remoteModifier().appendByte("C/c2"); + fakeFolder.remoteModifier().appendByte("C/c2"); + // expect: no dehydration, conflict + + QVERIFY(fakeFolder.syncOnce()); + + auto isDehydrated = [&](const QString &path) { + QString placeholder = path + ".nextcloud"; + return !fakeFolder.currentLocalState().find(path) + && fakeFolder.currentLocalState().find(placeholder); + }; + + QVERIFY(isDehydrated("A/a1")); + QVERIFY(isDehydrated("A/a2")); + + QVERIFY(!fakeFolder.currentLocalState().find("B/b1")); + QVERIFY(!fakeFolder.currentRemoteState().find("B/b1")); + QVERIFY(itemInstruction(completeSpy, "B/b1", CSYNC_INSTRUCTION_REMOVE)); + + QVERIFY(!fakeFolder.currentLocalState().find("B/b2")); + QVERIFY(!fakeFolder.currentRemoteState().find("B/b2")); + QVERIFY(isDehydrated("B/b3")); + QVERIFY(itemInstruction(completeSpy, "B/b2", CSYNC_INSTRUCTION_REMOVE)); + QVERIFY(itemInstruction(completeSpy, "B/b3.nextcloud", CSYNC_INSTRUCTION_NEW)); + + QCOMPARE(fakeFolder.currentRemoteState().find("C/c1")->size, 25); + QVERIFY(itemInstruction(completeSpy, "C/c1", CSYNC_INSTRUCTION_SYNC)); + + QCOMPARE(fakeFolder.currentRemoteState().find("C/c2")->size, 26); + QVERIFY(itemInstruction(completeSpy, "C/c2", CSYNC_INSTRUCTION_CONFLICT)); + cleanup(); + + auto expectedLocalState = fakeFolder.currentLocalState(); + auto expectedRemoteState = fakeFolder.currentRemoteState(); + QVERIFY(fakeFolder.syncOnce()); + QCOMPARE(fakeFolder.currentLocalState(), expectedLocalState); + QCOMPARE(fakeFolder.currentRemoteState(), expectedRemoteState); + } + + void testWipeVirtualSuffixFiles() + { + FakeFolder fakeFolder{ FileInfo{} }; + fakeFolder.syncEngine().setSyncOptions(vfsSyncOptions()); + + // Create a suffix-vfs baseline + + fakeFolder.remoteModifier().mkdir("A"); + fakeFolder.remoteModifier().mkdir("A/B"); + fakeFolder.remoteModifier().insert("f1"); + fakeFolder.remoteModifier().insert("A/a1"); + fakeFolder.remoteModifier().insert("A/a3"); + fakeFolder.remoteModifier().insert("A/B/b1"); + fakeFolder.localModifier().mkdir("A"); + fakeFolder.localModifier().mkdir("A/B"); + fakeFolder.localModifier().insert("f2"); + fakeFolder.localModifier().insert("A/a2"); + fakeFolder.localModifier().insert("A/B/b2"); + + QVERIFY(fakeFolder.syncOnce()); + + QVERIFY(fakeFolder.currentLocalState().find("f1.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a1.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a3.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/B/b1.nextcloud")); + + // Make local changes to a3 + fakeFolder.localModifier().remove("A/a3.nextcloud"); + fakeFolder.localModifier().insert("A/a3.nextcloud", 100); + + // Now wipe the virtuals + + SyncEngine::wipeVirtualFiles(fakeFolder.localPath(), fakeFolder.syncJournal(), fakeFolder.syncEngine().syncOptions()._vfs); + + QVERIFY(!fakeFolder.currentLocalState().find("f1.nextcloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/a1.nextcloud")); + QVERIFY(fakeFolder.currentLocalState().find("A/a3.nextcloud")); + QVERIFY(!fakeFolder.currentLocalState().find("A/B/b1.nextcloud")); + + fakeFolder.syncEngine().setSyncOptions(SyncOptions{}); + QVERIFY(fakeFolder.syncOnce()); + QVERIFY(fakeFolder.currentRemoteState().find("A/a3.nextcloud")); // regular upload + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + } }; QTEST_GUILESS_MAIN(TestSyncVirtualFiles)