csync tests: Fix test after db changes.

Found while investigating #2296

The problem is that we should not remove a directory locally if it contains
modified files.
But the modification time of the directory is not necessarily chaning (so
the instruction of the directory may still be NONE)

We have to move the child_modified test a bit down to be recursive

CMakeLists.txt

@@ -123,6 +123,12 @@ find_package(Sphinx)
 find_package(PdfLatex)
 
 
+find_package(SQLite3 3.8.0 REQUIRED)
+# On some OS, we want to use our own, not the system sqlite
+if (USE_OUR_OWN_SQLITE3)
+    include_directories(BEFORE ${SQLITE3_INCLUDE_DIR})
+endif()
+
 
 configure_file(config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config.h)
 

VERSION.cmake

@@ -4,7 +4,7 @@ set( MIRALL_VERSION_PATCH 0 )
 set( MIRALL_SOVERSION 0 )
 
 if ( NOT DEFINED MIRALL_VERSION_SUFFIX )
-    set( MIRALL_VERSION_SUFFIX "beta2") #e.g. beta1, beta2, rc1
+    set( MIRALL_VERSION_SUFFIX "beta4") #e.g. beta1, beta2, rc1
 endif( NOT DEFINED MIRALL_VERSION_SUFFIX )
 
 if( NOT DEFINED MIRALL_VERSION_BUILD )

admin/osx/macdeployqt.py

@@ -23,7 +23,7 @@ import sys
 from glob import glob
 
 def QueryQMake(attrib):
-    return subprocess.check_output(['qmake', '-query', attrib]).rstrip('\n')
+    return subprocess.check_output([qmake_path, '-query', attrib]).rstrip('\n')
 
 FRAMEWORK_SEARCH_PATH=[
     '/Library/Frameworks',
@@ -68,14 +68,18 @@ class CouldNotFindQtPluginError(Error):
 class CouldNotFindScriptPluginError(Error):
   pass
 
-if len(sys.argv) < 2:
+class CouldNotFindFrameworkError(Error):
-  print 'Usage: %s <bundle.app>' % sys.argv[0]
+  pass
+
+if len(sys.argv) < 3:
+  print 'Usage: %s <bundle.app> <path-to-qmake>' % sys.argv[0]
   exit()
 
 def is_exe(fpath):
   return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
 
 bundle_dir = sys.argv[1]
+qmake_path = sys.argv[2]
 
 bundle_name = os.path.basename(bundle_dir).split('.')[0]
 

cmake/modules/DefineCompilerFlags.cmake

@@ -14,7 +14,8 @@ if (${CMAKE_C_COMPILER_ID} MATCHES "(GNU|Clang)")
 
     # add -Wconversion ?
     # cannot be pedantic with sqlite3 directly linked
-    if (NOT CSYNC_STATIC_COMPILE_DIR)
+    # FIXME Can we somehow not use those flags for sqlite3.* but use them for the rest of csync?
+    if (NOT USE_OUR_OWN_SQLITE3)
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99 -pedantic -pedantic-errors")
     endif()
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Wshadow -Wmissing-prototypes")

cmake/modules/FindSQLite3.cmake

@@ -50,8 +50,18 @@ if (SQLite3_FIND_VERSION AND _SQLITE3_VERSION)
     set(SQLite3_VERSION _SQLITE3_VERSION)
 endif (SQLite3_FIND_VERSION AND _SQLITE3_VERSION)
 
-include(FindPackageHandleStandardArgs)
+if (APPLE OR WIN32)
-find_package_handle_standard_args(SQLite3 DEFAULT_MSG SQLITE3_LIBRARIES SQLITE3_INCLUDE_DIRS)
+    set(USE_OUR_OWN_SQLITE3 TRUE)
+    set(SQLITE3_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/src/3rdparty/sqlite3)
+    set(SQLITE3_LIBRARIES "")
+    set(SQLITE3_SOURCE ${SQLITE3_INCLUDE_DIR}/sqlite3.c)
+    MESSAGE(STATUS "Using own sqlite3 from " ${SQLITE3_INCLUDE_DIR})
+else()
+    include(FindPackageHandleStandardArgs)
+    find_package_handle_standard_args(SQLite3 DEFAULT_MSG SQLITE3_LIBRARIES SQLITE3_INCLUDE_DIRS)
+endif()
+
+
 
 # show the SQLITE3_INCLUDE_DIRS and SQLITE3_LIBRARIES variables only in the advanced view
 mark_as_advanced(SQLITE3_INCLUDE_DIRS SQLITE3_LIBRARIES)

cmake/modules/NSIS.template.in

@@ -391,6 +391,7 @@ Section "${APPLICATION_NAME}" SEC_APPLICATION
    ; Make sure only to copy qt, not qt_help, etc
    File "${MING_SHARE}\qt5\translations\qt_??.qm"
    File "${MING_SHARE}\qt5\translations\qt_??_??.qm"
+   File "${MING_SHARE}\qt5\translations\qtbase_*.qm"
    File "${MING_SHARE}\qt5\translations\qtkeychain_*.qm"
 
    SetOutPath "$INSTDIR\platforms"

cmake/modules/QtVersionAbstraction.cmake

@@ -153,7 +153,18 @@ if(NOT Qt5Core_FOUND)
         include( ${QT_USE_FILE} )
     endmacro()
 
-    add_definitions("-DQ_DECL_OVERRIDE=override")
+    if (CMAKE_COMPILER_IS_GNUCC)
+        execute_process(COMMAND ${CMAKE_C_COMPILER} -dumpversion
+                OUTPUT_VARIABLE GCC_VERSION)
+        if (GCC_VERSION VERSION_GREATER 4.7 OR GCC_VERSION VERSION_EQUAL 4.7)
+            add_definitions("-DQ_DECL_OVERRIDE=override")
+        else()
+            add_definitions("-DQ_DECL_OVERRIDE=")
+        endif()
+    else() #clang or others
+        add_definitions("-DQ_DECL_OVERRIDE=override")
+    endif()
+
 endif()
 
 if( Qt5Core_DIR )

csync/src/CMakeLists.txt

@@ -4,13 +4,6 @@ add_subdirectory(std)
 add_subdirectory(httpbf)
 
 # Statically include sqlite
-if (CSYNC_STATIC_COMPILE_DIR)
-    set(SQLITE3_INCLUDE_DIRS "")
-    set(SQLITE3_LIBRARIES "")
-    include_directories(${CSYNC_STATIC_COMPILE_DIR})
-else (CSYNC_STATIC_COMPILE_DIR)
-    find_package(SQLite3 3.3.9 REQUIRED)
-endif()
 
 set(CSYNC_PUBLIC_INCLUDE_DIRS
   ${CMAKE_CURRENT_BINARY_DIR}
@@ -86,8 +79,8 @@ set(csync_HDRS
 )
 
 # Statically include sqlite
-if (CSYNC_STATIC_COMPILE_DIR)
+if (USE_OUR_OWN_SQLITE3)
-    list(APPEND csync_SRCS ${CSYNC_STATIC_COMPILE_DIR}/dictionary.c  ${CSYNC_STATIC_COMPILE_DIR}/sqlite3.c)
+    list(APPEND csync_SRCS ${SQLITE3_SOURCE})
 endif()
 
 include_directories(

csync/src/csync.c

@@ -254,6 +254,13 @@ int csync_update(CSYNC *ctx) {
 
   csync_gettime(&finish);
 
+  /* Finalize the sql precompiled statements after the update run since
+   * it runs in its own thread. Precompiled statements shoult not be shared
+   * across thread borders according to
+   * http://www.sqlite.org/cvstrac/wiki?p=MultiThreading
+   */
+  csync_statedb_finalize_statements(ctx);
+
   CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG,
             "Update detection for remote replica took %.2f seconds "
             "walking %zu files.",

csync/src/csync_owncloud.c

@@ -110,7 +110,7 @@ static int ssl_callback_by_neon(void *userdata, int failures,
         }
     }
     DEBUG_WEBDAV("## VERIFY_SSL CERT: %d", ret  );
-      return ret;
+    return ret;
 }
 
 /*
@@ -761,6 +761,7 @@ int owncloud_commit(CSYNC* ctx) {
     }
 
     ctx->owncloud_context->is_first_propfind = true;
+    ctx->owncloud_context->dav_session.no_recursive_propfind = true;
   /* DEBUG_WEBDAV( "********** vio_module_shutdown" ); */
 
   ctx->owncloud_context->dav_session.ctx = 0;

csync/src/csync_private.h

@@ -100,6 +100,8 @@ struct csync_s {
     sqlite3_stmt* by_hash_stmt;
     sqlite3_stmt* by_fileid_stmt;
     sqlite3_stmt* by_inode_stmt;
+
+    int lastReturnValue;
   } statedb;
 
   struct {
@@ -146,8 +148,8 @@ struct csync_s {
   struct csync_owncloud_ctx_s *owncloud_context;
 
   /* hooks for checking the white list */
-  void *checkBlackListData;
+  void *checkSelectiveSyncBlackListData;
-  int (*checkBlackListHook)(void*, const char*);
+  int (*checkSelectiveSyncBlackListHook)(void*, const char*);
 };
 
 

csync/src/csync_reconcile.c

@@ -138,6 +138,12 @@ static int _csync_merge_algorithm_visitor(void *obj, void *data) {
                 /* Do not remove a directory that has ignored files */
                 break;
             }
+            CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Will Remove %s  %d", cur->path, cur->child_modified);
+            if (cur->child_modified) {
+                /* re-create directory that has modified contents */
+                cur->instruction = CSYNC_INSTRUCTION_NEW;
+                break;
+            }
             cur->instruction = CSYNC_INSTRUCTION_REMOVE;
             break;
         case CSYNC_INSTRUCTION_EVAL_RENAME:

csync/src/csync_statedb.c

@@ -50,6 +50,22 @@
 
 #define BUF_SIZE 16
 
+#define sqlite_open(A, B) sqlite3_open_v2(A,B, SQLITE_OPEN_READONLY+SQLITE_OPEN_NOMUTEX, NULL)
+
+#define SQLTM_TIME 150000
+#define SQLTM_COUNT 10
+
+#define SQLITE_BUSY_HANDLED(F) if(1) { \
+    int n = 0; \
+    do { rc = F ; \
+      if( (rc == SQLITE_BUSY) || (rc == SQLITE_LOCKED) ) { \
+         n++; \
+         usleep(SQLTM_TIME); \
+      } \
+    }while( (n < SQLTM_COUNT) && ((rc == SQLITE_BUSY) || (rc == SQLITE_LOCKED))); \
+  }
+
+
 void csync_set_statedb_exists(CSYNC *ctx, int val) {
   ctx->statedb.exists = val;
 }
@@ -73,6 +89,10 @@ static int _csync_check_db_integrity(sqlite3 *db) {
         c_strlist_destroy(result);
     }
 
+    if( sqlite3_threadsafe() == 0 ) {
+        CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "* WARNING: SQLite module is not threadsafe!");
+    }
+
     return rc;
 
 }
@@ -111,7 +131,7 @@ static int _csync_statedb_check(const char *statedb) {
                 if (r >= 0) {
                     buf[BUF_SIZE - 1] = '\0';
                     if (c_streq(buf, "SQLite format 3")) {
-                        if (sqlite3_open(statedb, &db ) == SQLITE_OK) {
+                        if( sqlite_open(statedb, &db ) == SQLITE_OK )  {
                             rc = _csync_check_db_integrity(db);
                             if( sqlite3_close(db) != 0 ) {
                                 CSYNC_LOG(CSYNC_LOG_PRIORITY_NOTICE, "WARN: sqlite3_close error!");
@@ -142,7 +162,9 @@ static int _csync_statedb_check(const char *statedb) {
 
   c_free_locale_string(wstatedb);
 
-  /* create database */
+  /* create database, use the original sqlite3_open function here as opening
+   * read only is not sufficient because that does not create a new db but
+   * bails out with error. */
   rc = sqlite3_open(statedb, &db);
   if (rc == SQLITE_OK) {
     sqlite3_close(db);
@@ -187,6 +209,8 @@ int csync_statedb_load(CSYNC *ctx, const char *statedb, sqlite3 **pdb) {
       return -1;
   }
 
+  ctx->statedb.lastReturnValue = SQLITE_OK;
+
   /* csync_statedb_check tries to open the statedb and creates it in case
    * its not there.
    */
@@ -199,7 +223,7 @@ int csync_statedb_load(CSYNC *ctx, const char *statedb, sqlite3 **pdb) {
   }
 
   /* Open or create the temporary database */
-  if (sqlite3_open(statedb, &db) != SQLITE_OK) {
+  if (sqlite_open(statedb, &db) != SQLITE_OK) {
     const char *errmsg= sqlite3_errmsg(ctx->statedb.db);
     CSYNC_LOG(CSYNC_LOG_PRIORITY_NOTICE, "ERR: Failed to sqlite3 open statedb - bail out: %s.",
               errmsg ? errmsg : "<no sqlite3 errormsg>");
@@ -216,8 +240,16 @@ int csync_statedb_load(CSYNC *ctx, const char *statedb, sqlite3 **pdb) {
     csync_set_statedb_exists(ctx, 1);
   }
 
+  /* Print out the version */
+  //
+  result = csync_statedb_query(db, "SELECT sqlite_version();");
+  if (result && result->count >= 1) {
+      CSYNC_LOG(CSYNC_LOG_PRIORITY_NOTICE, "sqlite3 version \"%s\"", *result->vector);
+  }
+  c_strlist_destroy(result);
+
   /* optimization for speeding up SQLite */
-  result = csync_statedb_query(db, "PRAGMA synchronous = FULL;");
+  result = csync_statedb_query(db, "PRAGMA synchronous = NORMAL;");
   c_strlist_destroy(result);
   result = csync_statedb_query(db, "PRAGMA case_sensitive_like = ON;");
   c_strlist_destroy(result);
@@ -243,21 +275,7 @@ int csync_statedb_close(CSYNC *ctx) {
       return -1;
   }
 
-  /* deallocate query resources */
+  csync_statedb_finalize_statements(ctx);
-  if( ctx->statedb.by_hash_stmt ) {
-      rc = sqlite3_finalize(ctx->statedb.by_hash_stmt);
-      ctx->statedb.by_hash_stmt = NULL;
-  }
-
-  if( ctx->statedb.by_fileid_stmt ) {
-      rc = sqlite3_finalize(ctx->statedb.by_fileid_stmt);
-      ctx->statedb.by_fileid_stmt = NULL;
-  }
-
-  if( ctx->statedb.by_inode_stmt ) {
-      rc = sqlite3_finalize(ctx->statedb.by_inode_stmt);
-      ctx->statedb.by_inode_stmt = NULL;
-  }
 
   sqlite3_close(ctx->statedb.db);
 
@@ -281,7 +299,7 @@ static int _csync_file_stat_from_metadata_table( csync_file_stat_t **st, sqlite3
 
     column_count = sqlite3_column_count(stmt);
 
-    rc = sqlite3_step(stmt);
+    SQLITE_BUSY_HANDLED( sqlite3_step(stmt) );
 
     if( rc == SQLITE_ROW ) {
         if(column_count > 7) {
@@ -344,7 +362,8 @@ csync_file_stat_t *csync_statedb_get_stat_by_hash(CSYNC *ctx,
   if( ctx->statedb.by_hash_stmt == NULL ) {
       const char *hash_query = "SELECT * FROM metadata WHERE phash=?1";
 
-      rc = sqlite3_prepare_v2(ctx->statedb.db, hash_query, strlen(hash_query), &ctx->statedb.by_hash_stmt, NULL);
+      SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, hash_query, strlen(hash_query), &ctx->statedb.by_hash_stmt, NULL));
+      ctx->statedb.lastReturnValue = rc;
       if( rc != SQLITE_OK ) {
           CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Unable to create stmt for hash query.");
           return NULL;
@@ -358,6 +377,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_hash(CSYNC *ctx,
   sqlite3_bind_int64(ctx->statedb.by_hash_stmt, 1, (long long signed int)phash);
 
   rc = _csync_file_stat_from_metadata_table(&st, ctx->statedb.by_hash_stmt);
+  ctx->statedb.lastReturnValue = rc;
   if( !(rc == SQLITE_ROW || rc == SQLITE_DONE) )  {
       CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Could not get line from metadata: %d!", rc);
   }
@@ -366,6 +386,28 @@ csync_file_stat_t *csync_statedb_get_stat_by_hash(CSYNC *ctx,
   return st;
 }
 
+void   csync_statedb_finalize_statements(CSYNC *ctx) {
+    if( !ctx ) {
+        return;
+    }
+
+    /* deallocate query resources */
+    if( ctx->statedb.by_fileid_stmt ) {
+        sqlite3_finalize(ctx->statedb.by_fileid_stmt);
+        ctx->statedb.by_fileid_stmt = NULL;
+    }
+    if( ctx->statedb.by_hash_stmt ) {
+        sqlite3_finalize(ctx->statedb.by_hash_stmt);
+        ctx->statedb.by_hash_stmt = NULL;
+    }
+    if( ctx->statedb.by_inode_stmt) {
+        sqlite3_finalize(ctx->statedb.by_inode_stmt);
+        ctx->statedb.by_inode_stmt = NULL;
+    }
+
+    ctx->statedb.lastReturnValue = SQLITE_OK;
+}
+
 csync_file_stat_t *csync_statedb_get_stat_by_file_id(CSYNC *ctx,
                                                       const char *file_id ) {
     csync_file_stat_t *st = NULL;
@@ -385,7 +427,8 @@ csync_file_stat_t *csync_statedb_get_stat_by_file_id(CSYNC *ctx,
     if( ctx->statedb.by_fileid_stmt == NULL ) {
         const char *query = "SELECT * FROM metadata WHERE fileid=?1";
 
-        rc = sqlite3_prepare_v2(ctx->statedb.db, query, strlen(query), &ctx->statedb.by_fileid_stmt, NULL);
+        SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, query, strlen(query), &ctx->statedb.by_fileid_stmt, NULL));
+        ctx->statedb.lastReturnValue = rc;
         if( rc != SQLITE_OK ) {
             CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Unable to create stmt for file id query.");
             return NULL;
@@ -396,6 +439,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_file_id(CSYNC *ctx,
     sqlite3_bind_text(ctx->statedb.by_fileid_stmt, 1, file_id, -1, SQLITE_STATIC);
 
     rc = _csync_file_stat_from_metadata_table(&st, ctx->statedb.by_fileid_stmt);
+    ctx->statedb.lastReturnValue = rc;
     if( !(rc == SQLITE_ROW || rc == SQLITE_DONE) ) {
         CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Could not get line from metadata: %d!", rc);
     }
@@ -423,7 +467,8 @@ csync_file_stat_t *csync_statedb_get_stat_by_inode(CSYNC *ctx,
   if( ctx->statedb.by_inode_stmt == NULL ) {
       const char *inode_query = "SELECT * FROM metadata WHERE inode=?1";
 
-      rc = sqlite3_prepare_v2(ctx->statedb.db, inode_query, strlen(inode_query), &ctx->statedb.by_inode_stmt, NULL);
+      SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, inode_query, strlen(inode_query), &ctx->statedb.by_inode_stmt, NULL));
+      ctx->statedb.lastReturnValue = rc;
       if( rc != SQLITE_OK ) {
           CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Unable to create stmt for inode query.");
           return NULL;
@@ -437,6 +482,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_inode(CSYNC *ctx,
   sqlite3_bind_int64(ctx->statedb.by_inode_stmt, 1, (long long signed int)inode);
 
   rc = _csync_file_stat_from_metadata_table(&st, ctx->statedb.by_inode_stmt);
+  ctx->statedb.lastReturnValue = rc;
   if( !(rc == SQLITE_ROW || rc == SQLITE_DONE) ) {
       CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Could not get line from metadata by inode: %d!", rc);
   }
@@ -485,7 +531,8 @@ int csync_statedb_get_below_path( CSYNC *ctx, const char *path ) {
         return -1;
     }
 
-    rc = sqlite3_prepare_v2(ctx->statedb.db, BELOW_PATH_QUERY, -1, &stmt, NULL);
+    SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, BELOW_PATH_QUERY, -1, &stmt, NULL));
+    ctx->statedb.lastReturnValue = rc;
     if( rc != SQLITE_OK ) {
       CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Unable to create stmt for below path query.");
       return -1;
@@ -507,6 +554,7 @@ int csync_statedb_get_below_path( CSYNC *ctx, const char *path ) {
 
     cnt = 0;
 
+    ctx->statedb.lastReturnValue = rc;
     do {
         csync_file_stat_t *st = NULL;
 
@@ -522,6 +570,7 @@ int csync_statedb_get_below_path( CSYNC *ctx, const char *path ) {
         }
     } while( rc == SQLITE_ROW );
 
+    ctx->statedb.lastReturnValue = rc;
     if( rc != SQLITE_DONE ) {
         ctx->status_code = CSYNC_STATUS_TREE_ERROR;
     } else {

csync/src/csync_statedb.h

@@ -99,25 +99,10 @@ int csync_statedb_get_below_path(CSYNC *ctx, const char *path);
 c_strlist_t *csync_statedb_query(sqlite3 *db, const char *statement);
 
 /**
- * @brief Insert function for the statedb.
+ * @brief csync_statedb_finalize_statements - Clear prepared statements
- *
+ * @param ctx The csync context
- * @param ctx        The csync context.
- * @param statement  The SQL statement to insert into the statedb.
- *
- * @return  The rowid of the most recent INSERT on success, 0 if the query
- *          wasn't successful.
  */
-typedef struct csync_progressinfo_s {
+void csync_statedb_finalize_statements(CSYNC *ctx);
-  struct csync_progressinfo_s *next;
-  uint64_t phash;
-  uint64_t modtime;
-  char *md5;
-  int error;
-  int chunk;
-  int transferId;
-  char *tmpfile;
-  char *error_string;
-} csync_progressinfo_t;
 
 #ifdef __cplusplus
 }

csync/src/csync_update.c

@@ -100,6 +100,10 @@ static bool _csync_sameextension(const char *p1, const char *p2) {
 }
 #endif
 
+static bool _last_db_return_error(CSYNC* ctx) {
+    return ctx->statedb.lastReturnValue != SQLITE_OK && ctx->statedb.lastReturnValue != SQLITE_DONE && ctx->statedb.lastReturnValue != SQLITE_ROW;
+}
+
 static int _csync_detect_update(CSYNC *ctx, const char *file,
     const csync_vio_file_stat_t *fs, const int type) {
   uint64_t h = 0;
@@ -157,8 +161,8 @@ static int _csync_detect_update(CSYNC *ctx, const char *file,
     }
   }
 
-  if (ctx->current == REMOTE_REPLICA && ctx->checkBlackListHook) {
+  if (ctx->current == REMOTE_REPLICA && ctx->checkSelectiveSyncBlackListHook) {
-      if (ctx->checkBlackListHook(ctx->checkBlackListData, path)) {
+      if (ctx->checkSelectiveSyncBlackListHook(ctx->checkSelectiveSyncBlackListData, path)) {
           return 1;
       }
   }
@@ -190,8 +194,15 @@ static int _csync_detect_update(CSYNC *ctx, const char *file,
     }
 
     if (fs->mtime == 0) {
-      tmp = csync_statedb_get_stat_by_hash(ctx, h);
       CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file: %s - mtime is zero!", path);
+
+      tmp = csync_statedb_get_stat_by_hash(ctx, h);
+      if(_last_db_return_error(ctx)) {
+          SAFE_FREE(st);
+          ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
+          return -1;
+      }
+
       if (tmp == NULL) {
         CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file: %s - not found in db, IGNORE!", path);
         st->instruction = CSYNC_INSTRUCTION_IGNORE;
@@ -228,6 +239,12 @@ static int _csync_detect_update(CSYNC *ctx, const char *file,
   if (csync_get_statedb_exists(ctx)) {
     tmp = csync_statedb_get_stat_by_hash(ctx, h);
 
+    if(_last_db_return_error(ctx)) {
+        SAFE_FREE(st);
+        ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
+        return -1;
+    }
+
     if(tmp && tmp->phash == h ) { /* there is an entry in the database */
         /* we have an update! */
         CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "Database entry found, compare: %" PRId64 " <-> %" PRId64 ", etag: %s <-> %s, inode: %" PRId64 " <-> %" PRId64 ", size: %" PRId64 " <-> %" PRId64,
@@ -289,6 +306,12 @@ static int _csync_detect_update(CSYNC *ctx, const char *file,
 
             tmp = csync_statedb_get_stat_by_inode(ctx, fs->inode);
 
+            if(_last_db_return_error(ctx)) {
+                SAFE_FREE(st);
+                ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
+                return -1;
+            }
+
             /* translate the file type between the two stat types csync has. */
             if( tmp && tmp->type == 0 ) {
                 tmp_vio_type = CSYNC_VIO_FILE_TYPE_REGULAR;
@@ -319,6 +342,12 @@ static int _csync_detect_update(CSYNC *ctx, const char *file,
         } else {
             /* Remote Replica Rename check */
             tmp = csync_statedb_get_stat_by_file_id(ctx, fs->file_id);
+
+            if(_last_db_return_error(ctx)) {
+                SAFE_FREE(st);
+                ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
+                return -1;
+            }
             if(tmp ) {                           /* tmp existing at all */
                 if ((tmp->type == CSYNC_FTW_TYPE_DIR && fs->type != CSYNC_VIO_FILE_TYPE_DIRECTORY) ||
                         (tmp->type == CSYNC_FTW_TYPE_FILE && fs->type != CSYNC_VIO_FILE_TYPE_REGULAR)) {
@@ -603,10 +632,13 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
     path = filename + ulen;
 
     /* skip ".csync_journal.db" and ".csync_journal.db.ctmp" */
+    /* Isn't this done via csync_exclude already? */
     if (c_streq(path, ".csync_journal.db")
             || c_streq(path, ".csync_journal.db.ctmp")
             || c_streq(path, ".csync_journal.db.ctmp-journal")
-            || c_streq(path, ".csync-progressdatabase")) {
+            || c_streq(path, ".csync-progressdatabase")
+            || c_streq(path, ".csync_journal.db-shm")
+            || c_streq(path, ".csync_journal.db-wal")) {
         csync_vio_file_stat_destroy(dirent);
         dirent = NULL;
         SAFE_FREE(filename);
@@ -652,6 +684,11 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
         uint64_t h = c_jhash64((uint8_t *) path, len, 0);
         etag = csync_statedb_get_etag( ctx, h );
 
+        if(_last_db_return_error(ctx)) {
+            ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL;
+            goto error;
+        }
+
         if( etag ) {
             SAFE_FREE(fs->etag);
             fs->etag = etag;
@@ -671,11 +708,6 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
     rc = fn(ctx, filename, fs, flag);
     /* this function may update ctx->current and ctx->read_from_db */
 
-    if (ctx->current_fs && previous_fs && ctx->current_fs->child_modified) {
-        /* If a directory has modified files, put the flag on the parent directory as well */
-        previous_fs->child_modified = ctx->current_fs->child_modified;
-    }
-
     /* Only for the local replica we have to destroy stat(), for the remote one it is a pointer to dirent */
     if (ctx->replica == LOCAL_REPLICA) {
         csync_vio_file_stat_destroy(fs);
@@ -710,6 +742,11 @@ int csync_ftw(CSYNC *ctx, const char *uri, csync_walker_fn fn,
       }
     }
 
+    if (ctx->current_fs && previous_fs && ctx->current_fs->child_modified) {
+        /* If a directory has modified files, put the flag on the parent directory as well */
+        previous_fs->child_modified = ctx->current_fs->child_modified;
+    }
+
     if (flag == CSYNC_FTW_FLAG_DIR && ctx->current_fs
         && (ctx->current_fs->instruction == CSYNC_INSTRUCTION_EVAL ||
             ctx->current_fs->instruction == CSYNC_INSTRUCTION_NEW)) {

csync/tests/csync_tests/check_csync_statedb_query.c

@@ -55,19 +55,11 @@ static void setup(void **state)
 
 static void setup_db(void **state)
 {
-    CSYNC *csync;
+    char *errmsg;
-    char *stmt = NULL;
     int rc = 0;
-    c_strlist_t *result = NULL;
+    sqlite3 *db = NULL;
 
-    setup(state);
+    const char *sql = "CREATE TABLE IF NOT EXISTS metadata ("
-    csync = *state;
-
-    // rc = csync_statedb_create_tables(csync->statedb.db);
-    assert_int_equal(rc, 0);
-
-    result = csync_statedb_query(csync->statedb.db,
-        "CREATE TABLE IF NOT EXISTS metadata ("
         "phash INTEGER(8),"
         "pathlen INTEGER,"
         "path VARCHAR(4096),"
@@ -79,29 +71,25 @@ static void setup_db(void **state)
         "type INTEGER,"
         "md5 VARCHAR(32),"
         "PRIMARY KEY(phash)"
-        ");"
+        ");";
-        );
 
-    assert_non_null(result);
+        const char *sql2 = "INSERT INTO metadata"
-    c_strlist_destroy(result);
-
-
-    stmt = sqlite3_mprintf("INSERT INTO metadata"
                            "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5) VALUES"
-                           "(%lu, %d, '%q', %d, %d, %d, %d, %lu, %d, %lu);",
+                           "(42, 42, 'Its funny stuff', 23, 42, 43, 55, 66, 2, 54);";
-                           42,
+
-                           42,
+
-                           "It's a rainy day",
+    setup(state);
-                           23,
+    rc = sqlite3_open( TESTDB, &db);
-                           42,
+    assert_int_equal(rc, SQLITE_OK);
-                           42,
+
-                           42,
+    rc = sqlite3_exec( db, sql, NULL, NULL, &errmsg );
-                           42,
+    assert_int_equal(rc, SQLITE_OK);
-                           2,
+
-                           43);
+    rc = sqlite3_exec( db, sql2, NULL, NULL, &errmsg );
+    assert_int_equal(rc, SQLITE_OK);
+
+    sqlite3_close(db);
 
-    // rc = csync_statedb_insert(csync->statedb.db, stmt);
-    sqlite3_free(stmt);
 }
 
 static void teardown(void **state) {
@@ -139,41 +127,6 @@ static void check_csync_statedb_query_statement(void **state)
     }
 }
 
-static void check_csync_statedb_create_error(void **state)
-{
-    CSYNC *csync = *state;
-    c_strlist_t *result;
-
-    result = csync_statedb_query(csync->statedb.db, "CREATE TABLE test(phash INTEGER, text VARCHAR(10));");
-    assert_non_null(result);
-    c_strlist_destroy(result);
-
-    result = csync_statedb_query(csync->statedb.db, "CREATE TABLE test(phash INTEGER, text VARCHAR(10));");
-    assert_null(result);
-
-    c_strlist_destroy(result);
-}
-
-static void check_csync_statedb_insert_statement(void **state)
-{
-    CSYNC *csync = *state;
-    c_strlist_t *result;
-    int rc = 0;
-
-    result = csync_statedb_query(csync->statedb.db, "CREATE TABLE test(phash INTEGER, text VARCHAR(10));");
-    assert_non_null(result);
-    c_strlist_destroy(result);
-
-    // rc = csync_statedb_insert(csync->statedb.db, "INSERT;");
-    assert_int_equal(rc, 0);
-    // rc = csync_statedb_insert(csync->statedb.db, "INSERT");
-    assert_int_equal(rc, 0);
-    // rc = csync_statedb_insert(csync->statedb.db, "");
-    assert_int_equal(rc, 0);
-}
-
-
-
 static void check_csync_statedb_drop_tables(void **state)
 {
     // CSYNC *csync = *state;
@@ -255,8 +208,6 @@ int torture_run_tests(void)
 {
     const UnitTest tests[] = {
         unit_test_setup_teardown(check_csync_statedb_query_statement, setup, teardown),
-        unit_test_setup_teardown(check_csync_statedb_create_error, setup, teardown),
-        unit_test_setup_teardown(check_csync_statedb_insert_statement, setup, teardown),
         unit_test_setup_teardown(check_csync_statedb_drop_tables, setup, teardown),
         unit_test_setup_teardown(check_csync_statedb_insert_metadata, setup, teardown),
         unit_test_setup_teardown(check_csync_statedb_write, setup, teardown),

csync/tests/csync_tests/check_csync_update.c

@@ -23,11 +23,64 @@
 
 #define TESTDB "/tmp/check_csync/journal.db"
 
+static int firstrun = 1;
+
+static void statedb_create_metadata_table(sqlite3 *db)
+{
+    int rc = 0;
+
+    if( db ) {
+        const char *sql = "CREATE TABLE IF NOT EXISTS metadata("
+                          "phash INTEGER(8),"
+                          "pathlen INTEGER,"
+                          "path VARCHAR(4096),"
+                          "inode INTEGER,"
+                          "uid INTEGER,"
+                          "gid INTEGER,"
+                          "mode INTEGER,"
+                          "modtime INTEGER(8),"
+                          "type INTEGER,"
+                          "md5 VARCHAR(32),"
+                          "PRIMARY KEY(phash));";
+
+        rc = sqlite3_exec(db, sql, NULL, NULL, NULL);
+        const char *msg = sqlite3_errmsg(db);
+        assert_int_equal( rc, SQLITE_OK );
+    }
+}
+
+static void statedb_insert_metadata(sqlite3 *db)
+{
+    int rc = 0;
+
+    if( db ) {
+        char *stmt = sqlite3_mprintf("INSERT INTO metadata"
+                                     "(phash, pathlen, path, inode, uid, gid, mode, modtime,type,md5) VALUES"
+                                     "(%lld, %d, '%q', %d, %d, %d, %d, %lld, %d, '%q');",
+                                     (long long signed int)42,
+                                     42,
+                                     "I_was_wurst_before_I_became_wurstsalat",
+                                     619070,
+                                     42,
+                                     42,
+                                     42,
+                                     (long long signed int)42,
+                                     0,
+                                     "4711");
+
+        char *errmsg;
+        rc = sqlite3_exec(db, stmt, NULL, NULL, &errmsg);
+        sqlite3_free(stmt);
+        assert_int_equal( rc, SQLITE_OK );
+    }
+}
+
 static void setup(void **state)
 {
     CSYNC *csync;
     int rc;
 
+    unlink(TESTDB);
     rc = system("mkdir -p /tmp/check_csync");
     assert_int_equal(rc, 0);
     rc = system("mkdir -p /tmp/check_csync1");
@@ -38,10 +91,21 @@ static void setup(void **state)
     assert_int_equal(rc, 0);
     rc = csync_init(csync);
     assert_int_equal(rc, 0);
+
+    /* Create a new db with metadata */
+    sqlite3 *db;
+    csync->statedb.file = c_strdup(TESTDB);
+    rc = sqlite3_open(csync->statedb.file, &db);
+    statedb_create_metadata_table(db);
+    if( firstrun ) {
+        statedb_insert_metadata(db);
+        firstrun = 0;
+    }
+    sqlite3_close(db);
+
     rc = csync_statedb_load(csync, TESTDB, &csync->statedb.db);
     assert_int_equal(rc, 0);
 
-
     *state = csync;
 }
 
@@ -71,6 +135,7 @@ static void teardown(void **state)
     CSYNC *csync = *state;
     int rc;
 
+    unlink( csync->statedb.file);
     rc = csync_destroy(csync);
     assert_int_equal(rc, 0);
 
@@ -249,6 +314,7 @@ static void check_csync_detect_update_db_eval(void **state)
     csync_vio_file_stat_destroy(fs);
 }
 
+
 static void check_csync_detect_update_db_rename(void **state)
 {
     CSYNC *csync = *state;
@@ -256,27 +322,6 @@ static void check_csync_detect_update_db_rename(void **state)
 
     csync_vio_file_stat_t *fs;
     int rc = 0;
-    char *stmt = NULL;
-
-    // rc = csync_statedb_create_tables(csync->statedb.db);
-
-    assert_int_equal(rc, 0);
-    stmt = sqlite3_mprintf("INSERT INTO metadata"
-                           "(phash, pathlen, path, inode, uid, gid, mode, modtime,type,md5) VALUES"
-                           "(%lld, %d, '%q', %d, %d, %d, %d, %lld, %d, '%q');",
-                           (long long signed int)42,
-                           42,
-                           "I_was_wurst_before_I_became_wurstsalat",
-                           619070,
-                           42,
-                           42,
-                           42,
-                           (long long signed int)42,
-                           0,
-                           "4711");
-
-    // rc = csync_statedb_insert(csync->statedb.db, stmt);
-    sqlite3_free(stmt);
 
     fs = create_fstat("wurst.txt", 0, 1, 42);
     assert_non_null(fs);

csync/tests/ownCloud/ownCloud/Test.pm

@@ -63,7 +63,7 @@ our %config;
 
 @ISA        = qw(Exporter);
 @EXPORT     = qw( initTesting createRemoteDir removeRemoteDir createLocalDir cleanup csync
-                  assertLocalDirs assertLocalAndRemoteDir glob_put put_to_dir 
+                  assertLocalDirs assertLocalAndRemoteDir glob_put put_to_dir
                   putToDirLWP localDir remoteDir localCleanup createLocalFile md5OfFile
                   remoteCleanup server initLocalDir initRemoteDir moveRemoteFile
                   printInfo remoteFileId createShare removeShare assert
@@ -73,12 +73,12 @@ sub server
 {
   return $owncloud;
 }
-                  
+
 sub fromFileName($)
 {
   my ($file) = @_;
   if ( $^O eq "darwin" ) {
-    my $fromFileName = NFC( Encode::decode('utf-8', $file) ); 
+    my $fromFileName = NFC( Encode::decode('utf-8', $file) );
     return $fromFileName;
   } else {
     return $file;
@@ -89,7 +89,7 @@ sub fromFileName($)
 sub initTesting(;$)
 {
   my ($prefix) = @_;
-  
+
   my $cfgFile = "./t1.cfg";
   $cfgFile = "/etc/ownCloud/t1.cfg" if( -r "/etc/ownCloud/t1.cfg" );
 
@@ -115,6 +115,8 @@ sub initTesting(;$)
 
   $owncloud .= "/" unless( $owncloud =~ /\/$/ );
 
+  $ENV{PERL_LWP_SSL_VERIFY_HOSTNAME} = 0;
+
   print "Connecting to ownCloud at ". $owncloud ."\n";
 
   # For SSL set the environment variable needed by the LWP module for SSL
@@ -129,15 +131,15 @@ sub initTesting(;$)
 		  -pass=> $passwd );
   # $d->DebugLevel(3);
   $prefix = "t1" unless( defined $prefix );
-  
+
   my $dirId = sprintf("%02d", rand(100));
   my $dateTime = strftime('%Y%m%d%H%M%S',localtime);
   my $dir = sprintf( "%s-%s-%s/", $prefix, $dateTime, $dirId );
-  
+
   $localDir = $dir;
   $localDir .= "/" unless( $localDir =~ /\/$/ );
   $remoteDir = $dir;
-  
+
   initRemoteDir();
   initLocalDir();
   printf( "Test directory name is %s\n", $dir );
@@ -166,7 +168,9 @@ sub testDirUrl()
 # the global var $remoteDir;
 sub initRemoteDir
 {
-  $d->open( $owncloud );
+  $d->open( $owncloud )
+       or die("Couldn't open $owncloud: " .$d->message . "\n");
+
   my $url = testDirUrl();
 
   my $re = $d->mkcol( $url );
@@ -204,7 +208,7 @@ sub removeRemoteDir($;$)
     if( $re == 0 ) {
 	print "Failed to remove directory <$url>:" . $d->message() ."\n";
     }
-  
+
     return $re;
 }
 
@@ -336,7 +340,7 @@ sub localDir()
   return $localDir;
 }
 
-sub remoteDir() 
+sub remoteDir()
 {
   return $remoteDir;
 }
@@ -381,7 +385,7 @@ sub traverse( $$;$ )
 {
     my ($remote, $acceptConflicts, $aurl) = @_;
     $remote .= '/' unless $remote =~ /(^|\/)$/;
-    
+
     my $url = testDirUrl() . $remote;
     if( $aurl ) {
 	$url = $aurl . $remote;
@@ -489,13 +493,13 @@ sub glob_put( $$;$ )
 	      print "   *** Putting $lfile to $puturl\n";
 	      # putToDirLWP( $lfile, $puturl );
 	      put_to_dir($lfile, $puturl, $optionsRef);
-	      
+
 	      # if( ! $d->put( -local=>$lfile, -url=> $puturl ) ) {
 	      #print "   ### FAILED to put: ". $d->message . '\n';
 	      # s}
 	    }
 	}
-	  
+
     }
 }
 
@@ -525,7 +529,7 @@ sub put_to_dir( $$;$ )
     }
 }
 
-# The HTTP DAV module often does a PROPFIND before it really PUTs. That 
+# The HTTP DAV module often does a PROPFIND before it really PUTs. That
 # is not neccessary if we know that the directory is really there.
 # Use this function in this case:
 sub putToDirLWP($$)
@@ -545,13 +549,13 @@ sub putToDirLWP($$)
     my $string = <FILE>;
     close FILE;
 
-    my $ua  = LWP::UserAgent->new();
+    my $ua  = LWP::UserAgent->new( ssl_opts => { verify_hostname => 0 });
     $ua->agent( "ownCloudTest_$localDir");
     my $req = PUT $puturl, Content_Type => 'application/octet-stream',
 		  Content => $string;
     $req->authorization_basic($user, $passwd);
     my $response = $ua->request($req);
-    
+
     if ($response->is_success()) {
       # print "OK: ", $response->content;
     } else {
@@ -579,7 +583,7 @@ sub getToFileLWP( $$ )
     my $geturl = testDirUrl() . $file;
     print "GETting $geturl to $localFile\n";
 
-    my $ua  = LWP::UserAgent->new();
+    my $ua  = LWP::UserAgent->new( ssl_opts => { verify_hostname => 0 });
     $ua->agent( "ownCloudTest_$localDir");
     $ua->credentials( server(), "foo", $user, $passwd);
     my $req = $ua->get($geturl, ":content_file" => $localFile);
@@ -594,15 +598,15 @@ sub getToFileLWP( $$ )
     }
 }
 
-sub createLocalFile( $$ ) 
+sub createLocalFile( $$ )
 {
   my ($fname, $size) = @_;
   $size = 1024 unless( $size );
-  
+
   my $md5 = Digest::MD5->new;
 
   open(FILE, ">", $fname) or die "Can't open $fname for writing ($!)";
-  
+
   my $minimum = 32;
   my $range = 96;
 
@@ -620,20 +624,20 @@ sub createLocalFile( $$ )
   print FILE $s;
   $md5->add($s);
   close FILE;
-  return $md5->hexdigest; 
+  return $md5->hexdigest;
 }
 
-sub md5OfFile( $ ) 
+sub md5OfFile( $ )
 {
   my ($file) = @_;
-  
+
   open FILE, "$file";
 
   my $ctx = Digest::MD5->new;
   $ctx->addfile (*FILE);
   my $hash = $ctx->hexdigest;
   close (FILE);
-  
+
   return $hash;
 }
 
@@ -647,27 +651,27 @@ sub moveRemoteFile($$;$)
 
   my $fromUrl = testDirUrl(). $from;
   my $toUrl = testDirUrl() . $to;
-  
+
   if( $no_testdir ) {
      $fromUrl = $from;
      $toUrl = $to;
   }
-  
+
   $d->move($fromUrl, $toUrl);
-  
+
 }
 
 sub printInfo($)
 {
   my ($info) = @_;
   my $tt = 6+length( $info );
-  
+
   print "#" x $tt;
   printf( "\n# %2d. %s", $infoCnt, $info );
   print "\n" unless $info =~ /\n$/;
   print "#" x $tt;
   print "\n";
-  
+
   $infoCnt++;
 }
 
@@ -718,16 +722,15 @@ sub createShare($$)
     my $re = $dd->mkcol( $url );
     if( $re == 0 ) {
 	print "Failed to create test dir $url\n";
-
     }
 
-    my $ua  = LWP::UserAgent->new();
+    my $ua  = LWP::UserAgent->new(ssl_opts => { verify_hostname => 0 } );
     $ua->agent( "ownCloudTest_sharing");
     # http://localhost/ocm/ocs/v1.php/apps/files_sharing/api/v1/shares
-    my $puturl = $ocs_url . "apps/files_sharing/api/v1/shares";
+    my $puturl = $ocs_url . "ocs/v1.php/apps/files_sharing/api/v1/shares";
 
     my $string = "path=$dir&shareType=0&shareWith=$user&publicUpload=false&permissions=$readWrite";
-    print ">>>>>>>>>> $string\n";
+    print ">>>>>>>>>> $puturl $string\n";
 
     my $req = POST $puturl, Content => $string;
     $req->authorization_basic($share_user, $share_passwd);
@@ -757,17 +760,16 @@ sub removeShare($$)
 		      -pass => $share_passwd );
     $dd->open( $owncloud);
 
-    my $ua  = LWP::UserAgent->new();
+    my $ua  = LWP::UserAgent->new(ssl_opts => { verify_hostname => 0 });
     $ua->agent( "ownCloudTest_sharing");
-    # http://localhost/ocm/ocs/v1.php/apps/files_sharing/api/v1/shares
+
-    my $url = $ocs_url . "apps/files_sharing/api/v1/shares/" . $shareId;
+    my $url = $ocs_url . "ocs/v1.php/apps/files_sharing/api/v1/shares/" . $shareId;
 
     my $req = DELETE $url;
     $req->authorization_basic($share_user, $share_passwd);
     my $response = $ua->request($req);
 
     if ($response->is_success()) {
-      # print "OK: ", $response->content;
 	print $response->decoded_content;
 	if( $response->decoded_content =~ /<status_code>(\d+)<\/status_code>/m) {
 	    my $code = $1;

csync/tests/ownCloud/t1.pl

@@ -169,6 +169,19 @@ assertLocalAndRemoteDir( '', 0);
 assert( -e localDir().'remoteToLocal1/rtlX' );
 assert( -e localDir().'remoteToLocal1/rtlX/rtl11/file.txt' );
 
+printInfo( "Remove a directory on the server with new files on the client");
+removeRemoteDir('remoteToLocal1/rtlX');
+system("echo hello > " . localDir(). "remoteToLocal1/rtlX/rtl11/hello.txt");
+csync();
+assertLocalAndRemoteDir( '', 0);
+# file.txt must be gone because the directory was removed on the server, but hello.txt must be there
+#   as it is a new file
+assert( ! -e localDir().'remoteToLocal1/rtlX/rtl11/file.txt' );
+assert( -e localDir().'remoteToLocal1/rtlX/rtl11/hello.txt' );
+
+
+
+
 # ==================================================================
 
 cleanup();

csync/tests/ownCloud/t6.pl

@@ -33,11 +33,13 @@ print "Hello, this is t6, a tester for csync with ownCloud.\n";
 
 initTesting();
 
-sub createPostUpdateScript()
+sub createPostUpdateScript($)
 {
-    my $srcFile = localDir()."BIG.file";
+  my ($name) = @_;
+
+    my $srcFile = localDir().'BIG1.file';
     my $cred = configValue("user") . ":" . configValue("passwd");
-    my $cmd = "curl -T $srcFile -u $cred " . testDirUrl();
+    my $cmd = "curl -T $srcFile -u $cred " . testDirUrl().$name;
     my $script = "/tmp/post_update_script.sh";
     open SC, ">$script" || die("Can not create script file");
     print SC "#!/bin/bash\n";
@@ -48,11 +50,11 @@ sub createPostUpdateScript()
     return $script;
 }
 
-sub getETagFromJournal($)
+sub getETagFromJournal($$)
 {
-    my ($num) = @_;
+    my ($name,$num) = @_;
-    
+
-    my $sql = "sqlite3 " . localDir() . ".csync_journal.db \"SELECT md5 FROM metadata WHERE path='BIG.file';\"";
+    my $sql = "sqlite3 " . localDir() . ".csync_journal.db \"SELECT md5 FROM metadata WHERE path='$name';\"";
     open(my $fh, '-|', $sql) or die $!;
     my $etag  = <$fh>;
     close $fh;
@@ -61,14 +63,14 @@ sub getETagFromJournal($)
     return $etag;
 }
 
-sub chunkFileTest( $$ ) 
+sub chunkFileTest( $$ )
 {
     my ($name, $size) = @_;
 
     # Big file chunking
     createLocalFile( localDir().$name, $size );
     assert( -e localDir().$name );
-    
+
     my $bigMd5 = md5OfFile( localDir().$name );
 
     csync();
@@ -89,26 +91,39 @@ sub chunkFileTest( $$ )
 }
 
 printInfo("Big file that needs chunking with default chunk size");
-chunkFileTest( "BIG.file", 23251233 );
+chunkFileTest( "BIG1.file", 23251233 );
 
 printInfo("Update the existing file and trigger reupload");
 # change the existing file again -> update
-chunkFileTest( "BIG.file", 21762122 );
+chunkFileTest( "BIG2.file", 21762122 );
 
 printInfo("Cause a precondition failed error");
 # Now overwrite the existing file to change it
-createLocalFile( localDir()."BIG.file", 21832199 );
+createLocalFile( localDir()."BIG3.file", 21832 );
+sleep(2);
+csync();
+createLocalFile( localDir().'BIG3.file', 34323 );
+sleep(2);
 # and create a post update script
-my $script = createPostUpdateScript();
+my $script = createPostUpdateScript('BIG3.file');
 $ENV{'OWNCLOUD_POST_UPDATE_SCRIPT'} = $script;
 
 # Save the etag before the sync
-my $firstETag = getETagFromJournal('First');
+my $firstETag = getETagFromJournal('BIG3.file', 'First');
-csync(); # Sync, which ends in a precondition failed error 
+sleep(2);
+csync(); # Sync, which ends in a precondition failed error
 # get the etag again. It has to be unchanged because of the error.
-my $secondETag = getETagFromJournal('Second');
+my $secondETag = getETagFromJournal('BIG3.file', 'Second');
-assert( $firstETag eq $secondETag, "Different ETags, no precondition error." );
 
+# Now the result is that there is a conflict file because since 1.7
+# the sync is stopped on preconditoin failed and done again.
+my $seen = 0;
+opendir(my $dh, localDir() );
+while(readdir $dh) {
+  $seen = 1 if ( /BIG3_conflict.*\.file/ );
+}
+closedir $dh;
+assert( $seen == 1, "No conflict file created on precondition failed!" );
 unlink($script);
 
 # Set a custom chunk size in environment.

csync/tests/ownCloud/t8.pl

@@ -48,6 +48,7 @@ mkdir($tmpdir);
 createLocalFile( $tmpdir . "HELLO.dat", 100 );
 createLocalFile( $tmpdir . "Hello.dat", 150 );
 createLocalFile( $tmpdir . "Normal.dat", 110 );
+createLocalFile( $tmpdir . "test.dat", 170 );
 
 #put them in some directories
 createRemoteDir( "dir" );
@@ -73,14 +74,20 @@ assertLocalAndRemoteDir( '', 0);
 
 printInfo( "Renaming one file to the same name as another one with different casing" );
 moveRemoteFile( 'dir/Hello.dat', 'dir/NORMAL.dat');
+moveRemoteFile( 'dir/test.dat', 'dir/TEST.dat');
 
 csync();
 
-#It should not have do the move
+# Hello -> NORMAL should not have do the move since the case conflict
 assert( -e localDir() . 'dir/Hello.dat' );
 assert( !-e localDir() . 'dir/NORMAL.dat' );
 assert( -e localDir() . 'dir/Normal.dat' );
 
+#test->TEST should have been worked.
+assert( -e localDir() . 'dir/TEST.dat' );
+assert( !-e localDir() . 'dir/test.dat' );
+
+
 printInfo( "Another directory with the same name but different casing is created" );
 
 createRemoteDir( "DIR" );

shell_integration/MacOSX/OwnCloudFinder/ContentManager.m

@@ -185,7 +185,7 @@ static ContentManager* sharedInstance = nil;
 	}
 	
 	if( [keysToDelete count] > 0 ) {
-		NSLog( @"Entries to delete: %d", [keysToDelete count]);
+		NSLog( @"Entries to delete: %lu", (unsigned long)[keysToDelete count]);
 		[_fileNamesCache removeObjectsForKeys:keysToDelete];
 	}
 }
@@ -233,7 +233,7 @@ static ContentManager* sharedInstance = nil;
 - (void)repaintAllWindowsIfNeeded
 {
 	if (!_hasChangedContent) {
-		NSLog(@"%@ Repaint scheduled but not needed", NSStringFromSelector(_cmd));
+		//NSLog(@"%@ Repaint scheduled but not needed", NSStringFromSelector(_cmd));
 		return;
 	}
 

shell_integration/MacOSX/OwnCloudFinder/FinishedIconCache.h

@@ -0,0 +1,24 @@
+//
+//  FinishedIconCache.h
+//  OwnCloudFinder
+//
+//  Created by Markus Goetz on 01/10/14.
+//
+//
+
+#import <Foundation/Foundation.h>
+#import <Cocoa/Cocoa.h>
+
+@interface FinishedIconCache : NSObject {
+	NSCache *_cache;
+	long long _hits;
+	long long _misses;
+}
+
++ (FinishedIconCache*)sharedInstance;
+
+- (NSImage*)getIcon:(NSString*)fileName overlayIconIndex:(int)idx width:(float)w height:(float)h;
+- (void)registerIcon:(NSImage*)icon withFileName:(NSString*)fileName overlayIconIndex:(int)idx width:(float)w height:(float)h;
+
+
+@end

shell_integration/MacOSX/OwnCloudFinder/FinishedIconCache.m

@@ -0,0 +1,91 @@
+//
+//  FinishedIconCache.m
+//  OwnCloudFinder
+//
+//  Created by Markus Goetz on 01/10/14.
+//
+//
+
+#import "FinishedIconCache.h"
+
+
+@interface FinishedIconCacheItem : NSObject
+@property (nonatomic, strong) NSImage *icon;
+@property (nonatomic) NSTimeInterval maxAge;
+@end
+
+@implementation FinishedIconCacheItem
+@synthesize icon;
+@synthesize maxAge;
+- (void)dealloc {
+	//NSLog(@"RELEASE %@ %@", self, self.icon);
+	if (self.icon) {
+		[self->icon release];
+	}
+	[super dealloc];
+}
+@end
+
+@implementation FinishedIconCache
+
+static FinishedIconCache* sharedInstance = nil;
+
+- init
+{
+	self = [super init];
+	if (self)
+	{
+		_cache = [[NSCache alloc] init];
+		_cache.totalCostLimit = (2880 * 1800); // mbp15 screen size
+		_hits = 0;
+		_misses = 0;
+	}
+	return self;
+}
+
+- (void)dealloc
+{
+	[_cache dealloc];
+	[super dealloc];
+}
+
++ (FinishedIconCache*)sharedInstance
+{
+	@synchronized(self)
+	{
+		if (sharedInstance == nil)
+		{
+			sharedInstance = [[self alloc] init];
+		}
+	}
+	return sharedInstance;
+}
+
+
+- (NSImage*)getIcon:(NSString*)fileName overlayIconIndex:(int)idx width:(float)w height:(float)h
+{
+	NSString *cacheKey = [NSString stringWithFormat:@"%@--%d--%f%f", fileName, idx, w,h];
+	FinishedIconCacheItem *item = [_cache objectForKey:cacheKey];
+	if (item) {
+		if (item.maxAge > [[NSDate date] timeIntervalSinceReferenceDate]) {
+			_hits++;
+			return item.icon;
+		}
+	}
+	_misses++;
+	return NULL;
+}
+
+- (void)registerIcon:(NSImage*)icon withFileName:(NSString*)fileName overlayIconIndex:(int)idx width:(float)w height:(float)h
+{
+	NSString *cacheKey = [NSString stringWithFormat:@"%@--%d--%f%f", fileName, idx, w, h];
+	FinishedIconCacheItem *item = [[FinishedIconCacheItem alloc] init];
+	item.icon = icon;
+	// max age between 1 sec and 5 sec
+	item.maxAge = [[NSDate date] timeIntervalSinceReferenceDate] + 1.0 + 4.0*((double)arc4random() / 0x100000000);
+	[_cache setObject:item forKey:cacheKey cost:w*h];
+	[item release];
+	//NSLog(@"CACHE hit/miss ratio: %f", (float)_hits/(float)_misses);
+}
+
+@end

shell_integration/MacOSX/OwnCloudFinder/IconOverlayHandlers.m

@@ -15,6 +15,7 @@
 #import <objc/runtime.h>
 #import "ContentManager.h"
 #import "IconCache.h"
+#import "FinishedIconCache.h"
 #import "IconOverlayHandlers.h"
 #import "Finder/Finder.h"
 
@@ -69,7 +70,6 @@
 
 	NSURL* url = [node previewItemURL];
 
-	NSError *error;
 	BOOL isDir;
 	if ([[NSFileManager defaultManager] fileExistsAtPath:[url path] isDirectory:&isDir] == NO) {
 		NSLog(@"ERROR: Could not determine file type of %@", [url path]);
@@ -83,15 +83,26 @@
 	{
 		NSImage* icon = [arg1 _nsImage];
 
-		[icon lockFocus];
+		// Use the short term icon cache that possibly has the finished icon
-
+		FinishedIconCache *finishedIconCache = [FinishedIconCache sharedInstance];
-		CGContextRef myContext = [[NSGraphicsContext currentContext] graphicsPort];
+		NSImage *finishedImage = [finishedIconCache getIcon:[url path] overlayIconIndex:imageIndex width:[icon size].width height:[icon size].height];
+		if (finishedImage) {
+			//NSLog(@"X Got finished image from cache %@ %@", finishedImage, [url path]);
+			return [[[IKImageWrapper alloc] initWithNSImage:finishedImage] autorelease];;
+		} else {
+			//NSLog(@"X Need to redraw %@", [url path]);
+		}
 
 		NSImage* iconimage = [[IconCache sharedInstance] getIcon:[NSNumber numberWithInt:[imageIndex intValue]]];
 
 		if (iconimage != nil)
 		{
+			[icon lockFocus];
+
+			CGContextRef myContext = [[NSGraphicsContext currentContext] graphicsPort];
+
 			CGRect destRect = CGRectMake(0, 0, [icon size].width, [icon size].height);
+
 			CGImageRef cgImage = [iconimage CGImageForProposedRect:&destRect
 														   context:[NSGraphicsContext currentContext]
 															 hints:nil];
@@ -103,9 +114,11 @@
 				NSLog(@"No image given!!!!!11 %@", [url path]);
 			}
 
+			[icon unlockFocus];
 		}
 
-		[icon unlockFocus];
+		// Insert into cache
+		[finishedIconCache registerIcon:icon withFileName:[url path] overlayIconIndex:imageIndex width:[icon size].width height:[icon size].height];
 
 		return [[[IKImageWrapper alloc] initWithNSImage:icon] autorelease];
 	}

shell_integration/MacOSX/OwnCloudFinder/JSONKit.h

@@ -1,251 +0,0 @@
-//
-//  JSONKit.h
-//  http://github.com/johnezang/JSONKit
-//  Dual licensed under either the terms of the BSD License, or alternatively
-//  under the terms of the Apache License, Version 2.0, as specified below.
-//
-
-/*
- Copyright (c) 2011, John Engelhart
- 
- All rights reserved.
- 
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are met:
- 
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- 
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
- 
- * Neither the name of the Zang Industries nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
- 
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-/*
- Copyright 2011 John Engelhart
- 
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
- 
- http://www.apache.org/licenses/LICENSE-2.0
- 
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-*/
-
-#include <stddef.h>
-#include <stdint.h>
-#include <limits.h>
-#include <TargetConditionals.h>
-#include <AvailabilityMacros.h>
-
-#ifdef    __OBJC__
-#import <Foundation/NSArray.h>
-#import <Foundation/NSData.h>
-#import <Foundation/NSDictionary.h>
-#import <Foundation/NSError.h>
-#import <Foundation/NSObjCRuntime.h>
-#import <Foundation/NSString.h>
-#endif // __OBJC__
- 
-#ifdef __cplusplus
-extern "C" {
-#endif
-  
-
-// For Mac OS X < 10.5.
-#ifndef   NSINTEGER_DEFINED
-#define   NSINTEGER_DEFINED
-#if       defined(__LP64__) || defined(NS_BUILD_32_LIKE_64)
-typedef long           NSInteger;
-typedef unsigned long  NSUInteger;
-#define NSIntegerMin   LONG_MIN
-#define NSIntegerMax   LONG_MAX
-#define NSUIntegerMax  ULONG_MAX
-#else  // defined(__LP64__) || defined(NS_BUILD_32_LIKE_64)
-typedef int            NSInteger;
-typedef unsigned int   NSUInteger;
-#define NSIntegerMin   INT_MIN
-#define NSIntegerMax   INT_MAX
-#define NSUIntegerMax  UINT_MAX
-#endif // defined(__LP64__) || defined(NS_BUILD_32_LIKE_64)
-#endif // NSINTEGER_DEFINED
-
-
-#ifndef _JSONKIT_H_
-#define _JSONKIT_H_
-
-#if defined(__GNUC__) && (__GNUC__ >= 4) && defined(__APPLE_CC__) && (__APPLE_CC__ >= 5465)
-#define JK_DEPRECATED_ATTRIBUTE __attribute__((deprecated))
-#else
-#define JK_DEPRECATED_ATTRIBUTE
-#endif
-  
-#define JSONKIT_VERSION_MAJOR 1
-#define JSONKIT_VERSION_MINOR 4
-
-typedef NSUInteger JKFlags;
-
-/*
-  JKParseOptionComments        : Allow C style // and /_* ... *_/ (without a _, obviously) comments in JSON.
-  JKParseOptionUnicodeNewlines : Allow Unicode recommended (?:\r\n|[\n\v\f\r\x85\p{Zl}\p{Zp}]) newlines.
-  JKParseOptionLooseUnicode    : Normally the decoder will stop with an error at any malformed Unicode.
-                                 This option allows JSON with malformed Unicode to be parsed without reporting an error.
-                                 Any malformed Unicode is replaced with \uFFFD, or "REPLACEMENT CHARACTER".
- */
-
-enum {
-  JKParseOptionNone                     = 0,
-  JKParseOptionStrict                   = 0,
-  JKParseOptionComments                 = (1 << 0),
-  JKParseOptionUnicodeNewlines          = (1 << 1),
-  JKParseOptionLooseUnicode             = (1 << 2),
-  JKParseOptionPermitTextAfterValidJSON = (1 << 3),
-  JKParseOptionValidFlags               = (JKParseOptionComments | JKParseOptionUnicodeNewlines | JKParseOptionLooseUnicode | JKParseOptionPermitTextAfterValidJSON),
-};
-typedef JKFlags JKParseOptionFlags;
-
-enum {
-  JKSerializeOptionNone                 = 0,
-  JKSerializeOptionPretty               = (1 << 0),
-  JKSerializeOptionEscapeUnicode        = (1 << 1),
-  JKSerializeOptionEscapeForwardSlashes = (1 << 4),
-  JKSerializeOptionValidFlags           = (JKSerializeOptionPretty | JKSerializeOptionEscapeUnicode | JKSerializeOptionEscapeForwardSlashes),
-};
-typedef JKFlags JKSerializeOptionFlags;
-
-#ifdef    __OBJC__
-
-typedef struct JKParseState JKParseState; // Opaque internal, private type.
-
-// As a general rule of thumb, if you use a method that doesn't accept a JKParseOptionFlags argument, it defaults to JKParseOptionStrict
-
-@interface JSONDecoder : NSObject {
-  JKParseState *parseState;
-}
-+ (id)decoder;
-+ (id)decoderWithParseOptions:(JKParseOptionFlags)parseOptionFlags;
-- (id)initWithParseOptions:(JKParseOptionFlags)parseOptionFlags;
-- (void)clearCache;
-
-// The parse... methods were deprecated in v1.4 in favor of the v1.4 objectWith... methods.
-- (id)parseUTF8String:(const unsigned char *)string length:(size_t)length                         JK_DEPRECATED_ATTRIBUTE; // Deprecated in JSONKit v1.4.  Use objectWithUTF8String:length:        instead.
-- (id)parseUTF8String:(const unsigned char *)string length:(size_t)length error:(NSError **)error JK_DEPRECATED_ATTRIBUTE; // Deprecated in JSONKit v1.4.  Use objectWithUTF8String:length:error:  instead.
-// The NSData MUST be UTF8 encoded JSON.
-- (id)parseJSONData:(NSData *)jsonData                                                            JK_DEPRECATED_ATTRIBUTE; // Deprecated in JSONKit v1.4.  Use objectWithData:                     instead.
-- (id)parseJSONData:(NSData *)jsonData error:(NSError **)error                                    JK_DEPRECATED_ATTRIBUTE; // Deprecated in JSONKit v1.4.  Use objectWithData:error:               instead.
-
-// Methods that return immutable collection objects.
-- (id)objectWithUTF8String:(const unsigned char *)string length:(NSUInteger)length;
-- (id)objectWithUTF8String:(const unsigned char *)string length:(NSUInteger)length error:(NSError **)error;
-// The NSData MUST be UTF8 encoded JSON.
-- (id)objectWithData:(NSData *)jsonData;
-- (id)objectWithData:(NSData *)jsonData error:(NSError **)error;
-
-// Methods that return mutable collection objects.
-- (id)mutableObjectWithUTF8String:(const unsigned char *)string length:(NSUInteger)length;
-- (id)mutableObjectWithUTF8String:(const unsigned char *)string length:(NSUInteger)length error:(NSError **)error;
-// The NSData MUST be UTF8 encoded JSON.
-- (id)mutableObjectWithData:(NSData *)jsonData;
-- (id)mutableObjectWithData:(NSData *)jsonData error:(NSError **)error;
-
-@end
-
-////////////
-#pragma mark Deserializing methods
-////////////
-
-@interface NSString (JSONKitDeserializing)
-- (id)objectFromJSONString;
-- (id)objectFromJSONStringWithParseOptions:(JKParseOptionFlags)parseOptionFlags;
-- (id)objectFromJSONStringWithParseOptions:(JKParseOptionFlags)parseOptionFlags error:(NSError **)error;
-- (id)mutableObjectFromJSONString;
-- (id)mutableObjectFromJSONStringWithParseOptions:(JKParseOptionFlags)parseOptionFlags;
-- (id)mutableObjectFromJSONStringWithParseOptions:(JKParseOptionFlags)parseOptionFlags error:(NSError **)error;
-@end
-
-@interface NSData (JSONKitDeserializing)
-// The NSData MUST be UTF8 encoded JSON.
-- (id)objectFromJSONData;
-- (id)objectFromJSONDataWithParseOptions:(JKParseOptionFlags)parseOptionFlags;
-- (id)objectFromJSONDataWithParseOptions:(JKParseOptionFlags)parseOptionFlags error:(NSError **)error;
-- (id)mutableObjectFromJSONData;
-- (id)mutableObjectFromJSONDataWithParseOptions:(JKParseOptionFlags)parseOptionFlags;
-- (id)mutableObjectFromJSONDataWithParseOptions:(JKParseOptionFlags)parseOptionFlags error:(NSError **)error;
-@end
-
-////////////
-#pragma mark Serializing methods
-////////////
-  
-@interface NSString (JSONKitSerializing)
-// Convenience methods for those that need to serialize the receiving NSString (i.e., instead of having to serialize a NSArray with a single NSString, you can "serialize to JSON" just the NSString).
-// Normally, a string that is serialized to JSON has quotation marks surrounding it, which you may or may not want when serializing a single string, and can be controlled with includeQuotes:
-// includeQuotes:YES `a "test"...` -> `"a \"test\"..."`
-// includeQuotes:NO  `a "test"...` -> `a \"test\"...`
-- (NSData *)JSONData;     // Invokes JSONDataWithOptions:JKSerializeOptionNone   includeQuotes:YES
-- (NSData *)JSONDataWithOptions:(JKSerializeOptionFlags)serializeOptions includeQuotes:(BOOL)includeQuotes error:(NSError **)error;
-- (NSString *)JSONString; // Invokes JSONStringWithOptions:JKSerializeOptionNone includeQuotes:YES
-- (NSString *)JSONStringWithOptions:(JKSerializeOptionFlags)serializeOptions includeQuotes:(BOOL)includeQuotes error:(NSError **)error;
-@end
-
-@interface NSArray (JSONKitSerializing)
-- (NSData *)JSONData;
-- (NSData *)JSONDataWithOptions:(JKSerializeOptionFlags)serializeOptions error:(NSError **)error;
-- (NSData *)JSONDataWithOptions:(JKSerializeOptionFlags)serializeOptions serializeUnsupportedClassesUsingDelegate:(id)delegate selector:(SEL)selector error:(NSError **)error;
-- (NSString *)JSONString;
-- (NSString *)JSONStringWithOptions:(JKSerializeOptionFlags)serializeOptions error:(NSError **)error;
-- (NSString *)JSONStringWithOptions:(JKSerializeOptionFlags)serializeOptions serializeUnsupportedClassesUsingDelegate:(id)delegate selector:(SEL)selector error:(NSError **)error;
-@end
-
-@interface NSDictionary (JSONKitSerializing)
-- (NSData *)JSONData;
-- (NSData *)JSONDataWithOptions:(JKSerializeOptionFlags)serializeOptions error:(NSError **)error;
-- (NSData *)JSONDataWithOptions:(JKSerializeOptionFlags)serializeOptions serializeUnsupportedClassesUsingDelegate:(id)delegate selector:(SEL)selector error:(NSError **)error;
-- (NSString *)JSONString;
-- (NSString *)JSONStringWithOptions:(JKSerializeOptionFlags)serializeOptions error:(NSError **)error;
-- (NSString *)JSONStringWithOptions:(JKSerializeOptionFlags)serializeOptions serializeUnsupportedClassesUsingDelegate:(id)delegate selector:(SEL)selector error:(NSError **)error;
-@end
-
-#ifdef __BLOCKS__
-
-@interface NSArray (JSONKitSerializingBlockAdditions)
-- (NSData *)JSONDataWithOptions:(JKSerializeOptionFlags)serializeOptions serializeUnsupportedClassesUsingBlock:(id(^)(id object))block error:(NSError **)error;
-- (NSString *)JSONStringWithOptions:(JKSerializeOptionFlags)serializeOptions serializeUnsupportedClassesUsingBlock:(id(^)(id object))block error:(NSError **)error;
-@end
-
-@interface NSDictionary (JSONKitSerializingBlockAdditions)
-- (NSData *)JSONDataWithOptions:(JKSerializeOptionFlags)serializeOptions serializeUnsupportedClassesUsingBlock:(id(^)(id object))block error:(NSError **)error;
-- (NSString *)JSONStringWithOptions:(JKSerializeOptionFlags)serializeOptions serializeUnsupportedClassesUsingBlock:(id(^)(id object))block error:(NSError **)error;
-@end
-  
-#endif
-
-
-#endif // __OBJC__
-
-#endif // _JSONKIT_H_
-
-#ifdef __cplusplus
-}  // extern "C"
-#endif

shell_integration/MacOSX/OwnCloudFinder/OwnCloudFinder.xcodeproj/project.pbxproj

@@ -7,9 +7,9 @@
 	objects = {
 
 /* Begin PBXBuildFile section */
-		0B13ECAE173C687400548DA1 /* GCDAsyncSocket.m in Sources */ = {isa = PBXBuildFile; fileRef = 0B13ECAD173C686A00548DA1 /* GCDAsyncSocket.m */; };
+		0B13ECAE173C687400548DA1 /* GCDAsyncSocket.m in Sources */ = {isa = PBXBuildFile; fileRef = 0B13ECAD173C686A00548DA1 /* GCDAsyncSocket.m */; settings = {COMPILER_FLAGS = "-fobjc-arc"; }; };
 		0B13ECAF173C687900548DA1 /* Security.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 0BFC9ACB173C57E400CDD329 /* Security.framework */; };
-		0BFAF21C16F8E6C10017EA7E /* JSONKit.m in Sources */ = {isa = PBXBuildFile; fileRef = 0BFAF21B16F8E6C10017EA7E /* JSONKit.m */; };
+		5BB74A8719DBF9BB001BAAAC /* FinishedIconCache.m in Sources */ = {isa = PBXBuildFile; fileRef = 5BB74A8619DBF9BB001BAAAC /* FinishedIconCache.m */; };
 		692C18A516660C4700BF6A53 /* ContextMenuHandlers.m in Sources */ = {isa = PBXBuildFile; fileRef = 692C18A416660C4600BF6A53 /* ContextMenuHandlers.m */; };
 		692C18A9166617F500BF6A53 /* IconOverlayHandlers.m in Sources */ = {isa = PBXBuildFile; fileRef = 692C18A8166617F500BF6A53 /* IconOverlayHandlers.m */; };
 		692C18AC1666392700BF6A53 /* MenuManager.m in Sources */ = {isa = PBXBuildFile; fileRef = 692C18AB1666392700BF6A53 /* MenuManager.m */; };
@@ -29,9 +29,9 @@
 		0B13ECAC173C686900548DA1 /* GCDAsyncSocket.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = GCDAsyncSocket.h; sourceTree = "<group>"; };
 		0B13ECAD173C686A00548DA1 /* GCDAsyncSocket.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = GCDAsyncSocket.m; sourceTree = "<group>"; };
 		0B2BF60B176A43DB001246CD /* Finder.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Finder.h; sourceTree = "<group>"; };
-		0BFAF21A16F8E6C10017EA7E /* JSONKit.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = JSONKit.h; sourceTree = "<group>"; };
-		0BFAF21B16F8E6C10017EA7E /* JSONKit.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = JSONKit.m; sourceTree = "<group>"; };
 		0BFC9ACB173C57E400CDD329 /* Security.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Security.framework; path = System/Library/Frameworks/Security.framework; sourceTree = SDKROOT; };
+		5BB74A8519DBF9BB001BAAAC /* FinishedIconCache.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = FinishedIconCache.h; sourceTree = "<group>"; };
+		5BB74A8619DBF9BB001BAAAC /* FinishedIconCache.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FinishedIconCache.m; sourceTree = "<group>"; };
 		692C18A316660C4600BF6A53 /* ContextMenuHandlers.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ContextMenuHandlers.h; sourceTree = "<group>"; };
 		692C18A416660C4600BF6A53 /* ContextMenuHandlers.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ContextMenuHandlers.m; sourceTree = "<group>"; };
 		692C18A7166617F500BF6A53 /* IconOverlayHandlers.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = IconOverlayHandlers.h; sourceTree = "<group>"; };
@@ -104,7 +104,6 @@
 			children = (
 				0B2BF60A176A43DB001246CD /* Finder */,
 				0B08BAC21759627700C8351E /* GCDAsyncSocket */,
-				0BFAF21916F8E6910017EA7E /* JSONKit */,
 				8C37DD99161593BD00016A95 /* FinderHook.h */,
 				8C37DD9A161593BD00016A95 /* FinderHook.m */,
 				692C18A316660C4600BF6A53 /* ContextMenuHandlers.h */,
@@ -117,6 +116,8 @@
 				69948B351636D50E0093B6CE /* ContentManager.m */,
 				8C99F6921622D145002D2135 /* IconCache.h */,
 				8C99F6931622D145002D2135 /* IconCache.m */,
+				5BB74A8519DBF9BB001BAAAC /* FinishedIconCache.h */,
+				5BB74A8619DBF9BB001BAAAC /* FinishedIconCache.m */,
 				692C18AA1666392700BF6A53 /* MenuManager.h */,
 				692C18AB1666392700BF6A53 /* MenuManager.m */,
 			);
@@ -140,15 +141,6 @@
 			path = Finder;
 			sourceTree = "<group>";
 		};
-		0BFAF21916F8E6910017EA7E /* JSONKit */ = {
-			isa = PBXGroup;
-			children = (
-				0BFAF21A16F8E6C10017EA7E /* JSONKit.h */,
-				0BFAF21B16F8E6C10017EA7E /* JSONKit.m */,
-			);
-			name = JSONKit;
-			sourceTree = "<group>";
-		};
 		19C28FB6FE9D52B211CA2CBB /* Products */ = {
 			isa = PBXGroup;
 			children = (
@@ -226,10 +218,10 @@
 				8C99F6941622D145002D2135 /* IconCache.m in Sources */,
 				69948B361636D50E0093B6CE /* ContentManager.m in Sources */,
 				6993878616494C000044E4DF /* RequestManager.m in Sources */,
+				5BB74A8719DBF9BB001BAAAC /* FinishedIconCache.m in Sources */,
 				692C18A516660C4700BF6A53 /* ContextMenuHandlers.m in Sources */,
 				692C18A9166617F500BF6A53 /* IconOverlayHandlers.m in Sources */,
 				692C18AC1666392700BF6A53 /* MenuManager.m in Sources */,
-				0BFAF21C16F8E6C10017EA7E /* JSONKit.m in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -302,8 +294,9 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				ARCHS = "$(ARCHS_STANDARD_32_64_BIT)";
-				COPY_PHASE_STRIP = YES;
+				COPY_PHASE_STRIP = NO;
 				GCC_C_LANGUAGE_STANDARD = gnu99;
+				GCC_OPTIMIZATION_LEVEL = 0;
 				GCC_WARN_ABOUT_RETURN_TYPE = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
 				MACOSX_DEPLOYMENT_TARGET = 10.7;

shell_integration/MacOSX/OwnCloudFinder/RequestManager.m

@@ -83,7 +83,7 @@ static RequestManager* sharedInstance = nil;
 	NSArray *regPathes = [_registeredPathes allKeys];
 	BOOL registered = NO;
 
-	NSString* checkPath = [[NSString alloc] initWithString:path];
+	NSString* checkPath = [NSString stringWithString:path];
 	if (isDir) {
 		// append a slash
 		checkPath = [path stringByAppendingString:@"/"];
@@ -127,17 +127,17 @@ static RequestManager* sharedInstance = nil;
 
 - (void)socket:(GCDAsyncSocket*)socket didReadData:(NSData*)data withTag:(long)tag
 {
-	NSArray *chunks;
 	NSString *answer = [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding];
+	NSArray *chunks = nil;
 	if (answer != nil && [answer length] > 0) {
 		// cut a trailing newline
 		answer = [answer substringToIndex:[answer length] - 1];
 		chunks = [answer componentsSeparatedByString: @":"];
 	}
-	NSLog(@"READ from socket (%ld): <%@>", tag, answer);
 	ContentManager *contentman = [ContentManager sharedInstance];
 
-	if( [chunks count] > 0 && tag == READ_TAG ) {
+	if( chunks && [chunks count] > 0 && tag == READ_TAG ) {
+		NSLog(@"READ from socket (%ld): <%@>", tag, answer);
 		if( [[chunks objectAtIndex:0] isEqualToString:@"STATUS"] ) {
 			NSString *path = [chunks objectAtIndex:2];
 			if( [chunks count] > 3 ) {
@@ -168,8 +168,8 @@ static RequestManager* sharedInstance = nil;
 		} else {
 			NSLog(@"Unknown command %@", [chunks objectAtIndex:0]);
 		}
-	} else {
+	} else if (tag != READ_TAG) {
-		NSLog(@"Received unknown tag %ld", tag);
+		NSLog(@"Received unknown tag %ld <%@>", tag, answer);
 	}
 	// Read on and on
 	NSData* stop = [@"\n" dataUsingEncoding:NSUTF8StringEncoding];
@@ -202,7 +202,7 @@ static RequestManager* sharedInstance = nil;
 	if( [_requestQueue count] > 0 ) {
 		NSLog( @"We have to empty the queue");
 		for( NSString *path in _requestQueue ) {
-			[self askOnSocket:path];
+			[self askOnSocket:path query:@"RETRIEVE_FILE_STATUS"];
 		}
 	}
 
@@ -253,7 +253,7 @@ static RequestManager* sharedInstance = nil;
 				NSLog(@"I goofed: %@", err);
 			}
 		} else if (!useTcp) {
-			NSURL *url;
+			NSURL *url = nil;
 			NSArray *paths = NSSearchPathForDirectoriesInDomains(NSCachesDirectory, NSUserDomainMask, YES);
 			if ([paths count])
 			{

shell_integration/nautilus/syncstate.py

@@ -34,6 +34,7 @@ class syncStateExtension(GObject.GObject, Nautilus.ColumnProvider, Nautilus.Info
             GObject.timeout_add(5000, self.connectToSocketServer)
 
     def connectToSocketServer(self):
+        do_reconnect = True
         try:
             self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
             postfix = "/"+self.appname+"/socket"
@@ -44,15 +45,18 @@ class syncStateExtension(GObject.GObject, Nautilus.ColumnProvider, Nautilus.Info
 		    print("Socket File: "+sock_file)
 		    self.sock.connect(sock_file)
 		    self.connected = True
+		    print("Setting connected to %r" % self.connected )
 		    self.watch_id = GObject.io_add_watch(self.sock, GObject.IO_IN, self.handle_notify)
-		except:
+		    do_reconnect = False
-		    print("Could not connect to unix socket.")
+		except Exception, e:
+		    print("Could not connect to unix socket." + str(e))
 	    else:
 		 print("Sock-File not valid: "+sock_file)
-        except:
+        except Exception, e:
-            print("Connect could not be established, try again later!")
+            print("Connect could not be established, try again later " + str(e))
             self.sock.close()
-        return not self.connected
+        # print("Returning %r" % do_reconnect)
+        return do_reconnect
 
     def sendCommand(self, cmd):
         if self.connected:
@@ -122,7 +126,7 @@ class syncStateExtension(GObject.GObject, Nautilus.ColumnProvider, Nautilus.Info
 			if( not itemStore['state'] or newState != itemStore['state'] ):
 			    item = itemStore['item']
 			    item.add_emblem(emblem)
-			    # print "Setting emblem on " + parts[2]
+			    # print "Setting emblem on " + parts[2]+ "<>"+emblem+"<>"
 			    self.nautilusVFSFile_table[parts[2]] = {'item': item, 'state':newState}
 
             elif action == 'UPDATE_VIEW':

shell_integration/windows/OCShellExtensions/OCOverlays/OCOverlay.cpp

@@ -36,17 +36,13 @@ extern HINSTANCE instanceHandle;
 #define IDM_DISPLAY 0  
 #define IDB_OK 101
 
-namespace {
-	static std::vector<std::wstring> s_watchedDirectories;
-}
-
 OCOverlay::OCOverlay(int state) 
-	: _communicationSocket(0)
+	: _referenceCount(1)
-	, _referenceCount(1)
-	, _checker(new RemotePathChecker(PORT))
 	, _state(state)
 
 {
+    static RemotePathChecker s_remotePathChecker;
+    _checker = &s_remotePathChecker;
 }
 
 OCOverlay::~OCOverlay(void)
@@ -121,23 +117,13 @@ IFACEMETHODIMP OCOverlay::GetPriority(int *pPriority)
 
  IFACEMETHODIMP OCOverlay::IsMemberOf(PCWSTR pwszPath, DWORD dwAttrib)
 {
-	
+    auto watchedDirectories = _checker->WatchedDirectories();
-	//if(!_IsOverlaysEnabled())
-	//{
-	//	return MAKE_HRESULT(S_FALSE, 0, 0);
-	//}
-
-	// FIXME: Use Registry instead, this will only trigger once
-	// and now follow any user changes in the client
-	if (s_watchedDirectories.empty()) {
-		s_watchedDirectories = _checker->WatchedDirectories();
-	}
 
 	wstring wpath(pwszPath);
-	wpath.append(L"\\");
+	//wpath.append(L"\\");
 	vector<wstring>::iterator it;
 	bool watched = false;
-	for (it = s_watchedDirectories.begin(); it != s_watchedDirectories.end(); ++it) {
+	for (it = watchedDirectories.begin(); it != watchedDirectories.end(); ++it) {
 		if (StringUtil::begins_with(wpath, *it)) {
 			watched = true;
 		}

shell_integration/windows/OCShellExtensions/OCOverlays/OCOverlay.h

@@ -35,14 +35,13 @@ public:
     IFACEMETHODIMP_(ULONG) Release();
 
 protected:
-    ~OCOverlay(void);
+    ~OCOverlay();
 
 private:
 	//bool _GenerateMessage(const wchar_t*, std::wstring*);
 
 	bool _IsOverlaysEnabled();
     long _referenceCount;
-	CommunicationSocket* _communicationSocket;
 	RemotePathChecker* _checker;
 	int _state;
 };

shell_integration/windows/OCShellExtensions/OCUtil/CommunicationSocket.cpp

@@ -21,6 +21,7 @@
 #include <windows.h>
 #include <iostream>
 #include <vector>
+#include <array>
 
 #include <fstream> 
 
@@ -30,8 +31,8 @@ using namespace std;
 
 #define DEFAULT_BUFLEN 4096
 
-CommunicationSocket::CommunicationSocket(int port)
+CommunicationSocket::CommunicationSocket()
-	: _port(port), _clientSocket(INVALID_SOCKET)
+    : _pipe(INVALID_HANDLE_VALUE)
 {
 }
 
@@ -43,64 +44,39 @@ CommunicationSocket::~CommunicationSocket()
 bool CommunicationSocket::Close()
 {
 	WSACleanup();
-	bool closed = (closesocket(_clientSocket) == 0);
+	if (_pipe == INVALID_HANDLE_VALUE) {
-	shutdown(_clientSocket, SD_BOTH);
+		return false;
-	_clientSocket = INVALID_SOCKET;
+	}
-	return closed;
+	CloseHandle(_pipe);
+	_pipe = INVALID_HANDLE_VALUE;
+	return true;
 }
 
 
-bool CommunicationSocket::Connect()
+bool CommunicationSocket::Connect(const std::wstring &pipename)
 {
-	WSADATA wsaData;
+	_pipe = CreateFile(pipename.data(), GENERIC_READ | GENERIC_WRITE, 0, NULL, OPEN_EXISTING, 0, NULL);
 
-	HRESULT iResult = WSAStartup(MAKEWORD(2, 2), &wsaData);
+    if (_pipe == INVALID_HANDLE_VALUE) {
-
+        return false;
-	if (iResult != NO_ERROR) {
+    }
-		int error = WSAGetLastError();
+    return true;
-	}
-
-
-	_clientSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
-
-	if (_clientSocket == INVALID_SOCKET) {
-		//int error = WSAGetLastError();
-		Close();
-		return false;
-	}
-
-	struct sockaddr_in clientService;
-
-	clientService.sin_family = AF_INET;
-	clientService.sin_addr.s_addr = inet_addr(PLUG_IN_SOCKET_ADDRESS);
-	clientService.sin_port = htons(_port);
-
-	iResult = connect(_clientSocket, (SOCKADDR*)&clientService, sizeof(clientService));
-	DWORD timeout = 500; // ms
-	setsockopt(_clientSocket, SOL_SOCKET, SO_RCVTIMEO, (const char*) &timeout, sizeof(DWORD));
-
-	if (iResult == SOCKET_ERROR) {
-		//int error = WSAGetLastError();
-		Close();
-		return false;
-	}
-	return true;
 }
 
 bool CommunicationSocket::SendMsg(const wchar_t* message)
 {
-	const char* utf8_msg = StringUtil::toUtf8(message);
+	auto utf8_msg = StringUtil::toUtf8(message);
-	size_t result = send(_clientSocket, utf8_msg, (int)strlen(utf8_msg), 0);
-	delete[] utf8_msg;
 
-	if (result == SOCKET_ERROR) {
+    DWORD numBytesWritten = 0;
-		//int error = WSAGetLastError();
+    auto result = WriteFile( _pipe, utf8_msg.c_str(), DWORD(utf8_msg.size()), &numBytesWritten, NULL);
-		closesocket(_clientSocket);
-		return false;
-	}
 
-	return true;
+    if (result) {
+        return true;
+    } else {
+        Close();
 
+        return false;
+    }
 }
 
 bool CommunicationSocket::ReadLine(wstring* response)
@@ -109,21 +85,43 @@ bool CommunicationSocket::ReadLine(wstring* response)
 		return false;
 	}
 
-	vector<char> resp_utf8;
+    response->clear();
-	char buffer;
+
+    if (_pipe == INVALID_HANDLE_VALUE) {
+        return false;
+    }
+
+
 	while (true) {
-		int bytesRead = recv(_clientSocket, &buffer, 1, 0);
+        int lbPos = 0;
-		if (bytesRead <= 0) {
+        auto it = std::find(_buffer.begin() + lbPos, _buffer.end(), '\n');
-			response = 0;
+        if (it != _buffer.end()) {
+            *response = StringUtil::toUtf16(_buffer.data(), DWORD(it - _buffer.begin()));
+            _buffer.erase(_buffer.begin(), it + 1);
+            return true;
+        }
+
+        std::array<char, 128> resp_utf8;
+        DWORD numBytesRead = 0;
+		DWORD totalBytesAvailable = 0;
+		auto result = PeekNamedPipe(_pipe, NULL, 0, 0, &totalBytesAvailable, 0);
+		if (!result) {
+			Close();
+			return false;
+		}
+		if (totalBytesAvailable == 0) {
 			return false;
 		}
 
-		if (buffer == '\n')	{
+        result = ReadFile(_pipe, resp_utf8.data(), DWORD(resp_utf8.size()), &numBytesRead, NULL);
-			resp_utf8.push_back(0);
+        if (!result) {
-			*response = StringUtil::toUtf16(&resp_utf8[0], resp_utf8.size());
+            Close();
-			return true;
+            return false;
-		} else {
+        }
-			resp_utf8.push_back(buffer);
+        if (numBytesRead <= 0) {
-		}
+            return false;
+        }
+		_buffer.insert(_buffer.end(), resp_utf8.begin(), resp_utf8.begin()+numBytesRead);
+        continue;
 	}
 }

shell_integration/windows/OCShellExtensions/OCUtil/CommunicationSocket.h

@@ -20,23 +20,27 @@
 #pragma warning (disable : 4251)
 
 #include <string>
+#include <vector>
 #include <WinSock2.h>
 
 class __declspec(dllexport) CommunicationSocket
 {
 public:
-	CommunicationSocket(int port);
+	CommunicationSocket();
 	~CommunicationSocket();
 
-	bool Connect();
+	bool Connect(const std::wstring& pipename);
 	bool Close();
 
 	bool SendMsg(const wchar_t*);
 	bool ReadLine(std::wstring*);
 
+    HANDLE Event() { return _pipe; }
+
 private:	
-	int _port;
+	HANDLE _pipe;
-	SOCKET _clientSocket;
+	std::vector<char> _buffer;
+    bool _connected;
 };
 
 #endif

shell_integration/windows/OCShellExtensions/OCUtil/RemotePathChecker.cpp

@@ -20,88 +20,168 @@
 #include <iostream>
 #include <sstream>
 #include <iterator>
+#include <unordered_set>
+#include <cassert>
+
+#include <shlobj.h>
 
 using namespace std;
 
-RemotePathChecker::RemotePathChecker(int port)
+
-	: _port(port)
+// This code is run in a thread
+void RemotePathChecker::workerThreadLoop()
 {
+    auto pipename = std::wstring(L"\\\\.\\pipe\\");
+    pipename += L"ownCloud";
+
+    bool connected = false;
+    CommunicationSocket socket;
+    std::unordered_set<std::wstring> asked;
+
+    while(!_stop) {
+		Sleep(50);
+
+        if (!connected) {
+            asked.clear();
+            if (!WaitNamedPipe(pipename.data(), 5 * 1000)) {
+                continue;
+            }
+            if (!socket.Connect(pipename)) {
+                continue;
+            }
+            connected = true;
+            std::unique_lock<std::mutex> lock(_mutex);
+            _connected = true;
+        }
+
+        {
+            std::unique_lock<std::mutex> lock(_mutex);
+            while (!_pending.empty() && !_stop) {
+                auto filePath = _pending.front();
+                _pending.pop();
+
+                lock.unlock();
+                if (!asked.count(filePath)) {
+                    asked.insert(filePath);
+                    socket.SendMsg(wstring(L"RETRIEVE_FILE_STATUS:" + filePath + L'\n').data());
+                }
+                lock.lock();
+            }
+        }
+
+        std::wstring response;
+        while (!_stop && socket.ReadLine(&response)) {
+			if (StringUtil::begins_with(response, wstring(L"REGISTER_PATH:"))) {
+				wstring responsePath = response.substr(14); // length of REGISTER_PATH:
+
+				{   std::unique_lock<std::mutex> lock(_mutex);
+					_watchedDirectories.push_back(responsePath);
+				}
+				SHChangeNotify(SHCNE_UPDATEDIR, SHCNF_PATH, responsePath.data(), NULL);
+			} else if (StringUtil::begins_with(response, wstring(L"UNREGISTER_PATH:"))) {
+                wstring responsePath = response.substr(16); // length of UNREGISTER_PATH:
+
+                {   std::unique_lock<std::mutex> lock(_mutex);
+                    _watchedDirectories.erase(
+                        std::remove(_watchedDirectories.begin(), _watchedDirectories.end(), responsePath),
+                        _watchedDirectories.end());
+
+                    // Remove any item from the cache
+                    for (auto it = _cache.begin(); it != _cache.end() ; ) {
+                        if (StringUtil::begins_with(it->first, responsePath)) {
+                            it = _cache.erase(it);
+                        } else {
+                            ++it;
+                        }
+                    }
+                }
+                SHChangeNotify(SHCNE_UPDATEDIR, SHCNF_PATH, responsePath.data(), NULL);
+            } else if (StringUtil::begins_with(response, wstring(L"STATUS:")) ||
+                    StringUtil::begins_with(response, wstring(L"BROADCAST:"))) {
+
+                auto statusBegin = response.find(L':', 0);
+                assert(statusBegin != std::wstring::npos);
+
+                auto statusEnd = response.find(L':', statusBegin + 1);
+                if (statusEnd == std::wstring::npos) {
+                    // the command do not contains two colon?
+                    continue;
+                }
+
+                auto responseStatus = response.substr(statusBegin+1, statusEnd - statusBegin-1);
+                auto responsePath = response.substr(statusEnd+1);
+                auto state = _StrToFileState(responseStatus);
+                auto erased = asked.erase(responsePath);
+
+                {   std::unique_lock<std::mutex> lock(_mutex);
+                    _cache[responsePath] = state;
+                }
+                SHChangeNotify(SHCNE_UPDATEITEM, SHCNF_PATH, responsePath.data(), NULL);
+            }
+        }
+
+		if (socket.Event() == INVALID_HANDLE_VALUE) {
+			std::unique_lock<std::mutex> lock(_mutex);
+			_cache.clear();
+			_watchedDirectories.clear();
+			_connected = connected = false;
+		}
+
+		if (_stop) return;
+
+		HANDLE handles[2] = { _newQueries, socket.Event() };
+		WaitForMultipleObjects(2, handles, false, 0);
+    }
+}
+
+
+
+RemotePathChecker::RemotePathChecker()
+    : _connected(false)
+    , _newQueries(CreateEvent(NULL, true, true, NULL))
+	, _thread([this]{ this->workerThreadLoop(); })
+{
+}
+
+RemotePathChecker::~RemotePathChecker()
+{
+    _stop = true;
+    //_newQueries.notify_all();
+    SetEvent(_newQueries);
+    _thread.join();
+    CloseHandle(_newQueries);
 }
 
 vector<wstring> RemotePathChecker::WatchedDirectories()
 {
-	vector<wstring> watchedDirectories;
+    std::unique_lock<std::mutex> lock(_mutex);
-	wstring response;
+    return _watchedDirectories;
-	bool needed = false;
-
-	CommunicationSocket socket(_port);
-	socket.Connect();
-
-	while (socket.ReadLine(&response)) {
-		if (StringUtil::begins_with(response, wstring(L"REGISTER_PATH:"))) {
-			size_t pathBegin = response.find(L':', 0);
-			if (pathBegin == -1) {
-				continue;
-			}
-
-			// chop trailing '\n'
-			wstring responsePath = response.substr(pathBegin + 1, response.length()-1);
-			watchedDirectories.push_back(responsePath);
-		}
-	}
-
-	return watchedDirectories;
 }
 
 bool RemotePathChecker::IsMonitoredPath(const wchar_t* filePath, int* state)
 {
-	wstring request;
+    assert(state); assert(filePath);
-	wstring response;
-	bool needed = false;
 
-	CommunicationSocket socket(_port);
+    std::unique_lock<std::mutex> lock(_mutex);
-	socket.Connect();
+    if (!_connected) {
-	request = L"RETRIEVE_FILE_STATUS:";
+        return false;
-	request += filePath;
+    }
-	request += L'\n';
 
-	if (!socket.SendMsg(request.c_str())) {
+    auto path = std::wstring(filePath);
-		return false;
-	}
 
-	while (socket.ReadLine(&response)) {
+    auto it = _cache.find(path);
-		// discard broadcast messages
+    if (it != _cache.end()) {
-		if (StringUtil::begins_with(response, wstring(L"STATUS:"))) {
+        *state = it->second;
-			break;
+        return true;
-		}
+    }
-	}
 
-	size_t statusBegin = response.find(L':', 0);
+    _pending.push(filePath);
-	if (statusBegin == -1)
+    SetEvent(_newQueries);
-		return false;
+    return false;
 
-	size_t statusEnd = response.find(L':', statusBegin + 1);
-	if (statusEnd == -1)
-		return false;
-
-
-	wstring responseStatus = response.substr(statusBegin+1, statusEnd - statusBegin-1);
-	wstring responsePath = response.substr(statusEnd+1);
-	if (responsePath == filePath) {
-		if (!state) {
-			return false;
-		}
-		*state = _StrToFileState(responseStatus);
-		if (*state == StateNone) {
-			return false;
-		}
-		needed = true;
-	}
-
-	return needed;
 }
 
-int RemotePathChecker::_StrToFileState(const std::wstring &str)
+RemotePathChecker::FileState RemotePathChecker::_StrToFileState(const std::wstring &str)
 {
 	if (str == L"NOP" || str == L"NONE") {
 		return StateNone;

shell_integration/windows/OCShellExtensions/OCUtil/RemotePathChecker.h

@@ -16,6 +16,12 @@
 
 #include <string>
 #include <vector>
+#include <unordered_map>
+#include <queue>
+#include <thread>
+#include <mutex>
+#include <atomic>
+#include <condition_variable>
 
 #pragma once    
 
@@ -29,14 +35,33 @@ public:
 		StateWarning, StateWarningSWM,
 		StateNone
 	};
-	RemotePathChecker(int port);
+	RemotePathChecker();
+    ~RemotePathChecker();
 	std::vector<std::wstring> WatchedDirectories();
 	bool IsMonitoredPath(const wchar_t* filePath, int* state);
 
 private:
-	int _StrToFileState(const std::wstring &str);
+	FileState _StrToFileState(const std::wstring &str);
-	int _port;
+    std::mutex _mutex;
+    std::atomic<bool> _stop;
 
+    // Everything here is protected by the _mutex
+
+    /** The list of paths we need to query. The main thread fill this, and the worker thread
+     * send that to the socket. */
+    std::queue<std::wstring> _pending;
+
+    std::unordered_map<std::wstring, FileState> _cache;
+    std::vector<std::wstring> _watchedDirectories;
+    bool _connected;
+
+
+    // The main thread notifies when there are new items in _pending
+    //std::condition_variable _newQueries;
+    HANDLE _newQueries;
+
+	std::thread _thread;
+    void workerThreadLoop();
 };
 
 #endif

shell_integration/windows/OCShellExtensions/OCUtil/StringUtil.cpp

@@ -11,22 +11,26 @@
 * details.
 */
 
-#include <Windows.h>
+#include <locale>
+#include <string>
+#include <codecvt>
 
 #include "StringUtil.h"
 
-char* StringUtil::toUtf8(const wchar_t *utf16, int len)
+std::string StringUtil::toUtf8(const wchar_t *utf16, int len)
 {
-	int newlen = WideCharToMultiByte(CP_UTF8, 0, utf16, len, NULL, 0, NULL, NULL);
+	if (len < 0) {
-	char* str = new char[newlen];
+		len = wcslen(utf16);
-	WideCharToMultiByte(CP_UTF8, 0, utf16, -1, str, newlen, NULL, NULL);
+	}
-	return str;
+    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t> > converter;
+    return converter.to_bytes(utf16, utf16+len);
 }
 
-wchar_t* StringUtil::toUtf16(const char *utf8, int len)
+std::wstring StringUtil::toUtf16(const char *utf8, int len)
 {
-	int newlen = MultiByteToWideChar(CP_UTF8, 0, utf8, len, NULL, 0);
+	if (len < 0) {
-	wchar_t* wstr = new wchar_t[newlen];
+		len = strlen(utf8);
-	MultiByteToWideChar(CP_UTF8, 0, utf8, -1, wstr, newlen);
+	}
-	return wstr;
+	std::wstring_convert<std::codecvt_utf8_utf16<wchar_t> > converter;
+    return converter.from_bytes(utf8, utf8+len);
 }

shell_integration/windows/OCShellExtensions/OCUtil/StringUtil.h

@@ -20,15 +20,14 @@
 
 class __declspec(dllexport) StringUtil {
 public:
-	static char*    toUtf8(const wchar_t* utf16, int len = -1);
+	static std::string  toUtf8(const wchar_t* utf16, int len = -1);
-	static wchar_t* toUtf16(const char* utf8, int len = -1);
+	static std::wstring toUtf16(const char* utf8, int len = -1);
-
 
 	template<class T>
 	static bool begins_with(const T& input, const T& match)
 	{
 		return input.size() >= match.size()
-			&& equal(match.begin(), match.end(), input.begin());
+			&& std::equal(match.begin(), match.end(), input.begin());
 	}
 };
 

src/CMakeLists.txt

@@ -75,6 +75,7 @@ set(libsync_SRCS
     mirall/propagator_legacy.cpp
     mirall/syncjournalfilerecord.cpp
     mirall/syncjournaldb.cpp
+    mirall/ownsql.cpp
     mirall/theme.cpp
     mirall/owncloudtheme.cpp
     mirall/logger.cpp
@@ -337,7 +338,6 @@ set(ownCloud ${ownCloud_old})
 if (WITH_DBUS)
     set(ADDITIONAL_APP_MODULES DBus)
 endif(WITH_DBUS)
-
 if(NOT BUILD_OWNCLOUD_OSX_BUNDLE AND NOT BUILD_LIBRARIES_ONLY)
     set(BIN_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
 
@@ -368,8 +368,15 @@ elseif(NOT BUILD_LIBRARIES_ONLY)
 
     set (QM_DIR ${OWNCLOUD_OSX_BUNDLE}/Contents/Resources/Translations)
     install(FILES ${mirall_I18N} DESTINATION ${QM_DIR})
+    get_target_property(_qmake Qt5::qmake LOCATION)
+    execute_process(COMMAND ${_qmake} -query QT_INSTALL_TRANSLATIONS
+        OUTPUT_VARIABLE QT_TRANSLATIONS_DIR
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
     file(GLOB qt_I18N ${QT_TRANSLATIONS_DIR}/qt_??.qm ${QT_TRANSLATIONS_DIR}/qt_??_??.qm)
     install(FILES ${qt_I18N} DESTINATION ${QM_DIR})
+    file(GLOB qtbase_I18N ${QT_TRANSLATIONS_DIR}/qtbase_??.qm ${QT_TRANSLATIONS_DIR}/qt_??_??.qm)
+    install(FILES ${qtbase_I18N} DESTINATION ${QM_DIR})
     file(GLOB qtkeychain_I18N ${QT_TRANSLATIONS_DIR}/qtkeychain*.qm)
     install(FILES ${qtkeychain_I18N} DESTINATION ${QM_DIR})
 endif()
@@ -383,7 +390,8 @@ if(NOT BUILD_LIBRARIES_ONLY)
           RUNTIME_OUTPUT_DIRECTORY  ${BIN_OUTPUT_DIRECTORY}
   )
   set_target_properties( ${APPLICATION_EXECUTABLE} PROPERTIES
-	  INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${LIB_INSTALL_DIR}/${APPLICATION_EXECUTABLE}" )
+          INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${LIB_INSTALL_DIR}/" )
+
 
   target_link_libraries( ${APPLICATION_EXECUTABLE} ${QT_LIBRARIES} )
   target_link_libraries( ${APPLICATION_EXECUTABLE} ${synclib_NAME} )
@@ -442,8 +450,9 @@ endif()
 # currently it needs to be done because the code right above needs to be executed no matter
 # if building a bundle or not and the install_qt4_executable needs to be called afterwards
 if(BUILD_OWNCLOUD_OSX_BUNDLE AND NOT BUILD_LIBRARIES_ONLY)
+   get_target_property (QT_QMAKE_EXECUTABLE Qt5::qmake IMPORTED_LOCATION)
    install(CODE "
     message(STATUS \"Deploying (Qt) dependencies and fixing library pathes...\")
-    execute_process(COMMAND \"${CMAKE_SOURCE_DIR}/admin/osx/macdeployqt.py\" ${CMAKE_INSTALL_PREFIX}/${OWNCLOUD_OSX_BUNDLE})
+    execute_process(COMMAND \"${CMAKE_SOURCE_DIR}/admin/osx/macdeployqt.py\" ${CMAKE_INSTALL_PREFIX}/${OWNCLOUD_OSX_BUNDLE} ${QT_QMAKE_EXECUTABLE})
     " COMPONENT RUNTIME)
 endif()

src/creds/httpcredentials.cpp

@@ -66,7 +66,11 @@ int getauth(const char *prompt,
         // qDebug() << "OOO Password requested!";
         qstrncpy( buf, pwd.toUtf8().constData(), len );
     } else {
-        re = handleNeonSSLProblems(prompt, buf, len, echo, verify, userdata);
+        if( http_credentials->sslIsTrusted() ) {
+            qstrcpy( buf, "yes" ); // Certificate is fine!
+        } else {
+            re = handleNeonSSLProblems(prompt, buf, len, echo, verify, userdata);
+        }
     }
     return re;
 }

src/creds/httpcredentials.h

@@ -52,6 +52,7 @@ public:
     virtual QString queryPassword(bool *ok) = 0;
     void invalidateToken(Account *account) Q_DECL_OVERRIDE;
     QString fetchUser(Account *account);
+    virtual bool sslIsTrusted() { return false; }
 
 private Q_SLOTS:
     void slotAuthentication(QNetworkReply*, QAuthenticator*);

src/main.cpp

@@ -82,6 +82,7 @@ int main(int argc, char **argv)
 
     // if the application is already running, notify it.
     if( app.isRunning() ) {
+        qDebug() << Q_FUNC_INFO << "Already running, exiting...";
         QStringList args = app.arguments();
         if ( args.size() > 1 && ! app.giveHelp() ) {
             QString msg = args.join( QLatin1String("|") );

src/mirall/accountsettings.cpp

@@ -605,7 +605,7 @@ void AccountSettings::slotSetProgress(const QString& folder, const Progress::Inf
     item->setData( QVariant(true), FolderStatusDelegate::AddProgressSpace );
 
     if (!progress._currentDiscoveredFolder.isEmpty()) {
-        item->setData( tr("Discovering %1").arg(progress._currentDiscoveredFolder) , FolderStatusDelegate::SyncProgressItemString );
+        item->setData( tr("Discovering '%1'").arg(progress._currentDiscoveredFolder) , FolderStatusDelegate::SyncProgressItemString );
         return;
     }
 

src/mirall/application.cpp

@@ -534,12 +534,15 @@ void Application::setupTranslations()
             setProperty("ui_lang", lang);
             const QString qtTrPath = QLibraryInfo::location(QLibraryInfo::TranslationsPath);
             const QString qtTrFile = QLatin1String("qt_") + lang;
-            if (qtTranslator->load(qtTrFile, qtTrPath)) {
+            const QString qtBaseTrFile = QLatin1String("qtbase_") + lang;
-                qtTranslator->load(qtTrFile, trPath);
+            if (!qtTranslator->load(qtTrFile, qtTrPath)) {
+                if (!qtTranslator->load(qtTrFile, trPath)) {
+                    qtTranslator->load(qtBaseTrFile, trPath);
+                }
             }
-            const QString qtkeychainFile = QLatin1String("qt_") + lang;
+            const QString qtkeychainTrFile = QLatin1String("qtkeychain_") + lang;
-            if (!qtkeychainTranslator->load(qtkeychainFile, qtTrPath)) {
+            if (!qtkeychainTranslator->load(qtkeychainTrFile, qtTrPath)) {
-               qtkeychainTranslator->load(qtkeychainFile, trPath);
+                qtkeychainTranslator->load(qtkeychainTrFile, trPath);
             }
             if (!translator->isEmpty())
                 installTranslator(translator);

src/mirall/discoveryphase.cpp

@@ -16,9 +16,11 @@
 #include <csync_private.h>
 #include <qdebug.h>
 
+#include <QUrl>
+
 namespace Mirall {
 
-bool DiscoveryJob::isInBlackList(const QString& path) const
+bool DiscoveryJob::isInSelectiveSyncBlackList(const QString& path) const
 {
     if (_selectiveSyncBlackList.isEmpty()) {
         // If there is no black list, everything is allowed
@@ -35,19 +37,24 @@ bool DiscoveryJob::isInBlackList(const QString& path) const
 
     auto it = std::lower_bound(_selectiveSyncBlackList.begin(), _selectiveSyncBlackList.end(), pathSlash);
 
+    if (it != _selectiveSyncBlackList.end() && *it == pathSlash) {
+        return true;
+    }
+
 	if (it == _selectiveSyncBlackList.begin()) {
         return false;
     }
     --it;
-    if (pathSlash.startsWith(*it + QLatin1Char('/'))) {
+    Q_ASSERT(it->endsWith(QLatin1Char('/'))); // SyncEngine::setSelectiveSyncBlackList makes sure of that
+    if (pathSlash.startsWith(*it)) {
         return true;
     }
     return false;
 }
 
-int DiscoveryJob::isInBlackListCallBack(void *data, const char *path)
+int DiscoveryJob::isInSelectiveSyncBlackListCallBack(void *data, const char *path)
 {
-    return static_cast<DiscoveryJob*>(data)->isInBlackList(QString::fromUtf8(path));
+    return static_cast<DiscoveryJob*>(data)->isInSelectiveSyncBlackList(QString::fromUtf8(path));
 }
 
 void DiscoveryJob::update_job_update_callback (bool local,
@@ -65,15 +72,15 @@ void DiscoveryJob::update_job_update_callback (bool local,
             updateJob->lastUpdateProgressCallbackCall.restart();
         }
 
-        QString path = QString::fromUtf8(dirUrl).section('/', -1);
+        QString path(QUrl::fromPercentEncoding(QByteArray(dirUrl)).section('/', -1));
         emit updateJob->folderDiscovered(local, path);
     }
 }
 
 void DiscoveryJob::start() {
     _selectiveSyncBlackList.sort();
-    _csync_ctx->checkBlackListHook = isInBlackListCallBack;
+    _csync_ctx->checkSelectiveSyncBlackListHook = isInSelectiveSyncBlackListCallBack;
-    _csync_ctx->checkBlackListData = this;
+    _csync_ctx->checkSelectiveSyncBlackListData = this;
 
     _csync_ctx->callbacks.update_callback = update_job_update_callback;
     _csync_ctx->callbacks.update_callback_userdata = this;
@@ -85,8 +92,8 @@ void DiscoveryJob::start() {
     lastUpdateProgressCallbackCall.invalidate();
     int ret = csync_update(_csync_ctx);
 
-    _csync_ctx->checkBlackListHook = 0;
+    _csync_ctx->checkSelectiveSyncBlackListHook = 0;
-    _csync_ctx->checkBlackListData = 0;
+    _csync_ctx->checkSelectiveSyncBlackListData = 0;
 
     _csync_ctx->callbacks.update_callback = 0;
     _csync_ctx->callbacks.update_callback_userdata = 0;

src/mirall/discoveryphase.h

@@ -40,8 +40,8 @@ class DiscoveryJob : public QObject {
      * return true if the given path should be synced,
      * false if the path should be ignored
      */
-    bool isInBlackList(const QString &path) const;
+    bool isInSelectiveSyncBlackList(const QString &path) const;
-    static int isInBlackListCallBack(void *, const char *);
+    static int isInSelectiveSyncBlackListCallBack(void *, const char *);
 
     static void update_job_update_callback (bool local,
                                             const char *dirname,

src/mirall/folder.cpp

@@ -60,6 +60,7 @@ Folder::Folder(const QString &alias, const QString &path, const QString& secondP
       , _csyncUnavail(false)
       , _wipeDb(false)
       , _proxyDirty(true)
+      , _forceSyncOnPollTimeout(false)
       , _journal(path)
       , _csync_ctx(0)
 {
@@ -269,10 +270,25 @@ void Folder::slotPollTimerTimeout()
         return;
     }
 
-    if (quint64(_timeSinceLastSync.elapsed()) > MirallConfigFile().forceSyncInterval() ||
+    bool forceSyncIntervalExpired =
-            _lastEtag.isNull() ||
+            quint64(_timeSinceLastSync.elapsed()) > MirallConfigFile().forceSyncInterval();
-            !(_syncResult.status() == SyncResult::Success ||_syncResult.status() == SyncResult::Problem)) {
+    bool okSyncResult =
-        qDebug() << "** Force Sync now, state is " << _syncResult.statusString();
+            _syncResult.status() == SyncResult::Success ||
+            _syncResult.status() == SyncResult::Problem;
+    if (forceSyncIntervalExpired ||
+            _forceSyncOnPollTimeout ||
+            !okSyncResult) {
+        if (forceSyncIntervalExpired) {
+            qDebug() << "** Force Sync, because it has been " << _timeSinceLastSync.elapsed() << "ms "
+                     << "since the last sync";
+        }
+        if (_forceSyncOnPollTimeout) {
+            qDebug() << "** Force Sync, because it was requested";
+        }
+        if (!okSyncResult) {
+            qDebug() << "** Force Sync, because the last sync had status: " << _syncResult.statusString();
+        }
+        _forceSyncOnPollTimeout = false;
         emit scheduleToSync(alias());
     } else {
         // do the ordinary etag check for the root folder.
@@ -496,11 +512,90 @@ QString Folder::configFile()
     return _configFile;
 }
 
+static void addErroredSyncItemPathsToList(const SyncFileItemVector& items, QSet<QString>* set) {
+    Q_FOREACH(const SyncFileItem &item, items) {
+        if (item.hasErrorStatus()) {
+            set->insert(item._file);
+        }
+    }
+}
+
 void Folder::slotThreadTreeWalkResult(const SyncFileItemVector& items)
 {
+    addErroredSyncItemPathsToList(items, &this->_stateLastSyncItemsWithError);
     _syncResult.setSyncFileItemVector(items);
 }
 
+void Folder::slotAboutToPropagate(const SyncFileItemVector& items)
+{
+    // empty the tainted list since the status generation code will use the _syncedItems
+    // (which imply the folder) to generate the syncing state icon now.
+    _stateTaintedFolders.clear();
+
+    addErroredSyncItemPathsToList(items, &this->_stateLastSyncItemsWithError);
+}
+
+
+bool Folder::estimateState(QString fn, csync_ftw_type_e t, SyncFileStatus* s)
+{
+    if (t == CSYNC_FTW_TYPE_DIR) {
+        qDebug() << Q_FUNC_INFO << "ASKING ERROR FOLDERS" << fn;
+        if (Utility::doesSetContainPrefix(_stateLastSyncItemsWithError, fn)) {
+            s->set(SyncFileStatus::STATUS_ERROR);
+            return true;
+        }
+        // If sync is running, check _syncedItems, possibly give it STATUS_EVAL (=syncing down)
+        if (!_engine.isNull()) {
+            qDebug() << Q_FUNC_INFO << "SYNC IS RUNNING, asking SyncEngine" << fn;
+            if (_engine->estimateState(fn, t, s)) {
+                return true;
+            }
+        }
+        qDebug() << Q_FUNC_INFO << "ASKING TAINTED FOLDERS" << fn;
+        if (Utility::doesSetContainPrefix(_stateTaintedFolders, fn)) {
+            qDebug() << Q_FUNC_INFO << "Folder is tainted, EVAL!" << fn;
+            s->set(SyncFileStatus::STATUS_EVAL);
+            return true;
+        }
+        return false;
+    } else if ( t== CSYNC_FTW_TYPE_FILE) {
+        // check if errorList has the directory/file
+        if (Utility::doesSetContainPrefix(_stateLastSyncItemsWithError, fn)) {
+            s->set(SyncFileStatus::STATUS_ERROR);
+            return true;
+        }
+        // If sync running: _syncedItems -> SyncingState
+        if (!_engine.isNull()) {
+            if (_engine->estimateState(fn, t, s)) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+void Folder::watcherSlot(QString fn)
+{
+    // FIXME: On OS X we could not do this "if" since on OS X the file watcher ignores events for ourselves
+    // however to have the same behaviour atm on all platforms, we don't do it
+    if (!_engine.isNull()) {
+        qDebug() << Q_FUNC_INFO << "Sync running, IGNORE event for " << fn;
+        return;
+    }
+    QFileInfo fi(fn);
+    if (fi.isFile()) {
+        fn = fi.path(); // depending on OS, file watcher might be for dir or file
+    }
+    // Make it a relative path depending on the folder
+    QString relativePath = fn.remove(0, path().length());
+    qDebug() << Q_FUNC_INFO << fi.canonicalFilePath() << fn << relativePath;
+    _stateTaintedFolders.insert(relativePath);
+
+    // Notify the SocketAPI?
+}
+
+
+
 void Folder::slotTerminateSync()
 {
     qDebug() << "folder " << alias() << " Terminating!";
@@ -623,6 +718,8 @@ void Folder::startSync(const QStringList &pathList)
 
     connect( _engine.data(), SIGNAL(treeWalkResult(const SyncFileItemVector&)),
               this, SLOT(slotThreadTreeWalkResult(const SyncFileItemVector&)), Qt::QueuedConnection);
+    connect( _engine.data(), SIGNAL(aboutToPropagate(const SyncFileItemVector&)),
+              this, SLOT(slotAboutToPropagate(const SyncFileItemVector&)), Qt::QueuedConnection);
 
     connect(_engine.data(), SIGNAL(started()),  SLOT(slotSyncStarted()), Qt::QueuedConnection);
     connect(_engine.data(), SIGNAL(finished()), SLOT(slotSyncFinished()), Qt::QueuedConnection);
@@ -689,8 +786,11 @@ void Folder::slotCsyncUnavailable()
 
 void Folder::slotSyncFinished()
 {
-    qDebug() << "-> CSync Finished slot with error " << _csyncError << "warn count" << _syncResult.warnCount();
+    if( _csyncError ) {
-
+        qDebug() << "-> SyncEngine finished with ERROR, warn count is" << _syncResult.warnCount();
+    } else {
+        qDebug() << "-> SyncEngine finished without problem.";
+    }
     bubbleUpSyncResult();
 
     bool anotherSyncNeeded = false;
@@ -701,6 +801,15 @@ void Folder::slotSyncFinished()
     // _watcher->setEventsEnabledDelayed(2000);
 
 
+    // This is for sync state calculation
+    _stateLastSyncItemsWithError = _stateLastSyncItemsWithErrorNew;
+    _stateLastSyncItemsWithErrorNew.clear();
+    _stateTaintedFolders.clear(); // heuristic: assume the sync had been done, new file watches needed to taint dirs
+    if (_csyncError || _csyncUnavail) {
+        // Taint the whole sync dir, we cannot give reliable state information
+        _stateTaintedFolders.insert(QLatin1String("/"));
+    }
+
 
     if (_csyncError) {
         _syncResult.setStatus(SyncResult::Error);
@@ -731,9 +840,9 @@ void Folder::slotSyncFinished()
         _pollTimer.start();
         _timeSinceLastSync.restart();
     } else {
-        // Another sync is required.  We will make sure that the poll timer occurs soon enough
+        // Another sync is required.  We will make sure that the poll timer occurs soon enough.
-        // and we clear the etag to force a sync
+        qDebug() << "another sync was requested by the finished sync";
-        _lastEtag.clear();
+        _forceSyncOnPollTimeout = true;
         QTimer::singleShot(1000, this, SLOT(slotPollTimerTimeout() ));
     }
 
@@ -767,6 +876,10 @@ void Folder::slotTransmissionProgress(const Progress::Info &pi)
 // a job is completed: count the errors and forward to the ProgressDispatcher
 void Folder::slotJobCompleted(const SyncFileItem &item)
 {
+    if (item.hasErrorStatus()) {
+        _stateLastSyncItemsWithError.insert(item._file);
+    }
+
     if (Progress::isWarningKind(item._status)) {
         // Count all error conditions.
         _syncResult.setWarnCount(_syncResult.warnCount()+1);
@@ -799,7 +912,8 @@ void Folder::slotAboutToRemoveAllFiles(SyncFileItem::Direction, bool *cancel)
     if (*cancel) {
         wipe();
         // speed up next sync
-        _lastEtag = QString();
+        _lastEtag.clear();
+        _forceSyncOnPollTimeout = true;
         QTimer::singleShot(50, this, SLOT(slotPollTimerTimeout()));
     }
 }

src/mirall/folder.h

@@ -27,6 +27,7 @@
 
 #include <QDir>
 #include <QHash>
+#include <QSet>
 #include <QObject>
 #include <QStringList>
 
@@ -34,15 +35,12 @@
 #include <QTimer>
 #include <qelapsedtimer.h>
 
-class QFileSystemWatcher;
 class QThread;
 
 namespace Mirall {
 
 class SyncEngine;
 
-class FolderWatcher;
-
 class Folder : public QObject
 {
     Q_OBJECT
@@ -119,12 +117,12 @@ public:
 
      // Used by the Socket API
      SyncJournalDb *journalDb() { return &_journal; }
-     CSYNC *csyncContext() { return _csync_ctx; }
 
      QStringList selectiveSyncBlackList() { return _selectiveSyncBlackList; }
      void setSelectiveSyncBlackList(const QStringList &blackList)
      { _selectiveSyncBlackList = blackList; }
 
+     bool estimateState(QString fn, csync_ftw_type_e t, SyncFileStatus* s);
 
 signals:
     void syncStateChange();
@@ -170,10 +168,13 @@ private slots:
     void etagRetreived(const QString &);
     void slotNetworkUnavailable();
 
-    void slotThreadTreeWalkResult(const SyncFileItemVector& );
+    void slotAboutToPropagate(const SyncFileItemVector& );
+    void slotThreadTreeWalkResult(const SyncFileItemVector& ); // after sync is done
 
     void slotEmitFinishedDelayed();
 
+    void watcherSlot(QString);
+
 private:
     bool init();
 
@@ -202,6 +203,12 @@ private:
     QTimer        _pollTimer;
     QString       _lastEtag;
     QElapsedTimer _timeSinceLastSync;
+    bool          _forceSyncOnPollTimeout;
+
+    // For the SocketAPI folder states
+    QSet<QString>   _stateLastSyncItemsWithErrorNew; // gets moved to _stateLastSyncItemsWithError at end of sync
+    QSet<QString>   _stateLastSyncItemsWithError;
+    QSet<QString>   _stateTaintedFolders;
 
     SyncJournalDb _journal;
 

src/mirall/folderman.cpp

@@ -147,6 +147,9 @@ void FolderMan::registerFolderMonitor( Folder *folder )
         connect(fw, SIGNAL(folderChanged(QString)), _folderWatcherSignalMapper, SLOT(map()));
         _folderWatcherSignalMapper->setMapping(fw, folder->alias());
         _folderWatchers.insert(folder->alias(), fw);
+
+        // This is at the moment only for the behaviour of the SocketApi.
+        connect(fw, SIGNAL(folderChanged(QString)), folder, SLOT(watcherSlot(QString)));
     }
 
     // register the folder with the socket API
@@ -442,6 +445,24 @@ void FolderMan::slotScheduleSync( const QString& alias )
         return;
     }
 
+    // The folder watcher fires a lot of bogus notifications during
+    // a sync operation, both for actual user files and the database
+    // and log. Never enqueue a folder for sync while it is syncing.
+    // We lose some genuine sync requests that way, but that can't be
+    // helped.
+    // ^^ FIXME: Note that this is not the case on OS X
+    if( _currentSyncFolder == alias ) {
+        qDebug() << "folder " << alias << " is currently syncing. NOT scheduling.";
+        return;
+    }
+
+    if( _socketApi ) {
+        // We want the SocketAPI to already now update so that it can show the EVAL icon
+        // for files/folders. Only do this when not syncing, else we might get a lot
+        // of those notifications.
+        _socketApi->slotUpdateFolderView(alias);
+    }
+
     qDebug() << "Schedule folder " << alias << " to sync!";
 
     if( ! _scheduleQueue.contains(alias) ) {
@@ -532,6 +553,8 @@ void FolderMan::slotFolderSyncStarted( )
 /*
   * a folder indicates that its syncing is finished.
   * Start the next sync after the system had some milliseconds to breath.
+  * This delay is particularly useful to avoid late file change notifications
+  * (that we caused ourselves by syncing) from triggering another spurious sync.
   */
 void FolderMan::slotFolderSyncFinished( const SyncResult& )
 {
@@ -566,11 +589,11 @@ Folder *FolderMan::folderForPath(const QString &path)
         const QString folderPath = QDir::cleanPath(folder->path())+QLatin1Char('/');
 
         if(absolutePath.startsWith(folderPath)) {
-            qDebug() << "found folder: " << folder->path() << " for " << absolutePath;
+            //qDebug() << "found folder: " << folder->path() << " for " << absolutePath;
             return folder;
         }
     }
-
+    qDebug() << "ERROR: could not find folder for " << absolutePath;
     return 0;
 }
 

src/mirall/folderwatcher.cpp

@@ -81,7 +81,7 @@ bool FolderWatcher::pathIsIgnored( const QString& path )
 
         QFileInfo fInfo(path);
         if( fInfo.isHidden() ) {
-            qDebug() << "* Discarded as is hidden!";
+            qDebug() << "* Discarded as is hidden!" << fInfo.filePath();
             return true;
         }
 

src/mirall/folderwizard.cpp

@@ -232,6 +232,9 @@ FolderWizardRemotePath::FolderWizardRemotePath()
     _ui.setupUi(this);
     _ui.warnFrame->hide();
 
+    _ui.folderTreeWidget->setSortingEnabled(true);
+    _ui.folderTreeWidget->sortByColumn(0, Qt::AscendingOrder);
+
     connect(_ui.addFolderButton, SIGNAL(clicked()), SLOT(slotAddRemoteFolder()));
     connect(_ui.refreshButton, SIGNAL(clicked()), SLOT(slotRefreshFolders()));
     connect(_ui.folderTreeWidget, SIGNAL(itemClicked(QTreeWidgetItem*,int)), SIGNAL(completeChanged()));

src/mirall/logger.cpp

@@ -16,6 +16,7 @@
 
 #include <QDir>
 #include <QStringList>
+#include <QThread>
 
 namespace Mirall {
 
@@ -85,6 +86,7 @@ void Logger::log(Log log)
     } else {
         // msg += "ownCloud - ";
     }
+    msg += QString().sprintf("%p ", (void*)QThread::currentThread());
     msg += log.message;
     // _logs.append(log);
     // std::cout << qPrintable(log.message) << std::endl;

src/mirall/mirallconfigfile.cpp

@@ -206,15 +206,16 @@ QString MirallConfigFile::configPathWithAppName() const
     return QFileInfo( configFile() ).dir().absolutePath().append("/");
 }
 
+static const QLatin1String exclFile("sync-exclude.lst");
+
 QString MirallConfigFile::excludeFile(Scope scope) const
 {
     // prefer sync-exclude.lst, but if it does not exist, check for
     // exclude.lst for compatibility reasons in the user writeable
     // directories.
-    const QString exclFile("sync-exclude.lst");
-    QFileInfo fi;
 
     if (scope != SystemScope) {
+        QFileInfo fi;
         fi.setFile( configPath(), exclFile );
 
         if( ! fi.isReadable() ) {
@@ -223,32 +224,37 @@ QString MirallConfigFile::excludeFile(Scope scope) const
         if( ! fi.isReadable() ) {
             fi.setFile( configPath(), exclFile );
         }
+        return fi.absoluteFilePath();
+    } else if (scope != UserScope) {
+        return MirallConfigFile::excludeFileFromSystem();
+    } else {
+        Q_ASSERT(false);
+        return QString(); // unreachable
     }
+}
 
-    if (scope != UserScope) {
+QString MirallConfigFile::excludeFileFromSystem()
-        // Check alternative places...
+{
-        if( ! fi.isReadable() ) {
+    QFileInfo fi;
 #ifdef Q_OS_WIN
-            fi.setFile( QCoreApplication::applicationDirPath(), exclFile );
+    fi.setFile( QCoreApplication::applicationDirPath(), exclFile );
 #endif
 #ifdef Q_OS_UNIX
-            fi.setFile( QString( SYSCONFDIR "/%1").arg(Theme::instance()->appName()), exclFile );
+    fi.setFile( QString( SYSCONFDIR "/%1").arg(Theme::instance()->appName()), exclFile );
-            if ( ! fi.exists() ) {
+    if ( ! fi.exists() ) {
-                // Prefer to return the preferred path! Only use the fallback location
+        // Prefer to return the preferred path! Only use the fallback location
-                // if the other path does not exist and the fallback is valid.
+        // if the other path does not exist and the fallback is valid.
-                QFileInfo nextToBinary( QCoreApplication::applicationDirPath(), exclFile );
+        QFileInfo nextToBinary( QCoreApplication::applicationDirPath(), exclFile );
-                if (nextToBinary.exists()) {
+        if (nextToBinary.exists()) {
-                    fi = nextToBinary;
+            fi = nextToBinary;
-                }
-            }
-#endif
-#ifdef Q_OS_MAC
-            // exec path is inside the bundle
-            fi.setFile( QCoreApplication::applicationDirPath(),
-                        QLatin1String("../Resources/") + exclFile );
-#endif
         }
     }
+#endif
+#ifdef Q_OS_MAC
+    // exec path is inside the bundle
+    fi.setFile( QCoreApplication::applicationDirPath(),
+                QLatin1String("../Resources/") + exclFile );
+#endif
     return fi.absoluteFilePath();
 }
 

src/mirall/mirallconfigfile.h

@@ -38,6 +38,7 @@ public:
     QString configPathWithAppName() const;
     QString configFile() const;
     QString excludeFile(Scope scope) const;
+    static QString excludeFileFromSystem(); // doesn't access config dir
 
     bool exists();
 

src/mirall/networkjobs.cpp

@@ -39,6 +39,8 @@ Q_DECLARE_METATYPE(QTimer*)
 
 namespace Mirall {
 
+bool AbstractNetworkJob::preOc7WasDetected = false;
+
 AbstractNetworkJob::AbstractNetworkJob(Account *account, const QString &path, QObject *parent)
     : QObject(parent)
     , _duration(0)
@@ -48,7 +50,12 @@ AbstractNetworkJob::AbstractNetworkJob(Account *account, const QString &path, QO
     , _path(path)
 {
     _timer.setSingleShot(true);
-    _timer.setInterval(10*1000); // default to 10 seconds.
+    if (!AbstractNetworkJob::preOc7WasDetected) {
+        _timer.setInterval(10*1000); // default to 10 seconds.
+    } else {
+        qDebug() << "Pre-oc7 server detected, adjusting timeout values";
+        _timer.setInterval(60*1000); // long PROPFINDs in oc6 might take too long
+    }
     connect(&_timer, SIGNAL(timeout()), this, SLOT(slotTimeout()));
 
     connect(this, SIGNAL(networkActivity()), SLOT(resetTimeout()));
@@ -430,7 +437,9 @@ bool CheckServerJob::finished()
 
     bool success = false;
     QByteArray body = reply()->readAll();
-    if( body.isEmpty() ) {
+    int httpStatus = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
+    if( body.isEmpty() || httpStatus != 200) {
+        qDebug() << "error: status.php replied " << httpStatus << body;
         emit instanceNotFound(reply());
     } else {
         QVariantMap status = QtJson::parse(QString::fromUtf8(body), success).toMap();
@@ -443,6 +452,12 @@ bool CheckServerJob::finished()
         if( status.contains("installed")
                 && status.contains("version")
                 && status.contains("versionstring") ) {
+
+            QString versionString = status.value("version").toString();
+            if (versionString.contains('.') && versionString.split('.')[0].toInt() < 7) {
+                AbstractNetworkJob::preOc7WasDetected = true;
+            }
+
             emit instanceFound(reply()->url(), status);
         } else {
             qDebug() << "No proper answer on " << requestedUrl;

src/mirall/networkjobs.h

@@ -94,6 +94,9 @@ protected:
     QElapsedTimer _durationTimer;
     quint64       _duration;
 
+    // Timeout workarounds (Because of PHP session locking)
+    static bool preOc7WasDetected;
+
 private slots:
     void slotFinished();
     virtual void slotTimeout();

src/mirall/owncloudgui.cpp

@@ -50,7 +50,6 @@ ownCloudGui::ownCloudGui(Application *parent) :
     _settingsDialog(new SettingsDialog(this)),
 #endif
     _logBrowser(0),
-    _contextMenu(0),
     _recentActionsMenu(0),
     _folderOpenActionMapper(new QSignalMapper(this)),
     _recentItemsMapper(new QSignalMapper(this)),
@@ -303,11 +302,11 @@ void ownCloudGui::setupContextMenu()
         _recentActionsMenu->addAction(tr("None."));
         _recentActionsMenu->addAction(_actionRecent);
     } else {
-        _contextMenu = new QMenu(_contextMenu);
+        _contextMenu.reset(new QMenu());
-        _recentActionsMenu = new QMenu(tr("Recent Changes"));
+        _recentActionsMenu = new QMenu(tr("Recent Changes"), _contextMenu.data());
         // this must be called only once after creating the context menu, or
         // it will trigger a bug in Ubuntu's SNI bridge patch (11.10, 12.04).
-        _tray->setContextMenu(_contextMenu);
+        _tray->setContextMenu(_contextMenu.data());
     }
     _contextMenu->setTitle(Theme::instance()->appNameGUI() );
     _contextMenu->addAction(_actionOpenoC);
@@ -464,7 +463,7 @@ void ownCloudGui::slotUpdateProgress(const QString &folder, const Progress::Info
     Q_UNUSED(folder);
 
      if (!progress._currentDiscoveredFolder.isEmpty()) {
-                 _actionStatus->setText( tr("Discovering %1")
+                 _actionStatus->setText( tr("Discovering '%1'")
                      .arg( progress._currentDiscoveredFolder ));
      } else if (progress._totalSize == 0 ) {
             quint64 currentFile =  progress._completedFileCount + progress._currentItems.count();           

src/mirall/owncloudgui.h

@@ -85,7 +85,7 @@ private:
 #endif
     QPointer<LogBrowser>_logBrowser;
        // tray's menu
-    QMenu *_contextMenu;
+    QScopedPointer<QMenu> _contextMenu;
     QMenu *_recentActionsMenu;
 
     QAction *_actionLogin;

src/mirall/owncloudpropagator.cpp

@@ -56,11 +56,10 @@ void PropagateItemJob::done(SyncFileItem::Status status, const QString &errorStr
         }
     }
 
-    if( _propagator->_abortRequested.fetchAndAddRelaxed(0) ) {
+    if( _propagator->_abortRequested.fetchAndAddRelaxed(0) &&
+            (status == SyncFileItem::NormalError || status == SyncFileItem::FatalError)) {
         // an abort request is ongoing. Change the status to Soft-Error
-
         status = SyncFileItem::SoftError;
-        _item._errorString = tr("Operation was canceled by user interaction.");
     }
 
     _item._status = status;
@@ -78,7 +77,7 @@ void PropagateItemJob::done(SyncFileItem::Status status, const QString &errorStr
         }
         retries = defaultRetriesCount.fetchAndAddAcquire(0);
     }
-    SyncJournalBlacklistRecord record(_item, retries);;
+    SyncJournalBlacklistRecord record(_item, retries);
 
     switch( status ) {
     case SyncFileItem::SoftError:
@@ -98,9 +97,13 @@ void PropagateItemJob::done(SyncFileItem::Status status, const QString &errorStr
         break;
     case SyncFileItem::Success:
     case SyncFileItem::Restoration:
-        if( _item._blacklistedInDb ) {
+        if( _item._hasBlacklistEntry ) {
             // wipe blacklist entry.
             _propagator->_journal->wipeBlacklistEntry(_item._file);
+            // remove a blacklist entry in case the file was moved.
+            if( _item._originalFile != _item._file ) {
+                _propagator->_journal->wipeBlacklistEntry(_item._originalFile);
+            }
         }
         break;
     case SyncFileItem::Conflict:
@@ -382,10 +385,10 @@ bool OwncloudPropagator::localFileNameClash( const QString& relFile )
             // returns false.
         } else {
             QString realFileName = QString::fromWCharArray( FindFileData.cFileName );
-            qDebug() << Q_FUNC_INFO << "Real file name is " << realFileName;
             FindClose(hFind);
 
             if( ! file.endsWith(realFileName, Qt::CaseSensitive) ) {
+                qDebug() << Q_FUNC_INFO << "Detected case clash between" << file << "and" << realFileName;
                 re = true;
             }
         }

src/mirall/owncloudtheme.cpp

@@ -52,7 +52,7 @@ QString ownCloudTheme::about() const
                "<p>Copyright ownCloud, Inc.</p>"
                "<p>Licensed under the GNU General Public License (GPL) Version 2.0<br/>"
                "ownCloud and the ownCloud Logo are registered trademarks of ownCloud, "
-               "Inc. in the United States, other countries, or both</p>"
+               "Inc. in the United States, other countries, or both.</p>"
                )
             .arg(MIRALL_VERSION_STRING)
             .arg("http://" MIRALL_STRINGIFY(APPLICATION_DOMAIN))

src/mirall/ownsql.cpp

@@ -0,0 +1,289 @@
+/*
+ * Copyright (C) by Klaas Freitag <freitag@owncloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+
+#include <QDateTime>
+#include <QString>
+#include <QDebug>
+
+#include "ownsql.h"
+#include "utility.h"
+
+#define SQLITE_SLEEP_TIME_USEC 100000
+#define SQLITE_REPEAT_COUNT 20
+
+#define SQLITE_DO(A) if(1) { \
+    _errId = (A); if(_errId != SQLITE_OK) { _error= QString::fromUtf8(sqlite3_errmsg(_db)); \
+     } }
+
+namespace Mirall {
+
+SqlDatabase::SqlDatabase()
+    :_db(0),
+      _errId(0)
+{
+
+}
+
+bool SqlDatabase::isOpen()
+{
+    return _db != 0;
+}
+
+bool SqlDatabase::open( const QString& filename )
+{
+    if(isOpen()) {
+        return true;
+    }
+
+    int flag = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_NOMUTEX;
+    SQLITE_DO( sqlite3_open_v2(filename.toUtf8().constData(), &_db, flag, 0) );
+
+    if( _errId != SQLITE_OK ) {
+        close(); // FIXME: Correct?
+        _db = 0;
+    }
+
+    sqlite3_busy_timeout(_db, 5000);
+
+    return isOpen();
+}
+
+QString SqlDatabase::error() const
+{
+    const QString err(_error);
+    // _error.clear();
+    return err;
+}
+
+void SqlDatabase::close()
+{
+    if( _db ) {
+        SQLITE_DO(sqlite3_close(_db) );
+        _db = 0;
+    }
+}
+
+bool SqlDatabase::transaction()
+{
+    if( ! _db ) {
+        return false;
+    }
+    SQLITE_DO(sqlite3_exec(_db, "BEGIN", 0, 0, 0));
+    return _errId == SQLITE_OK;
+}
+
+bool SqlDatabase::commit()
+{
+    if( ! _db ) {
+        return false;
+    }
+    SQLITE_DO(sqlite3_exec(_db, "COMMIT", 0, 0, 0));
+    return _errId == SQLITE_OK;
+}
+
+sqlite3* SqlDatabase::sqliteDb()
+{
+    return _db;
+}
+
+/* =========================================================================================== */
+
+SqlQuery::SqlQuery( SqlDatabase db )
+    :_db(db.sqliteDb()),
+      _stmt(0)
+{
+
+}
+
+SqlQuery::~SqlQuery()
+{
+    if( _stmt ) {
+        finish();
+    }
+}
+
+SqlQuery::SqlQuery(const QString& sql, SqlDatabase db)
+    :_db(db.sqliteDb()),
+      _stmt(0)
+{
+    prepare(sql);
+}
+
+int SqlQuery::prepare( const QString& sql)
+{
+    QString s(sql);
+    _sql = s.trimmed();
+    if(_stmt ) {
+        finish();
+    }
+    if(!_sql.isEmpty() ) {
+        int n = 0;
+        int rc;
+        do {
+            rc = sqlite3_prepare_v2(_db, _sql.toUtf8().constData(), -1, &_stmt, 0);
+            if( (rc == SQLITE_BUSY) || (rc == SQLITE_LOCKED) ) {
+                n++;
+                Mirall::Utility::usleep(SQLITE_SLEEP_TIME_USEC);
+            }
+        } while( (n < SQLITE_REPEAT_COUNT) && ((rc == SQLITE_BUSY) || (rc == SQLITE_LOCKED)));
+        _errId = rc;
+
+        if( _errId != SQLITE_OK ) {
+            qDebug() << "Sqlite prepare statement error:" << _error << "in"<<_sql;
+        }
+        // Q_ASSERT(_errId == SQLITE_OK);
+    }
+    return _errId;
+}
+
+bool SqlQuery::isSelect()
+{
+    return (!_sql.isEmpty() && _sql.startsWith("SELECT", Qt::CaseInsensitive));
+}
+
+bool SqlQuery::isPragma()
+{
+    return (!_sql.isEmpty() && _sql.startsWith("PRAGMA", Qt::CaseInsensitive));
+}
+
+bool SqlQuery::exec()
+{
+    // Don't do anything for selects, that is how we use the lib :-|
+    if(_stmt && !isSelect() && !isPragma() ) {
+        int rc, n = 0;
+        do {
+            rc = sqlite3_step(_stmt);
+            if( rc == SQLITE_LOCKED ) {
+                rc = sqlite3_reset(_stmt); /* This will also return SQLITE_LOCKED */
+                n++;
+                Mirall::Utility::usleep(SQLITE_SLEEP_TIME_USEC);
+            } else if( (rc == SQLITE_BUSY) ) {
+                Mirall::Utility::usleep(SQLITE_SLEEP_TIME_USEC);
+                n++;
+            }
+        } while( (n < SQLITE_REPEAT_COUNT) && ((rc == SQLITE_BUSY) || (rc == SQLITE_LOCKED)));
+        _errId = rc;
+
+        return (_errId == SQLITE_DONE); // either SQLITE_ROW or SQLITE_DONE
+    }
+
+    return true;
+}
+
+bool SqlQuery::next()
+{
+    SQLITE_DO(sqlite3_step(_stmt));
+    return _errId == SQLITE_ROW;
+}
+
+void SqlQuery::bindValue(int pos, const QVariant& value)
+{
+    int res = -1;
+    if( _stmt ) {
+        switch (value.type()) {
+        case QVariant::Int:
+        case QVariant::Bool:
+            res = sqlite3_bind_int(_stmt, pos, value.toInt());
+            break;
+        case QVariant::Double:
+            res = sqlite3_bind_double(_stmt, pos, value.toDouble());
+            break;
+        case QVariant::UInt:
+        case QVariant::LongLong:
+            res = sqlite3_bind_int64(_stmt, pos, value.toLongLong());
+            break;
+        case QVariant::DateTime: {
+            const QDateTime dateTime = value.toDateTime();
+            const QString str = dateTime.toString(QLatin1String("yyyy-MM-ddThh:mm:ss.zzz"));
+            res = sqlite3_bind_text16(_stmt, pos, str.utf16(),
+                                      str.size() * sizeof(ushort), SQLITE_TRANSIENT);
+            break;
+        }
+        case QVariant::Time: {
+            const QTime time = value.toTime();
+            const QString str = time.toString(QLatin1String("hh:mm:ss.zzz"));
+            res = sqlite3_bind_text16(_stmt, pos, str.utf16(),
+                                      str.size() * sizeof(ushort), SQLITE_TRANSIENT);
+            break;
+        }
+        case QVariant::String: {
+            if( !value.toString().isNull() ) {
+                // lifetime of string == lifetime of its qvariant
+                const QString *str = static_cast<const QString*>(value.constData());
+                res = sqlite3_bind_text16(_stmt, pos, str->utf16(),
+                                          (str->size()) * sizeof(QChar), SQLITE_TRANSIENT);
+            } else {
+                // unbound value create a null entry.
+                res = SQLITE_OK;
+            }
+            break; }
+        default: {
+            QString str = value.toString();
+            // SQLITE_TRANSIENT makes sure that sqlite buffers the data
+            res = sqlite3_bind_text16(_stmt, pos, str.utf16(),
+                                      (str.size()) * sizeof(QChar), SQLITE_TRANSIENT);
+            break; }
+        }
+    }
+    Q_ASSERT( res == SQLITE_OK );
+}
+
+QString SqlQuery::stringValue(int index)
+{
+    return QString::fromUtf16(static_cast<const ushort*>(sqlite3_column_text16(_stmt, index)));
+}
+
+int SqlQuery::intValue(int index)
+{
+    return sqlite3_column_int(_stmt, index);
+}
+
+quint64 SqlQuery::int64Value(int index)
+{
+    return sqlite3_column_int64(_stmt, index);
+}
+
+QByteArray SqlQuery::baValue(int index)
+{
+    return QByteArray( static_cast<const char*>(sqlite3_column_blob(_stmt, index)),
+                       sqlite3_column_bytes(_stmt, index));
+}
+
+QString SqlQuery::error() const
+{
+    return _error;
+}
+
+QString SqlQuery::lastQuery() const
+{
+    return _sql;
+}
+
+int SqlQuery::numRowsAffected()
+{
+    return 1;
+}
+
+void SqlQuery::finish()
+{
+    SQLITE_DO(sqlite3_finalize(_stmt));
+    _stmt = 0;
+}
+
+void SqlQuery::reset()
+{
+    SQLITE_DO(sqlite3_reset(_stmt));
+}
+
+} // namespace Mirall

src/mirall/ownsql.h

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) by Klaas Freitag <freitag@owncloud.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#ifndef OWNSQL_H
+#define OWNSQL_H
+
+#include <sqlite3.h>
+
+#include <QObject>
+#include <QVariant>
+
+#include "owncloudlib.h"
+
+namespace Mirall {
+
+class OWNCLOUDSYNC_EXPORT SqlDatabase
+{
+public:
+    explicit SqlDatabase();
+
+    bool isOpen();
+    bool open( const QString& filename );
+    bool transaction();
+    bool commit();
+    void close();
+    QString error() const;
+    sqlite3* sqliteDb();
+
+private:
+    sqlite3 *_db;
+    QString _error; // last error string
+    int _errId;
+
+};
+
+class OWNCLOUDSYNC_EXPORT SqlQuery
+{
+    Q_DISABLE_COPY(SqlQuery)
+public:
+    explicit SqlQuery();
+    explicit SqlQuery(SqlDatabase db);
+    explicit SqlQuery(const QString& sql, SqlDatabase db);
+
+    ~SqlQuery();
+    QString error() const;
+
+    QString stringValue(int index);
+    int intValue(int index);
+    quint64 int64Value(int index);
+    QByteArray baValue(int index);
+
+    bool isSelect();
+    bool isPragma();
+    bool exec();
+    int  prepare( const QString& sql );
+    bool next();
+    void bindValue(int pos, const QVariant& value);
+    QString lastQuery() const;
+    int numRowsAffected();
+    void reset();
+    void finish();
+
+private:
+    sqlite3 *_db;
+    sqlite3_stmt *_stmt;
+    QString _error;
+    int _errId;
+    QString _sql;
+};
+
+} // namespace Mirall
+
+#endif // OWNSQL_H

src/mirall/propagator_qnam.cpp

@@ -310,6 +310,7 @@ void PropagateUploadFileQNAM::slotPutFinished()
             // Precondition Failed:   Maybe the bad etag is in the database, we need to clear the
             // parent folder etag so we won't read from DB next sync.
             _propagator->_journal->avoidReadFromDbOnNextSync(_item._file);
+            _propagator->_anotherSyncNeeded = true;
         }
 
         done(classifyError(err, _item._httpErrorCode), errorString);
@@ -448,6 +449,12 @@ GETFileJob::GETFileJob(Account* account, const QUrl& url, QFile *device,
 
 
 void GETFileJob::start() {
+    if (_resumeStart > 0) {
+        _headers["Range"] = "bytes=" + QByteArray::number(_resumeStart) +'-';
+        _headers["Accept-Ranges"] = "bytes";
+        qDebug() << "Retry with range " << _headers["Range"];
+    }
+
     QNetworkRequest req;
     for(QMap<QByteArray, QByteArray>::const_iterator it = _headers.begin(); it != _headers.end(); ++it) {
         req.setRawHeader(it.key(), it.value());
@@ -476,9 +483,15 @@ void GETFileJob::start() {
 
 void GETFileJob::slotMetaDataChanged()
 {
-    if (reply()->error() != QNetworkReply::NoError
+    int httpStatus = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
-            || reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() / 100 != 2) {
+
-        // We will handle the error when the job is finished.
+    // If the status code isn't 2xx, don't write the reply body to the file.
+    // For any error: handle it when the job is finished, not here.
+    if (httpStatus / 100 != 2) {
+        _device->close();
+        return;
+    }
+    if (reply()->error() != QNetworkReply::NoError) {
         return;
     }
     _etag = get_etag_from_reply(reply());
@@ -548,13 +561,15 @@ void GETFileJob::slotReadyRead()
             return;
         }
 
-        qint64 w = _device->write(buffer.constData(), r);
+        if (_device->isOpen()) {
-        if (w != r) {
+            qint64 w = _device->write(buffer.constData(), r);
-            _errorString = _device->errorString();
+            if (w != r) {
-            _errorStatus = SyncFileItem::NormalError;
+                _errorString = _device->errorString();
-            qDebug() << "Error while writing to file" << w << r <<  _errorString;
+                _errorStatus = SyncFileItem::NormalError;
-            reply()->abort();
+                qDebug() << "Error while writing to file" << w << r <<  _errorString;
-            return;
+                reply()->abort();
+                return;
+            }
         }
     }
 }
@@ -626,18 +641,13 @@ void PropagateDownloadFileQNAM::start()
 
     QMap<QByteArray, QByteArray> headers;
 
-    quint64 startSize = 0;
+    quint64 startSize = _tmpFile.size();
-    if (_tmpFile.size() > 0) {
+    if (startSize > 0) {
-        quint64 done = _tmpFile.size();
+        if (startSize == _item._size) {
-        if (done == _item._size) {
             qDebug() << "File is already complete, no need to download";
             downloadFinished();
             return;
         }
-        headers["Range"] = "bytes=" + QByteArray::number(done) +'-';
-        headers["Accept-Ranges"] = "bytes";
-        qDebug() << "Retry with range " << headers["Range"];
-        startSize = done;
     }
 
     if (_item._directDownloadUrl.isEmpty()) {
@@ -647,14 +657,27 @@ void PropagateDownloadFileQNAM::start()
                             &_tmpFile, headers, expectedEtagForResume, startSize);
     } else {
         // We were provided a direct URL, use that one
+        qDebug() << Q_FUNC_INFO << "directDownloadUrl given for " << _item._file << _item._directDownloadUrl;
+
+        // Direct URLs don't support resuming, so clear an existing tmp file
+        if (startSize > 0) {
+            qDebug() << Q_FUNC_INFO << "resuming not supported for directDownloadUrl, deleting temporary";
+            _tmpFile.close();
+            if (!_tmpFile.open(QIODevice::WriteOnly | QIODevice::Unbuffered)) {
+                done(SyncFileItem::NormalError, _tmpFile.errorString());
+                return;
+            }
+            startSize = 0;
+        }
+
         if (!_item._directDownloadCookies.isEmpty()) {
             headers["Cookie"] = _item._directDownloadCookies.toUtf8();
         }
+
         QUrl url = QUrl::fromUserInput(_item._directDownloadUrl);
         _job = new GETFileJob(AccountManager::instance()->account(),
                               url,
                               &_tmpFile, headers);
-        qDebug() << Q_FUNC_INFO << "directDownloadUrl given for " << _item._file << _item._directDownloadUrl;
     }
     _job->setTimeout(_propagator->httpTimeout() * 1000);
     connect(_job, SIGNAL(finishedSignal()), this, SLOT(slotGetFinished()));
@@ -677,18 +700,34 @@ void PropagateDownloadFileQNAM::slotGetFinished()
 
     QNetworkReply::NetworkError err = job->reply()->error();
     if (err != QNetworkReply::NoError) {
-        if (_tmpFile.size() == 0) {
+        _item._httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
-            // don't keep the temporary file if it is empty.
+
+        // If we sent a 'Range' header and get 416 back, we want to retry
+        // without the header.
+        bool badRangeHeader = job->resumeStart() > 0 && _item._httpErrorCode == 416;
+        if (badRangeHeader) {
+            qDebug() << Q_FUNC_INFO << "server replied 416 to our range request, trying again without";
+            _propagator->_anotherSyncNeeded = true;
+        }
+
+        // Don't keep the temporary file if it is empty or we
+        // used a bad range header.
+        if (_tmpFile.size() == 0 || badRangeHeader) {
             _tmpFile.close();
             _tmpFile.remove();
             _propagator->_journal->setDownloadInfo(_item._file, SyncJournalDb::DownloadInfo());
         }
-        _item._httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
+
         _propagator->_activeJobs--;
         SyncFileItem::Status status = job->errorStatus();
         if (status == SyncFileItem::NoStatus) {
             status = classifyError(err, _item._httpErrorCode);
         }
+        if (badRangeHeader) {
+            // Can't do this in classifyError() because 416 without a
+            // Range header should result in NormalError.
+            status = SyncFileItem::SoftError;
+        }
         done(status, job->errorString());
         return;
     }

src/mirall/propagatorjobs.cpp

@@ -247,6 +247,17 @@ void PropagateLocalRename::start()
         qDebug() << "MOVE " << _propagator->_localDir + _item._file << " => " << _propagator->_localDir + _item._renameTarget;
         QFile file(_propagator->_localDir + _item._file);
 
+        if (QString::compare(_item._file, _item._renameTarget, Qt::CaseInsensitive) != 0
+                && _propagator->localFileNameClash(_item._renameTarget)) {
+            // Only use localFileNameClash for the destination if we know that the source was not
+            // the one conflicting  (renaming  A.txt -> a.txt is OK)
+
+            // Fixme: the file that is the reason for the clash could be named here,
+            // it would have to come out the localFileNameClash function
+            done(SyncFileItem::NormalError, tr( "File %1 can not be renamed to %2 because of a local file name clash")
+                 .arg(QDir::toNativeSeparators(_item._file)).arg(QDir::toNativeSeparators(_item._renameTarget)) );
+            return;
+        }
         if (!file.rename(_propagator->_localDir + _item._file, _propagator->_localDir + _item._renameTarget)) {
             done(SyncFileItem::NormalError, file.errorString());
             return;

src/mirall/protocolwidget.cpp

@@ -29,6 +29,8 @@
 
 #include "ui_protocolwidget.h"
 
+#include <climits>
+
 namespace Mirall {
 
 ProtocolWidget::ProtocolWidget(QWidget *parent) :
@@ -262,7 +264,7 @@ void ProtocolWidget::computeResyncButtonEnabled()
 
 void ProtocolWidget::slotProgressInfo( const QString& folder, const Progress::Info& progress )
 {
-    if( progress._completedFileCount == std::numeric_limits<quint64>::max() ) {
+    if( progress._completedFileCount == ULLONG_MAX ) {
         // The sync is restarting, clean the old items
         cleanIgnoreItems(folder);
         computeResyncButtonEnabled();

src/mirall/selectivesyncdialog.cpp

@@ -31,7 +31,7 @@
 namespace Mirall {
 
 SelectiveSyncTreeView::SelectiveSyncTreeView(Account *account, QWidget* parent)
-    : QTreeWidget(parent), _account(account)
+    : QTreeWidget(parent), _inserting(false), _account(account)
 {
     connect(this, SIGNAL(itemExpanded(QTreeWidgetItem*)), this, SLOT(slotItemExpanded(QTreeWidgetItem*)));
     connect(this, SIGNAL(itemChanged(QTreeWidgetItem*,int)), this, SLOT(slotItemChanged(QTreeWidgetItem*,int)));
@@ -262,7 +262,7 @@ void SelectiveSyncDialog::init(Account *account)
 {
     QVBoxLayout *layout = new QVBoxLayout(this);
     _treeView = new SelectiveSyncTreeView(account, this);
-    layout->addWidget(new QLabel(tr("Only checked folders will sync to this computer")));
+    layout->addWidget(new QLabel(tr("Unchecked folders will not be sync to this computer")));
     layout->addWidget(_treeView);
     QDialogButtonBox *buttonBox = new QDialogButtonBox(Qt::Horizontal);
     QPushButton *button;
@@ -276,6 +276,7 @@ void SelectiveSyncDialog::init(Account *account)
 void SelectiveSyncDialog::accept()
 {
     if (_folder) {
+        auto oldBlackListSet = _folder->selectiveSyncBlackList().toSet();
         QStringList blackList = _treeView->createBlackList();
         _folder->setSelectiveSyncBlackList(blackList);
 
@@ -283,11 +284,19 @@ void SelectiveSyncDialog::accept()
         QSettings settings(_folder->configFile(), QSettings::IniFormat);
         settings.beginGroup(FolderMan::escapeAlias(_folder->alias()));
         settings.setValue("blackList", blackList);
-
         FolderMan *folderMan = FolderMan::instance();
         if (_folder->isBusy()) {
             _folder->slotTerminateSync();
         }
+
+        //The part that changed should not be read from the DB on next sync because there might be new folders
+        // (the ones that are no longer in the blacklist)
+        auto blackListSet = blackList.toSet();
+        auto changes = (oldBlackListSet - blackListSet) + (blackListSet - oldBlackListSet);
+        foreach(const auto &it, changes) {
+            _folder->journalDb()->avoidReadFromDbOnNextSync(it);
+        }
+
         folderMan->slotScheduleSync(_folder->alias());
     }
     QDialog::accept();

src/mirall/selectivesyncdialog.h

@@ -46,7 +46,7 @@ private:
     QString _folderPath;
     QString _rootName;
     QStringList _oldBlackList;
-    bool _inserting = false; // set to true when we are inserting new items on the list
+    bool _inserting; // set to true when we are inserting new items on the list
     Account *_account;
 };
 

src/mirall/socketapi.cpp

@@ -35,6 +35,9 @@
 #include <QApplication>
 #include <QLocalSocket>
 
+#include <sqlite3.h>
+
+
 #if QT_VERSION >= QT_VERSION_CHECK(5, 0, 0)
 #include <QStandardPaths>
 #endif
@@ -60,179 +63,14 @@ CSYNC_EXCLUDE_TYPE csync_excluded_no_ctx(c_strlist_t *excludes, const char *path
 int csync_exclude_load(const char *fname, c_strlist_t **list);
 }
 
-namespace {
-    const int PORT = 34001;
-}
-
 namespace Mirall {
 
 #define DEBUG qDebug() << "SocketApi: "
 
-namespace SocketApiHelper {
-
-SyncFileStatus fileStatus(Folder *folder, const QString& systemFileName, c_strlist_t *excludes );
-
-/**
- * @brief recursiveFolderStatus
- * @param fileName - the relative file name to examine
- * @return the resulting status
- *
- * The resulting status can only be either SYNC which means all files
- * are in sync, ERROR if an error occured, or EVAL if something needs
- * to be synced underneath this dir.
- */
-// compute the file status of a directory recursively. It returns either
-// "all in sync" or "needs update" or "error", no more details.
-SyncFileStatus recursiveFolderStatus(Folder *folder, const QString& fileName, c_strlist_t *excludes  )
-{
-    QDir dir(folder->path() + fileName);
-
-    const QStringList dirEntries = dir.entryList( QDir::AllEntries | QDir::NoDotAndDotDot );
-
-    SyncFileStatus result(SyncFileStatus::STATUS_SYNC);
-
-    foreach( const QString entry, dirEntries ) {
-        QString normalizedFile = QString(fileName + QLatin1Char('/') + entry).normalized(QString::NormalizationForm_C);
-        QFileInfo fi(entry);
-        SyncFileStatus sfs;
-
-        if( fi.isDir() ) {
-            sfs = recursiveFolderStatus(folder, normalizedFile, excludes );
-        } else {
-            QString fs( normalizedFile );
-            if( fileName.isEmpty() ) {
-                // toplevel, no slash etc. needed.
-                fs = entry.normalized(QString::NormalizationForm_C);
-            }
-            sfs = fileStatus(folder, fs, excludes);
-        }
-
-        if( sfs.tag() == SyncFileStatus::STATUS_STAT_ERROR || sfs.tag() == SyncFileStatus::STATUS_ERROR ) {
-            return SyncFileStatus::STATUS_ERROR;
-        } else if( sfs.tag() == SyncFileStatus::STATUS_EVAL || sfs.tag() == SyncFileStatus::STATUS_NEW) {
-            result.set(SyncFileStatus::STATUS_EVAL);
-        }
-    }
-    return result;
-}
-
-SyncJournalFileRecord dbFileRecord( Folder *folder, QString fileName )
-{
-    QFileInfo fi(fileName);
-    if( !folder ) {
-        return SyncJournalFileRecord();
-    }
-    if( fi.isAbsolute() ) {
-        fileName.remove(0, folder->path().length());
-    }
-    return( folder->journalDb()->getFileRecord(fileName) );
-}
-
-/**
- * Get status about a single file.
- */
-SyncFileStatus fileStatus(Folder *folder, const QString& systemFileName, c_strlist_t *excludes )
-{
-    QString file = folder->path();
-    QString fileName = systemFileName.normalized(QString::NormalizationForm_C);
-
-    bool isSyncRootFolder = true;
-    if( fileName != QLatin1String("/") && !fileName.isEmpty() ) {
-        file = folder->path() + fileName;
-        isSyncRootFolder = false;
-    }
-
-    if( fileName.endsWith(QLatin1Char('/')) ) {
-        fileName.truncate(fileName.length()-1);
-        qDebug() << "Removed trailing slash: " << fileName;
-    }
-
-    QFileInfo fi(file);
-
-    if( !fi.exists() ) {
-        qDebug() << "OO File " << file << " is not existing";
-        return SyncFileStatus(SyncFileStatus::STATUS_STAT_ERROR);
-    }
-
-    // file is ignored?
-    if( fi.isSymLink() ) {
-        return SyncFileStatus(SyncFileStatus::STATUS_IGNORE);
-    }
-    int type = CSYNC_FTW_TYPE_FILE;
-    if( fi.isDir() ) {
-        type = CSYNC_FTW_TYPE_DIR;
-    }
-
-    // '\' is ignored, so convert to unix path before passing the path in.
-    QString unixFileName = QDir::fromNativeSeparators(fileName);
-
-    CSYNC_EXCLUDE_TYPE excl = csync_excluded_no_ctx(excludes, unixFileName.toUtf8(), type);
-    if( excl != CSYNC_NOT_EXCLUDED ) {
-        return SyncFileStatus(SyncFileStatus::STATUS_IGNORE);
-    }
-
-    // Problem: for the sync dir itself we do not have a record in the sync journal
-    // so the next check must not be used for the sync root folder.
-    SyncJournalFileRecord rec = dbFileRecord(folder, unixFileName );
-    if( !isSyncRootFolder && !rec.isValid() ) {
-        // check the parent folder if it is shared and if it is allowed to create a file/dir within
-        QDir d( fi.path() );
-        QString parentPath = d.path();
-        SyncJournalFileRecord dirRec = dbFileRecord(folder, parentPath);
-        while( !d.isRoot() && !(d.exists() && dirRec.isValid()) ) {
-            d.cdUp(); // returns true if the dir exists.
-
-            parentPath = d.path();
-            // cut the folder path
-            dirRec = dbFileRecord(folder, parentPath);
-        }
-        if( dirRec.isValid() ) {
-            if( dirRec._type == CSYNC_FTW_TYPE_DIR ) {
-                if( !dirRec._remotePerm.contains("K") ) {
-                    return SyncFileStatus::STATUS_ERROR;
-                }
-            } else {
-                if( !dirRec._remotePerm.contains("C") ) {
-                    return SyncFileStatus::STATUS_ERROR;
-                }
-            }
-        }
-        return SyncFileStatus(SyncFileStatus::STATUS_NEW);
-    }
-
-    SyncFileStatus status(SyncFileStatus::STATUS_NONE);
-    if( type == CSYNC_FTW_TYPE_DIR ) {
-        // compute recursive status of the directory
-        status = recursiveFolderStatus( folder, fileName, excludes );
-    } else if( FileSystem::getModTime(fi.absoluteFilePath()) != Utility::qDateTimeToTime_t(rec._modtime) ) {
-        // file was locally modified.
-        status.set(SyncFileStatus::STATUS_EVAL);
-    } else {
-        status.set(SyncFileStatus::STATUS_SYNC);
-    }
-
-    if (rec._remotePerm.contains("S")) {
-       status.setSharedWithMe(true);
-    }
-
-    return status;
-}
-
-
-}
-
 SocketApi::SocketApi(QObject* parent)
     : QObject(parent)
     , _excludes(0)
 {
-
-#ifdef SOCKETAPI_TCP
-    // setup socket
-    DEBUG << "Establishing SocketAPI server at" << PORT;
-    if (!_localServer.listen(QHostAddress::LocalHost, PORT)) {
-        DEBUG << "Failed to bind to port" << PORT;
-    }
-#else
     QString socketPath;
 
     if (Utility::isWindows()) {
@@ -274,7 +112,6 @@ SocketApi::SocketApi(QObject* parent)
         DEBUG << "server started, listening at " << socketPath;
     }
 
-#endif
     connect(&_localServer, SIGNAL(newConnection()), this, SLOT(slotNewConnection()));
 
     // folder watcher
@@ -387,6 +224,21 @@ void SocketApi::slotUnregisterPath( const QString& alias )
     Folder *f = FolderMan::instance()->folder(alias);
     if (f) {
         broadcastMessage(QLatin1String("UNREGISTER_PATH"), f->path(), QString::null, true );
+
+        if( _dbQueries.contains(f)) {
+            SqlQuery *h = _dbQueries[f];
+            if( h ) {
+                h->finish();
+            }
+            _dbQueries.remove(f);
+        }
+        if( _openDbs.contains(f) ) {
+            SqlDatabase *db = _openDbs[f];
+            if( db ) {
+                db->close();
+            }
+            _openDbs.remove(f);
+        }
     }
 }
 
@@ -406,6 +258,8 @@ void SocketApi::slotUpdateFolderView(const QString& alias)
             } else {
                 broadcastMessage(QLatin1String("UPDATE_VIEW"), f->path() );
             }
+        } else {
+            qDebug() << "Not sending UPDATE_VIEW for" << alias << "because status() is" << f->syncResult().status();
         }
     }
 }
@@ -452,11 +306,13 @@ void SocketApi::sendMessage(SocketType *socket, const QString& message, bool doW
     if( ! localMessage.endsWith(QLatin1Char('\n'))) {
         localMessage.append(QLatin1Char('\n'));
     }
-    qint64 sent = socket->write(localMessage.toUtf8());
+
+    QByteArray bytesToSend = localMessage.toUtf8();
+    qint64 sent = socket->write(bytesToSend);
     if( doWait ) {
         socket->waitForBytesWritten(1000);
     }
-    if( sent != localMessage.toUtf8().length() ) {
+    if( sent != bytesToSend.length() ) {
         qDebug() << "WARN: Could not send all data on socket for " << localMessage;
     }
 
@@ -475,7 +331,8 @@ void SocketApi::broadcastMessage( const QString& verb, const QString& path, cons
         msg.append(QDir::toNativeSeparators(path));
     }
 
-    DEBUG << "Broadcasting to" << _listeners.count() << "listeners: " << msg;
+    // sendMessage already has a debug output
+    //DEBUG << "Broadcasting to" << _listeners.count() << "listeners: " << msg;
     foreach(SocketType *socket, _listeners) {
         sendMessage(socket, msg, doWait);
     }
@@ -507,7 +364,7 @@ void SocketApi::command_RETRIEVE_FILE_STATUS(const QString& argument, SocketType
         statusString = QLatin1String("NOP");
     } else {
         const QString file = argument.mid(syncFolder->path().length());
-        SyncFileStatus fileStatus = SocketApiHelper::fileStatus(syncFolder, file, _excludes);
+        SyncFileStatus fileStatus = this->fileStatus(syncFolder, file, _excludes);
 
         statusString = fileStatus.toSocketAPIString();
     }
@@ -522,6 +379,181 @@ void SocketApi::command_VERSION(const QString&, SocketType* socket)
     sendMessage(socket, QLatin1String("VERSION:" MIRALL_VERSION_STRING ":" MIRALL_SOCKET_API_VERSION));
 }
 
+SqlQuery* SocketApi::getSqlQuery( Folder *folder )
+{
+    if( !folder ) {
+        return 0;
+    }
+
+    if( _dbQueries.contains(folder) ) {
+        return _dbQueries[folder];
+    }
+
+    /* No valid sql query object yet for this folder */
+    int rc;
+    const QString sql("SELECT inode, mode, modtime, type, md5, fileid, remotePerm FROM "
+                      "metadata WHERE phash=?1");
+    QString dbFileName = folder->journalDb()->databaseFilePath();
+
+    QFileInfo fi(dbFileName);
+    if( fi.exists() ) {
+        SqlDatabase *db = new SqlDatabase;
+
+        if( db && db->open(dbFileName) ) {
+            _openDbs.insert(folder, db);
+
+            SqlQuery *query = new SqlQuery(*db);
+            rc = query->prepare(sql);
+
+            if( rc != SQLITE_OK ) {
+                delete query;
+                qDebug() << "Unable to prepare the query statement:" << rc;
+                return 0; // do not insert into hash
+            }
+            _dbQueries.insert( folder, query);
+            return query;
+        }
+    } else {
+        qDebug() << Q_FUNC_INFO << "Journal to query does not yet exist.";
+    }
+    return 0;
+}
+
+SyncJournalFileRecord SocketApi::dbFileRecord_capi( Folder *folder, QString fileName )
+{
+    if( !(folder && folder->journalDb()) ) {
+        return SyncJournalFileRecord();
+    }
+
+    if( fileName.startsWith( folder->path() )) {
+        fileName.remove(0, folder->path().length());
+    }
+
+    SqlQuery *query = getSqlQuery(folder);
+    SyncJournalFileRecord rec;
+
+    if( query ) {
+        qlonglong phash = SyncJournalDb::getPHash( fileName );
+        query->bindValue(1, phash);
+        // int column_count = sqlite3_column_count(stmt);
+
+        if (query->next()) {
+            rec._path    = fileName;
+            rec._inode   = query->int64Value(0);
+            rec._mode    = query->intValue(1);
+            rec._modtime = Utility::qDateTimeFromTime_t( query->int64Value(2));
+            rec._type    = query->intValue(3);
+            rec._etag    = query->baValue(4);
+            rec._fileId  = query->baValue(5);
+            rec._remotePerm = query->baValue(6);
+        }
+        query->reset();
+    }
+    return rec;
+}
+
+/**
+ * Get status about a single file.
+ */
+SyncFileStatus SocketApi::fileStatus(Folder *folder, const QString& systemFileName, c_strlist_t *excludes )
+{
+    QString file = folder->path();
+    QString fileName = systemFileName.normalized(QString::NormalizationForm_C);
+
+    if( fileName != QLatin1String("/") && !fileName.isEmpty() ) {
+        file = folder->path() + fileName;
+    }
+
+    if( fileName.endsWith(QLatin1Char('/')) ) {
+        fileName.truncate(fileName.length()-1);
+        qDebug() << "Removed trailing slash: " << fileName;
+    }
+
+    QFileInfo fi(file);
+
+    if( !fi.exists() ) {
+        qDebug() << "OO File " << file << " is not existing";
+        return SyncFileStatus(SyncFileStatus::STATUS_STAT_ERROR);
+    }
+
+    // file is ignored?
+    if( fi.isSymLink() ) {
+        return SyncFileStatus(SyncFileStatus::STATUS_IGNORE);
+    }
+
+    csync_ftw_type_e type = CSYNC_FTW_TYPE_FILE;
+    if( fi.isDir() ) {
+        type = CSYNC_FTW_TYPE_DIR;
+    }
+
+    // '\' is ignored, so convert to unix path before passing the path in.
+    QString unixFileName = QDir::fromNativeSeparators(fileName);
+
+    CSYNC_EXCLUDE_TYPE excl = csync_excluded_no_ctx(excludes, unixFileName.toUtf8(), type);
+    if( excl != CSYNC_NOT_EXCLUDED ) {
+        return SyncFileStatus(SyncFileStatus::STATUS_IGNORE);
+    }
+
+
+    SyncFileStatus status(SyncFileStatus::STATUS_NONE);
+    if (type == CSYNC_FTW_TYPE_DIR) {
+        if (folder->estimateState(fileName, type, &status)) {
+            qDebug() << Q_FUNC_INFO << "Folder estimated status for" << fileName << "to" << status.toSocketAPIString();
+            return status;
+        }
+        if (fileName == "") {
+            // sync folder itself
+            if (folder->syncResult().status() == SyncResult::Undefined
+                    || folder->syncResult().status() == SyncResult::NotYetStarted
+                    || folder->syncResult().status() == SyncResult::SyncPrepare
+                    || folder->syncResult().status() == SyncResult::SyncRunning
+                    || folder->syncResult().status() == SyncResult::Paused) {
+                status.set(SyncFileStatus::STATUS_EVAL);
+                return status;
+            } else if (folder->syncResult().status() == SyncResult::Success
+                       || folder->syncResult().status() == SyncResult::Problem) {
+                status.set(SyncFileStatus::STATUS_SYNC);
+                return status;
+            }  else if (folder->syncResult().status() == SyncResult::Error
+                        || folder->syncResult().status() == SyncResult::SetupError
+                        || folder->syncResult().status() == SyncResult::SyncAbortRequested) {
+                status.set(SyncFileStatus::STATUS_ERROR);
+                return status;
+            }
+        }
+        SyncJournalFileRecord rec = dbFileRecord_capi(folder, unixFileName );
+        if (rec.isValid()) {
+            status.set(SyncFileStatus::STATUS_SYNC);
+            if (rec._remotePerm.contains("S")) {
+               status.setSharedWithMe(true);
+            }
+        } else {
+            status.set(SyncFileStatus::STATUS_EVAL);
+        }
+    } else if (type == CSYNC_FTW_TYPE_FILE) {
+        if (folder->estimateState(fileName, type, &status)) {
+            return status;
+        }
+        SyncJournalFileRecord rec = dbFileRecord_capi(folder, unixFileName );
+        if (rec.isValid()) {
+            if (rec._remotePerm.contains("S")) {
+               status.setSharedWithMe(true);
+            }
+            if( FileSystem::getModTime(fi.absoluteFilePath()) == Utility::qDateTimeToTime_t(rec._modtime) ) {
+                status.set(SyncFileStatus::STATUS_SYNC);
+                return status;
+            } else {
+                status.set(SyncFileStatus::STATUS_EVAL);
+                return status;
+            }
+        }
+        status.set(SyncFileStatus::STATUS_NEW);
+        return status;
+    }
+
+    return status;
+}
+
 
 } // namespace Mirall
 

src/mirall/socketapi.h

@@ -20,12 +20,16 @@ extern "C" {
 #include <std/c_string.h>
 }
 
+#include <sqlite3.h>
+
 #include <QWeakPointer>
 #include <QTcpSocket>
 #include <QTcpServer>
 #include <QLocalServer>
 
 #include "mirall/syncfileitem.h"
+#include "mirall/syncjournalfilerecord.h"
+#include "mirall/ownsql.h"
 
 class QUrl;
 class QLocalSocket;
@@ -33,19 +37,10 @@ class QStringList;
 
 namespace Mirall {
 
-//Define this to use the old school TCP API. Maybe we should offer both APIs
-// and have the old TCP one be enableable via command line switch?
-//#define SOCKETAPI_TCP
-#if defined(Q_OS_WIN)
-// Windows plugin has not been ported
-#define SOCKETAPI_TCP
-#endif
-
-#ifdef SOCKETAPI_TCP
-typedef QTcpSocket SocketType;
-#else
 typedef QLocalSocket SocketType;
-#endif
+
+class SyncFileStatus;
+class Folder;
 
 class SocketApi : public QObject
 {
@@ -69,6 +64,12 @@ private slots:
     void slotSyncItemDiscovered(const QString &, const SyncFileItem &);
 
 private:
+    SyncFileStatus fileStatus(Folder *folder, const QString& systemFileName, c_strlist_t *excludes );
+    SyncJournalFileRecord dbFileRecord( Folder *folder, QString fileName );
+    SyncJournalFileRecord dbFileRecord_capi( Folder *folder, QString fileName );
+    SyncFileStatus recursiveFolderStatus(Folder *folder, const QString& fileName, c_strlist_t *excludes  );
+    SqlQuery *getSqlQuery( Folder *folder );
+
     void sendMessage(SocketType* socket, const QString& message, bool doWait = false);
     void broadcastMessage(const QString& verb, const QString &path, const QString &status = QString::null, bool doWait = false);
 
@@ -77,7 +78,6 @@ private:
 
     Q_INVOKABLE void command_VERSION(const QString& argument, SocketType* socket);
 
-private:
 #ifdef SOCKETAPI_TCP
     QTcpServer _localServer;
 #else
@@ -85,6 +85,8 @@ private:
 #endif
     QList<SocketType*> _listeners;
     c_strlist_t *_excludes;
+    QHash<Folder*, SqlQuery*> _dbQueries;
+    QHash<Folder*, SqlDatabase*> _openDbs;
 };
 
 }

src/mirall/sslerrordialog.cpp

@@ -149,11 +149,10 @@ bool SslErrorDialog::checkFailingCertsKnown( const QList<QSslError> &errors )
     }
     msg += QL("</div></body></html>");
 
-    qDebug() << "#  # # # # # ";
+    //qDebug() << "#  # # # # # ";
-    qDebug() << msg;
+    //qDebug() << msg;
     QTextDocument *doc = new QTextDocument(0);
     QString style = styleSheet();
-    qDebug() << "Style: " << style;
     doc->addResource( QTextDocument::StyleSheetResource, QUrl( QL("format.css") ), style);
     doc->setHtml( msg );
 

src/mirall/syncengine.cpp

@@ -22,6 +22,7 @@
 #include "discoveryphase.h"
 #include "creds/abstractcredentials.h"
 #include "csync_util.h"
+#include "mirall/syncfilestatus.h"
 
 #ifdef Q_OS_WIN
 #include <windows.h>
@@ -29,6 +30,7 @@
 #include <unistd.h>
 #endif
 
+#include <climits>
 #include <assert.h>
 
 #include <QDebug>
@@ -196,12 +198,12 @@ bool SyncEngine::checkBlacklisting( SyncFileItem *item )
     }
 
     SyncJournalBlacklistRecord entry = _journal->blacklistEntry(item->_file);
-    item->_blacklistedInDb = false;
+    item->_hasBlacklistEntry = false;
 
     // if there is a valid entry in the blacklist table and the retry count is
     // already null or smaller than 0, the file is blacklisted.
     if( entry.isValid() ) {
-        item->_blacklistedInDb = true;
+        item->_hasBlacklistEntry = true;
 
         if( entry._retryCount <= 0 ) {
             re = true;
@@ -288,7 +290,7 @@ void SyncEngine::deleteStaleBlacklistEntries()
     // Find all blacklisted paths that we want to preserve.
     QSet<QString> blacklist_file_paths;
     foreach(const SyncFileItem& it, _syncedItems) {
-        if (it._status == SyncFileItem::FileIgnored)
+        if (it._hasBlacklistEntry)
             blacklist_file_paths.insert(it._file);
     }
 
@@ -401,7 +403,7 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote )
     int re = 0;
     switch(file->instruction) {
     case CSYNC_INSTRUCTION_NONE:
-        if (file->should_update_etag && !item._isDirectory) {
+        if (remote && item._should_update_etag && !item._isDirectory && item._instruction == CSYNC_INSTRUCTION_NONE) {
             // Update the database now already  (new fileid or etag or remotePerm)
             // Those are files that were detected as "resolved conflict".
             // They should have been a conflict because they both were new, or both
@@ -523,37 +525,39 @@ void SyncEngine::startSync()
 
     csync_resume(_csync_ctx);
 
+    int fileRecordCount = -1;
     if (!_journal->exists()) {
-        qDebug() << "=====sync looks new (no DB exists), activating recursive PROPFIND if csync supports it";
+        qDebug() << "=====sync looks new (no DB exists)";
+    } else {
+        qDebug() << "=====sync with existing DB";
+    }
+
+    fileRecordCount = _journal->getFileRecordCount(); // this creates the DB if it does not exist yet
+    bool isUpdateFrom_1_5 = _journal->isUpdateFrom_1_5();
+
+    if( fileRecordCount == -1 ) {
+        qDebug() << "No way to create a sync journal!";
+        emit csyncError(tr("Unable to initialize a sync journal."));
+        finalize();
+        return;
+        // database creation error!
+    }
+
+    if (fileRecordCount >= 1 && isUpdateFrom_1_5) {
+        qDebug() << "detected update from 1.5";
+        // Disable the read from DB to be sure to re-read all the fileid and etags.
+        csync_set_read_from_db(_csync_ctx, false);
+    }
+
+    bool usingSelectiveSync = (!_selectiveSyncBlackList.isEmpty());
+    qDebug() << (usingSelectiveSync ? "====Using Selective Sync" : "====NOT Using Selective Sync");
+    if (fileRecordCount >= 0 && fileRecordCount < 50 && !usingSelectiveSync) {
+        qDebug() << "===== Activating recursive PROPFIND (currently" << fileRecordCount << "file records)";
         bool no_recursive_propfind = false;
         csync_set_module_property(_csync_ctx, "no_recursive_propfind", &no_recursive_propfind);
     } else {
-        // retrieve the file count from the db and close it afterwards because
+        bool no_recursive_propfind = true;
-        // csync_update also opens the database.
+        csync_set_module_property(_csync_ctx, "no_recursive_propfind", &no_recursive_propfind);
-        int fileRecordCount = 0;
-        fileRecordCount = _journal->getFileRecordCount();
-        bool isUpdateFrom_1_5 = _journal->isUpdateFrom_1_5();
-        _journal->close();
-
-        if( fileRecordCount == -1 ) {
-            qDebug() << "No way to create a sync journal!";
-            emit csyncError(tr("Unable to initialize a sync journal."));
-            finalize();
-            return;
-            // database creation error!
-        } else if ( fileRecordCount < 50 ) {
-            qDebug() << "=====sync DB has only" << fileRecordCount << "items, enable recursive PROPFIND if csync supports it";
-            bool no_recursive_propfind = false;
-            csync_set_module_property(_csync_ctx, "no_recursive_propfind", &no_recursive_propfind);
-        } else {
-            qDebug() << "=====sync with existing DB";
-        }
-
-        if (fileRecordCount > 1 && isUpdateFrom_1_5) {
-            qDebug() << "detected update from 1.5";
-            // Disable the read from DB to be sure to re-read all the fileid and etags.
-            csync_set_read_from_db(_csync_ctx, false);
-        }
     }
 
     csync_set_userdata(_csync_ctx, this);
@@ -609,6 +613,17 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult)
     }
     qDebug() << "<<#### Discovery end #################################################### " << _stopWatch.addLapTime(QLatin1String("Discovery Finished"));
 
+    // Sanity check
+    if (!_journal->isConnected()) {
+        qDebug() << "Bailing out, DB failure";
+        emit csyncError(tr("Cannot open the sync journal"));
+        finalize();
+        return;
+    } else {
+        // Commits a possibly existing (should not though) transaction and starts a new one for the propagate phase
+        _journal->commitIfNeededAndStartNewTransaction("Post discovery");
+    }
+
     if( csync_reconcile(_csync_ctx) < 0 ) {
         handleSyncError(_csync_ctx, "csync_reconcile");
         return;
@@ -646,17 +661,9 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult)
     // make sure everything is allowed
     checkForPermission();
 
-    // Sanity check
-    if (!_journal->isConnected()) {
-        qDebug() << "Bailing out, DB failure";
-        emit csyncError(tr("Cannot open the sync journal"));
-        finalize();
-        return;
-    }
-
     // To announce the beginning of the sync
     emit aboutToPropagate(_syncedItems);
-    _progressInfo._completedFileCount = std::numeric_limits<quint64>::max(); // indicate the start with max
+    _progressInfo._completedFileCount = ULLONG_MAX; // indicate the start with max
     emit transmissionProgress(_progressInfo);
     _progressInfo._completedFileCount = 0;
 
@@ -680,14 +687,16 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult)
     Q_ASSERT(session);
 
     // post update phase script: allow to tweak stuff by a custom script in debug mode.
-#ifndef NDEBUG
     if( !qgetenv("OWNCLOUD_POST_UPDATE_SCRIPT").isEmpty() ) {
+#ifndef NDEBUG
         QString script = qgetenv("OWNCLOUD_POST_UPDATE_SCRIPT");
 
         qDebug() << "OOO => Post Update Script: " << script;
         QProcess::execute(script.toUtf8());
-    }
+#else
+    qDebug() << "**** Attention: POST_UPDATE_SCRIPT installed, but not executed because compiled with NDEBUG";
 #endif
+    }
 
     // do a database commit
     _journal->commit("post treewalk");
@@ -819,6 +828,11 @@ QString SyncEngine::adjustRenamedPath(const QString& original)
     return original;
 }
 
+/**
+ *
+ * Make sure that we are allowed to do what we do by checking the permissions and the selective sync list
+ *
+ */
 void SyncEngine::checkForPermission()
 {
     for (SyncFileItemVector::iterator it = _syncedItems.begin(); it != _syncedItems.end(); ++it) {
@@ -828,6 +842,25 @@ void SyncEngine::checkForPermission()
             continue;
         }
 
+        // Do not propagate anything in the server if it is in the selective sync blacklist
+        const QString path = it->destination() + QLatin1Char('/');
+        if (std::binary_search(_selectiveSyncBlackList.constBegin(), _selectiveSyncBlackList.constEnd(),
+                                path)) {
+            it->_instruction = CSYNC_INSTRUCTION_IGNORE;
+            it->_status = SyncFileItem::FileIgnored;
+            it->_errorString = tr("Ignored because of the \"choose what to sync\" blacklist");
+
+            if (it->_isDirectory) {
+                for (SyncFileItemVector::iterator it_next = it + 1; it_next != _syncedItems.end() && it_next->_file.startsWith(path); ++it_next) {
+                    it = it_next;
+                    it->_instruction = CSYNC_INSTRUCTION_IGNORE;
+                    it->_status = SyncFileItem::FileIgnored;
+                    it->_errorString = tr("Ignored because of the \"choose what to sync\" blacklist");
+                }
+            }
+            continue;
+        }
+
         switch(it->_instruction) {
             case CSYNC_INSTRUCTION_NEW: {
                 int slashPos = it->_file.lastIndexOf('/');
@@ -842,7 +875,6 @@ void SyncEngine::checkForPermission()
                     it->_status = SyncFileItem::NormalError;
                     it->_errorString = tr("Not allowed because you don't have permission to add sub-directories in that directory");
 
-                    const QString path = it->_file + QLatin1Char('/');
                     for (SyncFileItemVector::iterator it_next = it + 1; it_next != _syncedItems.end() && it_next->_file.startsWith(path); ++it_next) {
                         it = it_next;
                         it->_instruction = CSYNC_INSTRUCTION_ERROR;
@@ -895,7 +927,6 @@ void SyncEngine::checkForPermission()
 
                     if (it->_isDirectory) {
                         // restore all sub items
-                        const QString path = it->_file + QLatin1Char('/');
                         for (SyncFileItemVector::iterator it_next = it + 1;
                              it_next != _syncedItems.end() && it_next->_file.startsWith(path); ++it_next) {
                             it = it_next;
@@ -915,8 +946,7 @@ void SyncEngine::checkForPermission()
                             it->_errorString = tr("Not allowed to remove, restoring");
                         }
                     }
-                }
+                } else if(perms.contains("S") && perms.contains("D")) {
-                if(perms.contains("S") && perms.contains("D")) {
                     // this is a top level shared dir which can be removed to unshare it,
                     // regardless if it is a read only share or not.
                     // To avoid that we try to restore files underneath this dir which have
@@ -925,8 +955,6 @@ void SyncEngine::checkForPermission()
                     // underneath, propagator sees that.
                     if( it->_isDirectory ) {
                         SyncFileItemVector::iterator it_prev = it - 1;
-                        const QString path = it->_file + QLatin1Char('/');
-
 
                         // put a more descriptive message if really a top level share dir is removed.
                         if( it_prev != _syncedItems.begin() && !(path.startsWith(it_prev->_file)) ) {
@@ -1009,7 +1037,6 @@ void SyncEngine::checkForPermission()
 
 
                     if (it->_isDirectory) {
-                        const QString path = it->_renameTarget + QLatin1Char('/');
                         for (SyncFileItemVector::iterator it_next = it + 1;
                              it_next != _syncedItems.end() && it_next->destination().startsWith(path); ++it_next) {
                             it = it_next;
@@ -1040,6 +1067,29 @@ QByteArray SyncEngine::getPermissions(const QString& file) const
     return _remotePerms.value(file);
 }
 
+void SyncEngine::setSelectiveSyncBlackList(const QStringList& list)
+{
+    _selectiveSyncBlackList = list;
+    for (int i = 0; i < _selectiveSyncBlackList.count(); ++i) {
+        if (!_selectiveSyncBlackList.at(i).endsWith(QLatin1Char('/'))) {
+            _selectiveSyncBlackList[i].append(QLatin1Char('/'));
+        }
+    }
+}
+
+bool SyncEngine::estimateState(QString fn, csync_ftw_type_e t, SyncFileStatus* s)
+{
+    Q_UNUSED(t);
+    Q_FOREACH(const SyncFileItem &item, _syncedItems) {
+        //qDebug() << Q_FUNC_INFO << fn << item._file << fn.startsWith(item._file) << item._file.startsWith(fn);
+        if (item._file.startsWith(fn)) {
+            qDebug() << Q_FUNC_INFO << "Setting" << fn << " to STATUS_EVAL";
+            s->set(SyncFileStatus::STATUS_EVAL);
+            return true;
+        }
+    }
+    return false;
+}
 
 void SyncEngine::abort()
 {

src/mirall/syncengine.h

@@ -33,6 +33,7 @@
 #include "mirall/syncfileitem.h"
 #include "mirall/progressdispatcher.h"
 #include "mirall/utility.h"
+#include "mirall/syncfilestatus.h"
 
 class QProcess;
 
@@ -61,12 +62,13 @@ public:
 
     Utility::StopWatch &stopWatch() { return _stopWatch; }
 
-    void setSelectiveSyncBlackList(const QStringList &list)
+    void setSelectiveSyncBlackList(const QStringList &list);
-    { _selectiveSyncBlackList = list; }
 
     /* Return true if we detected that another sync is needed to complete the sync */
     bool isAnotherSyncNeeded() { return _anotherSyncNeeded; }
 
+    bool estimateState(QString fn, csync_ftw_type_e t, SyncFileStatus* s);
+
 signals:
     void csyncError( const QString& );
     void csyncUnavailable();
@@ -124,7 +126,11 @@ private:
     void finalize();
 
     static bool _syncRunning; //true when one sync is running somewhere (for debugging)
+
     QMap<QString, SyncFileItem> _syncItemMap;
+
+    // should be called _syncItems (present tense). It's the items from the _syncItemMap but
+    // sorted and re-adjusted based on permissions.
     SyncFileItemVector _syncedItems;
 
     CSYNC *_csync_ctx;

src/mirall/syncfileitem.h

@@ -46,13 +46,13 @@ public:
 
         Success, ///< The file was properly synced
         Conflict, ///< The file was properly synced, but a conflict was created
-        FileIgnored, ///< The file is in the ignored list
+        FileIgnored, ///< The file is in the ignored list (or blacklisted with no retries left)
         Restoration ///< The file was restored because what should have been done was not allowed
     };
 
     SyncFileItem() : _type(UnknownType),  _direction(None), _isDirectory(false),
         _instruction(CSYNC_INSTRUCTION_NONE), _modtime(0),
-        _size(0), _inode(0), _should_update_etag(false), _blacklistedInDb(false),
+        _size(0), _inode(0), _should_update_etag(false), _hasBlacklistEntry(false),
         _status(NoStatus), _httpErrorCode(0), _requestDuration(0), _isRestoration(false) {}
 
     friend bool operator==(const SyncFileItem& item1, const SyncFileItem& item2) {
@@ -72,6 +72,13 @@ public:
         return _file.isEmpty();
     }
 
+    bool hasErrorStatus() const {
+        return _status == SyncFileItem::SoftError
+                || _status == SyncFileItem::NormalError
+                || _status == SyncFileItem::FatalError
+                || !_errorString.isEmpty();
+    }
+
     // Variables usefull for everybody
     QString _file;
     QString _renameTarget;
@@ -91,7 +98,11 @@ public:
     QByteArray           _remotePerm;
     QString              _directDownloadUrl;
     QString              _directDownloadCookies;
-    bool                 _blacklistedInDb;
+
+    /// Whether there's an entry in the blacklist table.
+    /// Note: that entry may have retries left, so this can be true
+    /// without the status being FileIgnored.
+    bool                 _hasBlacklistEntry;
 
     // Variables usefull to report to the user
     Status               _status;

src/mirall/syncjournaldb.cpp

@@ -14,8 +14,7 @@
 #include <QFile>
 #include <QStringList>
 #include <QDebug>
-#include <QSqlError>
+#include "mirall/ownsql.h"
-#include <QSqlQuery>
 
 #include <inttypes.h>
 
@@ -26,8 +25,6 @@
 
 #include "../../csync/src/std/c_jhash.h"
 
-#define QSQLITE "QSQLITE"
-
 namespace Mirall {
 
 SyncJournalDb::SyncJournalDb(const QString& path, QObject *parent) :
@@ -49,15 +46,20 @@ bool SyncJournalDb::exists()
     return (!_dbFile.isEmpty() && QFile::exists(_dbFile));
 }
 
+QString SyncJournalDb::databaseFilePath()
+{
+    return _dbFile;
+}
+
 void SyncJournalDb::startTransaction()
 {
     if( _transaction == 0 ) {
         if( !_db.transaction() ) {
-            qDebug() << "ERROR starting transaction: " << _db.lastError().text();
+            qDebug() << "ERROR starting transaction: " << _db.error();
             return;
         }
         _transaction = 1;
-        // qDebug() << "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX Transaction start!";
+        // qDebug() << "XXX Transaction start!";
     } else {
         qDebug() << "Database Transaction is running, do not starting another one!";
     }
@@ -67,20 +69,20 @@ void SyncJournalDb::commitTransaction()
 {
     if( _transaction == 1 ) {
         if( ! _db.commit() ) {
-            qDebug() << "ERROR committing to the database: " << _db.lastError().text();
+            qDebug() << "ERROR committing to the database: " << _db.error();
             return;
         }
         _transaction = 0;
-        // qDebug() << "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX Transaction END!";
+        // qDebug() << "XXX Transaction END!";
     } else {
         qDebug() << "No database Transaction to commit";
     }
 }
 
-bool SyncJournalDb::sqlFail( const QString& log, const QSqlQuery& query )
+bool SyncJournalDb::sqlFail( const QString& log, const SqlQuery& query )
 {
     commitTransaction();
-    qWarning() << "SQL Error" << log << query.lastError().text();
+    qWarning() << "SQL Error" << log << query.error();
     Q_ASSERT(!"SQL ERROR");
     _db.close();
     return false;
@@ -92,37 +94,41 @@ bool SyncJournalDb::checkConnect()
         return true;
     }
 
-    if( _dbFile.isEmpty() || !QFile::exists(_dbFile) ) {
+    if( _dbFile.isEmpty()) {
-        qDebug() << "Database " + _dbFile + " is empty or does not exist";
+        qDebug() << "Database filename" + _dbFile + " is empty";
         return false;
     }
 
-    QStringList list = QSqlDatabase::drivers();
+    // The database file is created by this call (SQLITE_OPEN_CREATE)
-    if( list.size() == 0 ) {
+    if( !_db.open(_dbFile) ) {
-        qDebug() << "Database Drivers could not be loaded.";
+        QString error = _db.error();
-        return false ;
+        qDebug() << "Error opening the db: " << error;
+        return false;
+    }
+
+    if( !QFile::exists(_dbFile) ) {
+        qDebug() << "Database file" + _dbFile + " does not exist";
+        return false;
+    }
+
+    SqlQuery pragma1(_db);
+    pragma1.prepare("SELECT sqlite_version();");
+    if (!pragma1.exec()) {
+        return sqlFail("SELECT sqlite_version()", pragma1);
     } else {
-        if( list.indexOf( QSQLITE ) == -1 ) {
+        pragma1.next();
-            qDebug() << "Database Driver QSQLITE could not be loaded!";
+        qDebug() << "sqlite3 version" << pragma1.stringValue(0);
-            return false;
-        }
     }
 
-    // Add the connection
+    pragma1.prepare("PRAGMA journal_mode=WAL;");
-    _db = QSqlDatabase::addDatabase( QSQLITE,  _dbFile);
+    if (!pragma1.exec()) {
+        return sqlFail("Set PRAGMA synchronous", pragma1);
+    } else {
+        pragma1.next();
+        qDebug() << "sqlite3 journal_mode=" << pragma1.stringValue(0);
 
-    // Open the file
-    _db.setDatabaseName(_dbFile);
-
-    if (!_db.isOpen()) {
-        if( !_db.open() ) {
-            QSqlError error = _db.lastError();
-            qDebug() << "Error opening the db: " << error.text();
-            return false;
-        }
     }
 
-    QSqlQuery pragma1(_db);
     pragma1.prepare("PRAGMA synchronous = 1;");
     if (!pragma1.exec()) {
         return sqlFail("Set PRAGMA synchronous", pragma1);
@@ -135,7 +141,7 @@ bool SyncJournalDb::checkConnect()
     /* Because insert are so slow, e do everything in a transaction, and one need to call commit */
     startTransaction();
 
-    QSqlQuery createQuery(_db);
+    SqlQuery createQuery(_db);
     createQuery.prepare("CREATE TABLE IF NOT EXISTS metadata("
                          "phash INTEGER(8),"
                          "pathlen INTEGER,"
@@ -203,24 +209,19 @@ bool SyncJournalDb::checkConnect()
                                "custom VARCHAR(256)"
                                ");");
     if (!createQuery.exec()) {
-        return sqlFail("Create table blacklist", createQuery);
+        return sqlFail("Create table version", createQuery);
     }
 
-    QSqlQuery versionQuery("SELECT major, minor FROM version;", _db);
+    SqlQuery versionQuery("SELECT major, minor FROM version;", _db);
     if (!versionQuery.next()) {
         // If there was no entry in the table, it means we are likely upgrading from 1.5
         _possibleUpgradeFromMirall_1_5 = true;
-    } else {
-        // Delete the existing entry so we can replace it by the new one
-        createQuery.prepare("DELETE FROM version;");
-        if (!createQuery.exec()) {
-            return sqlFail("Remove version", createQuery);
-        }
     }
-    createQuery.prepare("INSERT INTO version (major, minor, patch) VALUES ( ? , ? , ? );");
+
-    createQuery.bindValue(0, MIRALL_VERSION_MAJOR);
+    createQuery.prepare("INSERT OR REPLACE INTO version (major, minor, patch) VALUES ( ?1, ?2 , ?3 );");
-    createQuery.bindValue(1, MIRALL_VERSION_MINOR);
+    createQuery.bindValue(1, MIRALL_VERSION_MAJOR);
-    createQuery.bindValue(2, MIRALL_VERSION_PATCH);
+    createQuery.bindValue(2, MIRALL_VERSION_MINOR);
+    createQuery.bindValue(3, MIRALL_VERSION_PATCH);
     if (!createQuery.exec()) {
         return sqlFail("Insert Version", createQuery);
     }
@@ -232,49 +233,58 @@ bool SyncJournalDb::checkConnect()
         qDebug() << "WARN: Failed to update the database structure!";
     }
 
-    _getFileRecordQuery.reset(new QSqlQuery(_db));
+    _getFileRecordQuery.reset(new SqlQuery(_db));
     _getFileRecordQuery->prepare("SELECT path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm FROM "
-                                 "metadata WHERE phash=:ph" );
+                                 "metadata WHERE phash=?1" );
 
-    _setFileRecordQuery.reset(new QSqlQuery(_db) );
+    _setFileRecordQuery.reset(new SqlQuery(_db) );
     _setFileRecordQuery->prepare("INSERT OR REPLACE INTO metadata "
                                  "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm) "
-                                 "VALUES ( ? , ?, ? , ? , ? , ? , ?,  ? , ? , ?, ?, ? )" );
+                                 "VALUES (?1 , ?2, ?3 , ?4 , ?5 , ?6 , ?7,  ?8 , ?9 , ?10, ?11, ?12);" );
 
-    _getDownloadInfoQuery.reset(new QSqlQuery(_db) );
+    _getDownloadInfoQuery.reset(new SqlQuery(_db) );
     _getDownloadInfoQuery->prepare( "SELECT tmpfile, etag, errorcount FROM "
-                                    "downloadinfo WHERE path=:pa" );
+                                    "downloadinfo WHERE path=?1" );
 
-    _setDownloadInfoQuery.reset(new QSqlQuery(_db) );
+    _setDownloadInfoQuery.reset(new SqlQuery(_db) );
     _setDownloadInfoQuery->prepare( "INSERT OR REPLACE INTO downloadinfo "
                                     "(path, tmpfile, etag, errorcount) "
-                                    "VALUES ( ? , ?, ? , ? )" );
+                                    "VALUES ( ?1 , ?2, ?3, ?4 )" );
 
-    _deleteDownloadInfoQuery.reset(new QSqlQuery(_db) );
+    _deleteDownloadInfoQuery.reset(new SqlQuery(_db) );
-    _deleteDownloadInfoQuery->prepare( "DELETE FROM downloadinfo WHERE path=?" );
+    _deleteDownloadInfoQuery->prepare( "DELETE FROM downloadinfo WHERE path=?1" );
 
-    _getUploadInfoQuery.reset(new QSqlQuery(_db));
+    _getUploadInfoQuery.reset(new SqlQuery(_db));
     _getUploadInfoQuery->prepare( "SELECT chunk, transferid, errorcount, size, modtime FROM "
-                                  "uploadinfo WHERE path=:pa" );
+                                  "uploadinfo WHERE path=?1" );
 
-    _setUploadInfoQuery.reset(new QSqlQuery(_db));
+    _setUploadInfoQuery.reset(new SqlQuery(_db));
     _setUploadInfoQuery->prepare( "INSERT OR REPLACE INTO uploadinfo "
                                   "(path, chunk, transferid, errorcount, size, modtime) "
-                                  "VALUES ( ? , ?, ? , ? ,  ? , ? )");
+                                  "VALUES ( ?1 , ?2, ?3 , ?4 ,  ?5, ?6 )");
 
-    _deleteUploadInfoQuery.reset(new QSqlQuery(_db));
+    _deleteUploadInfoQuery.reset(new SqlQuery(_db));
-    _deleteUploadInfoQuery->prepare("DELETE FROM uploadinfo WHERE path=?" );
+    _deleteUploadInfoQuery->prepare("DELETE FROM uploadinfo WHERE path=?1" );
 
 
-    _deleteFileRecordPhash.reset(new QSqlQuery(_db));
+    _deleteFileRecordPhash.reset(new SqlQuery(_db));
-    _deleteFileRecordPhash->prepare("DELETE FROM metadata WHERE phash=?");
+    _deleteFileRecordPhash->prepare("DELETE FROM metadata WHERE phash=?1");
 
-    _deleteFileRecordRecursively.reset(new QSqlQuery(_db));
+    _deleteFileRecordRecursively.reset(new SqlQuery(_db));
     _deleteFileRecordRecursively->prepare("DELETE FROM metadata WHERE path LIKE(?||'/%')");
 
-    _blacklistQuery.reset(new QSqlQuery(_db));
+    QString sql( "SELECT lastTryEtag, lastTryModtime, retrycount, errorstring "
-    _blacklistQuery->prepare("SELECT lastTryEtag, lastTryModtime, retrycount, errorstring "
+                 "FROM blacklist WHERE path=?1");
-                             "FROM blacklist WHERE path=:path");
+    if( Utility::fsCasePreserving() ) {
+        // if the file system is case preserving we have to check the blacklist
+        // case insensitively
+        sql += QLatin1String(" COLLATE NOCASE");
+    }
+    _blacklistQuery.reset(new SqlQuery(_db));
+    _blacklistQuery->prepare(sql);
+
+    // don't start a new transaction now
+    commitInternal(QString("checkConnect End"), false);
 
     return rc;
 }
@@ -282,6 +292,7 @@ bool SyncJournalDb::checkConnect()
 void SyncJournalDb::close()
 {
     QMutexLocker locker(&_mutex);
+    qDebug() << Q_FUNC_INFO << _dbFile;
 
     commitTransaction();
 
@@ -299,8 +310,7 @@ void SyncJournalDb::close()
     _possibleUpgradeFromMirall_1_5 = false;
 
     _db.close();
-    _db = QSqlDatabase(); // avoid the warning QSqlDatabasePrivate::removeDatabase: connection [...] still in use
+    _db = SqlDatabase(); // avoid the warning SqlDatabasePrivate::removeDatabase: connection [...] still in use
-    QSqlDatabase::removeDatabase(_dbFile);
     _avoidReadFromDbOnNextSyncFilter.clear();
 }
 
@@ -316,7 +326,7 @@ bool SyncJournalDb::updateDatabaseStructure()
     }
 
     if( columns.indexOf(QLatin1String("fileid")) == -1 ) {
-        QSqlQuery query(_db);
+        SqlQuery query(_db);
         query.prepare("ALTER TABLE metadata ADD COLUMN fileid VARCHAR(128);");
         if( !query.exec() ) {
             sqlFail("updateDatabaseStructure: Add column fileid", query);
@@ -332,7 +342,7 @@ bool SyncJournalDb::updateDatabaseStructure()
     }
     if( columns.indexOf(QLatin1String("remotePerm")) == -1 ) {
 
-        QSqlQuery query(_db);
+        SqlQuery query(_db);
         query.prepare("ALTER TABLE metadata ADD COLUMN remotePerm VARCHAR(128);");
         if( !query.exec()) {
             sqlFail("updateDatabaseStructure: add column remotePerm", query);
@@ -342,7 +352,7 @@ bool SyncJournalDb::updateDatabaseStructure()
     }
 
     if( 1 ) {
-        QSqlQuery query(_db);
+        SqlQuery query(_db);
         query.prepare("CREATE INDEX IF NOT EXISTS metadata_inode ON metadata(inode);");
         if( !query.exec()) {
             sqlFail("updateDatabaseStructure: create index inode", query);
@@ -353,7 +363,7 @@ bool SyncJournalDb::updateDatabaseStructure()
     }
 
     if( 1 ) {
-        QSqlQuery query(_db);
+        SqlQuery query(_db);
         query.prepare("CREATE INDEX IF NOT EXISTS metadata_pathlen ON metadata(pathlen);");
         if( !query.exec()) {
             sqlFail("updateDatabaseStructure: create index pathlen", query);
@@ -371,18 +381,18 @@ QStringList SyncJournalDb::tableColumns( const QString& table )
     if( !table.isEmpty() ) {
 
         if( checkConnect() ) {
-            QString q = QString("PRAGMA table_info(%1);").arg(table);
+            QString q = QString("PRAGMA table_info('%1');").arg(table);
-            QSqlQuery query(_db);
+            SqlQuery query(_db);
             query.prepare(q);
 
             if(!query.exec()) {
-                QString err = query.lastError().text();
+                QString err = query.error();
                 qDebug() << "Error creating prepared statement: " << query.lastQuery() << ", Error:" << err;;
                 return columns;
             }
 
             while( query.next() ) {
-                columns.append( query.value(1).toString() );
+                columns.append( query.stringValue(1) );
             }
         }
     }
@@ -391,7 +401,7 @@ QStringList SyncJournalDb::tableColumns( const QString& table )
     return columns;
 }
 
-qint64 SyncJournalDb::getPHash(const QString& file) const
+qint64 SyncJournalDb::getPHash(const QString& file)
 {
     QByteArray utf8File = file.toUtf8();
     int64_t h;
@@ -434,23 +444,22 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record )
         if( fileId.isEmpty() ) fileId = "";
         QString remotePerm (record._remotePerm);
         if (remotePerm.isEmpty()) remotePerm = QString(); // have NULL in DB (vs empty)
-
+        _setFileRecordQuery->bindValue(1, QString::number(phash));
-        _setFileRecordQuery->bindValue(0, QString::number(phash));
+        _setFileRecordQuery->bindValue(2, plen);
-        _setFileRecordQuery->bindValue(1, plen);
+        _setFileRecordQuery->bindValue(3, record._path );
-        _setFileRecordQuery->bindValue(2, record._path );
+        _setFileRecordQuery->bindValue(4, record._inode );
-        _setFileRecordQuery->bindValue(3, record._inode );
+        _setFileRecordQuery->bindValue(5, 0 ); // uid Not used
-        _setFileRecordQuery->bindValue(4, 0 ); // uid Not used
+        _setFileRecordQuery->bindValue(6, 0 ); // gid Not used
-        _setFileRecordQuery->bindValue(5, 0 ); // gid Not used
+        _setFileRecordQuery->bindValue(7, record._mode );
-        _setFileRecordQuery->bindValue(6, record._mode );
+        _setFileRecordQuery->bindValue(8, QString::number(Utility::qDateTimeToTime_t(record._modtime)));
-        _setFileRecordQuery->bindValue(7, QString::number(Utility::qDateTimeToTime_t(record._modtime)));
+        _setFileRecordQuery->bindValue(9, QString::number(record._type) );
-        _setFileRecordQuery->bindValue(8, QString::number(record._type) );
+        _setFileRecordQuery->bindValue(10, etag );
-        _setFileRecordQuery->bindValue(9, etag );
+        _setFileRecordQuery->bindValue(11, fileId );
-        _setFileRecordQuery->bindValue(10, fileId );
+        _setFileRecordQuery->bindValue(12, remotePerm );
-        _setFileRecordQuery->bindValue(11, remotePerm );
 
         if( !_setFileRecordQuery->exec() ) {
             qWarning() << "Error SQL statement setFileRecord: " << _setFileRecordQuery->lastQuery() <<  " :"
-                       << _setFileRecordQuery->lastError().text();
+                       << _setFileRecordQuery->error();
             return false;
         }
 
@@ -458,7 +467,7 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record )
                  << record._mode
                  << QString::number(Utility::qDateTimeToTime_t(record._modtime)) << QString::number(record._type)
                  << record._etag << record._fileId << record._remotePerm;
-        _setFileRecordQuery->finish();
+        _setFileRecordQuery->reset();
 
         return true;
     } else {
@@ -476,26 +485,26 @@ bool SyncJournalDb::deleteFileRecord(const QString& filename, bool recursively)
         // always delete the actual file.
 
         qlonglong phash = getPHash(filename);
-        _deleteFileRecordPhash->bindValue( 0, QString::number(phash) );
+        _deleteFileRecordPhash->bindValue( 1, QString::number(phash) );
 
         if( !_deleteFileRecordPhash->exec() ) {
             qWarning() << "Exec error of SQL statement: "
                        << _deleteFileRecordPhash->lastQuery()
-                       <<  " : " << _deleteFileRecordPhash->lastError().text();
+                       <<  " : " << _deleteFileRecordPhash->error();
             return false;
         }
-        qDebug() <<  _deleteFileRecordPhash->executedQuery() << phash << filename;
+        qDebug() <<  _deleteFileRecordPhash->lastQuery() << phash << filename;
-        _deleteFileRecordPhash->finish();
+        _deleteFileRecordPhash->reset();
         if( recursively) {
-            _deleteFileRecordRecursively->bindValue(0, filename);
+            _deleteFileRecordRecursively->bindValue(1, filename);
             if( !_deleteFileRecordRecursively->exec() ) {
                 qWarning() << "Exec error of SQL statement: "
                            << _deleteFileRecordRecursively->lastQuery()
-                           <<  " : " << _deleteFileRecordRecursively->lastError().text();
+                           <<  " : " << _deleteFileRecordRecursively->error();
                 return false;
             }
-            qDebug() <<  _deleteFileRecordRecursively->executedQuery()  << filename;
+            qDebug() <<  _deleteFileRecordRecursively->lastQuery()  << filename;
-            _deleteFileRecordRecursively->finish();
+            _deleteFileRecordRecursively->reset();
         }
         return true;
     } else {
@@ -513,31 +522,30 @@ SyncJournalFileRecord SyncJournalDb::getFileRecord( const QString& filename )
     SyncJournalFileRecord rec;
 
     if( checkConnect() ) {
-        _getFileRecordQuery->bindValue(":ph", QString::number(phash));
+        _getFileRecordQuery->bindValue(1, QString::number(phash));
 
         if (!_getFileRecordQuery->exec()) {
-            QString err = _getFileRecordQuery->lastError().text();
+            QString err = _getFileRecordQuery->error();
             qDebug() << "Error creating prepared statement: " << _getFileRecordQuery->lastQuery() << ", Error:" << err;;
             return rec;
         }
 
         if( _getFileRecordQuery->next() ) {
-            bool ok;
+            rec._path    = _getFileRecordQuery->stringValue(0);
-            rec._path    = _getFileRecordQuery->value(0).toString();
+            rec._inode   = _getFileRecordQuery->intValue(1);
-            rec._inode   = _getFileRecordQuery->value(1).toInt(&ok);
             //rec._uid     = _getFileRecordQuery->value(2).toInt(&ok); Not Used
             //rec._gid     = _getFileRecordQuery->value(3).toInt(&ok); Not Used
-            rec._mode    = _getFileRecordQuery->value(4).toInt(&ok);
+            rec._mode    = _getFileRecordQuery->intValue(4);
-            rec._modtime = Utility::qDateTimeFromTime_t(_getFileRecordQuery->value(5).toLongLong(&ok));
+            rec._modtime = Utility::qDateTimeFromTime_t(_getFileRecordQuery->int64Value(5));
-            rec._type    = _getFileRecordQuery->value(6).toInt(&ok);
+            rec._type    = _getFileRecordQuery->intValue(6);
-            rec._etag    = _getFileRecordQuery->value(7).toByteArray();
+            rec._etag    = _getFileRecordQuery->baValue(7);
-            rec._fileId  = _getFileRecordQuery->value(8).toByteArray();
+            rec._fileId  = _getFileRecordQuery->baValue(8);
-            rec._remotePerm = _getFileRecordQuery->value(9).toByteArray();
+            rec._remotePerm = _getFileRecordQuery->baValue(9);
 
-            _getFileRecordQuery->finish();
+            _getFileRecordQuery->reset();
         } else {
-            QString err = _getFileRecordQuery->lastError().text();
+            QString err = _getFileRecordQuery->error();
-	    qDebug() << "No journal entry found for " << filename;
+            qDebug() << "No journal entry found for " << filename;
         }
     }
     return rec;
@@ -551,11 +559,11 @@ bool SyncJournalDb::postSyncCleanup(const QSet<QString> &items )
         return false;
     }
 
-    QSqlQuery query(_db);
+    SqlQuery query(_db);
     query.prepare("SELECT phash, path FROM metadata order by path");
 
     if (!query.exec()) {
-        QString err = query.lastError().text();
+        QString err = query.error();
         qDebug() << "Error creating prepared statement: " << query.lastQuery() << ", Error:" << err;;
         return false;
     }
@@ -563,20 +571,20 @@ bool SyncJournalDb::postSyncCleanup(const QSet<QString> &items )
     QStringList superfluousItems;
 
     while(query.next()) {
-        const QString file = query.value(1).toString();
+        const QString file = query.stringValue(1);
         bool contained = items.contains(file);
         if( !contained ) {
-            superfluousItems.append(query.value(0).toString());
+            superfluousItems.append(query.stringValue(0));
         }
     }
 
     if( superfluousItems.count() )  {
         QString sql = "DELETE FROM metadata WHERE phash in ("+ superfluousItems.join(",")+")";
         qDebug() << "Sync Journal cleanup: " << sql;
-        QSqlQuery delQuery(_db);
+        SqlQuery delQuery(_db);
         delQuery.prepare(sql);
         if( !delQuery.exec() ) {
-            QString err = delQuery.lastError().text();
+            QString err = delQuery.error();
             qDebug() << "Error removing superfluous journal entries: " << delQuery.lastQuery() << ", Error:" << err;;
             return false;
         }
@@ -592,46 +600,49 @@ int SyncJournalDb::getFileRecordCount()
         return -1;
     }
 
-    QSqlQuery query(_db);
+    SqlQuery query(_db);
     query.prepare("SELECT COUNT(*) FROM metadata");
 
     if (!query.exec()) {
-        QString err = query.lastError().text();
+        QString err = query.error();
         qDebug() << "Error creating prepared statement: " << query.lastQuery() << ", Error:" << err;;
         return 0;
     }
 
     if (query.next()) {
-        int count = query.value(0).toInt();
+        int count = query.intValue(0);
         return count;
     }
 
     return 0;
 }
 
-static void toDownloadInfo(const QSqlQuery & query, SyncJournalDb::DownloadInfo * res)
+static void toDownloadInfo(SqlQuery &query, SyncJournalDb::DownloadInfo * res)
 {
     bool ok = true;
-    res->_tmpfile    = query.value(0).toString();
+    res->_tmpfile    = query.stringValue(0);
-    res->_etag       = query.value(1).toByteArray();
+    res->_etag       = query.baValue(1);
-    res->_errorCount = query.value(2).toInt(&ok);
+    res->_errorCount = query.intValue(2);
     res->_valid      = ok;
 }
 
-static bool deleteBatch(QSqlQuery & query, const QStringList & entries, const QString & name)
+static bool deleteBatch(SqlQuery & query, const QStringList & entries, const QString & name)
 {
     if (entries.isEmpty())
         return true;
 
     qDebug() << "Removing stale " << qPrintable(name) << " entries: " << entries.join(", ");
-    query.bindValue(0, entries);
+    // FIXME: Was ported from execBatch, check if correct!
-    if (!query.execBatch()) {
+    foreach( const QString& entry, entries ) {
-        QString err = query.lastError().text();
+        query.bindValue(1, entry);
-        qDebug() << "Error removing stale " << qPrintable(name) << " entries: "
+        if (!query.exec()) {
-                 << query.lastQuery() << ", Error:" << err;
+            QString err = query.error();
-        return false;
+            qDebug() << "Error removing stale " << qPrintable(name) << " entries: "
+                     << query.lastQuery() << ", Error:" << err;
+            return false;
+        }
     }
-    query.finish();
+    query.reset();
     return true;
 }
 
@@ -642,18 +653,20 @@ SyncJournalDb::DownloadInfo SyncJournalDb::getDownloadInfo(const QString& file)
     DownloadInfo res;
 
     if( checkConnect() ) {
-        _getDownloadInfoQuery->bindValue(":pa", file);
+        _getDownloadInfoQuery->bindValue(1, file);
 
         if (!_getDownloadInfoQuery->exec()) {
-            QString err = _getDownloadInfoQuery->lastError().text();
+            QString err = _getDownloadInfoQuery->error();
             qDebug() << "Database error for file " << file << " : " << _getDownloadInfoQuery->lastQuery() << ", Error:" << err;;
             return res;
         }
 
         if( _getDownloadInfoQuery->next() ) {
             toDownloadInfo(*_getDownloadInfoQuery, &res);
+        } else {
+            res._valid = false;
         }
-        _getDownloadInfoQuery->finish();
+        _getDownloadInfoQuery->reset();
     }
     return res;
 }
@@ -667,28 +680,28 @@ void SyncJournalDb::setDownloadInfo(const QString& file, const SyncJournalDb::Do
     }
 
     if (i._valid) {
-        _setDownloadInfoQuery->bindValue(0, file);
+        _setDownloadInfoQuery->bindValue(1, file);
-        _setDownloadInfoQuery->bindValue(1, i._tmpfile);
+        _setDownloadInfoQuery->bindValue(2, i._tmpfile);
-        _setDownloadInfoQuery->bindValue(2, i._etag );
+        _setDownloadInfoQuery->bindValue(3, i._etag );
-        _setDownloadInfoQuery->bindValue(3, i._errorCount );
+        _setDownloadInfoQuery->bindValue(4, i._errorCount );
 
         if( !_setDownloadInfoQuery->exec() ) {
-            qWarning() << "Exec error of SQL statement: " << _setDownloadInfoQuery->lastQuery() <<  " :"   << _setDownloadInfoQuery->lastError().text();
+            qWarning() << "Exec error of SQL statement: " << _setDownloadInfoQuery->lastQuery() <<  " :"   << _setDownloadInfoQuery->error();
             return;
         }
 
         qDebug() <<  _setDownloadInfoQuery->lastQuery() << file << i._tmpfile << i._etag << i._errorCount;
-        _setDownloadInfoQuery->finish();
+        _setDownloadInfoQuery->reset();
 
     } else {
-        _deleteDownloadInfoQuery->bindValue( 0, file );
+        _deleteDownloadInfoQuery->bindValue( 1, file );
 
         if( !_deleteDownloadInfoQuery->exec() ) {
-            qWarning() << "Exec error of SQL statement: " << _deleteDownloadInfoQuery->lastQuery() <<  " : " << _deleteDownloadInfoQuery->lastError().text();
+            qWarning() << "Exec error of SQL statement: " << _deleteDownloadInfoQuery->lastQuery() <<  " : " << _deleteDownloadInfoQuery->error();
             return;
         }
-        qDebug() <<  _deleteDownloadInfoQuery->executedQuery()  << file;
+        qDebug() <<  _deleteDownloadInfoQuery->lastQuery()  << file;
-        _deleteDownloadInfoQuery->finish();
+        _deleteDownloadInfoQuery->reset();
     }
 }
 
@@ -701,12 +714,12 @@ QVector<SyncJournalDb::DownloadInfo> SyncJournalDb::getAndDeleteStaleDownloadInf
         return empty_result;
     }
 
-    QSqlQuery query(_db);
+    SqlQuery query(_db);
     // The selected values *must* match the ones expected by toDownloadInfo().
     query.prepare("SELECT tmpfile, etag, errorcount, path FROM downloadinfo");
 
     if (!query.exec()) {
-        QString err = query.lastError().text();
+        QString err = query.error();
         qDebug() << "Error creating prepared statement: " << query.lastQuery() << ", Error:" << err;
         return empty_result;
     }
@@ -715,7 +728,7 @@ QVector<SyncJournalDb::DownloadInfo> SyncJournalDb::getAndDeleteStaleDownloadInf
     QVector<SyncJournalDb::DownloadInfo> deleted_entries;
 
     while (query.next()) {
-        const QString file = query.value(3).toString(); // path
+        const QString file = query.stringValue(3); // path
         if (!keep.contains(file)) {
             superfluousPaths.append(file);
             DownloadInfo info;
@@ -738,24 +751,24 @@ SyncJournalDb::UploadInfo SyncJournalDb::getUploadInfo(const QString& file)
 
     if( checkConnect() ) {
 
-        _getUploadInfoQuery->bindValue(":pa", file);
+        _getUploadInfoQuery->bindValue(1, file);
 
         if (!_getUploadInfoQuery->exec()) {
-            QString err = _getUploadInfoQuery->lastError().text();
+            QString err = _getUploadInfoQuery->error();
             qDebug() << "Database error for file " << file << " : " << _getUploadInfoQuery->lastQuery() << ", Error:" << err;
             return res;
         }
 
         if( _getUploadInfoQuery->next() ) {
             bool ok = true;
-            res._chunk      = _getUploadInfoQuery->value(0).toInt(&ok);
+            res._chunk      = _getUploadInfoQuery->intValue(0);
-            res._transferid = _getUploadInfoQuery->value(1).toInt(&ok);
+            res._transferid = _getUploadInfoQuery->intValue(1);
-            res._errorCount = _getUploadInfoQuery->value(2).toInt(&ok);
+            res._errorCount = _getUploadInfoQuery->intValue(2);
-            res._size       = _getUploadInfoQuery->value(3).toLongLong(&ok);
+            res._size       = _getUploadInfoQuery->int64Value(3);
-            res._modtime    = Utility::qDateTimeFromTime_t(_getUploadInfoQuery->value(4).toLongLong(&ok));
+            res._modtime    = Utility::qDateTimeFromTime_t(_getUploadInfoQuery->int64Value(4));
             res._valid      = ok;
         }
-        _getUploadInfoQuery->finish();
+        _getUploadInfoQuery->reset();
     }
     return res;
 }
@@ -769,29 +782,29 @@ void SyncJournalDb::setUploadInfo(const QString& file, const SyncJournalDb::Uplo
     }
 
     if (i._valid) {
-        _setUploadInfoQuery->bindValue(0, file);
+        _setUploadInfoQuery->bindValue(1, file);
-        _setUploadInfoQuery->bindValue(1, i._chunk);
+        _setUploadInfoQuery->bindValue(2, i._chunk);
-        _setUploadInfoQuery->bindValue(2, i._transferid );
+        _setUploadInfoQuery->bindValue(3, i._transferid );
-        _setUploadInfoQuery->bindValue(3, i._errorCount );
+        _setUploadInfoQuery->bindValue(4, i._errorCount );
-        _setUploadInfoQuery->bindValue(4, i._size );
+        _setUploadInfoQuery->bindValue(5, i._size );
-        _setUploadInfoQuery->bindValue(5, Utility::qDateTimeToTime_t(i._modtime) );
+        _setUploadInfoQuery->bindValue(6, Utility::qDateTimeToTime_t(i._modtime) );
 
         if( !_setUploadInfoQuery->exec() ) {
-            qWarning() << "Exec error of SQL statement: " << _setUploadInfoQuery->lastQuery() <<  " :"   << _setUploadInfoQuery->lastError().text();
+            qWarning() << "Exec error of SQL statement: " << _setUploadInfoQuery->lastQuery() <<  " :"   << _setUploadInfoQuery->error();
             return;
         }
 
         qDebug() <<  _setUploadInfoQuery->lastQuery() << file << i._chunk << i._transferid << i._errorCount;
-        _setUploadInfoQuery->finish();
+        _setUploadInfoQuery->reset();
     } else {
-        _deleteUploadInfoQuery->bindValue(0, file);
+        _deleteUploadInfoQuery->bindValue(1, file);
 
         if( !_deleteUploadInfoQuery->exec() ) {
-            qWarning() << "Exec error of SQL statement: " << _deleteUploadInfoQuery->lastQuery() <<  " : " << _deleteUploadInfoQuery->lastError().text();
+            qWarning() << "Exec error of SQL statement: " << _deleteUploadInfoQuery->lastQuery() <<  " : " << _deleteUploadInfoQuery->error();
             return;
         }
-        qDebug() <<  _deleteUploadInfoQuery->executedQuery() << file;
+        qDebug() <<  _deleteUploadInfoQuery->lastQuery() << file;
-        _deleteUploadInfoQuery->finish();
+        _deleteUploadInfoQuery->reset();
     }
 }
 
@@ -803,11 +816,11 @@ bool SyncJournalDb::deleteStaleUploadInfos(const QSet<QString> &keep)
         return false;
     }
 
-    QSqlQuery query(_db);
+    SqlQuery query(_db);
     query.prepare("SELECT path FROM uploadinfo");
 
     if (!query.exec()) {
-        QString err = query.lastError().text();
+        QString err = query.error();
         qDebug() << "Error creating prepared statement: " << query.lastQuery() << ", Error:" << err;
         return false;
     }
@@ -815,7 +828,7 @@ bool SyncJournalDb::deleteStaleUploadInfos(const QSet<QString> &keep)
     QStringList superfluousPaths;
 
     while (query.next()) {
-        const QString file = query.value(0).toString();
+        const QString file = query.stringValue(0);
         if (!keep.contains(file)) {
             superfluousPaths.append(file);
         }
@@ -834,21 +847,20 @@ SyncJournalBlacklistRecord SyncJournalDb::blacklistEntry( const QString& file )
     // SELECT lastTryEtag, lastTryModtime, retrycount, errorstring
 
     if( checkConnect() ) {
-        _blacklistQuery->bindValue( ":path", file );
+        _blacklistQuery->bindValue( 1, file );
         if( _blacklistQuery->exec() ){
             if( _blacklistQuery->next() ) {
-                bool ok;
+                entry._lastTryEtag    = _blacklistQuery->baValue(0);
-                entry._lastTryEtag    = _blacklistQuery->value(0).toByteArray();
+                entry._lastTryModtime = _blacklistQuery->int64Value(1);
-                entry._lastTryModtime = _blacklistQuery->value(1).toLongLong(&ok);
+                entry._retryCount     = _blacklistQuery->intValue(2);
-                entry._retryCount     = _blacklistQuery->value(2).toInt();
+                entry._errorString    = _blacklistQuery->stringValue(3);
-                entry._errorString    = _blacklistQuery->value(3).toString();
                 entry._file           = file;
             }
         } else {
             qWarning() << "Exec error blacklist: " << _blacklistQuery->lastQuery() <<  " : "
-                       << _blacklistQuery->lastError().text();
+                       << _blacklistQuery->error();
         }
-        _blacklistQuery->finish();
+        _blacklistQuery->reset();
     }
 
     return entry;
@@ -862,11 +874,11 @@ bool SyncJournalDb::deleteStaleBlacklistEntries(const QSet<QString> &keep)
         return false;
     }
 
-    QSqlQuery query(_db);
+    SqlQuery query(_db);
     query.prepare("SELECT path FROM blacklist");
 
     if (!query.exec()) {
-        QString err = query.lastError().text();
+        QString err = query.error();
         qDebug() << "Error creating prepared statement: " << query.lastQuery() << ", Error:" << err;
         return false;
     }
@@ -874,13 +886,13 @@ bool SyncJournalDb::deleteStaleBlacklistEntries(const QSet<QString> &keep)
     QStringList superfluousPaths;
 
     while (query.next()) {
-        const QString file = query.value(0).toString();
+        const QString file = query.stringValue(0);
         if (!keep.contains(file)) {
             superfluousPaths.append(file);
         }
     }
 
-    QSqlQuery delQuery(_db);
+    SqlQuery delQuery(_db);
     delQuery.prepare("DELETE FROM blacklist WHERE path = ?");
     return deleteBatch(delQuery, superfluousPaths, "blacklist");
 }
@@ -891,12 +903,13 @@ int SyncJournalDb::blackListEntryCount()
 
     QMutexLocker locker(&_mutex);
     if( checkConnect() ) {
-        QSqlQuery query(_db);
+        SqlQuery query("SELECT count(*) FROM blacklist", _db);
-        if( ! query.exec("SELECT count(*) FROM blacklist") ) {
+
+        if( ! query.exec() ) {
             sqlFail("Count number of blacklist entries failed", query);
         }
         if( query.next() ) {
-            re = query.value(0).toInt();
+            re = query.intValue(0);
         }
     }
     return re;
@@ -906,7 +919,7 @@ int SyncJournalDb::wipeBlacklist()
 {
     QMutexLocker locker(&_mutex);
     if( checkConnect() ) {
-        QSqlQuery query(_db);
+        SqlQuery query(_db);
 
         query.prepare("DELETE FROM blacklist");
 
@@ -921,12 +934,16 @@ int SyncJournalDb::wipeBlacklist()
 
 void SyncJournalDb::wipeBlacklistEntry( const QString& file )
 {
+    if( file.isEmpty() ) {
+        return;
+    }
+
     QMutexLocker locker(&_mutex);
     if( checkConnect() ) {
-        QSqlQuery query(_db);
+        SqlQuery query(_db);
 
-        query.prepare("DELETE FROM blacklist WHERE path=:path");
+        query.prepare("DELETE FROM blacklist WHERE path=?1");
-        query.bindValue(":path", file);
+        query.bindValue(1, file);
         if( ! query.exec() ) {
             sqlFail("Deletion of blacklist item failed.", query);
         }
@@ -935,55 +952,49 @@ void SyncJournalDb::wipeBlacklistEntry( const QString& file )
 
 void SyncJournalDb::updateBlacklistEntry( const SyncJournalBlacklistRecord& item )
 {
-    QMutexLocker locker(&_mutex);
-    QSqlQuery query(_db);
-
     if( !checkConnect() ) {
         return;
     }
 
-    QString sql("SELECT retrycount FROM blacklist WHERE path=:path");
+    int retries = 0;
+    SyncJournalBlacklistRecord rec = blacklistEntry( item._file );
 
-    if( Utility::fsCasePreserving() ) {
+    QMutexLocker locker(&_mutex);
-        // if the file system is case preserving we have to check the blacklist
+
-        // case insensitively
+    bool haveRecord = false;
-        sql += QLatin1String(" COLLATE NOCASE");
+
+    if( rec.isValid() ) {
+        haveRecord = true;
+        retries = rec._retryCount;
     }
 
-    query.prepare(sql);
+    SqlQuery iQuery(_db);
-    query.bindValue(":path", item._file);
 
-    if( !query.exec() ) {
+    if( haveRecord ) {
-        qDebug() << "SQL exec blacklistitem failed:" << query.lastError().text();
-        return;
-    }
-
-    QSqlQuery iQuery(_db);
-    if( query.next() ) {
-        int retries = query.value(0).toInt();
         retries--;
         if( retries < 0 ) retries = 0;
 
-        iQuery.prepare( "UPDATE blacklist SET lastTryEtag = :etag, lastTryModtime = :modtime, "
+        iQuery.prepare( "UPDATE blacklist SET lastTryEtag = ?1, lastTryModtime = ?2, "
-                        "retrycount = :retries, errorstring = :errStr WHERE path=:path");
+                        "retrycount = ?3, errorstring = ?4 WHERE path=?5;");
-        iQuery.bindValue(":etag", item._lastTryEtag);
+        iQuery.bindValue(1, item._lastTryEtag);
-        iQuery.bindValue(":modtime", QString::number(item._lastTryModtime));
+        iQuery.bindValue(2, QString::number(item._lastTryModtime));
-        iQuery.bindValue(":retries", retries);
+        iQuery.bindValue(3, retries);
-        iQuery.bindValue(":errStr", item._errorString);
+        iQuery.bindValue(4, item._errorString);
-        iQuery.bindValue(":path", item._file);
+        iQuery.bindValue(5, item._file);
     } else {
         // there is no entry yet.
         iQuery.prepare("INSERT INTO blacklist (path, lastTryEtag, lastTryModtime, retrycount, errorstring) "
-                         "VALUES (:path, :lastEtag, :lastMTime, :retrycount, :errorstring);");
+                         "VALUES (?1, ?2, ?3, ?4, ?5);");
 
-        iQuery.bindValue(":path", item._file );
+        iQuery.bindValue(1, item._file );
-        iQuery.bindValue(":lastEtag", item._lastTryEtag);
+        iQuery.bindValue(2, item._lastTryEtag);
-        iQuery.bindValue(":lastMTime", QString::number(item._lastTryModtime));
+        iQuery.bindValue(3, QString::number(item._lastTryModtime));
-        iQuery.bindValue(":retrycount", item._retryCount);
+        iQuery.bindValue(4, item._retryCount);
-        iQuery.bindValue(":errorstring", item._errorString);
+        iQuery.bindValue(5, item._errorString);
     }
     if( !iQuery.exec() ) {
-        qDebug() << "SQL exec blacklistitem insert/update failed: "<< iQuery.lastError().text();
+        QString bug = iQuery.error();
+        qDebug() << "SQL exec blacklistitem insert/update failed: "<< bug;
     }
 
 }
@@ -996,14 +1007,14 @@ void SyncJournalDb::avoidRenamesOnNextSync(const QString& path)
         return;
     }
 
-    QSqlQuery query(_db);
+    SqlQuery query(_db);
-    query.prepare("UPDATE metadata SET fileid = '', inode = '0' WHERE path == ? OR path LIKE(?||'/%')");
+    query.prepare("UPDATE metadata SET fileid = '', inode = '0' WHERE path == ?1 OR path LIKE(?2||'/%')");
-    query.bindValue(0, path);
     query.bindValue(1, path);
+    query.bindValue(2, path);
     if( !query.exec() ) {
-        qDebug() << Q_FUNC_INFO << "SQL error in avoidRenamesOnNextSync: "<< query.lastError().text();
+        qDebug() << Q_FUNC_INFO << "SQL error in avoidRenamesOnNextSync: "<< query.error();
     } else {
-        qDebug() << Q_FUNC_INFO << query.executedQuery()  << path << "(" << query.numRowsAffected() << " rows)";
+        qDebug() << Q_FUNC_INFO << query.lastQuery()  << path << "(" << query.numRowsAffected() << " rows)";
     }
 
     // We also need to remove the ETags so the update phase refreshes the directory paths
@@ -1024,14 +1035,14 @@ void SyncJournalDb::avoidReadFromDbOnNextSync(const QString& fileName)
         return;
     }
 
-    QSqlQuery query(_db);
+    SqlQuery query(_db);
     // This query will match entries for whitch the path is a prefix of fileName
-    query.prepare("UPDATE metadata SET md5='_invalid_' WHERE ? LIKE(path||'/%') AND type == 2"); // CSYNC_FTW_TYPE_DIR == 2
+    query.prepare("UPDATE metadata SET md5='_invalid_' WHERE ?1 LIKE(path||'/%') AND type == 2;"); // CSYNC_FTW_TYPE_DIR == 2
-    query.bindValue(0, fileName);
+    query.bindValue(1, fileName);
     if( !query.exec() ) {
-        qDebug() << Q_FUNC_INFO << "SQL error in avoidRenamesOnNextSync: "<< query.lastError().text();
+        qDebug() << Q_FUNC_INFO << "SQL error in avoidRenamesOnNextSync: "<< query.error();
     } else {
-        qDebug() << Q_FUNC_INFO << query.executedQuery()  << fileName << "(" << query.numRowsAffected() << " rows)";
+        qDebug() << Q_FUNC_INFO << query.lastQuery()  << fileName << "(" << query.numRowsAffected() << " rows)";
     }
 
     // Prevent future overwrite of the etag for this sync
@@ -1044,10 +1055,20 @@ void SyncJournalDb::commit(const QString& context, bool startTrans)
     commitInternal(context, startTrans);
 }
 
+void SyncJournalDb::commitIfNeededAndStartNewTransaction(const QString &context)
+{
+    QMutexLocker lock(&_mutex);
+    if( _transaction == 1 ) {
+        commitInternal(context, true);
+    } else {
+        startTransaction();
+    }
+}
+
 
 void SyncJournalDb::commitInternal(const QString& context, bool startTrans )
 {
-    qDebug() << "Transaction Start " << context;
+    qDebug() << Q_FUNC_INFO << "Transaction commit " << context << (startTrans ? "and starting new transaction" : "");
     commitTransaction();
 
     if( startTrans ) {

src/mirall/syncjournaldb.h

@@ -17,11 +17,10 @@
 #include <QObject>
 #include <qmutex.h>
 #include <QDateTime>
-#include <QSqlDatabase>
 #include <QHash>
-#include <QSqlQuery>
 
 #include "utility.h"
+#include "ownsql.h"
 
 namespace Mirall {
 class SyncJournalFileRecord;
@@ -44,6 +43,9 @@ public:
     int getFileRecordCount();
     bool exists();
 
+    QString databaseFilePath();
+    static qint64 getPHash(const QString& );
+
     void updateBlacklistEntry( const SyncJournalBlacklistRecord& item );
     void wipeBlacklistEntry(const QString& file);
     int wipeBlacklist();
@@ -91,6 +93,7 @@ public:
      * Commit will actually commit the transaction and create a new one.
      */
     void commit(const QString &context, bool startTrans = true);
+    void commitIfNeededAndStartNewTransaction(const QString &context);
 
     void close();
 
@@ -106,31 +109,30 @@ public:
     bool isUpdateFrom_1_5();
 
 private:
-    qint64 getPHash(const QString& ) const;
     bool updateDatabaseStructure();
-    bool sqlFail(const QString& log, const QSqlQuery &query );
+    bool sqlFail(const QString& log, const SqlQuery &query );
     void commitInternal(const QString &context, bool startTrans = true);
     void startTransaction();
     void commitTransaction();
     QStringList tableColumns( const QString& table );
     bool checkConnect();
 
-    QSqlDatabase _db;
+    SqlDatabase _db;
     QString _dbFile;
     QMutex _mutex; // Public functions are protected with the mutex.
     int _transaction;
     bool _possibleUpgradeFromMirall_1_5;
-    QScopedPointer<QSqlQuery> _getFileRecordQuery;
+    QScopedPointer<SqlQuery> _getFileRecordQuery;
-    QScopedPointer<QSqlQuery> _setFileRecordQuery;
+    QScopedPointer<SqlQuery> _setFileRecordQuery;
-    QScopedPointer<QSqlQuery> _getDownloadInfoQuery;
+    QScopedPointer<SqlQuery> _getDownloadInfoQuery;
-    QScopedPointer<QSqlQuery> _setDownloadInfoQuery;
+    QScopedPointer<SqlQuery> _setDownloadInfoQuery;
-    QScopedPointer<QSqlQuery> _deleteDownloadInfoQuery;
+    QScopedPointer<SqlQuery> _deleteDownloadInfoQuery;
-    QScopedPointer<QSqlQuery> _getUploadInfoQuery;
+    QScopedPointer<SqlQuery> _getUploadInfoQuery;
-    QScopedPointer<QSqlQuery> _setUploadInfoQuery;
+    QScopedPointer<SqlQuery> _setUploadInfoQuery;
-    QScopedPointer<QSqlQuery> _deleteUploadInfoQuery;
+    QScopedPointer<SqlQuery> _deleteUploadInfoQuery;
-    QScopedPointer<QSqlQuery> _deleteFileRecordPhash;
+    QScopedPointer<SqlQuery> _deleteFileRecordPhash;
-    QScopedPointer<QSqlQuery> _deleteFileRecordRecursively;
+    QScopedPointer<SqlQuery> _deleteFileRecordRecursively;
-    QScopedPointer<QSqlQuery> _blacklistQuery;
+    QScopedPointer<SqlQuery> _blacklistQuery;
 
     /* This is the list of paths we called avoidReadFromDbOnNextSync on.
      * It means that they should not be written to the DB in any case since doing

src/mirall/utility.cpp

@@ -224,6 +224,18 @@ QString Utility::toCSyncScheme(const QString &urlStr)
     return url.toString();
 }
 
+bool Utility::doesSetContainPrefix(QSet<QString> &l, QString &p) {
+
+    Q_FOREACH (const QString &setPath, l) {
+        //qDebug() << Q_FUNC_INFO << p << setPath << setPath.startsWith(p);
+        if (setPath.startsWith(p)) {
+            return true;
+        }
+    }
+    //qDebug() << "-> NOOOOO!!!" << p << l.count();
+    return false;
+}
+
 QString Utility::escape(const QString &in)
 {
 #if QT_VERSION < QT_VERSION_CHECK(5, 0, 0)

src/mirall/utility.h

@@ -42,6 +42,10 @@ namespace Utility
     OWNCLOUDSYNC_EXPORT QString toCSyncScheme(const QString &urlStr);
     /** Like QLocale::toString(double, 'f', prec), but drops trailing zeros after the decimal point */
 
+    OWNCLOUDSYNC_EXPORT bool doesSetContainPrefix(QSet<QString> &l, QString &p);
+
+
+
     /**
      * @brief compactFormatDouble - formats a double value human readable.
      *

src/owncloudcmd/owncloudcmd.cpp

@@ -27,12 +27,16 @@
 #include "csync.h"
 #include "mirall/clientproxy.h"
 #include "mirall/account.h"
+#include "mirall/mirallconfigfile.h" // ONLY ACCESS THE STATIC FUNCTIONS!
 #include "creds/httpcredentials.h"
 #include "owncloudcmd.h"
 #include "simplesslerrorhandler.h"
-
+#include "theme.h"
 #include "netrcparser.h"
 
+#include "version.h"
+#include "config.h"
+
 #ifdef Q_OS_WIN32
 #include <windows.h>
 #else
@@ -104,20 +108,37 @@ QString queryPassword(const QString &user)
 
 class HttpCredentialsText : public HttpCredentials {
 public:
-    HttpCredentialsText(const QString& user, const QString& password) : HttpCredentials(user, password) {}
+    HttpCredentialsText(const QString& user, const QString& password)
+        : HttpCredentials(user, password),
+          _sslTrusted(false)
+    {}
+
     QString queryPassword(bool *ok) {
         if (ok) {
             *ok = true;
         }
         return ::queryPassword(user());
     }
+
+    void setSSLTrusted( bool isTrusted ) {
+        _sslTrusted = isTrusted;
+    }
+
+    bool sslIsTrusted() {
+        return _sslTrusted;
+    }
+
+private:
+    bool _sslTrusted;
 };
 
 void help()
 {
-    std::cout << "owncloudcmd - command line ownCloud client tool." << std::endl;
+    const char *binaryName = APPLICATION_EXECUTABLE "cmd";
+
+    std::cout << binaryName << " - command line " APPLICATION_NAME " client tool" << std::endl;
     std::cout << "" << std::endl;
-    std::cout << "Usage: owncloudcmd <sourcedir> <owncloudurl>" << std::endl;
+    std::cout << "Usage: " << binaryName << " <sourcedir> <owncloudurl>" << std::endl;
     std::cout << "" << std::endl;
     std::cout << "A proxy can either be set manually using --httpproxy or it" << std::endl;
     std::cout << "uses the setting from a configured sync client." << std::endl;
@@ -127,37 +148,52 @@ void help()
     std::cout << "  --httpproxy [proxy]    Specify a http proxy to use." << std::endl;
     std::cout << "                         Proxy is http://server:port" << std::endl;
     std::cout << "  --trust                Trust the SSL certification." << std::endl;
-    std::cout << "  --exclude [file]       exclude list file" << std::endl;
+    std::cout << "  --exclude [file]       Exclude list file" << std::endl;
     std::cout << "  --user, -u [name]      Use [name] as the login name" << std::endl;
     std::cout << "  --password, -p [pass]  Use [pass] as password" << std::endl;
     std::cout << "  -n                     Use netrc (5) for login" << std::endl;
     std::cout << "  --non-interactive      Do not block execution with interaction" << std::endl;
+    std::cout << "  --version, -v          Display version and exit" << std::endl;
     std::cout << "" << std::endl;
     exit(1);
 
 }
 
+void showVersion() {
+    const char *binaryName = APPLICATION_EXECUTABLE "cmd";
+    std::cout << binaryName << " version " << qPrintable(Theme::instance()->version()) << std::endl;
+    exit(1);
+}
+
 void parseOptions( const QStringList& app_args, CmdOptions *options )
 {
     QStringList args(app_args);
 
-    if( args.count() < 3 ) {
+    int argCount = args.count();
+
+    if( argCount < 3 ) {
+        if (argCount >= 2) {
+            const QString option = args.at(1);
+            if (option == "-v" || option == "--version") {
+                showVersion();
+            }
+        }
         help();
     }
 
     options->target_url = args.takeLast();
     // check if the remote.php/webdav tail was added and append if not.
-    if( !options->target_url.contains("remote.php/webdav")) {
+    if(!options->target_url.endsWith("/")) {
-        if(!options->target_url.endsWith("/")) {
+        options->target_url.append("/");
-            options->target_url.append("/");
+    }
-        }
+    if( !options->target_url.contains("remote.php/webdav/")) {
-        options->target_url.append("remote.php/webdav");
+        options->target_url.append("remote.php/webdav/");
     }
     if (options->target_url.startsWith("http"))
         options->target_url.replace(0, 4, "owncloud");
     options->source_dir = args.takeLast();
     if( !QFile::exists( options->source_dir )) {
-        std::cerr << "Source dir does not exists." << std::endl;
+        std::cerr << "Source dir '" << qPrintable(options->source_dir) << "' does not exist." << std::endl;
         exit(1);
     }
 
@@ -181,7 +217,7 @@ void parseOptions( const QStringList& app_args, CmdOptions *options )
         } else if( (option == "-u" || option == "--user") && !it.peekNext().startsWith("-") ) {
                 options->user = it.next();
         } else if( (option == "-p" || option == "--password") && !it.peekNext().startsWith("-") ) {
-                options->user = it.next();
+                options->password = it.next();
         } else if( option == "--exclude" && !it.peekNext().startsWith("-") ) {
                 options->exclude = it.next();
         } else {
@@ -206,43 +242,45 @@ int main(int argc, char **argv) {
 
     parseOptions( app.arguments(), &options );
 
+    QUrl url = QUrl::fromUserInput(options.target_url);
 
-    QUrl url = QUrl::fromUserInput(options.target_url);    
+    // Order of retrieval attempt (later attempts override earlier ones):
+    // 1. From URL
+    // 2. From options
+    // 3. From netrc (if enabled)
+    // 4. From prompt (if interactive)
 
-    // Fetch username and password. If empty, try to retrieve
+    QString user = url.userName();
-    // from URL and strip URL
+    QString password = url.password();
-    QString user;
-    QString password;
 
-    if (options.useNetrc) {
+     if (!options.user.isEmpty()) {
-        NetrcParser parser;
+            user = options.user;
-        if (parser.parse()) {
+     }
-            NetrcParser::LoginPair pair = parser.find(url.host());
-            user = pair.first;
-            password = pair.second;
-        }
-    } else {
-        user = options.user;
-        if (user.isEmpty()) {
-            user = url.userName();
-        }
-        password = options.password;
-        if (password.isEmpty()) {
-            password = url.password();
-        }
 
-        if (options.interactive) {
+     if (!options.password.isEmpty()) {
-            if (user.isEmpty()) {
+         password = options.password;
-                std::cout << "Please enter user name: ";
+     }
-                std::string s;
+
-                std::getline(std::cin, s);
+     if (options.useNetrc) {
-                user = QString::fromStdString(s);
+         NetrcParser parser;
-            }
+         if (parser.parse()) {
-            if (password.isEmpty()) {
+             NetrcParser::LoginPair pair = parser.find(url.host());
-                password = queryPassword(user);
+             user = pair.first;
-            }
+             password = pair.second;
-        }
+         }
-    }
+     }
+
+     if (options.interactive) {
+         if (user.isEmpty()) {
+             std::cout << "Please enter user name: ";
+             std::string s;
+             std::getline(std::cin, s);
+             user = QString::fromStdString(s);
+         }
+         if (password.isEmpty()) {
+             password = queryPassword(user);
+         }
+     }
 
     // ### ensure URL is free of credentials
     if (url.userName().isEmpty()) {
@@ -252,18 +290,25 @@ int main(int argc, char **argv) {
         url.setPassword(password);
     }
 
+    // take the unmodified url to pass to csync_create()
+    QByteArray remUrl = options.target_url.toUtf8();
+
     Account account;
 
     // Find the folder and the original owncloud url
     QStringList splitted = url.path().split(account.davPath());
     url.setPath(splitted.value(0));
+
     url.setScheme(url.scheme().replace("owncloud", "http"));
     QString folder = splitted.value(1);
 
     SimpleSslErrorHandler *sslErrorHandler = new SimpleSslErrorHandler;
 
-    HttpCredentials *cred = new HttpCredentialsText(user, password);
+    HttpCredentialsText *cred = new HttpCredentialsText(user, password);
 
+    if( options.trustSSL ) {
+        cred->setSSLTrusted(true);
+    }
     account.setUrl(url);
     account.setCredentials(cred);
     account.setSslErrorHandler(sslErrorHandler);
@@ -273,8 +318,9 @@ int main(int argc, char **argv) {
 restart_sync:
 
     CSYNC *_csync_ctx;
+
     if( csync_create( &_csync_ctx, options.source_dir.toUtf8(),
-                      url.toEncoded().constData()) < 0 ) {
+                      remUrl.constData()) < 0 ) {
         qFatal("Unable to create csync-context!");
         return EXIT_FAILURE;
     }
@@ -329,14 +375,29 @@ restart_sync:
         clientProxy.setCSyncProxy(QUrl(url), _csync_ctx);
     }
 
+    // Exclude lists
+    QString systemExcludeListFn = MirallConfigFile::excludeFileFromSystem();
+    int loadedSystemExcludeList = false;
+    if (!systemExcludeListFn.isEmpty()) {
+        loadedSystemExcludeList = csync_add_exclude_list(_csync_ctx, systemExcludeListFn.toLocal8Bit());
+    }
+
+    int loadedUserExcludeList = false;
     if (!options.exclude.isEmpty()) {
-        csync_add_exclude_list(_csync_ctx, options.exclude.toLocal8Bit());
+        loadedUserExcludeList = csync_add_exclude_list(_csync_ctx, options.exclude.toLocal8Bit());
+    }
+
+    if (loadedSystemExcludeList != 0 && loadedUserExcludeList != 0) {
+        // Always make sure at least one list had been loaded
+        qFatal("Cannot load system exclude list or list supplied via --exclude");
+        return EXIT_FAILURE;
     }
 
     cred->syncContextPreStart(_csync_ctx);
 
     OwncloudCmd owncloudCmd;
     SyncJournalDb db(options.source_dir);
+
     SyncEngine engine(_csync_ctx, options.source_dir, QUrl(options.target_url).path(), folder, &db);
     QObject::connect(&engine, SIGNAL(finished()), &app, SLOT(quit()));
     QObject::connect(&engine, SIGNAL(transmissionProgress(Progress::Info)), &owncloudCmd, SLOT(transmissionProgressSlot()));

src/updater/updater.cpp

@@ -80,7 +80,11 @@ QString Updater::getSystemInfo()
 // To test, cmake with -DAPPLICATION_UPDATE_URL="http://127.0.0.1:8080/test.rss"
 Updater *Updater::create()
 {
-    QUrl updateBaseUrl = addQueryParams(QUrl(QLatin1String(APPLICATION_UPDATE_URL)));
+    QUrl updateBaseUrl(QString::fromLocal8Bit(qgetenv("OCC_UPDATE_URL")));
+    if (updateBaseUrl.isEmpty()) {
+        updateBaseUrl = QUrl(QLatin1String(APPLICATION_UPDATE_URL));
+    }
+    updateBaseUrl = addQueryParams(updateBaseUrl);
 #if defined(Q_OS_MAC) && defined(HAVE_SPARKLE)
     updateBaseUrl.addQueryItem( QLatin1String("sparkle"), QLatin1String("true"));
     return new SparkleUpdater(updateBaseUrl.toString());

test/CMakeLists.txt

@@ -27,5 +27,7 @@ endif(UNIX AND NOT APPLE)
 
 owncloud_add_test(CSyncSqlite "")
 owncloud_add_test(NetrcParser ../src/owncloudcmd/netrcparser.cpp)
+owncloud_add_test(OwnSql ../src/mirall/ownsql.cpp)
+
 
 

test/testownsql.h

@@ -0,0 +1,155 @@
+/*
+ *    This software is in the public domain, furnished "as is", without technical
+ *       support, and with no warranty, express or implied, as to its usefulness for
+ *          any purpose.
+ *          */
+
+#ifndef MIRALL_TESTOWNSQL_H
+#define MIRALL_TESTOWNSQL_H
+
+#include <QtTest>
+
+#include <sqlite3.h>
+
+#include "mirall/ownsql.h"
+
+using namespace Mirall;
+
+namespace {
+
+const char testdbC[] = "/tmp/testdb.sqlite";
+}
+
+class TestOwnSql : public QObject
+{
+    Q_OBJECT
+
+private slots:
+    void initTestCase() {
+        QFileInfo fi( testdbC );
+
+        if( fi.exists() ) {
+            QFile::remove(testdbC);
+        }
+        fi.refresh();
+        QVERIFY(!fi.exists());
+    }
+
+    void cleanupTestCase() {
+        // QFile::remove(testdbC);
+    }
+
+    void testOpenDb() {
+        QFileInfo fi( testdbC );
+        QVERIFY( !fi.exists() ); // must not exist
+        _db.open(testdbC);
+        fi.refresh();
+        QVERIFY(fi.exists());
+
+    }
+
+    void testCreate() {
+        const char *sql = "CREATE TABLE addresses ( id INTEGER, name VARCHAR(4096), "
+                "address VARCHAR(4096), entered INTEGER(8), PRIMARY KEY(id));";
+
+        SqlQuery q(_db);
+        q.prepare(sql);
+        QVERIFY(q.exec());
+    }
+
+    void testIsSelect() {
+        SqlQuery q(_db);
+        q.prepare("SELECT foo FROM bar;");
+        QVERIFY( q.isSelect() );
+
+        q.prepare("UPDATE bla SET foo = 1;");
+        QVERIFY( !q.isSelect());
+    }
+
+    void testInsert() {
+        const char *sql = "INSERT INTO addresses (id, name, address, entered) VALUES "
+                "(1, 'Gonzo Alberto', 'Moriabata 24, Palermo', 1403100844);";
+        SqlQuery q(_db);
+        q.prepare(sql);
+        QVERIFY(q.exec());
+    }
+
+    void testInsert2() {
+        const char *sql = "INSERT INTO addresses (id, name, address, entered) VALUES "
+                "(?1, ?2, ?3, ?4);";
+        SqlQuery q(_db);
+        q.prepare(sql);
+        q.bindValue(1, 2);
+        q.bindValue(2, "Brucely Lafayette");
+        q.bindValue(3, "Nurderway5, New York");
+        q.bindValue(4, 1403101224);
+        QVERIFY(q.exec());
+    }
+
+    void testSelect() {
+        const char *sql = "SELECT * FROM addresses;";
+
+        SqlQuery q(_db);
+        q.prepare(sql);
+
+        q.exec();
+        while( q.next() ) {
+            qDebug() << "Name: " << q.stringValue(1);
+            qDebug() << "Address: " << q.stringValue(2);
+        }
+    }
+
+    void testSelect2() {
+        const char *sql = "SELECT * FROM addresses WHERE id=?1";
+        SqlQuery q(_db);
+        q.prepare(sql);
+        q.bindValue(1, 2);
+        q.exec();
+        if( q.next() ) {
+            qDebug() << "Name:" << q.stringValue(1);
+            qDebug() << "Address:" << q.stringValue(2);
+        }
+    }
+
+    void testPragma() {
+        const char *sql = "PRAGMA table_info(addresses)";
+
+        SqlQuery q(_db);
+        int rc = q.prepare(sql);
+        qDebug() << "Pragma:" << rc;
+        q.exec();
+        if( q.next() ) {
+            qDebug() << "P:" << q.stringValue(1);
+        }
+    }
+
+    void testUnicode() {
+        const char *sql = "INSERT INTO addresses (id, name, address, entered) VALUES "
+                "(?1, ?2, ?3, ?4);";
+        SqlQuery q(_db);
+        q.prepare(sql);
+        q.bindValue(1, 3);
+        q.bindValue(2, QString::fromUtf8("пятницы"));
+        q.bindValue(3, QString::fromUtf8("проспект"));
+        q.bindValue(4, 1403002224);
+        QVERIFY(q.exec());
+    }
+
+    void testReadUnicode() {
+        const char *sql = "SELECT * FROM addresses WHERE id=3;";
+        SqlQuery q(_db);
+        q.prepare(sql);
+
+        if(q.next()) {
+            QString name = q.stringValue(1);
+            QString address = q.stringValue(2);
+            QVERIFY( name == QString::fromUtf8("пятницы") );
+            QVERIFY( address == QString::fromUtf8("проспект"));
+        }
+    }
+
+private:
+    SqlDatabase _db;
+};
+
+#endif