allow to change local files

- fuse/filecache:
     * add filecache_upload_patch
     * allow opening files in modes other than RDONLY
 - fuse/hashtbl:
     * add folder_tree_upload_patch
 - fuse/operations:
     * allow opening files in modes other than RDONLY
     * add members to private context which allow tracking of
       not-yet-uploaded files and files opened for writing and
       files opened in read-only mode
 - mfapi/apicalls/upload_patch:
     * supply x-filename and x-filesize headers
 - mfapi/apicalls/upload_simple:
     * do not supply the x-filehash header as it is not used by the
       server
 - utils/hash:
     * hex characters must be lower case for the server
 - utils/strings:
     * clean up unused functions strdup_join, strdup_substr,
       string_chomp
 - utils/stringv:
     * complete rewrite with different string vector implementation

fuse/filecache.c

@@ -26,6 +26,9 @@
 #endif
 #include <stdint.h>
 #include <stdlib.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <sys/types.h>
 
 #include "../utils/hash.h"
 #include "../utils/xdelta3.h"
@@ -54,25 +57,200 @@ static int      filecache_patch_file(const char *filecache_path,
                                      uint64_t source_revision,
                                      uint64_t target_revision);
 
+int filecache_upload_patch(const char *quickkey, uint64_t local_revision,
+                           const char *filecache_path, mfconn * conn)
+{
+    FILE           *source_fh;
+    FILE           *target_fh;
+    FILE           *patchfile_fh;
+    unsigned char   hash[SHA256_DIGEST_LENGTH];
+    char           *source_hash;
+    char           *target_hash;
+    uint64_t        target_size;
+    char           *cachefile;
+    char           *newfile;
+    char           *patch_file;
+    int             retval;
+    char           *upload_key;
+    int             status;
+    int             fileerror;
+
+    cachefile = strdup_printf("%s/%s_%d", filecache_path, quickkey,
+                              local_revision);
+
+    source_fh = fopen(cachefile, "r");
+    if (source_fh == NULL) {
+        fprintf(stderr, "cannot open %s\n", cachefile);
+        free(cachefile);
+        return -1;
+    }
+    free(cachefile);
+
+    newfile = strdup_printf("%s/%s_%d_new", filecache_path, quickkey,
+                            local_revision);
+
+    target_fh = fopen(newfile, "r");
+    if (target_fh == NULL) {
+        fprintf(stderr, "cannot open %s\n", newfile);
+        free(newfile);
+        fclose(source_fh);
+        return -1;
+    }
+    free(newfile);
+
+    retval = calc_sha256(source_fh, hash, NULL);
+
+    if (retval != 0) {
+        fprintf(stderr, "failed to calculate hash\n");
+        fclose(source_fh);
+        fclose(target_fh);
+        return -1;
+    }
+
+    source_hash = binary2hex(hash, SHA256_DIGEST_LENGTH);
+
+    retval = calc_sha256(target_fh, hash, &target_size);
+
+    if (retval != 0) {
+        fprintf(stderr, "failed to calculate hash\n");
+        fclose(source_fh);
+        fclose(target_fh);
+        return -1;
+    }
+
+    target_hash = binary2hex(hash, SHA256_DIGEST_LENGTH);
+
+    if (strcmp(source_hash, target_hash) == 0) {
+        // no changes were done
+        free(source_hash);
+        free(target_hash);
+        return 0;
+    }
+
+    patch_file = strdup_printf("%s/%s_patch_%d_new", filecache_path, quickkey,
+                               local_revision);
+
+    patchfile_fh = fopen(patch_file, "w");
+    if (patchfile_fh == NULL) {
+        fprintf(stderr, "cannot open %s\n", patch_file);
+        fclose(source_fh);
+        fclose(target_fh);
+        return -1;
+    }
+
+    rewind(source_fh);
+    rewind(target_fh);
+    retval = xdelta3_diff(source_fh, target_fh, patchfile_fh);
+    fclose(source_fh);
+    fclose(target_fh);
+    fclose(patchfile_fh);
+
+    upload_key = NULL;
+    retval = mfconn_api_upload_patch(conn, quickkey, source_hash, target_hash,
+                                     target_size, patch_file, &upload_key);
+
+    if (retval != 0 || upload_key == NULL) {
+        fprintf(stderr, "mfconn_api_upload_patch failed\n");
+        return -1;
+    }
+    // poll for completion
+    for (;;) {
+        // no need to update the secret key after this
+        retval = mfconn_api_upload_poll_upload(conn, upload_key,
+                                               &status, &fileerror);
+        if (retval != 0) {
+            fprintf(stderr, "mfconn_api_upload_poll_upload failed\n");
+            return -1;
+        }
+        fprintf(stderr, "status: %d, filerror: %d\n", status, fileerror);
+        if (status == 99) {
+            fprintf(stderr, "done\n");
+            break;
+        }
+        sleep(1);
+    }
+
+    free(upload_key);
+
+    return 0;
+}
+
 int filecache_open_file(const char *quickkey, uint64_t local_revision,
                         uint64_t remote_revision, uint64_t fsize,
                         const unsigned char *fhash,
-                        const char *filecache_path, mfconn * conn)
+                        const char *filecache_path, mfconn * conn, mode_t mode,
+                        bool update)
 {
     char           *cachefile;
+    char           *newfile;
     int             fd;
     int             retval;
+    const int       BUFSIZE = 4096;
+    char            buf[BUFSIZE];
+    size_t          size;
+    int             source;
+    int             dest;
 
-    /* check if the requested file is already in the cache */
-    cachefile =
-        strdup_printf("%s/%s_%d", filecache_path, quickkey, remote_revision);
-    fd = open(cachefile, O_RDWR);
-    if (fd > 0) {
-        /* file existed - return handle */
-        free(cachefile);
-        return fd;
+    if (update) {
+        cachefile = strdup_printf("%s/%s_%d", filecache_path, quickkey,
+                                  remote_revision);
+    } else {
+        cachefile = strdup_printf("%s/%s_%d", filecache_path, quickkey,
+                                  local_revision);
+    }
+    /* check if the requested file is already in the cache */
+    if ((mode & O_ACCMODE) == O_RDONLY) {
+        // if file is opened in readonly mode, we try to open it directly
+        fd = open(cachefile, mode);
+        free(cachefile);
+        if (fd > 0) {
+            /* file existed - return handle */
+            return fd;
+        }
+        // if the file cannot be opened, then it has to be retrieved
+    } else {
+        // if file is opened writable then a temporary file has to be opened
+        // instead to upload a patch if necessary
+        if (update) {
+            newfile = strdup_printf("%s/%s_%d_new", filecache_path, quickkey,
+                                    remote_revision);
+        } else {
+            newfile = strdup_printf("%s/%s_%d_new", filecache_path, quickkey,
+                                    local_revision);
+        }
+        fd = open(newfile, mode);
+        if (fd > 0) {
+            /* file existed - return handle */
+            free(newfile);
+            free(cachefile);
+            return fd;
+        }
+        // the temporary file wasn't available, so we copy it from the
+        // original
+        source = open(cachefile, O_RDONLY);
+        free(cachefile);
+        if (fd > 0) {
+            dest = open(newfile, O_WRONLY | O_CREAT, 0644);
+            while ((size = read(source, buf, BUFSIZE)) > 0) {
+                write(dest, buf, size);
+            }
+            close(source);
+            close(dest);
+            fd = open(newfile, mode);
+            free(newfile);
+            return fd;
+        }
+        free(newfile);
+        // if the source file cannot be opened for copying, then it has to be
+        // retrieved
+    }
+
+    // if no updating is requested and we end up here, then something failed
+    // but since we must not update, this is a failure
+    if (!update) {
+        fprintf(stderr, "no updating but cannot open\n");
+        return -1;
     }
-    free(cachefile);
 
     /* if the file with remote revision didn't exist, then check whether an
      * old revision exists and in that case update that.
@@ -81,7 +259,7 @@ int filecache_open_file(const char *quickkey, uint64_t local_revision,
 
     cachefile =
         strdup_printf("%s/%s_%d", filecache_path, quickkey, local_revision);
-    fd = open(cachefile, O_RDWR);
+    fd = open(cachefile, O_RDONLY);
     free(cachefile);
     if (fd > 0) {
         close(fd);
@@ -115,7 +293,24 @@ int filecache_open_file(const char *quickkey, uint64_t local_revision,
         return -1;
     }
 
-    fd = open(cachefile, O_RDWR);
+    if ((mode & O_ACCMODE) == O_RDONLY) {
+        // if file is opened in readonly mode, we open it directly
+        fd = open(cachefile, mode);
+    } else {
+        // if file is opened writable then a temporary file has to be opened
+        // instead to upload a patch if necessary
+        newfile = strdup_printf("%s/%s_%d_new", filecache_path, quickkey,
+                                remote_revision);
+        source = open(cachefile, O_RDONLY);
+        dest = open(newfile, O_WRONLY | O_CREAT, 0644);
+        while ((size = read(source, buf, BUFSIZE)) > 0) {
+            write(dest, buf, size);
+        }
+        close(source);
+        close(dest);
+        fd = open(newfile, mode);
+        free(newfile);
+    }
 
     free(cachefile);
 

fuse/filecache.h

@@ -23,6 +23,11 @@ int             filecache_open_file(const char *quickkey,
                                     uint64_t local_revision,
                                     uint64_t remote_revision, uint64_t fsize,
                                     const unsigned char *fhash,
-                                    const char *filecache, mfconn * conn);
+                                    const char *filecache, mfconn * conn,
+                                    mode_t mode, bool update);
+
+int             filecache_upload_patch(const char *quickkey,
+                                       uint64_t local_revision,
+                                       const char *filecache, mfconn * conn);
 
 #endif

fuse/hashtbl.c

@@ -34,24 +34,17 @@
 #include <fcntl.h>
 #include <stdint.h>
 #include <stddef.h>
-#include <pwd.h>
 #include <inttypes.h>
-#ifdef __linux
-#include <bits/fcntl-linux.h>
-#endif
 #include <openssl/sha.h>
 
 #include "hashtbl.h"
 #include "filecache.h"
 #include "../mfapi/mfconn.h"
 #include "../mfapi/file.h"
-#include "../mfapi/patch.h"
 #include "../mfapi/folder.h"
 #include "../mfapi/apicalls.h"
 #include "../utils/strings.h"
-#include "../utils/http.h"
 #include "../utils/hash.h"
-#include "../utils/xdelta3.h"
 
 /*
  * we build a hashtable using the first three characters of the file or folder
@@ -746,7 +739,31 @@ int folder_tree_tmp_open(folder_tree * tree)
     return fd;
 }
 
-int folder_tree_open_file(folder_tree * tree, mfconn * conn, const char *path)
+int folder_tree_upload_patch(folder_tree * tree, mfconn * conn,
+                             const char *path)
+{
+    struct h_entry *entry;
+    int             retval;
+
+    entry = folder_tree_lookup_path(tree, conn, path);
+    /* either file not found or found entry is not a file */
+    if (entry == NULL || entry->atime == 0) {
+        return -ENOENT;
+    }
+
+    retval = filecache_upload_patch(entry->key, entry->local_revision,
+                                    tree->filecache, conn);
+
+    if (retval != 0) {
+        fprintf(stderr, "filecache_upload_patch failed\n");
+        return -1;
+    }
+
+    return 0;
+}
+
+int folder_tree_open_file(folder_tree * tree, mfconn * conn, const char *path,
+                          mode_t mode, bool update)
 {
     struct h_entry *entry;
     int             retval;
@@ -762,16 +779,19 @@ int folder_tree_open_file(folder_tree * tree, mfconn * conn, const char *path)
 
     retval = filecache_open_file(entry->key, entry->local_revision,
                                  entry->remote_revision, entry->fsize,
-                                 entry->hash, tree->filecache, conn);
+                                 entry->hash, tree->filecache, conn, mode,
+                                 update);
     if (retval == -1) {
         fprintf(stderr, "filecache_open_file failed\n");
         return -1;
     }
 
-    /* make sure that the local_revision is equal to the remote revision
-     * because filecache_open_file took care of doing any updating if it was
-     * necessary */
-    entry->local_revision = entry->remote_revision;
+    if (update) {
+        /* make sure that the local_revision is equal to the remote revision
+         * because filecache_open_file took care of doing any updating if it
+         * was necessary */
+        entry->local_revision = entry->remote_revision;
+    }
 
     return retval;
 }

fuse/hashtbl.h

@@ -23,6 +23,7 @@
 #include <stdio.h>
 #include <stdbool.h>
 #include <stdint.h>
+#include <sys/types.h>
 
 #include "../mfapi/mfconn.h"
 
@@ -72,8 +73,12 @@ bool            folder_tree_path_is_file(folder_tree * tree, mfconn * conn,
                                          const char *path);
 
 int             folder_tree_open_file(folder_tree * tree, mfconn * conn,
-                                      const char *path);
+                                      const char *path, mode_t mode,
+                                      bool update);
 
 int             folder_tree_tmp_open(folder_tree * tree);
 
+int             folder_tree_upload_patch(folder_tree * tree, mfconn * conn,
+                                         const char *path);
+
 #endif

fuse/main.c

@@ -41,6 +41,7 @@
 #include "hashtbl.h"
 #include "operations.h"
 #include "../utils/strings.h"
+#include "../utils/stringv.h"
 
 enum {
     KEY_HELP,
@@ -410,8 +411,8 @@ int main(int argc, char *argv[])
 
     connect_mf(&options, ctx);
 
-    ctx->tmpfiles = NULL;
-    ctx->num_tmpfiles = 0;
+    ctx->sv_writefiles = stringv_alloc();
+    ctx->sv_readonlyfiles = stringv_alloc();
 
     ret = fuse_main(argc, argv, &mediafirefs_oper, ctx);
 
@@ -423,6 +424,8 @@ int main(int argc, char *argv[])
     free(ctx->configfile);
     free(ctx->dircache);
     free(ctx->filecache);
+    stringv_free(ctx->sv_writefiles);
+    stringv_free(ctx->sv_readonlyfiles);
     free(ctx);
 
     return ret;

fuse/operations.c

@@ -95,12 +95,11 @@
 struct mediafirefs_openfile {
     // to fread and fwrite from/to the file
     int             fd;
+    char           *path;
     // whether or not a patch has to be uploaded when closing
     bool            is_readonly;
     // whether or not to do a new file upload when closing
-    // if this is NULL then the file already existed remotely
-    // otherwise a new upload has to be done to the given remote path
-    char           *remote_path;
+    bool            is_local;
 };
 
 int mediafirefs_getattr(const char *path, struct stat *stbuf)
@@ -115,27 +114,21 @@ int mediafirefs_getattr(const char *path, struct stat *stbuf)
      */
     struct mediafirefs_context_private *ctx;
     int             retval;
-    size_t          i;
 
     ctx = fuse_get_context()->private_data;
     folder_tree_update(ctx->tree, ctx->conn, false);
     retval = folder_tree_getattr(ctx->tree, ctx->conn, path, stbuf);
 
-    if (retval != 0) {
-        for (i = 0; i < ctx->num_tmpfiles; i++) {
-            if (strcmp(ctx->tmpfiles[i], path) == 0) {
-                stbuf->st_uid = geteuid();
-                stbuf->st_gid = getegid();
-                stbuf->st_ctime = 0;
-                stbuf->st_mtime = 0;
-                stbuf->st_mode = S_IFREG | 0666;
-                stbuf->st_nlink = 1;
-                stbuf->st_atime = 0;
-                stbuf->st_size = 0;
-                retval = 0;
-                break;
-            }
-        }
+    if (retval != 0 && stringv_mem(ctx->sv_writefiles, path)) {
+        stbuf->st_uid = geteuid();
+        stbuf->st_gid = getegid();
+        stbuf->st_ctime = 0;
+        stbuf->st_mtime = 0;
+        stbuf->st_mode = S_IFREG | 0666;
+        stbuf->st_nlink = 1;
+        stbuf->st_atime = 0;
+        stbuf->st_size = 0;
+        retval = 0;
     }
 
     return retval;
@@ -308,20 +301,60 @@ int mediafirefs_unlink(const char *path)
     return 0;
 }
 
+/*
+ * the following restrictions apply:
+ *  1. a file can be opened in read-only mode more than once at a time
+ *  2. a file can only be be opened in write-only or read-write mode if it is
+ *     not open for writing at the same time
+ *  3. a file that is only local and has not been uploaded yet cannot be read
+ *     from
+ *  4. a file that has opened in any way will not be updated to its latest
+ *     remote revision until all its opened handles are closed
+ *
+ *  Point 2 is enforced by a lookup in the writefiles string vector. If the
+ *  path is in there then it was either just created locally or opened with
+ *  write-only or read-write. In both cases it must not be opened for
+ *  writing again.
+ *
+ *  Point 3 is enforced by the lookup in the hashtable failing.
+ *
+ *  Point 4 is enforced by checking if the current path is in the writefiles or
+ *  readonlyfiles string vector and if yes, no updating will be done.
+ */
 int mediafirefs_open(const char *path, struct fuse_file_info *file_info)
 {
     int             fd;
+    bool            is_open;
     struct mediafirefs_openfile *openfile;
     struct mediafirefs_context_private *ctx;
 
     ctx = fuse_get_context()->private_data;
 
-    if ((file_info->flags & O_ACCMODE) != O_RDONLY) {
-        fprintf(stderr, "can only open read-only\n");
+    /* if file is not opened read-only, check if it was already opened in a
+     * not read-only mode and abort if yes */
+    if ((file_info->flags & O_ACCMODE) != O_RDONLY
+        && stringv_mem(ctx->sv_writefiles, path)) {
+        fprintf(stderr, "file %s was already opened for writing\n", path);
         return -EACCES;
     }
 
-    fd = folder_tree_open_file(ctx->tree, ctx->conn, path);
+    is_open = false;
+    // check if the file was already opened
+    // check read-only files first
+    if (stringv_mem(ctx->sv_readonlyfiles, path)) {
+        is_open = true;
+    }
+    // check writable files only if the file was
+    //   - not yet found in the read-only files
+    //   - the file is opened in read-only mode (because otherwise the
+    //     writable files were already searched above without failing)
+    if (!is_open && (file_info->flags & O_ACCMODE) == O_RDONLY
+        && stringv_mem(ctx->sv_writefiles, path)) {
+        is_open = true;
+    }
+
+    fd = folder_tree_open_file(ctx->tree, ctx->conn, path, file_info->flags,
+                               !is_open);
     if (fd < 0) {
         fprintf(stderr, "folder_tree_file_open unsuccessful\n");
         return fd;
@@ -329,8 +362,19 @@ int mediafirefs_open(const char *path, struct fuse_file_info *file_info)
 
     openfile = malloc(sizeof(struct mediafirefs_openfile));
     openfile->fd = fd;
-    openfile->is_readonly = true;
-    openfile->remote_path = NULL;
+    openfile->is_local = false;
+    openfile->path = strdup(path);
+
+    if ((file_info->flags & O_ACCMODE) == O_RDONLY) {
+        openfile->is_readonly = true;
+        // add to readonlyfiles
+        stringv_add(ctx->sv_readonlyfiles, path);
+    } else {
+        openfile->is_readonly = false;
+        // add to writefiles
+        stringv_add(ctx->sv_writefiles, path);
+    }
+
     file_info->fh = (uint64_t) openfile;
     return 0;
 }
@@ -359,14 +403,13 @@ int mediafirefs_create(const char *path, mode_t mode,
 
     openfile = malloc(sizeof(struct mediafirefs_openfile));
     openfile->fd = fd;
+    openfile->is_local = true;
     openfile->is_readonly = false;
-    openfile->remote_path = strdup(path);
+    openfile->path = strdup(path);
     file_info->fh = (uint64_t) openfile;
 
-    // add to tmpfiles
-    ctx->num_tmpfiles++;
-    ctx->tmpfiles = realloc(ctx->tmpfiles, sizeof(char *) * ctx->num_tmpfiles);
-    ctx->tmpfiles[ctx->num_tmpfiles - 1] = openfile->remote_path;
+    // add to writefiles
+    stringv_add(ctx->sv_writefiles, path);
 
     return 0;
 }
@@ -411,19 +454,38 @@ int mediafirefs_release(const char *path, struct fuse_file_info *file_info)
 
     openfile = (struct mediafirefs_openfile *)file_info->fh;
 
-    // if the file was opened readonly, then no new updates have to be
-    // uploaded
+    // if file was opened as readonly then it just has to be closed
     if (openfile->is_readonly) {
+        // remove this entry from readonlyfiles
+        if (stringv_del(ctx->sv_readonlyfiles, openfile->path) != 0) {
+            fprintf(stderr, "FATAL: readonly entry %s not found\n",
+                    openfile->path);
+            exit(1);
+        }
+
         close(openfile->fd);
+        free(openfile->path);
         free(openfile);
         return 0;
     }
+    // if the file is not readonly, its entry in writefiles has to be removed
+    if (stringv_del(ctx->sv_writefiles, openfile->path) != 0) {
+        fprintf(stderr, "FATAL: writefiles entry %s not found\n",
+                openfile->path);
+        exit(1);
+    }
+    if (stringv_mem(ctx->sv_writefiles, openfile->path) != 0) {
+        fprintf(stderr,
+                "FATAL: writefiles entry %s was found more than once\n",
+                openfile->path);
+        exit(1);
+    }
     // if the file only exists locally, an initial upload has to be done
-    if (openfile->remote_path != NULL) {
+    if (openfile->is_local) {
         // pass a copy because dirname and basename may modify their argument
-        temp1 = strdup(openfile->remote_path);
+        temp1 = strdup(openfile->path);
         file_name = basename(temp1);
-        temp2 = strdup(openfile->remote_path);
+        temp2 = strdup(openfile->path);
         dir_name = dirname(temp2);
 
         fh = fdopen(openfile->fd, "r");
@@ -438,7 +500,7 @@ int mediafirefs_release(const char *path, struct fuse_file_info *file_info)
         fclose(fh);
         free(temp1);
         free(temp2);
-        free(openfile->remote_path);
+        free(openfile->path);
         free(openfile);
 
         if (retval != 0 || upload_key == NULL) {
@@ -462,11 +524,6 @@ int mediafirefs_release(const char *path, struct fuse_file_info *file_info)
             sleep(1);
         }
 
-        // remove from tmpfiles
-        ctx->num_tmpfiles--;
-        ctx->tmpfiles = realloc(ctx->tmpfiles,
-                                sizeof(char *) * ctx->num_tmpfiles);
-
         free(upload_key);
 
         folder_tree_update(ctx->tree, ctx->conn, true);
@@ -475,9 +532,18 @@ int mediafirefs_release(const char *path, struct fuse_file_info *file_info)
     // the file was not opened readonly and also existed on the remote
     // thus, we have to check whether any changes were made and if yes, upload
     // a patch
-    // FIXME: not implemented yet
+
     close(openfile->fd);
+
+    retval = folder_tree_upload_patch(ctx->tree, ctx->conn, openfile->path);
+    free(openfile->path);
     free(openfile);
+
+    if (retval != 0) {
+        fprintf(stderr, "folder_tree_upload_patch failed\n");
+        return -EACCES;
+    }
+
     folder_tree_update(ctx->tree, ctx->conn, true);
-    return -ENOSYS;
+    return 0;
 }

fuse/operations.h

@@ -24,6 +24,7 @@
 #include <sys/types.h>
 #include "../mfapi/mfconn.h"
 #include "hashtbl.h"
+#include "../utils/stringv.h"
 
 struct mediafirefs_context_private {
     mfconn         *conn;
@@ -31,12 +32,14 @@ struct mediafirefs_context_private {
     char           *configfile;
     char           *dircache;
     char           *filecache;
-    /* stores all currently open temporary files which are to be uploaded when
-     * they are closed.
-     * we use a normal array because the number of open temporary files will
-     * never be very high but is limited by the number of threads */
-    char          **tmpfiles;
-    size_t          num_tmpfiles;
+    /* stores:
+     *  - all currently open temporary files which are to be uploaded when
+     *    they are closed.
+     *  - all files that are opened for writing
+     */
+    stringv        *sv_writefiles;
+    /* stores all files that have been opened for reading only */
+    stringv        *sv_readonlyfiles;
 };
 
 int             mediafirefs_getattr(const char *path, struct stat *stbuf);