Allow to create new files (but changing them is not allowed yet)

- since mediafire cannot deal with empty files, we put new files into a
   temporary location and upload them once they get closed
 - add create and write functions to fuse
 - pass a file handle to mfconn_api_upload_simple instead of a path
 - allow calc_sha256 to also compute the file size
 - error out when the key returned by upload/simple is empty
 - make valgrind.supp more lenient

fuse/hashtbl.c

@@ -718,6 +718,28 @@ int folder_tree_readdir(folder_tree * tree, mfconn * conn, const char *path,
     return 0;
 }
 
+int folder_tree_tmp_open(folder_tree * tree)
+{
+    char           *tmpfilename;
+    int             fd;
+
+    tmpfilename = strdup_printf("%s/tmp_XXXXXX", tree->filecache);
+
+    fd = mkstemp(tmpfilename);
+
+    // this will cause the file to be removed immediately after it is closed
+    unlink(tmpfilename);
+
+    if (fd < 0) {
+        fprintf(stderr, "mkstemp failed\n");
+        return -1;
+    }
+
+    free(tmpfilename);
+
+    return fd;
+}
+
 int folder_tree_open_file(folder_tree * tree, mfconn * conn, const char *path)
 {
     struct h_entry *entry;

fuse/hashtbl.h

@@ -73,4 +73,6 @@ bool            folder_tree_path_is_file(folder_tree * tree, mfconn * conn,
 int             folder_tree_open_file(folder_tree * tree, mfconn * conn,
                                       const char *path);
 
+int             folder_tree_tmp_open(folder_tree * tree);
+
 #endif

fuse/main.c

@@ -62,10 +62,11 @@ static struct fuse_operations mediafirefs_oper = {
     .rmdir = mediafirefs_rmdir,
     .open = mediafirefs_open,
     .read = mediafirefs_read,
+    .write = mediafirefs_write,
     .release = mediafirefs_release,
     .unlink = mediafirefs_unlink,
-/*    .create = mediafirefs_create,
-    .fsync = mediafirefs_fsync,
+    .create = mediafirefs_create,
+/*    .fsync = mediafirefs_fsync,
     .getxattr = mediafirefs_getxattr,
     .init = mediafirefs_init,
     .listxattr = mediafirefs_listxattr,
@@ -74,8 +75,7 @@ static struct fuse_operations mediafirefs_oper = {
     .setxattr = mediafirefs_setxattr,
     .statfs = mediafirefs_statfs,
     .truncate = mediafirefs_truncate,
-    .utime = mediafirefs_utime,
-    .write = mediafirefs_write,*/
+    .utime = mediafirefs_utime,*/
 };
 
 static void usage(const char *progname)
@@ -405,6 +405,9 @@ int main(int argc, char *argv[])
 
     connect_mf(&options, ctx);
 
+    ctx->tmpfiles = NULL;
+    ctx->num_tmpfiles = 0;
+
     ret = fuse_main(argc, argv, &mediafirefs_oper, ctx);
 
     for (i = 0; i < argc; i++) {

fuse/operations.c

@@ -31,6 +31,7 @@
 #include <bits/fcntl-linux.h>
 #include <fuse/fuse_common.h>
 #include <stdint.h>
+#include <libgen.h>
 
 #include "../mfapi/mfconn.h"
 #include "../mfapi/apicalls.h"
@@ -87,7 +88,14 @@
  */
 
 struct mediafirefs_openfile {
+    // to fread and fwrite from/to the file
     int             fd;
+    // whether or not a patch has to be uploaded when closing
+    bool            is_readonly;
+    // whether or not to do a new file upload when closing
+    // if this is NULL then the file already existed remotely
+    // otherwise a new upload has to be done to the given remote path
+    char           *remote_path;
 };
 
 int mediafirefs_getattr(const char *path, struct stat *stbuf)
@@ -101,10 +109,31 @@ int mediafirefs_getattr(const char *path, struct stat *stbuf)
      * amount of time
      */
     struct mediafirefs_context_private *ctx;
+    int             retval;
+    size_t          i;
 
     ctx = fuse_get_context()->private_data;
     folder_tree_update(ctx->tree, ctx->conn);
-    return folder_tree_getattr(ctx->tree, ctx->conn, path, stbuf);
+    retval = folder_tree_getattr(ctx->tree, ctx->conn, path, stbuf);
+
+    if (retval != 0) {
+        for (i = 0; i < ctx->num_tmpfiles; i++) {
+            if (strcmp(ctx->tmpfiles[i], path) == 0) {
+                stbuf->st_uid = geteuid();
+                stbuf->st_gid = getegid();
+                stbuf->st_ctime = 0;
+                stbuf->st_mtime = 0;
+                stbuf->st_mode = S_IFREG | 0666;
+                stbuf->st_nlink = 1;
+                stbuf->st_atime = 0;
+                stbuf->st_size = 0;
+                retval = 0;
+                break;
+            }
+        }
+    }
+
+    return retval;
 }
 
 int mediafirefs_readdir(const char *path, void *buf, fuse_fill_dir_t filldir,
@@ -286,7 +315,7 @@ int mediafirefs_open(const char *path, struct fuse_file_info *file_info)
     ctx = fuse_get_context()->private_data;
 
     if ((file_info->flags & O_ACCMODE) != O_RDONLY) {
-        fprintf(stderr, "can only open read-only");
+        fprintf(stderr, "can only open read-only\n");
         return -EACCES;
     }
 
@@ -298,10 +327,48 @@ int mediafirefs_open(const char *path, struct fuse_file_info *file_info)
 
     openfile = malloc(sizeof(struct mediafirefs_openfile));
     openfile->fd = fd;
+    openfile->is_readonly = true;
+    openfile->remote_path = NULL;
     file_info->fh = (uint64_t) openfile;
     return 0;
 }
 
+/*
+ * this is called if the file does not exist yet. It will create a temporary
+ * file and open it.
+ * once the file gets closed, it will be uploaded.
+ */
+int mediafirefs_create(const char *path, mode_t mode,
+                       struct fuse_file_info *file_info)
+{
+    (void)mode;
+
+    int             fd;
+    struct mediafirefs_openfile *openfile;
+    struct mediafirefs_context_private *ctx;
+
+    ctx = fuse_get_context()->private_data;
+
+    fd = folder_tree_tmp_open(ctx->tree);
+    if (fd < 0) {
+        fprintf(stderr, "folder_tree_tmp_open failed\n");
+        return -EACCES;
+    }
+
+    openfile = malloc(sizeof(struct mediafirefs_openfile));
+    openfile->fd = fd;
+    openfile->is_readonly = false;
+    openfile->remote_path = strdup(path);
+    file_info->fh = (uint64_t) openfile;
+
+    // add to tmpfiles
+    ctx->num_tmpfiles++;
+    ctx->tmpfiles = realloc(ctx->tmpfiles, sizeof(char *) * ctx->num_tmpfiles);
+    ctx->tmpfiles[ctx->num_tmpfiles - 1] = openfile->remote_path;
+
+    return 0;
+}
+
 int mediafirefs_read(const char *path, char *buf, size_t size, off_t offset,
                      struct fuse_file_info *file_info)
 {
@@ -310,12 +377,105 @@ int mediafirefs_read(const char *path, char *buf, size_t size, off_t offset,
                  offset);
 }
 
+int mediafirefs_write(const char *path, const char *buf, size_t size,
+                      off_t offset, struct fuse_file_info *file_info)
+{
+    (void)path;
+    return pwrite(((struct mediafirefs_openfile *)file_info->fh)->fd, buf,
+                  size, offset);
+}
+
+/*
+ * note: the return value of release() is ignored by fuse
+ */
 int mediafirefs_release(const char *path, struct fuse_file_info *file_info)
 {
     (void)path;
-    struct mediafirefs_openfile *openfile =
-        (struct mediafirefs_openfile *)file_info->fh;
+
+    FILE           *fh;
+    char           *file_name;
+    char           *dir_name;
+    const char     *folder_key;
+    char           *upload_key;
+    char           *temp1;
+    char           *temp2;
+    int             retval;
+    struct mediafirefs_context_private *ctx;
+    struct mediafirefs_openfile *openfile;
+    int             status;
+    int             fileerror;
+
+    ctx = fuse_get_context()->private_data;
+
+    openfile = (struct mediafirefs_openfile *)file_info->fh;
+
+    // if the file was opened readonly, then no new updates have to be
+    // uploaded
+    if (openfile->is_readonly) {
         close(openfile->fd);
         free(openfile);
         return 0;
     }
+    // if the file only exists locally, an initial upload has to be done
+    if (openfile->remote_path != NULL) {
+        // pass a copy because dirname and basename may modify their argument
+        temp1 = strdup(openfile->remote_path);
+        file_name = basename(temp1);
+        temp2 = strdup(openfile->remote_path);
+        dir_name = dirname(temp2);
+
+        fh = fdopen(openfile->fd, "r");
+        rewind(fh);
+
+        folder_key = folder_tree_path_get_key(ctx->tree, ctx->conn, dir_name);
+
+        retval = mfconn_api_upload_simple(ctx->conn, folder_key,
+                                          fh, file_name, &upload_key);
+        mfconn_update_secret_key(ctx->conn);
+
+        fclose(fh);
+        free(temp1);
+        free(temp2);
+        free(openfile->remote_path);
+        free(openfile);
+
+        if (retval != 0 || upload_key == NULL) {
+            fprintf(stderr, "mfconn_api_upload_simple failed\n");
+            return -EACCES;
+        }
+        // poll for completion
+        for (;;) {
+            // no need to update the secret key after this
+            retval = mfconn_api_upload_poll_upload(ctx->conn, upload_key,
+                                                   &status, &fileerror);
+            if (retval != 0) {
+                fprintf(stderr, "mfconn_api_upload_poll_upload failed\n");
+                return -1;
+            }
+            fprintf(stderr, "status: %d, filerror: %d\n", status, fileerror);
+            if (status == 99) {
+                fprintf(stderr, "done\n");
+                break;
+            }
+            sleep(1);
+        }
+
+        // remove from tmpfiles
+        ctx->num_tmpfiles--;
+        ctx->tmpfiles = realloc(ctx->tmpfiles,
+                                sizeof(char *) * ctx->num_tmpfiles);
+
+        free(upload_key);
+
+        folder_tree_update(ctx->tree, ctx->conn);
+        return 0;
+    }
+    // the file was not opened readonly and also existed on the remote
+    // thus, we have to check whether any changes were made and if yes, upload
+    // a patch
+    // FIXME: not implemented yet
+    close(openfile->fd);
+    free(openfile);
+    folder_tree_update(ctx->tree, ctx->conn);
+    return -ENOSYS;
+}

fuse/operations.h

@@ -31,6 +31,12 @@ struct mediafirefs_context_private {
     char           *configfile;
     char           *dircache;
     char           *filecache;
+    /* stores all currently open temporary files which are to be uploaded when
+     * they are closed.
+     * we use a normal array because the number of open temporary files will
+     * never be very high but is limited by the number of threads */
+    char          **tmpfiles;
+    size_t          num_tmpfiles;
 };
 
 int             mediafirefs_getattr(const char *path, struct stat *stbuf);
@@ -46,7 +52,12 @@ int             mediafirefs_open(const char *path,
 int             mediafirefs_read(const char *path, char *buf, size_t size,
                                  off_t offset,
                                  struct fuse_file_info *file_info);
+int             mediafirefs_write(const char *path, const char *buf,
+                                  size_t size, off_t offset,
+                                  struct fuse_file_info *file_info);
 int             mediafirefs_release(const char *path,
                                     struct fuse_file_info *file_info);
+int             mediafirefs_create(const char *path, mode_t mode,
+                                   struct fuse_file_info *file_info);
 
 #endif

mfapi/apicalls.h

@@ -21,6 +21,7 @@
 #define _MFAPI_APICALLS_H_
 
 #include <stdint.h>
+#include <stdio.h>
 
 #include "file.h"
 #include "folder.h"
@@ -96,8 +97,7 @@ int             mfconn_api_device_get_patch(mfconn * conn, mfpatch * patch,
                                             uint64_t target_revision);
 
 int             mfconn_api_upload_simple(mfconn * conn, const char *folderkey,
-                                         const char *file_path,
-                                         const char *file_name,
+                                         FILE * fh, const char *file_name,
                                          char **upload_key);
 
 int             mfconn_api_upload_poll_upload(mfconn * conn,

mfapi/apicalls/upload_simple.c

@@ -24,8 +24,8 @@
 #include <string.h>
 #include <stddef.h>
 #include <stdio.h>
+#include <stdint.h>
 #include <openssl/sha.h>
-#include <sys/stat.h>
 
 #include "../../utils/http.h"
 #include "../../utils/json.h"
@@ -37,43 +37,32 @@ static int      _decode_upload_simple(mfhttp * conn, void *data);
 
 int
 mfconn_api_upload_simple(mfconn * conn, const char *folderkey,
-                         const char *file_path, const char *file_name,
-                         char **upload_key)
+                         FILE * fh, const char *file_name, char **upload_key)
 {
     const char     *api_call;
     int             retval;
     mfhttp         *http;
-    FILE           *fh;
     unsigned char   hash[SHA256_DIGEST_LENGTH];
     char           *file_hash;
-    struct stat     file_info;
+    uint64_t        file_size;
 
     if (conn == NULL)
         return -1;
 
-    fh = fopen(file_path, "r");
-    if (fh == NULL) {
-        perror("cannot open file");
-        fprintf(stderr, "cannot open %s\n", file_path);
+    if (fh == NULL)
         return -1;
-    }
 
-    retval = calc_sha256(fh, hash);
-    fclose(fh);
+    // make sure that we are at the beginning of the file
+    rewind(fh);
+
+    // calculate hash
+    retval = calc_sha256(fh, hash, &file_size);
 
     if (retval != 0) {
         fprintf(stderr, "failed to calculate hash\n");
         return -1;
     }
 
-    memset(&file_info, 0, sizeof(file_info));
-    retval = stat(file_path, &file_info);
-
-    if (retval != 0) {
-        fprintf(stderr, "stat failed\n");
-        return -1;
-    }
-
     file_hash = binary2hex(hash, SHA256_DIGEST_LENGTH);
 
     if (folderkey == NULL) {
@@ -87,9 +76,12 @@ mfconn_api_upload_simple(mfconn * conn, const char *folderkey,
                                             "&folder_key=%s", folderkey);
     }
 
+    // make sure that we are at the beginning of the file
+    rewind(fh);
+
     http = http_create();
-    retval = http_post_file(http, api_call, file_path, file_name,
-                            file_info.st_size, file_hash,
+    retval = http_post_file(http, api_call, fh, file_name,
+                            file_size, file_hash,
                             _decode_upload_simple, upload_key);
     http_destroy(http);
 
@@ -118,7 +110,11 @@ static int _decode_upload_simple(mfhttp * conn, void *user_ptr)
 
     j_obj = json_object_get(node, "key");
     if (j_obj != NULL) {
+        if (strcmp(json_string_value(j_obj), "") == 0) {
+            *upload_key = NULL;
+        } else {
             *upload_key = strdup(json_string_value(j_obj));
+        }
     } else {
         *upload_key = NULL;
     }

mfshell/commands/put.c

@@ -39,6 +39,7 @@ int mfshell_cmd_put(mfshell * mfshell, int argc, char *const argv[])
     char           *upload_key;
     int             status;
     int             fileerror;
+    FILE           *fh;
 
     if (mfshell == NULL)
         return -1;
@@ -58,25 +59,34 @@ int mfshell_cmd_put(mfshell * mfshell, int argc, char *const argv[])
 
     file_path = argv[1];
 
+    fh = fopen(file_path, "r");
+    if (fh == NULL) {
+        perror("fopen");
+        fprintf(stderr, "cannot open %s\n", file_path);
+        return -1;
+    }
     // create copies because basename modifies it
     temp = strdup(argv[1]);
     file_name = basename(temp);
 
     retval = mfconn_api_upload_simple(mfshell->conn,
                                       folder_get_key(mfshell->folder_curr),
-                                      file_path, file_name, &upload_key);
+                                      fh, file_name, &upload_key);
     mfconn_update_secret_key(mfshell->conn);
 
+    fclose(fh);
     free(temp);
 
-    if (retval != 0) {
+    if (retval != 0 || upload_key == NULL) {
         fprintf(stderr, "mfconn_api_upload_simple failed\n");
         return -1;
     }
 
     fprintf(stderr, "upload_key: %s\n", upload_key);
 
+    // poll for completion
     for (;;) {
+        // no need to update the secret key after this
         retval = mfconn_api_upload_poll_upload(mfshell->conn, upload_key,
                                                &status, &fileerror);
         if (retval != 0) {

utils/hash.c

@@ -96,21 +96,32 @@ int calc_md5(FILE * file, unsigned char *hash)
     return 0;
 }
 
-int calc_sha256(FILE * file, unsigned char *hash)
+/*
+ * calculate the SHA256 sum and optionally (if size != NULL) count the file
+ * size
+ */
+int calc_sha256(FILE * file, unsigned char *hash, uint64_t * size)
 {
     int             bytesRead;
     char           *buffer;
     SHA256_CTX      sha256;
+    uint64_t        bytesRead_sum;
 
     SHA256_Init(&sha256);
     buffer = malloc(bufsize);
     if (buffer == NULL) {
         return -1;
     }
+
+    bytesRead_sum = 0;
     while ((bytesRead = fread(buffer, 1, bufsize, file))) {
         SHA256_Update(&sha256, buffer, bytesRead);
+        bytesRead_sum += bytesRead;
     }
     SHA256_Final(hash, &sha256);
+    if (size != NULL) {
+        *size = bytesRead_sum;
+    }
     free(buffer);
     return 0;
 }
@@ -232,7 +243,7 @@ int file_check_integrity_hash(const char *path, const unsigned char *fhash)
         return -1;
     }
 
-    retval = calc_sha256(fh, hash);
+    retval = calc_sha256(fh, hash, NULL);
     if (retval != 0) {
         fprintf(stderr, "failed to calculate hash\n");
         fclose(fh);

utils/hash.h

@@ -20,7 +20,8 @@
 #define _MFSHELL_HASH_H_
 
 int             calc_md5(FILE * file, unsigned char *hash);
-int             calc_sha256(FILE * file, unsigned char *hash);
+int             calc_sha256(FILE * file, unsigned char *hash,
+                            uint64_t * file_size);
 int             base36_decode_triplet(const char *key);
 void            hex2binary(const char *hex, unsigned char *binary);
 char           *binary2hex(const unsigned char *binary, size_t length);

utils/http.c

@@ -277,7 +277,7 @@ http_read_file_cb(char *data, size_t size, size_t nmemb, void *user_ptr)
 }
 
 int
-http_post_file(mfhttp * conn, const char *url, const char *path,
+http_post_file(mfhttp * conn, const char *url, FILE * fh,
                const char *filename, uint64_t filesize, const char *fhash,
                int (*data_handler) (mfhttp * conn, void *data), void *data)
 {
@@ -318,10 +318,9 @@ http_post_file(mfhttp * conn, const char *url, const char *path,
     curl_easy_setopt(conn->curl_handle, CURLOPT_WRITEDATA, (void *)conn);
     curl_easy_setopt(conn->curl_handle, CURLOPT_POSTFIELDSIZE, filesize);
 
-    conn->stream = fopen(path, "r");
+    conn->stream = fh;
     fprintf(stderr, "POST: %s\n", url);
     retval = curl_easy_perform(conn->curl_handle);
-    fclose(conn->stream);
     curl_slist_free_all(custom_headers);
     if (retval != CURLE_OK) {
         fprintf(stderr, "error curl_easy_perform %s\n\r", conn->error_buf);

utils/http.h

@@ -39,7 +39,7 @@ int             http_get_file(mfhttp * conn, const char *url,
 json_t         *http_parse_buf_json(mfhttp * conn, size_t flags,
                                     json_error_t * error);
 int             http_post_file(mfhttp * conn, const char *url,
-                               const char *path, const char *filename,
+                               FILE * fh, const char *filename,
                                uint64_t filesize, const char *fhash,
                                int (*data_handler) (mfhttp * conn, void *data),
                                void *data);

valgrind.supp

@@ -116,7 +116,6 @@
 {
    fuse_opt_parse
    Memcheck:Leak
-   match-leak-kinds: definite
    fun:malloc
    fun:strdup
    obj:*libfuse.so*