add support for instant uploads

CMakeLists.txt

@@ -57,6 +57,8 @@ add_library(mfapi OBJECT
 	mfapi/apicalls/device_get_changes.c
 	mfapi/apicalls/device_get_patch.c
 	mfapi/apicalls/device_get_updates.c
+	mfapi/apicalls/upload_check.c
+	mfapi/apicalls/upload_instant.c
 	mfapi/apicalls/upload_simple.c
 	mfapi/apicalls/upload_patch.c
 	mfapi/apicalls/upload_poll_upload.c

fuse/operations.c

@@ -37,10 +37,12 @@
 #include <libgen.h>
 #include <stdbool.h>
 #include <time.h>
+#include <openssl/sha.h>
 
 #include "../mfapi/mfconn.h"
 #include "../mfapi/apicalls.h"
 #include "../utils/stringv.h"
+#include "../utils/hash.h"
 #include "hashtbl.h"
 #include "operations.h"
 
@@ -524,6 +526,10 @@ int mediafirefs_release(const char *path, struct fuse_file_info *file_info)
     int             retval;
     struct mediafirefs_context_private *ctx;
     struct mediafirefs_openfile *openfile;
+    struct mfconn_upload_check_result check_result;
+    unsigned char   bhash[SHA256_DIGEST_LENGTH];
+    char           *hash;
+    uint64_t        size;
 
     ctx = fuse_get_context()->private_data;
 
@@ -571,29 +577,82 @@ int mediafirefs_release(const char *path, struct fuse_file_info *file_info)
 
         folder_key = folder_tree_path_get_key(ctx->tree, ctx->conn, dir_name);
 
-        upload_key = NULL;
+        retval = calc_sha256(fh, bhash, &size);
-        retval = mfconn_api_upload_simple(ctx->conn, folder_key,
+        rewind(fh);
-                                          fh, file_name, &upload_key);
 
-        fclose(fh);
+        if (retval != 0) {
-        free(temp1);
+            fprintf(stderr, "failed to calculate hash\n");
-        free(temp2);
+            fclose(fh);
-        free(openfile->path);
+            free(temp1);
-        free(openfile);
+            free(temp2);
-
+            free(openfile->path);
-        if (retval != 0 || upload_key == NULL) {
+            free(openfile);
-            fprintf(stderr, "mfconn_api_upload_simple failed\n");
             pthread_mutex_unlock(&(ctx->mutex));
             return -EACCES;
         }
-        // poll for completion
+
-        retval = mfconn_upload_poll_for_completion(ctx->conn, upload_key);
+        hash = binary2hex(bhash, SHA256_DIGEST_LENGTH);
-        free(upload_key);
+
+        retval = mfconn_api_upload_check(ctx->conn, file_name, hash, size,
+                                         folder_key, &check_result);
 
         if (retval != 0) {
-            fprintf(stderr, "mfconn_upload_poll_for_completion failed\n");
+            fclose(fh);
+            free(temp1);
+            free(temp2);
+            free(openfile->path);
+            free(openfile);
+            free(hash);
+            fprintf(stderr, "mfconn_api_upload_check failed\n");
             pthread_mutex_unlock(&(ctx->mutex));
-            return -1;
+            return -EACCES;
+        }
+
+        if (check_result.hash_exists) {
+            // hash exists, so use upload/instant
+
+            retval = mfconn_api_upload_instant(ctx->conn, NULL,
+                    file_name, hash, size, folder_key);
+
+            fclose(fh);
+            free(temp1);
+            free(temp2);
+            free(openfile->path);
+            free(openfile);
+            free(hash);
+
+            if (retval != 0) {
+                fprintf(stderr, "mfconn_api_upload_instant failed\n");
+                pthread_mutex_unlock(&(ctx->mutex));
+                return -EACCES;
+            }
+        } else {
+            // hash does not exist, so do full upload
+            upload_key = NULL;
+            retval = mfconn_api_upload_simple(ctx->conn, folder_key,
+                    fh, file_name, &upload_key);
+
+            fclose(fh);
+            free(temp1);
+            free(temp2);
+            free(openfile->path);
+            free(openfile);
+            free(hash);
+
+            if (retval != 0 || upload_key == NULL) {
+                fprintf(stderr, "mfconn_api_upload_simple failed\n");
+                pthread_mutex_unlock(&(ctx->mutex));
+                return -EACCES;
+            }
+            // poll for completion
+            retval = mfconn_upload_poll_for_completion(ctx->conn, upload_key);
+            free(upload_key);
+
+            if (retval != 0) {
+                fprintf(stderr, "mfconn_upload_poll_for_completion failed\n");
+                pthread_mutex_unlock(&(ctx->mutex));
+                return -1;
+            }
         }
 
         folder_tree_update(ctx->tree, ctx->conn, true);

mfapi/apicalls.h

@@ -23,6 +23,7 @@
 #include <jansson.h>
 #include <stdint.h>
 #include <stdio.h>
+#include <stdbool.h>
 
 #include "file.h"
 #include "folder.h"
@@ -63,6 +64,14 @@ struct mfconn_device_change {
     char            parent[16];
 };
 
+struct mfconn_upload_check_result {
+    bool            hash_exists;
+    bool            in_account;
+    bool            file_exists;
+    bool            different_hash;
+    /* TODO: add resumable_upload */
+};
+
 int             mfapi_check_response(json_t * response, const char *apicall);
 
 int             mfapi_decode_common(mfhttp * conn, void *user_ptr);
@@ -133,6 +142,15 @@ int             mfconn_api_device_get_patch(mfconn * conn, mfpatch * patch,
                                             uint64_t source_revision,
                                             uint64_t target_revision);
 
+int             mfconn_api_upload_check(mfconn * conn, const char *filename,
+                                        const char *hash,
+                                        uint64_t size, const char *folder_key,
+                                        struct mfconn_upload_check_result *result);
+
+int             mfconn_api_upload_instant(mfconn * conn, const char *quick_key,
+                                          const char *filename, const char *hash,
+                                          uint64_t size, const char *folder_key);
+
 int             mfconn_api_upload_simple(mfconn * conn, const char *folderkey,
                                          FILE * fh, const char *file_name,
                                          char **upload_key);

mfapi/apicalls/upload_check.c

@@ -0,0 +1,227 @@
+/*
+ * Copyright (C) 2015 Johannes Schauer <j.schauer@email.de>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2, as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#include <jansson.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <inttypes.h>
+#include <stdio.h>
+
+#include "../../utils/http.h"
+#include "../mfconn.h"
+#include "../patch.h"
+#include "../apicalls.h"        // IWYU pragma: keep
+
+static int      _decode_upload_check(mfhttp * conn, void *data);
+
+int mfconn_api_upload_check(mfconn * conn, const char *filename, const char *hash,
+                            uint64_t size, const char *folder_key,
+                            struct mfconn_upload_check_result *result)
+{
+    const char     *api_call;
+    int             retval;
+    mfhttp         *http;
+    int             i;
+
+    if (conn == NULL)
+        return -1;
+
+    if (hash == NULL || hash[0] == '\0') {
+        fprintf(stderr, "hash must not be empty\n");
+        return -1;
+    }
+
+    if (filename == NULL || filename[0] == '\0') {
+        fprintf(stderr, "filename must not be empty\n");
+        return -1;
+    }
+
+    if (folder_key == NULL) {
+        fprintf(stderr, "folder_key must not be empty\n");
+        return -1;
+    }
+
+    for (i = 0; i < mfconn_get_max_num_retries(conn); i++) {
+        api_call = mfconn_create_signed_get(conn, 0, "upload/check.php",
+                                            "?response_format=json"
+                                            "&filename=%s"
+                                            "&size=%" PRIu64
+                                            "&hash=%s"
+                                            "&folder_key=%s", filename,
+                                            size, hash, folder_key);
+        if (api_call == NULL) {
+            fprintf(stderr, "mfconn_create_signed_get failed\n");
+            return -1;
+        }
+
+        http = http_create();
+        retval = http_get_buf(http, api_call, _decode_upload_check,
+                              (void *)result);
+        http_destroy(http);
+        mfconn_update_secret_key(conn);
+
+        free((void *)api_call);
+
+        if (retval != 127 && retval != 28)
+            break;
+
+        // if there was either a curl timeout or a token error, get a new
+        // token and try again
+        //
+        // on a curl timeout we get a new token because it is likely that we
+        // lost signature synchronization (we don't know whether the server
+        // accepted or rejected the last call)
+        fprintf(stderr, "got error %d - negotiate a new token\n", retval);
+        retval = mfconn_refresh_token(conn);
+        if (retval != 0) {
+            fprintf(stderr, "failed to get a new token\n");
+            break;
+        }
+    }
+
+    return retval;
+}
+
+static int      _decode_upload_check(mfhttp * conn, void *data)
+{
+    json_error_t    error;
+    json_t         *obj;
+    struct mfconn_upload_check_result *result;
+    json_t         *root;
+    json_t         *node;
+    int             retval;
+
+    if (data == NULL)
+        return -1;
+
+    result = (struct mfconn_upload_check_result *)data;
+
+    root = http_parse_buf_json(conn, 0, &error);
+
+    if (root == NULL) {
+        fprintf(stderr, "http_parse_buf_json failed at line %d\n", error.line);
+        fprintf(stderr, "error message: %s\n", error.text);
+        return -1;
+    }
+
+    node = json_object_get(root, "response");
+
+    retval = mfapi_check_response(node, "upload/check");
+    if (retval != 0) {
+        fprintf(stderr, "invalid response\n");
+        json_decref(root);
+        return retval;
+    }
+
+    /* retrieve response/hash_exists */
+    obj = json_object_get(node, "hash_exists");
+    if (obj == NULL) {
+        fprintf(stderr, "cannot get node response/hash_exists\n");
+        json_decref(root);
+        return -1;
+    }
+    if (!json_is_string(obj)) {
+        fprintf(stderr, "response/hash_exists is not expected type string\n");
+        json_decref(root);
+        return -1;
+    }
+    if (strcmp(json_string_value(obj), "yes") == 0)
+        result->hash_exists = true;
+    else if (strcmp(json_string_value(obj), "no") == 0)
+        result->hash_exists = false;
+    else {
+        fprintf(stderr, "response/hash_exists is neither yes nor no\n");
+        json_decref(root);
+        return -1;
+    }
+
+    /* retrieve response/in_account */
+    if (result->hash_exists) {
+        obj = json_object_get(node, "in_account");
+        if (obj == NULL) {
+            fprintf(stderr, "cannot get node response/in_account\n");
+            json_decref(root);
+            return -1;
+        }
+        if (!json_is_string(obj)) {
+            fprintf(stderr, "response/in_account is not expected type string\n");
+            json_decref(root);
+            return -1;
+        }
+        if (strcmp(json_string_value(obj), "yes") == 0)
+            result->in_account = true;
+        else if (strcmp(json_string_value(obj), "no") == 0)
+            result->in_account = false;
+        else {
+            fprintf(stderr, "response/in_account is neither yes nor no\n");
+            json_decref(root);
+            return -1;
+        }
+    }
+
+    /* retrieve response/file_exists */
+    obj = json_object_get(node, "file_exists");
+    if (obj == NULL) {
+        fprintf(stderr, "cannot get node response/file_exists\n");
+        json_decref(root);
+        return -1;
+    }
+    if (!json_is_string(obj)) {
+        fprintf(stderr, "response/file_exists is not expected type string\n");
+        json_decref(root);
+        return -1;
+    }
+    if (strcmp(json_string_value(obj), "yes") == 0)
+        result->file_exists = true;
+    else if (strcmp(json_string_value(obj), "no") == 0)
+        result->file_exists = false;
+    else {
+        fprintf(stderr, "response/file_exists is neither yes nor no\n");
+        json_decref(root);
+        return -1;
+    }
+
+    /* retrieve response/different_hash */
+    if (result->file_exists) {
+        obj = json_object_get(node, "different_hash");
+        if (obj == NULL) {
+            fprintf(stderr, "cannot get node response/different_hash\n");
+            json_decref(root);
+            return -1;
+        }
+        if (!json_is_string(obj)) {
+            fprintf(stderr, "response/different_hash is not expected type string\n");
+            json_decref(root);
+            return -1;
+        }
+        if (strcmp(json_string_value(obj), "yes") == 0)
+            result->different_hash = true;
+        else if (strcmp(json_string_value(obj), "no") == 0)
+            result->different_hash = false;
+        else {
+            fprintf(stderr, "response/different_hash is neither yes nor no\n");
+            json_decref(root);
+            return -1;
+        }
+    }
+
+    json_decref(root);
+
+    return 0;
+}

mfapi/apicalls/upload_instant.c

@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2015 Johannes Schauer <j.schauer@email.de>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2, as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <inttypes.h>
+
+#include "../../utils/http.h"
+#include "../mfconn.h"
+#include "../apicalls.h"        // IWYU pragma: keep
+
+int mfconn_api_upload_instant(mfconn * conn, const char *quick_key,
+                              const char *filename, const char *hash,
+                              uint64_t size, const char *folder_key)
+{
+    const char     *api_call;
+    int             retval;
+    mfhttp         *http;
+    int             i;
+
+    if (conn == NULL)
+        return -1;
+
+    if (hash == NULL || hash[0] == '\0') {
+        fprintf(stderr, "hash must not be empty\n");
+        return -1;
+    }
+
+    for (i = 0; i < mfconn_get_max_num_retries(conn); i++) {
+        if (quick_key != NULL && quick_key[0] != '\0') {
+            // update an existing file
+            api_call = mfconn_create_signed_get(conn, 0, "upload/instant.php",
+                                                "?quick_key=%s"
+                                                "&size=%" PRIu64
+                                                "&hash=%s"
+                                                "&response_format=json",
+                                                quick_key, size, hash);
+        } else if (filename != NULL && filename[0] != '\0'
+                && folder_key != 0) {
+            // upload a new file
+            api_call = mfconn_create_signed_get(conn, 0, "upload/instant.php",
+                                                "?folder_key=%s"
+                                                "&filename=%s"
+                                                "&size=%" PRIu64
+                                                "&hash=%s"
+                                                "&response_format=json",
+                                                folder_key, filename, size,
+                                                hash);
+        } else {
+            fprintf(stderr, "you must either pass a quick_key or a filename "
+                    "and folder_key\n");
+            return -1;
+        }
+
+        if (api_call == NULL) {
+            fprintf(stderr, "mfconn_create_signed_get failed\n");
+            return -1;
+        }
+
+        http = http_create();
+        retval =
+            http_get_buf(http, api_call, mfapi_decode_common, "upload/instant");
+        http_destroy(http);
+        mfconn_update_secret_key(conn);
+
+        free((void *)api_call);
+
+        if (retval != 127 && retval != 28)
+            break;
+
+        // if there was either a curl timeout or a token error, get a new
+        // token and try again
+        //
+        // on a curl timeout we get a new token because it is likely that we
+        // lost signature synchronization (we don't know whether the server
+        // accepted or rejected the last call)
+        fprintf(stderr, "got error %d - negotiate a new token\n", retval);
+        retval = mfconn_refresh_token(conn);
+        if (retval != 0) {
+            fprintf(stderr, "failed to get a new token\n");
+            break;
+        }
+    }
+
+    return retval;
+}