From 171fd815f2ec09fe59db9d5502ee98e947f273c1 Mon Sep 17 00:00:00 2001 From: josch Date: Mon, 8 Dec 2014 14:12:17 +0100 Subject: [PATCH] allow to change local files - fuse/filecache: * add filecache_upload_patch * allow opening files in modes other than RDONLY - fuse/hashtbl: * add folder_tree_upload_patch - fuse/operations: * allow opening files in modes other than RDONLY * add members to private context which allow tracking of not-yet-uploaded files and files opened for writing and files opened in read-only mode - mfapi/apicalls/upload_patch: * supply x-filename and x-filesize headers - mfapi/apicalls/upload_simple: * do not supply the x-filehash header as it is not used by the server - utils/hash: * hex characters must be lower case for the server - utils/strings: * clean up unused functions strdup_join, strdup_substr, string_chomp - utils/stringv: * complete rewrite with different string vector implementation --- fuse/filecache.c | 219 +++++++++++++++++++++++++++++++-- fuse/filecache.h | 7 +- fuse/hashtbl.c | 46 +++++-- fuse/hashtbl.h | 7 +- fuse/main.c | 7 +- fuse/operations.c | 150 +++++++++++++++------- fuse/operations.h | 15 ++- mfapi/apicalls.h | 7 +- mfapi/apicalls/upload_patch.c | 71 +++++------ mfapi/apicalls/upload_simple.c | 29 ++--- utils/hash.c | 47 +++---- utils/http.c | 1 + utils/strings.c | 126 ------------------- utils/strings.h | 7 -- utils/stringv.c | 200 ++++++++++-------------------- utils/stringv.h | 21 ++-- 16 files changed, 521 insertions(+), 439 deletions(-) diff --git a/fuse/filecache.c b/fuse/filecache.c index bb6406a..a45f1d3 100644 --- a/fuse/filecache.c +++ b/fuse/filecache.c @@ -26,6 +26,9 @@ #endif #include #include +#include +#include +#include #include "../utils/hash.h" #include "../utils/xdelta3.h" @@ -54,25 +57,200 @@ static int filecache_patch_file(const char *filecache_path, uint64_t source_revision, uint64_t target_revision); +int filecache_upload_patch(const char *quickkey, uint64_t local_revision, + const char *filecache_path, mfconn * conn) +{ + FILE *source_fh; + FILE *target_fh; + FILE *patchfile_fh; + unsigned char hash[SHA256_DIGEST_LENGTH]; + char *source_hash; + char *target_hash; + uint64_t target_size; + char *cachefile; + char *newfile; + char *patch_file; + int retval; + char *upload_key; + int status; + int fileerror; + + cachefile = strdup_printf("%s/%s_%d", filecache_path, quickkey, + local_revision); + + source_fh = fopen(cachefile, "r"); + if (source_fh == NULL) { + fprintf(stderr, "cannot open %s\n", cachefile); + free(cachefile); + return -1; + } + free(cachefile); + + newfile = strdup_printf("%s/%s_%d_new", filecache_path, quickkey, + local_revision); + + target_fh = fopen(newfile, "r"); + if (target_fh == NULL) { + fprintf(stderr, "cannot open %s\n", newfile); + free(newfile); + fclose(source_fh); + return -1; + } + free(newfile); + + retval = calc_sha256(source_fh, hash, NULL); + + if (retval != 0) { + fprintf(stderr, "failed to calculate hash\n"); + fclose(source_fh); + fclose(target_fh); + return -1; + } + + source_hash = binary2hex(hash, SHA256_DIGEST_LENGTH); + + retval = calc_sha256(target_fh, hash, &target_size); + + if (retval != 0) { + fprintf(stderr, "failed to calculate hash\n"); + fclose(source_fh); + fclose(target_fh); + return -1; + } + + target_hash = binary2hex(hash, SHA256_DIGEST_LENGTH); + + if (strcmp(source_hash, target_hash) == 0) { + // no changes were done + free(source_hash); + free(target_hash); + return 0; + } + + patch_file = strdup_printf("%s/%s_patch_%d_new", filecache_path, quickkey, + local_revision); + + patchfile_fh = fopen(patch_file, "w"); + if (patchfile_fh == NULL) { + fprintf(stderr, "cannot open %s\n", patch_file); + fclose(source_fh); + fclose(target_fh); + return -1; + } + + rewind(source_fh); + rewind(target_fh); + retval = xdelta3_diff(source_fh, target_fh, patchfile_fh); + fclose(source_fh); + fclose(target_fh); + fclose(patchfile_fh); + + upload_key = NULL; + retval = mfconn_api_upload_patch(conn, quickkey, source_hash, target_hash, + target_size, patch_file, &upload_key); + + if (retval != 0 || upload_key == NULL) { + fprintf(stderr, "mfconn_api_upload_patch failed\n"); + return -1; + } + // poll for completion + for (;;) { + // no need to update the secret key after this + retval = mfconn_api_upload_poll_upload(conn, upload_key, + &status, &fileerror); + if (retval != 0) { + fprintf(stderr, "mfconn_api_upload_poll_upload failed\n"); + return -1; + } + fprintf(stderr, "status: %d, filerror: %d\n", status, fileerror); + if (status == 99) { + fprintf(stderr, "done\n"); + break; + } + sleep(1); + } + + free(upload_key); + + return 0; +} + int filecache_open_file(const char *quickkey, uint64_t local_revision, uint64_t remote_revision, uint64_t fsize, const unsigned char *fhash, - const char *filecache_path, mfconn * conn) + const char *filecache_path, mfconn * conn, mode_t mode, + bool update) { char *cachefile; + char *newfile; int fd; int retval; + const int BUFSIZE = 4096; + char buf[BUFSIZE]; + size_t size; + int source; + int dest; - /* check if the requested file is already in the cache */ - cachefile = - strdup_printf("%s/%s_%d", filecache_path, quickkey, remote_revision); - fd = open(cachefile, O_RDWR); - if (fd > 0) { - /* file existed - return handle */ - free(cachefile); - return fd; + if (update) { + cachefile = strdup_printf("%s/%s_%d", filecache_path, quickkey, + remote_revision); + } else { + cachefile = strdup_printf("%s/%s_%d", filecache_path, quickkey, + local_revision); + } + /* check if the requested file is already in the cache */ + if ((mode & O_ACCMODE) == O_RDONLY) { + // if file is opened in readonly mode, we try to open it directly + fd = open(cachefile, mode); + free(cachefile); + if (fd > 0) { + /* file existed - return handle */ + return fd; + } + // if the file cannot be opened, then it has to be retrieved + } else { + // if file is opened writable then a temporary file has to be opened + // instead to upload a patch if necessary + if (update) { + newfile = strdup_printf("%s/%s_%d_new", filecache_path, quickkey, + remote_revision); + } else { + newfile = strdup_printf("%s/%s_%d_new", filecache_path, quickkey, + local_revision); + } + fd = open(newfile, mode); + if (fd > 0) { + /* file existed - return handle */ + free(newfile); + free(cachefile); + return fd; + } + // the temporary file wasn't available, so we copy it from the + // original + source = open(cachefile, O_RDONLY); + free(cachefile); + if (fd > 0) { + dest = open(newfile, O_WRONLY | O_CREAT, 0644); + while ((size = read(source, buf, BUFSIZE)) > 0) { + write(dest, buf, size); + } + close(source); + close(dest); + fd = open(newfile, mode); + free(newfile); + return fd; + } + free(newfile); + // if the source file cannot be opened for copying, then it has to be + // retrieved + } + + // if no updating is requested and we end up here, then something failed + // but since we must not update, this is a failure + if (!update) { + fprintf(stderr, "no updating but cannot open\n"); + return -1; } - free(cachefile); /* if the file with remote revision didn't exist, then check whether an * old revision exists and in that case update that. @@ -81,7 +259,7 @@ int filecache_open_file(const char *quickkey, uint64_t local_revision, cachefile = strdup_printf("%s/%s_%d", filecache_path, quickkey, local_revision); - fd = open(cachefile, O_RDWR); + fd = open(cachefile, O_RDONLY); free(cachefile); if (fd > 0) { close(fd); @@ -115,7 +293,24 @@ int filecache_open_file(const char *quickkey, uint64_t local_revision, return -1; } - fd = open(cachefile, O_RDWR); + if ((mode & O_ACCMODE) == O_RDONLY) { + // if file is opened in readonly mode, we open it directly + fd = open(cachefile, mode); + } else { + // if file is opened writable then a temporary file has to be opened + // instead to upload a patch if necessary + newfile = strdup_printf("%s/%s_%d_new", filecache_path, quickkey, + remote_revision); + source = open(cachefile, O_RDONLY); + dest = open(newfile, O_WRONLY | O_CREAT, 0644); + while ((size = read(source, buf, BUFSIZE)) > 0) { + write(dest, buf, size); + } + close(source); + close(dest); + fd = open(newfile, mode); + free(newfile); + } free(cachefile); diff --git a/fuse/filecache.h b/fuse/filecache.h index fd49d35..dbdff0d 100644 --- a/fuse/filecache.h +++ b/fuse/filecache.h @@ -23,6 +23,11 @@ int filecache_open_file(const char *quickkey, uint64_t local_revision, uint64_t remote_revision, uint64_t fsize, const unsigned char *fhash, - const char *filecache, mfconn * conn); + const char *filecache, mfconn * conn, + mode_t mode, bool update); + +int filecache_upload_patch(const char *quickkey, + uint64_t local_revision, + const char *filecache, mfconn * conn); #endif diff --git a/fuse/hashtbl.c b/fuse/hashtbl.c index b164304..7c884bf 100644 --- a/fuse/hashtbl.c +++ b/fuse/hashtbl.c @@ -34,24 +34,17 @@ #include #include #include -#include #include -#ifdef __linux -#include -#endif #include #include "hashtbl.h" #include "filecache.h" #include "../mfapi/mfconn.h" #include "../mfapi/file.h" -#include "../mfapi/patch.h" #include "../mfapi/folder.h" #include "../mfapi/apicalls.h" #include "../utils/strings.h" -#include "../utils/http.h" #include "../utils/hash.h" -#include "../utils/xdelta3.h" /* * we build a hashtable using the first three characters of the file or folder @@ -746,7 +739,31 @@ int folder_tree_tmp_open(folder_tree * tree) return fd; } -int folder_tree_open_file(folder_tree * tree, mfconn * conn, const char *path) +int folder_tree_upload_patch(folder_tree * tree, mfconn * conn, + const char *path) +{ + struct h_entry *entry; + int retval; + + entry = folder_tree_lookup_path(tree, conn, path); + /* either file not found or found entry is not a file */ + if (entry == NULL || entry->atime == 0) { + return -ENOENT; + } + + retval = filecache_upload_patch(entry->key, entry->local_revision, + tree->filecache, conn); + + if (retval != 0) { + fprintf(stderr, "filecache_upload_patch failed\n"); + return -1; + } + + return 0; +} + +int folder_tree_open_file(folder_tree * tree, mfconn * conn, const char *path, + mode_t mode, bool update) { struct h_entry *entry; int retval; @@ -762,16 +779,19 @@ int folder_tree_open_file(folder_tree * tree, mfconn * conn, const char *path) retval = filecache_open_file(entry->key, entry->local_revision, entry->remote_revision, entry->fsize, - entry->hash, tree->filecache, conn); + entry->hash, tree->filecache, conn, mode, + update); if (retval == -1) { fprintf(stderr, "filecache_open_file failed\n"); return -1; } - /* make sure that the local_revision is equal to the remote revision - * because filecache_open_file took care of doing any updating if it was - * necessary */ - entry->local_revision = entry->remote_revision; + if (update) { + /* make sure that the local_revision is equal to the remote revision + * because filecache_open_file took care of doing any updating if it + * was necessary */ + entry->local_revision = entry->remote_revision; + } return retval; } diff --git a/fuse/hashtbl.h b/fuse/hashtbl.h index 0648d57..262e6ed 100644 --- a/fuse/hashtbl.h +++ b/fuse/hashtbl.h @@ -23,6 +23,7 @@ #include #include #include +#include #include "../mfapi/mfconn.h" @@ -72,8 +73,12 @@ bool folder_tree_path_is_file(folder_tree * tree, mfconn * conn, const char *path); int folder_tree_open_file(folder_tree * tree, mfconn * conn, - const char *path); + const char *path, mode_t mode, + bool update); int folder_tree_tmp_open(folder_tree * tree); +int folder_tree_upload_patch(folder_tree * tree, mfconn * conn, + const char *path); + #endif diff --git a/fuse/main.c b/fuse/main.c index 926c796..d6dfd2c 100644 --- a/fuse/main.c +++ b/fuse/main.c @@ -41,6 +41,7 @@ #include "hashtbl.h" #include "operations.h" #include "../utils/strings.h" +#include "../utils/stringv.h" enum { KEY_HELP, @@ -410,8 +411,8 @@ int main(int argc, char *argv[]) connect_mf(&options, ctx); - ctx->tmpfiles = NULL; - ctx->num_tmpfiles = 0; + ctx->sv_writefiles = stringv_alloc(); + ctx->sv_readonlyfiles = stringv_alloc(); ret = fuse_main(argc, argv, &mediafirefs_oper, ctx); @@ -423,6 +424,8 @@ int main(int argc, char *argv[]) free(ctx->configfile); free(ctx->dircache); free(ctx->filecache); + stringv_free(ctx->sv_writefiles); + stringv_free(ctx->sv_readonlyfiles); free(ctx); return ret; diff --git a/fuse/operations.c b/fuse/operations.c index 97e7f16..a605666 100644 --- a/fuse/operations.c +++ b/fuse/operations.c @@ -95,12 +95,11 @@ struct mediafirefs_openfile { // to fread and fwrite from/to the file int fd; + char *path; // whether or not a patch has to be uploaded when closing bool is_readonly; // whether or not to do a new file upload when closing - // if this is NULL then the file already existed remotely - // otherwise a new upload has to be done to the given remote path - char *remote_path; + bool is_local; }; int mediafirefs_getattr(const char *path, struct stat *stbuf) @@ -115,27 +114,21 @@ int mediafirefs_getattr(const char *path, struct stat *stbuf) */ struct mediafirefs_context_private *ctx; int retval; - size_t i; ctx = fuse_get_context()->private_data; folder_tree_update(ctx->tree, ctx->conn, false); retval = folder_tree_getattr(ctx->tree, ctx->conn, path, stbuf); - if (retval != 0) { - for (i = 0; i < ctx->num_tmpfiles; i++) { - if (strcmp(ctx->tmpfiles[i], path) == 0) { - stbuf->st_uid = geteuid(); - stbuf->st_gid = getegid(); - stbuf->st_ctime = 0; - stbuf->st_mtime = 0; - stbuf->st_mode = S_IFREG | 0666; - stbuf->st_nlink = 1; - stbuf->st_atime = 0; - stbuf->st_size = 0; - retval = 0; - break; - } - } + if (retval != 0 && stringv_mem(ctx->sv_writefiles, path)) { + stbuf->st_uid = geteuid(); + stbuf->st_gid = getegid(); + stbuf->st_ctime = 0; + stbuf->st_mtime = 0; + stbuf->st_mode = S_IFREG | 0666; + stbuf->st_nlink = 1; + stbuf->st_atime = 0; + stbuf->st_size = 0; + retval = 0; } return retval; @@ -308,20 +301,60 @@ int mediafirefs_unlink(const char *path) return 0; } +/* + * the following restrictions apply: + * 1. a file can be opened in read-only mode more than once at a time + * 2. a file can only be be opened in write-only or read-write mode if it is + * not open for writing at the same time + * 3. a file that is only local and has not been uploaded yet cannot be read + * from + * 4. a file that has opened in any way will not be updated to its latest + * remote revision until all its opened handles are closed + * + * Point 2 is enforced by a lookup in the writefiles string vector. If the + * path is in there then it was either just created locally or opened with + * write-only or read-write. In both cases it must not be opened for + * writing again. + * + * Point 3 is enforced by the lookup in the hashtable failing. + * + * Point 4 is enforced by checking if the current path is in the writefiles or + * readonlyfiles string vector and if yes, no updating will be done. + */ int mediafirefs_open(const char *path, struct fuse_file_info *file_info) { int fd; + bool is_open; struct mediafirefs_openfile *openfile; struct mediafirefs_context_private *ctx; ctx = fuse_get_context()->private_data; - if ((file_info->flags & O_ACCMODE) != O_RDONLY) { - fprintf(stderr, "can only open read-only\n"); + /* if file is not opened read-only, check if it was already opened in a + * not read-only mode and abort if yes */ + if ((file_info->flags & O_ACCMODE) != O_RDONLY + && stringv_mem(ctx->sv_writefiles, path)) { + fprintf(stderr, "file %s was already opened for writing\n", path); return -EACCES; } - fd = folder_tree_open_file(ctx->tree, ctx->conn, path); + is_open = false; + // check if the file was already opened + // check read-only files first + if (stringv_mem(ctx->sv_readonlyfiles, path)) { + is_open = true; + } + // check writable files only if the file was + // - not yet found in the read-only files + // - the file is opened in read-only mode (because otherwise the + // writable files were already searched above without failing) + if (!is_open && (file_info->flags & O_ACCMODE) == O_RDONLY + && stringv_mem(ctx->sv_writefiles, path)) { + is_open = true; + } + + fd = folder_tree_open_file(ctx->tree, ctx->conn, path, file_info->flags, + !is_open); if (fd < 0) { fprintf(stderr, "folder_tree_file_open unsuccessful\n"); return fd; @@ -329,8 +362,19 @@ int mediafirefs_open(const char *path, struct fuse_file_info *file_info) openfile = malloc(sizeof(struct mediafirefs_openfile)); openfile->fd = fd; - openfile->is_readonly = true; - openfile->remote_path = NULL; + openfile->is_local = false; + openfile->path = strdup(path); + + if ((file_info->flags & O_ACCMODE) == O_RDONLY) { + openfile->is_readonly = true; + // add to readonlyfiles + stringv_add(ctx->sv_readonlyfiles, path); + } else { + openfile->is_readonly = false; + // add to writefiles + stringv_add(ctx->sv_writefiles, path); + } + file_info->fh = (uint64_t) openfile; return 0; } @@ -359,14 +403,13 @@ int mediafirefs_create(const char *path, mode_t mode, openfile = malloc(sizeof(struct mediafirefs_openfile)); openfile->fd = fd; + openfile->is_local = true; openfile->is_readonly = false; - openfile->remote_path = strdup(path); + openfile->path = strdup(path); file_info->fh = (uint64_t) openfile; - // add to tmpfiles - ctx->num_tmpfiles++; - ctx->tmpfiles = realloc(ctx->tmpfiles, sizeof(char *) * ctx->num_tmpfiles); - ctx->tmpfiles[ctx->num_tmpfiles - 1] = openfile->remote_path; + // add to writefiles + stringv_add(ctx->sv_writefiles, path); return 0; } @@ -411,19 +454,38 @@ int mediafirefs_release(const char *path, struct fuse_file_info *file_info) openfile = (struct mediafirefs_openfile *)file_info->fh; - // if the file was opened readonly, then no new updates have to be - // uploaded + // if file was opened as readonly then it just has to be closed if (openfile->is_readonly) { + // remove this entry from readonlyfiles + if (stringv_del(ctx->sv_readonlyfiles, openfile->path) != 0) { + fprintf(stderr, "FATAL: readonly entry %s not found\n", + openfile->path); + exit(1); + } + close(openfile->fd); + free(openfile->path); free(openfile); return 0; } + // if the file is not readonly, its entry in writefiles has to be removed + if (stringv_del(ctx->sv_writefiles, openfile->path) != 0) { + fprintf(stderr, "FATAL: writefiles entry %s not found\n", + openfile->path); + exit(1); + } + if (stringv_mem(ctx->sv_writefiles, openfile->path) != 0) { + fprintf(stderr, + "FATAL: writefiles entry %s was found more than once\n", + openfile->path); + exit(1); + } // if the file only exists locally, an initial upload has to be done - if (openfile->remote_path != NULL) { + if (openfile->is_local) { // pass a copy because dirname and basename may modify their argument - temp1 = strdup(openfile->remote_path); + temp1 = strdup(openfile->path); file_name = basename(temp1); - temp2 = strdup(openfile->remote_path); + temp2 = strdup(openfile->path); dir_name = dirname(temp2); fh = fdopen(openfile->fd, "r"); @@ -438,7 +500,7 @@ int mediafirefs_release(const char *path, struct fuse_file_info *file_info) fclose(fh); free(temp1); free(temp2); - free(openfile->remote_path); + free(openfile->path); free(openfile); if (retval != 0 || upload_key == NULL) { @@ -462,11 +524,6 @@ int mediafirefs_release(const char *path, struct fuse_file_info *file_info) sleep(1); } - // remove from tmpfiles - ctx->num_tmpfiles--; - ctx->tmpfiles = realloc(ctx->tmpfiles, - sizeof(char *) * ctx->num_tmpfiles); - free(upload_key); folder_tree_update(ctx->tree, ctx->conn, true); @@ -475,9 +532,18 @@ int mediafirefs_release(const char *path, struct fuse_file_info *file_info) // the file was not opened readonly and also existed on the remote // thus, we have to check whether any changes were made and if yes, upload // a patch - // FIXME: not implemented yet + close(openfile->fd); + + retval = folder_tree_upload_patch(ctx->tree, ctx->conn, openfile->path); + free(openfile->path); free(openfile); + + if (retval != 0) { + fprintf(stderr, "folder_tree_upload_patch failed\n"); + return -EACCES; + } + folder_tree_update(ctx->tree, ctx->conn, true); - return -ENOSYS; + return 0; } diff --git a/fuse/operations.h b/fuse/operations.h index df590e4..2e17d2b 100644 --- a/fuse/operations.h +++ b/fuse/operations.h @@ -24,6 +24,7 @@ #include #include "../mfapi/mfconn.h" #include "hashtbl.h" +#include "../utils/stringv.h" struct mediafirefs_context_private { mfconn *conn; @@ -31,12 +32,14 @@ struct mediafirefs_context_private { char *configfile; char *dircache; char *filecache; - /* stores all currently open temporary files which are to be uploaded when - * they are closed. - * we use a normal array because the number of open temporary files will - * never be very high but is limited by the number of threads */ - char **tmpfiles; - size_t num_tmpfiles; + /* stores: + * - all currently open temporary files which are to be uploaded when + * they are closed. + * - all files that are opened for writing + */ + stringv *sv_writefiles; + /* stores all files that have been opened for reading only */ + stringv *sv_readonlyfiles; }; int mediafirefs_getattr(const char *path, struct stat *stbuf); diff --git a/mfapi/apicalls.h b/mfapi/apicalls.h index 7269387..dd07616 100644 --- a/mfapi/apicalls.h +++ b/mfapi/apicalls.h @@ -107,8 +107,11 @@ int mfconn_api_upload_simple(mfconn * conn, const char *folderkey, char **upload_key); int mfconn_api_upload_patch(mfconn * conn, const char *quickkey, - FILE * source_fh, FILE * target_fh, - FILE * patch_fh, char **upload_key); + const char *source_hash, + const char *target_hash, + uint64_t target_size, + const char *patch_path, + char **upload_key); int mfconn_api_upload_poll_upload(mfconn * conn, const char *upload_key, diff --git a/mfapi/apicalls/upload_patch.c b/mfapi/apicalls/upload_patch.c index c9a290d..77d8fc2 100644 --- a/mfapi/apicalls/upload_patch.c +++ b/mfapi/apicalls/upload_patch.c @@ -26,10 +26,11 @@ #include #include #include -#include +#include +#include #include "../../utils/http.h" -#include "../../utils/hash.h" +#include "../../utils/strings.h" #include "../mfconn.h" #include "../apicalls.h" // IWYU pragma: keep @@ -37,52 +38,47 @@ static int _decode_upload_patch(mfhttp * conn, void *data); int mfconn_api_upload_patch(mfconn * conn, const char *quickkey, - FILE * source_fh, FILE * target_fh, - FILE * patch_fh, char **upload_key) + const char *source_hash, const char *target_hash, + uint64_t target_size, + const char *patch_path, char **upload_key) { const char *api_call; int retval; mfhttp *http; - uint64_t target_size; int i; - unsigned char hash[SHA256_DIGEST_LENGTH]; - char *source_hash; - char *target_hash; + FILE *patch_fh; + struct curl_slist *custom_headers = NULL; + char *tmpheader; + uint64_t patch_size; + struct stat file_info; if (conn == NULL) return -1; - if (source_fh == NULL) - return -1; - - if (target_fh == NULL) - return -1; - - rewind(source_fh); - retval = calc_sha256(source_fh, hash, NULL); - + memset(&file_info, 0, sizeof(file_info)); + retval = stat(patch_path, &file_info); if (retval != 0) { - fprintf(stderr, "failed to calculate hash\n"); + fprintf(stderr, "stat failed\n"); return -1; } - source_hash = binary2hex(hash, SHA256_DIGEST_LENGTH); - - rewind(target_fh); - retval = calc_sha256(target_fh, hash, &target_size); - - if (retval != 0) { - fprintf(stderr, "failed to calculate hash\n"); - return -1; - } - - target_hash = binary2hex(hash, SHA256_DIGEST_LENGTH); + patch_size = file_info.st_size; for (i = 0; i < mfconn_get_max_num_retries(conn); i++) { if (*upload_key != NULL) { free(*upload_key); *upload_key = NULL; } + if (custom_headers != NULL) { + curl_slist_free_all(custom_headers); + custom_headers = NULL; + } + + patch_fh = fopen(patch_path, "r"); + if (patch_fh == NULL) { + fprintf(stderr, "cannot open %s\n", patch_path); + return -1; + } api_call = mfconn_create_signed_get(conn, 0, "upload/patch.php", @@ -90,19 +86,23 @@ mfconn_api_upload_patch(mfconn * conn, const char *quickkey, "&source_hash=%s" "&target_hash=%s" "&target_size=%" PRIu64 - "&quick_key=%s", source_hash, + "&quickkey=%s", source_hash, target_hash, target_size, quickkey); - // make sure that we are at the beginning of the file - rewind(patch_fh); + custom_headers = curl_slist_append(custom_headers, + "x-filename: dummy.patch"); + tmpheader = strdup_printf("x-filesize: %" PRIu64, patch_size); + custom_headers = curl_slist_append(custom_headers, tmpheader); + free(tmpheader); http = http_create(); - retval = http_post_file(http, api_call, patch_fh, NULL, target_size, - _decode_upload_patch, upload_key); + retval = http_post_file(http, api_call, patch_fh, &custom_headers, + patch_size, _decode_upload_patch, upload_key); http_destroy(http); mfconn_update_secret_key(conn); + fclose(patch_fh); free((void *)api_call); if (retval != 127 && retval != 28) @@ -122,9 +122,6 @@ mfconn_api_upload_patch(mfconn * conn, const char *quickkey, } } - free(source_hash); - free(target_hash); - return retval; } diff --git a/mfapi/apicalls/upload_simple.c b/mfapi/apicalls/upload_simple.c index 5070538..4689ae4 100644 --- a/mfapi/apicalls/upload_simple.c +++ b/mfapi/apicalls/upload_simple.c @@ -26,11 +26,9 @@ #include #include #include -#include #include #include "../../utils/http.h" -#include "../../utils/hash.h" #include "../../utils/strings.h" #include "../mfconn.h" #include "../apicalls.h" // IWYU pragma: keep @@ -44,8 +42,7 @@ mfconn_api_upload_simple(mfconn * conn, const char *folderkey, const char *api_call; int retval; mfhttp *http; - unsigned char hash[SHA256_DIGEST_LENGTH]; - char *file_hash; + long l_file_size; uint64_t file_size; int i; struct curl_slist *custom_headers = NULL; @@ -57,18 +54,21 @@ mfconn_api_upload_simple(mfconn * conn, const char *folderkey, if (fh == NULL) return -1; - // make sure that we are at the beginning of the file - rewind(fh); - - // calculate hash - retval = calc_sha256(fh, hash, &file_size); - + retval = fseek(fh, 0, SEEK_END); if (retval != 0) { - fprintf(stderr, "failed to calculate hash\n"); + fprintf(stderr, "fseek failed\n"); return -1; } - file_hash = binary2hex(hash, SHA256_DIGEST_LENGTH); + l_file_size = ftell(fh); + if (l_file_size == -1) { + fprintf(stderr, "ftell failed\n"); + return -1; + } + file_size = l_file_size; + + // make sure that we are at the beginning of the file + rewind(fh); for (i = 0; i < mfconn_get_max_num_retries(conn); i++) { if (*upload_key != NULL) { @@ -102,9 +102,6 @@ mfconn_api_upload_simple(mfconn * conn, const char *folderkey, tmpheader = strdup_printf("x-filesize: %" PRIu64, file_size); custom_headers = curl_slist_append(custom_headers, tmpheader); free(tmpheader); - tmpheader = strdup_printf("x-filehash: %s", file_hash); - custom_headers = curl_slist_append(custom_headers, tmpheader); - free(tmpheader); http = http_create(); retval = http_post_file(http, api_call, fh, &custom_headers, file_size, @@ -135,8 +132,6 @@ mfconn_api_upload_simple(mfconn * conn, const char *folderkey, } } - free(file_hash); - return retval; } diff --git a/utils/hash.c b/utils/hash.c index b31aae8..dc00ee7 100644 --- a/utils/hash.c +++ b/utils/hash.c @@ -51,30 +51,33 @@ static unsigned char base36_decoding_table[] = { /* * table to convert from a byte into the two hexadecimal digits representing * it + * + * the hex chars have to be lower case until server-side is fixed to accept + * both cases */ static char base16_encoding_table[][2] = { - "00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "0A", "0B", - "0C", "0D", "0E", "0F", "10", "11", "12", "13", "14", "15", "16", "17", - "18", "19", "1A", "1B", "1C", "1D", "1E", "1F", "20", "21", "22", "23", - "24", "25", "26", "27", "28", "29", "2A", "2B", "2C", "2D", "2E", "2F", - "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "3A", "3B", - "3C", "3D", "3E", "3F", "40", "41", "42", "43", "44", "45", "46", "47", - "48", "49", "4A", "4B", "4C", "4D", "4E", "4F", "50", "51", "52", "53", - "54", "55", "56", "57", "58", "59", "5A", "5B", "5C", "5D", "5E", "5F", - "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "6A", "6B", - "6C", "6D", "6E", "6F", "70", "71", "72", "73", "74", "75", "76", "77", - "78", "79", "7A", "7B", "7C", "7D", "7E", "7F", "80", "81", "82", "83", - "84", "85", "86", "87", "88", "89", "8A", "8B", "8C", "8D", "8E", "8F", - "90", "91", "92", "93", "94", "95", "96", "97", "98", "99", "9A", "9B", - "9C", "9D", "9E", "9F", "A0", "A1", "A2", "A3", "A4", "A5", "A6", "A7", - "A8", "A9", "AA", "AB", "AC", "AD", "AE", "AF", "B0", "B1", "B2", "B3", - "B4", "B5", "B6", "B7", "B8", "B9", "BA", "BB", "BC", "BD", "BE", "BF", - "C0", "C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9", "CA", "CB", - "CC", "CD", "CE", "CF", "D0", "D1", "D2", "D3", "D4", "D5", "D6", "D7", - "D8", "D9", "DA", "DB", "DC", "DD", "DE", "DF", "E0", "E1", "E2", "E3", - "E4", "E5", "E6", "E7", "E8", "E9", "EA", "EB", "EC", "ED", "EE", "EF", - "F0", "F1", "F2", "F3", "F4", "F5", "F6", "F7", "F8", "F9", "FA", "FB", - "FC", "FD", "FE", "FF" + "00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "0a", "0b", + "0c", "0d", "0e", "0f", "10", "11", "12", "13", "14", "15", "16", "17", + "18", "19", "1a", "1b", "1c", "1d", "1e", "1f", "20", "21", "22", "23", + "24", "25", "26", "27", "28", "29", "2a", "2b", "2c", "2d", "2e", "2f", + "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "3a", "3b", + "3c", "3d", "3e", "3f", "40", "41", "42", "43", "44", "45", "46", "47", + "48", "49", "4a", "4b", "4c", "4d", "4e", "4f", "50", "51", "52", "53", + "54", "55", "56", "57", "58", "59", "5a", "5b", "5c", "5d", "5e", "5f", + "60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "6a", "6b", + "6c", "6d", "6e", "6f", "70", "71", "72", "73", "74", "75", "76", "77", + "78", "79", "7a", "7b", "7c", "7d", "7e", "7f", "80", "81", "82", "83", + "84", "85", "86", "87", "88", "89", "8a", "8b", "8c", "8d", "8e", "8f", + "90", "91", "92", "93", "94", "95", "96", "97", "98", "99", "9a", "9b", + "9c", "9d", "9e", "9f", "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", + "a8", "a9", "aa", "ab", "ac", "ad", "ae", "af", "b0", "b1", "b2", "b3", + "b4", "b5", "b6", "b7", "b8", "b9", "ba", "bb", "bc", "bd", "be", "bf", + "c0", "c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "ca", "cb", + "cc", "cd", "ce", "cf", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", + "d8", "d9", "da", "db", "dc", "dd", "de", "df", "e0", "e1", "e2", "e3", + "e4", "e5", "e6", "e7", "e8", "e9", "ea", "eb", "ec", "ed", "ee", "ef", + "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "fa", "fb", + "fc", "fd", "fe", "ff" }; int calc_md5(FILE * file, unsigned char *hash) diff --git a/utils/http.c b/utils/http.c index 0049497..fdde660 100644 --- a/utils/http.c +++ b/utils/http.c @@ -289,6 +289,7 @@ http_post_file(mfhttp * conn, const char *url, FILE * fh, if (custom_headers == NULL) { custom_headers = &fallback_headers; + *custom_headers = NULL; } // when using POST, curl implicitly sets // Content-Type: application/x-www-form-urlencoded diff --git a/utils/strings.c b/utils/strings.c index dae20a6..035a58f 100644 --- a/utils/strings.c +++ b/utils/strings.c @@ -59,132 +59,6 @@ char *strdup_printf(char *fmt, ...) return ret_str; } -char *strdup_join(char *string1, char *string2) -{ - char *new_string; - size_t string1_len; - size_t string2_len; - - if (string1 == NULL || string2 == NULL) - return NULL; - - string1_len = strlen(string1); - string2_len = strlen(string2); - - new_string = (char *)malloc(string1_len + string2_len + 1); - - strncpy(new_string, string1, string1_len); - strncat(new_string, string2, string2_len); - - new_string[string1_len + string2_len] = '\0'; - - return new_string; -} - -char *strdup_subst(char *string, char *token, char *subst, - unsigned int max) -{ - size_t string_len = 0; - size_t subst_len = 0; - size_t token_len = 0; - size_t total_len = 0; - size_t copy_len = 0; - size_t token_count; - char *str_new = NULL; - char **vectors; - char **rewind; - - if (string == NULL) - return NULL; - - if (token == NULL || subst == NULL) - return NULL; - - string_len = strlen(string); - token_len = strlen(token); - - // return on conditions that we could never handle. - if (token_len > string_len) - return NULL; - if (token[0] == '\0') - return NULL; - - vectors = stringv_find(string, token, max); - if (vectors == NULL) - return NULL; - rewind = vectors; - - // count the number of tokens found in the string - token_count = stringv_len(vectors); - - if (token_count > max) - token_count = max; - - // start with the original string size; - total_len = string_len; - - // subtract the total number of token chars to be removed - total_len -= (token_len * token_count); - - // add back the total number of subst chars to be inserted - total_len += (subst_len * token_count); - - str_new = (char *)malloc((total_len + 1) * sizeof(char)); - str_new[0] = '\0'; - - while (*vectors != NULL) { - // calculate distance to the next token from current position - copy_len = *vectors - string; - - if (copy_len > 0) { - strncat(str_new, string, copy_len); - string += copy_len; - - // when total_len == 0 the process is complete - total_len -= copy_len; - } - - strncat(str_new, token, token_len); - - // when total_len == 0 the process is complete - total_len -= token_len; - - vectors++; - } - - // might have one more copy operation to complete - if (total_len > 0) { - strcat(str_new, string); - } - // todo: can't free vectors directly cuz it was incremented - free(rewind); - - return str_new; -} - -void string_chomp(char *string) -{ - size_t len; - char *pos; - - if (string == NULL) - return; - - len = strlen(string); - - if (len == 0) - return; - - pos = &string[len - 1]; - - while (isspace((int)*pos) != 0) { - *pos = '\0'; - pos--; - } - - return; -} - char *string_line_from_stdin(bool hide) { char *line = NULL; diff --git a/utils/strings.h b/utils/strings.h index 2444e4f..069f72f 100644 --- a/utils/strings.h +++ b/utils/strings.h @@ -24,13 +24,6 @@ char *strdup_printf(char *fmt, ...); -char *strdup_join(char *string1, char *string2); - -char *strdup_subst(char *string, char *str_old, char *str_new, - unsigned int max); - -void string_chomp(char *string); - char *string_line_from_stdin(bool hide); #endif diff --git a/utils/stringv.c b/utils/stringv.c index d7d116c..fadb5ce 100644 --- a/utils/stringv.c +++ b/utils/stringv.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013 Bryan Christ + * Copyright (C) 2014 Johannes Schauer * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2, as published by @@ -18,166 +18,90 @@ #define _POSIX_C_SOURCE 200809L // for strdup -#include +#include #include +#include #include +#include #include "stringv.h" -size_t stringv_len(char **array) +struct stringv { + size_t len; + char **array; +}; + +stringv *stringv_alloc(void) { - size_t count = 0; - char **pos; + stringv *sv; - if (array == NULL) - return 0; - - pos = array; - while (pos[0] != NULL) { - pos++; - count++; - } - - return count; + sv = (stringv *) calloc(1, sizeof(stringv)); + sv->len = 0; + sv->array = NULL; + return sv; } -void stringv_free(char **array, int b_free) +void stringv_free(stringv * sv) { - char **pos; + size_t i; - if (array == NULL) - return; - - pos = array; - - while ((*pos) != NULL) { - free(*pos); - ++pos; + for (i = 0; i < sv->len; i++) { + free(sv->array[i]); } - - if (b_free == STRINGV_FREE_ALL) - free(array); - - return; + free(sv->array); + free(sv); } -char **stringv_copy(char **array) +bool stringv_mem(stringv * sv, const char *e) { - uint32_t array_len; - char **array_pos; + size_t i; - char **dup_array; - char **dup_pos; - - if (array == NULL) - return (char **)NULL; - - array_pos = array; - - array_len = stringv_len(array); - - if (array_len > UINT32_MAX - 1) - array_len = UINT32_MAX - 1; - - dup_array = (char **)calloc(array_len, sizeof(char *)); - dup_pos = dup_array; - - while ((*array_pos) != NULL) { - *dup_pos = strdup((const char *)*array_pos); - - array_pos++; - dup_pos++; + for (i = 0; i < sv->len; i++) { + if (strcmp(sv->array[i], e) == 0) + return true; } - - return dup_array; + return false; } -char **stringv_find(char *string, char *token, int limit) +int stringv_add(stringv * sv, const char *e) { - char **results = NULL; - char *pos = NULL; - int count = 0; + sv->len++; + sv->array = realloc(sv->array, sizeof(char *) * sv->len); + if (sv->array == NULL) { + fprintf(stderr, "failed to realloc\n"); + return -1; + } + sv->array[sv->len - 1] = strdup(e); + if (sv->array[sv->len - 1] == NULL) { + fprintf(stderr, "failed to strdup\n"); + return -1; + } + return 0; +} - if (string == NULL) - return (char **)NULL; - if (token == NULL) - return (char **)NULL; - if (limit == 0) - return (char **)NULL; +int stringv_del(stringv * sv, const char *e) +{ + size_t i; - pos = string; - - if (strlen(token) > strlen(string)) - return (char **)NULL; - - while (count != limit) { - pos = strstr(pos, token); - if (pos == NULL) + for (i = 0; i < sv->len; i++) { + if (strcmp(sv->array[i], e) == 0) { + free(sv->array[i]); break; - - count++; - results = - (char **)realloc((void *)results, sizeof(char *) * count + 1); - - results[count - 1] = pos; + } } - - if (count == 0) - return (char **)NULL; - - results[count] = (char *)NULL; - - return results; -} - -char **stringv_split(char *string, char *token, int limit) -{ - char **results = NULL; - char *curr = NULL; - char *next = NULL; - int count = 0; - unsigned int len; - size_t copy_len = 0; - - if (string == NULL) - return (char **)NULL; - if (token == NULL) - return (char **)NULL; - if (limit == 0) - return (char **)NULL; - - len = strlen(string); - if (strlen(token) > len) - return (char **)NULL; - - curr = string; - - do { - // alloc space for current item plus NULL vector terminator - results = (char **)realloc(results, sizeof(char *) * (count + 2)); - - // find the next occurrence - next = strstr(curr, token); - - if (next != NULL) - copy_len = next - curr; - else - copy_len = strlen(curr); - - results[count] = (char *)calloc(copy_len + 1, sizeof(char)); - memcpy(results[count], curr, copy_len); - - count++; - - if (next == NULL) - break; - - curr = next; - curr++; + if (i == sv->len) { + fprintf(stderr, "not found\n"); + return -1; } - while (count < limit); - - results[count] = NULL; - - return results; + // shift the remaining entries one place to the left + memmove(sv->array + i, sv->array + i + 1, + sizeof(char *) * (sv->len - i - 1)); + sv->len--; + if (sv->len == 0) { + free(sv->array); + sv->array = NULL; + } else { + sv->array = realloc(sv->array, sizeof(char *) * sv->len); + } + return 0; } diff --git a/utils/stringv.h b/utils/stringv.h index 2f929b4..ef190f4 100644 --- a/utils/stringv.h +++ b/utils/stringv.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013 Bryan Christ + * Copyright (C) 2014 Johannes Schauer * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2, as published by @@ -19,23 +19,18 @@ #ifndef _STRING_V_H_ #define _STRING_V_H_ -#include +#include -#define STRINGV_FREE_ALL 1 +typedef struct stringv stringv; -// count number of strings in a NULL in a string vector -size_t stringv_len(char **array); +stringv *stringv_alloc(void); -// free all of the strings in a vector and optionally the vector pointer -void stringv_free(char **array, int b_free); +void stringv_free(stringv * sv); -// deep copy of string vector. returns a new vector pointer -char **stringv_copy(char **array); +bool stringv_mem(stringv * sv, const char *e); -// returns a NULL terminated vector array to every location 'token' is found -char **stringv_find(char *string, char *token, int limit); +int stringv_add(stringv * sv, const char *e); -// returns a NULL terminated vector array of items delimited by 'token' -char **stringv_split(char *string, char *token, int limit); +int stringv_del(stringv * sv, const char *e); #endif