From 3db5ce5c42f9fd44ee5c67cecead489851d9daef Mon Sep 17 00:00:00 2001 From: josch Date: Mon, 27 Oct 2014 14:17:03 +0100 Subject: [PATCH] implement preliminary support to read files * no writing yet * no delta updates yet - new versions will be retrieved fully --- fuse/hashtbl.c | 140 ++++++++++++++++++++++++++++++++++++++++- fuse/hashtbl.h | 4 ++ fuse/main.c | 46 +++++++++----- mfapi/file.c | 55 ---------------- mfapi/file.h | 5 +- mfshell/commands/get.c | 37 ++++++++++- tests/valgrind_fuse.sh | 4 ++ 7 files changed, 215 insertions(+), 76 deletions(-) diff --git a/fuse/hashtbl.c b/fuse/hashtbl.c index b05f1da..0e70812 100644 --- a/fuse/hashtbl.c +++ b/fuse/hashtbl.c @@ -30,12 +30,17 @@ #include #include #include +#include +#include +#include #include "hashtbl.h" #include "../mfapi/mfconn.h" #include "../mfapi/file.h" #include "../mfapi/folder.h" #include "../mfapi/apicalls.h" +#include "../utils/strings.h" +#include "../utils/http.h" /* * we build a hashtable using the first three characters of the file or folder @@ -143,6 +148,7 @@ struct h_entry { struct folder_tree { uint64_t revision; + char *filecache; uint64_t bucket_lens[NUM_BUCKETS]; struct h_entry **buckets[NUM_BUCKETS]; struct h_entry root; @@ -321,6 +327,7 @@ folder_tree *folder_tree_load(FILE * stream) struct h_entry *tmp_entry; struct h_entry *parent; int bucket_id; + char *homedir; /* read and check the first four bytes */ ret = fread(tmp_buffer, 1, 4, stream); @@ -425,15 +432,28 @@ folder_tree *folder_tree_load(FILE * stream) free(ordered_entries); + /* set file cache */ + if ((homedir = getenv("HOME")) == NULL) { + homedir = getpwuid(getuid())->pw_dir; + } + tree->filecache = strdup_printf("%s/.mediafire-tools/cache", homedir); + return tree; } folder_tree *folder_tree_create(void) { folder_tree *tree; + char *homedir; tree = (folder_tree *) calloc(1, sizeof(folder_tree)); + /* set file cache */ + if ((homedir = getenv("HOME")) == NULL) { + homedir = getpwuid(getuid())->pw_dir; + } + tree->filecache = strdup_printf("%s/.mediafire-tools/cache", homedir); + return tree; } @@ -459,6 +479,7 @@ static void folder_tree_free_entries(folder_tree * tree) void folder_tree_destroy(folder_tree * tree) { folder_tree_free_entries(tree); + free(tree->filecache); free(tree); } @@ -729,6 +750,103 @@ int folder_tree_readdir(folder_tree * tree, mfconn * conn, const char *path, return 0; } +int folder_tree_open_file(folder_tree * tree, mfconn * conn, const char *path) +{ + struct h_entry *entry; + char *cachefile; + int fd; + struct stat file_info; + uint64_t bytes_read; + const char *url; + mffile *file; + int retval; + mfhttp *http; + + entry = folder_tree_lookup_path(tree, conn, path); + + /* either file not found or found entry is not a file */ + if (entry == NULL || entry->atime == 0) { + return -ENOENT; + } + + /* TODO: check entry->needs_update + * check if local size is equal to remote size + * check if local version exists and needs updating */ + + /* check if the requested file is already in the cache */ + cachefile = + strdup_printf("%s/%s_%d", tree->filecache, entry->key, + entry->revision); + fd = open(cachefile, O_RDWR); + if (fd > 0) { + /* file existed - return handle */ + free(cachefile); + return fd; + } + + /* download the file */ + file = file_alloc(); + retval = mfconn_api_file_get_links(conn, file, (char *)entry->key); + mfconn_update_secret_key(conn); + + if (retval == -1) { + fprintf(stderr, "mfconn_api_file_get_links failed\n"); + free(cachefile); + file_free(file); + return -1; + } + + url = file_get_direct_link(file); + + if (url == NULL) { + fprintf(stderr, "file_get_direct_link failed\n"); + free(cachefile); + file_free(file); + return -1; + } + + http = http_create(); + retval = http_get_file(http, url, cachefile); + http_destroy(http); + + if (retval != 0) { + fprintf(stderr, "download failed\n"); + free(cachefile); + file_free(file); + return -1; + } + + memset(&file_info, 0, sizeof(file_info)); + retval = stat(cachefile, &file_info); + + if (retval != 0) { + fprintf(stderr, "stat failed\n"); + free(cachefile); + file_free(file); + return -1; + } + + bytes_read = file_info.st_size; + + if (bytes_read != entry->fsize) { + fprintf(stderr, + "expected %" PRIu64 " bytes but got %" PRIu64 " bytes\n", + entry->fsize, bytes_read); + free(cachefile); + file_free(file); + return -1; + } + + file_free(file); + + fd = open(cachefile, O_RDWR); + + free(cachefile); + + /* return the file handle */ + return fd; +} + static bool folder_tree_is_root(struct h_entry *entry) { if (entry == NULL) { @@ -928,6 +1046,9 @@ static struct h_entry *folder_tree_add_file(folder_tree * tree, mffile * file, /* if the revisions of the old and new entry differ, we have to * update its content from the remote the next time the file is accessed + * + * we also have to fetch the remote content if the file was only just + * added and did not exist before */ if ((old_entry != NULL && old_revision < new_entry->revision) || old_entry == NULL) { @@ -1230,6 +1351,7 @@ static int folder_tree_update_file_info(folder_tree * tree, mfconn * conn, mffile *file; int retval; struct h_entry *parent; + struct h_entry *new_entry; file = file_alloc(); @@ -1247,11 +1369,19 @@ static int folder_tree_update_file_info(folder_tree * tree, mfconn * conn, parent = folder_tree_lookup_key(tree, file_get_parent(file)); if (parent == NULL) { - fprintf(stderr, "file_tree_lookup_key failed\n"); - return -1; + fprintf(stderr, "the parent of %s does not exist yet - retrieve it\n", + key); + folder_tree_update_folder_info(tree, conn, file_get_parent(file)); } - folder_tree_add_file(tree, file, parent); + /* store the updated entry in the hashtable */ + new_entry = folder_tree_add_file(tree, file, parent); + + if (new_entry == NULL) { + fprintf(stderr, "folder_tree_add_file failed\n"); + file_free(file); + return -1; + } file_free(file); @@ -1368,6 +1498,7 @@ void folder_tree_update(folder_tree * tree, mfconn * conn) mfconn_update_secret_key(conn); if (retval != 0) { fprintf(stderr, "device/get_changes() failed\n"); + free(changes); return; } @@ -1463,6 +1594,9 @@ void folder_tree_update(folder_tree * tree, mfconn * conn) folder_tree_housekeep(tree, conn); fprintf(stderr, "tree after cleaning:\n"); folder_tree_debug(tree); + + /* free allocated memory */ + free(changes); } /* diff --git a/fuse/hashtbl.h b/fuse/hashtbl.h index 3bfa891..bb54ac6 100644 --- a/fuse/hashtbl.h +++ b/fuse/hashtbl.h @@ -22,6 +22,7 @@ #include #include #include +#include #include "../mfapi/mfconn.h" @@ -69,4 +70,7 @@ bool folder_tree_path_is_root(folder_tree * tree, mfconn * conn, bool folder_tree_path_is_file(folder_tree * tree, mfconn * conn, const char *path); +int folder_tree_open_file(folder_tree * tree, mfconn * conn, + const char *path); + #endif diff --git a/fuse/main.c b/fuse/main.c index d8b37ad..f4431a7 100644 --- a/fuse/main.c +++ b/fuse/main.c @@ -33,6 +33,7 @@ #include #include #include +#include #include "../mfapi/mfconn.h" #include "../mfapi/apicalls.h" @@ -105,6 +106,10 @@ struct mediafirefs_user_options { char *api_key; }; +struct mediafirefs_openfile { + int fd; +}; + static void usage(const char *progname) { fprintf(stderr, "Usage %s [options] mountpoint\n" @@ -132,6 +137,9 @@ static int mediafirefs_getattr(const char *path, struct stat *stbuf) * since getattr is called before every other call (except for getattr, * read and write) wee only call folder_tree_update in the getattr call * and not the others + * + * FIXME: only call folder_tree_update if it has not been called for a set + * amount of time */ folder_tree_update(tree, conn); return folder_tree_getattr(tree, conn, path, stbuf); @@ -258,34 +266,43 @@ static int mediafirefs_rmdir(const char *path) static int mediafirefs_open(const char *path, struct fuse_file_info *file_info) { - (void)path; + int fd; + struct mediafirefs_openfile *openfile; if ((file_info->flags & O_ACCMODE) != O_RDONLY) { fprintf(stderr, "can only open read-only"); return -EACCES; } - /* check if file has already been downloaded */ + fd = folder_tree_open_file(tree, conn, path); + if (fd < 0) { + fprintf(stderr, "folder_tree_file_open unsuccessful\n"); + return fd; + } - /* check if downloaded version is the current one */ - - /* download file from remote */ - - /* update local file with patch */ - - return -ENOSYS; + openfile = malloc(sizeof(struct mediafirefs_openfile)); + openfile->fd = fd; + file_info->fh = (uint64_t) openfile; + return 0; } static int mediafirefs_read(const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *file_info) { (void)path; - (void)buf; - (void)size; - (void)offset; - (void)file_info; + return pread(((struct mediafirefs_openfile *)file_info->fh)->fd, buf, size, + offset); +} - return -ENOSYS; +static int mediafirefs_release(const char *path, + struct fuse_file_info *file_info) +{ + (void)path; + struct mediafirefs_openfile *openfile = + (struct mediafirefs_openfile *)file_info->fh; + close(openfile->fd); + free(openfile); + return 0; } static struct fuse_operations mediafirefs_oper = { @@ -296,6 +313,7 @@ static struct fuse_operations mediafirefs_oper = { .rmdir = mediafirefs_rmdir, .open = mediafirefs_open, .read = mediafirefs_read, + .release = mediafirefs_release, /* .create = mediafirefs_create, .fsync = mediafirefs_fsync, .getxattr = mediafirefs_getxattr, diff --git a/mfapi/file.c b/mfapi/file.c index c8ce413..70a4503 100644 --- a/mfapi/file.c +++ b/mfapi/file.c @@ -22,12 +22,8 @@ #include #include #include -#include -#include #include -#include "../utils/http.h" -#include "../utils/strings.h" #include "file.h" #include "apicalls.h" @@ -302,54 +298,3 @@ const char *file_get_onetime_link(mffile * file) return file->onetime_link; } - -ssize_t file_download_direct(mffile * file, const char *local_dir) -{ - const char *url; - const char *file_name; - const char *file_path; - struct stat file_info; - ssize_t bytes_read = 0; - int retval; - mfhttp *conn; - - if (file == NULL) - return -1; - if (local_dir == NULL) - return -1; - - url = file_get_direct_link(file); - if (url == NULL) - return -1; - - file_name = file_get_name(file); - if (file_name == NULL) - return -1; - if (strlen(file_name) < 1) - return -1; - - if (local_dir[strlen(local_dir) - 1] == '/') - file_path = strdup_printf("%s%s", local_dir, file_name); - else - file_path = strdup_printf("%s/%s", local_dir, file_name); - - conn = http_create(); - retval = http_get_file(conn, url, file_path); - http_destroy(conn); - - /* - it is preferable to have the vfs tell us how many bytes the - transfer actually is. it's really all that matters. - */ - memset(&file_info, 0, sizeof(file_info)); - retval = stat(file_path, &file_info); - - free((void *)file_path); - - if (retval != 0) - return -1; - - bytes_read = file_info.st_size; - - return bytes_read; -} diff --git a/mfapi/file.h b/mfapi/file.h index 1cabb5e..a8f8be9 100644 --- a/mfapi/file.h +++ b/mfapi/file.h @@ -20,7 +20,8 @@ #ifndef __MFAPI_FILE_H__ #define __MFAPI_FILE_H__ -#include +#include +#include typedef struct mffile mffile; @@ -56,8 +57,6 @@ int file_set_onetime_link(mffile * file, const char *onetime_link); const char *file_get_onetime_link(mffile * file); -ssize_t file_download_direct(mffile * file, const char *local_dir); - int file_set_size(mffile * file, uint64_t size); uint64_t file_get_size(mffile * file); diff --git a/mfshell/commands/get.c b/mfshell/commands/get.c index 119a1ec..4e62c54 100644 --- a/mfshell/commands/get.c +++ b/mfshell/commands/get.c @@ -24,12 +24,15 @@ #include #include #include +#include #include "../../mfapi/apicalls.h" #include "../mfshell.h" #include "../../mfapi/file.h" #include "../../mfapi/mfconn.h" #include "../commands.h" // IWYU pragma: keep +#include "../../utils/strings.h" +#include "../../utils/http.h" int mfshell_cmd_get(mfshell * mfshell, int argc, char *const argv[]) { @@ -38,6 +41,11 @@ int mfshell_cmd_get(mfshell * mfshell, int argc, char *const argv[]) int retval; ssize_t bytes_read; const char *quickkey; + const char *file_path; + const char *file_name; + const char *url; + struct stat file_info; + mfhttp *http; if (mfshell == NULL) return -1; @@ -86,7 +94,34 @@ int mfshell_cmd_get(mfshell * mfshell, int argc, char *const argv[]) getcwd(mfshell->local_working_dir, PATH_MAX); } - bytes_read = file_download_direct(file, mfshell->local_working_dir); + file_name = file_get_name(file); + if (file_name == NULL) + return -1; + if (strlen(file_name) < 1) + return -1; + + file_path = strdup_printf("%s/%s", mfshell->local_working_dir, file_name); + + url = file_get_direct_link(file); + if (url == NULL) + return -1; + + http = http_create(); + retval = http_get_file(http, url, file_path); + http_destroy(http); + + if (retval != 0) + return -1; + + memset(&file_info, 0, sizeof(file_info)); + retval = stat(file_path, &file_info); + + free((void *)file_path); + + if (retval != 0) + return -1; + + bytes_read = file_info.st_size; if (bytes_read != -1) printf("\r Downloaded %zd bytes OK!\n\r", bytes_read); diff --git a/tests/valgrind_fuse.sh b/tests/valgrind_fuse.sh index 13eced3..4082929 100755 --- a/tests/valgrind_fuse.sh +++ b/tests/valgrind_fuse.sh @@ -49,6 +49,10 @@ fi tree /mnt +printf "foobar" | diff - /mnt/Untitled.txt + +sleep 2 + fusermount -u /mnt wait "$fusepid"