diff --git a/TODO b/TODO index d807039..f1fc027 100644 --- a/TODO +++ b/TODO @@ -15,9 +15,7 @@ - make h_entry more efficient in memory (for example by making filenames zero terminated) - when handling device/get_changes, make sure to only use the latest revision of the same file-/folderkey - allow different cache directory (useful for running test suite) - - delete old files in cache that have been updated - delete patches in cache that have been applied - - check the hash of all files in the cache on startup - after uploading a file it is immediately downloaded - instead, the existing local file should be used by checking the remote hash - add an option to only call device/get_status in configurable intervals - write man pages diff --git a/fuse/hashtbl.c b/fuse/hashtbl.c index f0ea276..5af5a64 100644 --- a/fuse/hashtbl.c +++ b/fuse/hashtbl.c @@ -36,6 +36,8 @@ #include #include #include +#include +#include #include "hashtbl.h" #include "filecache.h" @@ -147,6 +149,9 @@ static struct h_entry *folder_tree_add_folder(folder_tree * tree, static void folder_tree_remove(folder_tree * tree, const char *key); static bool folder_tree_is_parent_of(struct h_entry *parent, struct h_entry *child); +static bool is_valid_cache_filename(const char *name, char key[], + uint64_t *revision); +static int atime_compare(const void *a, const void *b); /* functions with remote access */ static struct h_entry *folder_tree_lookup_path(folder_tree * tree, @@ -793,6 +798,9 @@ int folder_tree_open_file(folder_tree * tree, mfconn * conn, const char *path, entry->local_revision = entry->remote_revision; } + // however the file was opened, its access time has to be updated + entry->atime = time(NULL); + return retval; } @@ -1746,3 +1754,223 @@ void folder_tree_debug(folder_tree * tree) { folder_tree_debug_helper(tree, NULL, 0); } + +static int atime_compare(const void *a, const void *b) +{ + return ((struct h_entry *)a)->atime - ((struct h_entry *)b)->atime; +} + +/* + * to be a valid cache file, the first 15 bytes have to be letters + * from a-z and numbers from 0-9, the 16th has to be an underscore, + * the 17th has to be a number from 1-9 and the remaining characters + * (if any) be a number from 0-9 + */ +static bool is_valid_cache_filename(const char *name, char key[], + uint64_t *revision) +{ + int i; + + for (i = 0; i < 15; i++) { + if (!islower(name[i]) && !isdigit(name[i])) + return false; + } + if (name[i] != '_') + return false; + i++; + if (name[i] < 49 || name[i] > 57) + return false; + for (; name[i] != '\0'; i++) { + if (!isdigit(name[i])) + return false; + } + + // now copy the first 15 bytes from the name to the key + memcpy(key, name, 15); + key[15] = '\0'; + + *revision = atoll(name+16); + + return true; +} + +/* go through all files in the filecache and check: + * + * - does the filename match the known pattern? + * (do not act on other files to avoid accidentally touching user + * files) + * - is the quickkey known by the hashtable? + * - if no, delete + * - check if its revision is equal the remote revision + * - if no, delete + * - check if its size and hash verifies + * - if no, delete + * - once all files in the cache have been processed this way, check if + * the sum of their sizes is greater than X and delete the oldest + */ +void folder_tree_cleanup_filecache(folder_tree *tree, uint64_t allowed_size) +{ + struct dirent *endp; + struct dirent *entryp; + DIR *dirp; + int retval; + long name_max; + char *filepath; + char key[MFAPI_MAX_LEN_KEY + 1]; + uint64_t revision; + struct h_entry *entry; + size_t num_cachefiles; + size_t i; + uint64_t sum_size; + struct h_entry **cachefiles; + + // from the readdir_r man page + name_max = pathconf(tree->filecache, _PC_NAME_MAX); + if (name_max == -1) /* Limit not defined, or error */ + name_max = 255; /* Take a guess */ + entryp = malloc(offsetof(struct dirent, d_name) + name_max + 1); + + dirp = opendir(tree->filecache); + if (dirp == NULL) { + fprintf(stderr, "cannot open filecache\n"); + return; + } + + num_cachefiles = 0; + cachefiles = NULL; + + for (;;) { + endp = NULL; + retval = readdir_r(dirp, entryp, &endp); + if (retval != 0) { + fprintf(stderr, "readdir_r failed\n"); + free(entryp); + closedir(dirp); + if (cachefiles != NULL) + free(cachefiles); + return; + } + if (endp == NULL) { + break; + } + if (strcmp(entryp->d_name, ".") == 0 || + strcmp(entryp->d_name, "..") == 0) + continue; + + if (!is_valid_cache_filename(entryp->d_name, key, &revision)) { + fprintf(stderr, "not a valid cachefile: %s (ignoring)\n", + entryp->d_name); + continue; + } + + filepath = strdup_printf("%s/%s", tree->filecache, entryp->d_name); + + entry = folder_tree_lookup_key(tree, key); + if (entry == NULL) { + fprintf(stderr, "delete file not in hashtable: %s\n", + entryp->d_name); + retval = unlink(filepath); + if (retval != 0) { + fprintf(stderr, "unlink failed\n"); + } + free(filepath); + continue; + } + + if (revision != entry->remote_revision) { + fprintf(stderr, "delete file with revision %" PRIu64 + " different from remote %" PRIu64 ": %s\n", revision, + entry->remote_revision, entryp->d_name); + retval = unlink(filepath); + if (retval != 0) { + fprintf(stderr, "unlink failed\n"); + } + entry->local_revision = 0; + free(filepath); + continue; + } + + if (revision != entry->local_revision) { + fprintf(stderr, "delete file with revision %" PRIu64 + " different from local %" PRIu64 ": %s\n", revision, + entry->local_revision, entryp->d_name); + retval = unlink(filepath); + if (retval != 0) { + fprintf(stderr, "unlink failed\n"); + } + entry->local_revision = 0; + free(filepath); + continue; + } + + retval = file_check_integrity(filepath, entry->fsize, entry->hash); + if (retval != 0) { + fprintf(stderr, "delete file with invalid content: %s\n", + entryp->d_name); + retval = unlink(filepath); + if (retval != 0) { + fprintf(stderr, "unlink failed\n"); + } + entry->local_revision = 0; + free(filepath); + continue; + } + free(filepath); + + // everything is okay with this one, so append it to the list of files + // in the cache + num_cachefiles++; + cachefiles = + (struct h_entry **)realloc(cachefiles, + num_cachefiles * + sizeof(struct h_entry *)); + if (cachefiles == NULL) { + fprintf(stderr, "realloc failed\n"); + free(entryp); + closedir(dirp); + return; + } + cachefiles[num_cachefiles - 1] = entry; + } + + free(entryp); + closedir(dirp); + + // return if there are no files in the cache + if (num_cachefiles == 0) + return; + + // now calculate the sum of valid files in the cache and check whether it + // is larger than allowed + sum_size = 0; + for (i = 0; i < num_cachefiles; i++) { + sum_size += cachefiles[i]->fsize; + } + + // if the summed size is below the allowed, return + if (sum_size <= allowed_size) { + free(cachefiles); + return; + } + + // sort the files in the cache by their access time + qsort(cachefiles, num_cachefiles, sizeof(struct h_entry *), atime_compare); + + // delete the oldest file until sum is below allowed size + for (i = 0; i < num_cachefiles && sum_size > allowed_size; i++) { + entry = cachefiles[i]; + fprintf(stderr, "delete file to free space: %s_%" PRIu64 "\n", + entry->key, entry->remote_revision); + filepath = strdup_printf("%s/%s_%" PRIu64, tree->filecache, entry->key, + entry->remote_revision); + retval = unlink(filepath); + if (retval != 0) { + fprintf(stderr, "unlink failed\n"); + } + entry->local_revision = 0; + free(filepath); + sum_size -= entry->fsize; + } + + free(cachefiles); +} diff --git a/fuse/hashtbl.h b/fuse/hashtbl.h index c88026b..3444a00 100644 --- a/fuse/hashtbl.h +++ b/fuse/hashtbl.h @@ -53,6 +53,9 @@ int folder_tree_store(folder_tree * tree, FILE * stream); folder_tree *folder_tree_load(FILE * stream, const char *filecache); +void folder_tree_cleanup_filecache(folder_tree *tree, + uint64_t allowed_size); + bool folder_tree_path_exists(folder_tree * tree, mfconn * conn, const char *path); diff --git a/fuse/main.c b/fuse/main.c index 33dec3e..0dbd3a5 100644 --- a/fuse/main.c +++ b/fuse/main.c @@ -319,6 +319,10 @@ static void open_hashtbl(const char *dircache, const char *filecache, fclose(fp); + // TODO: make the maximum cache size configurable + // size is given in bytes and current default is 1 GiB + folder_tree_cleanup_filecache(*tree, 1073741824); + folder_tree_update(*tree, conn, false); } else { // file doesn't exist @@ -456,7 +460,7 @@ int main(int argc, char *argv[]) ctx->sv_writefiles = stringv_alloc(); ctx->sv_readonlyfiles = stringv_alloc(); ctx->last_status_check = 0; - ctx->interval_status_check = 60; + ctx->interval_status_check = 60; // TODO: make this configurable pthread_mutex_init(&(ctx->mutex), NULL);