Ark/mediafire-fuse
1
0
Fork 0
mirror of https://github.com/xorgy/mediafire-fuse synced 2026-01-13 13:14:29 -08:00
Code Issues Packages Projects Releases Wiki Activity

Optimize device/get_changes updates

Browse Source 
- don't use folder/get_content but rely on parent information
 - add many more debugging information and sanity checks
 - ignore trash
 - correctly update local device revision using device_response
 - remove "visited" member and change housekeeping function to check for
   consistency of parents and children (and retrieve fix from remote if
   necessary)
 - print key and parent key in the debugging function
 - check for remote updates on every fs function
 - introduce global defines for key and filename length
 - add parent member, getters and setters to file.h
This commit is contained in:
josch
2014-09-27 13:53:44 +02:00
parent da914e67b1
commit b6f35763fa
11 changed files with 459 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files

528
fuse/hashtbl.c
View File

@@ -70,17 +70,17 @@ struct h_entry {
* keys are either 13 (folders) or 15 (files) long since the structure * keys are either 13 (folders) or 15 (files) long since the structure
* members are most likely 8-byte aligned anyways, it does not make sense * members are most likely 8-byte aligned anyways, it does not make sense
* to differentiate between them */ * to differentiate between them */
char key[16]; char key[MFAPI_MAX_LEN_KEY + 1];
/* a filename is maximum 255 characters long */ /* a filename is maximum 255 characters long */
char name[256]; char name[MFAPI_MAX_LEN_NAME + 1];
/* we do not add the parent here because it is not used anywhere */ /* we do not add the parent here because it is not used anywhere */
/* char parent[20]; */ /* char parent[20]; */
/* local revision */ /* local revision */
uint64_t revision; uint64_t revision;
/* creation time */ /* creation time */
uint64_t ctime; uint64_t ctime;
/* for maintenance, set to zero when storing on disk */ /* key of the containing folder */
char visited; h_entry *parent;
/******************** /********************
* only for folders * * only for folders *
@@ -92,7 +92,7 @@ struct h_entry {
* This member could also be an array of keys which would not require * This member could also be an array of keys which would not require
* lookups on updating but we expect more reads than writes so we * lookups on updating but we expect more reads than writes so we
* sacrifice slower updates for faster lookups */ * sacrifice slower updates for faster lookups */
struct h_entry **children; h_entry **children;
/****************** /******************
* only for files * * only for files *
@@ -169,6 +169,9 @@ static void folder_tree_free_entries(folder_tree * tree)
tree->buckets[i] = NULL; tree->buckets[i] = NULL;
tree->bucket_lens[i] = 0; tree->bucket_lens[i] = 0;
} }
free(tree->root.children);
tree->root.children = NULL;
tree->root.num_children = 0;
} }
void folder_tree_destroy(folder_tree * tree) void folder_tree_destroy(folder_tree * tree)
@@ -189,7 +192,7 @@ static h_entry *folder_tree_lookup_key(folder_tree * tree, const char *key)
int bucket_id; int bucket_id;
uint64_t i; uint64_t i;
if (key == NULL) { if (key == NULL || key[0] == '\0') {
return &(tree->root); return &(tree->root);
} }
/* retrieve the right bucket for this key */ /* retrieve the right bucket for this key */
@@ -201,6 +204,7 @@ static h_entry *folder_tree_lookup_key(folder_tree * tree, const char *key)
} }
} }
fprintf(stderr, "cannot find h_entry for key %s\n", key);
return NULL; return NULL;
} }
@@ -349,21 +353,47 @@ int folder_tree_readdir(folder_tree * tree, const char *path, void *buf,
return 0; return 0;
} }
static bool folder_tree_is_root(h_entry * entry)
{
if (entry == NULL) {
fprintf(stderr, "entry is NULL\n");
return false;
}
return (entry->name[0] == '\0'
|| (strncmp(entry->name, "myfiles", sizeof(entry->name)) == 0))
&& entry->key[0] == '\0';
}
/* /*
* When adding an existing key, the old key is overwritten. * given a key and the new parent, this function makes sure to allocate new
* Return the inserted or updated key * memory if necessary and adjust the children arrays of the former and new
* parent to accommodate for the change
*/ */
static h_entry *folder_tree_add_file(folder_tree * tree, mffile * file) static h_entry *folder_tree_allocate_entry(folder_tree * tree, const char *key,
h_entry * new_parent)
{ {
h_entry *entry; h_entry *entry;
int bucket_id; int bucket_id;
const char *key; h_entry *old_parent;
uint64_t i;
bool found;
key = file_get_key(file); if (tree == NULL) {
fprintf(stderr, "tree cannot be NULL\n");
return NULL;
}
if (new_parent == NULL) {
fprintf(stderr, "new parent cannot be NULL\n");
return NULL;
}
entry = folder_tree_lookup_key(tree, key); entry = folder_tree_lookup_key(tree, key);
if (entry == NULL) { if (entry == NULL) {
fprintf(stderr,
"key is NULL but this is fine, we just create it now\n");
/* entry was not found, so append it to the end of the bucket */ /* entry was not found, so append it to the end of the bucket */
entry = (h_entry *) calloc(1, sizeof(h_entry)); entry = (h_entry *) calloc(1, sizeof(h_entry));
bucket_id = HASH_OF_KEY(key); bucket_id = HASH_OF_KEY(key);
@@ -376,10 +406,133 @@ static h_entry *folder_tree_add_file(folder_tree * tree, mffile * file)
return NULL; return NULL;
} }
tree->buckets[bucket_id][tree->bucket_lens[bucket_id] - 1] = entry; tree->buckets[bucket_id][tree->bucket_lens[bucket_id] - 1] = entry;
/* since this entry is new, add it to the children of its parent
*
* since the key of this file or folder did not exist in the
* hashtable, we do not have to check whether the parent already has
* it as a child but can just append to its list of children
*/
new_parent->num_children++;
new_parent->children =
(h_entry **) realloc(new_parent->children,
new_parent->num_children * sizeof(h_entry *));
if (new_parent->children == NULL) {
fprintf(stderr, "realloc failed\n");
return NULL;
} }
new_parent->children[new_parent->num_children - 1] = entry;
return entry;
}
/* Entry was found so check if the old parent is different from the
* new parent. If yes, we need to adjust the children of the old and new
* parent.
*/
old_parent = entry->parent;
/* parent stays the same - nothing to do */
if (old_parent == new_parent)
return entry;
/* check whether entry does not have a parent (this is the case for the
* root node) */
if (old_parent != NULL) {
/* remove the file or folder from the old parent */
for (i = 0; i < old_parent->num_children; i++) {
if (old_parent->children[i] == entry) {
/* move the entries on the right one place to the left */
memmove(old_parent->children[i], old_parent->children[i + 1],
sizeof(h_entry *) * (old_parent->num_children - i -
1));
old_parent->num_children--;
/* change the children size */
if (old_parent->num_children) {
free(old_parent->children);
old_parent->children = NULL;
} else {
old_parent->children =
(h_entry **) realloc(old_parent->children,
old_parent->num_children *
sizeof(h_entry *));
if (old_parent->children == NULL) {
fprintf(stderr, "realloc failed\n");
return NULL;
}
}
}
}
} else {
/* sanity check: if the parent was NULL then this entry must be the
* root */
if (!folder_tree_is_root(entry)) {
fprintf(stderr,
"the parent was NULL so this node should be root but is not\n");
fprintf(stderr, "name: %s, key: %s\n", entry->name, entry->key);
return NULL;
}
}
/* and add it to the new */
/* since the entry already existed, it can be that the new parent
* already contains the child */
found = false;
for (i = 0; i < new_parent->num_children; i++) {
if (new_parent->children[i] == entry) {
found = true;
break;
}
}
if (!found) {
new_parent->num_children++;
new_parent->children =
(h_entry **) realloc(new_parent->children,
new_parent->num_children * sizeof(h_entry *));
if (new_parent->children == 0) {
fprintf(stderr, "realloc failed\n");
return NULL;
}
new_parent->children[new_parent->num_children - 1] = entry;
}
return entry;
}
/*
* When adding an existing key, the old key is overwritten.
* Return the inserted or updated key
*/
static h_entry *folder_tree_add_file(folder_tree * tree, mffile * file,
h_entry * parent)
{
h_entry *entry;
const char *key;
if (tree == NULL) {
fprintf(stderr, "tree cannot be NULL\n");
return NULL;
}
if (file == NULL) {
fprintf(stderr, "file cannot be NULL\n");
return NULL;
}
if (parent == NULL) {
fprintf(stderr, "parent cannot be NULL\n");
return NULL;
}
key = file_get_key(file);
entry = folder_tree_allocate_entry(tree, key, parent);
strncpy(entry->key, key, sizeof(entry->key)); strncpy(entry->key, key, sizeof(entry->key));
strncpy(entry->name, file_get_name(file), sizeof(entry->name)); strncpy(entry->name, file_get_name(file), sizeof(entry->name));
entry->parent = parent;
entry->revision = file_get_revision(file); entry->revision = file_get_revision(file);
entry->ctime = file_get_created(file); entry->ctime = file_get_created(file);
entry->fsize = file_get_size(file); entry->fsize = file_get_size(file);
@@ -396,31 +549,32 @@ static h_entry *folder_tree_add_file(folder_tree * tree, mffile * file)
* *
* returns the added or updated h_entry * returns the added or updated h_entry
*/ */
static h_entry *folder_tree_add_folder(folder_tree * tree, mffolder * folder) static h_entry *folder_tree_add_folder(folder_tree * tree, mffolder * folder,
h_entry * parent)
{ {
h_entry *entry; h_entry *entry;
int bucket_id;
const char *key; const char *key;
const char *name; const char *name;
key = folder_get_key(folder); if (tree == NULL) {
fprintf(stderr, "tree cannot be NULL\n");
entry = folder_tree_lookup_key(tree, key);
if (entry == NULL) {
/* entry was not found, so append it to the end of the bucket */
entry = (h_entry *) calloc(1, sizeof(h_entry));
bucket_id = HASH_OF_KEY(key);
tree->bucket_lens[bucket_id]++;
tree->buckets[bucket_id] =
realloc(tree->buckets[bucket_id],
sizeof(h_entry *) * tree->bucket_lens[bucket_id]);
if (tree->buckets[bucket_id] == NULL) {
fprintf(stderr, "realloc failed\n");
return NULL; return NULL;
} }
tree->buckets[bucket_id][tree->bucket_lens[bucket_id] - 1] = entry;
if (folder == NULL) {
fprintf(stderr, "folder cannot be NULL\n");
return NULL;
} }
if (parent == NULL) {
fprintf(stderr, "parent cannot be NULL\n");
return NULL;
}
key = folder_get_key(folder);
entry = folder_tree_allocate_entry(tree, key, parent);
/* can be NULL for root */ /* can be NULL for root */
if (key != NULL) if (key != NULL)
strncpy(entry->key, key, sizeof(entry->key)); strncpy(entry->key, key, sizeof(entry->key));
@@ -430,6 +584,7 @@ static h_entry *folder_tree_add_folder(folder_tree * tree, mffolder * folder)
strncpy(entry->name, name, sizeof(entry->name)); strncpy(entry->name, name, sizeof(entry->name));
entry->revision = folder_get_revision(folder); entry->revision = folder_get_revision(folder);
entry->ctime = folder_get_created(folder); entry->ctime = folder_get_created(folder);
entry->parent = parent;
return entry; return entry;
} }
@@ -490,6 +645,9 @@ static void folder_tree_remove(folder_tree * tree, const char *key)
* better performance * better performance
* *
* thus, this function relies on the fact that only one h_entry per key exists * thus, this function relies on the fact that only one h_entry per key exists
*
* This function does not use the parent member of the child. If you want to
* rely on that, then use it directly.
*/ */
static bool folder_tree_is_parent_of(h_entry * parent, h_entry * child) static bool folder_tree_is_parent_of(h_entry * parent, h_entry * child)
{ {
@@ -554,24 +712,7 @@ static int folder_tree_rebuild_helper(folder_tree * tree, mfconn * conn,
fprintf(stderr, "folder_get_key returned NULL\n"); fprintf(stderr, "folder_get_key returned NULL\n");
folder_free(folder_result[i]); folder_free(folder_result[i]);
} }
entry = folder_tree_add_folder(tree, folder_result[i]); entry = folder_tree_add_folder(tree, folder_result[i], curr_entry);
/*
* make sure that this entry is not already a child of the current
* folder
*/
if (!folder_tree_is_parent_of(curr_entry, entry)) {
/* add this entry to the children of the current folder */
curr_entry->num_children++;
curr_entry->children =
realloc(curr_entry->children,
curr_entry->num_children * sizeof(h_entry));
if (curr_entry->children == NULL) {
fprintf(stderr, "realloc failed\n");
folder_free(folder_result[i]);
continue;
}
curr_entry->children[curr_entry->num_children - 1] = entry;
}
/* recurse */ /* recurse */
if (recurse) if (recurse)
folder_tree_rebuild_helper(tree, conn, entry, true); folder_tree_rebuild_helper(tree, conn, entry, true);
@@ -601,24 +742,7 @@ static int folder_tree_rebuild_helper(folder_tree * tree, mfconn * conn,
fprintf(stderr, "file_get_key returned NULL\n"); fprintf(stderr, "file_get_key returned NULL\n");
file_free(file_result[i]); file_free(file_result[i]);
} }
entry = folder_tree_add_file(tree, file_result[i]); entry = folder_tree_add_file(tree, file_result[i], curr_entry);
/*
* make sure that this entry is not already a child of the current
* folder
*/
if (!folder_tree_is_parent_of(curr_entry, entry)) {
/* add this entry to the children of the current file */
curr_entry->num_children++;
curr_entry->children =
realloc(curr_entry->children,
curr_entry->num_children * sizeof(h_entry));
if (curr_entry->children == NULL) {
fprintf(stderr, "realloc failed\n");
file_free(file_result[i]);
continue;
}
curr_entry->children[curr_entry->num_children - 1] = entry;
}
file_free(file_result[i]); file_free(file_result[i]);
} }
free(file_result); free(file_result);
@@ -634,6 +758,7 @@ static int folder_tree_update_file_info(folder_tree * tree, mfconn * conn,
{ {
mffile *file; mffile *file;
int retval; int retval;
h_entry *parent;
file = file_alloc(); file = file_alloc();
@@ -641,11 +766,19 @@ static int folder_tree_update_file_info(folder_tree * tree, mfconn * conn,
mfconn_update_secret_key(conn); mfconn_update_secret_key(conn);
if (retval != 0) { if (retval != 0) {
fprintf(stderr, "api call unsuccessful\n"); fprintf(stderr, "api call unsuccessful\n");
/* TODO: check reason. It might be that the remote object does not
* exist anymore in which case it has to be removed locally */
file_free(file); file_free(file);
return -1; return -1;
} }
folder_tree_add_file(tree, file); parent = folder_tree_lookup_key(tree, file_get_parent(file));
if (parent == NULL) {
fprintf(stderr, "file_tree_lookup_key failed\n");
return -1;
}
folder_tree_add_file(tree, file, parent);
file_free(file); file_free(file);
@@ -663,6 +796,12 @@ static int folder_tree_update_folder_info(folder_tree * tree, mfconn * conn,
{ {
mffolder *folder; mffolder *folder;
int retval; int retval;
h_entry *parent;
if (key != NULL && strcmp(key, "trash")) {
fprintf(stderr, "cannot get folder info of trash\n");
return -1;
}
folder = folder_alloc(); folder = folder_alloc();
@@ -670,38 +809,21 @@ static int folder_tree_update_folder_info(folder_tree * tree, mfconn * conn,
mfconn_update_secret_key(conn); mfconn_update_secret_key(conn);
if (retval != 0) { if (retval != 0) {
fprintf(stderr, "api call unsuccessful\n"); fprintf(stderr, "api call unsuccessful\n");
/* TODO: check reason. It might be that the remote object does not
* exist anymore in which case it has to be removed locally */
folder_free(folder); folder_free(folder);
return -1; return -1;
} }
folder_tree_add_folder(tree, folder); parent = folder_tree_lookup_key(tree, folder_get_parent(folder));
if (parent == NULL) {
folder_free(folder);
return 0;
}
/*
* update the contents of a folder and its direct children. Does not recurse
* further
*
* this function cannot complete if there is no reference to the new children
* in the hashtable
*
*/
static int folder_tree_update_folder_contents(folder_tree * tree,
mfconn * conn, char *key)
{
h_entry *curr_entry;
curr_entry = folder_tree_lookup_key(tree, key);
if (curr_entry == NULL) {
fprintf(stderr, "folder_tree_lookup_key failed\n"); fprintf(stderr, "folder_tree_lookup_key failed\n");
return -1; return -1;
} }
folder_tree_rebuild_helper(tree, conn, curr_entry, false); folder_tree_add_folder(tree, folder, parent);
folder_free(folder);
return 0; return 0;
} }
@@ -711,7 +833,7 @@ static int folder_tree_update_folder_contents(folder_tree * tree,
* *
* if yes, integrate those changes * if yes, integrate those changes
*/ */
static void folder_tree_update(folder_tree * tree, mfconn * conn) void folder_tree_update(folder_tree * tree, mfconn * conn)
{ {
uint64_t revision_remote; uint64_t revision_remote;
uint64_t i; uint64_t i;
@@ -725,20 +847,13 @@ static void folder_tree_update(folder_tree * tree, mfconn * conn)
fprintf(stderr, "Request to update but nothing to do\n"); fprintf(stderr, "Request to update but nothing to do\n");
return; return;
} }
/*
* we can ignore the information about the parent because if a file was
* created or moved, then the parent directory was updated as well
* if only the content of a file was modified, then the information about
* the parent is useless
*/
/* /*
* we have to manually check the root because it never shows up in the * root never shows up in device_get_changes but since we rely on the
* results from device_get_changes * parent information of files and folders, we do not manually retrieve
* its content
*/ */
folder_tree_update_folder_contents(tree, conn, NULL);
/* /*
* changes have to be applied in the right order but the result of * changes have to be applied in the right order but the result of
* mfconn_api_device_get_changes is already sorted by revision * mfconn_api_device_get_changes is already sorted by revision
@@ -752,27 +867,67 @@ static void folder_tree_update(folder_tree * tree, mfconn * conn)
return; return;
} }
for (i = 0; changes[i].revision != 0; i++) { /*
* TODO: before calling remote functions here, check if the revision of
* the local object is lower than the one of the reported change
*
* TODO: only use the latest revision of the same file-/folderkey
*/
for (i = 0; changes[i].change != MFCONN_DEVICE_CHANGE_END; i++) {
switch (changes[i].change) { switch (changes[i].change) {
case MFCONN_DEVICE_CHANGE_DELETED_FOLDER: case MFCONN_DEVICE_CHANGE_DELETED_FOLDER:
case MFCONN_DEVICE_CHANGE_DELETED_FILE: case MFCONN_DEVICE_CHANGE_DELETED_FILE:
folder_tree_remove(tree, changes[i].key); folder_tree_remove(tree, changes[i].key);
break; break;
case MFCONN_DEVICE_CHANGE_UPDATED_FOLDER: case MFCONN_DEVICE_CHANGE_UPDATED_FOLDER:
/* if a folder has been updated then either its name... */ /* ignore updates of the folder key "trash" or folders with
* the parent folder key "trash" */
if (strcmp(changes[i].key, "trash") == 0)
continue;
if (strcmp(changes[i].parent, "trash") == 0)
continue;
/* if a folder has been updated then its name or location
* might have changed */
folder_tree_update_folder_info(tree, conn, changes[i].key); folder_tree_update_folder_info(tree, conn, changes[i].key);
/* ...or its contents changed */ /* its content might also have changed but we use the changes
folder_tree_update_folder_contents(tree, conn, changes[i].key); * to files to update it */
break; break;
case MFCONN_DEVICE_CHANGE_UPDATED_FILE: case MFCONN_DEVICE_CHANGE_UPDATED_FILE:
/* ignore files updated in trash */
if (strcmp(changes[i].parent, "trash") == 0)
continue;
/* if a file changed, update its info */ /* if a file changed, update its info */
folder_tree_update_file_info(tree, conn, changes[i].key); folder_tree_update_file_info(tree, conn, changes[i].key);
break; break;
case MFCONN_DEVICE_CHANGE_END:
break;
} }
} }
/* the new revision of the tree is the revision of the last change */ /* the new revision of the tree is the revision of the terminating change
tree->revision = changes[i - 1].revision; * */
tree->revision = changes[i].revision;
/*
* it can happen that another change happened remotely while we were
* trying to integrate the changes reported by the last device/get_changes
* results. In that case, the file and folder information we retrieve will
* have a revision greater than the local device revision we store. This
* can also lead to lost changes. But this will only be temporary as the
* situation should be rectified once the next device/get_changes is done.
*
* Just remember that due to this it can happen that the revision of the
* local tree is less than the highest revision of a h_entry it stores.
*/
/* now fix up any possible errors */
/* clean the resulting folder_tree of any dangling objects */
fprintf(stderr, "tree before cleaning:\n");
folder_tree_debug(tree, NULL, 0);
folder_tree_housekeep(tree, conn);
fprintf(stderr, "tree after cleaning:\n");
folder_tree_debug(tree, NULL, 0);
} }
/* /*
@@ -816,104 +971,107 @@ int folder_tree_rebuild(folder_tree * tree, mfconn * conn)
*/ */
folder_tree_update(tree, conn); folder_tree_update(tree, conn);
/* clean the resulting folder_tree of any dangling objects */
folder_tree_housekeep(tree);
return 0; return 0;
} }
static void mark_visited(h_entry * ent) /*
{ * clean up files and folders that are never referenced
uint64_t i; *
* first find all folders that have children that do not reference their
* parent. If a discrepancy is found, ask the remote for the true list of
* children of that folder.
*
* then find all files and folders that have a parent that does not reference
* them. If a discrepancy is found, ask the remote for the true parent of that
* file.
*/
ent->visited = 1; void folder_tree_housekeep(folder_tree * tree, mfconn * conn)
for (i = 0; i < ent->num_children; i++) {
if (ent->children[i]->atime == 0) {
/* folder */
mark_visited(ent->children[i]);
} else {
/* file */
ent->children[i]->visited = 1;
}
}
}
/* clean up files and folders that are never referenced */
void folder_tree_housekeep(folder_tree * tree)
{ {
uint64_t i, uint64_t i,
j; j,
k;
/* mark all objects as unvisited, just in case */ bool found;
for (i = 0; i < NUM_BUCKETS; i++) {
for (j = 0; j < tree->bucket_lens[i]; j++) {
tree->buckets[i][j]->visited = 0;
}
}
/* walk the tree, mark found objects as visited */
mark_visited(&(tree->root));
/* /*
* remove unvisited objects * find objects with children who claim to have a different parent
*
* do not remove unreferenced objects which have a revision that is equal
* to the device revision because they might only be dangling right now
* because we did this cleaning in the middle of a file or directory
* movement and at some point during the movement, the moved object is not
* referenced by either its source nor its destination
*/ */
/* first check the root as a special case */
found = false;
for (k = 0; k < tree->root.num_children; k++) {
/* only compare pointers and not keys. This relies on keys
* being unique */
if (tree->root.children[k]->parent != &(tree->root)) {
fprintf(stderr,
"root claims that %s is its child but %s doesn't think so\n",
tree->root.children[k]->key, tree->root.children[k]->key);
found = true;
break;
}
}
if (found) {
folder_tree_rebuild_helper(tree, conn, &(tree->root), false);
}
/* then check the hashtable */
for (i = 0; i < NUM_BUCKETS; i++) { for (i = 0; i < NUM_BUCKETS; i++) {
for (j = 0; j < tree->bucket_lens[i]; j++) { for (j = 0; j < tree->bucket_lens[i]; j++) {
if (tree->buckets[i][j]->visited == 0 && found = false;
tree->buckets[i][j]->revision != tree->revision) { for (k = 0; k < tree->buckets[i][j]->num_children; k++) {
fprintf(stderr, "remove unvisited %s\n", /* only compare pointers and not keys. This relies on keys
tree->buckets[i][j]->key); * being unique */
/* free memory of h_entry */ if (tree->buckets[i][j]->children[k]->parent !=
free(tree->buckets[i][j]); tree->buckets[i][j]) {
/* move the items on the right one place to the left */ fprintf(stderr,
memmove(tree->buckets[i][j], tree->buckets[i][j + 1], "%s claims that %s is its child but %s doesn't think so\n",
sizeof(h_entry *) * (tree->bucket_lens[i] - j - 1)); tree->buckets[i][j]->key,
/* change bucket size */ tree->buckets[i][j]->children[k]->key,
tree->bucket_lens[i]--; tree->buckets[i][j]->children[k]->key);
if (tree->bucket_lens[i] == 0) { found = true;
free(tree->buckets[i]); break;
tree->buckets[i] = NULL;
} else {
tree->buckets[i] =
realloc(tree->buckets[i],
sizeof(h_entry) * tree->bucket_lens[i]);
if (tree->buckets[i] == NULL) {
fprintf(stderr, "realloc failed\n");
return;
} }
} }
if (found) {
/* an entry was found that claims to have a different parent,
* so ask the remote to retrieve the real list of children */
folder_tree_rebuild_helper(tree, conn, tree->buckets[i][j],
false);
}
}
}
/* find objects whose parents do not match their actual parents */
for (i = 0; i < NUM_BUCKETS; i++) {
for (j = 0; j < tree->bucket_lens[i]; j++) {
if (!folder_tree_is_parent_of
(tree->buckets[i][j]->parent, tree->buckets[i][j])) {
fprintf(stderr,
"%s claims that %s is its parent but it is not\n",
tree->buckets[i][j]->key,
tree->buckets[i][j]->parent->key);
if (tree->buckets[i][j]->atime == 0) {
/* folder */
folder_tree_update_folder_info(tree, conn,
tree->buckets[i][j]->key);
} else {
/* file */
folder_tree_update_file_info(tree, conn,
tree->buckets[i][j]->key);
}
} }
} }
} }
/* TODO: remove unvisited cached files */ /* TODO: remove unreferenced cached files */
/* /*
* Never remove an unreferenced cached file if its revision is equal to
* the device revision
*
* for file in cache directory: * for file in cache directory:
* e = folder_tree_lookup_key(file.key) * e = folder_tree_lookup_key(file.key)
* if e == None: * if e == None:
* delete(file) * delete(file)
* elif e->visited == 0 && e->revision != tree->revision:
* delete(file)
*/ */
/* mark all objects as unvisited again */
tree->root.visited = 0;
for (i = 0; i < NUM_BUCKETS; i++) {
for (j = 0; j < tree->bucket_lens[i]; j++) {
tree->buckets[i][j]->visited = 0;
}
}
} }
void folder_tree_debug(folder_tree * tree, h_entry * ent, int depth) void folder_tree_debug(folder_tree * tree, h_entry * ent, int depth)
@@ -927,13 +1085,15 @@ void folder_tree_debug(folder_tree * tree, h_entry * ent, int depth)
for (i = 0; i < ent->num_children; i++) { for (i = 0; i < ent->num_children; i++) {
if (ent->children[i]->atime == 0) { if (ent->children[i]->atime == 0) {
/* folder */ /* folder */
fprintf(stderr, "%*s d:%s\n", depth + 1, " ", fprintf(stderr, "%*s d:%s k:%s p:%s\n", depth + 1, " ",
ent->children[i]->name); ent->children[i]->name, ent->children[i]->key,
ent->children[i]->parent->key);
folder_tree_debug(tree, ent->children[i], depth + 1); folder_tree_debug(tree, ent->children[i], depth + 1);
} else { } else {
/* file */ /* file */
fprintf(stderr, "%*s f:%s\n", depth + 1, " ", fprintf(stderr, "%*s f:%s k:%s p:%s\n", depth + 1, " ",
ent->children[i]->name); ent->children[i]->name, ent->children[i]->key,
ent->children[i]->parent->key);
} }
} }
} }

4
fuse/hashtbl.h
View File

@@ -32,7 +32,7 @@ void folder_tree_destroy(folder_tree * tree);
int folder_tree_rebuild(folder_tree * tree, mfconn * conn); int folder_tree_rebuild(folder_tree * tree, mfconn * conn);
void folder_tree_housekeep(folder_tree * tree); void folder_tree_housekeep(folder_tree * tree, mfconn * conn);
void folder_tree_debug(folder_tree * tree, h_entry * ent, void folder_tree_debug(folder_tree * tree, h_entry * ent,
int depth); int depth);
@@ -43,4 +43,6 @@ int folder_tree_getattr(folder_tree * tree, const char *path,
int folder_tree_readdir(folder_tree * tree, const char *path, int folder_tree_readdir(folder_tree * tree, const char *path,
void *buf, fuse_fill_dir_t filldir); void *buf, fuse_fill_dir_t filldir);
void folder_tree_update(folder_tree * tree, mfconn * conn);
#endif #endif

4
fuse/main.c
View File

@@ -88,6 +88,7 @@ static void usage(const char *progname)
static int mediafirefs_getattr(const char *path, struct stat *stbuf) static int mediafirefs_getattr(const char *path, struct stat *stbuf)
{ {
folder_tree_update(tree, conn);
return folder_tree_getattr(tree, path, stbuf); return folder_tree_getattr(tree, path, stbuf);
} }
@@ -98,6 +99,7 @@ static int mediafirefs_readdir(const char *path, void *buf,
(void)offset; (void)offset;
(void)info; (void)info;
folder_tree_update(tree, conn);
return folder_tree_readdir(tree, path, buf, filldir); return folder_tree_readdir(tree, path, buf, filldir);
} }
@@ -187,7 +189,7 @@ int main(int argc, char *argv[])
//folder_tree_housekeep(tree); //folder_tree_housekeep(tree);
folder_tree_debug(tree, NULL, 0); //folder_tree_debug(tree, NULL, 0);
//folder_tree_destroy(tree); //folder_tree_destroy(tree);

9
mfapi/apicalls.h
View File

@@ -26,17 +26,22 @@
#include "folder.h" #include "folder.h"
#include "mfconn.h" #include "mfconn.h"
#define MFAPI_MAX_LEN_KEY 15
#define MFAPI_MAX_LEN_NAME 255
enum mfconn_device_change_type { enum mfconn_device_change_type {
MFCONN_DEVICE_CHANGE_DELETED_FOLDER, MFCONN_DEVICE_CHANGE_DELETED_FOLDER,
MFCONN_DEVICE_CHANGE_DELETED_FILE, MFCONN_DEVICE_CHANGE_DELETED_FILE,
MFCONN_DEVICE_CHANGE_UPDATED_FOLDER, MFCONN_DEVICE_CHANGE_UPDATED_FOLDER,
MFCONN_DEVICE_CHANGE_UPDATED_FILE MFCONN_DEVICE_CHANGE_UPDATED_FILE,
MFCONN_DEVICE_CHANGE_END
}; };
struct mfconn_device_change { struct mfconn_device_change {
enum mfconn_device_change_type change; enum mfconn_device_change_type change;
char key[20]; char key[16];
uint64_t revision; uint64_t revision;
char parent[16];
}; };
int mfconn_api_file_get_info(mfconn * conn, mffile * file, int mfconn_api_file_get_info(mfconn * conn, mffile * file,

46
mfapi/apicalls/device_get_changes.c
View File

@@ -23,6 +23,7 @@
#include <stdint.h> #include <stdint.h>
#include <string.h> #include <string.h>
#include <stddef.h> #include <stddef.h>
#include <stdio.h>
#include "../../utils/http.h" #include "../../utils/http.h"
#include "../../utils/json.h" #include "../../utils/json.h"
@@ -62,14 +63,16 @@ int mfconn_api_device_get_changes(mfconn * conn, uint64_t revision,
return retval; return retval;
} }
static void aux(json_t * key, json_t * revision, static void aux(json_t * key, json_t * parent, json_t * revision,
enum mfconn_device_change_type change, enum mfconn_device_change_type change,
struct mfconn_device_change **changes, size_t * len_changes) struct mfconn_device_change **changes, size_t * len_changes)
{ {
struct mfconn_device_change *tmp_change; struct mfconn_device_change *tmp_change;
if (key == NULL || revision == NULL) if (key == NULL || revision == NULL || parent == NULL) {
fprintf(stderr, "change without either key, revision or parent");
return; return;
}
(*len_changes)++; (*len_changes)++;
*changes = (struct mfconn_device_change *) *changes = (struct mfconn_device_change *)
realloc(*changes, realloc(*changes,
@@ -77,6 +80,8 @@ static void aux(json_t * key, json_t * revision,
tmp_change = *changes + (*len_changes) - 1; tmp_change = *changes + (*len_changes) - 1;
tmp_change->change = change; tmp_change->change = change;
strncpy(tmp_change->key, json_string_value(key), sizeof(tmp_change->key)); strncpy(tmp_change->key, json_string_value(key), sizeof(tmp_change->key));
strncpy(tmp_change->parent, json_string_value(parent),
sizeof(tmp_change->parent));
tmp_change->revision = atoll(json_string_value(revision)); tmp_change->revision = atoll(json_string_value(revision));
} }
@@ -95,7 +100,9 @@ static int _decode_device_get_changes(mfhttp * conn, void *user_ptr)
json_t *obj_array; json_t *obj_array;
json_t *key; json_t *key;
json_t *parent;
json_t *revision; json_t *revision;
json_t *device_revision;
int array_sz; int array_sz;
int i = 0; int i = 0;
@@ -109,6 +116,15 @@ static int _decode_device_get_changes(mfhttp * conn, void *user_ptr)
root = http_parse_buf_json(conn, 0, &error); root = http_parse_buf_json(conn, 0, &error);
node = json_object_by_path(root, "response");
device_revision = json_object_get(node, "device_revision");
if (device_revision == NULL) {
fprintf(stderr,
"response/device_revision is not part of the result\n");
return -1;
}
len_changes = 0; len_changes = 0;
node = json_object_by_path(root, "response/updated"); node = json_object_by_path(root, "response/updated");
@@ -121,9 +137,10 @@ static int _decode_device_get_changes(mfhttp * conn, void *user_ptr)
if (!json_is_object(data)) if (!json_is_object(data))
continue; continue;
key = json_object_get(data, "quickkey"); key = json_object_get(data, "quickkey");
parent = json_object_get(data, "parent_folderkey");
revision = json_object_get(data, "revision"); revision = json_object_get(data, "revision");
aux(key, revision, MFCONN_DEVICE_CHANGE_UPDATED_FILE, changes, aux(key, parent, revision, MFCONN_DEVICE_CHANGE_UPDATED_FILE,
&len_changes); changes, &len_changes);
} }
} }
@@ -135,9 +152,10 @@ static int _decode_device_get_changes(mfhttp * conn, void *user_ptr)
if (!json_is_object(data)) if (!json_is_object(data))
continue; continue;
key = json_object_get(data, "folderkey"); key = json_object_get(data, "folderkey");
parent = json_object_get(data, "parent_folderkey");
revision = json_object_get(data, "revision"); revision = json_object_get(data, "revision");
aux(key, revision, MFCONN_DEVICE_CHANGE_UPDATED_FOLDER, changes, aux(key, parent, revision, MFCONN_DEVICE_CHANGE_UPDATED_FOLDER,
&len_changes); changes, &len_changes);
} }
} }
@@ -151,9 +169,10 @@ static int _decode_device_get_changes(mfhttp * conn, void *user_ptr)
if (!json_is_object(data)) if (!json_is_object(data))
continue; continue;
key = json_object_get(data, "quickkey"); key = json_object_get(data, "quickkey");
parent = json_object_get(data, "parent_folderkey");
revision = json_object_get(data, "revision"); revision = json_object_get(data, "revision");
aux(key, revision, MFCONN_DEVICE_CHANGE_DELETED_FILE, changes, aux(key, parent, revision, MFCONN_DEVICE_CHANGE_DELETED_FILE,
&len_changes); changes, &len_changes);
} }
} }
@@ -165,20 +184,25 @@ static int _decode_device_get_changes(mfhttp * conn, void *user_ptr)
if (!json_is_object(data)) if (!json_is_object(data))
continue; continue;
key = json_object_get(data, "folderkey"); key = json_object_get(data, "folderkey");
parent = json_object_get(data, "parent_folderkey");
revision = json_object_get(data, "revision"); revision = json_object_get(data, "revision");
aux(key, revision, MFCONN_DEVICE_CHANGE_DELETED_FOLDER, changes, aux(key, parent, revision, MFCONN_DEVICE_CHANGE_DELETED_FOLDER,
&len_changes); changes, &len_changes);
} }
} }
// sort // sort
qsort(*changes, len_changes, sizeof(struct mfconn_device_change), qsort(*changes, len_changes, sizeof(struct mfconn_device_change),
change_compare); change_compare);
// put a zero valued entry at the end // put an entry with change type MFCONN_DEVICE_CHANGE_END at the end
// encode the current device revision as its revision
len_changes++; len_changes++;
*changes = (struct mfconn_device_change *) *changes = (struct mfconn_device_change *)
realloc(*changes, len_changes * sizeof(struct mfconn_device_change)); realloc(*changes, len_changes * sizeof(struct mfconn_device_change));
memset(*changes + len_changes - 1, 0, sizeof(struct mfconn_device_change)); memset(*changes + len_changes - 1, 0, sizeof(struct mfconn_device_change));
(*changes)[len_changes - 1].change = MFCONN_DEVICE_CHANGE_END;
(*changes)[len_changes - 1].revision =
atoll(json_string_value(device_revision));
if (root != NULL) if (root != NULL)
json_decref(root); json_decref(root);

23
mfapi/apicalls/file_get_info.c
View File

@@ -69,9 +69,8 @@ static int _decode_file_get_info(mfhttp * conn, void *data)
json_error_t error; json_error_t error;
json_t *root; json_t *root;
json_t *node; json_t *node;
json_t *obj;
json_t *quickkey; json_t *quickkey;
json_t *file_hash;
json_t *file_name;
int retval = 0; int retval = 0;
mffile *file; mffile *file;
@@ -88,15 +87,23 @@ static int _decode_file_get_info(mfhttp * conn, void *data)
if (quickkey != NULL) if (quickkey != NULL)
file_set_key(file, json_string_value(quickkey)); file_set_key(file, json_string_value(quickkey));
file_name = json_object_get(node, "filename"); obj = json_object_get(node, "filename");
if (file_name != NULL) if (obj != NULL)
file_set_name(file, json_string_value(file_name)); file_set_name(file, json_string_value(obj));
file_hash = json_object_get(node, "hash"); obj = json_object_get(node, "hash");
if (file_hash != NULL) { if (obj != NULL) {
file_set_hash(file, json_string_value(file_hash)); file_set_hash(file, json_string_value(obj));
} }
obj = json_object_get(node, "parent_folderkey");
if (obj != NULL) {
file_set_parent(file, json_string_value(obj));
}
// infer that the parent folder must be root
if (obj == NULL && quickkey != NULL)
file_set_parent(file, NULL);
if (quickkey == NULL) if (quickkey == NULL)
retval = -1; retval = -1;

3
mfapi/apicalls/folder_get_content.c
View File

@@ -23,6 +23,7 @@
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <stddef.h> #include <stddef.h>
#include <stdio.h>
#include "../../utils/http.h" #include "../../utils/http.h"
#include "../../utils/json.h" #include "../../utils/json.h"
@@ -129,6 +130,7 @@ static int _decode_folder_get_content_folders(mfhttp * conn, void *user_ptr)
folders_array = json_object_get(node, "folders"); folders_array = json_object_get(node, "folders");
if (!json_is_array(folders_array)) { if (!json_is_array(folders_array)) {
fprintf(stderr, "is not an array: folders");
json_decref(root); json_decref(root);
return -1; return -1;
} }
@@ -225,6 +227,7 @@ static int _decode_folder_get_content_files(mfhttp * conn, void *user_ptr)
files_array = json_object_get(node, "files"); files_array = json_object_get(node, "files");
if (!json_is_array(files_array)) { if (!json_is_array(files_array)) {
fprintf(stderr, "is not an array: files");
json_decref(root); json_decref(root);
return -1; return -1;
} }

37
mfapi/file.c
View File

@@ -27,11 +27,13 @@
#include "../utils/http.h" #include "../utils/http.h"
#include "../utils/strings.h" #include "../utils/strings.h"
#include "file.h" #include "file.h"
#include "apicalls.h"
struct mffile { struct mffile {
char quickkey[18]; char quickkey[MFAPI_MAX_LEN_KEY + 1];
char parent[MFAPI_MAX_LEN_NAME + 1];
char hash[65]; char hash[65];
char name[256]; char name[MFAPI_MAX_LEN_NAME + 1];
time_t created; time_t created;
uint64_t revision; uint64_t revision;
uint64_t size; uint64_t size;
@@ -81,7 +83,7 @@ int file_set_key(mffile * file, const char *key)
return -1; return -1;
memset(file->quickkey, 0, sizeof(file->quickkey)); memset(file->quickkey, 0, sizeof(file->quickkey));
strncpy(file->quickkey, key, sizeof(file->quickkey) - 1); strncpy(file->quickkey, key, sizeof(file->quickkey));
return 0; return 0;
} }
@@ -94,6 +96,33 @@ const char *file_get_key(mffile * file)
return file->quickkey; return file->quickkey;
} }
int file_set_parent(mffile * file, const char *parent_key)
{
if (file == NULL)
return -1;
if (parent_key == NULL) {
memset(file->parent, 0, sizeof(file->parent));
} else {
memset(file->parent, 0, sizeof(file->parent));
strncpy(file->parent, parent_key, sizeof(file->parent));
}
return 0;
}
const char *file_get_parent(mffile * file)
{
if (file == NULL)
return NULL;
if (file->parent[0] == '\0') {
return NULL;
} else {
return file->parent;
}
}
int file_set_hash(mffile * file, const char *hash) int file_set_hash(mffile * file, const char *hash)
{ {
if (file == NULL) if (file == NULL)
@@ -130,7 +159,7 @@ int file_set_name(mffile * file, const char *name)
return -1; return -1;
memset(file->name, 0, sizeof(file->name)); memset(file->name, 0, sizeof(file->name));
strncpy(file->name, name, sizeof(file->name) - 1); strncpy(file->name, name, sizeof(file->name));
return 0; return 0;
} }

4
mfapi/file.h
View File

@@ -32,6 +32,10 @@ int file_set_key(mffile * file, const char *quickkey);
const char *file_get_key(mffile * file); const char *file_get_key(mffile * file);
int file_set_parent(mffile * file, const char *parent_key);
const char *file_get_parent(mffile * file);
int file_set_hash(mffile * file, const char *hash); int file_set_hash(mffile * file, const char *hash);
const char *file_get_hash(mffile * file); const char *file_get_hash(mffile * file);

21
mfapi/folder.c
View File

@@ -22,11 +22,12 @@
#include <string.h> #include <string.h>
#include "folder.h" #include "folder.h"
#include "apicalls.h"
struct mffolder { struct mffolder {
char folderkey[20]; char folderkey[MFAPI_MAX_LEN_KEY + 1];
char name[41]; char name[MFAPI_MAX_LEN_NAME + 1];
char parent[20]; char parent[MFAPI_MAX_LEN_KEY + 1];
uint64_t revision; uint64_t revision;
time_t created; time_t created;
}; };
@@ -62,7 +63,7 @@ int folder_set_key(mffolder * folder, const char *key)
return -1; return -1;
memset(folder->folderkey, 0, sizeof(folder->folderkey)); memset(folder->folderkey, 0, sizeof(folder->folderkey));
strncpy(folder->folderkey, key, sizeof(folder->folderkey) - 1); strncpy(folder->folderkey, key, sizeof(folder->folderkey));
} }
return 0; return 0;
@@ -88,12 +89,8 @@ int folder_set_parent(mffolder * folder, const char *parent_key)
if (parent_key == NULL) { if (parent_key == NULL) {
memset(folder->parent, 0, sizeof(folder->parent)); memset(folder->parent, 0, sizeof(folder->parent));
} else { } else {
if (strlen(parent_key) != 13) {
return -1;
}
memset(folder->parent, 0, sizeof(folder->parent)); memset(folder->parent, 0, sizeof(folder->parent));
strncpy(folder->parent, parent_key, sizeof(folder->parent) - 1); strncpy(folder->parent, parent_key, sizeof(folder->parent));
} }
return 0; return 0;
@@ -104,7 +101,11 @@ const char *folder_get_parent(mffolder * folder)
if (folder == NULL) if (folder == NULL)
return NULL; return NULL;
if (folder->parent[0] == '\0') {
return NULL;
} else {
return folder->parent; return folder->parent;
}
} }
int folder_set_name(mffolder * folder, const char *name) int folder_set_name(mffolder * folder, const char *name)
@@ -118,7 +119,7 @@ int folder_set_name(mffolder * folder, const char *name)
return -1; return -1;
memset(folder->name, 0, sizeof(folder->name)); memset(folder->name, 0, sizeof(folder->name));
strncpy(folder->name, name, sizeof(folder->name) - 1); strncpy(folder->name, name, sizeof(folder->name));
return 0; return 0;
} }

4
mfshell/commands/changes.c
View File

@@ -54,7 +54,7 @@ int mfshell_cmd_changes(mfshell * mfshell, int argc, char *const argv[])
retval = mfconn_api_device_get_changes(mfshell->conn, revision, &changes); retval = mfconn_api_device_get_changes(mfshell->conn, revision, &changes);
mfconn_update_secret_key(mfshell->conn); mfconn_update_secret_key(mfshell->conn);
for (i = 0; changes[i].revision != 0; i++) { for (i = 0; changes[i].change != MFCONN_DEVICE_CHANGE_END; i++) {
switch (changes[i].change) { switch (changes[i].change) {
case MFCONN_DEVICE_CHANGE_DELETED_FOLDER: case MFCONN_DEVICE_CHANGE_DELETED_FOLDER:
printf("%" PRIu64 " deleted folder: %s\n", changes[i].revision, printf("%" PRIu64 " deleted folder: %s\n", changes[i].revision,
@@ -72,6 +72,8 @@ int mfshell_cmd_changes(mfshell * mfshell, int argc, char *const argv[])
printf("%" PRIu64 " updated file: %s\n", changes[i].revision, printf("%" PRIu64 " updated file: %s\n", changes[i].revision,
changes[i].key); changes[i].key);
break; break;
case MFCONN_DEVICE_CHANGE_END:
break;
} }
} }