mirror of
https://github.com/xorgy/mediafire-fuse
synced 2026-01-13 13:14:29 -08:00
Add support for updating local cache through patches
- refactor some code: - code handling the file cache is now in fuse/filecache.c - base36 and base16 encoding and decoding moved to utils/hash.c - mfshell: add "updates" command calling device/get_updates - add container class to store patch information as mfapi/patch.c - apicalls: store more info retrieved by file/get_info - apicalls: add device/get_updates - apicalls: add device/get_patch
This commit is contained in:
josch
477
fuse/filecache.c
Normal file
477
fuse/filecache.c
Normal file
@@ -0,0 +1,477 @@
|
||||
/*
|
||||
* Copyright (C) 2014 Johannes Schauer <j.schauer@email.de>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License version 2, as published by
|
||||
* the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program; if not, write to the Free Software Foundation, Inc., 51
|
||||
* Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <unistd.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <inttypes.h>
|
||||
#include <openssl/sha.h>
|
||||
|
||||
#include "../utils/hash.h"
|
||||
#include "../utils/xdelta3.h"
|
||||
#include "../mfapi/file.h"
|
||||
#include "../mfapi/apicalls.h"
|
||||
#include "../mfapi/patch.h"
|
||||
#include "../utils/http.h"
|
||||
#include "../utils/strings.h"
|
||||
|
||||
static int filecache_check_integrity(const char * cachefile, uint64_t fsize, const unsigned char * fhash);
|
||||
static int filecache_check_integrity_size(const char * cachefile, uint64_t fsize);
|
||||
static int filecache_check_integrity_hash(const char * cachefile, const unsigned char * fhash);
|
||||
static int filecache_update_file(const char * filecache_path, mfconn * conn, const char * quickkey, uint64_t local_revision, uint64_t remote_revision);
|
||||
static int filecache_download_file(const char * filecache_path, const char * quickkey, uint64_t remote_revision, mfconn * conn);
|
||||
static int filecache_download_patch(mfconn * conn, const char * quickkey, uint64_t source_revision, uint64_t target_revision, const char * phash, const char * filecache_path);
|
||||
static int filecache_patch_file(const char * filecache_path, const char * quickkey, uint64_t source_revision, uint64_t target_revision);
|
||||
|
||||
int filecache_open_file(const char * quickkey, uint64_t local_revision,
|
||||
uint64_t remote_revision, uint64_t fsize, const unsigned char * fhash,
|
||||
const char * filecache_path, mfconn * conn)
|
||||
{
|
||||
char *cachefile;
|
||||
int fd;
|
||||
int retval;
|
||||
|
||||
/* check if the requested file is already in the cache */
|
||||
cachefile =
|
||||
strdup_printf("%s/%s_%d", filecache_path, quickkey,
|
||||
remote_revision);
|
||||
fd = open(cachefile, O_RDWR);
|
||||
if (fd > 0) {
|
||||
/* file existed - return handle */
|
||||
free(cachefile);
|
||||
return fd;
|
||||
}
|
||||
free(cachefile);
|
||||
|
||||
/* if the file with remote revision didn't exist, then check whether an
|
||||
* old revision exists and in that case update that.
|
||||
*
|
||||
* Otherwise, download the file anew */
|
||||
|
||||
cachefile =
|
||||
strdup_printf("%s/%s_%d", filecache_path, quickkey,
|
||||
local_revision);
|
||||
fd = open(cachefile, O_RDWR);
|
||||
free(cachefile);
|
||||
if (fd > 0) {
|
||||
close(fd);
|
||||
/* file exists, so we have to update it with one or more patches from
|
||||
* the remote */
|
||||
retval = filecache_update_file(filecache_path, conn, quickkey, local_revision, remote_revision);
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "update_file failed\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
} else {
|
||||
/* download the file */
|
||||
retval = filecache_download_file(filecache_path, quickkey, remote_revision, conn);
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "filecache_download_file failed\n");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
/* check whether the patched or newly downloaded file matches the hash we
|
||||
* have stored */
|
||||
cachefile =
|
||||
strdup_printf("%s/%s_%d", filecache_path, quickkey,
|
||||
remote_revision);
|
||||
retval = filecache_check_integrity(cachefile, fsize, fhash);
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "checking integrity failed\n");
|
||||
free(cachefile);
|
||||
return -1;
|
||||
}
|
||||
|
||||
fd = open(cachefile, O_RDWR);
|
||||
|
||||
free(cachefile);
|
||||
|
||||
/* return the file handle */
|
||||
return fd;
|
||||
}
|
||||
|
||||
static int filecache_check_integrity(const char * path, uint64_t fsize, const unsigned char * fhash) {
|
||||
int retval;
|
||||
|
||||
retval = filecache_check_integrity_size(path, fsize);
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "filecache_check_integrity_size failed\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
retval = filecache_check_integrity_hash(path, fhash);
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "filecache_check_integrity_hash failed\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int filecache_check_integrity_size(const char * path, uint64_t fsize)
|
||||
{
|
||||
struct stat file_info;
|
||||
int retval;
|
||||
uint64_t bytes_read;
|
||||
|
||||
/* check if the size of the downloaded file matches the expected size */
|
||||
memset(&file_info, 0, sizeof(file_info));
|
||||
retval = stat(path, &file_info);
|
||||
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "stat failed\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
bytes_read = file_info.st_size;
|
||||
|
||||
if (bytes_read != fsize) {
|
||||
fprintf(stderr,
|
||||
"expected %" PRIu64 " bytes but got %" PRIu64 " bytes\n",
|
||||
fsize, bytes_read);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int filecache_check_integrity_hash(const char * path, const unsigned char * fhash)
|
||||
{
|
||||
int retval;
|
||||
FILE *fh;
|
||||
unsigned char hash[SHA256_DIGEST_LENGTH];
|
||||
char *hexhash;
|
||||
|
||||
fh = fopen(path, "r");
|
||||
if (fh == NULL) {
|
||||
perror("cannot open file");
|
||||
return -1;
|
||||
}
|
||||
|
||||
retval = calc_sha256(fh, hash);
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "failed to calculate hash\n");
|
||||
fclose(fh);
|
||||
return -1;
|
||||
}
|
||||
|
||||
fclose(fh);
|
||||
|
||||
if (memcmp(fhash, hash, SHA256_DIGEST_LENGTH) != 0) {
|
||||
fprintf(stderr, "hashes are not equal\n");
|
||||
hexhash = binary2hex(fhash, SHA256_DIGEST_LENGTH);
|
||||
fprintf(stderr, "remote: %s\n", hexhash);
|
||||
free(hexhash);
|
||||
hexhash = binary2hex(hash, SHA256_DIGEST_LENGTH);
|
||||
fprintf(stderr, "downloaded: %s\n", hexhash);
|
||||
free(hexhash);
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int filecache_download_file(const char * filecache_path, const char * quickkey, uint64_t remote_revision, mfconn * conn)
|
||||
{
|
||||
const char *url;
|
||||
mffile *file;
|
||||
mfhttp *http;
|
||||
char *cachefile;
|
||||
int retval;
|
||||
|
||||
cachefile = strdup_printf("%s/%s_%d", filecache_path, quickkey,
|
||||
remote_revision);
|
||||
|
||||
file = file_alloc();
|
||||
retval = mfconn_api_file_get_links(conn, file, (char *)quickkey);
|
||||
mfconn_update_secret_key(conn);
|
||||
|
||||
if (retval == -1) {
|
||||
fprintf(stderr, "mfconn_api_file_get_links failed\n");
|
||||
free(cachefile);
|
||||
file_free(file);
|
||||
return -1;
|
||||
}
|
||||
|
||||
url = file_get_direct_link(file);
|
||||
|
||||
if (url == NULL) {
|
||||
fprintf(stderr, "file_get_direct_link failed\n");
|
||||
free(cachefile);
|
||||
file_free(file);
|
||||
return -1;
|
||||
}
|
||||
|
||||
http = http_create();
|
||||
retval = http_get_file(http, url, cachefile);
|
||||
http_destroy(http);
|
||||
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "download failed\n");
|
||||
free(cachefile);
|
||||
file_free(file);
|
||||
return -1;
|
||||
}
|
||||
|
||||
free(cachefile);
|
||||
file_free(file);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int filecache_update_file(const char * filecache_path, mfconn * conn, const char * quickkey, uint64_t local_revision, uint64_t remote_revision)
|
||||
{
|
||||
unsigned char hash2[SHA256_DIGEST_LENGTH];
|
||||
int retval;
|
||||
int i;
|
||||
uint64_t last_target_revision;
|
||||
char *cachefile;
|
||||
|
||||
mfpatch **patches = NULL;
|
||||
|
||||
retval = mfconn_api_device_get_updates(conn, quickkey,
|
||||
local_revision, remote_revision,
|
||||
&patches);
|
||||
mfconn_update_secret_key(conn);
|
||||
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "device/get_updates api call unsuccessful\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
// if no patches are returned, then the full file has to be downloaded
|
||||
if (patches[0] == NULL) {
|
||||
free(patches);
|
||||
|
||||
retval = filecache_download_file(filecache_path, quickkey, remote_revision, conn);
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "filecache_download_file failed\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
last_target_revision = local_revision;
|
||||
// else, go through all patches and download and apply them
|
||||
for (i = 0; patches[i] != NULL; i++) {
|
||||
/* verify that the source revision is equal to the last target
|
||||
* revision */
|
||||
if (patch_get_source_revision(patches[i]) != last_target_revision) {
|
||||
fprintf(stderr, "the source revision is unequal the last target revision\n");
|
||||
break;
|
||||
}
|
||||
last_target_revision = patch_get_target_revision(patches[i]);
|
||||
|
||||
retval = filecache_download_patch(conn, quickkey, patch_get_source_revision(patches[i]), patch_get_target_revision(patches[i]), patch_get_hash(patches[i]), filecache_path);
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "filecache_download_patch failed\n");
|
||||
break;
|
||||
}
|
||||
|
||||
/* verify that the file to patch has the right hash */
|
||||
cachefile =
|
||||
strdup_printf("%s/%s_%d", filecache_path, quickkey,
|
||||
patch_get_source_revision(patches[i]));
|
||||
hex2binary(patch_get_source_hash(patches[i]), hash2);
|
||||
retval = filecache_check_integrity_hash(cachefile, hash2);
|
||||
free(cachefile);
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "the source file has the wrong hash\n");
|
||||
break;
|
||||
}
|
||||
|
||||
/* now apply the patch in patchfile to the file in cachefile */
|
||||
retval = filecache_patch_file(filecache_path, quickkey, patch_get_source_revision(patches[i]), patch_get_target_revision(patches[i]));
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "filecache_patch_file failed\n");
|
||||
break;
|
||||
}
|
||||
|
||||
/* verify that the patched file has the right hash */
|
||||
cachefile =
|
||||
strdup_printf("%s/%s_%d", filecache_path, quickkey,
|
||||
patch_get_target_revision(patches[i]));
|
||||
hex2binary(patch_get_target_hash(patches[i]), hash2);
|
||||
retval = filecache_check_integrity_hash(cachefile, hash2);
|
||||
free(cachefile);
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "the target file has the wrong hash\n");
|
||||
break;
|
||||
}
|
||||
|
||||
free(patches[i]);
|
||||
}
|
||||
|
||||
/* check if the terminating NULL was reached or if processing was aborted
|
||||
* before that */
|
||||
if (patches[i] != NULL) {
|
||||
for (; patches[i] != NULL; i++) free(patches[i]);
|
||||
free(patches);
|
||||
return -1;
|
||||
}
|
||||
|
||||
free(patches);
|
||||
|
||||
/* verify that the last target revision is equal to the requested remote
|
||||
* revision */
|
||||
if (last_target_revision != remote_revision) {
|
||||
fprintf(stderr, "last_target_revision is not equal to the requested remote revision\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int filecache_download_patch(mfconn * conn, const char * quickkey, uint64_t source_revision, uint64_t target_revision, const char * phash, const char * filecache_path)
|
||||
{
|
||||
mfpatch *patch;
|
||||
const char *url;
|
||||
mfhttp *http;
|
||||
int retval;
|
||||
char *patchfile;
|
||||
unsigned char hash2[SHA256_DIGEST_LENGTH];
|
||||
|
||||
/* first retrieve the patch url */
|
||||
patch = patch_alloc();
|
||||
retval = mfconn_api_device_get_patch(conn, patch, quickkey,
|
||||
source_revision,
|
||||
target_revision);
|
||||
mfconn_update_secret_key(conn);
|
||||
|
||||
if (retval == -1) {
|
||||
fprintf(stderr, "mfconn_api_device_get_patch failed\n");
|
||||
patch_free(patch);
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* verify if the retrieved patch hash is the expected patch hash */
|
||||
if (strcmp(phash, patch_get_hash(patch)) != 0) {
|
||||
fprintf(stderr, "the expected patch hash is not equal the hash returned by device/get_patch\n");
|
||||
patch_free(patch);
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* then download the patch */
|
||||
url = patch_get_link(patch);
|
||||
|
||||
if (url == NULL || url[0] == '\0') {
|
||||
fprintf(stderr, "patch_get_link failed\n");
|
||||
patch_free(patch);
|
||||
return -1;
|
||||
}
|
||||
|
||||
patchfile =
|
||||
strdup_printf("%s/%s_patch_%d_%d", filecache_path, quickkey,
|
||||
source_revision, target_revision);
|
||||
|
||||
http = http_create();
|
||||
retval = http_get_file(http, url, patchfile);
|
||||
http_destroy(http);
|
||||
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "download failed\n");
|
||||
free(patchfile);
|
||||
patch_free(patch);
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* verify the integrity of the patch */
|
||||
hex2binary(patch_get_hash(patch), hash2);
|
||||
retval = filecache_check_integrity_hash(patchfile, hash2);
|
||||
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "filecache_check_integrity_hash failed for patch\n");
|
||||
patch_free(patch);
|
||||
return -1;
|
||||
}
|
||||
|
||||
patch_free(patch);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int filecache_patch_file(const char * filecache_path, const char * quickkey, uint64_t source_revision, uint64_t target_revision)
|
||||
{
|
||||
char *patchfile;
|
||||
char *sourcefile;
|
||||
char *targetfile;
|
||||
FILE *sourcefile_fh;
|
||||
FILE *patchfile_fh;
|
||||
FILE *targetfile_fh;
|
||||
int retval;
|
||||
|
||||
sourcefile =
|
||||
strdup_printf("%s/%s_%d", filecache_path, quickkey,
|
||||
source_revision);
|
||||
sourcefile_fh = fopen(sourcefile, "r");
|
||||
if (sourcefile_fh == NULL) {
|
||||
fprintf(stderr, "cannot open %s\n", sourcefile);
|
||||
free(sourcefile);
|
||||
return -1;
|
||||
}
|
||||
|
||||
free(sourcefile);
|
||||
|
||||
patchfile =
|
||||
strdup_printf("%s/%s_patch_%d_%d", filecache_path, quickkey,
|
||||
source_revision, target_revision);
|
||||
patchfile_fh = fopen(patchfile, "r");
|
||||
if (patchfile_fh == NULL) {
|
||||
fprintf(stderr, "cannot open %s\n", patchfile);
|
||||
free(patchfile);
|
||||
fclose(sourcefile_fh);
|
||||
return -1;
|
||||
}
|
||||
|
||||
free(patchfile);
|
||||
|
||||
targetfile =
|
||||
strdup_printf("%s/%s_%d", filecache_path, quickkey,
|
||||
target_revision);
|
||||
targetfile_fh = fopen(targetfile, "w");
|
||||
if (targetfile_fh == NULL) {
|
||||
fprintf(stderr, "cannot open %s\n", targetfile);
|
||||
fclose(sourcefile_fh);
|
||||
fclose(patchfile_fh);
|
||||
free(targetfile);
|
||||
return -1;
|
||||
}
|
||||
|
||||
free(targetfile);
|
||||
|
||||
retval = xdelta3_patch(sourcefile_fh, patchfile_fh, targetfile_fh);
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "unable to patch\n");
|
||||
fclose(sourcefile_fh);
|
||||
fclose(patchfile_fh);
|
||||
fclose(targetfile_fh);
|
||||
return -1;
|
||||
}
|
||||
|
||||
fclose(sourcefile_fh);
|
||||
fclose(patchfile_fh);
|
||||
fclose(targetfile_fh);
|
||||
|
||||
return 0;
|
||||
}
|
||||
27
fuse/filecache.h
Normal file
27
fuse/filecache.h
Normal file
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Copyright (C) 2014 Johannes Schauer <j.schauer@email.de>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify it
|
||||
* under the terms of the GNU General Public License version 2, as published by
|
||||
* the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
||||
* more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along with
|
||||
* this program; if not, write to the Free Software Foundation, Inc., 51
|
||||
* Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifndef __FUSE_FILECACHE_H__
|
||||
#define __FUSE_FILECACHE_H__
|
||||
|
||||
int filecache_open_file(const char * quickkey, uint64_t local_revision,
|
||||
uint64_t remote_revision, uint64_t fsize, const unsigned char * fhash,
|
||||
const char * filecache, mfconn * conn);
|
||||
|
||||
|
||||
#endif
|
||||
253
fuse/hashtbl.c
253
fuse/hashtbl.c
@@ -36,13 +36,16 @@
|
||||
#include <openssl/sha.h>
|
||||
|
||||
#include "hashtbl.h"
|
||||
#include "filecache.h"
|
||||
#include "../mfapi/mfconn.h"
|
||||
#include "../mfapi/file.h"
|
||||
#include "../mfapi/patch.h"
|
||||
#include "../mfapi/folder.h"
|
||||
#include "../mfapi/apicalls.h"
|
||||
#include "../utils/strings.h"
|
||||
#include "../utils/http.h"
|
||||
#include "../utils/hash.h"
|
||||
#include "../utils/xdelta3.h"
|
||||
|
||||
/*
|
||||
* we build a hashtable using the first three characters of the file or folder
|
||||
@@ -51,108 +54,6 @@
|
||||
*/
|
||||
#define NUM_BUCKETS 46656
|
||||
|
||||
/*
|
||||
* we use this table to convert from a base36 char (ignoring case) to an
|
||||
* integer or from a hex string to binary (in the latter case letters g-z and
|
||||
* G-Z remain unused)
|
||||
* we "waste" these 128 bytes of memory so that we don't need branching
|
||||
* instructions when decoding
|
||||
* we only need 128 bytes because the input is a *signed* char
|
||||
*/
|
||||
static unsigned char base36_decoding_table[] = {
|
||||
/* 0x00 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
/* 0x10 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
/* 0x20 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
/* 0x30 */ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0,
|
||||
/* 0x40 */ 0, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
|
||||
/* 0x50 */ 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 0, 0, 0, 0, 0,
|
||||
/* 0x60 */ 0, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
|
||||
/* 0x70 */ 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 0, 0, 0, 0, 0
|
||||
};
|
||||
|
||||
/*
|
||||
* table to convert from a byte into the two hexadecimal digits representing
|
||||
* it
|
||||
*/
|
||||
static char base16_encoding_table[][2] = {
|
||||
"00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "0A", "0B",
|
||||
"0C", "0D", "0E", "0F", "10", "11", "12", "13", "14", "15", "16", "17",
|
||||
"18", "19", "1A", "1B", "1C", "1D", "1E", "1F", "20", "21", "22", "23",
|
||||
"24", "25", "26", "27", "28", "29", "2A", "2B", "2C", "2D", "2E", "2F",
|
||||
"30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "3A", "3B",
|
||||
"3C", "3D", "3E", "3F", "40", "41", "42", "43", "44", "45", "46", "47",
|
||||
"48", "49", "4A", "4B", "4C", "4D", "4E", "4F", "50", "51", "52", "53",
|
||||
"54", "55", "56", "57", "58", "59", "5A", "5B", "5C", "5D", "5E", "5F",
|
||||
"60", "61", "62", "63", "64", "65", "66", "67", "68", "69", "6A", "6B",
|
||||
"6C", "6D", "6E", "6F", "70", "71", "72", "73", "74", "75", "76", "77",
|
||||
"78", "79", "7A", "7B", "7C", "7D", "7E", "7F", "80", "81", "82", "83",
|
||||
"84", "85", "86", "87", "88", "89", "8A", "8B", "8C", "8D", "8E", "8F",
|
||||
"90", "91", "92", "93", "94", "95", "96", "97", "98", "99", "9A", "9B",
|
||||
"9C", "9D", "9E", "9F", "A0", "A1", "A2", "A3", "A4", "A5", "A6", "A7",
|
||||
"A8", "A9", "AA", "AB", "AC", "AD", "AE", "AF", "B0", "B1", "B2", "B3",
|
||||
"B4", "B5", "B6", "B7", "B8", "B9", "BA", "BB", "BC", "BD", "BE", "BF",
|
||||
"C0", "C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9", "CA", "CB",
|
||||
"CC", "CD", "CE", "CF", "D0", "D1", "D2", "D3", "D4", "D5", "D6", "D7",
|
||||
"D8", "D9", "DA", "DB", "DC", "DD", "DE", "DF", "E0", "E1", "E2", "E3",
|
||||
"E4", "E5", "E6", "E7", "E8", "E9", "EA", "EB", "EC", "ED", "EE", "EF",
|
||||
"F0", "F1", "F2", "F3", "F4", "F5", "F6", "F7", "F8", "F9", "FA", "FB",
|
||||
"FC", "FD", "FE", "FF"
|
||||
};
|
||||
|
||||
/*
|
||||
* a macro to convert a char* of the key into a hash of its first three
|
||||
* characters, treating those first three characters as if they represented a
|
||||
* number in base36
|
||||
*
|
||||
* in the future this could be made more dynamic by using the ability of
|
||||
* strtoll to convert numbers of base36 and then only retrieving the desired
|
||||
* amount of high-bits for the desired size of the hashtable
|
||||
*/
|
||||
#define HASH_OF_KEY(key) base36_decoding_table[(int)(key)[0]]*36*36+\
|
||||
base36_decoding_table[(int)(key)[1]]*36+\
|
||||
base36_decoding_table[(int)(key)[2]]
|
||||
|
||||
/* decodes a zero terminated string containing hex characters into their
|
||||
* binary representation. The length of the string must be even as pairs of
|
||||
* characters are converted to one output byte. The output buffer must be at
|
||||
* least half the length of the input string.
|
||||
*/
|
||||
static void hex2binary(const char *hex, unsigned char *binary)
|
||||
{
|
||||
unsigned char val1,
|
||||
val2;
|
||||
const char *c1,
|
||||
*c2;
|
||||
unsigned char *b;
|
||||
|
||||
for (b = binary, c1 = hex, c2 = hex + 1;
|
||||
*c1 != '\0' && *c2 != '\0'; b++, c1 += 2, c2 += 2) {
|
||||
val1 = base36_decoding_table[(int)(*c1)];
|
||||
val2 = base36_decoding_table[(int)(*c2)];
|
||||
*b = (val1 << 4) | val2;
|
||||
}
|
||||
}
|
||||
|
||||
static char *binary2hex(unsigned char *binary, size_t length)
|
||||
{
|
||||
char *out;
|
||||
char *p;
|
||||
size_t i;
|
||||
|
||||
out = malloc(length * 2 + 1);
|
||||
if (out == NULL) {
|
||||
fprintf(stderr, "cannot allocate memory\n");
|
||||
return NULL;
|
||||
}
|
||||
for (i = 0; i < length; i++) {
|
||||
p = base16_encoding_table[binary[i]];
|
||||
out[i * 2] = p[0];
|
||||
out[i * 2 + 1] = p[1];
|
||||
}
|
||||
out[length * 2] = '\0';
|
||||
return out;
|
||||
}
|
||||
|
||||
struct h_entry {
|
||||
/*
|
||||
* keys are either 13 (folders) or 15 (files) long since the structure
|
||||
@@ -165,12 +66,13 @@ struct h_entry {
|
||||
/* char parent[20]; */
|
||||
/* local revision */
|
||||
uint64_t remote_revision;
|
||||
/* the revision of the local version. For folders, this is the last
|
||||
* revision for which the folder contents have been retrieved. For files
|
||||
* this is the last revision for which the file contents have been
|
||||
* retrieved */
|
||||
uint64_t local_revision;
|
||||
/* creation time */
|
||||
uint64_t ctime;
|
||||
/* Whether or not this h_entry needs to be updated before it can be used.
|
||||
* This value is set to true when directories and files are added as
|
||||
* children of a directory but their content has not bee retrieved yet */
|
||||
uint64_t local_revision;
|
||||
/* the containing folder */
|
||||
union {
|
||||
/* during runtime this is a pointer to the containing h_entry struct */
|
||||
@@ -350,7 +252,7 @@ int folder_tree_store(folder_tree * tree, FILE * stream)
|
||||
if (tmp_parent == &(tree->root)) {
|
||||
tree->buckets[i][j]->parent_offs = 0;
|
||||
} else {
|
||||
bucket_id = HASH_OF_KEY(tmp_parent->key);
|
||||
bucket_id = base36_decode_triplet(tmp_parent->key);
|
||||
found = false;
|
||||
for (k = 0; k < tree->bucket_lens[bucket_id]; k++) {
|
||||
if (tree->buckets[bucket_id][k] == tmp_parent) {
|
||||
@@ -489,7 +391,7 @@ folder_tree *folder_tree_load(FILE * stream, char *filecache)
|
||||
parent->children[parent->num_children - 1] = ordered_entries[i];
|
||||
|
||||
/* put the entry into the hashtable */
|
||||
bucket_id = HASH_OF_KEY(ordered_entries[i]->key);
|
||||
bucket_id = base36_decode_triplet(ordered_entries[i]->key);
|
||||
tree->bucket_lens[bucket_id]++;
|
||||
tree->buckets[bucket_id] =
|
||||
(struct h_entry **)realloc(tree->buckets[bucket_id],
|
||||
@@ -564,7 +466,7 @@ static struct h_entry *folder_tree_lookup_key(folder_tree * tree,
|
||||
return &(tree->root);
|
||||
}
|
||||
/* retrieve the right bucket for this key */
|
||||
bucket_id = HASH_OF_KEY(key);
|
||||
bucket_id = base36_decode_triplet(key);
|
||||
|
||||
for (i = 0; i < tree->bucket_lens[bucket_id]; i++) {
|
||||
if (strcmp(tree->buckets[bucket_id][i]->key, key) == 0) {
|
||||
@@ -630,7 +532,7 @@ static struct h_entry *folder_tree_lookup_path(folder_tree * tree,
|
||||
result = curr_dir->children[i];
|
||||
|
||||
// make sure that result is up to date
|
||||
if (result->atime == 0 &&
|
||||
if (result->atime == 0
|
||||
&& result->local_revision != result->remote_revision) {
|
||||
folder_tree_rebuild_helper(tree, conn, result);
|
||||
}
|
||||
@@ -819,17 +721,7 @@ int folder_tree_readdir(folder_tree * tree, mfconn * conn, const char *path,
|
||||
int folder_tree_open_file(folder_tree * tree, mfconn * conn, const char *path)
|
||||
{
|
||||
struct h_entry *entry;
|
||||
char *cachefile;
|
||||
int fd;
|
||||
struct stat file_info;
|
||||
uint64_t bytes_read;
|
||||
const char *url;
|
||||
mffile *file;
|
||||
int retval;
|
||||
mfhttp *http;
|
||||
FILE *fh;
|
||||
unsigned char hash[SHA256_DIGEST_LENGTH];
|
||||
char *hexhash;
|
||||
int retval;
|
||||
|
||||
entry = folder_tree_lookup_path(tree, conn, path);
|
||||
|
||||
@@ -837,116 +729,23 @@ int folder_tree_open_file(folder_tree * tree, mfconn * conn, const char *path)
|
||||
if (entry == NULL || entry->atime == 0) {
|
||||
return -ENOENT;
|
||||
}
|
||||
fprintf(stderr, "opening %s with local %" PRIu64 " and remote %" PRIu64
|
||||
"\n", entry->key, entry->local_revision, entry->remote_revision);
|
||||
|
||||
/* TODO: check entry->needs_update
|
||||
* check if local size is equal to remote size
|
||||
* check if local version exists and needs updating */
|
||||
|
||||
/* check if the requested file is already in the cache */
|
||||
cachefile =
|
||||
strdup_printf("%s/%s_%d", tree->filecache, entry->key,
|
||||
entry->remote_revision);
|
||||
fd = open(cachefile, O_RDWR);
|
||||
if (fd > 0) {
|
||||
/* file existed - return handle */
|
||||
free(cachefile);
|
||||
return fd;
|
||||
}
|
||||
|
||||
/* download the file */
|
||||
file = file_alloc();
|
||||
retval = mfconn_api_file_get_links(conn, file, (char *)entry->key);
|
||||
mfconn_update_secret_key(conn);
|
||||
|
||||
retval = filecache_open_file(entry->key, entry->local_revision,
|
||||
entry->remote_revision, entry->fsize,
|
||||
entry->hash, tree->filecache, conn);
|
||||
if (retval == -1) {
|
||||
fprintf(stderr, "mfconn_api_file_get_links failed\n");
|
||||
free(cachefile);
|
||||
file_free(file);
|
||||
fprintf(stderr, "filecache_open_file failed\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
url = file_get_direct_link(file);
|
||||
/* make sure that the local_revision is equal to the remote revision
|
||||
* because filecache_open_file took care of doing any updating if it was
|
||||
* necessary */
|
||||
entry->local_revision = entry->remote_revision;
|
||||
|
||||
if (url == NULL) {
|
||||
fprintf(stderr, "file_get_direct_link failed\n");
|
||||
free(cachefile);
|
||||
file_free(file);
|
||||
return -1;
|
||||
}
|
||||
|
||||
http = http_create();
|
||||
retval = http_get_file(http, url, cachefile);
|
||||
http_destroy(http);
|
||||
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "download failed\n");
|
||||
free(cachefile);
|
||||
file_free(file);
|
||||
return -1;
|
||||
}
|
||||
|
||||
memset(&file_info, 0, sizeof(file_info));
|
||||
retval = stat(cachefile, &file_info);
|
||||
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "stat failed\n");
|
||||
free(cachefile);
|
||||
file_free(file);
|
||||
return -1;
|
||||
}
|
||||
|
||||
bytes_read = file_info.st_size;
|
||||
|
||||
if (bytes_read != entry->fsize) {
|
||||
fprintf(stderr,
|
||||
"expected %" PRIu64 " bytes but got %" PRIu64 " bytes\n",
|
||||
entry->fsize, bytes_read);
|
||||
free(cachefile);
|
||||
file_free(file);
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* size matches - now compare the hash as well */
|
||||
fh = fopen(cachefile, "r");
|
||||
if (fh == NULL) {
|
||||
perror("cannot open file");
|
||||
free(cachefile);
|
||||
file_free(file);
|
||||
return -1;
|
||||
}
|
||||
|
||||
retval = calc_sha256(fh, hash);
|
||||
if (retval != 0) {
|
||||
fprintf(stderr, "failed to calculate hash\n");
|
||||
free(cachefile);
|
||||
file_free(file);
|
||||
fclose(fh);
|
||||
return -1;
|
||||
}
|
||||
|
||||
fclose(fh);
|
||||
|
||||
if (memcmp(entry->hash, hash, SHA256_DIGEST_LENGTH) != 0) {
|
||||
fprintf(stderr, "hashes are not equal\n");
|
||||
hexhash = binary2hex(entry->hash, SHA256_DIGEST_LENGTH);
|
||||
fprintf(stderr, "remote: %s\n", hexhash);
|
||||
free(hexhash);
|
||||
hexhash = binary2hex(hash, SHA256_DIGEST_LENGTH);
|
||||
fprintf(stderr, "downloaded: %s\n", hexhash);
|
||||
free(hexhash);
|
||||
free(cachefile);
|
||||
file_free(file);
|
||||
return -1;
|
||||
}
|
||||
|
||||
file_free(file);
|
||||
|
||||
fd = open(cachefile, O_RDWR);
|
||||
|
||||
free(cachefile);
|
||||
|
||||
/* return the file handle */
|
||||
return fd;
|
||||
return retval;
|
||||
}
|
||||
|
||||
static bool folder_tree_is_root(struct h_entry *entry)
|
||||
@@ -993,7 +792,7 @@ static struct h_entry *folder_tree_allocate_entry(folder_tree * tree,
|
||||
"key is NULL but this is fine, we just create it now\n");
|
||||
/* entry was not found, so append it to the end of the bucket */
|
||||
entry = (struct h_entry *)calloc(1, sizeof(struct h_entry));
|
||||
bucket_id = HASH_OF_KEY(key);
|
||||
bucket_id = base36_decode_triplet(key);
|
||||
tree->bucket_lens[bucket_id]++;
|
||||
tree->buckets[bucket_id] =
|
||||
realloc(tree->buckets[bucket_id],
|
||||
@@ -1328,7 +1127,7 @@ static void folder_tree_remove(folder_tree * tree, const char *key)
|
||||
return;
|
||||
}
|
||||
|
||||
bucket_id = HASH_OF_KEY(key);
|
||||
bucket_id = base36_decode_triplet(key);
|
||||
|
||||
/* check if the key exists */
|
||||
found = 0;
|
||||
|
||||
Reference in New Issue
Block a user