From b5bad6671e90a0bc055f8e4a357920a022341936 Mon Sep 17 00:00:00 2001
From: Edward Rudd <urkle@outoforder.cc>
Date: Tue, 14 Oct 2014 14:48:46 +0000
Subject: [PATCH] add local IDB caching of file contents
---
client/library/library_cloudfs.js | 243 +++++++++++++++++++++++++++++++++++++++++-------
1 files changed, 206 insertions(+), 37 deletions(-)
diff --git a/client/library/library_cloudfs.js b/client/library/library_cloudfs.js
index c7f450c..a885949 100644
--- a/client/library/library_cloudfs.js
+++ b/client/library/library_cloudfs.js
@@ -23,22 +23,71 @@
return MEMFS.mount.apply(null, arguments);
},
syncfs: function(mount, populate, callback) {
- if (mount.opts.disabled) {
- return callback(new Error("Syncing Disabled"));
+ if (mount.opts.disabled) {
+ return callback(new Error("Syncing Disabled"));
+ }
+
+ if (populate) {
+ if (!mount.opts.disabled) {
+ CLOUDFS.syncfs_load_from_cloud(mount, function (err) {
+ if (err) return callback(err);
+ CLOUDFS.syncfs_load_from_idb(mount, callback);
+ });
+ } else {
+ CLOUDFS.syncfs_load_from_idb(mount, callback);
}
+ } else {
+ CLOUDFS.syncfs_save_to_idb(mount, function(err) {
+ if (err) return callback(err);
+ if (!mount.opts.disabled) {
+ CLOUDFS.syncfs_save_to_cloud(mount, callback);
+ }
+ });
+ }
+ },
+ syncfs_load_from_cloud: function(mount, callback) {
+ CLOUDFS.getRemoteSet(mount, function(err, remote) {
+ if (err) return callback(err);
+
+ CLOUDFS.getIDBSet(mount, function(err, idb) {
+ if (err) return callback(err);
+
+ CLOUDFS.reconcile(mount, remote, idb, callback);
+ });
+ });
+ },
+ syncfs_load_from_idb: function(mount, callback) {
+ CLOUDFS.getIDBSet(mount, function(err, idb) {
+ if (err) return callback(err);
CLOUDFS.getLocalSet(mount, function(err, local) {
- if (err) return callback(err);
+ if (err) return callback(err);
- CLOUDFS.getRemoteSet(mount, function(err, remote) {
- if (err) return callback(err);
-
- var src = populate ? remote : local;
- var dst = populate ? local : remote;
-
- CLOUDFS.reconcile(mount, src, dst, callback);
- });
+ CLOUDFS.reconcile(mount, idb, local, callback);
});
+ });
+ },
+ syncfs_save_to_idb: function(mount, callback) {
+ CLOUDFS.getLocalSet(mount, function(err, local) {
+ if (err) return callback(err);
+
+ CLOUDFS.getIDBSet(mount, function(err, idb) {
+ if (err) return callback(err);
+
+ CLOUDFS.reconcile(mount, local, idb, callback);
+ });
+ });
+ },
+ syncfs_save_to_cloud: function(mount, callback) {
+ CLOUDFS.getIDBSet(mount, function(err, idb) {
+ if (err) return callback(err);
+
+ CLOUDFS.getRemoteSet(mount, function(err, remote) {
+ if (err) return callback(err);
+
+ CLOUDFS.reconcile(mount, idb, remote, callback);
+ });
+ });
},
// handling the diffing of "haves" and "have nots"
reconcile: function(mount, src, dst, callback) {
@@ -83,31 +132,25 @@
}
};
+ var srcFunc = CLOUDFS.getFuncSet(src.type),
+ dstFunc = CLOUDFS.getFuncSet(dst.type);
+ var srcCtx = srcFunc.start(mount, src, done),
+ dstCtx = dstFunc.start(mount, dst, done);
+
// sort paths in ascending order so directory entries are created
// before the files inside them
create.sort().forEach(function (path) {
var pathinfo = src.entries[path];
- if (dst.type === 'local') {
- CLOUDFS.loadRemoteEntry(mount, pathinfo, function (err, entry) {
- if (err) return done(err);
- CLOUDFS.storeLocalEntry(path, entry, done);
- });
- } else {
- CLOUDFS.loadLocalEntry(path, function (err, entry) {
- if (err) return done(err);
- CLOUDFS.storeRemoteEntry(mount, pathinfo, entry, done);
- });
- }
+ srcFunc.load(mount, srcCtx, pathinfo, function(err, entry) {
+ if (err) return done(err);
+ dstFunc.store(mount, dstCtx, pathinfo, entry, done);
+ });
});
// sort paths in descending order so files are deleted before their
// parent directories
remove.sort().reverse().forEach(function(path) {
- if (dst.type === 'local') {
- CLOUDFS.removeLocalEntry(path, done);
- } else {
- CLOUDFS.removeRemoteEntry(mount, dst.entries[path], done);
- }
+ dstFunc.remove(mount, dstCtx, dst.entries[path], done);
});
},
// Utility functions
@@ -153,6 +196,80 @@
prefix = p;
});
}
+ },
+ getFuncSet: function(type) {
+ if (type == 'local') {
+ return {
+ start: function() {},
+ load: CLOUDFS.loadLocalEntry,
+ store: CLOUDFS.storeLocalEntry,
+ remove: CLOUDFS.removeLocalEntry
+ };
+ } else if (type == 'remote') {
+ return {
+ start: function() {},
+ load: CLOUDFS.loadRemoteEntry,
+ store: CLOUDFS.storeRemoteEntry,
+ remove: CLOUDFS.removeRemoteEntry
+ };
+ } else if (type == 'idb') {
+ return {
+ start: CLOUDFS.startIDBEntry,
+ load: CLOUDFS.loadIDBEntry,
+ store: CLOUDFS.storeIDBEntry,
+ remove: CLOUDFS.removeIDBEntry
+ };
+ }
+ },
+ // Indexed DB Utility functions
+ db: null,
+ indexedDB: function() {
+ if (typeof indexedDB !== 'undefined') return indexedDB;
+ var ret = null;
+ if (typeof window === 'object') ret = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB;
+ assert(ret, 'CLOUDFS used, but indexedDB not supported');
+ return ret;
+ },
+ DB_VERSION: 1,
+ DB_NAME: 'CLOUDFS',
+ DB_STORE_NAME: 'FILE_DATA',
+ getDB: function(callback) {
+ // check the cache first
+ var db = CLOUDFS.db;
+ if (db) {
+ return callback(null, db);
+ }
+
+ var req;
+ try {
+ req = CLOUDFS.indexedDB().open(CLOUDFS.DB_NAME, CLOUDFS.DB_VERSION);
+ } catch (e) {
+ return callback(e);
+ }
+ req.onupgradeneeded = function(e) {
+ var db = e.target.result;
+ var transaction = e.target.transaction;
+
+ var fileStore;
+
+ if (db.objectStoreNames.contains(CLOUDFS.DB_STORE_NAME)) {
+ fileStore = transaction.objectStore(CLOUDFS.DB_STORE_NAME);
+ } else {
+ fileStore = db.createObjectStore(CLOUDFS.DB_STORE_NAME);//, {keyPath: 'key'});
+ }
+
+ fileStore.createIndex('scope','scope',{ unique: false });
+ };
+ req.onsuccess = function() {
+ db = req.result;
+
+ // add to the cache
+ CLOUDFS.db = db;
+ callback(null, db);
+ };
+ req.onerror = function() {
+ callback(this.error);
+ };
},
// Getting list of entities
getLocalSet: function(mount, callback) {
@@ -236,14 +353,18 @@
var store = transaction.objectStore(CLOUDFS.DB_STORE_NAME);
var index = store.index('scope');
- index.openKeyCursor(IDBKeyRange.only(mount.opts.scope)).onsuccess = function(event) {
+ index.openCursor(IDBKeyRange.only(mount.opts.scope)).onsuccess = function(event) {
var cursor = event.target.result;
if (!cursor) {
return callback(null, { type: 'idb', db: db, entries: entries });
}
- entries[cursor.primaryKey] = { timestamp: cursor.key };
+ entries[PATH.join2(mount.mountpoint, cursor.value.path)] = {
+ path: cursor.value.path,
+ type: FS.isDir(cursor.value.mode) ? 'dir' : 'file',
+ timestamp: cursor.value.timestamp
+ };
cursor.continue();
};
@@ -273,8 +394,9 @@
});
},
// Local file access
- loadLocalEntry: function(path, callback) {
- var stat, node;
+ loadLocalEntry: function(mount, ctx, pathinfo, callback) {
+ var stat, node,
+ path = PATH.join2(mount.mountpoint, pathinfo.path);
try {
var lookup = FS.lookupPath(path);
@@ -295,7 +417,8 @@
return callback(new Error('node type not supported'));
}
},
- storeLocalEntry: function(path, entry, callback) {
+ storeLocalEntry: function(mount, ctx, pathinfo, entry, callback) {
+ var path = PATH.join2(mount.mountpoint, pathinfo.path);
try {
if (FS.isDir(entry.mode)) {
try {
@@ -316,7 +439,8 @@
callback(null);
},
- removeLocalEntry: function(path, callback) {
+ removeLocalEntry: function(mount, ctx, pathinfo, callback) {
+ var path = PATH.join2(mount.mountpoint, pathinfo.path);
try {
var lookup = FS.lookupPath(path);
var stat = FS.stat(path);
@@ -336,12 +460,57 @@
callback(null);
},
+ // IDB File access
+ buildIDBKey: function(mount, pathinfo) {
+ return (mount.opts.scope || '') + ':' + pathinfo.path;
+ },
+ startIDBEntry: function(mount, dataset, ondone) {
+ return {db: dataset.db};
+ },
+ loadIDBEntry: function(mount, ctx, pathinfo, callback) {
+ var tx = ctx.db.transaction([CLOUDFS.DB_STORE_NAME], 'readwrite');
+ tx.onerror = function() { callback(this.error); };
+ var store = tx.objectStore(CLOUDFS.DB_STORE_NAME);
+
+ var req = store.get(CLOUDFS.buildIDBKey(mount, pathinfo));
+ req.onsuccess = function(event) { callback(null, event.target.result); };
+ req.onerror = function() { callback(this.error); };
+ },
+ storeIDBEntry: function(mount, ctx, pathinfo, entry, callback) {
+ var tx = ctx.db.transaction([CLOUDFS.DB_STORE_NAME], 'readwrite');
+ tx.onerror = function() { callback(this.error); };
+ var store = tx.objectStore(CLOUDFS.DB_STORE_NAME);
+
+ // keep scope with entry
+ var d = {
+ scope: mount.opts.scope,
+ path: pathinfo.path,
+ mode: entry.mode,
+ timestamp: entry.timestamp
+ };
+ if (entry.contents) d.contents = entry.contents;
+
+ var req = store.put(d, CLOUDFS.buildIDBKey(mount, pathinfo));
+ req.onsuccess = function() { callback(null); };
+ req.onerror = function() { callback(this.error); };
+ },
+ removeIDBEntry: function(mount, ctx, pathinfo, callback) {
+ var tx = ctx.db.transaction([CLOUDFS.DB_STORE_NAME], 'readwrite');
+ tx.onerror = function() { callback(this.error); };
+ var store = tx.objectStore(CLOUDFS.DB_STORE_NAME);
+
+ var req = store.delete(CLOUDFS.buildIDBKey(mount, pathinfo));
+ req.onsuccess = function() { callback(null); };
+ req.onerror = function() { callback(this.error); };
+ },
// Remote file access
- loadRemoteEntry: function(mount, pathinfo, callback) {
+ loadRemoteEntry: function(mount, ctx, pathinfo, callback) {
if (pathinfo.type == 'file') {
mount.opts.provider.read(mount.opts.cloud, pathinfo.url,
function(data) {
- callback(null, { contents: data, timestamp: pathinfo.timestamp, mode: CLOUDFS._FILE_MODE });
+ // ensure data is in Uint8Array
+ var u8data = new Uint8Array(data);
+ callback(null, { contents: u8data, timestamp: pathinfo.timestamp, mode: CLOUDFS._FILE_MODE });
},
function(e) {
callback(e);
@@ -350,7 +519,7 @@
callback(null, { timestamp: pathinfo.timestamp, mode: CLOUDFS._DIR_MODE });
}
},
- storeRemoteEntry: function(mount, pathinfo, entry, callback) {
+ storeRemoteEntry: function(mount, ctx, pathinfo, entry, callback) {
if (FS.isFile(entry.mode)) {
mount.opts.provider.write(mount.opts.cloud, pathinfo, entry.contents, function() {
callback(null);
@@ -360,7 +529,7 @@
})
}
},
- removeRemoteEntry: function(mount, pathinfo, callback) {
+ removeRemoteEntry: function(mount, ctx, pathinfo, callback) {
if (pathinfo.type == 'file') {
mount.opts.provider.rm(mount.opts.cloud, pathinfo, function() {
callback(null);
--
Gitblit v1.9.3