add local IDB caching of file contents

Edward Rudd

2014-10-14 b5bad6671e90a0bc055f8e4a357920a022341936

 client/library/library_cloudfs.js

@@ -23,22 +23,71 @@
            return MEMFS.mount.apply(null, arguments);
        },
        syncfs: function(mount, populate, callback) {
            if (mount.opts.disabled) {
                return callback(new Error("Syncing Disabled"));
          if (mount.opts.disabled) {
            return callback(new Error("Syncing Disabled"));
          }

          if (populate) {
            if (!mount.opts.disabled) {
              CLOUDFS.syncfs_load_from_cloud(mount, function (err) {
                if (err) return callback(err);
                CLOUDFS.syncfs_load_from_idb(mount, callback);
              });
            } else {
              CLOUDFS.syncfs_load_from_idb(mount, callback);
            }
          } else {
            CLOUDFS.syncfs_save_to_idb(mount, function(err) {
              if (err) return callback(err);
              if (!mount.opts.disabled) {
                CLOUDFS.syncfs_save_to_cloud(mount, callback);
              }
            });
          }
        },
        syncfs_load_from_cloud: function(mount, callback) {
          CLOUDFS.getRemoteSet(mount, function(err, remote) {
            if (err) return callback(err);

            CLOUDFS.getIDBSet(mount, function(err, idb) {
              if (err) return callback(err);

              CLOUDFS.reconcile(mount, remote, idb, callback);
            });
          });
        },
        syncfs_load_from_idb: function(mount, callback) {
          CLOUDFS.getIDBSet(mount, function(err, idb) {
            if (err) return callback(err);

            CLOUDFS.getLocalSet(mount, function(err, local) {
                if (err) return callback(err);
              if (err) return callback(err);

                CLOUDFS.getRemoteSet(mount, function(err, remote) {
                    if (err) return callback(err);

                    var src = populate ? remote : local;
                    var dst = populate ? local : remote;

                    CLOUDFS.reconcile(mount, src, dst, callback);
                });
              CLOUDFS.reconcile(mount, idb, local, callback);
            });
          });
        },
        syncfs_save_to_idb: function(mount, callback) {
          CLOUDFS.getLocalSet(mount, function(err, local) {
            if (err) return callback(err);

            CLOUDFS.getIDBSet(mount, function(err, idb) {
              if (err) return callback(err);

              CLOUDFS.reconcile(mount, local, idb, callback);
            });
          });
        },
        syncfs_save_to_cloud: function(mount, callback) {
          CLOUDFS.getIDBSet(mount, function(err, idb) {
            if (err) return callback(err);

            CLOUDFS.getRemoteSet(mount, function(err, remote) {
              if (err) return callback(err);

              CLOUDFS.reconcile(mount, idb, remote, callback);
            });
          });
        },
        // handling the diffing of "haves" and "have nots"
        reconcile: function(mount, src, dst, callback) {
@@ -83,31 +132,25 @@
            }
          };

          var srcFunc = CLOUDFS.getFuncSet(src.type),
            dstFunc = CLOUDFS.getFuncSet(dst.type);
          var srcCtx = srcFunc.start(mount, src, done),
            dstCtx = dstFunc.start(mount, dst, done);

          // sort paths in ascending order so directory entries are created
          // before the files inside them
          create.sort().forEach(function (path) {
            var pathinfo = src.entries[path];
            if (dst.type === 'local') {
              CLOUDFS.loadRemoteEntry(mount, pathinfo, function (err, entry) {
                if (err) return done(err);
                CLOUDFS.storeLocalEntry(path, entry, done);
              });
            } else {
              CLOUDFS.loadLocalEntry(path, function (err, entry) {
                if (err) return done(err);
                CLOUDFS.storeRemoteEntry(mount, pathinfo, entry, done);
              });
            }
            srcFunc.load(mount, srcCtx, pathinfo, function(err, entry) {
              if (err) return done(err);
              dstFunc.store(mount, dstCtx, pathinfo, entry, done);
            });
          });

          // sort paths in descending order so files are deleted before their
          // parent directories
          remove.sort().reverse().forEach(function(path) {
            if (dst.type === 'local') {
              CLOUDFS.removeLocalEntry(path, done);
            } else {
              CLOUDFS.removeRemoteEntry(mount, dst.entries[path], done);
            }
            dstFunc.remove(mount, dstCtx, dst.entries[path], done);
          });
        },
        // Utility functions
@@ -153,6 +196,80 @@
              prefix = p;
            });
          }
        },
        getFuncSet: function(type) {
          if (type == 'local') {
            return {
              start: function() {},
              load: CLOUDFS.loadLocalEntry,
              store: CLOUDFS.storeLocalEntry,
              remove: CLOUDFS.removeLocalEntry
            };
          } else if (type == 'remote') {
            return {
              start: function() {},
              load: CLOUDFS.loadRemoteEntry,
              store: CLOUDFS.storeRemoteEntry,
              remove: CLOUDFS.removeRemoteEntry
            };
          } else if (type == 'idb') {
            return {
              start: CLOUDFS.startIDBEntry,
              load: CLOUDFS.loadIDBEntry,
              store: CLOUDFS.storeIDBEntry,
              remove: CLOUDFS.removeIDBEntry
            };
          }
        },
        // Indexed DB Utility functions
        db: null,
        indexedDB: function() {
          if (typeof indexedDB !== 'undefined') return indexedDB;
          var ret = null;
          if (typeof window === 'object') ret = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB;
          assert(ret, 'CLOUDFS used, but indexedDB not supported');
          return ret;
        },
        DB_VERSION: 1,
        DB_NAME: 'CLOUDFS',
        DB_STORE_NAME: 'FILE_DATA',
        getDB: function(callback) {
          // check the cache first
          var db = CLOUDFS.db;
          if (db) {
            return callback(null, db);
          }

          var req;
          try {
            req = CLOUDFS.indexedDB().open(CLOUDFS.DB_NAME, CLOUDFS.DB_VERSION);
          } catch (e) {
            return callback(e);
          }
          req.onupgradeneeded = function(e) {
            var db = e.target.result;
            var transaction = e.target.transaction;

            var fileStore;

            if (db.objectStoreNames.contains(CLOUDFS.DB_STORE_NAME)) {
              fileStore = transaction.objectStore(CLOUDFS.DB_STORE_NAME);
            } else {
              fileStore = db.createObjectStore(CLOUDFS.DB_STORE_NAME);//, {keyPath: 'key'});
            }

            fileStore.createIndex('scope','scope',{ unique: false });
          };
          req.onsuccess = function() {
            db = req.result;

            // add to the cache
            CLOUDFS.db = db;
            callback(null, db);
          };
          req.onerror = function() {
            callback(this.error);
          };
        },
        // Getting list of entities
        getLocalSet: function(mount, callback) {
@@ -236,14 +353,18 @@
            var store = transaction.objectStore(CLOUDFS.DB_STORE_NAME);
            var index = store.index('scope');

            index.openKeyCursor(IDBKeyRange.only(mount.opts.scope)).onsuccess = function(event) {
            index.openCursor(IDBKeyRange.only(mount.opts.scope)).onsuccess = function(event) {
              var cursor = event.target.result;

              if (!cursor) {
                return callback(null, { type: 'idb', db: db, entries: entries });
              }

              entries[cursor.primaryKey] = { timestamp: cursor.key };
              entries[PATH.join2(mount.mountpoint, cursor.value.path)] = {
                path: cursor.value.path,
                type: FS.isDir(cursor.value.mode) ? 'dir' : 'file',
                timestamp: cursor.value.timestamp
              };

              cursor.continue();
            };
@@ -273,8 +394,9 @@
          });
        },
        // Local file access
        loadLocalEntry: function(path, callback) {
            var stat, node;
        loadLocalEntry: function(mount, ctx, pathinfo, callback) {
            var stat, node,
                path = PATH.join2(mount.mountpoint, pathinfo.path);

            try {
                var lookup = FS.lookupPath(path);
@@ -295,7 +417,8 @@
                return callback(new Error('node type not supported'));
            }
        },
        storeLocalEntry: function(path, entry, callback) {
        storeLocalEntry: function(mount, ctx, pathinfo, entry, callback) {
          var path = PATH.join2(mount.mountpoint, pathinfo.path);
          try {
            if (FS.isDir(entry.mode)) {
              try {
@@ -316,7 +439,8 @@

          callback(null);
        },
        removeLocalEntry: function(path, callback) {
        removeLocalEntry: function(mount, ctx, pathinfo, callback) {
          var path = PATH.join2(mount.mountpoint, pathinfo.path);
          try {
            var lookup = FS.lookupPath(path);
            var stat = FS.stat(path);
@@ -336,12 +460,57 @@

          callback(null);
        },
        // IDB File access
        buildIDBKey: function(mount, pathinfo) {
          return (mount.opts.scope || '') + ':' + pathinfo.path;
        },
        startIDBEntry: function(mount, dataset, ondone) {
          return {db: dataset.db};
        },
        loadIDBEntry: function(mount, ctx, pathinfo, callback) {
          var tx = ctx.db.transaction([CLOUDFS.DB_STORE_NAME], 'readwrite');
          tx.onerror = function() { callback(this.error); };
          var store = tx.objectStore(CLOUDFS.DB_STORE_NAME);

          var req = store.get(CLOUDFS.buildIDBKey(mount, pathinfo));
          req.onsuccess = function(event) { callback(null, event.target.result); };
          req.onerror = function() { callback(this.error); };
        },
        storeIDBEntry: function(mount, ctx, pathinfo, entry, callback) {
          var tx = ctx.db.transaction([CLOUDFS.DB_STORE_NAME], 'readwrite');
          tx.onerror = function() { callback(this.error); };
          var store = tx.objectStore(CLOUDFS.DB_STORE_NAME);

          // keep scope with entry
          var d = {
            scope: mount.opts.scope,
            path: pathinfo.path,
            mode: entry.mode,
            timestamp: entry.timestamp
          };
          if (entry.contents) d.contents = entry.contents;

          var req = store.put(d, CLOUDFS.buildIDBKey(mount, pathinfo));
          req.onsuccess = function() { callback(null); };
          req.onerror = function() { callback(this.error); };
        },
        removeIDBEntry: function(mount, ctx, pathinfo, callback) {
          var tx = ctx.db.transaction([CLOUDFS.DB_STORE_NAME], 'readwrite');
          tx.onerror = function() { callback(this.error); };
          var store = tx.objectStore(CLOUDFS.DB_STORE_NAME);

          var req = store.delete(CLOUDFS.buildIDBKey(mount, pathinfo));
          req.onsuccess = function() { callback(null); };
          req.onerror = function() { callback(this.error); };
        },
        // Remote file access
        loadRemoteEntry: function(mount, pathinfo, callback) {
        loadRemoteEntry: function(mount, ctx, pathinfo, callback) {
            if (pathinfo.type == 'file') {
                mount.opts.provider.read(mount.opts.cloud, pathinfo.url,
                    function(data) {
                        callback(null, { contents: data, timestamp: pathinfo.timestamp, mode: CLOUDFS._FILE_MODE });
                        // ensure data is in Uint8Array
                        var u8data = new Uint8Array(data);
                        callback(null, { contents: u8data, timestamp: pathinfo.timestamp, mode: CLOUDFS._FILE_MODE });
                    },
                    function(e) {
                        callback(e);
@@ -350,7 +519,7 @@
                callback(null, { timestamp: pathinfo.timestamp, mode: CLOUDFS._DIR_MODE });
            }
        },
        storeRemoteEntry: function(mount, pathinfo, entry, callback) {
        storeRemoteEntry: function(mount, ctx, pathinfo, entry, callback) {
            if (FS.isFile(entry.mode)) {
                mount.opts.provider.write(mount.opts.cloud, pathinfo, entry.contents, function() {
                    callback(null);
@@ -360,7 +529,7 @@
                })
            }
        },
        removeRemoteEntry: function(mount, pathinfo, callback) {
        removeRemoteEntry: function(mount, ctx, pathinfo, callback) {
          if (pathinfo.type == 'file') {
            mount.opts.provider.rm(mount.opts.cloud, pathinfo, function() {
              callback(null);