client/library/library_cloudfs.js

@@ -6,29 +6,88 @@
            var provider = CLOUDFS.fetchProvider(mount);
            if (provider) {
                mount.opts.provider = provider;
                if (!mount.opts.scope) {
                  // backwards compat
                  mount.opts.scope = mount.opts.cloud.applicationtoken;
                }
                mount.opts.cloud.scope = mount.opts.scope;
                Module.print('Cloud provider vendor: ' + provider.vendor);
                if (!provider.isAvailable(mount.opts.cloud)) {
                  mount.opts.disabled = true;
                  Module.print("WARNING: Cloud not available. Disabling Cloud Sync");
                }
            } else {
                mount.opts.disabled = true;
                Module.print("WARNING: Cloud provider not available. Disabling Sync");
                Module.print("WARNING: Cloud provider not available. Disabling Cloud Sync");
            }
            return MEMFS.mount.apply(null, arguments);
        },
        syncfs: function(mount, populate, callback) {
            if (mount.opts.disabled) {
                return callback(new Error("Syncing Disabled"));
          if (mount.opts.disabled) {
            return callback(new Error("Syncing Disabled"));
          }

          if (populate) {
            if (!mount.opts.disabled) {
              CLOUDFS.syncfs_load_from_cloud(mount, function (err) {
                if (err) return callback(err);
                CLOUDFS.syncfs_load_from_idb(mount, callback);
              });
            } else {
              CLOUDFS.syncfs_load_from_idb(mount, callback);
            }
          } else {
            CLOUDFS.syncfs_save_to_idb(mount, function(err) {
              if (err) return callback(err);
              if (!mount.opts.disabled) {
                CLOUDFS.syncfs_save_to_cloud(mount, callback);
              }
            });
          }
        },
        syncfs_load_from_cloud: function(mount, callback) {
          CLOUDFS.getRemoteSet(mount, function(err, remote) {
            if (err) return callback(err);

            CLOUDFS.getIDBSet(mount, function(err, idb) {
              if (err) return callback(err);

              CLOUDFS.reconcile(mount, remote, idb, callback);
            });
          });
        },
        syncfs_load_from_idb: function(mount, callback) {
          CLOUDFS.getIDBSet(mount, function(err, idb) {
            if (err) return callback(err);

            CLOUDFS.getLocalSet(mount, function(err, local) {
                if (err) return callback(err);
              if (err) return callback(err);

                CLOUDFS.getRemoteSet(mount, function(err, remote) {
                    if (err) return callback(err);

                    var src = populate ? remote : local;
                    var dst = populate ? local : remote;

                    CLOUDFS.reconcile(mount, src, dst, callback);
                });
              CLOUDFS.reconcile(mount, idb, local, callback);
            });
          });
        },
        syncfs_save_to_idb: function(mount, callback) {
          CLOUDFS.getLocalSet(mount, function(err, local) {
            if (err) return callback(err);

            CLOUDFS.getIDBSet(mount, function(err, idb) {
              if (err) return callback(err);

              CLOUDFS.reconcile(mount, local, idb, callback);
            });
          });
        },
        syncfs_save_to_cloud: function(mount, callback) {
          CLOUDFS.getIDBSet(mount, function(err, idb) {
            if (err) return callback(err);

            CLOUDFS.getRemoteSet(mount, function(err, remote) {
              if (err) return callback(err);

              CLOUDFS.reconcile(mount, idb, remote, callback);
            });
          });
        },
        // handling the diffing of "haves" and "have nots"
        reconcile: function(mount, src, dst, callback) {
@@ -73,31 +132,25 @@
            }
          };

          var srcFunc = CLOUDFS.getFuncSet(src.type),
            dstFunc = CLOUDFS.getFuncSet(dst.type);
          var srcCtx = srcFunc.start(mount, src, done),
            dstCtx = dstFunc.start(mount, dst, done);

          // sort paths in ascending order so directory entries are created
          // before the files inside them
          create.sort().forEach(function (path) {
            var pathinfo = src.entries[path];
            if (dst.type === 'local') {
              CLOUDFS.loadRemoteEntry(mount, pathinfo, function (err, entry) {
                if (err) return done(err);
                CLOUDFS.storeLocalEntry(path, entry, done);
              });
            } else {
              CLOUDFS.loadLocalEntry(path, function (err, entry) {
                if (err) return done(err);
                CLOUDFS.storeRemoteEntry(mount, pathinfo, entry, done);
              });
            }
            srcFunc.load(mount, srcCtx, pathinfo, function(err, entry) {
              if (err) return done(err);
              dstFunc.store(mount, dstCtx, pathinfo, entry, done);
            });
          });

          // sort paths in descending order so files are deleted before their
          // parent directories
          remove.sort().reverse().forEach(function(path) {
            if (dst.type === 'local') {
              CLOUDFS.removeLocalEntry(path, done);
            } else {
              CLOUDFS.removeRemoteEntry(mount, dst.entries[path], done);
            }
            dstFunc.remove(mount, dstCtx, dst.entries[path], done);
          });
        },
        // Utility functions
@@ -105,7 +158,7 @@
            var provider = CLOUD_PROVIDERS[provider_name];
            if (provider === undefined) return false;

            var requiredMethods = ['allFiles', 'read', 'write', 'rm'];
            var requiredMethods = ['allFiles', 'read', 'write', 'rm','isAvailable'];
            return requiredMethods.every(function(method) {
                return (method in provider);
            });
@@ -115,12 +168,108 @@
                return false;
            }
            if (CLOUDFS.validateProvider( mount.opts.provider ) ) {
                var provider = CLOUD_PROVIDERS[mount.opts.provider];
                Module.print('Cloud provider vendor: ' + provider.vendor);
                return provider;
                return CLOUD_PROVIDERS[mount.opts.provider];
            } else {
                return false;
            }
        },
        populateDirs: function(entries, f, toAbsolute) {
          if (f.path.indexOf('/') !== -1) {
            // we have folders.. stuff them in the list
            var parts = f.path.split('/'),
                prefix = '';
            // remove the "file" from the end
            parts.pop();
            // remove the empty directory from the beginning
            if (parts[0] == '') parts.shift();

            parts.forEach(function(e) {
              var p = prefix.length ? PATH.join2(prefix, e) : e,
                abs = toAbsolute(p);
              if (!(abs in entries)) {
                entries[abs] = {
                  path: p,
                  type: 'dir',
                  timestamp: f.timestamp
                };
              }
              prefix = p;
            });
          }
        },
        getFuncSet: function(type) {
          if (type == 'local') {
            return {
              start: function() {},
              load: CLOUDFS.loadLocalEntry,
              store: CLOUDFS.storeLocalEntry,
              remove: CLOUDFS.removeLocalEntry
            };
          } else if (type == 'remote') {
            return {
              start: function() {},
              load: CLOUDFS.loadRemoteEntry,
              store: CLOUDFS.storeRemoteEntry,
              remove: CLOUDFS.removeRemoteEntry
            };
          } else if (type == 'idb') {
            return {
              start: CLOUDFS.startIDBEntry,
              load: CLOUDFS.loadIDBEntry,
              store: CLOUDFS.storeIDBEntry,
              remove: CLOUDFS.removeIDBEntry
            };
          }
        },
        // Indexed DB Utility functions
        db: null,
        indexedDB: function() {
          if (typeof indexedDB !== 'undefined') return indexedDB;
          var ret = null;
          if (typeof window === 'object') ret = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB;
          assert(ret, 'CLOUDFS used, but indexedDB not supported');
          return ret;
        },
        DB_VERSION: 1,
        DB_NAME: 'CLOUDFS',
        DB_STORE_NAME: 'FILE_DATA',
        getDB: function(callback) {
          // check the cache first
          var db = CLOUDFS.db;
          if (db) {
            return callback(null, db);
          }

          var req;
          try {
            req = CLOUDFS.indexedDB().open(CLOUDFS.DB_NAME, CLOUDFS.DB_VERSION);
          } catch (e) {
            return callback(e);
          }
          req.onupgradeneeded = function(e) {
            var db = e.target.result;
            var transaction = e.target.transaction;

            var fileStore;

            if (db.objectStoreNames.contains(CLOUDFS.DB_STORE_NAME)) {
              fileStore = transaction.objectStore(CLOUDFS.DB_STORE_NAME);
            } else {
              fileStore = db.createObjectStore(CLOUDFS.DB_STORE_NAME);//, {keyPath: 'key'});
            }

            fileStore.createIndex('scope','scope',{ unique: false });
          };
          req.onsuccess = function() {
            db = req.result;

            // add to the cache
            CLOUDFS.db = db;
            callback(null, db);
          };
          req.onerror = function() {
            callback(this.error);
          };
        },
        // Getting list of entities
        getLocalSet: function(mount, callback) {
@@ -164,6 +313,7 @@
                var path = check.pop(),
                    stat,
                    keep = true,
                    is_dir = false,
                    abs_path = PATH.join2(mount.mountpoint, path);

                try {
@@ -174,6 +324,7 @@

                if (FS.isDir(stat.mode)) {
                    check.push.apply(check, FS.readdir(abs_path).filter(isRealDir).map(toAbsolute(path)));
                    is_dir = true;
                } else if (shouldFilter) {
                    keep = checkPath(path);
                }
@@ -181,12 +332,43 @@
                if (keep) {
                    entries[abs_path] = {
                        timestamp: stat.mtime,
                        path: path
                        path: path,
                        type: is_dir ? 'dir' : 'file'
                    };
                }
            }

            return callback(null, { type: 'local', entries: entries });
        },
        getIDBSet: function(mount, callback) {
          var entries = {},
              toAbsolute = function(p) { return PATH.join2(mount.mountpoint, p); };

          CLOUDFS.getDB(function(err, db) {
            if (err) return callback(err);

            var transaction = db.transaction([CLOUDFS.DB_STORE_NAME], 'readonly');
            transaction.onerror = function() { callback(this.error); };

            var store = transaction.objectStore(CLOUDFS.DB_STORE_NAME);
            var index = store.index('scope');

            index.openCursor(IDBKeyRange.only(mount.opts.scope)).onsuccess = function(event) {
              var cursor = event.target.result;

              if (!cursor) {
                return callback(null, { type: 'idb', db: db, entries: entries });
              }

              entries[PATH.join2(mount.mountpoint, cursor.value.path)] = {
                path: cursor.value.path,
                type: FS.isDir(cursor.value.mode) ? 'dir' : 'file',
                timestamp: cursor.value.timestamp
              };

              cursor.continue();
            };
          });
        },
        getRemoteSet: function(mount, callback) {
          mount.opts.provider.allFiles(mount.opts.cloud, function(data) {
@@ -195,28 +377,8 @@
            for(var k in data) {
              var f = data[k];

              if (f.path.indexOf('/') !== -1) {
                  // we have folders.. stuff them in the list
                  var parts = f.path.split('/'),
                      prefix = '';
                  // remove the "file" from the end
                  parts.pop();
                  // remove the empty directory from the beginning
                  if (parts[0] == '') parts.shift();
              CLOUDFS.populateDirs(entries, f, toAbsolute);

                  parts.forEach(function(e) {
                      var p = prefix.length ? PATH.join2(prefix, e) : e,
                          abs = toAbsolute(p);
                      if (!(abs in entries)) {
                          entries[abs] = {
                              path: p,
                              type: 'dir',
                              timestamp: f.timestamp
                          };
                      }
                      prefix = p;
                  });
              }
              var p = toAbsolute(f.path);
              entries[p] = {
                url: f.url,
@@ -231,9 +393,10 @@
            callback(e || new Error('failed request'));
          });
        },
        // Fetching local and remote files
        loadLocalEntry: function(path, callback) {
            var stat, node;
        // Local file access
        loadLocalEntry: function(mount, ctx, pathinfo, callback) {
            var stat, node,
                path = PATH.join2(mount.mountpoint, pathinfo.path);

            try {
                var lookup = FS.lookupPath(path);
@@ -254,53 +417,30 @@
                return callback(new Error('node type not supported'));
            }
        },
        loadRemoteEntry: function(mount, pathinfo, callback) {
            if (pathinfo.type == 'file') {
                mount.opts.provider.read(mount.opts.cloud, pathinfo.url,
                    function(data) {
                        callback(null, { contents: data, timestamp: pathinfo.timestamp, mode: {{{ cDefine('S_IFREG') | 0777 }}} });
                    },
                    function(e) {
                        callback(e);
                    });
        storeLocalEntry: function(mount, ctx, pathinfo, entry, callback) {
          var path = PATH.join2(mount.mountpoint, pathinfo.path);
          try {
            if (FS.isDir(entry.mode)) {
              try {
                FS.mkdir(path, entry.mode);
              } catch(e) {
                // ignore existing dirs
              }
            } else if (FS.isFile(entry.mode)) {
              FS.writeFile(path, entry.contents, { encoding: 'binary', canOwn: true });
            } else {
                callback(null, { timestamp: pathinfo.timestamp, mode: {{{ cDefine('S_IFDIR') | 0777 }}} });
            }
        },
        // storing local and remote files
        storeLocalEntry: function(path, entry, callback) {
            try {
                if (FS.isDir(entry.mode)) {
                    try {
                        FS.mkdir(path, entry.mode);
                    } catch(e) {
                        // ignore existing dirs
                    }
                } else if (FS.isFile(entry.mode)) {
                    FS.writeFile(path, entry.contents, { encoding: 'binary', canOwn: true });
                } else {
                    return callback(new Error('node type not supported'));
                }

                FS.utime(path, entry.timestamp, entry.timestamp);
            } catch (e) {
                return callback(e);
              return callback(new Error('node type not supported'));
            }

            callback(null);
            FS.utime(path, entry.timestamp, entry.timestamp);
          } catch (e) {
            return callback(e);
          }

          callback(null);
        },
        storeRemoteEntry: function(mount, pathinfo, entry, callback) {
            if (FS.isFile(entry.mode)) {
                mount.opts.provider.write(mount.opts.cloud, pathinfo, entry.contents, function() {
                    callback(null);
                },
                function(e) {
                    callback(e);
                })
            }
        },
        // remove local and remote files
        removeLocalEntry: function(path, callback) {
        removeLocalEntry: function(mount, ctx, pathinfo, callback) {
          var path = PATH.join2(mount.mountpoint, pathinfo.path);
          try {
            var lookup = FS.lookupPath(path);
            var stat = FS.stat(path);
@@ -320,7 +460,76 @@

          callback(null);
        },
        removeRemoteEntry: function(mount, pathinfo, callback) {
        // IDB File access
        buildIDBKey: function(mount, pathinfo) {
          return (mount.opts.scope || '') + ':' + pathinfo.path;
        },
        startIDBEntry: function(mount, dataset, ondone) {
          return {db: dataset.db};
        },
        loadIDBEntry: function(mount, ctx, pathinfo, callback) {
          var tx = ctx.db.transaction([CLOUDFS.DB_STORE_NAME], 'readwrite');
          tx.onerror = function() { callback(this.error); };
          var store = tx.objectStore(CLOUDFS.DB_STORE_NAME);

          var req = store.get(CLOUDFS.buildIDBKey(mount, pathinfo));
          req.onsuccess = function(event) { callback(null, event.target.result); };
          req.onerror = function() { callback(this.error); };
        },
        storeIDBEntry: function(mount, ctx, pathinfo, entry, callback) {
          var tx = ctx.db.transaction([CLOUDFS.DB_STORE_NAME], 'readwrite');
          tx.onerror = function() { callback(this.error); };
          var store = tx.objectStore(CLOUDFS.DB_STORE_NAME);

          // keep scope with entry
          var d = {
            scope: mount.opts.scope,
            path: pathinfo.path,
            mode: entry.mode,
            timestamp: entry.timestamp
          };
          if (entry.contents) d.contents = entry.contents;

          var req = store.put(d, CLOUDFS.buildIDBKey(mount, pathinfo));
          req.onsuccess = function() { callback(null); };
          req.onerror = function() { callback(this.error); };
        },
        removeIDBEntry: function(mount, ctx, pathinfo, callback) {
          var tx = ctx.db.transaction([CLOUDFS.DB_STORE_NAME], 'readwrite');
          tx.onerror = function() { callback(this.error); };
          var store = tx.objectStore(CLOUDFS.DB_STORE_NAME);

          var req = store.delete(CLOUDFS.buildIDBKey(mount, pathinfo));
          req.onsuccess = function() { callback(null); };
          req.onerror = function() { callback(this.error); };
        },
        // Remote file access
        loadRemoteEntry: function(mount, ctx, pathinfo, callback) {
            if (pathinfo.type == 'file') {
                mount.opts.provider.read(mount.opts.cloud, pathinfo.url,
                    function(data) {
                        // ensure data is in Uint8Array
                        var u8data = new Uint8Array(data);
                        callback(null, { contents: u8data, timestamp: pathinfo.timestamp, mode: CLOUDFS._FILE_MODE });
                    },
                    function(e) {
                        callback(e);
                    });
            } else {
                callback(null, { timestamp: pathinfo.timestamp, mode: CLOUDFS._DIR_MODE });
            }
        },
        storeRemoteEntry: function(mount, ctx, pathinfo, entry, callback) {
            if (FS.isFile(entry.mode)) {
                mount.opts.provider.write(mount.opts.cloud, pathinfo, entry.contents, function() {
                    callback(null);
                },
                function(e) {
                    callback(e);
                })
            }
        },
        removeRemoteEntry: function(mount, ctx, pathinfo, callback) {
          if (pathinfo.type == 'file') {
            mount.opts.provider.rm(mount.opts.cloud, pathinfo, function() {
              callback(null);
@@ -329,7 +538,9 @@
              callback(e);
            });
          }
        }
        },
        _FILE_MODE: {{{ cDefine('S_IFREG') | 0777 }}},
        _DIR_MODE: {{{ cDefine('S_IFDIR') | 0777 }}}
    }
};


 client/library/library_humble.js

@@ -1,6 +1,5 @@
var LibraryHUMBLE = {
    $HUMBLE_API: {
        file_cache: {},
        options: {
            /**
             * allows altering the incoming URL before it is fetched from the network

 client/test/main.cpp

@@ -5,6 +5,7 @@
#include "humble_api.h"

#include <dirent.h>
#include <sys/stat.h>

extern "C" {
    void test_list_files();
@@ -28,7 +29,9 @@
            if ((entry->d_type & DT_DIR)>0) {
                list_subfolder(folder + '/' + entry->d_name, entry->d_name, prefix + "  ");
            } else {
                std::cout << prefix << "  " << entry->d_name << "\n";
                struct stat st;
                stat((folder + '/' + entry->d_name).c_str(), &st);
                std::cout << prefix << "  " << entry->d_name << " S:" << st.st_size << "\n";
            }
        }
        closedir(d);

 client/test/shell.html

@@ -1237,12 +1237,6 @@
          if (element) element.value = ''; // clear browser cache
          return function(text) {
            text = Array.prototype.slice.call(arguments).join(' ');
            // These replacements are necessary if you render to raw HTML
            //text = text.replace(/&/g, "&amp;");
            //text = text.replace(/</g, "&lt;");
            //text = text.replace(/>/g, "&gt;");
            //text = text.replace('\n', '<br>', 'g');
//            console.log(text);
            if (element) {
              element.value += text + "\n";
              element.scrollTop = element.scrollHeight; // focus on bottom
@@ -1320,7 +1314,6 @@
      });
      Module['preRun'].push(function() {
          var provider_name = '@TEST_CLOUD_PROVIDER@';
          addRunDependency('CLOUDFS_setup');

          FS.createFolder('/', 'user_data', true, true);

@@ -1345,12 +1338,6 @@
          } else { // Cloud provider not available
              FS.mount(IDBFS, {}, '/user_data');
          }

          FS.syncfs(true, function(err) {
              if(err) console.log('ERROR!', err);
              console.log('finished syncing.. YEAH!!!');
              removeRunDependency('CLOUDFS_setup');
          });
      });
      var test_functions = {};
      var test_function_args = {
@@ -1369,13 +1356,13 @@
      function fetch_from_cloud()
      {
          FS.syncfs(true, function(e) {
              Module.print('Sync from Remote', e);
              console.log('Sync from Remote', e);
          });
      }
      function push_to_cloud()
      {
          FS.syncfs(function(e) {
              Module.print('Sync to Remote', e);
              console.log('Sync to Remote', e);
          });
      }
      function populate_user_data()

 humble_cloud/urkle_cloud.js

@@ -4,10 +4,10 @@

(function() {
    // @todo: This may need adjusting to support fetching binary.
    function xhrGET(url, onload, onerror) {
    function xhrGET(url, isBin, onload, onerror) {
        var xhr = new XMLHttpRequest();
        xhr.open('GET', url, true);
        xhr.responseType = 'text';
        xhr.responseType = isBin ? 'arraybuffer' : 'text';
        xhr.onload = function () {
            if (xhr.status == 200 || (xhr.status == 0 && xhr.response)) {
                onload(xhr.response);
@@ -54,7 +54,7 @@
    var provider = {
        vendor: 'Urkle!',
        allFiles: function(options, onsuccess, onerror) {
            xhrGET(settings.remoteAPIEndpoint + '/storage/files?appToken=' + encodeURIComponent(options.applicationtoken), function(data) {
            xhrGET(settings.remoteAPIEndpoint + '/storage/files?appToken=' + encodeURIComponent(options.applicationtoken), false, function(data) {
                var json = JSON.parse(data),
                    ret = [];
                ret = json.data.map(function(f) {
@@ -68,7 +68,7 @@
            }, onerror);
        },
        read: function(options, url, onsuccess, onerror) {
            xhrGET(url, onsuccess, onerror);
            xhrGET(url, true, onsuccess, onerror);
        },
        write: function(options, fileinfo, data, onsuccess, onerror) {
            var q = 'appToken=' + encodeURIComponent(options.applicationtoken)
@@ -78,6 +78,9 @@
        },
        rm: function(options, fileinfo, onsuccess, onerror) {
          xhrDELETE(settings.remoteAPIEndpoint + '/storage/files/' + fileinfo.path, onsuccess, onerror);
        },
        isAvailable: function(options) {
          return true;
        }
    };