Filesystem access inside Web Workers

The HTML5 FileSystem API and Web Workers are powerful on their own. Combined, they enable file I/O and hierarchical storage inside a true multi-threaded JavaScript environment. A synchronous variant of the FileSystem API exists specifically for use inside Worker contexts, eliminating the need to layer one asynchronous API on top of another.

How the synchronous variant differs

The synchronous API is nearly identical to its asynchronous counterpart. Methods, properties, and features are the same; the key differences are:

  • The synchronous API is only available inside a Web Worker. The asynchronous API works both inside and outside Workers.
  • Callback-based methods are replaced with methods that return values directly.
  • Global methods become requestFileSystemSync() and resolveLocalFileSystemSyncURL(), rather than requestFileSystem() and resolveLocalFileSystemURL().

Requesting a filesystem and quota handling

From within a Worker, obtain a LocalFileSystemSync object using requestFileSystemSync(), available on the Worker's global scope. The lack of callbacks means the filesystem object is returned directly:

var fs = requestFileSystemSync(TEMPORARY, 1024*1024 /*1MB*/);

As with the asynchronous API, the methods remain prefixed for now:

self.requestFileSystemSync = self.webkitRequestFileSystemSync ||
                                 self.requestFileSystemSync;

One important limitation: PERSISTENT quota cannot be requested from within a Worker. Handle quota outside the Worker by coordinating through message passing:

  1. Wrap FileSystem code in the Worker in a try/catch to catch QUOTA_EXCEED_ERR errors.
  2. When quota errors occur, send a postMessage('get me more quota') to the main app.
  3. The main app requests additional storage via window.webkitStorageInfo.requestQuota().
  4. Once the user grants more quota, send postMessage('resume writes') back to the Worker.

Working with files and directories

Synchronous getFile() and getDirectory() calls return FileEntrySync and DirectoryEntrySync objects. Creating an empty file named log.txt in the root directory:

var fileEntry = fs.root.getFile('log.txt', {create: true});

Creating a directory in the root folder follows the same pattern:

var dirEntry = fs.root.getDirectory('mydir', {create: true});

Error handling and debugging

Debugging Worker code is difficult, and the absence of error callbacks makes it trickier still. Wrapping all relevant Worker code in a try/catch block and forwarding any errors to the main app via postMessage() simplifies the process:

function onError(e) {
    postMessage('ERROR: ' + e.toString());
}

try {
    // Error thrown if "log.txt" already exists.
    var fileEntry = fs.root.getFile('log.txt', {create: true, exclusive: true});
} catch (e) {
    onError(e);
}

Shifting binary data between app and Worker

Early Workers only supported string data in postMessage(), then serializable JSON. Browsers now support structured cloning, enabling binary data like Typed Arrays, ArrayBuffers, Files, and Blobs to pass between the main app and Worker. The data is still copied, but passing a File directly is a performance improvement over base64-encoding the contents beforehand.

This example passes a user-selected list of files to a dedicated Worker, which passes the list back. The main app reads each file as an ArrayBuffer. The example also demonstrates the inline Web Worker technique:

<!DOCTYPE html>
<html>
<head>
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="chrome=1">
    <title>Passing a FileList to a Worker</title>
    <script type="javascript/worker" id="fileListWorker">
    self.onmessage = function(e) {
    // TODO: do something interesting with the files.
    postMessage(e.data); // Pass through.
    };
    </script>
</head>
<body>
</body>

<input type="file" multiple>

<script>
document.querySelector('input[type="file"]').addEventListener('change', function(e) {
    var files = this.files;
    loadInlineWorker('#fileListWorker', function(worker) {

    // Setup handler to process messages from the worker.
    worker.onmessage = function(e) {

        // Read each file aysnc. as an array buffer.
        for (var i = 0, file; file = files[i]; ++i) {
        var reader = new FileReader();
        reader.onload = function(e) {
            console.log(this.result); // this.result is the read file as an ArrayBuffer.
        };
        reader.onerror = function(e) {
            console.log(e);
        };
        reader.readAsArrayBuffer(file);
        }

    };

    worker.postMessage(files);
    });
}, false);

function loadInlineWorker(selector, callback) {
    window.URL = window.URL || window.webkitURL || null;

    var script = document.querySelector(selector);
    if (script.type === 'javascript/worker') {
    var blob = new Blob([script.textContent]);
    callback(new Worker(window.URL.createObjectURL(blob));
    }
}
</script>
</html>

Reading files with FileReaderSync

While the asynchronous FileReader API works fine inside Workers, the synchronous FileReaderSync avoids callback nesting. The readAs* methods simply return the read data.

Main app:

<!DOCTYPE html>
<html>
<head>
    <title>Using FileReaderSync Example</title>
    <style>
    #error { color: red; }
    </style>
</head>
<body>
<input type="file" multiple />
<output id="error"></output>
<script>
    var worker = new Worker('worker.js');

    worker.onmessage = function(e) {
    console.log(e.data); // e.data should be an array of ArrayBuffers.
    };

    worker.onerror = function(e) {
    document.querySelector('#error').textContent = [
        'ERROR: Line ', e.lineno, ' in ', e.filename, ': ', e.message].join('');
    };

    document.querySelector('input[type="file"]').addEventListener('change', function(e) {
    worker.postMessage(this.files);
    }, false);
</script>
</body>
</html>

worker.js:

self.addEventListener('message', function(e) {
    var files = e.data;
    var buffers = [];

    // Read each file synchronously as an ArrayBuffer and
    // stash it in a global array to return to the main app.
    [].forEach.call(files, function(file) {
    var reader = new FileReaderSync();
    buffers.push(reader.readAsArrayBuffer(file));
    });

    postMessage(buffers);
}, false);

Available snippets not included

The source article contains additional code examples that are referenced but not included here.

Fetching all entries example:

self.requestFileSystemSync = self.webkitRequestFileSystemSync ||
                                self.requestFileSystemSync;

var paths = []; // Global to hold the list of entry filesystem URLs.

function getAllEntries(dirReader) {
    var entries = dirReader.readEntries();

    for (var i = 0, entry; entry = entries[i]; ++i) {
    paths.push(entry.toURL()); // Stash this entry's filesystem: URL.

    // If this is a directory, we have more traversing to do.
    if (entry.isDirectory) {
        getAllEntries(entry.createReader());
    }
    }
}

function onError(e) {
    postMessage('ERROR: ' + e.toString()); // Forward the error to main app.
}

self.onmessage = function(e) {
    var data = e.data;

    // Ignore everything else except our 'list' command.
    if (!data.cmd || data.cmd != 'list') {
    return;
    }

    try {
    var fs = requestFileSystemSync(TEMPORARY, 1024*1024 /*1MB*/);

    getAllEntries(fs.root.createReader());

    self.postMessage({entries: paths});
    } catch (e) {
    onError(e);
    }
};

Downloading files with XHR2 example:

self.requestFileSystemSync = self.webkitRequestFileSystemSync ||
                                self.requestFileSystemSync;

function makeRequest(url) {
    try {
    var xhr = new XMLHttpRequest();
    xhr.open('GET', url, false); // Note: synchronous
    xhr.responseType = 'arraybuffer';
    xhr.send();
    return xhr.response;
    } catch(e) {
    return "XHR Error " + e.toString();
    }
}

function onError(e) {
    postMessage('ERROR: ' + e.toString());
}

onmessage = function(e) {
    var data = e.data;

    // Make sure we have the right parameters.
    if (!data.fileName || !data.url || !data.type) {
    return;
    }
    
    try {
    var fs = requestFileSystemSync(TEMPORARY, 1024 * 1024 /*1MB*/);

    postMessage('Got file system.');

    var fileEntry = fs.root.getFile(data.fileName, {create: true});

    postMessage('Got file entry.');

    var arrayBuffer = makeRequest(data.url);
    var blob = new Blob([new Uint8Array(arrayBuffer)], {type: data.type});

    try {
        postMessage('Begin writing');
        fileEntry.createWriter().write(blob);
        postMessage('Writing complete');
        postMessage(fileEntry.toURL());
    } catch (e) {
        onError(e);
    }

    } catch (e) {
    onError(e);
    }
};