Merge pull request emscripten-core#23 from rstz/file_packager_readme

Added Readme for file packager

Author: rstz

pthreadfs/examples/Makefile

@@ -151,7 +151,7 @@ dist/$(NODEJSTESTS)/%.js : $(OBJ)/%.o $(OBJ)/pthreadfs.o $(PTHREADFS_JS)
 # - $(PACKAGERTESTS)/input/mediumlarge/subfolder/mediumfile.txt: Size 138670 bytes, first line "Begin mediumfile.txt -------------------------------------------"
 # - $(PACKAGERTESTS)/input/mediumlarge/bigfile.txt: Size 212992000 bytes, first line "Begin bigfile.txt ----------------------------------------------"
 .PHONY: packager-tests
-packager-tests: dist/$(PACKAGERTESTS)/preloading_without_pthreadfs.html dist/$(PACKAGERTESTS)/preloading.html dist/$(PACKAGERTESTS)/intermediate_loading.html
+packager-tests: dist/$(PACKAGERTESTS)/preloading_without_pthreadfs.html dist/$(PACKAGERTESTS)/preloading.html dist/$(PACKAGERTESTS)/load_package_sync.html dist/$(PACKAGERTESTS)/load_package_async.html
 
 # Create the file packages
 dist/$(PACKAGERTESTS)/pkg_preload_small.js: $(PACKAGER_INPUT_SMALL) $(FILE_PACKAGER)
@@ -177,7 +177,11 @@ dist/$(PACKAGERTESTS)/preloading.html: $(OBJ)/preloading.o $(OBJ)/pthreadfs.o di
 	mkdir -p dist/$(PACKAGERTESTS)
 	$(EMCC) $(LINK_FLAGS) -o $@ --js-library=$(PTHREADFS_JS)  --pre-js $(word 3,$^) --pre-js $(word 4,$^)  $< $(word 2,$^) 
 
-dist/$(PACKAGERTESTS)/intermediate_loading.html: $(OBJ)/intermediate_loading.o $(OBJ)/pthreadfs.o dist/$(PACKAGERTESTS)/pkg_intermediate_small.js dist/$(PACKAGERTESTS)/pkg_intermediate_mediumlarge.js $(PTHREADFS_JS)
+dist/$(PACKAGERTESTS)/load_package_sync.html: $(OBJ)/load_package_sync.o $(OBJ)/pthreadfs.o dist/$(PACKAGERTESTS)/pkg_intermediate_small.js dist/$(PACKAGERTESTS)/pkg_intermediate_mediumlarge.js $(PTHREADFS_JS)
+	mkdir -p dist/$(PACKAGERTESTS)
+	$(EMCC) $(LINK_FLAGS) -o $@ --js-library=$(PTHREADFS_JS) $< $(word 2,$^)
+
+dist/$(PACKAGERTESTS)/load_package_async.html: $(OBJ)/load_package_async.o $(OBJ)/pthreadfs.o dist/$(PACKAGERTESTS)/pkg_intermediate_small.js $(PTHREADFS_JS)
 	mkdir -p dist/$(PACKAGERTESTS)
 	$(EMCC) $(LINK_FLAGS) -o $@ --js-library=$(PTHREADFS_JS) $< $(word 2,$^)
 

pthreadfs/examples/packager-tests/load_package_async.cpp

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2021 The Emscripten Authors.  All rights reserved.
+ * Emscripten is available under two separate licenses, the MIT license and the
+ * University of Illinois/NCSA Open Source License.  Both these licenses can be
+ * found in the LICENSE file.
+ */
+
+
+#include <assert.h>
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <sys/stat.h>
+#include "pthreadfs.h"
+
+struct file_info {
+ public:
+  file_info(std::string path, std::string first_line, int size): path_(path), first_line_(first_line), size_(size) {} 
+  std::string path_;
+  std::string first_line_;
+  int size_;
+};
+
+void test(void* arg) {
+  file_info* file = (file_info*) arg;
+  std::cout << "Start reading first line of file " << file->path_ << std::endl;
+  std::ifstream stream(file->path_);
+  std::string read_line;
+  std::getline(stream, read_line);
+  stream.close();
+
+  std::cout << "  " << read_line << std::endl;
+  assert(read_line == file->first_line_);
+
+  struct stat s;
+  int err = stat(file->path_.c_str(), &s);
+  assert(!err);
+  assert(s.st_size == file->size_);
+  
+  std::cout << "Success.\n";
+}
+
+int main() {
+  std::cout << "Do some work before loading files.\n";
+
+  file_info* small_file  = new file_info("persistent/intermediate_loading/smallfile.txt", "These are the contents of a very small file.", 188);
+  file_info* medium_file  = new file_info("persistent/intermediate_loading/subfolder/mediumfile.txt",
+    "Begin mediumfile.txt -------------------------------------------", 138670);
+  file_info* big_file  = new file_info("persistent/intermediate_loading/bigfile.txt", "Begin bigfile.txt ----------------------------------------------",
+    212992000);
+
+  EM_ASM({
+    importScripts("pkg_intermediate_small.js");
+    PThreadFS.init('persistent').then(async () => {
+      let load_fct = Module["pthreadfs_available_packages"].pop();
+      await load_fct(); 
+
+      // Load the second function.
+      importScripts("pkg_intermediate_mediumlarge.js");
+      PThreadFS.init('persistent').then(async () => {
+        let load_fct = Module["pthreadfs_available_packages"].pop();
+        await load_fct(); 
+      });
+    });
+  });
+  // Wait 1 second for loading the small package.
+  emscripten_async_call(test, small_file, 1000);
+
+  // Wait another second for loading the larger package.
+  emscripten_async_call(test, medium_file, 2000);
+  emscripten_async_call(test, big_file, 2000);
+
+  return 0;
+}

pthreadfs/examples/packager-tests/intermediate_loading.cpp renamed to pthreadfs/examples/packager-tests/load_package_sync.cpp

@@ -0,0 +1,55 @@
+# PThreadFS File Packager Manual
+
+The PThreadFS file packager adapts [Emscripten's file packager](https://emscripten.org/docs/porting/files/packaging_files.html) for PThreadFS. Support for PThreadFS is activated with the switch `--use-pthreadfs`, e.g.
+```
+python3 file_packager.py --preload ./input/files@/persistent --use_pthreadfs --js-output=my_package.js my_package.data
+```
+
+The packaged files can be loaded into PThreadFS via multiple paths. In the remainder of this manual, `my_package.js` will be the _javascript helper_ file generated by the file packager (i.e., the file specified in the `--js-output=` switch). The generated `mypackage.data` file is called the _package_ and may hold multiple files.
+
+## Howto for common scenarios
+
+### Loading files on startup
+If files need to be available before the first file I/O (or very early during the program's execution), they can be loaded as soon as PThreadFS initializes. This can be achieved by running `my_package.js` before the Wasm module is instantiated (e.g. during `--pre-js`).
+
+It is possible to load multiple packages at startup by adding all of their respective javascript helpers before instantiation.
+
+See `examples/preloading.cpp` for an example.
+
+### Loading files during execution (synchronously)
+Additional packages can be loaded synchronously during execution by running the following from the WebAssembly module
+```c++
+#include "pthreadfs.h"
+// [other code]
+pthreadfs_load_package("my_package.js");
+```
+This command can also be triggered from the Javascript side by exporting the `pthreadfs_load_package` function. Calling `pthreadfs_load_package` from the Javascript's main thread is *unsupported*, since doing so may block the main thread. See the next section for alternatives.
+
+See `examples/load_package_sync.cpp` for an example.
+
+### Loading files during execution (asynchronously)
+
+It is possible to asynchronously load packages from a Javascript context. The first step is to execute `my_package.js` script (e.g. by using `importScripts()` in workers, or any other method). Right after that, it suffices to run the following code:
+```
+// This requires a JS context that has defined the `Module` object.
+await PThreadFS.init('persistent');
+let load_fct = Module["pthreadfs_available_packages"].pop();
+await load_fct();
+```
+Due to limitations of OPFS Access Handles, loading packages from the main thread may be quite slow. The expected slowdown is between 200% and 1000%. The same applies to calling PThreadFS functions such as `PThreadFS.writeFile()` directly. Asynchronously loading files during execution from a worker thread should not experience any slowdown when compared to loading on startup.
+
+See `examples/load_package_async.cpp` for an example.
+
+### Compiling the examples
+
+The examples for the file packager are found in `examples/packager-tests`.
+Building the packager tests requires manual creation of the following files:
+- packager-tests/input/small/smallfile.txt: Size 188 bytes, first line "These are the contents of a very small file."
+- packager-tests/input/mediumlarge/subfolder/mediumfile.txt: Size 138670 bytes, first line "Begin mediumfile.txt -------------------------------------------"
+- packager-tests/input/mediumlarge/bigfile.txt: Size 212992000 bytes, first line "Begin bigfile.txt ----------------------------------------------"
+
+After creating these files, the tests can be built by running `make packager-tests`.
+
+## Known limitations
+- File packager options `--embed` and `--lz4` cannot be used.
+- Asynchronously loading packages is not possible when using the MEMFS backend

pthreadfs/file_packager_readme.md

@@ -2438,6 +2438,10 @@ mergeInto(LibraryManager.library, SyscallsLibrary);
       }
     },
     init: async function(folder) {
+      if (PThreadFS.initialized) {
+        return;
+      }
+      PThreadFS.initialized = true;
       if (typeof folder !== 'string' || folder.includes('/')) {
         console.log("PThreadFS warning: Bad folder name: " + folder);
         console.log("                   The folder name should be a string that does not include /");

pthreadfs/library_pthreadfs.js

@@ -1521,6 +1521,10 @@
       }
     },
     init: async function(folder) {
+      if (PThreadFS.initialized) {
+        return;
+      }
+      PThreadFS.initialized = true;
       if (typeof folder !== 'string' || folder.includes('/')) {
         console.log("PThreadFS warning: Bad folder name: " + folder);
         console.log("                   The folder name should be a string that does not include /");