Merge pull request emscripten-core#21 from rstz/remove-sfa-support

Removed Storage Foundation Support, Simplify file packager code

Author: rstz

pthreadfs/README.md

@@ -5,20 +5,20 @@ The Emscripten Pthread File System (PThreadFS) unlocks using (partly) asynchrono
 PThreadFS works by replacing Emscripten's file system API with a new API that proxies all file system operations to a dedicated pthread. This dedicated thread maintains a virtual file system that can use different APIs as backend (very similar to the way Emscripten's VFS is designed). In particular, PThreadFS comes with built-in support for asynchronous backends such as [OPFS Access Handles](https://docs.google.com/document/d/121OZpRk7bKSF7qU3kQLqAEUVSNxqREnE98malHYwWec/edit#heading=h.gj2fudnvy982).
 Although the underlying storage API is asynchronous, PThreadFS makes it appear synchronous to the C++ application.
 
-If OPFS Access Handles are not available, PThreadFS will attempt to use the [Storage Foundation API](https://github.com/WICG/storage-foundation-api-explainer). As a last fallback, Emscripten's MEMFS in-memory file system is used.
+If OPFS Access Handles are not available, PThreadFS will use an in-memory file system.
 
 The code is still prototype quality and **should not be used in a production environment** for the time being.
 
 ## Enable OPFS Access Handles in Chrome
 
-OPFS Access Handles require recent versions of Google Chrome Canary. "Experimental Web Platform Features" must be enabled in [chrome://flags](chrome://flags).
+OPFS Access Handles require Google Chrome 96. "Experimental Web Platform Features" must be enabled in [chrome://flags](chrome://flags).
 
 Alternatively, you may enable the API with Chrome's " --enable-runtime-features=FileSystemAccessAccessHandle". On MacOS, this can be done through
 ```
 open -a /Applications/Google\ Chrome\ Canary.app --args --enable-runtime-features=FileSystemAccessAccessHandle
 ```
 
-Both Storage Foundation API and OPFS are available as [origin trials](https://developer.chrome.com/origintrials/) in Chrome 95.
+OPFS Access Handles are available as [origin trial](https://developer.chrome.com/origintrials/) in Chrome 95.
 ## Getting the code
 
 PthreadFS is available on Github in the [emscripten-pthreadfs](https://github.com/rstz/emscripten-pthreadfs) repository. All code resides in the `pthreadfs` folder. It should be usable with any up-to-date Emscripten version. 
@@ -74,7 +74,6 @@ See `pthreadfs/examples/emscripten-tests/fsafs.cpp` for exemplary usage.
 - PThreadFS depends on C++ libraries. `EM_PTHREADFS_ASM()` cannot be used within C files.
 - Performance is good if and only if optimizations (compiler option `-O2`) are enabled and DevTools are closed.
 - Accessing the file system before `main()` requires linker option `PTHREAD_POOL_SIZE=<expression>` to be active. Doing so may lead to some blocking of the main thread, which is risky. Check out `examples/early_syscall.cpp` for an example.
-- The Storage Foundation backend requires case-insensitive file names.
 - Compiling with the Closure Compiler is not supported.
 - Compiling with optimization `-O3` is not yet supported and may lead to a faulty build.
 
@@ -98,23 +97,23 @@ python3 -m http.server 8888
 Then open the following link in a Chrome instance with the
 _OPFS Access Handles_ [enabled](#enable-and-detect-opfs-in-chrome):
 
-[localhost:8888/sqlite-speedtest](http://localhost:8888/sqlite-speedtest). The results of the speedtest can be found in the DevTools console.
+[localhost:8888/sqlite-speedtest](http://localhost:8888/sqlite-speedtest).
 
 ### Other tests
 
-The folder `pthreadfs/examples/emscripten-tests` contains a number of other file system tests, mostly taken from Emscripten's standard test suite.
+The folder `pthreadfs/examples/` contains a number of other tests.
 
 To compile, navigate to the `pthreadfs/examples/` directory and run
 
 ```shell
-make emscripten-tests
-cd dist/emscripten-tests
+make all
+cd dist/
 python3 -m http.server 8888
 ```
 Then open the following link in a Chrome instance with the
 _OPFS Access Handles_ [enabled](#enable-and-detect-opfs-in-chrome):
 
-[localhost:8888/emscripten-tests](http://localhost:8888/emscripten-tests) and choose a test. The results of the test can be found in the DevTools console.
+[localhost:8888](http://localhost:8888) and choose a test. The results of the test can be found in the DevTools console.
 
 ## Authors
 - Richard Stotz (<[email protected]>)

pthreadfs/examples/Makefile

@@ -28,7 +28,7 @@ SQLITE_SPEEDTEST_URL = https://sqlite.org/src/raw/5e5b805f24cc939656058f6a498f5a
 SQLITE_SPEEDTEST_SHA1 = f0edde2ad68f090e4676ac30042e0f6b765a8528
 
 .PHONY: all 
-all: sqlite-speedtest emscripten-tests packager-tests
+all: sqlite-speedtest emscripten-tests packager-tests jsops-tests
 
 ## cache
 cache/$(SQLITE_AMALGAMATION).zip:
@@ -179,4 +179,20 @@ dist/$(PACKAGERTESTS)/preloading.html: $(OBJ)/preloading.o $(OBJ)/pthreadfs.o di
 
 dist/$(PACKAGERTESTS)/intermediate_loading.html: $(OBJ)/intermediate_loading.o $(OBJ)/pthreadfs.o dist/$(PACKAGERTESTS)/pkg_intermediate_small.js dist/$(PACKAGERTESTS)/pkg_intermediate_mediumlarge.js $(PTHREADFS_JS)
 	mkdir -p dist/$(PACKAGERTESTS)
-	$(EMCC) $(LINK_FLAGS) -o $@ --js-library=$(PTHREADFS_JS) $< $(word 2,$^)
+	$(EMCC) $(LINK_FLAGS) -o $@ --js-library=$(PTHREADFS_JS) $< $(word 2,$^)
+
+JSOPSTESTS = jsops-tests
+.PHONY: jsops-tests
+jsops-tests: dist/$(JSOPSTESTS)/write_from_main_thread.html dist/$(JSOPSTESTS)/read_from_main_thread.html
+
+$(OBJ)/%.o : $(JSOPSTESTS)/%.cpp
+	mkdir -p $(OBJ)
+	$(EMCC) -c $(CFLAGS) $< -o $@
+
+dist/$(JSOPSTESTS)/write_from_main_thread.html: $(OBJ)/write_from_main_thread.o $(OBJ)/pthreadfs.o $(JSOPSTESTS)/write_from_main_thread_pre.js $(PTHREADFS_JS)
+	mkdir -p dist/$(JSOPSTESTS)
+	$(EMCC) $(LINK_FLAGS) -o $@ --js-library=$(PTHREADFS_JS) --pre-js $(word 3,$^) $< $(word 2,$^)
+
+dist/$(JSOPSTESTS)/read_from_main_thread.html: $(OBJ)/read_from_main_thread.o $(OBJ)/pthreadfs.o $(PTHREADFS_JS)
+	mkdir -p dist/$(JSOPSTESTS)
+	$(EMCC) $(LINK_FLAGS) -o $@ --js-library=$(PTHREADFS_JS) $< $(word 2,$^)  

pthreadfs/examples/emscripten-tests/stat.cpp

@@ -13,45 +13,37 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/stat.h>
+#include <sys/sysmacros.h>
+#include <sys/types.h>
 #include <time.h>
 #include <unistd.h>
 #include <utime.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <sys/sysmacros.h>
 
-void create_file(const char *path, const char *buffer, int mode) {
+void create_file(const char* path, const char* buffer, int mode) {
   int fd = open(path, O_CREAT | O_TRUNC | O_RDWR, mode);
   assert(fd >= 0);
 
   int err = write(fd, buffer, sizeof(char) * strlen(buffer));
-  assert(err ==  (sizeof(char) * strlen(buffer)));
+  assert(err == (sizeof(char) * strlen(buffer)));
 
   close(fd);
 }
 
 void setup() {
-  // struct utimbuf t = {1200000000, 1200000000};
-
   mkdir("persistent/folder", 0777);
   create_file("persistent/folder/file", "abcdef", 0777);
-  //symlink("file", "folder/file-link");
-
-  //utime("folder/file", &t);
-  //utime("folder", &t);
 }
 
 void cleanup() {
   rmdir("persistent/folder/subdir");
   unlink("persistent/folder/file");
-  // unlink("folder/file-link");
   rmdir("persistent/folder");
 }
 
 void test() {
   int err;
   struct stat s;
-  // struct utimbuf t = {1200000000, 1200000000};
 
   // non-existent
   err = stat("persistent/does_not_exist", &s);
@@ -110,7 +102,8 @@ void test() {
 
   // fstat a file (should match file stat from above)
   memset(&s, 0, sizeof(s));
-  err = fstat(open("persistent/folder/file", O_RDONLY), &s);
+  int fd = open("persistent/folder/file", O_RDONLY);
+  err = fstat(fd, &s);
   assert(!err);
   assert(s.st_dev);
   assert(s.st_ino);
@@ -125,6 +118,8 @@ void test() {
   assert(s.st_blksize == 4096);
   assert(s.st_blocks == 1);
 #endif
+  err = close(fd);
+  assert(!err);
 
   // stat a device
   memset(&s, 0, sizeof(s));
@@ -146,54 +141,6 @@ void test() {
   assert(s.st_blksize == 4096);
   assert(s.st_blocks == 0);
 #endif
-
-  // stat a link (should match the file stat from above)
-//   memset(&s, 0, sizeof(s));
-//   err = stat("folder/file-link", &s);
-//   assert(!err);
-//   assert(s.st_dev);
-//   assert(s.st_ino);
-//   assert(S_ISREG(s.st_mode));
-//   assert(s.st_nlink);
-//   assert(s.st_rdev == 0);
-//   assert(s.st_size == 6);
-//   assert(s.st_atime == 1200000000);
-//   assert(s.st_mtime == 1200000000);
-//   assert(s.st_ctime);
-// #ifdef __EMSCRIPTEN__
-//   assert(s.st_blksize == 4096);
-//   assert(s.st_blocks == 1);
-// #endif
-
-  // lstat a link (should NOT match the file stat from above)
-//   memset(&s, 0, sizeof(s));
-//   err = lstat("folder/file-link", &s);
-//   assert(!err);
-//   assert(s.st_dev);
-//   assert(s.st_ino);
-//   assert(S_ISLNK(s.st_mode));
-//   assert(s.st_nlink);
-//   assert(s.st_rdev == 0);
-//   assert(s.st_size == 4);  // strlen("file")
-//   assert(s.st_atime != 1200000000);  // should NOT match the utime call we did for dir/file
-//   assert(s.st_mtime != 1200000000);
-//   assert(s.st_ctime);
-// #ifdef __EMSCRIPTEN__
-//   assert(s.st_blksize == 4096);
-//   assert(s.st_blocks == 1);
-// #endif
-
-  // create and unlink files inside a directory and check that mtime updates
-  mkdir("persistent/folder/subdir", 0777);
-  // utime("persistent/folder/subdir", &t);
-  create_file("persistent/folder/subdir/file", "abcdef", 0777);
-  err = stat("persistent/folder/subdir", &s);
-  // assert(s.st_mtime != 1200000000);
-  // utime("persistent/folder/subdir", &t);
-  unlink("persistent/folder/subdir/file");
-  err = stat("persistent/folder/subdir", &s);
-  // assert(s.st_mtime != 1200000000);
-
   puts("success");
 }
 

pthreadfs/examples/jsops-tests/read_from_main_thread.cpp

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2013 The Emscripten Authors.  All rights reserved.
+ * Emscripten is available under two separate licenses, the MIT license and the
+ * University of Illinois/NCSA Open Source License.  Both these licenses can be
+ * found in the LICENSE file.
+ */
+
+#include <assert.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "emscripten.h"
+
+void create_file(const char* path, const char* contents, int mode) {
+  int fd = open(path, O_CREAT | O_TRUNC | O_RDWR, mode);
+  assert(fd >= 0);
+
+  int err = write(fd, contents, sizeof(char) * strlen(contents));
+  assert(err == (sizeof(char) * strlen(contents)));
+
+  close(fd);
+}
+
+void test_file_contents(const char* path, const char* contents) {
+  MAIN_THREAD_ASYNC_EM_ASM({
+    (async() => {
+      let path = UTF8ToString($0);
+      await PThreadFS.init("persistent");
+      let content = await PThreadFS.readFile(path);
+      content = new TextDecoder().decode(content);
+      if (content != UTF8ToString($1)) {
+        throw new Error('Incorrect contents read: ' + content);
+      }
+      await PThreadFS.unlink(path);
+      console.log("Success");
+    })();
+  }, path, contents);
+}
+
+int main() {
+  const char* path = "persistent/read_from_main_file.txt";
+  const char* contents = "file_contents :)";
+  create_file(path, contents, 0777);
+
+  test_file_contents(path, contents);
+
+  puts("Check the console for success");
+  return EXIT_SUCCESS;
+}

pthreadfs/examples/jsops-tests/write_from_main_thread.cpp

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2013 The Emscripten Authors.  All rights reserved.
+ * Emscripten is available under two separate licenses, the MIT license and the
+ * University of Illinois/NCSA Open Source License.  Both these licenses can be
+ * found in the LICENSE file.
+ */
+
+#include <assert.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+void cleanup() {
+  rmdir("persistent/mainthreadfolder/subfolder");
+  rmdir("persistent/mainthreadfolder");
+}
+
+void test_file_contents(const char* path) {
+  printf("Test contents for file %s\n", path);
+
+  int fd = open(path, O_RDONLY, 0777);
+  assert(fd >= 0);
+
+  char readbuf[1000];
+  int bytes_read = read(fd, readbuf, sizeof(char) * (strlen(path) + 20));
+  assert(bytes_read > 0);
+  printf("Content: %s\n", readbuf);
+  close(fd);
+
+  unlink(path);
+}
+
+int main() {
+  const char* paths[] = {"persistent/file.txt", "persistent/mainthreadfolder/subfolder/ok now"};
+
+  for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) {
+    test_file_contents(paths[i]);
+  }
+  cleanup();
+
+  puts("success");
+  return EXIT_SUCCESS;
+}

pthreadfs/examples/jsops-tests/write_from_main_thread_pre.js

@@ -0,0 +1,9 @@
+if ("preRun" in Module) {
+  Module["preRun"].push(async () => {
+    await PThreadFS.init('persistent');
+    await PThreadFS.writeFile("persistent/file.txt", "the contents of persistent/file.txt !!", {});
+    await PThreadFS.createPath("/", "persistent/mainthreadfolder/subfolder/", true, true);
+    await PThreadFS.writeFile("persistent/mainthreadfolder/subfolder/ok now",
+      "the contents of persistent/mainthreadfolder/subfolder/ok now !!", {});
+  });
+}