Move webassembly binary handling to webassembly.py

emcc.py

@@ -49,6 +49,7 @@
 from tools.toolchain_profiler import ToolchainProfiler
 from tools import js_manipulation
 from tools import wasm2c
+from tools import webassembly
 
 if __name__ == '__main__':
   ToolchainProfiler.record_process_start()
@@ -2621,10 +2622,10 @@ def do_binaryen(target, options, memfile, wasm_target,
 
   # TODO: do this with upstream
   # if shared.Settings.SIDE_MODULE:
-  #   shared.WebAssembly.add_dylink_section(wasm_target, shared.Settings.RUNTIME_LINKED_LIBS)
+  #   webassembly.add_dylink_section(wasm_target, shared.Settings.RUNTIME_LINKED_LIBS)
 
   if shared.Settings.EMIT_EMSCRIPTEN_METADATA:
-    shared.WebAssembly.add_emscripten_metadata(final, wasm_target)
+    webassembly.add_emscripten_metadata(final, wasm_target)
 
   if shared.Settings.SIDE_MODULE:
     return # and we are done.

tests/test_other.py

@@ -17,7 +17,6 @@
 import select
 import shlex
 import shutil
-import struct
 import subprocess
 import sys
 import time
@@ -39,7 +38,8 @@
 from tools import shared, building
 import jsrun
 import clang_native
-import tools.line_endings
+from tools import line_endings
+from tools import webassembly
 
 scons_path = shared.which('scons')
 emmake = shared.bat_suffix(path_from_root('emmake'))
@@ -89,13 +89,6 @@ def decorated(self):
   return decorated
 
 
-def encode_leb(number):
-  # TODO(sbc): handle larger numbers
-  assert(number < 255)
-  # pack the integer then take only the first (little end) byte
-  return struct.pack('<i', number)[:1]
-
-
 def parse_wasm(filename):
   wat = shared.run_process([os.path.join(building.get_binaryen_bin(), 'wasm-dis'), filename], stdout=PIPE).stdout
   imports = []
@@ -6747,7 +6740,7 @@ def test_output_eol(self):
             else:
               expected_ending = '\r\n'
 
-            ret = tools.line_endings.check_line_endings(f, expect_only=expected_ending)
+            ret = line_endings.check_line_endings(f, expect_only=expected_ending)
             assert ret == 0
 
           for f in files:
@@ -7757,7 +7750,7 @@ def test_check_sourcemapurl(self):
     self.run_process([EMCC, path_from_root('tests', 'hello_123.c'), '-g4', '-o', 'a.js', '--source-map-base', 'dir/'])
     output = open('a.wasm', 'rb').read()
     # has sourceMappingURL section content and points to 'dir/a.wasm.map' file
-    source_mapping_url_content = encode_leb(len('sourceMappingURL')) + b'sourceMappingURL' + encode_leb(len('dir/a.wasm.map')) + b'dir/a.wasm.map'
+    source_mapping_url_content = webassembly.toLEB(len('sourceMappingURL')) + b'sourceMappingURL' + webassembly.toLEB(len('dir/a.wasm.map')) + b'dir/a.wasm.map'
     self.assertEqual(output.count(source_mapping_url_content), 1)
     # make sure no DWARF debug info sections remain - they would just waste space
     self.assertNotIn(b'.debug_', output)
@@ -7782,7 +7775,7 @@ def test_check_sourcemapurl_default(self, *args):
     self.run_process([EMCC, path_from_root('tests', 'hello_123.c'), '-g4', '-o', 'a.js'] + list(args))
     output = open('a.wasm', 'rb').read()
     # has sourceMappingURL section content and points to 'a.wasm.map' file
-    source_mapping_url_content = encode_leb(len('sourceMappingURL')) + b'sourceMappingURL' + encode_leb(len('a.wasm.map')) + b'a.wasm.map'
+    source_mapping_url_content = webassembly.toLEB(len('sourceMappingURL')) + b'sourceMappingURL' + webassembly.toLEB(len('a.wasm.map')) + b'a.wasm.map'
     self.assertIn(source_mapping_url_content, output)
 
   def test_wasm_sourcemap(self):

tools/building.py

@@ -21,6 +21,7 @@
 from . import diagnostics
 from . import response_file
 from . import shared
+from . import webassembly
 from .toolchain_profiler import ToolchainProfiler
 from .shared import Settings, CLANG_CC, CLANG_CXX, PYTHON
 from .shared import LLVM_NM, EMCC, EMAR, EMXX, EMRANLIB, NODE_JS, WASM_LD, LLVM_AR
@@ -30,7 +31,7 @@
 from .shared import asmjs_mangle, DEBUG, WINDOWS, JAVA
 from .shared import EM_BUILD_VERBOSE, TEMP_DIR, print_compiler_stage, BINARYEN_ROOT
 from .shared import CANONICAL_TEMP_DIR, LLVM_DWARFDUMP, demangle_c_symbol_name, asbytes
-from .shared import get_emscripten_temp_dir, exe_suffix, WebAssembly, which, is_c_symbol
+from .shared import get_emscripten_temp_dir, exe_suffix, which, is_c_symbol
 
 logger = logging.getLogger('building')
 
@@ -1378,11 +1379,11 @@ def emit_debug_on_side(wasm_file, wasm_file_with_dwarf):
   # see https://yurydelendik.github.io/webassembly-dwarf/#external-DWARF
   section_name = b'\x13external_debug_info' # section name, including prefixed size
   filename_bytes = asbytes(embedded_path)
-  contents = WebAssembly.toLEB(len(filename_bytes)) + filename_bytes
+  contents = webassembly.toLEB(len(filename_bytes)) + filename_bytes
   section_size = len(section_name) + len(contents)
   with open(wasm_file, 'ab') as f:
     f.write(b'\0') # user section is code 0
-    f.write(WebAssembly.toLEB(section_size))
+    f.write(webassembly.toLEB(section_size))
     f.write(section_name)
     f.write(contents)
 

tools/shared.py

@@ -12,7 +12,6 @@
 import difflib
 import json
 import logging
-import math
 import os
 import re
 import shutil
@@ -1295,148 +1294,6 @@ def is_function_table(name):
     return name.startswith('FUNCTION_TABLE_')
 
 
-class WebAssembly(object):
-  @staticmethod
-  def toLEB(x):
-    assert x >= 0, 'TODO: signed'
-    ret = []
-    while 1:
-      byte = x & 127
-      x >>= 7
-      more = x != 0
-      if more:
-        byte = byte | 128
-      ret.append(byte)
-      if not more:
-        break
-    return bytearray(ret)
-
-  @staticmethod
-  def readLEB(buf, offset):
-    result = 0
-    shift = 0
-    while True:
-      byte = bytearray(buf[offset:offset + 1])[0]
-      offset += 1
-      result |= (byte & 0x7f) << shift
-      if not (byte & 0x80):
-        break
-      shift += 7
-    return (result, offset)
-
-  @staticmethod
-  def add_emscripten_metadata(js_file, wasm_file):
-    WASM_PAGE_SIZE = 65536
-
-    mem_size = Settings.INITIAL_MEMORY // WASM_PAGE_SIZE
-    table_size = Settings.WASM_TABLE_SIZE
-    global_base = Settings.GLOBAL_BASE
-
-    js = open(js_file).read()
-    m = re.search(r"(^|\s)DYNAMIC_BASE\s+=\s+(\d+)", js)
-    dynamic_base = int(m.group(2))
-
-    logger.debug('creating wasm emscripten metadata section with mem size %d, table size %d' % (mem_size, table_size,))
-    name = b'\x13emscripten_metadata' # section name, including prefixed size
-    contents = (
-      # metadata section version
-      WebAssembly.toLEB(EMSCRIPTEN_METADATA_MAJOR) +
-      WebAssembly.toLEB(EMSCRIPTEN_METADATA_MINOR) +
-
-      # NB: The structure of the following should only be changed
-      #     if EMSCRIPTEN_METADATA_MAJOR is incremented
-      # Minimum ABI version
-      WebAssembly.toLEB(EMSCRIPTEN_ABI_MAJOR) +
-      WebAssembly.toLEB(EMSCRIPTEN_ABI_MINOR) +
-
-      # Wasm backend, always 1 now
-      WebAssembly.toLEB(1) +
-
-      WebAssembly.toLEB(mem_size) +
-      WebAssembly.toLEB(table_size) +
-      WebAssembly.toLEB(global_base) +
-      WebAssembly.toLEB(dynamic_base) +
-      # dynamictopPtr, always 0 now
-      WebAssembly.toLEB(0) +
-
-      # tempDoublePtr, always 0 in wasm backend
-      WebAssembly.toLEB(0) +
-
-      WebAssembly.toLEB(int(Settings.STANDALONE_WASM))
-
-      # NB: more data can be appended here as long as you increase
-      #     the EMSCRIPTEN_METADATA_MINOR
-    )
-
-    orig = open(wasm_file, 'rb').read()
-    with open(wasm_file, 'wb') as f:
-      f.write(orig[0:8]) # copy magic number and version
-      # write the special section
-      f.write(b'\0') # user section is code 0
-      # need to find the size of this section
-      size = len(name) + len(contents)
-      f.write(WebAssembly.toLEB(size))
-      f.write(name)
-      f.write(contents)
-      f.write(orig[8:])
-
-  @staticmethod
-  def add_dylink_section(wasm_file, needed_dynlibs):
-    # a wasm shared library has a special "dylink" section, see tools-conventions repo
-    # TODO: use this in the wasm backend
-    assert False
-    mem_align = Settings.MAX_GLOBAL_ALIGN
-    mem_size = Settings.STATIC_BUMP
-    table_size = Settings.WASM_TABLE_SIZE
-    mem_align = int(math.log(mem_align, 2))
-    logger.debug('creating wasm dynamic library with mem size %d, table size %d, align %d' % (mem_size, table_size, mem_align))
-
-    # Write new wasm binary with 'dylink' section
-    wasm = open(wasm_file, 'rb').read()
-    section_name = b"\06dylink" # section name, including prefixed size
-    contents = (WebAssembly.toLEB(mem_size) + WebAssembly.toLEB(mem_align) +
-                WebAssembly.toLEB(table_size) + WebAssembly.toLEB(0))
-
-    # we extend "dylink" section with information about which shared libraries
-    # our shared library needs. This is similar to DT_NEEDED entries in ELF.
-    #
-    # In theory we could avoid doing this, since every import in wasm has
-    # "module" and "name" attributes, but currently emscripten almost always
-    # uses just "env" for "module". This way we have to embed information about
-    # required libraries for the dynamic linker somewhere, and "dylink" section
-    # seems to be the most relevant place.
-    #
-    # Binary format of the extension:
-    #
-    #   needed_dynlibs_count        varuint32       ; number of needed shared libraries
-    #   needed_dynlibs_entries      dynlib_entry*   ; repeated dynamic library entries as described below
-    #
-    # dynlib_entry:
-    #
-    #   dynlib_name_len             varuint32       ; length of dynlib_name_str in bytes
-    #   dynlib_name_str             bytes           ; name of a needed dynamic library: valid UTF-8 byte sequence
-    #
-    # a proposal has been filed to include the extension into "dylink" specification:
-    # https://github.com/WebAssembly/tool-conventions/pull/77
-    contents += WebAssembly.toLEB(len(needed_dynlibs))
-    for dyn_needed in needed_dynlibs:
-      dyn_needed = bytes(asbytes(dyn_needed))
-      contents += WebAssembly.toLEB(len(dyn_needed))
-      contents += dyn_needed
-
-    section_size = len(section_name) + len(contents)
-    with open(wasm_file, 'wb') as f:
-      # copy magic number and version
-      f.write(wasm[0:8])
-      # write the special section
-      f.write(b'\0') # user section is code 0
-      f.write(WebAssembly.toLEB(section_size))
-      f.write(section_name)
-      f.write(contents)
-      # copy rest of binary
-      f.write(wasm[8:])
-
-
 # Python 2-3 compatibility helper function:
 # Converts a string to the native str type.
 def asstr(s):
@@ -1663,22 +1520,6 @@ def read_and_preprocess(filename, expand_macros=False):
 
 set_version_globals()
 
-# For the Emscripten-specific WASM metadata section, follows semver, changes
-# whenever metadata section changes structure.
-# NB: major version 0 implies no compatibility
-# NB: when changing the metadata format, we should only append new fields, not
-#     reorder, modify, or remove existing ones.
-EMSCRIPTEN_METADATA_MAJOR, EMSCRIPTEN_METADATA_MINOR = (0, 3)
-# For the JS/WASM ABI, specifies the minimum ABI version required of
-# the WASM runtime implementation by the generated WASM binary. It follows
-# semver and changes whenever C types change size/signedness or
-# syscalls change signature. By semver, the maximum ABI version is
-# implied to be less than (EMSCRIPTEN_ABI_MAJOR + 1, 0). On an ABI
-# change, increment EMSCRIPTEN_ABI_MINOR if EMSCRIPTEN_ABI_MAJOR == 0
-# or the ABI change is backwards compatible, otherwise increment
-# EMSCRIPTEN_ABI_MAJOR and set EMSCRIPTEN_ABI_MINOR = 0.
-EMSCRIPTEN_ABI_MAJOR, EMSCRIPTEN_ABI_MINOR = (0, 27)
-
 # Tools/paths
 if LLVM_ADD_VERSION is None:
   LLVM_ADD_VERSION = os.getenv('LLVM_ADD_VERSION')