Duplicate function elimination

This change adds support for duplicate function elimination (DFE) to the
JavaScript optimizer.  A new JS file has been added -
eliminate-duplicate-functions.js - which is used to postprocess the
output generated by Emscripten. We add a new file, rather than
augmenting the existing JS optimizer file, for a variety of reasons -
pass independence, reduced coupling between Python scripts and the JS
optimizer, etc.

We introduce a multipass algorithm in which each pass consists of the
following four phases:

Phase 1 - identify duplicate functions using a hash of the function body
Phase 2 - identify variable names that would conflict after renaming
function calls
Phase 3 - generate mapping from equivalent functions to their
replacement function - use the information from Phase 2 to ensure that
the replacement function is not a variable name
Phase 4 - use the mapping generated in Phase 3 to perform the reduction

NOTE: In some rare cases, we may actually not be able to move on from
Phase 3 if we find that we cannot generate a mapping because of
conflicts with variable names.

One pass can reveal new sets of identical functions which in turn can be
reduced by further passes. Empirically, four or five passes are
sufficient to eliminate all duplicate functions. Internally, therefore,
the elimination will perform 5 passes by default. This can be overridden
by setting ELIMINATE_DUPLICATE_FUNCTIONS_PASSES to 1 in settings.js or
on the Emscripten command line.

Generated asm.js is broken into several batches (at function boundaries)
to enable parallelization of the elimination. This saves on memory and
makes use of more CPU cores to save on build time. A number of tests
have been introduced to test this functionality as well.

The change also introduces various tweaks to the amount of diagnostic
information that is dumped out by the JavaScript optimizer. Verbose
logging is now only enabled in debug mode (via the EMCC_LOG_DEBUG
environment variable). We also dump backtraces on encountering unhandled
exceptions: this is useful when Emscripten runs as part of a large build
process. In order to view detailed information about which functions
were merged, set the
ELIMINATE_DUPLICATE_FUNCTIONS_DUMP_EQUIVALENT_FUNCTIONS value to 1 in
settings.js or via the Emscripten command line. This generates a log
file in the same directory as the generated JavaScript listing the sets
of merged functions. This can be decoded using the symbol map generated
by Emscripten. It is, therefore, recommended that developers enable
symbol map generation when attempting to modify or debug this feature.

Since DFE increases build time significantly, it is disabled by default.
It can be enabled by setting ELIMINATE_DUPLICATE_FUNCTIONS to 1 either
in settings.js or by adding "-s ELIMINATE_DUPLICATE_FUNCTIONS=1" on the
Emscripten command line. The poppler test has been updated to also run
with the ELIMINATE_DUPLICATE_FUNCTIONS setting set to 1.

Improvements/future work

It has been observed that on average we experience a code size reduction
of 25% when transpiling large C++ code bases.  Typically, C++ code that
makes heavy use of templates will experience the greatest reduction in
code size. There are several directions that future work might take:

* Deduplication of code across templates: e.g. reduction of
std::vector<long> and std::vector<int> to single instantiations of
template code when appropriate

* Histogram-based selection of candidates for replacement: improved code
size should be attainable by assigning the shortest identifiers to the
most frequently referenced functions (in the style of Huffman coding)

* Convergence: the five-pass default chosen in this implementation is
based on empirical observations on a 150,000LOC C++ code base

* Candidate selection: this will, most likely, influence both the
convergence time (i.e. number of passes) and the code size reduction;
currently, when selecting candidates, we choose the shortest identifier
from the list that is not also a variable name

Author: achoudhury85

AUTHORS

@@ -232,4 +232,6 @@ a license to everyone to use it as detailed in LICENSE.)
 * Nick Shin <[email protected]>
 * Gregg Tavares <[email protected]>
 * Tanner Rogalsky <[email protected]>
-
+* Richard Cook <[email protected]> (copyright owned by Tableau Software, Inc.)
+* Arnab Choudhury <[email protected]> (copyright owned by Tableau Software, Inc.)
+* Charles Vaughn <[email protected]> (copyright owned by Tableau Software, Inc.)

emcc.py

@@ -1634,6 +1634,11 @@ def do_minify(): # minifies the code. this is also when we do certain optimizati
         else:
           JSOptimizer.queue += ['registerize']
 
+      # NOTE: Important that this comes after registerize/registerizeHarder
+      if shared.Settings.ELIMINATE_DUPLICATE_FUNCTIONS and opt_level >= 2:
+        JSOptimizer.flush()
+        shared.Building.eliminate_duplicate_funcs(final)
+
       if not shared.Settings.EMTERPRETIFY:
         do_minify()
 

src/settings.js

@@ -681,4 +681,9 @@ var PTHREADS_PROFILING = 0; // True when building with --threadprofiler
 
 var MAX_GLOBAL_ALIGN = -1; // received from the backend
 
+// Duplicate function elimination
+var ELIMINATE_DUPLICATE_FUNCTIONS = 0; // disabled by default
+var ELIMINATE_DUPLICATE_FUNCTIONS_PASSES = 5;
+var ELIMINATE_DUPLICATE_FUNCTIONS_DUMP_EQUIVALENT_FUNCTIONS = 0;
+
 // Reserved: variables containing POINTER_MASKING.

tests/optimizer/test-function-eliminator-double-parsed-correctly-output.js

@@ -0,0 +1,14 @@
+// EMSCRIPTEN_START_ASM
+var asm = (function(global, env, buffer) {
+ "use asm";
+ var e = 0;
+ 
+// EMSCRIPTEN_START_FUNCS
+function a() {
+ var c = 0.0;
+ return 0;
+}
+// EMSCRIPTEN_END_FUNCS
+ var f = 0;
+})(Module.asmGlobalArg, Module.asmLibraryArg, buffer);
+// EMSCRIPTEN_END_ASM

tests/optimizer/test-function-eliminator-double-parsed-correctly.js

@@ -0,0 +1,22 @@
+// EMSCRIPTEN_START_ASM
+var asm = (function(global, env, buffer) {
+ "use asm";
+ var e = 0;
+ 
+// EMSCRIPTEN_START_FUNCS
+ function a() {
+  var c = +0;
+  return 0;
+ }
+ function b() {
+  var c = +0;
+  return 0;
+ }
+// EMSCRIPTEN_END_FUNCS
+ var f = 0;
+})(Module.asmGlobalArg, Module.asmLibraryArg, buffer);
+// EMSCRIPTEN_END_ASM
+// EMSCRIPTEN_GENERATED_FUNCTIONS
+
+
+

tests/optimizer/test-function-eliminator-replace-array-value-output.js

@@ -0,0 +1,25 @@
+// EMSCRIPTEN_START_ASM
+var asm = (function(global, env, buffer) {
+ "use asm";
+ 
+// EMSCRIPTEN_START_FUNCS
+function d() {
+ a();
+ e();
+ return;
+}
+
+function c() {
+ a();
+ return;
+}
+
+function a() {
+ return 0;
+}
+
+// EMSCRIPTEN_END_FUNCS
+
+ var f = [ a ];
+})(Module.asmGlobalArg, Module.asmLibraryArg, buffer);
+// EMSCRIPTEN_END_ASM

tests/optimizer/test-function-eliminator-replace-array-value-with-hash-info.js

@@ -0,0 +1,32 @@
+var asm = (function(global, env, buffer) {
+"use asm";
+function a()
+{
+return 0;
+}
+
+function b()
+{
+return 0;
+}
+
+function c()
+{
+  a();
+  return;
+}
+
+function d()
+{
+  b();
+  
+  // We expect that b gets replaced by a below
+  var f = [b];
+  e();
+
+  return;
+}
+
+})(Module.asmGlobalArg, Module.asmLibraryArg, buffer);
+
+// {"b":"a"}

tests/optimizer/test-function-eliminator-replace-array-value.js

@@ -0,0 +1,23 @@
+// EMSCRIPTEN_START_ASM
+var asm = (function(global, env, buffer) {
+ "use asm";
+// EMSCRIPTEN_START_FUNCS
+ function a() {
+  return 0;
+ }
+ function b() {
+  return 0;
+ }
+ function c() {
+  a();
+  return;
+ }
+ function d() {
+  b();
+  e();
+  return;
+ }
+// EMSCRIPTEN_END_FUNCS
+  var f = [ b ];
+})(Module.asmGlobalArg, Module.asmLibraryArg, buffer);
+// EMSCRIPTEN_END_ASM

tests/optimizer/test-function-eliminator-replace-function-call-output-with-hash-info.js

@@ -0,0 +1,22 @@
+var asm = (function(global, env, buffer) {
+"use asm";
+function a()
+{
+return 0;
+}
+
+function c()
+{
+  a();
+  return;
+}
+
+function d()
+{
+  a();
+  return;
+}
+
+})(Module.asmGlobalArg, Module.asmLibraryArg, buffer);
+
+// {"d":"c"}

tests/optimizer/test-function-eliminator-replace-function-call-output.js

@@ -0,0 +1,14 @@
+var asm = (function(global, env, buffer) {
+ "use asm";
+ function a() {
+  return 0;
+ }
+ function c() {
+  a();
+  return;
+ }
+ function d() {
+  a();
+  return;
+ }
+})(Module.asmGlobalArg, Module.asmLibraryArg, buffer);

tests/optimizer/test-function-eliminator-replace-function-call-two-passes-output.js

@@ -0,0 +1,10 @@
+var asm = (function(global, env, buffer) {
+ "use asm";
+ function a() {
+  return 0;
+ }
+ function c() {
+  a();
+  return;
+ }
+})(Module.asmGlobalArg, Module.asmLibraryArg, buffer);

tests/optimizer/test-function-eliminator-replace-function-call-with-hash-info.js

@@ -0,0 +1,27 @@
+var asm = (function(global, env, buffer) {
+"use asm";
+function a()
+{
+return 0;
+}
+
+function b()
+{
+return 0;
+}
+
+function c()
+{
+  a();
+  return;
+}
+
+function d()
+{
+  b();
+  return;
+}
+
+})(Module.asmGlobalArg, Module.asmLibraryArg, buffer);
+
+// {"b":"a"}

tests/optimizer/test-function-eliminator-replace-function-call.js

@@ -0,0 +1,20 @@
+var asm = (function(global, env, buffer) {
+ "use asm";
+ function a() {
+  return 0;
+ }
+ function b() {
+  return 0;
+ }
+ function c() {
+  a();
+  return;
+ }
+ function d() {
+  b();
+  return;
+ }
+})(Module.asmGlobalArg, Module.asmLibraryArg, buffer);
+
+
+

tests/optimizer/test-function-eliminator-replace-object-value-assignment-output.js

@@ -0,0 +1,18 @@
+var asm = (function(global, env, buffer) {
+ "use asm";
+ function a() {
+  return 0;
+ }
+ function c() {
+  a();
+  return;
+ }
+ function d() {
+  a();
+  var f = {
+   g: a
+  };
+  e();
+  return;
+ }
+})(Module.asmGlobalArg, Module.asmLibraryArg, buffer);

tests/optimizer/test-function-eliminator-replace-object-value-assignment-with-hash-info.js

@@ -0,0 +1,34 @@
+var asm = (function(global, env, buffer) {
+"use asm";
+function a()
+{
+return 0;
+}
+
+function b()
+{
+return 0;
+}
+
+function c()
+{
+  a();
+  return;
+}
+
+function d()
+{
+  b();
+  
+  // We expect that b gets replaced by a below
+  var f = {
+    g: b
+  };
+  e();
+
+  return;
+}
+
+})(Module.asmGlobalArg, Module.asmLibraryArg, buffer);
+
+// {"b":"a"}

tests/optimizer/test-function-eliminator-replace-object-value-assignment.js

@@ -0,0 +1,24 @@
+var asm = (function(global, env, buffer) {
+ "use asm";
+ function a() {
+  return 0;
+ }
+ function b() {
+  return 0;
+ }
+ function c() {
+  a();
+  return;
+ }
+ function d() {
+  b();
+  var f = {
+   g: b
+  };
+  e();
+  return;
+ }
+})(Module.asmGlobalArg, Module.asmLibraryArg, buffer);
+
+
+

tests/optimizer/test-function-eliminator-replace-variable-value-output.js

@@ -0,0 +1,16 @@
+var asm = (function(global, env, buffer) {
+ "use asm";
+ function a() {
+  return 0;
+ }
+ function c() {
+  a();
+  return;
+ }
+ function d() {
+  a();
+  var e = a;
+  e();
+  return;
+ }
+})(Module.asmGlobalArg, Module.asmLibraryArg, buffer);

tests/optimizer/test-function-eliminator-replace-variable-value-with-hash-info.js

@@ -0,0 +1,32 @@
+var asm = (function(global, env, buffer) {
+"use asm";
+function a()
+{
+return 0;
+}
+
+function b()
+{
+return 0;
+}
+
+function c()
+{
+  a();
+  return;
+}
+
+function d()
+{
+  b();
+  
+  // We expect that b gets replaced by a below
+  var e = b;  
+  e();
+
+  return;
+}
+
+})(Module.asmGlobalArg, Module.asmLibraryArg, buffer);
+
+// {"b" : "a"}