Switch merkle libs

modules/contracts/src.ts/services/ethService.ts

@@ -23,8 +23,7 @@ import {
   encodeTransferResolver,
   encodeTransferState,
   getRandomBytes32,
-  generateMerkleTreeData,
-  hashCoreTransferState,
+  getMerkleProof,
 } from "@connext/vector-utils";
 import { Signer } from "@ethersproject/abstract-signer";
 import { BigNumber } from "@ethersproject/bignumber";
@@ -1029,15 +1028,15 @@ export class EthereumChainService extends EthereumChainReader implements IVector
     }
 
     // Generate merkle root
-    const { tree } = generateMerkleTreeData(activeTransfers);
+    const proof = getMerkleProof(activeTransfers, transferIdToDispute);
 
     return this.sendTxWithRetries(
       transferState.channelAddress,
       transferState.chainId,
       TransactionReason.disputeTransfer,
       () => {
         const channel = new Contract(transferState.channelAddress, VectorChannel.abi, signer);
-        return channel.disputeTransfer(transferState, tree.getHexProof(hashCoreTransferState(transferState)));
+        return channel.disputeTransfer(transferState, proof);
       },
     ) as Promise<Result<TransactionResponseWithResult, ChainError>>;
   }

modules/contracts/src.ts/tests/cmcs/adjudicator.spec.ts

@@ -15,6 +15,7 @@ import {
   hashCoreTransferState,
   hashTransferState,
   signChannelMessage,
+  getMerkleProof,
 } from "@connext/vector-utils";
 import { BigNumber, BigNumberish } from "@ethersproject/bignumber";
 import { AddressZero, HashZero, Zero } from "@ethersproject/constants";
@@ -69,7 +70,7 @@ describe("CMCAdjudicator.sol", async function () {
   const verifyTransferDispute = async (cts: FullTransferState, disputeBlockNumber: number) => {
     const { timestamp } = await provider.getBlock(disputeBlockNumber);
     const transferDispute = await channel.getTransferDispute(cts.transferId);
-    expect(transferDispute.transferStateHash).to.be.eq(`0x` + hashCoreTransferState(cts).toString("hex"));
+    expect(transferDispute.transferStateHash).to.be.eq(hashCoreTransferState(cts));
     expect(transferDispute.isDefunded).to.be.false;
     expect(transferDispute.transferDisputeExpiry).to.be.eq(BigNumber.from(timestamp).add(cts.transferTimeout));
   };
@@ -115,14 +116,16 @@ describe("CMCAdjudicator.sol", async function () {
   };
 
   // Get merkle proof of transfer
-  const getMerkleProof = (cts: FullTransferState[] = [transferState], toProve: string = transferState.transferId) => {
-    const { tree } = generateMerkleTreeData(cts);
-    return tree.getHexProof(hashCoreTransferState(cts.find((t) => t.transferId === toProve)!));
+  const getMerkleProofTest = (
+    cts: FullTransferState[] = [transferState],
+    toProve: string = transferState.transferId,
+  ) => {
+    return getMerkleProof(cts, toProve);
   };
 
   // Helper to dispute transfers + bring to defund phase
   const disputeTransfer = async (cts: FullTransferState = transferState) => {
-    await (await channel.disputeTransfer(cts, getMerkleProof([cts], cts.transferId))).wait();
+    await (await channel.disputeTransfer(cts, getMerkleProofTest([cts], cts.transferId))).wait();
   };
 
   // Helper to defund channels and verify transfers
@@ -536,7 +539,7 @@ describe("CMCAdjudicator.sol", async function () {
       }
       await disputeChannel();
       await expect(
-        channel.disputeTransfer({ ...transferState, channelAddress: getRandomAddress() }, getMerkleProof()),
+        channel.disputeTransfer({ ...transferState, channelAddress: getRandomAddress() }, getMerkleProofTest()),
       ).revertedWith("CMCAdjudicator: INVALID_TRANSFER");
     });
 
@@ -546,7 +549,7 @@ describe("CMCAdjudicator.sol", async function () {
       }
       await disputeChannel();
       await expect(
-        channel.disputeTransfer({ ...transferState, transferId: getRandomBytes32() }, getMerkleProof()),
+        channel.disputeTransfer({ ...transferState, transferId: getRandomBytes32() }, getMerkleProofTest()),
       ).revertedWith("CMCAdjudicator: INVALID_MERKLE_PROOF");
     });
 
@@ -558,7 +561,7 @@ describe("CMCAdjudicator.sol", async function () {
       // the defund phase
       const tx = await channel.disputeChannel(channelState, aliceSignature, bobSignature);
       await tx.wait();
-      await expect(channel.disputeTransfer(transferState, getMerkleProof())).revertedWith(
+      await expect(channel.disputeTransfer(transferState, getMerkleProofTest())).revertedWith(
         "CMCAdjudicator: INVALID_PHASE",
       );
     });
@@ -569,9 +572,9 @@ describe("CMCAdjudicator.sol", async function () {
       }
       const longerTimeout = { ...channelState, timeout: "4" };
       await disputeChannel(longerTimeout);
-      const tx = await channel.disputeTransfer(transferState, getMerkleProof());
+      const tx = await channel.disputeTransfer(transferState, getMerkleProofTest());
       await tx.wait();
-      await expect(channel.disputeTransfer(transferState, getMerkleProof())).revertedWith(
+      await expect(channel.disputeTransfer(transferState, getMerkleProofTest())).revertedWith(
         "CMCAdjudicator: TRANSFER_ALREADY_DISPUTED",
       );
     });
@@ -581,7 +584,7 @@ describe("CMCAdjudicator.sol", async function () {
         this.skip();
       }
       await disputeChannel();
-      const tx = await channel.disputeTransfer(transferState, getMerkleProof());
+      const tx = await channel.disputeTransfer(transferState, getMerkleProofTest());
       const { blockNumber } = await tx.wait();
       await verifyTransferDispute(transferState, blockNumber);
     });
@@ -597,14 +600,15 @@ describe("CMCAdjudicator.sol", async function () {
         { ...transferState, transferId: getRandomBytes32() },
         { ...transferState, transferId: getRandomBytes32() },
       ];
-      const { root, tree } = generateMerkleTreeData(transfers);
+      const { root } = generateMerkleTreeData(transfers);
 
       const newState = { ...channelState, merkleRoot: root };
       await disputeChannel(newState);
 
       const txs = [];
       for (const t of transfers) {
-        const tx = await channel.disputeTransfer(t, tree.getHexProof(hashCoreTransferState(t)));
+        const proof = getMerkleProofTest(transfers, t.transferId);
+        const tx = await channel.disputeTransfer(t, proof);
         txs.push(tx);
       }
       const receipts = await Promise.all(txs.map((tx) => tx.wait()));

modules/iframe-app/config-overrides.js

@@ -0,0 +1,29 @@
+// WASM support inspired by https://stackoverflow.com/a/59720645
+
+module.exports = function override(config, env) {
+  const wasmExtensionRegExp = /\.wasm$/;
+
+  config.resolve.extensions.push(".wasm");
+
+  // make file-loader ignore WASM files
+  config.module.rules.forEach((rule) => {
+    (rule.oneOf || []).forEach((oneOf) => {
+      if (oneOf.loader && oneOf.loader.indexOf("file-loader") >= 0) {
+        oneOf.exclude.push(wasmExtensionRegExp);
+      }
+    });
+  });
+
+  // add a dedicated loader for WASM
+  config.module.rules.push({
+    test: wasmExtensionRegExp,
+
+    // necessary to avoid "Module parse failed: magic header not detected" errors;
+    // see https://github.com/pine/arraybuffer-loader/issues/12#issuecomment-390834140
+    type: "javascript/auto",
+
+    use: [{ loader: require.resolve("wasm-loader"), options: {} }],
+  });
+
+  return config;
+};

modules/iframe-app/package.json

@@ -22,14 +22,16 @@
     "react": "17.0.1",
     "react-dom": "17.0.1",
     "react-scripts": "3.4.3",
-    "typescript": "4.2.4"
+    "react-app-rewired": "2.1.8",
+    "typescript": "4.2.4",
+    "wasm-loader": "1.3.0"
   },
   "scripts": {
-    "start": "BROWSER=none PORT=3030 react-scripts start",
-    "build": "REACT_APP_VECTOR_CONFIG=$(cat \"../../ops/config/browser.default.json\") SKIP_PREFLIGHT_CHECK=true react-scripts build",
-    "build-prod": "SKIP_PREFLIGHT_CHECK=true react-scripts build",
-    "test": "react-scripts test",
-    "eject": "react-scripts eject"
+    "start": "BROWSER=none PORT=3030 react-app-rewired start",
+    "build": "REACT_APP_VECTOR_CONFIG=$(cat \"../../ops/config/browser.default.json\") SKIP_PREFLIGHT_CHECK=true react-app-rewired build",
+    "build-prod": "SKIP_PREFLIGHT_CHECK=true react-app-rewired build",
+    "test": "react-app-rewired test",
+    "eject": "react-app-rewired eject"
   },
   "eslintConfig": {
     "extends": [

modules/protocol/src/testing/validate.spec.ts

@@ -1147,16 +1147,6 @@ describe("validateAndApplyInboundUpdate", () => {
           overrides: { transferEncodings: "fail" },
           error: "should be array",
         },
-        {
-          name: "no merkleProofData",
-          overrides: { merkleProofData: undefined },
-          error: "should have required property 'merkleProofData'",
-        },
-        {
-          name: "malformed merkleProofData",
-          overrides: { merkleProofData: "fail" },
-          error: "should be array",
-        },
         {
           name: "no merkleRoot",
           overrides: { merkleRoot: undefined },

modules/protocol/src/update.ts

@@ -508,7 +508,6 @@ async function generateCreateUpdate(
       balance,
       transferInitialState,
       transferEncodings: [stateEncoding, resolverEncoding],
-      merkleProofData: tree.getHexProof(hashCoreTransferState(transferState)),
       merkleRoot: root,
       meta: { ...(meta ?? {}), createdAt: Date.now() },
     },

modules/server-node/prisma-postgres/schema.prisma

@@ -114,7 +114,6 @@ model Update {
   transferTimeout      String?
   transferInitialState String? // JSON string
   transferEncodings    String?
-  merkleProofData      String? // proofs.join(",")
   meta                 String?
   responder            String?
 

modules/server-node/prisma-sqlite/schema.prisma

@@ -114,7 +114,6 @@ model Update {
   transferTimeout      String?
   transferInitialState String? // JSON string
   transferEncodings    String?
-  merkleProofData      String? // proofs.join(",")
   meta                 String?
   responder            String?
 

modules/server-node/src/services/store.ts

@@ -124,7 +124,6 @@ const convertChannelEntityToFullChannelState = (
             to: [channelEntity.latestUpdate.transferToA!, channelEntity.latestUpdate.transferToB!],
             amount: [channelEntity.latestUpdate.transferAmountA!, channelEntity.latestUpdate.transferAmountB!],
           },
-          merkleProofData: channelEntity.latestUpdate.merkleProofData!.split(","),
           merkleRoot: channelEntity.latestUpdate.merkleRoot!,
           transferDefinition: channelEntity.latestUpdate.transferDefinition!,
           transferTimeout: channelEntity.latestUpdate.transferTimeout!,
@@ -676,7 +675,6 @@ export class PrismaStore implements IServerNodeStore {
         transferAmountB: (channelState.latestUpdate!.details as CreateUpdateDetails).balance?.amount[1] ?? undefined,
         transferToB: (channelState.latestUpdate!.details as CreateUpdateDetails).balance?.to[1] ?? undefined,
         merkleRoot: (channelState.latestUpdate!.details as CreateUpdateDetails).merkleRoot,
-        merkleProofData: (channelState.latestUpdate!.details as CreateUpdateDetails).merkleProofData?.join(),
         transferDefinition: (channelState.latestUpdate!.details as CreateUpdateDetails).transferDefinition,
         transferEncodings: (channelState.latestUpdate!.details as CreateUpdateDetails).transferEncodings
           ? (channelState.latestUpdate!.details as CreateUpdateDetails).transferEncodings.join("$") // comma separation doesnt work
@@ -891,7 +889,6 @@ export class PrismaStore implements IServerNodeStore {
         transferAmountB: (channel.latestUpdate!.details as CreateUpdateDetails).balance?.amount[1] ?? undefined,
         transferToB: (channel.latestUpdate!.details as CreateUpdateDetails).balance?.to[1] ?? undefined,
         merkleRoot: (channel.latestUpdate!.details as CreateUpdateDetails).merkleRoot,
-        merkleProofData: (channel.latestUpdate!.details as CreateUpdateDetails).merkleProofData?.join(),
         transferDefinition: (channel.latestUpdate!.details as CreateUpdateDetails).transferDefinition,
         transferEncodings: (channel.latestUpdate!.details as CreateUpdateDetails).transferEncodings
           ? (channel.latestUpdate!.details as CreateUpdateDetails).transferEncodings.join("$") // comma separation doesnt work
@@ -945,7 +942,6 @@ export class PrismaStore implements IServerNodeStore {
             transferTimeout: transfer.transferTimeout,
             transferInitialState: JSON.stringify(transfer.transferState),
             transferEncodings: transfer.transferEncodings.join("$"),
-            merkleProofData: "", // could recreate, but y tho
             meta: transfer.meta ? JSON.stringify(transfer.meta) : undefined,
             responder: transfer.responder,
           },

modules/types/src/channel.ts

@@ -201,7 +201,6 @@ export type CreateUpdateDetails = {
   transferTimeout: string;
   transferInitialState: TransferState;
   transferEncodings: string[]; // Included for `applyUpdate`
-  merkleProofData: string[];
   merkleRoot: string;
   meta?: BasicMeta;
 };

modules/types/src/schemas/basic.ts

@@ -127,7 +127,6 @@ export const TCreateUpdateDetails = Type.Object({
   transferTimeout: TIntegerString,
   transferInitialState: TransferStateSchema,
   transferEncodings: TransferEncodingSchema,
-  merkleProofData: Type.Array(Type.String()),
   merkleRoot: TBytes32,
   meta: TBasicMeta,
 });

modules/utils/package.json

@@ -13,6 +13,7 @@
     "test": "nyc ts-mocha --check-leaks --exit 'src/**/*.spec.ts'"
   },
   "dependencies": {
+    "@connext/vector-merkle-tree": "0.1.2",
     "@connext/vector-types": "0.2.5-alpha.2",
     "@ethersproject/abi": "5.1.0",
     "@ethersproject/abstract-provider": "5.1.0",
@@ -52,6 +53,7 @@
     "@types/chai-subset": "1.3.3",
     "@types/mocha": "8.2.1",
     "@types/node": "14.14.31",
+    "copy-webpack-plugin": "6.2.1",
     "mocha": "8.3.0",
     "nyc": "15.1.0",
     "sinon": "10.0.0",

modules/utils/src/merkle.spec.ts

@@ -1,13 +1,22 @@
 import { createCoreTransferState, expect } from "./test";
 import { getRandomBytes32, isValidBytes32 } from "./hexStrings";
 import { generateMerkleTreeData } from "./merkle";
-import { HashZero } from "@ethersproject/constants";
 import { hashCoreTransferState } from "./transfers";
 
+import * as merkle from "@connext/vector-merkle-tree";
 import { MerkleTree } from "merkletreejs";
 import { keccak256 } from "ethereumjs-util";
 import { keccak256 as solidityKeccak256 } from "@ethersproject/solidity";
 import { bufferify } from "./crypto";
+import { CoreTransferState } from "@connext/vector-types";
+
+const generateMerkleTreeJs = (transfers: CoreTransferState[]) => {
+  const sorted = transfers.sort((a, b) => a.transferId.localeCompare(b.transferId));
+
+  const leaves = sorted.map((transfer) => hashCoreTransferState(transfer));
+  const tree = new MerkleTree(leaves, keccak256, { sortPairs: true });
+  return tree;
+};
 
 describe("generateMerkleTreeData", () => {
   const generateTransfers = (noTransfers = 1) => {
@@ -18,24 +27,55 @@ describe("generateMerkleTreeData", () => {
       });
   };
 
+  let toFree: merkle.Tree | undefined;
+
+  const getMerkleTreeRoot = (transfers: CoreTransferState[]): string => {
+    const data = generateMerkleTreeData(transfers);
+    toFree = data.tree;
+    return data.root;
+  };
+
+  beforeEach(() => {
+    toFree = undefined;
+  });
+
+  afterEach(() => {
+    if (toFree) {
+      toFree.free();
+    }
+  });
+
   it("should work for a single transfer", () => {
     const [transfer] = generateTransfers();
-    const { root, tree } = generateMerkleTreeData([transfer]);
-    expect(root).to.not.be.eq(HashZero);
+    const root = getMerkleTreeRoot([transfer]);
+    const tree = generateMerkleTreeJs([transfer]);
+    expect(root).to.be.eq(tree.getHexRoot());
     expect(isValidBytes32(root)).to.be.true;
 
     const leaf = hashCoreTransferState(transfer);
     expect(tree.verify(tree.getHexProof(leaf), leaf, root)).to.be.true;
   });
 
+  it("should generate the same root for both libs", () => {
+    const transfers = generateTransfers(15);
+    const root = getMerkleTreeRoot(transfers);
+
+    const sorted = transfers.sort((a, b) => a.transferId.localeCompare(b.transferId));
+
+    const leaves = sorted.map((transfer) => hashCoreTransferState(transfer));
+    const tree = new MerkleTree(leaves, keccak256, { sortPairs: true });
+    expect(root).to.be.eq(tree.getHexRoot());
+  });
+
   it("should work for multiple transfers", () => {
-    const transfers = generateTransfers(1);
+    const transfers = generateTransfers(15);
 
     const randomIdx = Math.floor(Math.random() * 1);
     const toProve = transfers[randomIdx];
 
-    const { root, tree } = generateMerkleTreeData(transfers);
-    expect(root).to.not.be.eq(HashZero);
+    const root = getMerkleTreeRoot(transfers);
+    const tree = generateMerkleTreeJs(transfers);
+    expect(root).to.be.eq(tree.getHexRoot());
     expect(isValidBytes32(root)).to.be.true;
 
     const leaf = hashCoreTransferState(toProve);