From 583097f6eaa2c561b75388e38ebc8653357eb90f Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Fri, 4 Oct 2019 22:11:38 +0300 Subject: [PATCH 01/24] WIP Add FlatListProof tests --- .../binding/common/proofs/CheckedProof.java | 1 + .../common/proofs/list/CheckedListProof.java | 6 + .../proofs/list/CheckedListProofImpl.java | 6 + .../common/proofs/list/FlatListProof.java | 42 ++ .../proofs/list/InvalidProofException.java | 42 ++ .../proofs/list/ListProofElementEntry.java | 36 ++ .../common/proofs/list/ListProofEntry.java | 57 ++ .../proofs/list/ListProofHashedEntry.java | 40 ++ .../proofs/list/UncheckedListProof.java | 2 +- .../common/proofs/list/FlatListProofTest.java | 596 ++++++++++++++++++ 10 files changed, 827 insertions(+), 1 deletion(-) create mode 100644 exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java create mode 100644 exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/InvalidProofException.java create mode 100644 exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofElementEntry.java create mode 100644 exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofEntry.java create mode 100644 exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofHashedEntry.java create mode 100644 exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/CheckedProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/CheckedProof.java index a573a1f1c2..3af9198373 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/CheckedProof.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/CheckedProof.java @@ -25,6 +25,7 @@ * If it is valid, the proof contents may be accessed. See {@link CheckedListProof} * and {@link CheckedMapProof} for available contents description. */ +// todo: Why do we need to represent invalid proofs? public interface CheckedProof { /** diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/CheckedListProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/CheckedListProof.java index 6b9954efc4..da83f18eee 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/CheckedListProof.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/CheckedListProof.java @@ -35,6 +35,12 @@ * } */ public interface CheckedListProof extends CheckedProof { + + /** + * Returns the size of the list: the total number of elements in it. + */ + long size(); + /** * Get all list proof elements. There might be several consecutive ranges. * diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/CheckedListProofImpl.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/CheckedListProofImpl.java index 4f65f056ff..eeff30c2fd 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/CheckedListProofImpl.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/CheckedListProofImpl.java @@ -58,6 +58,12 @@ public CheckedListProofImpl(HashCode calculatedIndexHash, this.proofStatus = checkNotNull(proofStatus); } + @Override + public long size() { + // todo: + return 0; + } + @Override public NavigableMap getElements() { checkValid(); diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java new file mode 100644 index 0000000000..6355a4d2d3 --- /dev/null +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 The Exonum Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exonum.binding.common.proofs.list; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.List; + +/** + * A flat list proof. It proves that certain elements are present in a proof list + * of a certain size. + */ +class FlatListProof { + private final List elements; + private final List proof; + private final long size; + + FlatListProof(List elements, + List proof, long size) { + this.elements = checkNotNull(elements); + this.proof = checkNotNull(proof); + this.size = size; + } + + CheckedListProof verify() { + return null; + } +} diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/InvalidProofException.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/InvalidProofException.java new file mode 100644 index 0000000000..675b5dd1ef --- /dev/null +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/InvalidProofException.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 The Exonum Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exonum.binding.common.proofs.list; + +/** + * todo: + */ +public class InvalidProofException extends RuntimeException { + + /** + * Constructs a new runtime exception with {@code null} as its detail message. The cause is not + * initialized, and may subsequently be initialized by a call to {@link #initCause}. + */ + public InvalidProofException() { + super(); + } + + /** + * Constructs a new runtime exception with the specified detail message. The cause is not + * initialized, and may subsequently be initialized by a call to {@link #initCause}. + * + * @param message the detail message. The detail message is saved for later retrieval by the + * {@link #getMessage()} method. + */ + public InvalidProofException(String message) { + super(message); + } +} diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofElementEntry.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofElementEntry.java new file mode 100644 index 0000000000..2f2922c474 --- /dev/null +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofElementEntry.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 The Exonum Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exonum.binding.common.proofs.list; + +import com.google.auto.value.AutoValue; + +/** + * A value stored in the Merkle tree at its bottom level (at height 0). + */ +@AutoValue +abstract class ListProofElementEntry implements ListProofEntry { + + /** + * Returns a value of the element stored at this index in the list. + */ + abstract byte[] getElement(); + + static ListProofElementEntry newInstance(long index, byte[] element) { + ListProofEntry.checkIndex(index); + return new AutoValue_ListProofElementEntry(index, 0, element); + } +} diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofEntry.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofEntry.java new file mode 100644 index 0000000000..42b956d031 --- /dev/null +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofEntry.java @@ -0,0 +1,57 @@ +/* + * Copyright 2019 The Exonum Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exonum.binding.common.proofs.list; + +import static com.google.common.base.Preconditions.checkArgument; + +interface ListProofEntry { + + /** + * The maximum height of a list proof tree. + */ + int MAX_HEIGHT = 56; + + /** + * The maximum index of a list proof node: 2^56 - 1. + */ + long MAX_INDEX = 0xFF_FFFF_FFFF_FFFFL; + + /** + * Returns the index of the proof tree node at the height of its level. Indexes start + * from 0 for the leftmost node and up to 2^d - 1 for the rightmost node, + * where d = ceil(log2(N)) - h is the depth of the node at height h; + * N is the number of elements in the tree. + */ + long getIndex(); + + /** + * Returns the height of the proof tree node corresponding to this entry. + * The height of leaf nodes is equal to 0; the height of the root, or top node: + * ceil(log2(N)). + */ + int getHeight(); + + static void checkIndex(long index) { + checkArgument(0 <= index && index <= MAX_INDEX, + "Entry index (%s) is out of range [0; 2^56]", index); + } + + static void checkHeight(int height) { + checkArgument(0 <= height && height <= MAX_HEIGHT, + "Entry height (%s) is out of range [0; 56]", height); + } +} diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofHashedEntry.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofHashedEntry.java new file mode 100644 index 0000000000..43c889f3d3 --- /dev/null +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofHashedEntry.java @@ -0,0 +1,40 @@ +/* + * Copyright 2019 The Exonum Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exonum.binding.common.proofs.list; + +import com.exonum.binding.common.hash.HashCode; +import com.google.auto.value.AutoValue; + +/** + * A hash of a sub-tree in a Merkle proof tree. + */ +@AutoValue +abstract class ListProofHashedEntry implements ListProofEntry { + // todo: do we need the interface (are we going to operate on the entries using the interface?) + // Do we need entries at all, or just hashes? + + /** + * Returns the hash of the sub-tree this entry represents. + */ + abstract HashCode getHash(); + + static ListProofHashedEntry newInstance(long index, int height, HashCode nodeHash) { + ListProofEntry.checkIndex(index); + ListProofEntry.checkHeight(height); + return new AutoValue_ListProofHashedEntry(index, height, nodeHash); + } +} diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/UncheckedListProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/UncheckedListProof.java index afbe57b1f2..9ee6765e0e 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/UncheckedListProof.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/UncheckedListProof.java @@ -17,7 +17,7 @@ package com.exonum.binding.common.proofs.list; /** - * A proof that some elements exist in a proof list. You must + * A proof that some elements exist in a proof list of a certain size. You must * {@link #check} its structure and index hash before accessing the elements. */ public interface UncheckedListProof { diff --git a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java new file mode 100644 index 0000000000..5deeb07a97 --- /dev/null +++ b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java @@ -0,0 +1,596 @@ +/* + * Copyright 2019 The Exonum Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.exonum.binding.common.proofs.list; + +import static com.exonum.binding.common.proofs.list.ListProofEntry.MAX_HEIGHT; +import static com.exonum.binding.common.proofs.list.ListProofEntry.MAX_INDEX; +import static com.exonum.binding.test.Bytes.bytes; +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static java.util.stream.Collectors.toList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.entry; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.exonum.binding.common.hash.HashCode; +import com.exonum.binding.common.hash.Hashing; +import com.exonum.binding.test.Bytes; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.IntStream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; + +/* +The test uses the following illustrations to aid understanding of the proof structure: + + H — height of nodes at each level + 2 o + / \ + 1 o o + / \ / \ + 0 e e e h + _________________ + Legend: + e — element proof entry (ListProofElementEntry) + h — hashed proof entry (ListProofHashedEntry) + o — 'virtual' node — not present in the proof, inferred during verification. + Shown mostly to communicate the tree structure of the proof. + ? — absent node — a node that is expected to be present in a valid proof, but isn't + x — redundant node — a node that shall not be present in a valid proof, but is + */ +class FlatListProofTest { + + private static final HashCode EMPTY_LIST_INDEX_HASH = + HashCode.fromString("c6c0aa07f27493d2f2e5cff56c890a353a20086d6c25ec825128e12ae752b2d9"); + + private static final List ELEMENTS = createElements(4); + private static final List ELEMENT_ENTRIES = createElementEntries(ELEMENTS); + + @Test + void emptyListValidProof() { + FlatListProof proof = new FlatListProof(emptyList(), emptyList(), 0L); + + CheckedListProof checked = proof.verify(); + + assertTrue(checked.isValid()); + assertThat(checked.size()).isZero(); + assertThat(checked.getIndexHash()).isEqualTo(EMPTY_LIST_INDEX_HASH); + } + + @Test + void emptyListInvalidProofIfHasElements() { + ListProofElementEntry unexpectedElement = ELEMENT_ENTRIES.get(0); + FlatListProof proof = new FlatListProof(singletonList(unexpectedElement), emptyList(), 0L); + + InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); + + assertThat(e).hasMessageContaining("Unexpected element entry") + .hasMessageContaining(unexpectedElement.toString()); + } + + @Test + void emptyListInvalidProofIfHasHashedEntries() { + ListProofHashedEntry unexpectedEntry = hashedEntry(0, 0, HashCode.fromInt(1)); + FlatListProof proof = new FlatListProof(emptyList(), singletonList(unexpectedEntry), 0L); + + InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); + + assertThat(e).hasMessageContaining("Unexpected proof entry") + .hasMessageContaining(unexpectedEntry.toString()); + } + + @Test + void singletonListValidProof() { + int index = 0; + ListProofElementEntry element = ELEMENT_ENTRIES.get(index); + long size = 1L; + FlatListProof proof = new FlatListProof(singletonList(element), emptyList(), size); + + CheckedListProof checked = proof.verify(); + + assertTrue(checked.isValid()); + assertThat(checked.size()).isEqualTo(size); + assertThat(checked.getElements()).containsExactly(entry((long) index, ELEMENTS.get(index))); + // todo: hash. + } + + // todo: more singletonListInvalid + // todo: more twoItemValid + @Test + void twoElementListInvalidProofMissingHashNode1() { + /* + H + 1 o + / \ + 0 e ? + */ + int index = 0; + ListProofElementEntry element = ELEMENT_ENTRIES.get(index); + long size = 2L; + FlatListProof proof = new FlatListProof(singletonList(element), emptyList(), size); + + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + + assertThat(e).hasMessageContaining("Missing proof entry at position (1, 0)"); + } + + @Test + void twoElementListInvalidProofMissingHashNode2() { + /* + H + 1 o + / \ + 0 ? e + */ + int index = 0; + ListProofElementEntry element = ELEMENT_ENTRIES.get(index); + long size = 2L; + FlatListProof proof = new FlatListProof(singletonList(element), emptyList(), size); + + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + + assertThat(e).hasMessageContaining("Missing proof entry at position (1, 0)"); + } + + @ParameterizedTest + @MethodSource("twoElementListInvalidProofNodesAboveMaxHeightSource") + void twoElementListInvalidProofNodesAboveMaxHeight(List invalidEntries) { + /* + H + k {x} <— invalid nodes at various heights k > 1 + + 1 o + / \ + 0 e h + */ + FlatListProof proof = twoElementListAt0() + .addProofEntries(invalidEntries) + .build(); + + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + + assertThat(e).hasMessageContaining("Invalid node") + .hasMessageContaining("at height above the max (1)"); + } + + static List> twoElementListInvalidProofNodesAboveMaxHeightSource() { + return asList( + // | index | height | + singletonList(hashedEntry(0, 2)), + singletonList(hashedEntry(1, 2)), + singletonList(hashedEntry(0, 3)), + singletonList(hashedEntry(0, MAX_HEIGHT)), + asList(hashedEntry(0, 2), hashedEntry(1, 2)) + ); + } + + @ParameterizedTest + @MethodSource("twoElementListInvalidProofExtraNodesAtInvalidIndexesSource") + void twoElementListInvalidProofExtraNodesAtInvalidIndexes( + List invalidProofEntries) { + /* + H + 1 o {x} <- invalid nodes at various indexes exceeding max + / \ + 0 e h {x} <— + */ + FlatListProof proof = twoElementListAt0() + .addAnyEntries(invalidProofEntries) + .build(); + + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + + assertThat(e).hasMessageContaining("Invalid node") + .hasMessageFindingMatch("(?i)at index .+ exceeding the max"); + } + + private static Collection> + twoElementListInvalidProofExtraNodesAtInvalidIndexesSource() { + /* + Target proof tree: + H + 1 o + / \ + 0 e h + */ + byte[] value = bytes(1, 2, 3); + return asList( + // | index | height | + singletonList(hashedEntry(2, 0)), + singletonList(hashedEntry(3, 0)), + singletonList(elementEntry(2, value)), + singletonList(elementEntry(3, value)), + singletonList(elementEntry(MAX_INDEX, value)), + singletonList(hashedEntry(ListProofEntry.MAX_INDEX, 0)), + asList(hashedEntry(2, 0), hashedEntry(3, 0)), + singletonList(hashedEntry(1, 1)), + singletonList(hashedEntry(2, 1)) + ); + } + + @ParameterizedTest + @CsvSource({ + /* + H + 1 o + / / \ + 0 e x h + */ + "0, 0", + /* + H + 1 x o + / / \ + 0 e h + */ + "0, 1" + }) + void twoElementListInvalidProofExtraNodesRedundantCalculated(long index, int height) { + ListProofHashedEntry redundantNode = hashedEntry(index, height); + FlatListProof proof = twoElementListAt0() + .addProofEntry(redundantNode) + .build(); + + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + + assertThat(e).hasMessageContaining("Redundant proof node") + .hasMessageContaining(redundantNode.toString()) + .hasMessageContaining("must be inferred"); + } + + @Test + void twoElementListInvalidProofExtraNodesDuplicateHashed() { + /* + H + 1 o + / \ \ + 0 e h h + */ + ListProofHashedEntry duplicateNode = hashedEntry(1, 0); + FlatListProof proof = twoElementListAt0() + .addProofEntry(duplicateNode) + .build(); + + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + + assertThat(e).hasMessageFindingMatch("Duplicate.+node") + .hasMessageContaining(duplicateNode.toString()); + } + + @Test + void twoElementListInvalidProofExtraNodesDuplicateElementSameValue() { + /* + H + 1 o + / / \ + 0 e e h + */ + ListProofElementEntry duplicateEntry = ELEMENT_ENTRIES.get(0); + FlatListProof proof = twoElementListAt0() + .addElement(duplicateEntry) + .build(); + + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + + assertThat(e).hasMessageFindingMatch("Duplicate.+node") + .hasMessageContaining(duplicateEntry.toString()); + } + + @Test + void twoElementListInvalidProofExtraNodesDuplicateElementDiffValue() { + /* + H + 1 o + / / \ + 0 e e h + */ + ListProofElementEntry entry = ELEMENT_ENTRIES.get(0); + ListProofElementEntry conflictingEntry = elementEntry(0, bytes(1, 2, 3, 4)); + FlatListProof proof = twoElementListAt0() + .addElement(conflictingEntry) + .build(); + + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + + assertThat(e).hasMessageFindingMatch("Duplicate.+node") + .hasMessageContaining(entry.toString()) + .hasMessageContaining(conflictingEntry.toString()); + } + + @Test + void twoElementListValidProofAt0() { + // Check the proof returned by twoElementListAt0 is valid + FlatListProof proof = twoElementListAt0().build(); + CheckedListProof checked = proof.verify(); + assertTrue(checked.isValid()); + } + + /** + * Returns a builder with a structurally valid flat proof for a tree of size 2 + * and an element at index 0. + */ + private FlatListProofBuilder twoElementListAt0() {/* + H + + 1 o + / \ + 0 e h + */ + long size = 2L; + ListProofElementEntry e0 = ELEMENT_ENTRIES.get(0); + ListProofHashedEntry h1 = hashedEntry(1, 0); + return new FlatListProofBuilder() + .size(size) + .addElement(e0) + .addProofEntry(h1); + } + + @Test + void threeElementListValidProofE2() { + /* + H — height of nodes at each level + 2 o + / \ + 1 h o + / + 0 e + */ + int index = 2; + ListProofElementEntry elementEntry = ELEMENT_ENTRIES.get(index); + long size = 3; + FlatListProof proof = new FlatListProofBuilder() + .size(size) + .addElement(elementEntry) + .addProofEntry(hashedEntry(0, 1)) + .build(); + + CheckedListProof checked = proof.verify(); + + assertThat(checked.size()).isEqualTo(size); + assertThat(checked.getElements()) + .containsExactly(entry((long) index, elementEntry.getElement())); + } + + @Test + void fourElementListInvalidProofMissingHashNode1() { + /* + H + 2 o + / \ + 1 o ? + / \ + 0 e h + */ + FlatListProof proof = new FlatListProofBuilder() + .size(4) + .addElement(ELEMENT_ENTRIES.get(0)) + .addProofEntry(hashedEntry(1, 0)) + .build(); + + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + + assertThat(e).hasMessageContaining("Missing proof entry at position (1, 1)"); + } + + @Test + void fourElementListInvalidProofMissingHashNode2() { + /* + H — height of nodes at each level + 2 o + / \ + 1 ? o + / \ + 0 e h + */ + FlatListProof proof = new FlatListProofBuilder() + .size(4) + .addElement(ELEMENT_ENTRIES.get(2)) + .addProofEntry(hashedEntry(3, 0)) + .build(); + + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + + assertThat(e).hasMessageContaining("Missing proof entry at position (1, 1)"); + } + + @Test + void fourElementListInvalidProofExtraNodesRedundantTotally() { + /* + H + 2 o + / \ + 1 o h + / \ / \ + 0 e h x x — the last two hashes are totally redundant as their parent must be + and is present in the proof tree + */ + ListProofHashedEntry r1 = hashedEntry(2, 0); + ListProofHashedEntry r2 = hashedEntry(3, 0); + FlatListProof proof = new FlatListProofBuilder() + .size(4) + .addElement(ELEMENT_ENTRIES.get(0)) + .addProofEntries(hashedEntry(1, 0), hashedEntry(1, 1)) + // Totally redundant hashed entries at height 0 + .addProofEntries(r1, r2) + .build(); + + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + + assertThat(e).hasMessageContaining("Redundant proof entries") + .hasMessageContaining(r1.toString()) + .hasMessageStartingWith(r2.toString()); + } + + + @Test + void eightElementListInvalidProofExtraNodesRedundantTotally( + List invalidEntries + ) { + /* + H + 3 o + / \ + 2 o h + / \ + 1 o h + /\ + 0 e h + */ + FlatListProof proof = new FlatListProofBuilder() + .size(8) + .addElement(ELEMENT_ENTRIES.get(0)) + .addProofEntries(hashedEntry(1, 0), hashedEntry(1, 1), hashedEntry(1, 2)) + // Totally redundant hashed entries at height 0 + .addProofEntries(invalidEntries) + .build(); + + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + + assertThat(e).hasMessageContaining("Redundant proof entries"); + } + + static Collection> + eightElementListInvalidProofExtraNodesRedundantTotallySource() { + /* + H + 2 o + / \ + 1 o h + / \ + 0 o h 1 2 + /\ + 0 e h 2 3 4 5 6 7 + + index is used instead of 'x' to show all possible redundant hashed entries + */ + return asList( + // At height 0 + singletonList(hashedEntry(2, 0)), + singletonList(hashedEntry(3, 0)), + singletonList(hashedEntry(7, 0)), + asList(hashedEntry(6,0), hashedEntry(7, 0)), + // At height 1 + singletonList(hashedEntry(2, 1)), + singletonList(hashedEntry(3, 1)), + asList(hashedEntry(2, 1), hashedEntry(3, 1)) + ); + } + + /** + * Creates a given number of single-byte elements, with their first and only byte + * equal to the index they are stored at. + */ + private static List createElements(int size) { + return IntStream.range(0, size) + .mapToObj(Bytes::bytes) + .collect(toList()); + } + + /** + * Creates list proof entries for each element of the given list. + * @param list a list of elements + */ + private static List createElementEntries(List list) { + int size = list.size(); + List entries = new ArrayList<>(size); + return IntStream.range(0, size) + .mapToObj(i -> elementEntry(0, list.get(i))) + .collect(toList()); + } + + private static ListProofElementEntry elementEntry(long index, byte[] element) { + return ListProofElementEntry.newInstance(index, element); + } + + private static ListProofHashedEntry hashedEntry(long index, int height) { + HashCode nodeHash = Hashing.sha256().newHasher() + .putLong(index) + .putInt(height) + .hash(); + return ListProofHashedEntry.newInstance(index, height, nodeHash); + } + + private static ListProofHashedEntry hashedEntry(long index, int height, HashCode nodeHash) { + return ListProofHashedEntry.newInstance(index, height, nodeHash); + } + + private static class FlatListProofBuilder { + List elements = new ArrayList<>(); + List proof = new ArrayList<>(); + long size = 0L; + + FlatListProofBuilder size(long size) { + this.size = size; + return this; + } + + FlatListProofBuilder addElement(ListProofElementEntry elementEntry) { + elements.add(elementEntry); + return this; + } + + FlatListProofBuilder addProofEntry(ListProofHashedEntry h1) { + proof.add(h1); + return this; + } + + FlatListProofBuilder addProofEntries(ListProofHashedEntry... proofHashedEntries) { + return addProofEntries(asList(proofHashedEntries)); + } + + FlatListProofBuilder addProofEntries(List proofHashedEntries) { + proof.addAll(proofHashedEntries); + return this; + } + + FlatListProofBuilder addAnyEntries(List entries) { + for (ListProofEntry e : entries) { + if (e instanceof ListProofHashedEntry) { + addProofEntry((ListProofHashedEntry) e); + } else if (e instanceof ListProofElementEntry) { + addElement((ListProofElementEntry) e); + } else { + throw new AssertionError("" + e); + } + } + return this; + } + + FlatListProof build() { + return new FlatListProof(elements, proof, size); + } + } +} From 4b00ad34e07746d90c6a1a1743f2e57b1b9c47d5 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Mon, 7 Oct 2019 19:01:50 +0300 Subject: [PATCH 02/24] Add tests for 2-element valid proofs --- .../common/proofs/list/FlatListProofTest.java | 109 +++++++++++++++++- .../list/ListProofHashCalculatorTest.java | 18 +-- .../common/proofs/list/ListProofUtils.java | 8 +- 3 files changed, 121 insertions(+), 14 deletions(-) diff --git a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java index 5deeb07a97..9e092ab2f8 100644 --- a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java +++ b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java @@ -18,6 +18,9 @@ import static com.exonum.binding.common.proofs.list.ListProofEntry.MAX_HEIGHT; import static com.exonum.binding.common.proofs.list.ListProofEntry.MAX_INDEX; +import static com.exonum.binding.common.proofs.list.ListProofUtils.getBranchHashCode; +import static com.exonum.binding.common.proofs.list.ListProofUtils.getLeafHashCode; +import static com.exonum.binding.common.proofs.list.ListProofUtils.getProofListHash; import static com.exonum.binding.test.Bytes.bytes; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; @@ -110,12 +113,106 @@ void singletonListValidProof() { assertTrue(checked.isValid()); assertThat(checked.size()).isEqualTo(size); - assertThat(checked.getElements()).containsExactly(entry((long) index, ELEMENTS.get(index))); - // todo: hash. + byte[] e0Value = ELEMENTS.get(index); + assertThat(checked.getElements()).containsExactly(entry((long) index, e0Value)); + HashCode rootHash = getLeafHashCode(e0Value); + HashCode expectedListHash = getProofListHash(rootHash, size); + assertThat(checked.getIndexHash()).isEqualTo(expectedListHash); } // todo: more singletonListInvalid - // todo: more twoItemValid + + @Test + void twoElementListValidProofE0() { + /* + H + 1 o + / \ + 0 e h + */ + int index = 0; + long size = 2L; + ListProofHashedEntry h1 = hashedEntry(1, 0); + FlatListProof proof = new FlatListProofBuilder() + .size(size) + .addElement(ELEMENT_ENTRIES.get(index)) + .addProofEntry(h1) + .build(); + + CheckedListProof checked = proof.verify(); + + assertTrue(checked.isValid()); + assertThat(checked.size()).isEqualTo(size); + byte[] e0value = ELEMENTS.get(index); + assertThat(checked.getElements()).containsExactly(entry((long) index, e0value)); + + HashCode node0Hash = getLeafHashCode(e0value); + HashCode node1Hash = h1.getHash(); + HashCode rootHash = getBranchHashCode(node0Hash, node1Hash); + HashCode expectedListHash = getProofListHash(rootHash, size); + assertThat(checked.getIndexHash()).isEqualTo(expectedListHash); + } + + @Test + void twoElementListValidProofE1() { + /* + H + 1 o + / \ + 0 h e + */ + int index = 1; + long size = 2L; + ListProofHashedEntry h0 = hashedEntry(0, 0); + FlatListProof proof = new FlatListProofBuilder() + .size(size) + .addElement(ELEMENT_ENTRIES.get(index)) + .addProofEntry(h0) + .build(); + + CheckedListProof checked = proof.verify(); + + assertTrue(checked.isValid()); + assertThat(checked.size()).isEqualTo(size); + byte[] e1value = ELEMENTS.get(index); + assertThat(checked.getElements()).containsExactly(entry((long) index, e1value)); + + HashCode node0Hash = h0.getHash(); + HashCode node1Hash = getLeafHashCode(e1value); + HashCode rootHash = getBranchHashCode(node0Hash, node1Hash); + HashCode expectedListHash = getProofListHash(rootHash, size); + assertThat(checked.getIndexHash()).isEqualTo(expectedListHash); + } + + @Test + void twoElementListValidProofFullProof() { + /* + H + 1 o + / \ + 0 e e + */ + long size = 2L; + FlatListProof proof = new FlatListProofBuilder() + .size(size) + .addElements(ELEMENT_ENTRIES.subList(0, (int) size)) + .build(); + + CheckedListProof checked = proof.verify(); + + assertTrue(checked.isValid()); + assertThat(checked.size()).isEqualTo(size); + assertThat(checked.getElements()).containsExactly( + entry(0L, ELEMENTS.get(0)), + entry(1L, ELEMENTS.get(1))); + + HashCode node0Hash = getLeafHashCode(ELEMENTS.get(0)); + HashCode node1Hash = getLeafHashCode(ELEMENTS.get(1)); + HashCode rootHash = getBranchHashCode(node0Hash, node1Hash); + HashCode expectedListHash = getProofListHash(rootHash, size); + assertThat(checked.getIndexHash()).isEqualTo(expectedListHash); + } + @Test void twoElementListInvalidProofMissingHashNode1() { /* @@ -241,6 +338,7 @@ void twoElementListInvalidProofExtraNodesAtInvalidIndexes( 0 e x h */ "0, 0", + /* H 1 x o @@ -561,6 +659,11 @@ FlatListProofBuilder addElement(ListProofElementEntry elementEntry) { return this; } + FlatListProofBuilder addElements(List elementEntries) { + elements.addAll(elementEntries); + return this; + } + FlatListProofBuilder addProofEntry(ListProofHashedEntry h1) { proof.add(h1); return this; diff --git a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/ListProofHashCalculatorTest.java b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/ListProofHashCalculatorTest.java index 4ef203c93f..203186a9c0 100644 --- a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/ListProofHashCalculatorTest.java +++ b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/ListProofHashCalculatorTest.java @@ -17,7 +17,7 @@ package com.exonum.binding.common.proofs.list; import static com.exonum.binding.common.proofs.list.ListProofUtils.getBranchHashCode; -import static com.exonum.binding.common.proofs.list.ListProofUtils.getNodeHashCode; +import static com.exonum.binding.common.proofs.list.ListProofUtils.getLeafHashCode; import static com.exonum.binding.common.proofs.list.ListProofUtils.getProofListHash; import static com.google.common.collect.ImmutableMap.of; import static org.hamcrest.CoreMatchers.equalTo; @@ -47,7 +47,7 @@ void visit_SingletonListProof() { long length = 1; calculator = new ListProofHashCalculator(root, length); - HashCode expectedProofListHash = getProofListHash(getNodeHashCode(V1), length); + HashCode expectedProofListHash = getProofListHash(getLeafHashCode(V1), length); assertThat(calculator.getElements(), equalTo(of(0L, V1))); assertEquals(expectedProofListHash, calculator.getHash()); @@ -63,7 +63,7 @@ void visit_FullProof2elements() { calculator = new ListProofHashCalculator(root, length); // Calculate expected proof list hash - HashCode expectedRootHash = getBranchHashCode(getNodeHashCode(V1), getNodeHashCode(V2)); + HashCode expectedRootHash = getBranchHashCode(getLeafHashCode(V1), getLeafHashCode(V2)); HashCode expectedProofListHash = getProofListHash(expectedRootHash, length); assertThat(calculator.getElements(), equalTo(of(0L, V1, @@ -88,8 +88,8 @@ void visit_FullProof4elements() { calculator = new ListProofHashCalculator(root, length); // Calculate expected proof list hash - HashCode leftBranchHash = getBranchHashCode(getNodeHashCode(V1), getNodeHashCode(V2)); - HashCode rightBranchHash = getBranchHashCode(getNodeHashCode(V3), getNodeHashCode(V4)); + HashCode leftBranchHash = getBranchHashCode(getLeafHashCode(V1), getLeafHashCode(V2)); + HashCode rightBranchHash = getBranchHashCode(getLeafHashCode(V3), getLeafHashCode(V4)); HashCode expectedRootHash = getBranchHashCode(leftBranchHash, rightBranchHash); HashCode expectedProofListHash = getProofListHash(expectedRootHash, length); @@ -111,7 +111,7 @@ void visit_ProofLeftValue() { long length = 2; calculator = new ListProofHashCalculator(root, length); - HashCode expectedRootHash = getBranchHashCode(getNodeHashCode(V1), H2); + HashCode expectedRootHash = getBranchHashCode(getLeafHashCode(V1), H2); HashCode expectedProofListHash = getProofListHash(expectedRootHash, length); assertThat(calculator.getElements(), equalTo(of(0L, V1))); @@ -127,7 +127,7 @@ void visit_ProofRightValue() { long length = 2; calculator = new ListProofHashCalculator(root, length); - HashCode expectedRootHash = getBranchHashCode(H1, getNodeHashCode(V2)); + HashCode expectedRootHash = getBranchHashCode(H1, getLeafHashCode(V2)); HashCode expectedProofListHash = getProofListHash(expectedRootHash, length); assertThat(calculator.getElements(), equalTo(of(1L, V2))); @@ -150,8 +150,8 @@ void visit_FullProof3elements() { long length = 3; calculator = new ListProofHashCalculator(root, length); - HashCode leftBranchHash = getBranchHashCode(getNodeHashCode(V1), getNodeHashCode(V2)); - HashCode rightBranchHash = getBranchHashCode(getNodeHashCode(V3), null); + HashCode leftBranchHash = getBranchHashCode(getLeafHashCode(V1), getLeafHashCode(V2)); + HashCode rightBranchHash = getBranchHashCode(getLeafHashCode(V3), null); HashCode expectedRootHash = getBranchHashCode(leftBranchHash, rightBranchHash); HashCode expectedProofListHash = getProofListHash(expectedRootHash, length); diff --git a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/ListProofUtils.java b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/ListProofUtils.java index 3b7450e791..243d25adb9 100644 --- a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/ListProofUtils.java +++ b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/ListProofUtils.java @@ -54,10 +54,14 @@ static ListProofNode generateRightLeaningProofTree(int depth, ListProofNode leaf return root; } - static HashCode getNodeHashCode(ByteString v) { + static HashCode getLeafHashCode(ByteString value) { + return getLeafHashCode(value.toByteArray()); + } + + static HashCode getLeafHashCode(byte[] value) { return Hashing.defaultHashFunction().newHasher() .putByte(BLOB_PREFIX) - .putBytes(v.toByteArray()) + .putBytes(value) .hash(); } From b2188b4b178a41b63456b3bbe41b591488f83d7f Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Mon, 7 Oct 2019 19:02:17 +0300 Subject: [PATCH 03/24] Add todos on incomplete trees --- .../exonum/binding/common/proofs/list/FlatListProofTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java index 9e092ab2f8..799b27573b 100644 --- a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java +++ b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java @@ -453,6 +453,7 @@ private FlatListProofBuilder twoElementListAt0() {/* @Test void threeElementListValidProofE2() { + // todo: this or that: https://wiki.bf.local/display/EXN/Flat+list+proofs?focusedCommentId=34901842#comment-34901842 /* H — height of nodes at each level 2 o @@ -475,6 +476,7 @@ void threeElementListValidProofE2() { assertThat(checked.size()).isEqualTo(size); assertThat(checked.getElements()) .containsExactly(entry((long) index, elementEntry.getElement())); + // todo: verify hashing with odd number of elements } @Test From b2821c5dbe383a778ca4b991d18d8f1b5565a131 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Mon, 7 Oct 2019 19:34:17 +0300 Subject: [PATCH 04/24] Add tests for trees having odd numbers of nodes on some levels --- .../common/proofs/list/FlatListProofTest.java | 95 ++++++++++++++++++- 1 file changed, 92 insertions(+), 3 deletions(-) diff --git a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java index 799b27573b..cbd3ee8516 100644 --- a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java +++ b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java @@ -465,18 +465,26 @@ void threeElementListValidProofE2() { int index = 2; ListProofElementEntry elementEntry = ELEMENT_ENTRIES.get(index); long size = 3; + ListProofHashedEntry h0 = hashedEntry(0, 1); FlatListProof proof = new FlatListProofBuilder() .size(size) .addElement(elementEntry) - .addProofEntry(hashedEntry(0, 1)) + .addProofEntry(h0) .build(); CheckedListProof checked = proof.verify(); assertThat(checked.size()).isEqualTo(size); assertThat(checked.getElements()) - .containsExactly(entry((long) index, elementEntry.getElement())); - // todo: verify hashing with odd number of elements + .containsExactly(entry((long) index, ELEMENTS.get(index))); + + // Check the hash code + HashCode leafHash = getLeafHashCode(ELEMENTS.get(index)); + HashCode node0Hash = h0.getHash(); + HashCode node1Hash = getBranchHashCode(leafHash, null); + HashCode rootHash = getBranchHashCode(node0Hash, node1Hash); + HashCode expectedListHash = getProofListHash(rootHash, size); + assertThat(checked.getIndexHash()).isEqualTo(expectedListHash); } @Test @@ -552,8 +560,89 @@ void fourElementListInvalidProofExtraNodesRedundantTotally() { .hasMessageStartingWith(r2.toString()); } + @ParameterizedTest + @MethodSource("fiveElementListInvalidProofExtraNodesAtInvalidIndexesSource") + void fiveElementListInvalidProofExtraNodesAtInvalidIndexes( + List invalidProofEntries) { + /* + H + 3 o + / \ + 2 h o + / + 1 o {x} <- invalid nodes at various heights with indexes exceeding max + / + 0 e {x} <— + */ + FlatListProof proof = fiveElementListAt4() + .addAnyEntries(invalidProofEntries) + .build(); + + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + + assertThat(e).hasMessageContaining("Invalid node") + .hasMessageFindingMatch("(?i)at index .+ exceeding the max"); + } + + private static Collection> + fiveElementListInvalidProofExtraNodesAtInvalidIndexesSource() { + /* + Target proof tree: + H + 3 o + / \ + 2 h o + / + 1 o {x} <- invalid nodes at various heights with indexes exceeding max + / + 0 e {x} <— + */ + byte[] value = bytes(1, 2, 3); + return asList( + // | index | height | + // At height 0 + singletonList(hashedEntry(5, 0)), + singletonList(hashedEntry(6, 0)), + singletonList(hashedEntry(ListProofEntry.MAX_INDEX, 0)), + singletonList(elementEntry(5, value)), + singletonList(elementEntry(6, value)), + singletonList(elementEntry(MAX_INDEX, value)), + asList(hashedEntry(5, 0), hashedEntry(7, 0)), + // At height 1 + singletonList(hashedEntry(3, 1)), + singletonList(hashedEntry(4, 1)) + ); + } @Test + void fiveElementListValidProofAt4() { + // Check the proof returned by fiveElementListAt4 is valid + FlatListProof proof = fiveElementListAt4().build(); + CheckedListProof checked = proof.verify(); + assertTrue(checked.isValid()); + assertThat(checked.getElements()).containsExactly(entry(4L, ELEMENTS.get(4))); + } + + private FlatListProofBuilder fiveElementListAt4() { + /* + H + 3 o + / \ + 2 h o + / + 1 o + / + 0 e + */ + return new FlatListProofBuilder() + .size(5) + .addElement(ELEMENT_ENTRIES.get(4)) + .addProofEntry(hashedEntry(0, 2)); + } + + @ParameterizedTest + @MethodSource("eightElementListInvalidProofExtraNodesRedundantTotallySource") void eightElementListInvalidProofExtraNodesRedundantTotally( List invalidEntries ) { From 6df880ad51caa14aa68151449caf2899a0366b3e Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Mon, 7 Oct 2019 19:36:47 +0300 Subject: [PATCH 05/24] Fix the number of elements --- .../exonum/binding/common/proofs/list/FlatListProofTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java index cbd3ee8516..c1f274cc0f 100644 --- a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java +++ b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java @@ -66,7 +66,7 @@ class FlatListProofTest { private static final HashCode EMPTY_LIST_INDEX_HASH = HashCode.fromString("c6c0aa07f27493d2f2e5cff56c890a353a20086d6c25ec825128e12ae752b2d9"); - private static final List ELEMENTS = createElements(4); + private static final List ELEMENTS = createElements(8); private static final List ELEMENT_ENTRIES = createElementEntries(ELEMENTS); @Test From c37810b0927923f9d476820bc6dc1d8d4e42a2fc Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Mon, 7 Oct 2019 20:58:33 +0300 Subject: [PATCH 06/24] Extend the integration test to cover various odd-numbered lists --- .../indices/ProofListIndexProxyIntegrationTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListIndexProxyIntegrationTest.java b/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListIndexProxyIntegrationTest.java index 9772612a55..51e1f5d916 100644 --- a/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListIndexProxyIntegrationTest.java +++ b/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListIndexProxyIntegrationTest.java @@ -34,6 +34,8 @@ import java.util.function.Consumer; import java.util.function.Function; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; /** * Contains tests of ProofListIndexProxy methods @@ -140,10 +142,11 @@ void getRangeProofSingletonList() { }); } - @Test - void getProofMultipleItemList() { + @ParameterizedTest + @ValueSource(ints = {2, 3, 4, 5, 7, 8, 9}) + void getProofMultipleItemList(int size) { runTestWithView(database::createFork, (list) -> { - List values = TestStorageItems.values; + List values = TestStorageItems.values.subList(0, size); list.addAll(values); From 388b0a6ad9a69ed143ec29fbef808f5f57147f78 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Tue, 8 Oct 2019 21:15:41 +0300 Subject: [PATCH 07/24] Add tests for empty-range proof. --- .../common/proofs/list/FlatListProofTest.java | 116 +++++++++++++++++- .../indices/ProofListContainsMatcher.java | 1 + .../ProofListIndexProxyIntegrationTest.java | 13 ++ 3 files changed, 129 insertions(+), 1 deletion(-) diff --git a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java index c1f274cc0f..3c0464866b 100644 --- a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java +++ b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java @@ -42,6 +42,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; /* The test uses the following illustrations to aid understanding of the proof structure: @@ -122,6 +123,120 @@ void singletonListValidProof() { // todo: more singletonListInvalid + @ParameterizedTest + @CsvSource({ + // size of the list, height of the tree (and the root node) + "1, 0", + "2, 1", + "3, 2", + "4, 2", + "5, 3", + "8, 3" + }) + void emptyRangeValidProof(long size, int height) { + /* + H + height h — a single hash entry for the root, no elements + */ + ListProofHashedEntry h1 = hashedEntry(0L, height); + FlatListProof proof = new FlatListProofBuilder() + .size(size) + .addProofEntry(h1) + .build(); + + CheckedListProof checked = proof.verify(); + + assertTrue(checked.isValid()); + assertThat(checked.size()).isEqualTo(size); + assertThat(checked.getElements()).isEmpty(); + + HashCode rootHash = h1.getHash(); + HashCode expectedListHash = getProofListHash(rootHash, size); + assertThat(checked.getIndexHash()).isEqualTo(expectedListHash); + } + + @ParameterizedTest + @ValueSource(longs = {1, 2, 3, 4}) + void emptyRangeInvalidProofNoHash(long size) { + FlatListProof proof = new FlatListProofBuilder() + .size(size) + .build(); + + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + assertThat(e).hasMessageContaining("No root hash in empty range proof"); + } + + @ParameterizedTest + @CsvSource({ + // index | height. The only correct pair is 0, 2 for a 4-element list. + // Invalid indices + "1, 2", + "2, 2", + // Invalid heights + // Below + "0, 0", + "0, 1", + // Above + "0, 3", + "0, 4" + }) + void emptyRangeInvalidProofBadRootHashEntry(long index, int height) { + /* + H + 2 ? — root hashed entry at position (0, 2) + + 1 {x} — various hashed entries scattered among levels + + 0 + */ + ListProofHashedEntry h1 = hashedEntry(index, height); + FlatListProof proof = new FlatListProofBuilder() + .size(4L) + .addProofEntry(h1) + .build(); + + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + assertThat(e).hasMessageContaining(h1.toString()); + } + + @ParameterizedTest + @MethodSource("emptyRangeInvalidProofRedundantHashNodesSource") + void emptyRangeInvalidProofRedundantHashNodes(List redundantEntries) { + /* + H + 2 h — root hashed entry + + 1 {x} — various hashed entries scattered among levels, + with valid and invalid indexes + + 0 + */ + ListProofHashedEntry rootHashEntry = hashedEntry(0L, 2); + FlatListProof proof = new FlatListProofBuilder() + .size(4L) + .addProofEntry(rootHashEntry) + .addProofEntries(redundantEntries) + .build(); + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + assertThat(e).hasMessageContaining("A single root hash proof entry expected, but (%d) found"); + } + + static List> emptyRangeInvalidProofRedundantHashNodesSource() { + return asList( + singletonList(hashedEntry(0L, 1)), + // Sibling nodes at height 1, with valid indices + asList(hashedEntry(0L, 1), hashedEntry(1L, 1)), + // Nodes with invalid heights/indexes + // Invalid heights + singletonList(hashedEntry(0L, 3)), + // Invalid indices + singletonList(hashedEntry(2L, 1)) + ); + } + @Test void twoElementListValidProofE0() { /* @@ -453,7 +568,6 @@ private FlatListProofBuilder twoElementListAt0() {/* @Test void threeElementListValidProofE2() { - // todo: this or that: https://wiki.bf.local/display/EXN/Flat+list+proofs?focusedCommentId=34901842#comment-34901842 /* H — height of nodes at each level 2 o diff --git a/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListContainsMatcher.java b/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListContainsMatcher.java index 193df2a802..bab5eaa4cf 100644 --- a/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListContainsMatcher.java +++ b/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListContainsMatcher.java @@ -61,6 +61,7 @@ protected boolean matchesSafely(ProofListIndexProxy list) { .collect(toMap(Map.Entry::getKey, e -> e.getValue().toStringUtf8())); return checkedProof.isValid() + // todo: verify the list size also && elementsMatcher.matches(actualElements) && list.getIndexHash().equals(checkedProof.getIndexHash()); } diff --git a/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListIndexProxyIntegrationTest.java b/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListIndexProxyIntegrationTest.java index 51e1f5d916..f7c9b7a0dd 100644 --- a/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListIndexProxyIntegrationTest.java +++ b/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListIndexProxyIntegrationTest.java @@ -20,6 +20,7 @@ import static com.exonum.binding.core.storage.indices.ProofListContainsMatcher.provesAbsence; import static com.exonum.binding.core.storage.indices.ProofListContainsMatcher.provesThatContains; import static com.exonum.binding.core.storage.indices.TestStorageItems.V1; +import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.IsEqual.equalTo; @@ -190,6 +191,18 @@ void getRangeProofMultipleItemList_2ndHalf() { }); } + @ParameterizedTest + @ValueSource(ints = {1, 2, 3, 4}) + void getRangeProofMultipleItemList_EmptyRange(int size) { + runTestWithView(database::createFork, (list) -> { + List values = TestStorageItems.values.subList(0, size); + + list.addAll(values); + + assertThat(list, provesThatContains(0, emptyList())); + }); + } + private static void runTestWithView(Function viewFactory, Consumer> listTest) { runTestWithView(viewFactory, (ignoredView, list) -> listTest.accept(list)); From 9a3f6b0bb860c0992cd4a4054b91dd6a418100d2 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Wed, 9 Oct 2019 22:25:15 +0300 Subject: [PATCH 08/24] WIP Impl proof verification: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 78 of 81 tests pass; 3 — fail. --- .../proofs/list/CheckedListProofImpl.java | 11 +- .../common/proofs/list/FlatListProof.java | 328 +++++++++++++++++- .../proofs/list/ListProofElementEntry.java | 8 +- .../list/UncheckedListProofAdapter.java | 2 +- .../common/proofs/list/FlatListProofTest.java | 93 +++-- 5 files changed, 403 insertions(+), 39 deletions(-) diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/CheckedListProofImpl.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/CheckedListProofImpl.java index eeff30c2fd..9d056a9da3 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/CheckedListProofImpl.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/CheckedListProofImpl.java @@ -37,12 +37,11 @@ * {@link #getProofStatus()} with description of why the proof is not * valid. */ -public class CheckedListProofImpl implements CheckedListProof { +public class CheckedListProofImpl implements CheckedListProof { + private final long size; private final HashCode calculatedIndexHash; - private final NavigableMap elements; - private final ListProofStatus proofStatus; /** @@ -51,8 +50,9 @@ public class CheckedListProofImpl implements CheckedListProof { * @param elements proof elements collection (empty in case of a proof of absence) * @param proofStatus a status of proof verification */ - public CheckedListProofImpl(HashCode calculatedIndexHash, + public CheckedListProofImpl(long size, HashCode calculatedIndexHash, NavigableMap elements, ListProofStatus proofStatus) { + this.size = size; this.calculatedIndexHash = checkNotNull(calculatedIndexHash); this.elements = checkNotNull(elements); this.proofStatus = checkNotNull(proofStatus); @@ -60,8 +60,7 @@ public CheckedListProofImpl(HashCode calculatedIndexHash, @Override public long size() { - // todo: - return 0; + return size; } @Override diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java index 6355a4d2d3..b80ca84376 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java @@ -16,15 +16,79 @@ package com.exonum.binding.common.proofs.list; +import static com.exonum.binding.common.hash.Funnels.hashCodeFunnel; +import static com.exonum.binding.common.hash.Hashing.sha256; +import static com.exonum.binding.common.proofs.list.ListProofEntry.MAX_INDEX; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.math.BigIntegerMath.log2; +import static java.util.stream.Collectors.toMap; +import com.exonum.binding.common.hash.HashCode; +import com.exonum.binding.common.hash.Hasher; +import com.exonum.binding.common.hash.Hashing; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.function.BinaryOperator; +import org.checkerframework.checker.nullness.qual.Nullable; /** * A flat list proof. It proves that certain elements are present in a proof list * of a certain size. */ class FlatListProof { + + @VisibleForTesting + static final byte BLOB_PREFIX = 0x00; + @VisibleForTesting + static final byte LIST_BRANCH_PREFIX = 0x01; + @VisibleForTesting + static final byte LIST_ROOT_PREFIX = 0x02; + + private static final long MAX_SIZE = MAX_INDEX + 1; + + private static HashCode EMPTY_LIST_INDEX_HASH = hashListIndex(0L, + HashCode.fromBytes(new byte[Hashing.DEFAULT_HASH_SIZE_BYTES])); + /* + Proof lists are represented as BSTs, where all leaf elements are at the same height. + Here is a tree for a three-element proof list: + + H — height of nodes at each level + 2 o + / \ + 1 o o + / \ / + 0 e e e + + Proofs for elements of proof lists include the requested elements; and hashes of all + the sub-trees that are adjacent to the paths from the root node to the leaf nodes containing + the elements. + + Here is a proof for the element at index 1 from the list shown above: + + H + 2 o + / \ + 1 o h + / \ + 0 h e + _________________ + Legend: + e — element proof entry (ListProofElementEntry) + h — hashed proof entry (ListProofHashedEntry) + o — 'virtual' node — not present in the proof, inferred during verification. + Shown mostly to communicate the tree structure of the proof. + */ private final List elements; private final List proof; private final long size; @@ -37,6 +101,268 @@ class FlatListProof { } CheckedListProof verify() { - return null; + // Check the size + checkArgument(0 <= size && size <= MAX_SIZE, + "Invalid size (%s), must be in range [0; 2^56]", size); + + // Handle special cases + // Empty list + if (size == 0) { + // Check there are no elements or proof entries + if (!elements.isEmpty()) { + throw new InvalidProofException("Proof for empty list must not have elements, but has: " + + elements); + } + if (!proof.isEmpty()) { + throw new InvalidProofException( + "Proof for empty list must not have proof entries, but has: " + proof); + } + return new CheckedListProofImpl<>(size, EMPTY_LIST_INDEX_HASH, new TreeMap<>(), + ListProofStatus.VALID); + } else if (elements.isEmpty()) { + // Empty range: must have a single root hash node + if (proof.size() != 1) { + throw new InvalidProofException(String.format( + "Proof for an empty range must have a single proof node, but has %d: %s", + proof.size(), proof)); + } + ListProofHashedEntry h = proof.get(0); + int treeHeight = calcTreeHeight(size); + // Check height + if (h.getHeight() != treeHeight) { + throw new InvalidProofException( + String.format("Proof node for an empty range at invalid height (%d)," + + "must be at height (%d) for a list of size %d: %s", h.getHeight(), + treeHeight, size, h)); + } + // Check index + if (h.getIndex() != 0L) { + throw new InvalidProofException( + String.format("Proof node for an empty range at invalid index (%d)," + + "must be always at index 0: %s", h.getIndex(), h)); + } + HashCode rootHash = h.getHash(); + HashCode listHash = hashListIndex(rootHash); + return new CheckedListProofImpl<>(size, listHash, new TreeMap<>(), ListProofStatus.VALID); + } else { + // A list of size 1+ + + // Calculate the expected tree height + int treeHeight = calcTreeHeight(size); + + // Index proof entries and verify their local correctness: no out-of-range nodes; no duplicates. + List> proofByHeight = new ArrayList<>(treeHeight); + for (int i = 0; i < treeHeight; i++) { + // For single-element proofs, only a single proof node is expected on each level. + // For contiguous range proofs, up to two proof nodes are expected on any level. + // Multiple-range proofs might have up to 'elements.size' proof nodes on the lowest level, + // but Exonum does not currently produce such. + int initialCapacity = (elements.size() <= 1) ? 1 : 2; + Map proofAtHeight = new HashMap<>(initialCapacity); + proofByHeight.add(proofAtHeight); + } + for (ListProofHashedEntry hashedEntry : proof) { + // Check height + int height = hashedEntry.getHeight(); + if (height < 0 || treeHeight <= height) { + throw new InvalidProofException( + String.format("Proof entry at invalid height (%d), must be in range [0; %d): %s", + height, treeHeight, hashedEntry)); + } + // Check index + long levelSize = levelSizeAt(height); + long index = hashedEntry.getIndex(); + if (index < 0L || levelSize <= index) { + throw new InvalidProofException(String + .format( + "Proof entry at invalid index (%d); it must be in range [0; %d) at height %d: %s", + index, levelSize, height, hashedEntry)); + } + // Add the entry at the height, checking for duplicates + Map proofsAtHeight = proofByHeight.get(height); + ListProofHashedEntry present = proofsAtHeight.putIfAbsent(index, hashedEntry); + if (present != null) { + throw new InvalidProofException( + String.format("Multiple proof entries at the same position: %s and %s", + present, hashedEntry)); + } + } + + // Check element entries: have unique indexes that are in range [0; size) + Map elementsByIndex = Maps + .uniqueIndex(elements, ListProofEntry::getIndex); + for (ListProofElementEntry elementEntry : elements) { + long index = elementEntry.getIndex(); + if (index < 0L || size <= index) { + throw new InvalidProofException( + String.format("Entry at invalid index (%d), must be in range [0; %d): %s", + index, size, elementEntry)); + } + } + + // Hash the element entries, and obtain the first level of calculated hashes +// Map calculated = elements.stream() +// .map(FlatListProof::hashLeafNode) +// .collect(toMap(ListProofEntry::getIndex, Functions.identity())); + Map calculated = Maps + .transformValues(elementsByIndex, FlatListProof::hashLeafNode); + + // For each tree level, starting at the bottom + for (int height = 0; height < treeHeight; height++) { + // Verify nodes: + // - For an inferred node n there is a sibling either in the inferred + // nodes or in hash nodes; or it is the last node in a odd-sized level. + Map proofAtLevel = proofByHeight.get(height); + for (ListProofHashedEntry inferredNode : calculated.values()) { + if (isLastOnOddLevel(inferredNode)) { + // The last node on an odd-sized level does not have a sibling. + continue; + } + long index = inferredNode.getIndex(); + long siblingIndex = getSiblingIndex(index); + if (!(calculated.containsKey(siblingIndex) || proofAtLevel.containsKey(siblingIndex))) { + throw new InvalidProofException( + String.format("Missing proof entry at index (%d) for the calculated one: %s", + siblingIndex, inferredNode)); + } + } + // Verify proof nodes: + for (ListProofHashedEntry proofNode : proofAtLevel.values()) { + long index = proofNode.getIndex(); + // No hash nodes overriding the inferred nodes (i.e., have same index) + if (calculated.containsKey(index)) { + throw new InvalidProofException( + String.format("Redundant proof entry (%s) with the same index (%d) as " + + "the calculated node (%s)", proofNode, index, calculated.get(index))); + } + // No redundant hash nodes that have no siblings in the inferred nodes. + long siblingIndex = getSiblingIndex(index); + if (!calculated.containsKey(siblingIndex)) { + throw new InvalidProofException( + String.format("Redundant proof entry (%s) not needed for verification", proofNode)); + } + } + + // Merge calculated on the previous level hashes with the proof entries of the current level, + // ordered by indexes. The sibling nodes will go in adjacent pairs. + SortedMap merged = new TreeMap<>(); + merged.putAll(calculated); + merged.putAll(proofAtLevel); + + // Reduce the adjacent nodes to produce the nodes on the upper level + Iterator mergedIter = merged.values().iterator(); + Map nextLevel = new HashMap<>((merged.size() + 1) / 2); + while (mergedIter.hasNext()) { + ListProofHashedEntry left = mergedIter.next(); + ListProofHashedEntry right = null; + if (mergedIter.hasNext()) { + right = mergedIter.next(); + } + ListProofHashedEntry parent = hashBranchNode(left, right); + nextLevel.put(parent.getIndex(), parent); + } + // Proceed to the next level + calculated = nextLevel; + } + + // Take the root hash and calculate the index hash. + HashCode rootHash = calculated.get(0L).getHash(); + HashCode indexHash = hashListIndex(rootHash); + + // Extract the entries. + // todo: Extract into newCheckedProof(indexHash) — all the other things can be inferred. + return new CheckedListProofImpl<>(size, indexHash, indexElements(), ListProofStatus.VALID); + } + } + + @VisibleForTesting + static int calcTreeHeight(long size) { + // Beware, toIntExact(round(ceil(log(size) / log(2.)))) won't work for large sizes because + // `log(size) / log(2.)` might produce a number slightly larger than the actual, with + // subsequent `ceil` making it off by +1. + // For instance, ceil(log(2^55 as long) / log(2.)) -> 56. + return (size == 0L) ? 0 + : log2(BigInteger.valueOf(size), RoundingMode.CEILING); + } + + private long levelSizeAt(int height) { + // todo: Is there a one-liner for ceil(ceil(... ceil(size / 2) / 2 ... / 2)))? + // todo: consider memoizing if no efficient ^ + checkArgument(height >= 0); + long levelSize = size; + while (height-- != 0) { + levelSize = (levelSize + 1) / 2; + } + return levelSize; + } + + private boolean isLastOnOddLevel(ListProofHashedEntry inferredNode) { + long levelSize = levelSizeAt(inferredNode.getHeight()); + long index = inferredNode.getIndex(); + return isOdd(levelSize) && (index == levelSize - 1); + } + + private static long getSiblingIndex(long index) { + if (isEven(index)) { + return index + 1; + } else { + return index - 1; + } + } + + private static boolean isOdd(long v) { + return !isEven(v); + } + + private static boolean isEven(long v) { + return (v & 1L) == 0L; + } + + private static ListProofHashedEntry hashLeafNode(ListProofElementEntry elementEntry) { + long index = elementEntry.getIndex(); + HashCode hash = newHasher() + .putByte(BLOB_PREFIX) + .putBytes(elementEntry.getElement()) + .hash(); + return ListProofHashedEntry.newInstance(index, 0, hash); + } + + private static ListProofHashedEntry hashBranchNode(ListProofHashedEntry leftChild, + @Nullable ListProofHashedEntry rightChild) { + long index = leftChild.getIndex() / 2; + int height = leftChild.getHeight() + 1; + Hasher hasher = newHasher() + .putByte(LIST_BRANCH_PREFIX) + .putObject(leftChild.getHash(), hashCodeFunnel()); + if (rightChild != null) { + hasher.putObject(rightChild.getHash(), hashCodeFunnel()); + } + return ListProofHashedEntry.newInstance(index, height, hasher.hash()); + } + + private HashCode hashListIndex(HashCode rootHash) { + return hashListIndex(size, rootHash); + } + + private static HashCode hashListIndex(long size, HashCode rootHash) { + return newHasher() + .putByte(LIST_ROOT_PREFIX) + .putLong(size) + .putObject(rootHash, hashCodeFunnel()) + .hash(); + } + + private static Hasher newHasher() { + return sha256().newHasher(); + } + + private NavigableMap indexElements() { + return elements.stream() + .collect(toMap(ListProofEntry::getIndex, ListProofElementEntry::getElement, + throwingMerger(), TreeMap::new)); + } + + private static BinaryOperator throwingMerger() { + return (u, v) -> { throw new IllegalArgumentException("Duplicate values with the same key"); }; } } diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofElementEntry.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofElementEntry.java index 2f2922c474..a6a5a9ae4a 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofElementEntry.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofElementEntry.java @@ -24,6 +24,12 @@ @AutoValue abstract class ListProofElementEntry implements ListProofEntry { + @Override + public int getHeight() { + // Elements are always at height 0 + return 0; + } + /** * Returns a value of the element stored at this index in the list. */ @@ -31,6 +37,6 @@ abstract class ListProofElementEntry implements ListProofEntry { static ListProofElementEntry newInstance(long index, byte[] element) { ListProofEntry.checkIndex(index); - return new AutoValue_ListProofElementEntry(index, 0, element); + return new AutoValue_ListProofElementEntry(index, element); } } diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/UncheckedListProofAdapter.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/UncheckedListProofAdapter.java index 52728a1041..b0dd6ec674 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/UncheckedListProofAdapter.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/UncheckedListProofAdapter.java @@ -51,7 +51,7 @@ public CheckedListProof check() { ListProofStatus structureCheckStatus = listProofStructureValidator.getProofStatus(); HashCode calculatedIndexHash = listProofHashCalculator.getHash(); - return new CheckedListProofImpl<>( + return new CheckedListProofImpl<>(0, calculatedIndexHash, listProofHashCalculator.getElements(), structureCheckStatus); } diff --git a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java index 3c0464866b..dc47a61389 100644 --- a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java +++ b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java @@ -16,6 +16,7 @@ package com.exonum.binding.common.proofs.list; +import static com.exonum.binding.common.proofs.list.FlatListProof.calcTreeHeight; import static com.exonum.binding.common.proofs.list.ListProofEntry.MAX_HEIGHT; import static com.exonum.binding.common.proofs.list.ListProofEntry.MAX_INDEX; import static com.exonum.binding.common.proofs.list.ListProofUtils.getBranchHashCode; @@ -30,6 +31,7 @@ import static org.assertj.core.api.Assertions.entry; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.params.provider.Arguments.arguments; import com.exonum.binding.common.hash.HashCode; import com.exonum.binding.common.hash.Hashing; @@ -40,6 +42,7 @@ import java.util.stream.IntStream; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; @@ -70,6 +73,33 @@ class FlatListProofTest { private static final List ELEMENTS = createElements(8); private static final List ELEMENT_ENTRIES = createElementEntries(ELEMENTS); + @ParameterizedTest + @MethodSource("calcTreeHeightSource") + void testCalcTreeHeight(long size, int expectedHeight) { + assertThat(calcTreeHeight(size)).isEqualTo(expectedHeight); + } + + private static List calcTreeHeightSource() { + int maxHeight = 56; + long maxSize = 0x100_0000_0000_0000L; // 2^56 + return asList( + // | size | height | + arguments(0, 0), + arguments(1, 0), + arguments(2, 1), + arguments(3, 2), + arguments(4, 2), + arguments(5, 3), + arguments(8, 3), + arguments(9, 4), + arguments(maxSize / 2 - 1, maxHeight - 1), + arguments(maxSize / 2, maxHeight - 1), + arguments(maxSize / 2 + 1, maxHeight), + arguments(maxSize - 1, maxHeight), + arguments(maxSize, maxHeight) + ); + } + @Test void emptyListValidProof() { FlatListProof proof = new FlatListProof(emptyList(), emptyList(), 0L); @@ -88,7 +118,7 @@ void emptyListInvalidProofIfHasElements() { InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageContaining("Unexpected element entry") + assertThat(e).hasMessageContaining("must not have elements") .hasMessageContaining(unexpectedElement.toString()); } @@ -99,7 +129,7 @@ void emptyListInvalidProofIfHasHashedEntries() { InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageContaining("Unexpected proof entry") + assertThat(e).hasMessageContaining("Proof for empty list must not have proof entries") .hasMessageContaining(unexpectedEntry.toString()); } @@ -164,7 +194,7 @@ void emptyRangeInvalidProofNoHash(long size) { InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageContaining("No root hash in empty range proof"); + assertThat(e).hasMessageContaining("Proof for an empty range must have a single proof node"); } @ParameterizedTest @@ -221,7 +251,8 @@ void emptyRangeInvalidProofRedundantHashNodes(List redunda .build(); InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageContaining("A single root hash proof entry expected, but (%d) found"); + assertThat(e).hasMessageContaining("Proof for an empty range must have a single proof node") + .hasMessageContaining(String.valueOf(redundantEntries.size() + 1)); } static List> emptyRangeInvalidProofRedundantHashNodesSource() { @@ -329,7 +360,7 @@ void twoElementListValidProofFullProof() { } @Test - void twoElementListInvalidProofMissingHashNode1() { + void twoElementListInvalidProofMissingHashNodeAt1() { /* H 1 o @@ -344,18 +375,18 @@ void twoElementListInvalidProofMissingHashNode1() { InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageContaining("Missing proof entry at position (1, 0)"); + assertThat(e).hasMessageContaining("Missing proof entry at index (1)"); } @Test - void twoElementListInvalidProofMissingHashNode2() { + void twoElementListInvalidProofMissingHashNodeAt0() { /* H 1 o / \ 0 ? e */ - int index = 0; + int index = 1; ListProofElementEntry element = ELEMENT_ENTRIES.get(index); long size = 2L; FlatListProof proof = new FlatListProof(singletonList(element), emptyList(), size); @@ -363,7 +394,7 @@ void twoElementListInvalidProofMissingHashNode2() { InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageContaining("Missing proof entry at position (1, 0)"); + assertThat(e).hasMessageContaining("Missing proof entry at index (0)"); } @ParameterizedTest @@ -384,8 +415,8 @@ void twoElementListInvalidProofNodesAboveMaxHeight(List in InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageContaining("Invalid node") - .hasMessageContaining("at height above the max (1)"); + assertThat(e).hasMessageContaining("Proof entry at invalid height") + .hasMessageContaining("must be in range [0; 1)"); } static List> twoElementListInvalidProofNodesAboveMaxHeightSource() { @@ -416,8 +447,10 @@ void twoElementListInvalidProofExtraNodesAtInvalidIndexes( InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageContaining("Invalid node") - .hasMessageFindingMatch("(?i)at index .+ exceeding the max"); + String message = e.getMessage(); + assertThat(message) + .containsIgnoringCase("entry at invalid index") + .contains("must be in range [0; "); } private static Collection> @@ -438,9 +471,7 @@ void twoElementListInvalidProofExtraNodesAtInvalidIndexes( singletonList(elementEntry(3, value)), singletonList(elementEntry(MAX_INDEX, value)), singletonList(hashedEntry(ListProofEntry.MAX_INDEX, 0)), - asList(hashedEntry(2, 0), hashedEntry(3, 0)), - singletonList(hashedEntry(1, 1)), - singletonList(hashedEntry(2, 1)) + asList(hashedEntry(2, 0), hashedEntry(3, 0)) ); } @@ -471,9 +502,9 @@ void twoElementListInvalidProofExtraNodesRedundantCalculated(long index, int hei InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageContaining("Redundant proof node") + assertThat(e).hasMessageContaining("Redundant proof entry") .hasMessageContaining(redundantNode.toString()) - .hasMessageContaining("must be inferred"); + .hasMessageFindingMatch("with the same index .+ as the calculated node"); } @Test @@ -492,7 +523,7 @@ void twoElementListInvalidProofExtraNodesDuplicateHashed() { InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageFindingMatch("Duplicate.+node") + assertThat(e).hasMessageContaining("Multiple proof entries at the same position") .hasMessageContaining(duplicateNode.toString()); } @@ -512,7 +543,7 @@ void twoElementListInvalidProofExtraNodesDuplicateElementSameValue() { InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageFindingMatch("Duplicate.+node") + assertThat(e).hasMessageFindingMatch("Duplicate.+entries") .hasMessageContaining(duplicateEntry.toString()); } @@ -620,7 +651,7 @@ void fourElementListInvalidProofMissingHashNode1() { InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageContaining("Missing proof entry at position (1, 1)"); + assertThat(e).hasMessageContaining("Missing proof entry at index (1)"); } @Test @@ -642,7 +673,7 @@ void fourElementListInvalidProofMissingHashNode2() { InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageContaining("Missing proof entry at position (1, 1)"); + assertThat(e).hasMessageContaining("Missing proof entry at index (0)"); } @Test @@ -669,9 +700,12 @@ void fourElementListInvalidProofExtraNodesRedundantTotally() { InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageContaining("Redundant proof entries") - .hasMessageContaining(r1.toString()) - .hasMessageStartingWith(r2.toString()); + assertThat(e).hasMessageContaining("Redundant proof entry") + .hasMessageContaining("not needed for verification"); + assertThat(e).satisfiesAnyOf( + e1 -> assertThat(e1).hasMessageContaining(r1.toString()), + e1 -> assertThat(e1).hasMessageContaining(r2.toString()) + ); } @ParameterizedTest @@ -695,8 +729,7 @@ void fiveElementListInvalidProofExtraNodesAtInvalidIndexes( InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageContaining("Invalid node") - .hasMessageFindingMatch("(?i)at index .+ exceeding the max"); + assertThat(e.getMessage()).containsIgnoringCase("entry at invalid index"); } private static Collection> @@ -781,7 +814,8 @@ void eightElementListInvalidProofExtraNodesRedundantTotally( InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageContaining("Redundant proof entries"); + assertThat(e).hasMessageContaining("Redundant proof entry") + .hasMessageContaining("not needed for verification"); } static Collection> @@ -827,9 +861,8 @@ private static List createElements(int size) { */ private static List createElementEntries(List list) { int size = list.size(); - List entries = new ArrayList<>(size); return IntStream.range(0, size) - .mapToObj(i -> elementEntry(0, list.get(i))) + .mapToObj(i -> elementEntry(i, list.get(i))) .collect(toList()); } From b48cd7c268b21c3ba42ed50877230378eb4cefcf Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Thu, 10 Oct 2019 14:21:18 +0300 Subject: [PATCH 09/24] Fix duplicate elements detection --- .../common/proofs/list/FlatListProof.java | 30 ++++++++++++------- .../common/proofs/list/FlatListProofTest.java | 12 ++++---- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java index b80ca84376..17aca2af0f 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java @@ -28,10 +28,13 @@ import com.exonum.binding.common.hash.Hasher; import com.exonum.binding.common.hash.Hashing; import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Maps; +import com.google.common.base.Functions; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; import java.math.BigInteger; import java.math.RoundingMode; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -189,23 +192,28 @@ CheckedListProof verify() { } // Check element entries: have unique indexes that are in range [0; size) - Map elementsByIndex = Maps - .uniqueIndex(elements, ListProofEntry::getIndex); - for (ListProofElementEntry elementEntry : elements) { - long index = elementEntry.getIndex(); + Multimap elementsByIndex = Multimaps + .index(elements, ListProofEntry::getIndex); + for (Map.Entry> e : elementsByIndex.asMap() + .entrySet()) { + Collection elementsAtIndex = e.getValue(); + Long index = e.getKey(); + if (elementsAtIndex.size() != 1) { + throw new InvalidProofException( + String.format("Multiple element entries at the same index (%d): %s", index, + elementsAtIndex)); + } if (index < 0L || size <= index) { throw new InvalidProofException( String.format("Entry at invalid index (%d), must be in range [0; %d): %s", - index, size, elementEntry)); + index, size, elementsAtIndex)); } } // Hash the element entries, and obtain the first level of calculated hashes -// Map calculated = elements.stream() -// .map(FlatListProof::hashLeafNode) -// .collect(toMap(ListProofEntry::getIndex, Functions.identity())); - Map calculated = Maps - .transformValues(elementsByIndex, FlatListProof::hashLeafNode); + Map calculated = elements.stream() + .map(FlatListProof::hashLeafNode) + .collect(toMap(ListProofEntry::getIndex, Functions.identity())); // For each tree level, starting at the bottom for (int height = 0; height < treeHeight; height++) { diff --git a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java index dc47a61389..f8db2a4c63 100644 --- a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java +++ b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java @@ -543,8 +543,9 @@ void twoElementListInvalidProofExtraNodesDuplicateElementSameValue() { InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageFindingMatch("Duplicate.+entries") - .hasMessageContaining(duplicateEntry.toString()); + assertThat(e.getMessage()) + .containsIgnoringCase("multiple element entries at the same index (0)") + .contains(duplicateEntry.toString()); } @Test @@ -564,9 +565,10 @@ void twoElementListInvalidProofExtraNodesDuplicateElementDiffValue() { InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); - assertThat(e).hasMessageFindingMatch("Duplicate.+node") - .hasMessageContaining(entry.toString()) - .hasMessageContaining(conflictingEntry.toString()); + assertThat(e.getMessage()) + .containsIgnoringCase("multiple element entries at the same index (0)") + .contains(entry.toString()) + .contains(conflictingEntry.toString()); } @Test From a169dc6abe25b679c9e68681076456d03cce2a42 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Thu, 10 Oct 2019 17:18:36 +0300 Subject: [PATCH 10/24] Add a test for invalid hash entry at the top. --- .../com/exonum/binding/common/proofs/list/FlatListProofTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java index f8db2a4c63..18c5702eec 100644 --- a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java +++ b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java @@ -422,6 +422,7 @@ void twoElementListInvalidProofNodesAboveMaxHeight(List in static List> twoElementListInvalidProofNodesAboveMaxHeightSource() { return asList( // | index | height | + singletonList(hashedEntry(0, 1)), singletonList(hashedEntry(0, 2)), singletonList(hashedEntry(1, 2)), singletonList(hashedEntry(0, 3)), From ab577b963780ed2ef5ccfa4e560ade1a5433ef62 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Thu, 10 Oct 2019 17:31:24 +0300 Subject: [PATCH 11/24] Improve test for redundant hash nodes overriding the calculated one. --- .../common/proofs/list/FlatListProofTest.java | 73 ++++++++++++++----- 1 file changed, 53 insertions(+), 20 deletions(-) diff --git a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java index 18c5702eec..5ca290057d 100644 --- a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java +++ b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java @@ -476,26 +476,15 @@ void twoElementListInvalidProofExtraNodesAtInvalidIndexes( ); } - @ParameterizedTest - @CsvSource({ - /* - H - 1 o - / / \ - 0 e x h - */ - "0, 0", - - /* - H - 1 x o - / / \ - 0 e h - */ - "0, 1" - }) - void twoElementListInvalidProofExtraNodesRedundantCalculated(long index, int height) { - ListProofHashedEntry redundantNode = hashedEntry(index, height); + @Test + void twoElementListInvalidProofExtraNodesRedundantCalculated() { + /* + H + 1 o + / / \ + 0 e x h + */ + ListProofHashedEntry redundantNode = hashedEntry(0L, 0); FlatListProof proof = twoElementListAt0() .addProofEntry(redundantNode) .build(); @@ -711,6 +700,46 @@ void fourElementListInvalidProofExtraNodesRedundantTotally() { ); } + @ParameterizedTest + @CsvSource({ + // Overriding the calculated hashes of element entries (h=0) + "0, 0", + "1, 0", + "4, 0", + // Overriding the calculated hashes at intermediate levels + "0, 1", + "2, 1", + "0, 2", + "1, 2", + }) + void fiveElementListInvalidProofExtraNodesRedundantCalculated(long index, int height) { + /* + Target proof tree of size 5 with two ranges [0, 1], [4]: + H + 3 o + / \ + 2 o o + / \ / + 1 o h o + /\ / + 0 e e e + */ + ListProofHashedEntry redundantNode = hashedEntry(index, height); + FlatListProof proof = new FlatListProofBuilder() + .size(5L) + .addElements(ELEMENT_ENTRIES.get(0), ELEMENT_ENTRIES.get(1), ELEMENT_ENTRIES.get(4)) + .addProofEntry(hashedEntry(1L, 1)) + .addProofEntry(redundantNode) + .build(); + + InvalidProofException e = assertThrows(InvalidProofException.class, + proof::verify); + + assertThat(e.getMessage()).containsIgnoringCase("Redundant proof entry") + .contains(redundantNode.toString()) + .contains(String.format("with the same index (%d) as the calculated node", index)); + } + @ParameterizedTest @MethodSource("fiveElementListInvalidProofExtraNodesAtInvalidIndexesSource") void fiveElementListInvalidProofExtraNodesAtInvalidIndexes( @@ -900,6 +929,10 @@ FlatListProofBuilder addElement(ListProofElementEntry elementEntry) { return this; } + FlatListProofBuilder addElements(ListProofElementEntry... elementEntries) { + return addElements(asList(elementEntries)); + } + FlatListProofBuilder addElements(List elementEntries) { elements.addAll(elementEntries); return this; From 06abb62453aec62cbcf7d71f535d8fad388ce3ce Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Thu, 10 Oct 2019 17:34:04 +0300 Subject: [PATCH 12/24] Migrate off multimap --- .../common/proofs/list/FlatListProof.java | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java index 17aca2af0f..6f1cba5a56 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java @@ -29,12 +29,9 @@ import com.exonum.binding.common.hash.Hashing; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Functions; -import com.google.common.collect.Multimap; -import com.google.common.collect.Multimaps; import java.math.BigInteger; import java.math.RoundingMode; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -192,21 +189,19 @@ CheckedListProof verify() { } // Check element entries: have unique indexes that are in range [0; size) - Multimap elementsByIndex = Multimaps - .index(elements, ListProofEntry::getIndex); - for (Map.Entry> e : elementsByIndex.asMap() - .entrySet()) { - Collection elementsAtIndex = e.getValue(); - Long index = e.getKey(); - if (elementsAtIndex.size() != 1) { - throw new InvalidProofException( - String.format("Multiple element entries at the same index (%d): %s", index, - elementsAtIndex)); - } + Map elementsByIndex = new HashMap<>(elements.size()); + for (ListProofElementEntry e : elements) { + long index = e.getIndex(); if (index < 0L || size <= index) { throw new InvalidProofException( String.format("Entry at invalid index (%d), must be in range [0; %d): %s", - index, size, elementsAtIndex)); + index, size, e)); + } + ListProofElementEntry present = elementsByIndex.putIfAbsent(index, e); + if (present != null) { + throw new InvalidProofException( + String.format("Multiple element entries at the same index (%d): %s and %s", index, + present, e)); } } From 1731e8be8e59b04e64656fb5bdbeec27cd19eb6b Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Thu, 10 Oct 2019 17:38:52 +0300 Subject: [PATCH 13/24] Extract newCheckedProof --- .../binding/common/proofs/list/FlatListProof.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java index 6f1cba5a56..38fecdf0a1 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java @@ -117,8 +117,7 @@ CheckedListProof verify() { throw new InvalidProofException( "Proof for empty list must not have proof entries, but has: " + proof); } - return new CheckedListProofImpl<>(size, EMPTY_LIST_INDEX_HASH, new TreeMap<>(), - ListProofStatus.VALID); + return newCheckedProof(EMPTY_LIST_INDEX_HASH); } else if (elements.isEmpty()) { // Empty range: must have a single root hash node if (proof.size() != 1) { @@ -143,7 +142,7 @@ CheckedListProof verify() { } HashCode rootHash = h.getHash(); HashCode listHash = hashListIndex(rootHash); - return new CheckedListProofImpl<>(size, listHash, new TreeMap<>(), ListProofStatus.VALID); + return newCheckedProof(listHash); } else { // A list of size 1+ @@ -273,8 +272,7 @@ CheckedListProof verify() { HashCode indexHash = hashListIndex(rootHash); // Extract the entries. - // todo: Extract into newCheckedProof(indexHash) — all the other things can be inferred. - return new CheckedListProofImpl<>(size, indexHash, indexElements(), ListProofStatus.VALID); + return newCheckedProof(indexHash); } } @@ -359,6 +357,10 @@ private static Hasher newHasher() { return sha256().newHasher(); } + private CheckedListProofImpl newCheckedProof(HashCode indexHash) { + return new CheckedListProofImpl<>(size, indexHash, indexElements(), ListProofStatus.VALID); + } + private NavigableMap indexElements() { return elements.stream() .collect(toMap(ListProofEntry::getIndex, ListProofElementEntry::getElement, From 0bca025a66446df06e6b5e55af3683a284536385 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Thu, 10 Oct 2019 17:44:43 +0300 Subject: [PATCH 14/24] Fix the checkstyle --- .../common/proofs/list/FlatListProof.java | 13 +++++--- .../proofs/list/InvalidProofException.java | 2 +- .../common/proofs/list/FlatListProofTest.java | 32 +++++++++---------- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java index 38fecdf0a1..9b667f1367 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java @@ -149,7 +149,8 @@ CheckedListProof verify() { // Calculate the expected tree height int treeHeight = calcTreeHeight(size); - // Index proof entries and verify their local correctness: no out-of-range nodes; no duplicates. + // Index proof entries and verify their local correctness: no out-of-range nodes; + // no duplicates. List> proofByHeight = new ArrayList<>(treeHeight); for (int i = 0; i < treeHeight; i++) { // For single-element proofs, only a single proof node is expected on each level. @@ -245,8 +246,8 @@ CheckedListProof verify() { } } - // Merge calculated on the previous level hashes with the proof entries of the current level, - // ordered by indexes. The sibling nodes will go in adjacent pairs. + // Merge calculated on the previous level hashes with the proof entries of the current + // level, ordered by indexes. The sibling nodes will go in adjacent pairs. SortedMap merged = new TreeMap<>(); merged.putAll(calculated); merged.putAll(proofAtLevel); @@ -336,7 +337,7 @@ private static ListProofHashedEntry hashBranchNode(ListProofHashedEntry leftChil .putByte(LIST_BRANCH_PREFIX) .putObject(leftChild.getHash(), hashCodeFunnel()); if (rightChild != null) { - hasher.putObject(rightChild.getHash(), hashCodeFunnel()); + hasher.putObject(rightChild.getHash(), hashCodeFunnel()); } return ListProofHashedEntry.newInstance(index, height, hasher.hash()); } @@ -368,6 +369,8 @@ private NavigableMap indexElements() { } private static BinaryOperator throwingMerger() { - return (u, v) -> { throw new IllegalArgumentException("Duplicate values with the same key"); }; + return (u, v) -> { + throw new IllegalArgumentException("Duplicate values with the same key"); + }; } } diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/InvalidProofException.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/InvalidProofException.java index 675b5dd1ef..4186ae0694 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/InvalidProofException.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/InvalidProofException.java @@ -17,7 +17,7 @@ package com.exonum.binding.common.proofs.list; /** - * todo: + * Indicates that the corresponding list proof has invalid structure and must be rejected. */ public class InvalidProofException extends RuntimeException { diff --git a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java index 5ca290057d..e4e85b7fbd 100644 --- a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java +++ b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java @@ -455,7 +455,7 @@ void twoElementListInvalidProofExtraNodesAtInvalidIndexes( } private static Collection> - twoElementListInvalidProofExtraNodesAtInvalidIndexesSource() { + twoElementListInvalidProofExtraNodesAtInvalidIndexesSource() { /* Target proof tree: H @@ -499,12 +499,12 @@ void twoElementListInvalidProofExtraNodesRedundantCalculated() { @Test void twoElementListInvalidProofExtraNodesDuplicateHashed() { - /* - H - 1 o - / \ \ - 0 e h h - */ + /* + H + 1 o + / \ \ + 0 e h h + */ ListProofHashedEntry duplicateNode = hashedEntry(1, 0); FlatListProof proof = twoElementListAt0() .addProofEntry(duplicateNode) @@ -519,12 +519,12 @@ void twoElementListInvalidProofExtraNodesDuplicateHashed() { @Test void twoElementListInvalidProofExtraNodesDuplicateElementSameValue() { - /* - H - 1 o - / / \ - 0 e e h - */ + /* + H + 1 o + / / \ + 0 e e h + */ ListProofElementEntry duplicateEntry = ELEMENT_ENTRIES.get(0); FlatListProof proof = twoElementListAt0() .addElement(duplicateEntry) @@ -573,9 +573,9 @@ void twoElementListValidProofAt0() { * Returns a builder with a structurally valid flat proof for a tree of size 2 * and an element at index 0. */ - private FlatListProofBuilder twoElementListAt0() {/* + private FlatListProofBuilder twoElementListAt0() { + /* H - 1 o / \ 0 e h @@ -765,7 +765,7 @@ void fiveElementListInvalidProofExtraNodesAtInvalidIndexes( } private static Collection> - fiveElementListInvalidProofExtraNodesAtInvalidIndexesSource() { + fiveElementListInvalidProofExtraNodesAtInvalidIndexesSource() { /* Target proof tree: H From f1dc209b91b26ec0834d830a03da3bc9ca835ab2 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Thu, 10 Oct 2019 19:04:28 +0300 Subject: [PATCH 15/24] Refactor the verification code, improve documentation --- .../common/proofs/list/FlatListProof.java | 418 ++++++++++-------- 1 file changed, 239 insertions(+), 179 deletions(-) diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java index 9b667f1367..723c45a074 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java @@ -18,7 +18,6 @@ import static com.exonum.binding.common.hash.Funnels.hashCodeFunnel; import static com.exonum.binding.common.hash.Hashing.sha256; -import static com.exonum.binding.common.proofs.list.ListProofEntry.MAX_INDEX; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.math.BigIntegerMath.log2; @@ -47,18 +46,6 @@ * of a certain size. */ class FlatListProof { - - @VisibleForTesting - static final byte BLOB_PREFIX = 0x00; - @VisibleForTesting - static final byte LIST_BRANCH_PREFIX = 0x01; - @VisibleForTesting - static final byte LIST_ROOT_PREFIX = 0x02; - - private static final long MAX_SIZE = MAX_INDEX + 1; - - private static HashCode EMPTY_LIST_INDEX_HASH = hashListIndex(0L, - HashCode.fromBytes(new byte[Hashing.DEFAULT_HASH_SIZE_BYTES])); /* Proof lists are represented as BSTs, where all leaf elements are at the same height. Here is a tree for a three-element proof list: @@ -82,13 +69,26 @@ class FlatListProof { 1 o h / \ 0 h e - _________________ + ________________ Legend: e — element proof entry (ListProofElementEntry) h — hashed proof entry (ListProofHashedEntry) o — 'virtual' node — not present in the proof, inferred during verification. Shown mostly to communicate the tree structure of the proof. - */ + */ + + private static final long MAX_SIZE = ListProofEntry.MAX_INDEX + 1; + + private static HashCode EMPTY_LIST_INDEX_HASH = hashListIndex(0L, + HashCode.fromBytes(new byte[Hashing.DEFAULT_HASH_SIZE_BYTES])); + + @VisibleForTesting + static final byte BLOB_PREFIX = 0x00; + @VisibleForTesting + static final byte LIST_BRANCH_PREFIX = 0x01; + @VisibleForTesting + static final byte LIST_ROOT_PREFIX = 0x02; + private final List elements; private final List proof; private final long size; @@ -106,185 +106,244 @@ CheckedListProof verify() { "Invalid size (%s), must be in range [0; 2^56]", size); // Handle special cases - // Empty list if (size == 0) { - // Check there are no elements or proof entries - if (!elements.isEmpty()) { - throw new InvalidProofException("Proof for empty list must not have elements, but has: " - + elements); - } - if (!proof.isEmpty()) { - throw new InvalidProofException( - "Proof for empty list must not have proof entries, but has: " + proof); - } - return newCheckedProof(EMPTY_LIST_INDEX_HASH); + // Empty list + return verifyEmptyListProof(); } else if (elements.isEmpty()) { - // Empty range: must have a single root hash node - if (proof.size() != 1) { - throw new InvalidProofException(String.format( - "Proof for an empty range must have a single proof node, but has %d: %s", - proof.size(), proof)); - } - ListProofHashedEntry h = proof.get(0); - int treeHeight = calcTreeHeight(size); + // Empty range proof + return verifyEmptyRangeProof(); + } else { + // 1+ element proof + return verifyNonEmptyListProof(); + } + } + + private CheckedListProof verifyEmptyListProof() { + // Check there are no elements or proof entries + if (!elements.isEmpty()) { + throw new InvalidProofException("Proof for empty list must not have elements, but has: " + + elements); + } + if (!proof.isEmpty()) { + throw new InvalidProofException( + "Proof for empty list must not have proof entries, but has: " + proof); + } + return newCheckedProof(EMPTY_LIST_INDEX_HASH); + } + + private CheckedListProof verifyEmptyRangeProof() { + // Empty range: must have a single root hash node + if (proof.size() != 1) { + throw new InvalidProofException(String.format( + "Proof for an empty range must have a single proof node, but has %d: %s", + proof.size(), proof)); + } + ListProofHashedEntry rootHashEntry = proof.get(0); + int treeHeight = calcTreeHeight(size); + // Check height + if (rootHashEntry.getHeight() != treeHeight) { + throw new InvalidProofException( + String.format("Proof node for an empty range at invalid height (%d)," + + "must be at height (%d) for a list of size %d: %s", rootHashEntry.getHeight(), + treeHeight, size, rootHashEntry)); + } + // Check index + if (rootHashEntry.getIndex() != 0L) { + throw new InvalidProofException( + String.format("Proof node for an empty range at invalid index (%d)," + + "must be always at index 0: %s", rootHashEntry.getIndex(), rootHashEntry)); + } + HashCode rootHash = rootHashEntry.getHash(); + HashCode listHash = hashListIndex(rootHash); + return newCheckedProof(listHash); + } + + private CheckedListProof verifyNonEmptyListProof() { + // Calculate the expected tree height + int treeHeight = calcTreeHeight(size); + + // Index proof entries by height and verify their local correctness: no out-of-range nodes; + // no duplicates. + List> proofByHeight = indexHashedEntriesByHeight(treeHeight); + + // Check element entries: have unique indexes that are in range [0; size) + checkElementEntries(); + + // Compute the Merkle root hash + HashCode rootHash = computeRootHash(proofByHeight, treeHeight); + + // Compute the list object hash + HashCode indexHash = hashListIndex(rootHash); + return newCheckedProof(indexHash); + } + + @VisibleForTesting + static int calcTreeHeight(long size) { + return (size == 0L) ? 0 + : log2(BigInteger.valueOf(size), RoundingMode.CEILING); + } + + /** + * Indexes proof entries by their height, also verifying their local correctness: + * no out-of-range nodes; no duplicates. + * + * @param treeHeight the height of the proof list tree + * @return a list of proof entries at each height from 0 to treeHeight; + * entries at each level are indexed by their index + */ + private List> indexHashedEntriesByHeight(int treeHeight) { + List> proofByHeight = new ArrayList<>(treeHeight); + for (int i = 0; i < treeHeight; i++) { + // For single-element proofs, only a single proof node is expected on each level. + // For contiguous range proofs, up to two proof nodes are expected on any level. + // Multiple-range proofs might have up to 'elements.size' proof nodes on the lowest level, + // but Exonum does not currently produce such. + int initialCapacity = (elements.size() <= 1) ? 1 : 2; + Map proofAtHeight = new HashMap<>(initialCapacity); + proofByHeight.add(proofAtHeight); + } + + for (ListProofHashedEntry hashedEntry : proof) { // Check height - if (h.getHeight() != treeHeight) { + int height = hashedEntry.getHeight(); + if (height < 0 || treeHeight <= height) { throw new InvalidProofException( - String.format("Proof node for an empty range at invalid height (%d)," - + "must be at height (%d) for a list of size %d: %s", h.getHeight(), - treeHeight, size, h)); + String.format("Proof entry at invalid height (%d), must be in range [0; %d): %s", + height, treeHeight, hashedEntry)); } // Check index - if (h.getIndex() != 0L) { + long levelSize = levelSizeAt(height); + long index = hashedEntry.getIndex(); + if (index < 0L || levelSize <= index) { + throw new InvalidProofException(String + .format( + "Proof entry at invalid index (%d); it must be in range [0; %d) at height %d: %s", + index, levelSize, height, hashedEntry)); + } + // Add the entry at the height, checking for duplicates + Map proofsAtHeight = proofByHeight.get(height); + ListProofHashedEntry present = proofsAtHeight.putIfAbsent(index, hashedEntry); + if (present != null) { throw new InvalidProofException( - String.format("Proof node for an empty range at invalid index (%d)," - + "must be always at index 0: %s", h.getIndex(), h)); + String.format("Multiple proof entries at the same position: %s and %s", + present, hashedEntry)); } - HashCode rootHash = h.getHash(); - HashCode listHash = hashListIndex(rootHash); - return newCheckedProof(listHash); - } else { - // A list of size 1+ - - // Calculate the expected tree height - int treeHeight = calcTreeHeight(size); - - // Index proof entries and verify their local correctness: no out-of-range nodes; - // no duplicates. - List> proofByHeight = new ArrayList<>(treeHeight); - for (int i = 0; i < treeHeight; i++) { - // For single-element proofs, only a single proof node is expected on each level. - // For contiguous range proofs, up to two proof nodes are expected on any level. - // Multiple-range proofs might have up to 'elements.size' proof nodes on the lowest level, - // but Exonum does not currently produce such. - int initialCapacity = (elements.size() <= 1) ? 1 : 2; - Map proofAtHeight = new HashMap<>(initialCapacity); - proofByHeight.add(proofAtHeight); + } + return proofByHeight; + } + + /** + * Checks the element entries: no out-of-range elements; no duplicate indexes. + */ + private void checkElementEntries() { + Map elementsByIndex = new HashMap<>(elements.size()); + for (ListProofElementEntry e : elements) { + long index = e.getIndex(); + if (index < 0L || size <= index) { + throw new InvalidProofException( + String.format("Entry at invalid index (%d), must be in range [0; %d): %s", + index, size, e)); } - for (ListProofHashedEntry hashedEntry : proof) { - // Check height - int height = hashedEntry.getHeight(); - if (height < 0 || treeHeight <= height) { - throw new InvalidProofException( - String.format("Proof entry at invalid height (%d), must be in range [0; %d): %s", - height, treeHeight, hashedEntry)); - } - // Check index - long levelSize = levelSizeAt(height); - long index = hashedEntry.getIndex(); - if (index < 0L || levelSize <= index) { - throw new InvalidProofException(String - .format( - "Proof entry at invalid index (%d); it must be in range [0; %d) at height %d: %s", - index, levelSize, height, hashedEntry)); - } - // Add the entry at the height, checking for duplicates - Map proofsAtHeight = proofByHeight.get(height); - ListProofHashedEntry present = proofsAtHeight.putIfAbsent(index, hashedEntry); - if (present != null) { - throw new InvalidProofException( - String.format("Multiple proof entries at the same position: %s and %s", - present, hashedEntry)); - } + ListProofElementEntry present = elementsByIndex.putIfAbsent(index, e); + if (present != null) { + throw new InvalidProofException( + String.format("Multiple element entries at the same index (%d): %s and %s", index, + present, e)); } + } + } - // Check element entries: have unique indexes that are in range [0; size) - Map elementsByIndex = new HashMap<>(elements.size()); - for (ListProofElementEntry e : elements) { - long index = e.getIndex(); - if (index < 0L || size <= index) { - throw new InvalidProofException( - String.format("Entry at invalid index (%d), must be in range [0; %d): %s", - index, size, e)); - } - ListProofElementEntry present = elementsByIndex.putIfAbsent(index, e); - if (present != null) { - throw new InvalidProofException( - String.format("Multiple element entries at the same index (%d): %s and %s", index, - present, e)); - } - } + /** + * Computes the root hash of the proof list tree, also verifying the correctness of + * proof entries with regard to the calculated ones. + * + * @param proofByHeight proof entries indexed by their height + * @param treeHeight the height of the tree + */ + private HashCode computeRootHash(List> proofByHeight, + int treeHeight) { + // Hash the element entries, and obtain the first level of calculated hashes + Map calculated = hashElements(); + + // For each tree level, starting at the bottom + for (int height = 0; height < treeHeight; height++) { + // Take the proof (hashed) entries at this height + Map proofAtLevel = proofByHeight.get(height); + // Merge the calculated entries with the proof entries at this height + calculated = reduce(calculated, proofAtLevel); + } - // Hash the element entries, and obtain the first level of calculated hashes - Map calculated = elements.stream() - .map(FlatListProof::hashLeafNode) - .collect(toMap(ListProofEntry::getIndex, Functions.identity())); - - // For each tree level, starting at the bottom - for (int height = 0; height < treeHeight; height++) { - // Verify nodes: - // - For an inferred node n there is a sibling either in the inferred - // nodes or in hash nodes; or it is the last node in a odd-sized level. - Map proofAtLevel = proofByHeight.get(height); - for (ListProofHashedEntry inferredNode : calculated.values()) { - if (isLastOnOddLevel(inferredNode)) { - // The last node on an odd-sized level does not have a sibling. - continue; - } - long index = inferredNode.getIndex(); - long siblingIndex = getSiblingIndex(index); - if (!(calculated.containsKey(siblingIndex) || proofAtLevel.containsKey(siblingIndex))) { - throw new InvalidProofException( - String.format("Missing proof entry at index (%d) for the calculated one: %s", - siblingIndex, inferredNode)); - } - } - // Verify proof nodes: - for (ListProofHashedEntry proofNode : proofAtLevel.values()) { - long index = proofNode.getIndex(); - // No hash nodes overriding the inferred nodes (i.e., have same index) - if (calculated.containsKey(index)) { - throw new InvalidProofException( - String.format("Redundant proof entry (%s) with the same index (%d) as " - + "the calculated node (%s)", proofNode, index, calculated.get(index))); - } - // No redundant hash nodes that have no siblings in the inferred nodes. - long siblingIndex = getSiblingIndex(index); - if (!calculated.containsKey(siblingIndex)) { - throw new InvalidProofException( - String.format("Redundant proof entry (%s) not needed for verification", proofNode)); - } - } - - // Merge calculated on the previous level hashes with the proof entries of the current - // level, ordered by indexes. The sibling nodes will go in adjacent pairs. - SortedMap merged = new TreeMap<>(); - merged.putAll(calculated); - merged.putAll(proofAtLevel); - - // Reduce the adjacent nodes to produce the nodes on the upper level - Iterator mergedIter = merged.values().iterator(); - Map nextLevel = new HashMap<>((merged.size() + 1) / 2); - while (mergedIter.hasNext()) { - ListProofHashedEntry left = mergedIter.next(); - ListProofHashedEntry right = null; - if (mergedIter.hasNext()) { - right = mergedIter.next(); - } - ListProofHashedEntry parent = hashBranchNode(left, right); - nextLevel.put(parent.getIndex(), parent); - } - // Proceed to the next level - calculated = nextLevel; - } + // Take the root hash and calculate the index hash. + return calculated.get(0L).getHash(); + } - // Take the root hash and calculate the index hash. - HashCode rootHash = calculated.get(0L).getHash(); - HashCode indexHash = hashListIndex(rootHash); + private Map hashElements() { + return elements.stream() + .map(FlatListProof::hashLeafNode) + .collect(toMap(ListProofEntry::getIndex, Functions.identity())); + } - // Extract the entries. - return newCheckedProof(indexHash); + /** + * Combines the nodes at height h to produce their parent nodes at height h + 1. + * + * @param calculated the nodes inferred from the elements and the proof nodes from levels [0, h-1] + * @param proofAtLevel the proof nodes at height h + * @return the calculated nodes at height h + 1 + */ + private Map reduce(Map calculated, + Map proofAtLevel) { + // Verify nodes: + // - For an inferred node n there is a sibling either in the inferred + // nodes or in hash nodes; or it is the last node in an odd-sized level. + for (ListProofHashedEntry inferredNode : calculated.values()) { + if (isLastOnOddLevel(inferredNode)) { + // The last node on an odd-sized level does not have a sibling. + continue; + } + long index = inferredNode.getIndex(); + long siblingIndex = getSiblingIndex(index); + if (!(calculated.containsKey(siblingIndex) || proofAtLevel.containsKey(siblingIndex))) { + throw new InvalidProofException( + String.format("Missing proof entry at index (%d) for the calculated one: %s", + siblingIndex, inferredNode)); + } + } + // Verify proof nodes: + for (ListProofHashedEntry proofNode : proofAtLevel.values()) { + long index = proofNode.getIndex(); + // No hash nodes overriding the inferred nodes (i.e., have same index) + if (calculated.containsKey(index)) { + throw new InvalidProofException( + String.format("Redundant proof entry (%s) with the same index (%d) as " + + "the calculated node (%s)", proofNode, index, calculated.get(index))); + } + // No redundant hash nodes that have no siblings in the inferred nodes. + long siblingIndex = getSiblingIndex(index); + if (!calculated.containsKey(siblingIndex)) { + throw new InvalidProofException( + String.format("Redundant proof entry (%s) not needed for verification", proofNode)); + } } - } - @VisibleForTesting - static int calcTreeHeight(long size) { - // Beware, toIntExact(round(ceil(log(size) / log(2.)))) won't work for large sizes because - // `log(size) / log(2.)` might produce a number slightly larger than the actual, with - // subsequent `ceil` making it off by +1. - // For instance, ceil(log(2^55 as long) / log(2.)) -> 56. - return (size == 0L) ? 0 - : log2(BigInteger.valueOf(size), RoundingMode.CEILING); + // Merge calculated on the previous level hashes with the proof entries of the current + // level, ordered by indexes. The sibling nodes will go in adjacent pairs. + SortedMap merged = new TreeMap<>(); + merged.putAll(calculated); + merged.putAll(proofAtLevel); + + // Reduce the adjacent nodes to produce the calculated nodes on the upper level + Iterator mergedIter = merged.values().iterator(); + Map nextLevel = new HashMap<>((merged.size() + 1) / 2); + while (mergedIter.hasNext()) { + ListProofHashedEntry left = mergedIter.next(); + ListProofHashedEntry right = null; + if (mergedIter.hasNext()) { + right = mergedIter.next(); + } + ListProofHashedEntry parent = hashBranchNode(left, right); + nextLevel.put(parent.getIndex(), parent); + } + return nextLevel; } private long levelSizeAt(int height) { @@ -359,7 +418,8 @@ private static Hasher newHasher() { } private CheckedListProofImpl newCheckedProof(HashCode indexHash) { - return new CheckedListProofImpl<>(size, indexHash, indexElements(), ListProofStatus.VALID); + NavigableMap elements = indexElements(); + return new CheckedListProofImpl<>(size, indexHash, elements, ListProofStatus.VALID); } private NavigableMap indexElements() { From 0f6e9b69c18743d5ad3b7679a27789158b15390b Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Thu, 10 Oct 2019 19:27:20 +0300 Subject: [PATCH 16/24] Link the article --- .../exonum/binding/common/proofs/list/FlatListProof.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java index 723c45a074..7a72f94782 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java @@ -57,9 +57,9 @@ class FlatListProof { / \ / 0 e e e - Proofs for elements of proof lists include the requested elements; and hashes of all - the sub-trees that are adjacent to the paths from the root node to the leaf nodes containing - the elements. + Proofs for elements of proof lists include the list size; the requested elements; + and hashes of all sub-trees that are adjacent to the paths from the root node to the leaf nodes + containing the elements. Here is a proof for the element at index 1 from the list shown above: @@ -75,6 +75,8 @@ class FlatListProof { h — hashed proof entry (ListProofHashedEntry) o — 'virtual' node — not present in the proof, inferred during verification. Shown mostly to communicate the tree structure of the proof. + + See also: https://wiki.bf.local/display/EXN/Flat+list+proofs */ private static final long MAX_SIZE = ListProofEntry.MAX_INDEX + 1; From 7d8e09647d5db99cfc5ccf67f3f8b9fd73276dc3 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Thu, 10 Oct 2019 20:20:47 +0300 Subject: [PATCH 17/24] Disable the new IT --- .../storage/indices/ProofListIndexProxyIntegrationTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListIndexProxyIntegrationTest.java b/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListIndexProxyIntegrationTest.java index f7c9b7a0dd..3e8eaf6289 100644 --- a/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListIndexProxyIntegrationTest.java +++ b/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListIndexProxyIntegrationTest.java @@ -34,6 +34,7 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -193,6 +194,8 @@ void getRangeProofMultipleItemList_2ndHalf() { @ParameterizedTest @ValueSource(ints = {1, 2, 3, 4}) + @Disabled("ECR-3673: empty ranges are not supported with the current tree format; " + + "need a flat one") void getRangeProofMultipleItemList_EmptyRange(int size) { runTestWithView(database::createFork, (list) -> { List values = TestStorageItems.values.subList(0, size); From 978230edbfcea971ff5ac28350328d9bfa8bc3fe Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Thu, 10 Oct 2019 20:39:35 +0300 Subject: [PATCH 18/24] Add a test for size checks --- .../binding/common/proofs/list/FlatListProof.java | 6 ++++-- .../common/proofs/list/FlatListProofTest.java | 12 ++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java index 7a72f94782..d25ad6f5fd 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java @@ -104,8 +104,10 @@ class FlatListProof { CheckedListProof verify() { // Check the size - checkArgument(0 <= size && size <= MAX_SIZE, - "Invalid size (%s), must be in range [0; 2^56]", size); + if (size < 0 || MAX_SIZE < size) { + throw new InvalidProofException(String.format("Invalid size (%s), must be in range [0; 2^56]", + size)); + } // Handle special cases if (size == 0) { diff --git a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java index e4e85b7fbd..e24048d15c 100644 --- a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java +++ b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java @@ -100,6 +100,18 @@ private static List calcTreeHeightSource() { ); } + @ParameterizedTest + @ValueSource(longs = {Long.MIN_VALUE,-1L, /* Max size + 1: */ 0x100_0000_0000_0001L, + Long.MAX_VALUE}) + void invalidSizeListInvalidProof(long size) { + FlatListProof proof = new FlatListProof(emptyList(), emptyList(), size); + + InvalidProofException e = assertThrows(InvalidProofException.class, proof::verify); + + assertThat(e.getMessage()).containsIgnoringCase("Invalid size") + .contains(Long.toString(size)); + } + @Test void emptyListValidProof() { FlatListProof proof = new FlatListProof(emptyList(), emptyList(), 0L); From 3a461e16ac0ccffa25ac8979cdf0ef659131ae2b Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Thu, 10 Oct 2019 20:41:15 +0300 Subject: [PATCH 19/24] Fix final --- .../com/exonum/binding/common/proofs/list/FlatListProof.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java index d25ad6f5fd..2531f0a1cd 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java @@ -81,7 +81,7 @@ class FlatListProof { private static final long MAX_SIZE = ListProofEntry.MAX_INDEX + 1; - private static HashCode EMPTY_LIST_INDEX_HASH = hashListIndex(0L, + private static final HashCode EMPTY_LIST_INDEX_HASH = hashListIndex(0L, HashCode.fromBytes(new byte[Hashing.DEFAULT_HASH_SIZE_BYTES])); @VisibleForTesting From e9ef8ab341ebdcf8c777f73662c8890483c98a36 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Thu, 10 Oct 2019 22:49:26 +0300 Subject: [PATCH 20/24] Fix the HashMap with expected size instantiation: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A HashMap constructor accepts the initial capacity of the internal table of hash buckets, not the expected number of elements (as ArrayList constructor), hence the code used to down-size that table (given the default 0.75 load factor). The benchmark with GC profiler confirms the fix: Profile 1, no fix: Benchmark (height) Mode Cnt Score Error Units FlatListProofBenchmark.verify 10 thrpt 10 85,289 ± 0,920 ops/ms FlatListProofBenchmark.verify:·gc.alloc.rate 10 thrpt 10 931,355 ± 10,365 MB/sec FlatListProofBenchmark.verify:·gc.alloc.rate.norm 10 thrpt 10 14320,003 ± 0,001 B/op FlatListProofBenchmark.verify:·gc.count 10 thrpt 10 292,000 counts FlatListProofBenchmark.verify:·gc.time 10 thrpt 10 159,000 ms Profile 2, with exp. size: Benchmark (height) Mode Cnt Score Error Units FlatListProofBenchmark.verify 10 thrpt 10 87,436 ± 0,961 ops/ms FlatListProofBenchmark.verify:·gc.alloc.rate 10 thrpt 10 921,130 ± 10,306 MB/sec FlatListProofBenchmark.verify:·gc.alloc.rate.norm 10 thrpt 10 13816,003 ± 0,001 B/op FlatListProofBenchmark.verify:·gc.count 10 thrpt 10 290,000 counts FlatListProofBenchmark.verify:·gc.time 10 thrpt 10 157,000 ms --- .../exonum/binding/common/proofs/list/FlatListProof.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java index 2531f0a1cd..eb0fbc8ecc 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java @@ -20,6 +20,7 @@ import static com.exonum.binding.common.hash.Hashing.sha256; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.Maps.newHashMapWithExpectedSize; import static com.google.common.math.BigIntegerMath.log2; import static java.util.stream.Collectors.toMap; @@ -203,7 +204,7 @@ private List> indexHashedEntriesByHeight(int tre // Multiple-range proofs might have up to 'elements.size' proof nodes on the lowest level, // but Exonum does not currently produce such. int initialCapacity = (elements.size() <= 1) ? 1 : 2; - Map proofAtHeight = new HashMap<>(initialCapacity); + Map proofAtHeight = newHashMapWithExpectedSize(initialCapacity); proofByHeight.add(proofAtHeight); } @@ -240,7 +241,7 @@ private List> indexHashedEntriesByHeight(int tre * Checks the element entries: no out-of-range elements; no duplicate indexes. */ private void checkElementEntries() { - Map elementsByIndex = new HashMap<>(elements.size()); + Map elementsByIndex = newHashMapWithExpectedSize(elements.size()); for (ListProofElementEntry e : elements) { long index = e.getIndex(); if (index < 0L || size <= index) { @@ -337,7 +338,7 @@ private Map reduce(Map c // Reduce the adjacent nodes to produce the calculated nodes on the upper level Iterator mergedIter = merged.values().iterator(); - Map nextLevel = new HashMap<>((merged.size() + 1) / 2); + Map nextLevel = newHashMapWithExpectedSize((merged.size() + 1) / 2); while (mergedIter.hasNext()) { ListProofHashedEntry left = mergedIter.next(); ListProofHashedEntry right = null; From 1228987ff1dfe6ebdc38cff7879e990b64137bb4 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Fri, 11 Oct 2019 12:28:02 +0300 Subject: [PATCH 21/24] Fix unused import --- .../com/exonum/binding/common/proofs/list/FlatListProof.java | 1 - 1 file changed, 1 deletion(-) diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java index eb0fbc8ecc..92d6d8bc9c 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java @@ -32,7 +32,6 @@ import java.math.BigInteger; import java.math.RoundingMode; import java.util.ArrayList; -import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; From c023f4ce25725a66e7b5fd5c182bd835cce8f536 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Fri, 11 Oct 2019 21:58:37 +0300 Subject: [PATCH 22/24] Resolve the todo --- .../com/exonum/binding/common/proofs/list/FlatListProof.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java index 92d6d8bc9c..19b22e7762 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/FlatListProof.java @@ -351,8 +351,7 @@ private Map reduce(Map c } private long levelSizeAt(int height) { - // todo: Is there a one-liner for ceil(ceil(... ceil(size / 2) / 2 ... / 2)))? - // todo: consider memoizing if no efficient ^ + // Consider memoizing the level sizes to avoid re-calculation checkArgument(height >= 0); long levelSize = size; while (height-- != 0) { From 8ecbff9b4b9f5d1284b48b0338e9167f12b7b668 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Fri, 11 Oct 2019 22:07:51 +0300 Subject: [PATCH 23/24] WIP resolve some todos --- .../java/com/exonum/binding/common/proofs/CheckedProof.java | 2 +- .../exonum/binding/common/proofs/list/ListProofHashedEntry.java | 1 - .../exonum/binding/common/proofs/list/FlatListProofTest.java | 2 -- .../binding/core/storage/indices/ProofListContainsMatcher.java | 2 +- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/CheckedProof.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/CheckedProof.java index 3af9198373..8029bceeb7 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/CheckedProof.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/CheckedProof.java @@ -25,7 +25,7 @@ * If it is valid, the proof contents may be accessed. See {@link CheckedListProof} * and {@link CheckedMapProof} for available contents description. */ -// todo: Why do we need to represent invalid proofs? +// todo: [ECR-2410] Why do we need to represent invalid proofs with checked proof? public interface CheckedProof { /** diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofHashedEntry.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofHashedEntry.java index 43c889f3d3..b8f917c585 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofHashedEntry.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofHashedEntry.java @@ -25,7 +25,6 @@ @AutoValue abstract class ListProofHashedEntry implements ListProofEntry { // todo: do we need the interface (are we going to operate on the entries using the interface?) - // Do we need entries at all, or just hashes? /** * Returns the hash of the sub-tree this entry represents. diff --git a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java index e24048d15c..62feb783ed 100644 --- a/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java +++ b/exonum-java-binding/common/src/test/java/com/exonum/binding/common/proofs/list/FlatListProofTest.java @@ -163,8 +163,6 @@ void singletonListValidProof() { assertThat(checked.getIndexHash()).isEqualTo(expectedListHash); } - // todo: more singletonListInvalid - @ParameterizedTest @CsvSource({ // size of the list, height of the tree (and the root node) diff --git a/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListContainsMatcher.java b/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListContainsMatcher.java index bab5eaa4cf..cf5319c5b6 100644 --- a/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListContainsMatcher.java +++ b/exonum-java-binding/core/src/test/java/com/exonum/binding/core/storage/indices/ProofListContainsMatcher.java @@ -61,7 +61,7 @@ protected boolean matchesSafely(ProofListIndexProxy list) { .collect(toMap(Map.Entry::getKey, e -> e.getValue().toStringUtf8())); return checkedProof.isValid() - // todo: verify the list size also + // todo: [ECR-3673] verify the list size also && elementsMatcher.matches(actualElements) && list.getIndexHash().equals(checkedProof.getIndexHash()); } From cb286cba40a066c8221675e3199b86bfe5c6a3f6 Mon Sep 17 00:00:00 2001 From: Dmitry Timofeev Date: Fri, 11 Oct 2019 22:21:13 +0300 Subject: [PATCH 24/24] Push #getHeight to the hashed entry: Push #getHeight to the hashed entry as the element entries are always at height 0 and their height is neither requested nor included in the proof originally. --- .../binding/common/proofs/list/ListProofElementEntry.java | 6 ------ .../exonum/binding/common/proofs/list/ListProofEntry.java | 7 ------- .../binding/common/proofs/list/ListProofHashedEntry.java | 8 +++++++- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofElementEntry.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofElementEntry.java index a6a5a9ae4a..f62a952e81 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofElementEntry.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofElementEntry.java @@ -24,12 +24,6 @@ @AutoValue abstract class ListProofElementEntry implements ListProofEntry { - @Override - public int getHeight() { - // Elements are always at height 0 - return 0; - } - /** * Returns a value of the element stored at this index in the list. */ diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofEntry.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofEntry.java index 42b956d031..8c462d4780 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofEntry.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofEntry.java @@ -38,13 +38,6 @@ interface ListProofEntry { */ long getIndex(); - /** - * Returns the height of the proof tree node corresponding to this entry. - * The height of leaf nodes is equal to 0; the height of the root, or top node: - * ceil(log2(N)). - */ - int getHeight(); - static void checkIndex(long index) { checkArgument(0 <= index && index <= MAX_INDEX, "Entry index (%s) is out of range [0; 2^56]", index); diff --git a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofHashedEntry.java b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofHashedEntry.java index b8f917c585..2e72bda0a3 100644 --- a/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofHashedEntry.java +++ b/exonum-java-binding/common/src/main/java/com/exonum/binding/common/proofs/list/ListProofHashedEntry.java @@ -24,7 +24,13 @@ */ @AutoValue abstract class ListProofHashedEntry implements ListProofEntry { - // todo: do we need the interface (are we going to operate on the entries using the interface?) + + /** + * Returns the height of the proof tree node corresponding to this entry. + * The height of leaf nodes is equal to 0; the height of the root, or top node: + * ceil(log2(N)). + */ + abstract int getHeight(); /** * Returns the hash of the sub-tree this entry represents.