From 1485d8969cf631842e8426508d495dfb1c42783e Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 26 Nov 2024 16:41:19 +0800 Subject: [PATCH 01/61] feat: pbss implemention --- cmd/geth/dbcmd.go | 31 ++ cmd/geth/main.go | 2 + cmd/geth/usage.go | 2 + cmd/utils/flags.go | 18 + common/types.go | 69 ++++ core/blockchain.go | 101 ++++-- core/blockchain_l2.go | 8 +- core/genesis.go | 57 +++- core/rawdb/accessors_state.go | 127 +++++++ core/rawdb/accessors_trie.go | 66 ++++ core/rawdb/schema.go | 30 ++ core/state/database.go | 30 +- core/state/iterator.go | 2 +- core/state/pruner/zk-pruner.go | 6 +- core/state/state_object.go | 12 +- core/state/state_prove.go | 15 +- core/state/statedb.go | 8 + core/types/block.go | 7 +- eth/backend.go | 7 + eth/ethconfig/config.go | 8 + go.mod | 8 +- go.sum | 16 +- les/server_requests.go | 2 +- light/trie.go | 2 +- params/config.go | 7 + trie/database.go | 269 ++++++++++++++- trie/hbss2pbss.go | 176 ++++++++++ trie/morph_zk_trie.go | 228 +++++++++++++ trie/zk_trie.go | 8 +- trie/zk_trie_proof_test.go | 8 +- trie/zk_trie_test.go | 97 +++++- triedb/hashdb/metrics.go | 43 +++ triedb/hashdb/zk_trie_database.go | 232 +++++++++++++ triedb/pathdb/asyncnodebuffer.go | 357 ++++++++++++++++++++ triedb/pathdb/database.go | 383 +++++++++++++++++++++ triedb/pathdb/difflayer.go | 305 +++++++++++++++++ triedb/pathdb/difflayer_test.go | 170 ++++++++++ triedb/pathdb/disklayer.go | 268 +++++++++++++++ triedb/pathdb/errors.go | 74 +++++ triedb/pathdb/journal.go | 535 ++++++++++++++++++++++++++++++ triedb/pathdb/layertree.go | 301 +++++++++++++++++ triedb/pathdb/metrics.go | 56 ++++ triedb/pathdb/nodebuffer.go | 205 ++++++++++++ triedb/types/database_types.go | 56 ++++ triedb/types/metrics.go | 43 +++ 45 files changed, 4344 insertions(+), 111 deletions(-) create mode 100644 core/rawdb/accessors_trie.go create mode 100644 trie/hbss2pbss.go create mode 100644 trie/morph_zk_trie.go create mode 100644 triedb/hashdb/metrics.go create mode 100644 triedb/hashdb/zk_trie_database.go create mode 100644 triedb/pathdb/asyncnodebuffer.go create mode 100644 triedb/pathdb/database.go create mode 100644 triedb/pathdb/difflayer.go create mode 100644 triedb/pathdb/difflayer_test.go create mode 100644 triedb/pathdb/disklayer.go create mode 100644 triedb/pathdb/errors.go create mode 100644 triedb/pathdb/journal.go create mode 100644 triedb/pathdb/layertree.go create mode 100644 triedb/pathdb/metrics.go create mode 100644 triedb/pathdb/nodebuffer.go create mode 100644 triedb/types/database_types.go create mode 100644 triedb/types/metrics.go diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 0ccb0cc18..7bd232077 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -70,6 +70,7 @@ Remove blockchain and state databases`, dbDumpFreezerIndex, dbImportCmd, dbExportCmd, + dbHbss2PbssCmd, }, } dbInspectCmd = cli.Command{ @@ -254,6 +255,18 @@ WARNING: This is a low-level operation which may cause database corruption!`, }, Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.", } + dbHbss2PbssCmd = cli.Command{ + Action: hbss2pbss, + Name: "hbss-to-pbss", + ArgsUsage: "", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.AncientFlag, + }, + Usage: "Convert Hash-Base to Path-Base trie node.", + Description: `This command iterates the entire trie node database and convert the hash-base node to path-base node.`, + } ) func removeDB(ctx *cli.Context) error { @@ -706,3 +719,21 @@ func exportChaindata(ctx *cli.Context) error { db := utils.MakeChainDatabase(ctx, stack, true) return utils.ExportChaindata(ctx.Args().Get(1), kind, exporter(db), stop) } + +func hbss2pbss(ctx *cli.Context) error { + stack, config := makeConfigNode(ctx) + defer stack.Close() + + chaindb := utils.MakeChainDatabase(ctx, stack, false) + h2p, err := trie.NewHbss2Pbss(chaindb, stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal)) + if err != nil { + return err + } + + if ctx.NArg() > 0 { + log.Error("Too many arguments given") + return errors.New("too many arguments") + } + + return h2p.Run() +} diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 20198b992..6374ec10c 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -158,6 +158,8 @@ var ( utils.GpoIgnoreGasPriceFlag, configFileFlag, utils.CatalystFlag, + utils.MorphZkTrieFlag, + utils.PathDBSyncFlag, } rpcFlags = []cli.Flag{ diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index be4804157..085ba294f 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -57,6 +57,8 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.IdentityFlag, utils.LightKDFFlag, utils.WhitelistFlag, + utils.MorphZkTrieFlag, + utils.PathDBSyncFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 0f79394c4..20bc8388e 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -851,6 +851,15 @@ var ( Name: "rpc.getlogs.maxrange", Usage: "Limit max fetched block range for `eth_getLogs` method", } + + MorphZkTrieFlag = cli.BoolFlag{ + Name: "morphzktrie", + Usage: "Use MorphZkTrie instead of ZkTrie in state", + } + PathDBSyncFlag = cli.BoolFlag{ + Name: "pathdb.sync", + Usage: "Sync flush nodes cache to disk in path schema", + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -1718,6 +1727,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { cfg.EthDiscoveryURLs = SplitAndTrim(urls) } } + if ctx.GlobalIsSet(PathDBSyncFlag.Name) { + cfg.PathSyncFlush = true + } // Override any default configs for hard coded networks. switch { case ctx.GlobalBool(MainnetFlag.Name): @@ -1765,6 +1777,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { // disable prefetch log.Info("Prefetch disabled") cfg.NoPrefetch = true + + // use morph zktrie + cfg.Genesis.Config.Morph.MorphZkTrie = ctx.GlobalBool(MorphZkTrieFlag.Name) case ctx.GlobalBool(MorphHoleskyFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 2810 @@ -1780,6 +1795,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { // disable prefetch log.Info("Prefetch disabled") cfg.NoPrefetch = true + + // use morph zktrie + cfg.Genesis.Config.Morph.MorphZkTrie = ctx.GlobalBool(MorphZkTrieFlag.Name) case ctx.GlobalBool(DeveloperFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337 diff --git a/common/types.go b/common/types.go index d438a1649..42e972738 100644 --- a/common/types.go +++ b/common/types.go @@ -65,6 +65,11 @@ func BigToHash(b *big.Int) Hash { return BytesToHash(b.Bytes()) } // If b is larger than len(h), b will be cropped from the left. func HexToHash(s string) Hash { return BytesToHash(FromHex(s)) } +// Cmp compares two hashes. +func (h Hash) Cmp(other Hash) int { + return bytes.Compare(h[:], other[:]) +} + // Bytes gets the byte representation of the underlying hash. func (h Hash) Bytes() []byte { return h[:] } @@ -226,6 +231,11 @@ func IsHexAddress(s string) bool { return len(s) == 2*AddressLength && isHex(s) } +// Cmp compares two addresses. +func (a Address) Cmp(other Address) int { + return bytes.Compare(a[:], other[:]) +} + // Bytes gets the string representation of the underlying address. func (a Address) Bytes() []byte { return a[:] } @@ -433,3 +443,62 @@ func (ma *MixedcaseAddress) ValidChecksum() bool { func (ma *MixedcaseAddress) Original() string { return ma.original } + +func ReverseBytes(b []byte) []byte { + o := make([]byte, len(b)) + for i := range b { + o[len(b)-1-i] = b[i] + } + return o +} + +func bitReverseForNibble(b byte) byte { + switch b { + case 0: + return 0 + case 1: + return 8 + case 2: + return 4 + case 3: + return 12 + case 4: + return 2 + case 5: + return 10 + case 6: + return 6 + case 7: + return 14 + case 8: + return 1 + case 9: + return 9 + case 10: + return 5 + case 11: + return 13 + case 12: + return 3 + case 13: + return 11 + case 14: + return 7 + case 15: + return 15 + default: + panic("unexpected input") + } +} + +func BitReverse(inp []byte) (out []byte) { + + l := len(inp) + out = make([]byte, l) + + for i, b := range inp { + out[l-i-1] = bitReverseForNibble(b&15)<<4 + bitReverseForNibble(b>>4) + } + + return +} diff --git a/core/blockchain.go b/core/blockchain.go index f23faaf43..4130a0d46 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -45,6 +45,8 @@ import ( "github.com/morph-l2/go-ethereum/metrics" "github.com/morph-l2/go-ethereum/params" "github.com/morph-l2/go-ethereum/trie" + "github.com/morph-l2/go-ethereum/triedb/hashdb" + "github.com/morph-l2/go-ethereum/triedb/pathdb" ) var ( @@ -135,6 +137,9 @@ type CacheConfig struct { Preimages bool // Whether to store preimage of trie key to the disk SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it + + PathSyncFlush bool // Whether sync flush the trienodebuffer of pathdb to disk. + JournalFilePath string } // defaultCacheConfig are the default caching values if none are specified by the @@ -239,16 +244,37 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par log.Warn("Using fee vault address", "FeeVaultAddress", *chainConfig.Morph.FeeVaultAddress) } + var hashdbConfig *hashdb.Config + var pathdbConfig *pathdb.Config + if chainConfig.Morph.ZktrieEnabled() && chainConfig.Morph.MorphZktrieEnabled() { + pathdbConfig = &pathdb.Config{ + SyncFlush: cacheConfig.PathSyncFlush, + CleanCacheSize: cacheConfig.TrieCleanLimit * 1024 * 1024, + DirtyCacheSize: cacheConfig.TrieCleanLimit * 1024 * 1024, + JournalFilePath: cacheConfig.JournalFilePath, + } + } else { + if chainConfig.Morph.ZktrieEnabled() { + hashdbConfig = &hashdb.Config{ + Cache: cacheConfig.TrieCleanLimit, + Journal: cacheConfig.TrieCleanJournal, + } + } + } + bc := &BlockChain{ chainConfig: chainConfig, cacheConfig: cacheConfig, db: db, triegc: prque.New(nil), stateCache: state.NewDatabaseWithConfig(db, &trie.Config{ - Cache: cacheConfig.TrieCleanLimit, - Journal: cacheConfig.TrieCleanJournal, - Preimages: cacheConfig.Preimages, - Zktrie: chainConfig.Morph.ZktrieEnabled(), + Cache: cacheConfig.TrieCleanLimit, + Journal: cacheConfig.TrieCleanJournal, + Preimages: cacheConfig.Preimages, + Zktrie: chainConfig.Morph.ZktrieEnabled(), + MorphZkTrie: chainConfig.Morph.ZktrieEnabled() && chainConfig.Morph.MorphZktrieEnabled(), + HashDB: hashdbConfig, + PathDB: pathdbConfig, }), quit: make(chan struct{}), chainmu: syncx.NewClosableMutex(), @@ -809,43 +835,50 @@ func (bc *BlockChain) Stop() { } } - // Ensure the state of a recent block is also stored to disk before exiting. - // We're writing three different states to catch different restart scenarios: - // - HEAD: So we don't need to reprocess any blocks in the general case - // - HEAD-1: So we don't do large reorgs if our HEAD becomes an uncle - // - HEAD-127: So we have a hard limit on the number of blocks reexecuted - if !bc.cacheConfig.TrieDirtyDisabled { - triedb := bc.stateCache.TrieDB() - - for _, offset := range []uint64{0, 1, TriesInMemory - 1} { - if number := bc.CurrentBlock().NumberU64(); number > offset { - recent := bc.GetBlockByNumber(number - offset) - - log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root()) - if err := triedb.Commit(recent.Root(), true, nil); err != nil { + if bc.stateCache.TrieDB().Scheme() == rawdb.PathScheme { + // Ensure that the in-memory trie nodes are journaled to disk properly. + if err := bc.stateCache.TrieDB().Journal(bc.CurrentBlock().Root()); err != nil { + log.Info("Failed to journal in-memory trie nodes", "err", err) + } + } else { + // Ensure the state of a recent block is also stored to disk before exiting. + // We're writing three different states to catch different restart scenarios: + // - HEAD: So we don't need to reprocess any blocks in the general case + // - HEAD-1: So we don't do large reorgs if our HEAD becomes an uncle + // - HEAD-127: So we have a hard limit on the number of blocks reexecuted + if !bc.cacheConfig.TrieDirtyDisabled { + triedb := bc.stateCache.TrieDB() + + for _, offset := range []uint64{0, 1, TriesInMemory - 1} { + if number := bc.CurrentBlock().NumberU64(); number > offset { + recent := bc.GetBlockByNumber(number - offset) + + log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root()) + if err := triedb.Commit(recent.Root(), true, nil); err != nil { + log.Error("Failed to commit recent state trie", "err", err) + } + } + } + if snapBase != (common.Hash{}) { + log.Info("Writing snapshot state to disk", "root", snapBase) + if err := triedb.Commit(snapBase, true, nil); err != nil { log.Error("Failed to commit recent state trie", "err", err) } } - } - if snapBase != (common.Hash{}) { - log.Info("Writing snapshot state to disk", "root", snapBase) - if err := triedb.Commit(snapBase, true, nil); err != nil { - log.Error("Failed to commit recent state trie", "err", err) + for !bc.triegc.Empty() { + triedb.Dereference(bc.triegc.PopItem().(common.Hash)) + } + if size, _ := triedb.Size(); size != 0 { + log.Error("Dangling trie nodes after full cleanup") } } - for !bc.triegc.Empty() { - triedb.Dereference(bc.triegc.PopItem().(common.Hash)) - } - if size, _ := triedb.Size(); size != 0 { - log.Error("Dangling trie nodes after full cleanup") + // Ensure all live cached entries be saved into disk, so that we can skip + // cache warmup when node restarts. + if bc.cacheConfig.TrieCleanJournal != "" { + triedb := bc.stateCache.TrieDB() + triedb.SaveCache(bc.cacheConfig.TrieCleanJournal) } } - // Ensure all live cached entries be saved into disk, so that we can skip - // cache warmup when node restarts. - if bc.cacheConfig.TrieCleanJournal != "" { - triedb := bc.stateCache.TrieDB() - triedb.SaveCache(bc.cacheConfig.TrieCleanJournal) - } log.Info("Blockchain stopped") } diff --git a/core/blockchain_l2.go b/core/blockchain_l2.go index 0bbdb1a2a..e1eb6232d 100644 --- a/core/blockchain_l2.go +++ b/core/blockchain_l2.go @@ -107,6 +107,10 @@ func (bc *BlockChain) writeBlockStateWithoutHead(block *types.Block, receipts [] if err := blockBatch.Write(); err != nil { log.Crit("Failed to write block into disk", "err", err) } + + current := block.NumberU64() + origin := state.GetOriginRoot() + // Commit all cached state changes into underlying memory database. root, err := state.Commit(bc.chainConfig.IsEIP158(block.Number())) if err != nil { @@ -116,13 +120,15 @@ func (bc *BlockChain) writeBlockStateWithoutHead(block *types.Block, receipts [] triedb := bc.stateCache.TrieDB() // If we're running an archive node, always flush if bc.cacheConfig.TrieDirtyDisabled { + if triedb.Scheme() == rawdb.PathScheme { + return triedb.CommitState(root, origin, current, false) + } return triedb.Commit(root, false, nil) } // Full but not archive node, do proper garbage collection triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive bc.triegc.Push(root, -int64(block.NumberU64())) - current := block.NumberU64() // Flush limits are not considered for the first TriesInMemory blocks. if current <= TriesInMemory { return nil diff --git a/core/genesis.go b/core/genesis.go index b0e168f66..7e04ff0c9 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -38,6 +38,7 @@ import ( "github.com/morph-l2/go-ethereum/params" "github.com/morph-l2/go-ethereum/rlp" "github.com/morph-l2/go-ethereum/trie" + zkt "github.com/scroll-tech/zktrie/types" ) //go:generate gencodec -type Genesis -field-override genesisSpecMarshaling -out gen_genesis.go @@ -201,28 +202,41 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override if storedcfg == nil { log.Warn("Found genesis block without chain config") } else { - trieCfg = &trie.Config{Zktrie: storedcfg.Morph.ZktrieEnabled()} + trieCfg = &trie.Config{Zktrie: storedcfg.Morph.ZktrieEnabled(), MorphZkTrie: storedcfg.Morph.MorphZktrieEnabled()} } } else { - trieCfg = &trie.Config{Zktrie: genesis.Config.Morph.ZktrieEnabled()} + trieCfg = &trie.Config{Zktrie: genesis.Config.Morph.ZktrieEnabled(), MorphZkTrie: genesis.Config.Morph.MorphZktrieEnabled()} } - if _, err := state.New(header.Root, state.NewDatabaseWithConfig(db, trieCfg), nil); err != nil { - log.Error("failed to new state in SetupGenesisBlockWithOverride", "header root", header.Root.String(), "error", err) - if genesis == nil { - genesis = DefaultGenesisBlock() - } - // Ensure the stored genesis matches with the given one. - hash := genesis.ToBlock(nil).Hash() - if hash != stored { - return genesis.Config, hash, &GenesisMismatchError{stored, hash} - } - block, err := genesis.Commit(db) - if err != nil { - return genesis.Config, hash, err + _, diskroot := rawdb.ReadAccountTrieNode(db, zkt.TrieRootPathKey[:]) + head := rawdb.ReadPersistentStateID(db) + var verify bool = true + // new states already overide genesis states. + // if trieCfg.MorphZkTrie && (diskroot == types.EmptyRootHash1 || header.Root != diskroot) { + if trieCfg.MorphZkTrie && (diskroot == types.GenesisRootHash || head > 0) { + verify = false + } + + if verify { + if _, err := state.New(header.Root, state.NewDatabaseWithConfig(db, trieCfg), nil); err != nil { + log.Error("failed to new state in SetupGenesisBlockWithOverride", "header root", header.Root.String(), "error", err) + if genesis == nil { + genesis = DefaultGenesisBlock() + } + // Ensure the stored genesis matches with the given one. + hash := genesis.ToBlock(nil).Hash() + if hash != stored { + return genesis.Config, hash, &GenesisMismatchError{stored, hash} + } + + block, err := genesis.Commit(db) + if err != nil { + return genesis.Config, hash, err + } + return genesis.Config, block.Hash(), nil } - return genesis.Config, block.Hash(), nil } + // Check whether the genesis block is already written. if genesis != nil { hash := genesis.ToBlock(nil).Hash() @@ -252,6 +266,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override storedcfg.Morph.MaxTxPerBlock = storedcfg.Scroll.MaxTxPerBlock storedcfg.Morph.MaxTxPayloadBytesPerBlock = storedcfg.Scroll.MaxTxPayloadBytesPerBlock storedcfg.Morph.FeeVaultAddress = storedcfg.Scroll.FeeVaultAddress + storedcfg.Morph.MorphZkTrie = storedcfg.Scroll.MorphZkTrie } // Special case: don't change the existing config of a non-mainnet chain if no new @@ -296,12 +311,14 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { // ToBlock creates the genesis block and writes state of a genesis specification // to the given database (or discards it if nil). func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { + toDiskDb := true if db == nil { + toDiskDb = false db = rawdb.NewMemoryDatabase() } var trieCfg *trie.Config if g.Config != nil { - trieCfg = &trie.Config{Zktrie: g.Config.Morph.ZktrieEnabled()} + trieCfg = &trie.Config{Zktrie: g.Config.Morph.ZktrieEnabled(), MorphZkTrie: g.Config.Morph.MorphZktrieEnabled()} } statedb, err := state.New(common.Hash{}, state.NewDatabaseWithConfig(db, trieCfg), nil) if err != nil { @@ -344,7 +361,11 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { } } statedb.Commit(false) - statedb.Database().TrieDB().Commit(root, true, nil) + if toDiskDb { + statedb.Database().TrieDB().CommitGenesis(root) + } else { + statedb.Database().TrieDB().Commit(root, true, nil) + } return types.NewBlock(head, nil, nil, nil, trie.NewStackTrie(nil)) } diff --git a/core/rawdb/accessors_state.go b/core/rawdb/accessors_state.go index 4812cc245..155a2b2bd 100644 --- a/core/rawdb/accessors_state.go +++ b/core/rawdb/accessors_state.go @@ -17,6 +17,8 @@ package rawdb import ( + "encoding/binary" + "github.com/morph-l2/go-ethereum/common" "github.com/morph-l2/go-ethereum/ethdb" "github.com/morph-l2/go-ethereum/log" @@ -81,6 +83,11 @@ func ReadTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte { return data } +// ReadTrieNode retrieves the trie node of the provided hash. +func ReadTrieNodeByKey(db ethdb.KeyValueReader, key []byte) ([]byte, error) { + return db.Get(key) +} + // WriteTrieNode writes the provided trie node database. func WriteTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) { if err := db.Put(hash.Bytes(), node); err != nil { @@ -88,9 +95,129 @@ func WriteTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) { } } +// WriteTrieNodeByKey writes the provided trie node into database. +func WriteTrieNodeByKey(db ethdb.KeyValueWriter, path []byte, node []byte) { + if err := db.Put(path, node); err != nil { + log.Crit("Failed to store trie node", "err", err) + } +} + // DeleteTrieNode deletes the specified trie node from the database. func DeleteTrieNode(db ethdb.KeyValueWriter, hash common.Hash) { if err := db.Delete(hash.Bytes()); err != nil { log.Crit("Failed to delete trie node", "err", err) } } + +// ExistsAccountTrieNode checks the presence of the account trie node with the +// specified node path, regardless of the node hash. +func ExistsAccountTrieNode(db ethdb.KeyValueReader, path []byte) bool { + has, err := db.Has(accountTrieNodeKey(path)) + if err != nil { + return false + } + return has +} + +// WriteAccountTrieNode writes the provided account trie node into database. +func WriteAccountTrieNode(db ethdb.KeyValueWriter, path []byte, node []byte) { + if err := db.Put(accountTrieNodeKey(path), node); err != nil { + log.Crit("Failed to store account trie node", "err", err) + } +} + +// DeleteAccountTrieNode deletes the specified account trie node from the database. +func DeleteAccountTrieNode(db ethdb.KeyValueWriter, path []byte) { + if err := db.Delete(accountTrieNodeKey(path)); err != nil { + log.Crit("Failed to delete account trie node", "err", err) + } +} + +// ExistsStorageTrieNode checks the presence of the storage trie node with the +// specified account hash and node path, regardless of the node hash. +func ExistsStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte) bool { + has, err := db.Has(storageTrieNodeKey(accountHash, path)) + if err != nil { + return false + } + return has +} + +// WriteStorageTrieNode writes the provided storage trie node into database. +func WriteStorageTrieNode(db ethdb.KeyValueWriter, accountHash common.Hash, path []byte, node []byte) { + if err := db.Put(storageTrieNodeKey(accountHash, path), node); err != nil { + log.Crit("Failed to store storage trie node", "err", err) + } +} + +// DeleteStorageTrieNode deletes the specified storage trie node from the database. +func DeleteStorageTrieNode(db ethdb.KeyValueWriter, accountHash common.Hash, path []byte) { + if err := db.Delete(storageTrieNodeKey(accountHash, path)); err != nil { + log.Crit("Failed to delete storage trie node", "err", err) + } +} + +// ReadTrieJournal retrieves the serialized in-memory trie nodes of layers saved at +// the last shutdown. +func ReadTrieJournal(db ethdb.KeyValueReader) []byte { + data, _ := db.Get(trieJournalKey) + return data +} + +// WriteTrieJournal stores the serialized in-memory trie nodes of layers to save at +// shutdown. +func WriteTrieJournal(db ethdb.KeyValueWriter, journal []byte) { + if err := db.Put(trieJournalKey, journal); err != nil { + log.Crit("Failed to store tries journal", "err", err) + } +} + +// DeleteTrieJournal deletes the serialized in-memory trie nodes of layers saved at +// the last shutdown. +func DeleteTrieJournal(db ethdb.KeyValueWriter) { + if err := db.Delete(trieJournalKey); err != nil { + log.Crit("Failed to remove tries journal", "err", err) + } +} + +// ReadPersistentStateID retrieves the id of the persistent state from the database. +func ReadPersistentStateID(db ethdb.KeyValueReader) uint64 { + data, _ := db.Get(persistentStateIDKey) + if len(data) != 8 { + return 0 + } + return binary.BigEndian.Uint64(data) +} + +// WritePersistentStateID stores the id of the persistent state into database. +func WritePersistentStateID(db ethdb.KeyValueWriter, number uint64) { + if err := db.Put(persistentStateIDKey, encodeBlockNumber(number)); err != nil { + log.Crit("Failed to store the persistent state ID", "err", err) + } +} + +// ReadStateID retrieves the state id with the provided state root. +func ReadStateID(db ethdb.KeyValueReader, root common.Hash) *uint64 { + data, err := db.Get(stateIDKey(root)) + if err != nil || len(data) == 0 { + return nil + } + number := binary.BigEndian.Uint64(data) + return &number +} + +// WriteStateID writes the provided state lookup to database. +func WriteStateID(db ethdb.KeyValueWriter, root common.Hash, id uint64) { + var buff [8]byte + binary.BigEndian.PutUint64(buff[:], id) + if err := db.Put(stateIDKey(root), buff[:]); err != nil { + log.Crit("Failed to store state ID", "err", err) + } +} + +// DeleteStateID deletes the specified state lookup from the database. +func DeleteStateID(db ethdb.KeyValueWriter, root common.Hash) { + if err := db.Delete(stateIDKey(root)); err != nil { + log.Crit("Failed to delete state ID", "err", err) + } +} diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go new file mode 100644 index 000000000..245b306b1 --- /dev/null +++ b/core/rawdb/accessors_trie.go @@ -0,0 +1,66 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package rawdb + +import ( + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/ethdb" + + zktrie "github.com/scroll-tech/zktrie/trie" +) + +// HashScheme is the legacy hash-based state scheme with which trie nodes are +// stored in the disk with node hash as the database key. The advantage of this +// scheme is that different versions of trie nodes can be stored in disk, which +// is very beneficial for constructing archive nodes. The drawback is it will +// store different trie nodes on the same path to different locations on the disk +// with no data locality, and it's unfriendly for designing state pruning. +// +// Now this scheme is still kept for backward compatibility, and it will be used +// for archive node and some other tries(e.g. light trie). +const HashScheme = "hash" + +const ZkHashScheme = "hashZk" + +// PathScheme is the new path-based state scheme with which trie nodes are stored +// in the disk with node path as the database key. This scheme will only store one +// version of state data in the disk, which means that the state pruning operation +// is native. At the same time, this scheme will put adjacent trie nodes in the same +// area of the disk with good data locality property. But this scheme needs to rely +// on extra state diffs to survive deep reorg. +const PathScheme = "path" + +// ReadAccountTrieNode retrieves the account trie node and the associated node +// hash with the specified node path. +func ReadAccountTrieNode(db ethdb.KeyValueReader, path []byte) ([]byte, common.Hash) { + data, err := db.Get(accountTrieNodeKey(path)) + if err != nil { + return nil, common.Hash{} + } + + n, err := zktrie.NewNodeFromBytes(data) + if err != nil { + return nil, common.Hash{} + } + + zkHash, err := n.NodeHash() + if err != nil { + return nil, common.Hash{} + } + + return data, common.BytesToHash(zkHash.Bytes()) +} diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 8b7ffd07a..3032b826b 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -43,6 +43,9 @@ var ( // headFastBlockKey tracks the latest known incomplete block's hash during fast sync. headFastBlockKey = []byte("LastFast") + // persistentStateIDKey tracks the id of latest stored state(for path-based only). + persistentStateIDKey = []byte("LastStateID") + // lastPivotKey tracks the last pivot block used by fast sync (to reenable on sethead). lastPivotKey = []byte("LastPivot") @@ -70,6 +73,9 @@ var ( // skeletonSyncStatusKey tracks the skeleton sync status across restarts. skeletonSyncStatusKey = []byte("SkeletonSyncStatus") + // trieJournalKey tracks the in-memory trie node layers across restarts. + trieJournalKey = []byte("TrieJournal") + // txIndexTailKey tracks the oldest block whose transactions have been indexed. txIndexTailKey = []byte("TransactionIndexTail") @@ -122,6 +128,11 @@ var ( rollupBatchSignaturePrefix = []byte("R-bs") rollupBatchL1DataFeePrefix = []byte("R-df") rollupBatchHeadBatchHasFeeKey = []byte("R-hbf") + + // Path-based storage scheme of merkle patricia trie. + TrieNodeAccountPrefix = []byte("A") // trieNodeAccountPrefix + hexPath -> trie node + TrieNodeStoragePrefix = []byte("O") // trieNodeStoragePrefix + accountHash + hexPath -> trie node + stateIDPrefix = []byte("L") // stateIDPrefix + state root -> state id ) const ( @@ -310,3 +321,22 @@ func RollupBatchSignatureSignerKey(batchHash common.Hash, signer common.Address) func RollupBatchL1DataFeeKey(batchIndex uint64) []byte { return append(rollupBatchL1DataFeePrefix, encodeBigEndian(batchIndex)...) } + +// accountTrieNodeKey = trieNodeAccountPrefix + nodePath. +func accountTrieNodeKey(path []byte) []byte { + return append(TrieNodeAccountPrefix, path...) +} + +// storageTrieNodeKey = trieNodeStoragePrefix + accountHash + nodePath. +func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte { + buf := make([]byte, len(TrieNodeStoragePrefix)+common.HashLength+len(path)) + n := copy(buf, TrieNodeStoragePrefix) + n += copy(buf[n:], accountHash.Bytes()) + copy(buf[n:], path) + return buf +} + +// stateIDKey = stateIDPrefix + root (32 bytes) +func stateIDKey(root common.Hash) []byte { + return append(stateIDPrefix, root.Bytes()...) +} diff --git a/core/state/database.go b/core/state/database.go index a9c78e121..f26f9afab 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -28,6 +28,7 @@ import ( "github.com/morph-l2/go-ethereum/core/types" "github.com/morph-l2/go-ethereum/ethdb" "github.com/morph-l2/go-ethereum/trie" + zkt "github.com/scroll-tech/zktrie/types" ) const ( @@ -44,7 +45,7 @@ type Database interface { OpenTrie(root common.Hash) (Trie, error) // OpenStorageTrie opens the storage trie of an account. - OpenStorageTrie(addrHash, root common.Hash) (Trie, error) + OpenStorageTrie(addrHash, root, origin common.Hash) (Trie, error) // CopyTrie returns an independent copy of the given trie. CopyTrie(Trie) Trie @@ -121,6 +122,7 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { csc, _ := lru.New(codeSizeCacheSize) return &cachingDB{ zktrie: config != nil && config.Zktrie, + morphZktrie: config != nil && config.Zktrie && config.MorphZkTrie, db: trie.NewDatabaseWithConfig(db, config), codeSizeCache: csc, codeCache: lru2.NewSizeConstrainedLRU(codeCacheSize), @@ -132,12 +134,21 @@ type cachingDB struct { codeSizeCache *lru.Cache codeCache *lru2.SizeConstrainedLRU zktrie bool + morphZktrie bool } // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { + if db.morphZktrie { + tr, err := trie.NewMorphZkTrie(root, root, db.db, rawdb.TrieNodeAccountPrefix) + if err != nil { + return nil, err + } + return tr, nil + } + if db.zktrie { - tr, err := trie.NewZkTrie(root, trie.NewZktrieDatabaseFromTriedb(db.db)) + tr, err := trie.NewZkTrie(root, db.db) if err != nil { return nil, err } @@ -151,9 +162,18 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { } // OpenStorageTrie opens the storage trie of an account. -func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { +func (db *cachingDB) OpenStorageTrie(addrHash, root, origin common.Hash) (Trie, error) { + if db.morphZktrie { + prefix := append(rawdb.TrieNodeStoragePrefix, zkt.ReverseByteOrder(addrHash.Bytes())...) + tr, err := trie.NewMorphZkTrie(root, origin, db.db, prefix) + if err != nil { + return nil, err + } + return tr, nil + } + if db.zktrie { - tr, err := trie.NewZkTrie(root, trie.NewZktrieDatabaseFromTriedb(db.db)) + tr, err := trie.NewZkTrie(root, db.db) if err != nil { return nil, err } @@ -173,6 +193,8 @@ func (db *cachingDB) CopyTrie(t Trie) Trie { return t.Copy() case *trie.ZkTrie: return t.Copy() + case *trie.MorphZkTrie: + return t.Copy() default: panic(fmt.Errorf("unknown trie type %T", t)) } diff --git a/core/state/iterator.go b/core/state/iterator.go index 6ad0d37bf..0c28af74f 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -109,7 +109,7 @@ func (it *NodeIterator) step() error { if err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob()), &account); err != nil { return err } - dataTrie, err := it.state.db.OpenStorageTrie(common.BytesToHash(it.stateIt.LeafKey()), account.Root) + dataTrie, err := it.state.db.OpenStorageTrie(common.BytesToHash(it.stateIt.LeafKey()), account.Root, common.Hash{}) if err != nil { return err } diff --git a/core/state/pruner/zk-pruner.go b/core/state/pruner/zk-pruner.go index c72433969..bff037860 100644 --- a/core/state/pruner/zk-pruner.go +++ b/core/state/pruner/zk-pruner.go @@ -47,7 +47,7 @@ func NewZKPruner(chaindb ethdb.Database, bloomSize uint64, datadir, trieCachePat headBlock := rawdb.ReadHeadBlock(chaindb) root := headBlock.Root() log.Info("current head block", "block number", headBlock.NumberU64(), "root", root.Hex()) - zkTrie, err := trie.NewZkTrie(root, trie.NewZktrieDatabaseFromTriedb(stateCache.TrieDB())) + zkTrie, err := trie.NewZkTrie(root, stateCache.TrieDB()) if err != nil { return nil, err } @@ -249,9 +249,9 @@ func (p *ZKPruner) extractTrieNodes(root *common.Hash, accountTrie bool) { } } - p.stateBloom.Put(trie.BitReverse(nodeHash[:]), nil) + p.stateBloom.Put(common.BitReverse(nodeHash[:]), nil) case zktrie.NodeTypeBranch_0, zktrie.NodeTypeBranch_1, zktrie.NodeTypeBranch_2, zktrie.NodeTypeBranch_3, zktrie.DBEntryTypeRoot: - p.stateBloom.Put(trie.BitReverse(nodeHash[:]), nil) + p.stateBloom.Put(common.BitReverse(nodeHash[:]), nil) case zktrie.NodeTypeEmpty, zktrie.NodeTypeLeaf, zktrie.NodeTypeParent: panic("encounter unsupported deprecated node type") default: diff --git a/core/state/state_object.go b/core/state/state_object.go index 073791214..e61e8f252 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -29,6 +29,8 @@ import ( "github.com/morph-l2/go-ethereum/crypto/codehash" "github.com/morph-l2/go-ethereum/metrics" "github.com/morph-l2/go-ethereum/rlp" + + zkt "github.com/scroll-tech/zktrie/types" ) var emptyPoseidonCodeHash = codehash.EmptyPoseidonCodeHash.Bytes() @@ -163,9 +165,15 @@ func (s *stateObject) getTrie(db Database) Trie { } if s.trie == nil { var err error - s.trie, err = db.OpenStorageTrie(s.addrHash, s.data.Root) + addrHash := s.addrHash + if db.TrieDB().IsMorphZk() { + addr_s, _ := zkt.ToSecureKey(s.address.Bytes()) + addrHash = common.BigToHash(addr_s) + } + + s.trie, err = db.OpenStorageTrie(addrHash, s.data.Root, s.db.originalRoot) if err != nil { - s.trie, _ = db.OpenStorageTrie(s.addrHash, common.Hash{}) + s.trie, _ = db.OpenStorageTrie(addrHash, common.Hash{}, s.db.originalRoot) s.setError(fmt.Errorf("can't create storage trie: %v", err)) } } diff --git a/core/state/state_prove.go b/core/state/state_prove.go index aba0981ba..dec259104 100644 --- a/core/state/state_prove.go +++ b/core/state/state_prove.go @@ -52,10 +52,19 @@ func (s *StateDB) GetStorageTrieForProof(addr common.Address) (Trie, error) { // try the trie in stateObject first, else we would create one stateObject := s.getStateObject(addr) + + addrHash := crypto.Keccak256Hash(addr[:]) + if s.IsMorphZktrie() { + k, err := zkt.ToSecureKey(addr.Bytes()) + if err != nil { + return nil, fmt.Errorf("can't create storage trie on ToSecureKey %s: %v ", addr.Hex(), err) + } + addrHash = common.BigToHash(k) + } + if stateObject == nil { // still return a empty trie - addrHash := crypto.Keccak256Hash(addr[:]) - dummy_trie, _ := s.db.OpenStorageTrie(addrHash, common.Hash{}) + dummy_trie, _ := s.db.OpenStorageTrie(addrHash, common.Hash{}, common.Hash{}) return dummy_trie, nil } @@ -63,7 +72,7 @@ func (s *StateDB) GetStorageTrieForProof(addr common.Address) (Trie, error) { var err error if trie == nil { // use a new, temporary trie - trie, err = s.db.OpenStorageTrie(stateObject.addrHash, stateObject.data.Root) + trie, err = s.db.OpenStorageTrie(addrHash, stateObject.data.Root, s.originalRoot) if err != nil { return nil, fmt.Errorf("can't create storage trie on root %s: %v ", stateObject.data.Root, err) } diff --git a/core/state/statedb.go b/core/state/statedb.go index 2a18c9d35..b50abb22f 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -156,6 +156,10 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) return sdb, nil } +func (s *StateDB) GetOriginRoot() common.Hash { + return s.originalRoot +} + // StartPrefetcher initializes a new trie prefetcher to pull in nodes from the // state trie concurrently while the state is mutated so that when we reach the // commit phase, most of the needed data is already hot. @@ -193,6 +197,10 @@ func (s *StateDB) IsZktrie() bool { return s.db.TrieDB().Zktrie } +func (s *StateDB) IsMorphZktrie() bool { + return s.db.TrieDB().Zktrie && s.db.TrieDB().IsMorphZk() +} + func (s *StateDB) AddLog(log *types.Log) { s.journal.append(addLogChange{txhash: s.thash}) diff --git a/core/types/block.go b/core/types/block.go index e14252dad..2fd4a032f 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -32,9 +32,10 @@ import ( ) var ( - EmptyRootHash = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") - EmptyUncleHash = rlpHash([]*Header(nil)) - EmptyAddress = common.Address{} + EmptyRootHash = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + EmptyUncleHash = rlpHash([]*Header(nil)) + EmptyAddress = common.Address{} + GenesisRootHash = common.HexToHash("09688bec5d876538664e62247c2f64fc7a02c54a3f898b42020730c7dd4933aa") ) // A BlockNonce is a 64-bit hash which proves (combined with the diff --git a/eth/backend.go b/eth/backend.go index 986ca6740..7ba630c95 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -121,6 +121,11 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } config.TrieDirtyCache = 0 } + + if config.JournalFileName == "" { + config.JournalFileName = ethconfig.Defaults.JournalFileName + } + log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024) // Assemble the Ethereum object @@ -186,6 +191,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TrieTimeLimit: config.TrieTimeout, SnapshotLimit: config.SnapshotCache, Preimages: config.Preimages, + PathSyncFlush: config.PathSyncFlush, + JournalFilePath: stack.ResolvePath(config.JournalFileName), } ) // TODO (MariusVanDerWijden) get rid of shouldPreserve in a follow-up PR diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 13a92cd55..06c6eb9a9 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -90,6 +90,8 @@ var Defaults = Config{ GPO: FullNodeGPO, RPCTxFeeCap: 1, // 1 ether MaxBlockRange: -1, // Default unconfigured value: no block range limit for backward compatibility + JournalFileName: "trie.journal", + PathSyncFlush: true, } func init() { @@ -211,6 +213,12 @@ type Config struct { // Max block range for eth_getLogs api method MaxBlockRange int64 + + // Path schema journal file name + JournalFileName string + + // State scheme used to store ethereum state and merkle trie nodes on top + PathSyncFlush bool } // CreateConsensusEngine creates a consensus engine for the given chain config. diff --git a/go.mod b/go.mod index 996ac7e8a..e789185ea 100644 --- a/go.mod +++ b/go.mod @@ -59,11 +59,11 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/tyler-smith/go-bip39 v1.1.0 golang.org/x/crypto v0.21.0 - golang.org/x/sync v0.5.0 + golang.org/x/sync v0.6.0 golang.org/x/sys v0.18.0 golang.org/x/text v0.14.0 golang.org/x/time v0.3.0 - golang.org/x/tools v0.15.0 + golang.org/x/tools v0.18.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 @@ -117,7 +117,7 @@ require ( github.com/supranational/blst v0.3.11 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - golang.org/x/mod v0.14.0 // indirect + golang.org/x/mod v0.15.0 // indirect golang.org/x/net v0.21.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect @@ -126,3 +126,5 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) + +replace github.com/scroll-tech/zktrie v0.8.4 => github.com/ryanmorphl2/zktrie v1.8.9 diff --git a/go.sum b/go.sum index fcb0ccc45..a2c0a749f 100644 --- a/go.sum +++ b/go.sum @@ -432,8 +432,8 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/scroll-tech/zktrie v0.8.4 h1:UagmnZ4Z3ITCk+aUq9NQZJNAwnWl4gSxsLb2Nl7IgRE= -github.com/scroll-tech/zktrie v0.8.4/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk= +github.com/ryanmorphl2/zktrie v1.8.9 h1:Ssjldo56GNjiVDJLc8NwhHJL+ZRC2AdgkhviVrtbEts= +github.com/ryanmorphl2/zktrie v1.8.9/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -526,8 +526,8 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -566,8 +566,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -663,8 +663,8 @@ golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8= -golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk= +golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/les/server_requests.go b/les/server_requests.go index ac04e9320..888681b59 100644 --- a/les/server_requests.go +++ b/les/server_requests.go @@ -428,7 +428,7 @@ func handleGetProofs(msg Decoder) (serveRequestFn, uint64, uint64, error) { p.bumpInvalid() continue } - trie, err = statedb.OpenStorageTrie(common.BytesToHash(request.AccKey), account.Root) + trie, err = statedb.OpenStorageTrie(common.BytesToHash(request.AccKey), account.Root, root) if trie == nil || err != nil { p.Log().Warn("Failed to open storage trie for proof", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "root", account.Root, "err", err) continue diff --git a/light/trie.go b/light/trie.go index 17698b4c3..c62be6827 100644 --- a/light/trie.go +++ b/light/trie.go @@ -51,7 +51,7 @@ func (db *odrDatabase) OpenTrie(root common.Hash) (state.Trie, error) { return &odrTrie{db: db, id: db.id}, nil } -func (db *odrDatabase) OpenStorageTrie(addrHash, root common.Hash) (state.Trie, error) { +func (db *odrDatabase) OpenStorageTrie(addrHash, root, origin common.Hash) (state.Trie, error) { return &odrTrie{db: db, id: StorageTrieID(db.id, addrHash, root)}, nil } diff --git a/params/config.go b/params/config.go index 48c6755f8..88ebbbf79 100644 --- a/params/config.go +++ b/params/config.go @@ -564,6 +564,9 @@ type MorphConfig struct { // Transaction fee vault address [optional] FeeVaultAddress *common.Address `json:"feeVaultAddress,omitempty"` + + // Use MorphZkTrie instead of ZkTrie in state + MorphZkTrie bool `json:"morphZkTrie,omitempty"` } func (s MorphConfig) FeeVaultEnabled() bool { @@ -574,6 +577,10 @@ func (s MorphConfig) ZktrieEnabled() bool { return s.UseZktrie } +func (s MorphConfig) MorphZktrieEnabled() bool { + return s.MorphZkTrie +} + func (s MorphConfig) String() string { maxTxPerBlock := "" if s.MaxTxPerBlock != nil { diff --git a/trie/database.go b/trie/database.go index 16e9c6bbc..7f7ea5710 100644 --- a/trie/database.go +++ b/trie/database.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "io" + "math/big" "reflect" "runtime" "sync" @@ -33,6 +34,9 @@ import ( "github.com/morph-l2/go-ethereum/log" "github.com/morph-l2/go-ethereum/metrics" "github.com/morph-l2/go-ethereum/rlp" + "github.com/morph-l2/go-ethereum/triedb/hashdb" + "github.com/morph-l2/go-ethereum/triedb/pathdb" + zkt "github.com/scroll-tech/zktrie/types" ) var ( @@ -71,7 +75,8 @@ type Database struct { diskdb ethdb.KeyValueStore // Persistent storage for matured trie nodes // zktrie related stuff - Zktrie bool + Zktrie bool + MorphZkTrie bool // TODO: It's a quick&dirty implementation. FIXME later. rawDirties KvMap @@ -93,6 +98,8 @@ type Database struct { preimages *preimageStore // The store for caching preimages lock sync.RWMutex + + backend backend // The backend for managing trie nodes } // rawNode is a simple binary blob used to differentiate between collapsed trie @@ -278,10 +285,58 @@ func expandNode(hash hashNode, n node) node { // Config defines all necessary options for database. type Config struct { - Cache int // Memory allowance (MB) to use for caching trie nodes in memory - Journal string // Journal of clean cache to survive node restarts - Preimages bool // Flag whether the preimage of trie key is recorded - Zktrie bool // use zktrie + Cache int // Memory allowance (MB) to use for caching trie nodes in memory + Journal string // Journal of clean cache to survive node restarts + Preimages bool // Flag whether the preimage of trie key is recorded + Zktrie bool // use zktrie + MorphZkTrie bool // use morph zktrie + + HashDB *hashdb.Config // Configs for hash-based scheme + PathDB *pathdb.Config // Configs for experimental path-based scheme +} + +// HashDefaults represents a config for using hash-based scheme with +// default settings. +var HashDefaults = &Config{ + Preimages: false, + HashDB: hashdb.Defaults, +} + +var ZkHashDefaults = &Config{ + Preimages: false, + HashDB: hashdb.Defaults, + Zktrie: true, +} + +// Reader wraps the Node method of a backing trie reader. +type Reader interface { + // Node retrieves the trie node blob with the provided trie identifier, + // node path and the corresponding node hash. No error will be returned + // if the node is not found. + Node(path []byte) ([]byte, error) +} + +// backend defines the methods needed to access/update trie nodes in different +// state scheme. +type backend interface { + // Scheme returns the identifier of used storage scheme. + Scheme() string + + // Size returns the current storage size of the memory cache in front of the + // persistent database layer. + Size() (common.StorageSize, common.StorageSize, common.StorageSize) + + // Commit writes all relevant trie nodes belonging to the specified state + // to disk. Report specifies whether logs will be displayed in info level. + CommitState(root common.Hash, parentRoot common.Hash, blockNumber uint64, report bool) error + + CommitGenesis(root common.Hash) error + + // Close closes the trie database backend and releases all held resources. + Close() error + + // Put nodes to database + Put(k, v []byte) error } // NewDatabase creates a new trie database to store ephemeral trie content before @@ -291,6 +346,10 @@ func NewDatabase(diskdb ethdb.KeyValueStore) *Database { return NewDatabaseWithConfig(diskdb, nil) } +func NewZkDatabase(diskdb ethdb.KeyValueStore) *Database { + return NewDatabaseWithConfig(diskdb, &Config{Zktrie: true}) +} + // NewDatabaseWithConfig creates a new trie database to store ephemeral trie content // before its written out to disk or garbage collected. It also acts as a read cache // for nodes loaded from disk. @@ -313,8 +372,23 @@ func NewDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *Database dirties: map[common.Hash]*cachedNode{{}: { children: make(map[common.Hash]uint16), }}, - rawDirties: make(KvMap), - preimages: preimage, + rawDirties: make(KvMap), + preimages: preimage, + Zktrie: config != nil && config.Zktrie, + MorphZkTrie: config != nil && config.Zktrie && config.MorphZkTrie, + } + if config.HashDB != nil && config.PathDB != nil { + log.Crit("Both 'hash' and 'path' mode are configured") + } + + if db.MorphZkTrie { + log.Info("PathDB", "NewDatabaseWithConfig use morphzktrie", db.MorphZkTrie) + db.backend = pathdb.New(diskdb, config.PathDB) + } else { + log.Info("PathDB", "NewDatabaseWithConfig use zktrie", db.Zktrie) + if db.Zktrie { + db.backend = hashdb.NewZkDatabaseWithConfig(diskdb, config.HashDB) + } } return db } @@ -437,6 +511,10 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) { // This method is extremely expensive and should only be used to validate internal // states in test code. func (db *Database) Nodes() []common.Hash { + if db.backend != nil { + panic("Database not support Nodes()") + } + db.lock.RLock() defer db.lock.RUnlock() @@ -454,6 +532,15 @@ func (db *Database) Nodes() []common.Hash { // and external node(e.g. storage trie root), all internal trie nodes // are referenced together by database itself. func (db *Database) Reference(child common.Hash, parent common.Hash) { + if db.backend != nil { + zdb, ok := db.backend.(*hashdb.ZktrieDatabase) + if !ok { + return + } + zdb.Reference(child, parent) + return + } + db.lock.Lock() defer db.lock.Unlock() @@ -483,6 +570,16 @@ func (db *Database) reference(child common.Hash, parent common.Hash) { // Dereference removes an existing reference from a root node. func (db *Database) Dereference(root common.Hash) { + if db.backend != nil { + zdb, ok := db.backend.(*hashdb.ZktrieDatabase) + if !ok { + log.Error("Databse not support dereference") + return + } + zdb.Dereference(root) + return + } + // Sanity check to ensure that the meta-root is not removed if root == (common.Hash{}) { log.Error("Attempted to dereference the trie cache meta root") @@ -562,6 +659,17 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) { // Note, this method is a non-synchronized mutator. It is unsafe to call this // concurrently with other mutators. func (db *Database) Cap(limit common.StorageSize) error { + if db.backend != nil { + zdb, ok := db.backend.(*hashdb.ZktrieDatabase) + if !ok { + return errors.New("not supported") + } + if db.preimages != nil { + db.preimages.commit(false) + } + return zdb.Cap(limit) + } + // Create a database batch to flush persistent data out. It is important that // outside code doesn't see an inconsistent state (referenced data removed from // memory cache during commit but not yet in persistent storage). This is ensured @@ -640,6 +748,13 @@ func (db *Database) Cap(limit common.StorageSize) error { return nil } +func (db *Database) CommitGenesis(root common.Hash) error { + if db.backend != nil { + db.backend.CommitGenesis(root) + } + return db.Commit(root, true, nil) +} + // Commit iterates over all the children of a particular node, writes them out // to disk, forcefully tearing down all references in both directions. As a side // effect, all pre-images accumulated up to this point are also written. @@ -647,6 +762,14 @@ func (db *Database) Cap(limit common.StorageSize) error { // Note, this method is a non-synchronized mutator. It is unsafe to call this // concurrently with other mutators. func (db *Database) Commit(node common.Hash, report bool, callback func(common.Hash)) error { + if db.backend != nil { + zdb, ok := db.backend.(*hashdb.ZktrieDatabase) + if !ok { + return errors.New("not supported") + } + return zdb.CommitState(node, common.Hash{}, 0, report) + } + // Create a database batch to flush persistent data out. It is important that // outside code doesn't see an inconsistent state (referenced data removed from // memory cache during commit but not yet in persistent storage). This is ensured @@ -835,12 +958,29 @@ func (db *Database) saveCache(dir string, threads int) error { // SaveCache atomically saves fast cache data to the given dir using all // available CPU cores. func (db *Database) SaveCache(dir string) error { + if db.backend != nil { + zdb, ok := db.backend.(*hashdb.ZktrieDatabase) + if !ok { + return errors.New("not supported") + } + return zdb.SaveCache(dir) + } + return db.saveCache(dir, runtime.GOMAXPROCS(0)) } // SaveCachePeriodically atomically saves fast cache data to the given dir with // the specified interval. All dump operation will only use a single CPU core. func (db *Database) SaveCachePeriodically(dir string, interval time.Duration, stopCh <-chan struct{}) { + if db.backend != nil { + zdb, ok := db.backend.(*hashdb.ZktrieDatabase) + if !ok { + return + } + zdb.SaveCachePeriodically(dir, interval, stopCh) + return + } + ticker := time.NewTicker(interval) defer ticker.Stop() @@ -856,10 +996,123 @@ func (db *Database) SaveCachePeriodically(dir string, interval time.Duration, st // EmptyRoot indicate what root is for an empty trie, it depends on its underlying implement (zktrie or common trie) func (db *Database) EmptyRoot() common.Hash { - if db.Zktrie { return common.Hash{} } else { return emptyRoot } } + +// Reader returns a reader for accessing all trie nodes with provided state root. +// An error will be returned if the requested state is not available. +func (db *Database) Reader(blockRoot common.Hash) (Reader, error) { + switch b := db.backend.(type) { + case *hashdb.ZktrieDatabase: + return b.Reader(blockRoot) + case *pathdb.Database: + return b.Reader(blockRoot) + } + return nil, errors.New("unknown backend") +} + +func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockNumber uint64, report bool) error { + if db.backend != nil { + if db.preimages != nil { + db.preimages.commit(true) + } + return db.backend.CommitState(root, parentRoot, blockNumber, report) + } + return db.Commit(root, report, nil) +} + +// Scheme returns the node scheme used in the database. +func (db *Database) Scheme() string { + if db.backend != nil { + return db.backend.Scheme() + } + return rawdb.HashScheme +} + +// Close flushes the dangling preimages to disk and closes the trie database. +// It is meant to be called when closing the blockchain object, so that all +// resources held can be released correctly. +func (db *Database) Close() error { + if db.preimages != nil { + db.preimages.commit(true) + } + + if db.backend != nil { + return db.backend.Close() + } + return nil +} + +// Preimage retrieves a cached trie node pre-image from memory. If it cannot be +// found cached, the method queries the persistent database for the content. +func (db *Database) Preimage(hash common.Hash) []byte { + if db.preimages == nil { + return nil + } + return db.preimages.preimage(hash) +} + +// Journal commits an entire diff hierarchy to disk into a single journal entry. +// This is meant to be used during shutdown to persist the snapshot without +// flattening everything down (bad for reorgs). It's only supported by path-based +// database and will return an error for others. +func (db *Database) Journal(root common.Hash) error { + if db.backend != nil { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return errors.New("not supported") + } + return pdb.Journal(root) + } + return nil +} + +func (db *Database) IsZk() bool { return db.Zktrie } +func (db *Database) IsMorphZk() bool { return db.Zktrie && db.MorphZkTrie } + +// ZkTrie database interface +func (db *Database) UpdatePreimage(preimage []byte, hashField *big.Int) { + if db.backend != nil { + if db.preimages != nil { + // we must copy the input key + preimages := make(map[common.Hash][]byte) + preimages[common.BytesToHash(hashField.Bytes())] = common.CopyBytes(preimage) + db.preimages.insertPreimage(preimages) + } + } +} + +func (db *Database) Put(k, v []byte) error { + if db.backend != nil { + return db.backend.Put(k, v) + } + return nil +} + +func (db *Database) Get(key []byte) ([]byte, error) { + if db.backend != nil { + zdb, ok := db.backend.(*hashdb.ZktrieDatabase) + if !ok { + return nil, errors.New("not supported") + } + return zdb.Get(key) + } + return nil, nil +} + +func (db *Database) GetFrom(root, key []byte) ([]byte, error) { + if db.backend != nil { + pdb, ok := db.backend.(*pathdb.Database) + if !ok { + return nil, errors.New("backend GetFrom not supported") + } + + reader, _ := pdb.Reader(common.BytesToHash(zkt.ReverseByteOrder(root[:]))) + return reader.Node(key) + } + return nil, nil +} diff --git a/trie/hbss2pbss.go b/trie/hbss2pbss.go new file mode 100644 index 000000000..2309b5730 --- /dev/null +++ b/trie/hbss2pbss.go @@ -0,0 +1,176 @@ +package trie + +import ( + "bytes" + "errors" + "fmt" + "os" + "sync" + "time" + + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/core/rawdb" + "github.com/morph-l2/go-ethereum/core/types" + "github.com/morph-l2/go-ethereum/ethdb" + "github.com/morph-l2/go-ethereum/log" + zktrie "github.com/scroll-tech/zktrie/trie" + zkt "github.com/scroll-tech/zktrie/types" +) + +type Hbss2Pbss struct { + zkTrie *ZkTrie + db ethdb.Database + wg *sync.WaitGroup + datadir string + trieCachePath string + errChan chan error +} + +func NewHbss2Pbss(chaindb ethdb.Database, datadir, trieCachePath string) (*Hbss2Pbss, error) { + stateCache := NewDatabaseWithConfig(chaindb, &Config{ + Zktrie: true, + }) + + headBlock := rawdb.ReadHeadBlock(chaindb) + root := headBlock.Root() + log.Info("current head block", "block number", headBlock.NumberU64(), "root", root.Hex()) + zkTrie, err := NewZkTrie(root, stateCache) + if err != nil { + return nil, err + } + + return &Hbss2Pbss{ + db: chaindb, + zkTrie: zkTrie, + datadir: datadir, + trieCachePath: trieCachePath, + errChan: make(chan error), + wg: &sync.WaitGroup{}, + }, nil +} + +func (h2p *Hbss2Pbss) Run() error { + if _, err := os.Stat(h2p.trieCachePath); os.IsNotExist(err) { + log.Warn("The clean trie cache is not found.") + } else { + os.RemoveAll(h2p.trieCachePath) + log.Info("Deleted trie clean cache", "path", h2p.trieCachePath) + } + + // Write genesis in new state db + h2p.handleGenesis() + + // Convert hbss state db to new state db + root, err := h2p.zkTrie.Tree().Root() + if err != nil { + return err + } + start := time.Now() + go func() { + defer func() { + close(h2p.errChan) + }() + + h2p.concurrentTraversal(root, []bool{}, common.Hash{}) + h2p.wg.Wait() + }() + + for err := range h2p.errChan { // wait until p.errChan is closed, or an error occurs + if err != nil { + return err + } + } + log.Info("Hbss2Pbss complete", "elapsed", common.PrettyDuration(time.Since(start))) + + return nil +} + +func (h2p *Hbss2Pbss) handleGenesis() error { + genesisHash := rawdb.ReadCanonicalHash(h2p.db, 0) + if genesisHash == (common.Hash{}) { + return errors.New("missing genesis hash") + } + genesis := rawdb.ReadBlock(h2p.db, genesisHash, 0) + if genesis == nil { + return errors.New("missing genesis block") + } + genesisRoot := genesis.Root() + + log.Info(":", "genesistRoot:", genesisRoot.Hex()) + + h2p.concurrentTraversal(zkt.NewHashFromBytes(genesisRoot[:]), []bool{}, common.Hash{}) + + return nil +} + +func (h2p *Hbss2Pbss) writeNode(pathKey []bool, n *zktrie.Node, owner common.Hash) { + if owner == (common.Hash{}) { + h, _ := n.NodeHash() + rawdb.WriteAccountTrieNode(h2p.db, zkt.PathToKey(pathKey)[:], n.CanonicalValue()) + log.Debug("WriteNodes account trie node", "type", n.Type, "path", zkt.PathToString(pathKey), "hash", h.Hex()) + } else { + rawdb.WriteStorageTrieNode(h2p.db, owner, zkt.PathToKey(pathKey)[:], n.CanonicalValue()) + log.Debug("WriteNodes storage trie node", "owner", owner.Hex(), "type", n.Type, "path", zkt.PathToString(pathKey)) + } +} + +func (h2p *Hbss2Pbss) concurrentTraversal(nodeHash *zkt.Hash, path []bool, owner common.Hash) error { + n, err := h2p.zkTrie.Tree().GetNode(nodeHash) + if err != nil { + return err + } + + switch n.Type { + case zktrie.NodeTypeEmpty, zktrie.NodeTypeEmpty_New: + h2p.writeNode(path, n, owner) + return nil + case zktrie.NodeTypeLeaf_New: + h2p.writeNode(path, n, owner) + + if bytes.Equal(owner[:], common.Hash{}.Bytes()) { + data, err := types.UnmarshalStateAccount(n.Data()) + if err != nil { + h2p.errChan <- err + return err + } + + if data.CodeSize > 0 { + if !bytes.Equal(data.Root[:], common.Hash{}.Bytes()) { + // log.Info("/-------------------\\") + // log.Info("concurrentTraversal", "owner", owner.Hex(), "path", zkt.PathToString(path), "nodeHash", nodeHash.Hex(), "key", n.NodeKey.Hex(), "root", data.Root.Hex()) + + // codeHash := data.KeccakCodeHash + h2p.concurrentTraversal(zkt.NewHashFromBytes(data.Root[:]), []bool{}, common.BytesToHash(n.NodeKey[:])) + // h2p.wg.Add(1) + // go func(root, o common.Hash) { + // defer h2p.wg.Done() + // log.Info("/-------------------\\") + // log.Info("concurrentTraversal", "owner", owner.Hex(), "path", zkt.PathToString(path), "nodeHash", nodeHash.Hex(), "key", n.NodeKey.Hex()) + // h2p.concurrentTraversal(zkt.NewHashFromBytes(root[:]), []bool{}, o) + // log.Info("\\-------------------/") + // }(data.Root, common.BytesToHash(n.NodeKey.Bytes())) + // log.Info("\\-------------------/") + } + } + } + + return nil + case zktrie.NodeTypeBranch_0, zktrie.NodeTypeBranch_1, zktrie.NodeTypeBranch_2, zktrie.NodeTypeBranch_3: + h2p.writeNode(path, n, owner) + + leftErr := h2p.concurrentTraversal(n.ChildL, append(path, false), owner) + if leftErr != nil { + h2p.errChan <- leftErr + return leftErr + } + rightErr := h2p.concurrentTraversal(n.ChildR, append(path, true), owner) + if rightErr != nil { + h2p.errChan <- rightErr + return rightErr + } + default: + return errors.New(fmt.Sprint("unexpected node type", n.Type)) + } + + return nil +} diff --git a/trie/morph_zk_trie.go b/trie/morph_zk_trie.go new file mode 100644 index 000000000..940fb3d60 --- /dev/null +++ b/trie/morph_zk_trie.go @@ -0,0 +1,228 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + + zkt "github.com/scroll-tech/zktrie/types" + varienttrie "github.com/scroll-tech/zktrie/varienttrie" + + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/core/types" + "github.com/morph-l2/go-ethereum/crypto/poseidon" + "github.com/morph-l2/go-ethereum/ethdb" + "github.com/morph-l2/go-ethereum/log" +) + +// wrap zktrie for trie interface +type MorphZkTrie struct { + *varienttrie.ZkTrie + db *Database +} + +func init() { + zkt.InitHashScheme(poseidon.HashFixedWithDomain) +} + +// NewZkTrie creates a trie +// NewZkTrie bypasses all the buffer mechanism in *Database, it directly uses the +// underlying diskdb +func NewMorphZkTrie(root common.Hash, origin common.Hash, db *Database, prefix []byte) (*MorphZkTrie, error) { + tr, err := varienttrie.NewZkTrieWithPrefix(*zkt.NewByte32FromBytes(root.Bytes()), *zkt.NewByte32FromBytes(origin.Bytes()), db, prefix) + if err != nil { + return nil, err + } + return &MorphZkTrie{tr, db}, nil +} + +// Get returns the value for key stored in the trie. +// The value bytes must not be modified by the caller. +func (t *MorphZkTrie) Get(key []byte) []byte { + sanityCheckByte32Key(key) + res, err := t.TryGet(key) + if err != nil { + log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + } + return res +} + +// TryUpdateAccount will abstract the write of an account to the +// secure trie. +func (t *MorphZkTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { + sanityCheckByte32Key(key) + value, flag := acc.MarshalFields() + return t.ZkTrie.TryUpdate(key, flag, value) +} + +// Update associates key with value in the trie. Subsequent calls to +// Get will return value. If value has length zero, any existing value +// is deleted from the trie and calls to Get will return nil. +// +// The value bytes must not be modified by the caller while they are +// stored in the trie. +func (t *MorphZkTrie) Update(key, value []byte) { + if err := t.TryUpdate(key, value); err != nil { + log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + } +} + +// NOTE: value is restricted to length of bytes32. +// we override the underlying zktrie's TryUpdate method +func (t *MorphZkTrie) TryUpdate(key, value []byte) error { + sanityCheckByte32Key(key) + return t.ZkTrie.TryUpdate(key, 1, []zkt.Byte32{*zkt.NewByte32FromBytes(value)}) +} + +// Delete removes any existing value for key from the trie. +func (t *MorphZkTrie) Delete(key []byte) { + sanityCheckByte32Key(key) + if err := t.TryDelete(key); err != nil { + log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + } +} + +// GetKey returns the preimage of a hashed key that was +// previously used to store a value. +func (t *MorphZkTrie) GetKey(kHashBytes []byte) []byte { + // TODO: use a kv cache in memory + k, err := zkt.NewBigIntFromHashBytes(kHashBytes) + if err != nil { + log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + } + if t.db.preimages != nil { + return t.db.preimages.preimage(common.BytesToHash(k.Bytes())) + } + return nil +} + +// Commit writes all nodes and the secure hash pre-images to the trie's database. +// Nodes are stored with their sha3 hash as the key. +// +// Committing flushes nodes from memory. Subsequent Get calls will load nodes +// from the database. +func (t *MorphZkTrie) Commit(LeafCallback) (common.Hash, int, error) { + if err := t.ZkTrie.Commit(); err != nil { + return common.Hash{}, 0, err + } + // in current implmentation, every update of trie already writes into database + // so Commmit does nothing + return t.Hash(), 0, nil +} + +// Hash returns the root hash of SecureBinaryTrie. It does not write to the +// database and can be used even if the trie doesn't have one. +func (t *MorphZkTrie) Hash() common.Hash { + var hash common.Hash + hash.SetBytes(t.ZkTrie.Hash()) + return hash +} + +// Copy returns a copy of SecureBinaryTrie. +func (t *MorphZkTrie) Copy() *MorphZkTrie { + return &MorphZkTrie{t.ZkTrie.Copy(), t.db} +} + +// NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration +// starts at the key after the given start key. +func (t *MorphZkTrie) NodeIterator(start []byte) NodeIterator { + /// FIXME + panic("not implemented") +} + +// hashKey returns the hash of key as an ephemeral buffer. +// The caller must not hold onto the return value because it will become +// invalid on the next call to hashKey or secKey. +/*func (t *ZkTrie) hashKey(key []byte) []byte { + if len(key) != 32 { + panic("non byte32 input to hashKey") + } + low16 := new(big.Int).SetBytes(key[:16]) + high16 := new(big.Int).SetBytes(key[16:]) + hash, err := poseidon.Hash([]*big.Int{low16, high16}) + if err != nil { + panic(err) + } + return hash.Bytes() +} +*/ + +// Prove constructs a merkle proof for key. The result contains all encoded nodes +// on the path to the value at key. The value itself is also included in the last +// node and can be retrieved by verifying the proof. +// +// If the trie does not contain a value for key, the returned proof contains all +// nodes of the longest existing prefix of the key (at least the root node), ending +// with the node that proves the absence of the key. +func (t *MorphZkTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { + err := t.ZkTrie.Prove(key, fromLevel, func(n *varienttrie.Node) error { + nodeHash, err := n.NodeHash() + if err != nil { + return err + } + + if n.Type == varienttrie.NodeTypeLeaf_New { + preImage := t.GetKey(n.NodeKey.Bytes()) + if len(preImage) > 0 { + n.KeyPreimage = &zkt.Byte32{} + copy(n.KeyPreimage[:], preImage) + //return fmt.Errorf("key preimage not found for [%x] ref %x", n.NodeKey.Bytes(), k.Bytes()) + } + } + return proofDb.Put(nodeHash[:], n.Value()) + }) + if err != nil { + return err + } + + // we put this special kv pair in db so we can distinguish the type and + // make suitable Proof + return proofDb.Put(magicHash, varienttrie.ProofMagicBytes()) +} + +// VerifyProof checks merkle proofs. The given proof must contain the value for +// key in a trie with the given root hash. VerifyProof returns an error if the +// proof contains invalid trie nodes or the wrong value. +func VerifyProofSMT2(rootHash common.Hash, key []byte, proofDb ethdb.KeyValueReader) (value []byte, err error) { + h := zkt.NewHashFromBytes(rootHash.Bytes()) + k, err := zkt.ToSecureKey(key) + if err != nil { + return nil, err + } + + proof, n, err := varienttrie.BuildZkTrieProof(h, k, len(key)*8, func(key *zkt.Hash) (*varienttrie.Node, error) { + buf, _ := proofDb.Get(key[:]) + if buf == nil { + return nil, varienttrie.ErrKeyNotFound + } + n, err := varienttrie.NewNodeFromBytes(buf) + return n, err + }) + + if err != nil { + // do not contain the key + return nil, err + } else if !proof.Existence { + return nil, nil + } + + if varienttrie.VerifyProofZkTrie(h, proof, n) { + return n.Data(), nil + } else { + return nil, fmt.Errorf("bad proof node %v", proof) + } +} diff --git a/trie/zk_trie.go b/trie/zk_trie.go index 2d01fcc33..b46a6dda2 100644 --- a/trie/zk_trie.go +++ b/trie/zk_trie.go @@ -34,7 +34,7 @@ var magicHash []byte = []byte("THIS IS THE MAGIC INDEX FOR ZKTRIE") // wrap zktrie for trie interface type ZkTrie struct { *zktrie.ZkTrie - db *ZktrieDatabase + db *Database } func init() { @@ -50,7 +50,7 @@ func sanityCheckByte32Key(b []byte) { // NewZkTrie creates a trie // NewZkTrie bypasses all the buffer mechanism in *Database, it directly uses the // underlying diskdb -func NewZkTrie(root common.Hash, db *ZktrieDatabase) (*ZkTrie, error) { +func NewZkTrie(root common.Hash, db *Database) (*ZkTrie, error) { tr, err := zktrie.NewZkTrie(*zkt.NewByte32FromBytes(root.Bytes()), db) if err != nil { return nil, err @@ -112,8 +112,8 @@ func (t *ZkTrie) GetKey(kHashBytes []byte) []byte { if err != nil { log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) } - if t.db.db.preimages != nil { - return t.db.db.preimages.preimage(common.BytesToHash(k.Bytes())) + if t.db.preimages != nil { + return t.db.preimages.preimage(common.BytesToHash(k.Bytes())) } return nil } diff --git a/trie/zk_trie_proof_test.go b/trie/zk_trie_proof_test.go index 5b23ae7b8..1d42f60a2 100644 --- a/trie/zk_trie_proof_test.go +++ b/trie/zk_trie_proof_test.go @@ -63,7 +63,7 @@ func verifyValue(proveVal []byte, vPreimage []byte) bool { } func TestSMTOneElementProof(t *testing.T) { - tr, _ := NewZkTrie(common.Hash{}, NewZktrieDatabase((memorydb.New()))) + tr, _ := NewZkTrie(common.Hash{}, NewZkDatabase((memorydb.New()))) mt := &zkTrieImplTestWrapper{tr.Tree()} err := mt.UpdateWord( zkt.NewByte32FromBytesPaddingZero(bytes.Repeat([]byte("k"), 32)), @@ -148,7 +148,7 @@ func TestSMTBadProof(t *testing.T) { // Tests that missing keys can also be proven. The test explicitly uses a single // entry trie and checks for missing keys both before and after the single entry. func TestSMTMissingKeyProof(t *testing.T) { - tr, _ := NewZkTrie(common.Hash{}, NewZktrieDatabase((memorydb.New()))) + tr, _ := NewZkTrie(common.Hash{}, NewZkDatabase((memorydb.New()))) mt := &zkTrieImplTestWrapper{tr.Tree()} err := mt.UpdateWord( zkt.NewByte32FromBytesPaddingZero(bytes.Repeat([]byte("k"), 32)), @@ -180,7 +180,7 @@ func TestSMTMissingKeyProof(t *testing.T) { } func randomZktrie(t *testing.T, n int) (*ZkTrie, map[string]*kv) { - tr, err := NewZkTrie(common.Hash{}, NewZktrieDatabase((memorydb.New()))) + tr, err := NewZkTrie(common.Hash{}, NewZkDatabase((memorydb.New()))) if err != nil { panic(err) } @@ -210,7 +210,7 @@ func randomZktrie(t *testing.T, n int) (*ZkTrie, map[string]*kv) { // Tests that new "proof trace" feature func TestProofWithDeletion(t *testing.T) { - tr, _ := NewZkTrie(common.Hash{}, NewZktrieDatabase((memorydb.New()))) + tr, _ := NewZkTrie(common.Hash{}, NewZkDatabase((memorydb.New()))) mt := &zkTrieImplTestWrapper{tr.Tree()} key1 := bytes.Repeat([]byte("l"), 32) key2 := bytes.Repeat([]byte("m"), 32) diff --git a/trie/zk_trie_test.go b/trie/zk_trie_test.go index f93e067dd..0c413c818 100644 --- a/trie/zk_trie_test.go +++ b/trie/zk_trie_test.go @@ -30,6 +30,7 @@ import ( zkt "github.com/scroll-tech/zktrie/types" "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/core/rawdb" "github.com/morph-l2/go-ethereum/ethdb/leveldb" "github.com/morph-l2/go-ethereum/ethdb/memorydb" ) @@ -37,19 +38,16 @@ import ( func newEmptyZkTrie() *ZkTrie { trie, _ := NewZkTrie( common.Hash{}, - &ZktrieDatabase{ - db: NewDatabaseWithConfig(memorydb.New(), - &Config{Preimages: true}), - prefix: []byte{}, - }, + NewDatabaseWithConfig(memorydb.New(), + &Config{Preimages: true, Zktrie: true}), ) return trie } // makeTestSecureTrie creates a large enough secure trie for testing. -func makeTestZkTrie() (*ZktrieDatabase, *ZkTrie, map[string][]byte) { +func makeTestZkTrie() (*Database, *ZkTrie, map[string][]byte) { // Create an empty trie - triedb := NewZktrieDatabase(memorydb.New()) + triedb := NewZkDatabase(memorydb.New()) trie, _ := NewZkTrie(common.Hash{}, triedb) // Fill it with some arbitrary data @@ -173,9 +171,9 @@ const benchElemCountZk = 10000 func BenchmarkZkTrieGet(b *testing.B) { _, tmpdb := tempDBZK(b) - zkTrie, _ := NewZkTrie(common.Hash{}, NewZktrieDatabaseFromTriedb(tmpdb)) + zkTrie, _ := NewZkTrie(common.Hash{}, tmpdb) defer func() { - ldb := zkTrie.db.db.diskdb.(*leveldb.Database) + ldb := zkTrie.db.diskdb.(*leveldb.Database) ldb.Close() os.RemoveAll(ldb.Path()) }() @@ -188,7 +186,7 @@ func BenchmarkZkTrieGet(b *testing.B) { assert.NoError(b, err) } - zkTrie.db.db.Commit(common.Hash{}, true, nil) + zkTrie.db.Commit(common.Hash{}, true, nil) b.ResetTimer() for i := 0; i < b.N; i++ { binary.LittleEndian.PutUint64(k, uint64(i)) @@ -200,9 +198,9 @@ func BenchmarkZkTrieGet(b *testing.B) { func BenchmarkZkTrieUpdate(b *testing.B) { _, tmpdb := tempDBZK(b) - zkTrie, _ := NewZkTrie(common.Hash{}, NewZktrieDatabaseFromTriedb(tmpdb)) + zkTrie, _ := NewZkTrie(common.Hash{}, tmpdb) defer func() { - ldb := zkTrie.db.db.diskdb.(*leveldb.Database) + ldb := zkTrie.db.diskdb.(*leveldb.Database) ldb.Close() os.RemoveAll(ldb.Path()) }() @@ -219,7 +217,7 @@ func BenchmarkZkTrieUpdate(b *testing.B) { binary.LittleEndian.PutUint64(k, benchElemCountZk/2) //zkTrie.Commit(nil) - zkTrie.db.db.Commit(common.Hash{}, true, nil) + zkTrie.db.Commit(common.Hash{}, true, nil) b.ResetTimer() for i := 0; i < b.N; i++ { binary.LittleEndian.PutUint64(k, uint64(i)) @@ -264,3 +262,76 @@ func TestZkTrieDelete(t *testing.T) { assert.Equal(t, hashes[i].Hex(), hash.Hex()) } } + +func TestMorphAccountTrie(t *testing.T) { + // dir := "/data/morph/ethereum/geth/chaindata" + dir := "" + if len(dir) == 0 { + return + } + + db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false) + if err != nil { + t.Fatalf("error opening database at %v: %v", dir, err) + } + + stored := rawdb.ReadCanonicalHash(db, 0) + header := rawdb.ReadHeader(db, stored, 0) + + println("stored:", stored.Hex()) + println("header:", header.Root.Hex()) + + headBlock := rawdb.ReadHeadBlock(db) + println("headBlock:", headBlock.NumberU64()) + root := headBlock.Root() + println("head state root:", root.Hex()) + + _, diskroot := rawdb.ReadAccountTrieNode(db, zkt.TrieRootPathKey[:]) + // diskroot = types.TrieRootHash(diskroot) + println("test state root:", diskroot.Hex()) + + // trieDb := NewDatabaseWithConfig(db, &Config{Zktrie: true, MorphZkTrie: true, PathDB: pathdb.Defaults}) + // varientTrie, _ := varienttrie.NewZkTrieWithPrefix(*zkt.NewByte32FromBytes(root.Bytes()), trieDb, rawdb.TrieNodeAccountPrefix) + + // calroot, _ := varientTrie.Tree().Root() + // println("recalculate state root:", calroot.Hex()) + + // var buffer bytes.Buffer + // err = varientTrie.Tree().GraphViz(&buffer, nil) + // assert.NoError(t, err) +} + +func TestZkSecureKey(t *testing.T) { + addr1 := common.HexToAddress("0x5300000000000000000000000000000000000004").Bytes() + addr2 := common.HexToAddress("0xc0D3c0D3c0D3c0d3c0d3C0d3C0d3c0D3C0D30004").Bytes() + addr3 := common.HexToAddress("0x523bff68043C818e9b449dd3Bee8ecCfa85D7E50").Bytes() + addr4 := common.HexToAddress("0x803DcE4D3f4Ae2e17AF6C51343040dEe320C149D").Bytes() + addr5 := common.HexToAddress("0x530000000000000000000000000000000000000D").Bytes() + addr6 := common.HexToAddress("0x530000000000000000000000000000000000000b").Bytes() + + k, _ := zkt.ToSecureKey(addr1) + hk := zkt.NewHashFromBigInt(k) + ck := common.BigToHash(k) + assert.Equal(t, hk.Hex(), "20e9fb498ff9c35246d527da24aa1710d2cc9b055ecf9a95a8a2a11d3d836cdf") + assert.Equal(t, ck.Hex(), "0x20e9fb498ff9c35246d527da24aa1710d2cc9b055ecf9a95a8a2a11d3d836cdf") + + k, _ = zkt.ToSecureKey(addr2) + hk = zkt.NewHashFromBigInt(k) + assert.Equal(t, hk.Hex(), "22e3957366ea5ad008a980d4bd48e10b72472a7838a7187d04b926fe80fd9c06") + + k, _ = zkt.ToSecureKey(addr3) + hk = zkt.NewHashFromBigInt(k) + assert.Equal(t, hk.Hex(), "0accc4c88536715cc403dbec52286fc5a0ee01e7e0f7257a2aa84f57b706b0e0") + + k, _ = zkt.ToSecureKey(addr4) + hk = zkt.NewHashFromBigInt(k) + assert.Equal(t, hk.Hex(), "1616e66d32ff1b5ad90c5e7db84045202dc719261bf13b0bb543873dcc135cc6") + + k, _ = zkt.ToSecureKey(addr5) + hk = zkt.NewHashFromBigInt(k) + assert.Equal(t, hk.Hex(), "19fd1d3fef5e6662c505f44512137d7f0fd6d5637856b4abd05a6438e2f07a7f") + + k, _ = zkt.ToSecureKey(addr6) + hk = zkt.NewHashFromBigInt(k) + assert.Equal(t, hk.Hex(), "0c3730cadca93736aec00976a03becc8b73c60233161bc87e213e5fdf3fe4c85") +} diff --git a/triedb/hashdb/metrics.go b/triedb/hashdb/metrics.go new file mode 100644 index 000000000..527c6ba75 --- /dev/null +++ b/triedb/hashdb/metrics.go @@ -0,0 +1,43 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package hashdb + +import "github.com/morph-l2/go-ethereum/metrics" + +var ( + memcacheCleanHitMeter = metrics.NewRegisteredMeter("trie/memcache/clean/hit", nil) + memcacheCleanMissMeter = metrics.NewRegisteredMeter("trie/memcache/clean/miss", nil) + memcacheCleanReadMeter = metrics.NewRegisteredMeter("trie/memcache/clean/read", nil) + memcacheCleanWriteMeter = metrics.NewRegisteredMeter("trie/memcache/clean/write", nil) + + memcacheDirtyHitMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/hit", nil) + memcacheDirtyMissMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/miss", nil) + memcacheDirtyReadMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/read", nil) + memcacheDirtyWriteMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/write", nil) + + memcacheFlushTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/flush/time", nil) + memcacheFlushNodesMeter = metrics.NewRegisteredMeter("trie/memcache/flush/nodes", nil) + memcacheFlushSizeMeter = metrics.NewRegisteredMeter("trie/memcache/flush/size", nil) + + memcacheGCTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/gc/time", nil) + memcacheGCNodesMeter = metrics.NewRegisteredMeter("trie/memcache/gc/nodes", nil) + memcacheGCSizeMeter = metrics.NewRegisteredMeter("trie/memcache/gc/size", nil) + + memcacheCommitTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/commit/time", nil) + memcacheCommitNodesMeter = metrics.NewRegisteredMeter("trie/memcache/commit/nodes", nil) + memcacheCommitSizeMeter = metrics.NewRegisteredMeter("trie/memcache/commit/size", nil) +) diff --git a/triedb/hashdb/zk_trie_database.go b/triedb/hashdb/zk_trie_database.go new file mode 100644 index 000000000..a02511462 --- /dev/null +++ b/triedb/hashdb/zk_trie_database.go @@ -0,0 +1,232 @@ +package hashdb + +import ( + "runtime" + "sync" + "time" + + "github.com/VictoriaMetrics/fastcache" + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/core/rawdb" + "github.com/morph-l2/go-ethereum/ethdb" + "github.com/morph-l2/go-ethereum/log" + "github.com/morph-l2/go-ethereum/triedb/types" + "github.com/syndtr/goleveldb/leveldb" + + zktrie "github.com/scroll-tech/zktrie/trie" +) + +// Config defines all necessary options for database. +type Config struct { + Cache int // Memory allowance (MB) to use for caching trie nodes in memory + Journal string // Journal of clean cache to survive node restarts +} + +// Defaults is the default setting for database if it's not specified. +// Notably, clean cache is disabled explicitly, +var Defaults = &Config{ + // Explicitly set clean cache size to 0 to avoid creating fastcache, + // otherwise database must be closed when it's no longer needed to + // prevent memory leak. + Cache: 0, +} + +type ZktrieDatabase struct { + diskdb ethdb.KeyValueStore // Persistent storage for matured trie nodes + cleans *fastcache.Cache // GC friendly memory cache of clean node RLPs + prefix []byte + dirties types.KvMap + + lock sync.RWMutex + dirtiesSize common.StorageSize // Storage size of the dirty node cache (exc. metadata) +} + +func NewZkDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *ZktrieDatabase { + if config == nil { + config = Defaults + } + + var cleans *fastcache.Cache + if config != nil && config.Cache > 0 { + if config.Journal == "" { + cleans = fastcache.New(config.Cache * 1024 * 1024) + } else { + cleans = fastcache.LoadFromFileOrNew(config.Journal, config.Cache*1024*1024) + } + } + + return &ZktrieDatabase{ + diskdb: diskdb, + cleans: cleans, + dirties: make(types.KvMap), + } +} + +func (db *ZktrieDatabase) Scheme() string { return rawdb.ZkHashScheme } + +func (db *ZktrieDatabase) Size() (common.StorageSize, common.StorageSize, common.StorageSize) { + db.lock.RLock() + defer db.lock.RUnlock() + + // db.dirtiesSize only contains the useful data in the cache, but when reporting + // the total memory consumption, the maintenance metadata is also needed to be + // counted. + var metadataSize = common.StorageSize(len(db.dirties)) + return 0, 0, db.dirtiesSize + metadataSize +} + +func (db *ZktrieDatabase) CommitState(root common.Hash, parentRoot common.Hash, blockNumber uint64, report bool) error { + beforeDirtyCount, beforeDirtySize := len(db.dirties), db.dirtiesSize + + start := time.Now() + if err := db.commitAllDirties(); err != nil { + return err + } + memcacheCommitTimeTimer.Update(time.Since(start)) + // memcacheCommitBytesMeter.Mark(int64(beforeDirtySize - db.dirtiesSize)) + memcacheCommitNodesMeter.Mark(int64(beforeDirtyCount - len(db.dirties))) + + logger := log.Debug + if report { + logger = log.Info + } + logger( + "Persisted trie from memory database", + "nodes", beforeDirtyCount-len(db.dirties), + "size", beforeDirtySize-db.dirtiesSize, + "time", time.Since(start), + "livenodes", len(db.dirties), + "livesize", db.dirtiesSize, + ) + return nil +} + +func (db *ZktrieDatabase) CommitGenesis(root common.Hash) error { + return db.CommitState(root, common.Hash{}, 0, true) +} + +func (db *ZktrieDatabase) commitAllDirties() error { + batch := db.diskdb.NewBatch() + + db.lock.Lock() + for _, v := range db.dirties { + batch.Put(v.K, v.V) + } + for k := range db.dirties { + delete(db.dirties, k) + } + db.lock.Unlock() + + if err := batch.Write(); err != nil { + return err + } + + batch.Reset() + return nil +} + +func (db *ZktrieDatabase) Close() error { return nil } +func (db *ZktrieDatabase) Cap(_ common.StorageSize) error { return nil } +func (db *ZktrieDatabase) Reference(_ common.Hash, _ common.Hash) {} +func (db *ZktrieDatabase) Dereference(_ common.Hash) {} + +func (db *ZktrieDatabase) Node(hash common.Hash) ([]byte, error) { + panic("ZktrieDatabase not implement Node()") +} + +// Put saves a key:value into the Storage +func (db *ZktrieDatabase) Put(k, v []byte) error { + k = common.ReverseBytes(k) + + db.lock.Lock() + db.dirties.Put(k, v) + db.lock.Unlock() + + if db.cleans != nil { + db.cleans.Set(k[:], v) + memcacheCleanMissMeter.Mark(1) + memcacheCleanWriteMeter.Mark(int64(len(v))) + } + return nil +} + +// Get retrieves a value from a key in the Storage +func (db *ZktrieDatabase) Get(key []byte) ([]byte, error) { + key = common.ReverseBytes(key[:]) + + db.lock.RLock() + value, ok := db.dirties.Get(key) + db.lock.RUnlock() + if ok { + return value, nil + } + + if db.cleans != nil { + if enc := db.cleans.Get(nil, key); enc != nil { + memcacheCleanHitMeter.Mark(1) + memcacheCleanReadMeter.Mark(int64(len(enc))) + return enc, nil + } + } + + v, err := db.diskdb.Get(key) + if err == leveldb.ErrNotFound { + return nil, zktrie.ErrKeyNotFound + } + if err != nil && db.cleans != nil { + db.cleans.Set(key[:], v) + memcacheCleanMissMeter.Mark(1) + memcacheCleanWriteMeter.Mark(int64(len(v))) + } + return v, err +} + +// saveCache saves clean state cache to given directory path +// using specified CPU cores. +func (db *ZktrieDatabase) saveCache(dir string, threads int) error { + if db.cleans == nil { + return nil + } + log.Info("Writing clean trie cache to disk", "path", dir, "threads", threads) + + start := time.Now() + err := db.cleans.SaveToFileConcurrent(dir, threads) + if err != nil { + log.Error("Failed to persist clean trie cache", "error", err) + return err + } + log.Info("Persisted the clean trie cache", "path", dir, "elapsed", common.PrettyDuration(time.Since(start))) + return nil +} + +// SaveCache atomically saves fast cache data to the given dir using all +// available CPU cores. +func (db *ZktrieDatabase) SaveCache(dir string) error { + return db.saveCache(dir, runtime.GOMAXPROCS(0)) +} + +// SaveCachePeriodically atomically saves fast cache data to the given dir with +// the specified interval. All dump operation will only use a single CPU core. +func (db *ZktrieDatabase) SaveCachePeriodically(dir string, interval time.Duration, stopCh <-chan struct{}) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + db.saveCache(dir, 1) + case <-stopCh: + return + } + } +} + +func (db *ZktrieDatabase) Reader(root common.Hash) (*zkReader, error) { + return &zkReader{db: db}, nil +} + +type zkReader struct{ db *ZktrieDatabase } + +func (z zkReader) Node(path []byte) ([]byte, error) { + return z.db.Get(path) +} diff --git a/triedb/pathdb/asyncnodebuffer.go b/triedb/pathdb/asyncnodebuffer.go new file mode 100644 index 000000000..9d256ebab --- /dev/null +++ b/triedb/pathdb/asyncnodebuffer.go @@ -0,0 +1,357 @@ +package pathdb + +import ( + "bytes" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/VictoriaMetrics/fastcache" + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/core/rawdb" + "github.com/morph-l2/go-ethereum/ethdb" + "github.com/morph-l2/go-ethereum/log" + dbtypes "github.com/morph-l2/go-ethereum/triedb/types" +) + +var _ trienodebuffer = &asyncnodebuffer{} + +// asyncnodebuffer implement trienodebuffer interface, and async the nodecache +// to disk. +type asyncnodebuffer struct { + mux sync.RWMutex + current *nodecache + background *nodecache + isFlushing atomic.Bool + stopFlushing atomic.Bool +} + +// newAsyncNodeBuffer initializes the async node buffer with the provided nodes. +func newAsyncNodeBuffer(limit int, nodes dbtypes.KvMap, layers uint64) *asyncnodebuffer { + if nodes == nil { + nodes = make(dbtypes.KvMap) + } + var size uint64 + for _, v := range nodes { + size += uint64(len(v.K) + len(v.V)) + } + + return &asyncnodebuffer{ + current: newNodeCache(uint64(limit), size, nodes, layers), + background: newNodeCache(uint64(limit), 0, make(dbtypes.KvMap), 0), + } +} + +// node retrieves the trie node with given node info. +func (a *asyncnodebuffer) node(path []byte) ([]byte, error) { + a.mux.RLock() + defer a.mux.RUnlock() + + node, err := a.current.node(path) + if err != nil { + return nil, err + } + if node == nil { + return a.background.node(path) + } + return node, nil +} + +// commit merges the dirty nodes into the nodebuffer. This operation won't take +// the ownership of the nodes map which belongs to the bottom-most diff layer. +// It will just hold the node references from the given map which are safe to +// copy. +func (a *asyncnodebuffer) commit(nodes dbtypes.KvMap) trienodebuffer { + a.mux.Lock() + defer a.mux.Unlock() + + err := a.current.commit(nodes) + if err != nil { + log.Crit("[BUG] Failed to commit nodes to asyncnodebuffer", "error", err) + } + return a +} + +// setSize is unsupported in asyncnodebuffer, due to the double buffer, blocking will occur. +func (a *asyncnodebuffer) setSize(size int, db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64) error { + return errors.New("not supported") +} + +// reset cleans up the disk cache. +func (a *asyncnodebuffer) reset() { + a.mux.Lock() + defer a.mux.Unlock() + + a.current.reset() + a.background.reset() +} + +// empty returns an indicator if nodebuffer contains any state transition inside. +func (a *asyncnodebuffer) empty() bool { + a.mux.RLock() + defer a.mux.RUnlock() + + return a.current.empty() && a.background.empty() +} + +// flush persists the in-memory dirty trie node into the disk if the configured +// memory threshold is reached. Note, all data must be written atomically. +func (a *asyncnodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64, force bool) error { + a.mux.Lock() + defer a.mux.Unlock() + + if a.stopFlushing.Load() { + return nil + } + + if force { + for { + if atomic.LoadUint64(&a.background.immutable) == 1 { + time.Sleep(time.Duration(DefaultBackgroundFlushInterval) * time.Second) + log.Info("Waiting background memory table flushed into disk for forcing flush node buffer") + continue + } + atomic.StoreUint64(&a.current.immutable, 1) + return a.current.flush(db, clean, id) + } + } + + if a.current.size < a.current.limit { + return nil + } + + // background flush doing + if atomic.LoadUint64(&a.background.immutable) == 1 { + return nil + } + + atomic.StoreUint64(&a.current.immutable, 1) + a.current, a.background = a.background, a.current + + a.isFlushing.Store(true) + go func(persistID uint64) { + defer a.isFlushing.Store(false) + for { + err := a.background.flush(db, clean, persistID) + if err == nil { + log.Debug("Succeed to flush background nodecache to disk", "state_id", persistID) + return + } + log.Error("Failed to flush background nodecache to disk", "state_id", persistID, "error", err) + } + }(id) + return nil +} + +func (a *asyncnodebuffer) waitAndStopFlushing() { + a.stopFlushing.Store(true) + for a.isFlushing.Load() { + time.Sleep(time.Second) + log.Warn("Waiting background memory table flushed into disk") + } +} + +func (a *asyncnodebuffer) getAllNodes() dbtypes.KvMap { + a.mux.Lock() + defer a.mux.Unlock() + + cached, err := a.current.merge(a.background) + if err != nil { + log.Crit("[BUG] Failed to merge node cache under revert async node buffer", "error", err) + } + return cached.nodes +} + +func (a *asyncnodebuffer) getLayers() uint64 { + a.mux.RLock() + defer a.mux.RUnlock() + + return a.current.layers + a.background.layers +} + +func (a *asyncnodebuffer) getSize() (uint64, uint64) { + a.mux.RLock() + defer a.mux.RUnlock() + + return a.current.size, a.background.size +} + +type nodecache struct { + layers uint64 // The number of diff layers aggregated inside + size uint64 // The size of aggregated writes + limit uint64 // The maximum memory allowance in bytes + nodes dbtypes.KvMap // The dirty node set, mapped by owner and path + immutable uint64 // The flag equal 1, flush nodes to disk background +} + +func newNodeCache(limit, size uint64, nodes dbtypes.KvMap, layers uint64) *nodecache { + return &nodecache{ + layers: layers, + size: size, + limit: limit, + nodes: nodes, + immutable: 0, + } +} + +func (nc *nodecache) node(path []byte) ([]byte, error) { + n, ok := nc.nodes.Get(path) + if ok { + return n, nil + } + return nil, nil +} + +func (nc *nodecache) commit(nodes dbtypes.KvMap) error { + if atomic.LoadUint64(&nc.immutable) == 1 { + return errWriteImmutable + } + + var ( + delta int64 + overwrite int64 + overwriteSize int64 + ) + + for _, v := range nodes { + current, exist := nc.nodes.Get(v.K) + if !exist { + nc.nodes.Put(v.K, v.V) + delta += int64(len(v.K) + len(v.V)) + + continue + } + + if !bytes.Equal(current, v.V) { + delta += int64(len(v.V) - len(current)) + overwrite++ + overwriteSize += int64(len(v.V) + len(v.K)) + } + + nc.nodes.Put(v.K, v.V) + } + + nc.updateSize(delta) + nc.layers++ + gcNodesMeter.Mark(overwrite) + gcBytesMeter.Mark(overwriteSize) + return nil +} + +func (nc *nodecache) updateSize(delta int64) { + size := int64(nc.size) + delta + if size >= 0 { + nc.size = uint64(size) + return + } + s := nc.size + nc.size = 0 + log.Error("Invalid pathdb buffer size", "prev", common.StorageSize(s), "delta", common.StorageSize(delta)) +} + +func (nc *nodecache) reset() { + atomic.StoreUint64(&nc.immutable, 0) + nc.layers = 0 + nc.size = 0 + nc.nodes = make(dbtypes.KvMap) +} + +func (nc *nodecache) empty() bool { + return nc.layers == 0 +} + +func (nc *nodecache) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64) error { + if atomic.LoadUint64(&nc.immutable) != 1 { + return errFlushMutable + } + + // Ensure the target state id is aligned with the internal counter. + head := rawdb.ReadPersistentStateID(db) + if head+nc.layers != id { + return fmt.Errorf("buffer layers (%d) cannot be applied on top of persisted state id (%d) to reach requested state id (%d)", nc.layers, head, id) + } + var ( + start = time.Now() + batch = db.NewBatchWithSize(int(float64(nc.size) * DefaultBatchRedundancyRate)) + ) + nodes := writeNodes(batch, nc.nodes, clean) + rawdb.WritePersistentStateID(batch, id) + + // Flush all mutations in a single batch + size := batch.ValueSize() + if err := batch.Write(); err != nil { + return err + } + commitBytesMeter.Mark(int64(size)) + commitNodesMeter.Mark(int64(nodes)) + commitTimeTimer.UpdateSince(start) + log.Debug("Persisted pathdb nodes", "nodes", len(nc.nodes), "bytes", common.StorageSize(size), "elapsed", common.PrettyDuration(time.Since(start))) + nc.reset() + return nil +} + +func (nc *nodecache) merge(nc1 *nodecache) (*nodecache, error) { + if nc == nil && nc1 == nil { + return nil, nil + } + if nc == nil || nc.empty() { + res := copyNodeCache(nc1) + atomic.StoreUint64(&res.immutable, 0) + return res, nil + } + if nc1 == nil || nc1.empty() { + res := copyNodeCache(nc) + atomic.StoreUint64(&res.immutable, 0) + return res, nil + } + if atomic.LoadUint64(&nc.immutable) == atomic.LoadUint64(&nc1.immutable) { + return nil, errIncompatibleMerge + } + + var ( + immutable *nodecache + mutable *nodecache + res = &nodecache{} + ) + if atomic.LoadUint64(&nc.immutable) == 1 { + immutable = nc + mutable = nc1 + } else { + immutable = nc1 + mutable = nc + } + res.size = immutable.size + mutable.size + res.layers = immutable.layers + mutable.layers + res.limit = immutable.limit + res.nodes = make(dbtypes.KvMap) + for _, v := range immutable.nodes { + res.nodes.Put(v.K, v.K) + } + + for _, v := range mutable.nodes { + res.nodes.Put(v.K, v.K) + } + + return res, nil +} + +func copyNodeCache(n *nodecache) *nodecache { + if n == nil { + return nil + } + nc := &nodecache{ + layers: n.layers, + size: n.size, + limit: n.limit, + immutable: atomic.LoadUint64(&n.immutable), + nodes: make(dbtypes.KvMap), + } + + for _, v := range n.nodes { + nc.nodes.Put(v.K, v.V) + } + + return nc +} diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go new file mode 100644 index 000000000..a504755f4 --- /dev/null +++ b/triedb/pathdb/database.go @@ -0,0 +1,383 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "fmt" + "io" + "os" + "sort" + "strconv" + "sync" + + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/core/rawdb" + "github.com/morph-l2/go-ethereum/core/types" + "github.com/morph-l2/go-ethereum/ethdb" + "github.com/morph-l2/go-ethereum/log" + "github.com/morph-l2/go-ethereum/params" + dbtypes "github.com/morph-l2/go-ethereum/triedb/types" +) + +const ( + // maxDiffLayers is the maximum diff layers allowed in the layer tree. + maxDiffLayers = 128 + + // defaultCleanSize is the default memory allowance of clean cache. + defaultCleanSize = 16 * 1024 * 1024 + + // MaxDirtyBufferSize is the maximum memory allowance of node buffer. + // Too large nodebuffer will cause the system to pause for a long + // time when write happens. Also, the largest batch that pebble can + // support is 4GB, node will panic if batch size exceeds this limit. + MaxDirtyBufferSize = 256 * 1024 * 1024 + + // DefaultDirtyBufferSize is the default memory allowance of node buffer + // that aggregates the writes from above until it's flushed into the + // disk. It's meant to be used once the initial sync is finished. + // Do not increase the buffer size arbitrarily, otherwise the system + // pause time will increase when the database writes happen. + DefaultDirtyBufferSize = 64 * 1024 * 1024 + + // DefaultBackgroundFlushInterval defines the default the wait interval + // that background node cache flush disk. + DefaultBackgroundFlushInterval = 3 + + // DefaultBatchRedundancyRate defines the batch size, compatible write + // size calculation is inaccurate + DefaultBatchRedundancyRate = 1.1 +) + +type JournalType int + +const ( + JournalKVType JournalType = iota + JournalFileType +) + +// layer is the interface implemented by all state layers which includes some +// public methods and some additional methods for internal usage. +type layer interface { + // Node retrieves the trie node with the node info. An error will be returned + // if the read operation exits abnormally. For example, if the layer is already + // stale, or the associated state is regarded as corrupted. Notably, no error + // will be returned if the requested node is not found in database. + Node(path []byte) ([]byte, error) + + // rootHash returns the root hash for which this layer was made. + rootHash() common.Hash + + // stateID returns the associated state id of layer. + stateID() uint64 + + // parentLayer returns the subsequent layer of it, or nil if the disk was reached. + parentLayer() layer + + // update creates a new layer on top of the existing layer diff tree with + // the provided dirty trie nodes along with the state change set. + // + // Note, the maps are retained by the method to avoid copying everything. + update(root common.Hash, id uint64, block uint64, nodes dbtypes.KvMap) *diffLayer + + // journal commits an entire diff hierarchy to disk into a single journal entry. + // This is meant to be used during shutdown to persist the layer without + // flattening everything down (bad for reorgs). + journal(w io.Writer, journalType JournalType) error +} + +// Config contains the settings for database. +type Config struct { + SyncFlush bool // Flag of trienodebuffer sync flush cache to disk + StateHistory uint64 // Number of recent blocks to maintain state history for + CleanCacheSize int // Maximum memory allowance (in bytes) for caching clean nodes + DirtyCacheSize int // Maximum memory allowance (in bytes) for caching dirty nodes + ReadOnly bool // Flag whether the database is opened in read only mode. + NoTries bool + JournalFilePath string + JournalFile bool +} + +// sanitize checks the provided user configurations and changes anything that's +// unreasonable or unworkable. +func (c *Config) sanitize() *Config { + conf := *c + if conf.DirtyCacheSize > MaxDirtyBufferSize { + log.Warn("Sanitizing invalid node buffer size", "provided", common.StorageSize(conf.DirtyCacheSize), "updated", common.StorageSize(MaxDirtyBufferSize)) + conf.DirtyCacheSize = MaxDirtyBufferSize + } + return &conf +} + +// Defaults contains default settings for Ethereum mainnet. +var Defaults = &Config{ + StateHistory: params.FullImmutabilityThreshold, + CleanCacheSize: defaultCleanSize, + DirtyCacheSize: DefaultDirtyBufferSize, +} + +// ReadOnly is the config in order to open database in read only mode. +var ReadOnly = &Config{ReadOnly: true} + +// Database is a multiple-layered structure for maintaining in-memory trie nodes. +// It consists of one persistent base layer backed by a key-value store, on top +// of which arbitrarily many in-memory diff layers are stacked. The memory diffs +// can form a tree with branching, but the disk layer is singleton and common to +// all. If a reorg goes deeper than the disk layer, a batch of reverse diffs can +// be applied to rollback. The deepest reorg that can be handled depends on the +// amount of state histories tracked in the disk. +// +// At most one readable and writable database can be opened at the same time in +// the whole system which ensures that only one database writer can operate disk +// state. Unexpected open operations can cause the system to panic. +type Database struct { + // readOnly is the flag whether the mutation is allowed to be applied. + // It will be set automatically when the database is journaled during + // the shutdown to reject all following unexpected mutations. + readOnly bool // Flag if database is opened in read only mode + bufferSize int // Memory allowance (in bytes) for caching dirty nodes + config *Config // Configuration for database + diskdb ethdb.KeyValueStore // Persistent storage for matured trie nodes + tree *layerTree // The group for all known layers + lock sync.RWMutex // Lock to prevent mutations from happening at the same time + dirties dbtypes.KvMap +} + +// New attempts to load an already existing layer from a persistent key-value +// store (with a number of memory layers from a journal). If the journal is not +// matched with the base persistent layer, all the recorded diff layers are discarded. +func New(diskdb ethdb.KeyValueStore, config *Config) *Database { + if config == nil { + config = Defaults + } + config = config.sanitize() + db := &Database{ + readOnly: config.ReadOnly, + bufferSize: config.DirtyCacheSize, + config: config, + diskdb: diskdb, + dirties: make(dbtypes.KvMap), + } + // Construct the layer tree by resolving the in-disk singleton state + // and in-memory layer journal. + db.tree = newLayerTree(db.loadLayers()) + + return db +} + +// Reader retrieves a layer belonging to the given state root. +func (db *Database) Reader(root common.Hash) (layer, error) { + l := db.tree.get(root) + if l == nil { + return nil, fmt.Errorf("state %#x is not available", root) + } + return l, nil +} + +func (db *Database) CommitGenesis(root common.Hash) error { + log.Info("pathdb write genesis state to disk", "root", root.Hex()) + batch := db.diskdb.NewBatch() + for _, v := range db.dirties { + batch.Put(v.K, v.V) + } + for k := range db.dirties { + delete(db.dirties, k) + } + if err := batch.Write(); err != nil { + return err + } + batch.Reset() + return nil +} + +// Commit traverses downwards the layer tree from a specified layer with the +// provided state root and all the layers below are flattened downwards. It +// can be used alone and mostly for test purposes. +func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockNumber uint64, report bool) error { + // Hold the lock to prevent concurrent mutations. + db.lock.Lock() + defer db.lock.Unlock() + + // Short circuit if the mutation is not allowed. + if err := db.modifyAllowed(); err != nil { + return err + } + + // only 1 entry, state not change + // some block maybe has no txns, so state do not change + if root == parentRoot && len(db.dirties) == 1 { + return nil + } + + nodes := db.dirties.Flatten() + for k := range db.dirties { + delete(db.dirties, k) + } + if err := db.tree.add(root, parentRoot, blockNumber, nodes); err != nil { + return err + } + + // Keep 128 diff layers in the memory, persistent layer is 129th. + // - head layer is paired with HEAD state + // - head-1 layer is paired with HEAD-1 state + // - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state + // - head-128 layer(disk layer) is paired with HEAD-128 state + return db.tree.cap(root, maxDiffLayers) +} + +// Close closes the trie database and the held freezer. +func (db *Database) Close() error { + db.lock.Lock() + defer db.lock.Unlock() + + // Set the database to read-only mode to prevent all + // following mutations. + db.readOnly = true + + // Release the memory held by clean cache. + db.tree.bottom().resetCache() + + return nil +} + +// Size returns the current storage size of the memory cache in front of the +// persistent database layer. +func (db *Database) Size() (diffs common.StorageSize, nodes common.StorageSize, immutableNodes common.StorageSize) { + db.tree.forEach(func(layer layer) { + if diff, ok := layer.(*diffLayer); ok { + diffs += common.StorageSize(diff.memory) + } + if disk, ok := layer.(*diskLayer); ok { + nodes, immutableNodes = disk.size() + } + }) + return diffs, nodes, immutableNodes +} + +// Initialized returns an indicator if the state data is already +// initialized in path-based scheme. +func (db *Database) Initialized(genesisRoot common.Hash) bool { + var inited bool + db.tree.forEach(func(layer layer) { + if layer.rootHash() != types.EmptyRootHash { + inited = true + } + }) + + return inited +} + +// SetBufferSize sets the node buffer size to the provided value(in bytes). +func (db *Database) SetBufferSize(size int) error { + db.lock.Lock() + defer db.lock.Unlock() + + if size > MaxDirtyBufferSize { + log.Info("Capped node buffer size", "provided", common.StorageSize(size), "adjusted", common.StorageSize(MaxDirtyBufferSize)) + size = MaxDirtyBufferSize + } + db.bufferSize = size + return db.tree.bottom().setBufferSize(db.bufferSize) +} + +// Scheme returns the node scheme used in the database. +func (db *Database) Scheme() string { + return rawdb.PathScheme +} + +// Head return the top non-fork difflayer/disklayer root hash for rewinding. +func (db *Database) Head() common.Hash { + db.lock.Lock() + defer db.lock.Unlock() + return db.tree.front() +} + +// modifyAllowed returns the indicator if mutation is allowed. This function +// assumes the db.lock is already held. +func (db *Database) modifyAllowed() error { + if db.readOnly { + return errDatabaseReadOnly + } + return nil +} + +// GetAllRootHash returns all diffLayer and diskLayer root hash +func (db *Database) GetAllRootHash() [][]string { + db.lock.Lock() + defer db.lock.Unlock() + + data := make([][]string, 0, len(db.tree.layers)) + for _, v := range db.tree.layers { + if dl, ok := v.(*diffLayer); ok { + data = append(data, []string{fmt.Sprintf("%d", dl.block), dl.rootHash().String()}) + } + } + sort.Slice(data, func(i, j int) bool { + block1, _ := strconv.Atoi(data[i][0]) + block2, _ := strconv.Atoi(data[j][0]) + return block1 > block2 + }) + + data = append(data, []string{"-1", db.tree.bottom().rootHash().String()}) + return data +} + +// DetermineJournalTypeForWriter is used when persisting the journal. It determines JournalType based on the config passed in by the Config. +func (db *Database) DetermineJournalTypeForWriter() JournalType { + if db.config.JournalFile { + return JournalFileType + } else { + return JournalKVType + } +} + +// DetermineJournalTypeForReader is used when loading the journal. It loads based on whether JournalKV or JournalFile currently exists. +func (db *Database) DetermineJournalTypeForReader() JournalType { + if journal := rawdb.ReadTrieJournal(db.diskdb); len(journal) != 0 { + return JournalKVType + } + + if fileInfo, stateErr := os.Stat(db.config.JournalFilePath); stateErr == nil && !fileInfo.IsDir() { + return JournalFileType + } + + return JournalKVType +} + +func (db *Database) DeleteTrieJournal(writer ethdb.KeyValueWriter) error { + // To prevent any remnants of old journals after converting from JournalKV to JournalFile or vice versa, all deletions must be completed. + rawdb.DeleteTrieJournal(writer) + + // delete from journal file, may not exist + filePath := db.config.JournalFilePath + if _, err := os.Stat(filePath); os.IsNotExist(err) { + return nil + } + errRemove := os.Remove(filePath) + if errRemove != nil { + log.Crit("Failed to remove tries journal", "journal path", filePath, "err", errRemove) + } + return nil +} + +// zk-trie put dirties +func (db *Database) Put(k, v []byte) error { + db.lock.Lock() + defer db.lock.Unlock() + + db.dirties.Put(k, v) + return nil +} diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go new file mode 100644 index 000000000..1b1e6fa5a --- /dev/null +++ b/triedb/pathdb/difflayer.go @@ -0,0 +1,305 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "crypto/sha256" + "fmt" + "sync" + + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/log" + dbtypes "github.com/morph-l2/go-ethereum/triedb/types" +) + +type RefTrieNode struct { + refCount uint32 + blob []byte +} + +type HashNodeCache struct { + lock sync.RWMutex + cache map[[sha256.Size]byte]*RefTrieNode +} + +func (h *HashNodeCache) length() int { + if h == nil { + return 0 + } + h.lock.RLock() + defer h.lock.RUnlock() + return len(h.cache) +} + +func (h *HashNodeCache) set(key, val []byte) { + if h == nil { + return + } + h.lock.Lock() + defer h.lock.Unlock() + hash := sha256.Sum256(key) + if n, ok := h.cache[hash]; ok { + n.refCount++ + n.blob = val + } else { + h.cache[hash] = &RefTrieNode{1, val} + } +} + +func (h *HashNodeCache) Get(key []byte) []byte { + if h == nil { + return nil + } + h.lock.RLock() + defer h.lock.RUnlock() + hash := sha256.Sum256(key) + if n, ok := h.cache[hash]; ok { + return n.blob + } + return nil +} + +func (h *HashNodeCache) del(key []byte) { + if h == nil { + return + } + h.lock.Lock() + defer h.lock.Unlock() + + hash := sha256.Sum256(key) + n, ok := h.cache[hash] + if !ok { + return + } + if n.refCount > 0 { + n.refCount-- + } + if n.refCount == 0 { + delete(h.cache, hash) + } +} + +func (h *HashNodeCache) Add(ly layer) { + if h == nil { + return + } + dl, ok := ly.(*diffLayer) + if !ok { + return + } + beforeAdd := h.length() + for _, v := range dl.nodes { + h.set(v.K, v.V) + } + diffHashCacheLengthGauge.Update(int64(h.length())) + log.Debug("Add difflayer to hash map", "root", ly.rootHash(), "block_number", dl.block, "map_len", h.length(), "add_delta", h.length()-beforeAdd) +} + +func (h *HashNodeCache) Remove(ly layer) { + if h == nil { + return + } + dl, ok := ly.(*diffLayer) + if !ok { + return + } + go func() { + beforeDel := h.length() + for _, v := range dl.nodes { + h.del(v.K) + } + diffHashCacheLengthGauge.Update(int64(h.length())) + log.Debug("Remove difflayer from hash map", "root", ly.rootHash(), "block_number", dl.block, "map_len", h.length(), "del_delta", beforeDel-h.length()) + }() +} + +// diffLayer represents a collection of modifications made to the in-memory tries +// along with associated state changes after running a block on top. +// +// The goal of a diff layer is to act as a journal, tracking recent modifications +// made to the state, that have not yet graduated into a semi-immutable state. +type diffLayer struct { + // Immutables + root common.Hash // Root hash to which this layer diff belongs to + id uint64 // Corresponding state id + block uint64 // Associated block number + nodes dbtypes.KvMap // Cached trie nodes indexed by owner and path + memory uint64 // Approximate guess as to how much memory we use + cache *HashNodeCache // trienode cache by hash key. cache is immutable, but cache's item can be add/del. + + // mutables + origin *diskLayer // The current difflayer corresponds to the underlying disklayer and is updated during cap. + parent layer // Parent layer modified by this one, never nil, **can be changed** + lock sync.RWMutex // Lock used to protect parent +} + +// newDiffLayer creates a new diff layer on top of an existing layer. +func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes dbtypes.KvMap) *diffLayer { + var ( + size int64 + count int + ) + dl := &diffLayer{ + root: root, + id: id, + block: block, + nodes: nodes, + parent: parent, + } + + switch l := parent.(type) { + case *diskLayer: + dl.origin = l + dl.cache = &HashNodeCache{ + cache: make(map[[sha256.Size]byte]*RefTrieNode), + } + case *diffLayer: + dl.origin = l.originDiskLayer() + dl.cache = l.cache + default: + panic("unknown parent type") + } + + for _, v := range nodes { + dl.memory += uint64(len(v.K) + len(v.V)) + size += int64(len(v.K) + len(v.V)) + count += 1 + } + + dirtyWriteMeter.Mark(size) + diffLayerNodesMeter.Mark(int64(count)) + diffLayerBytesMeter.Mark(int64(dl.memory)) + log.Debug("Created new diff layer", "id", id, "block", block, "nodes", count, "size", common.StorageSize(dl.memory), "root", dl.root) + return dl +} + +func (dl *diffLayer) originDiskLayer() *diskLayer { + dl.lock.RLock() + defer dl.lock.RUnlock() + return dl.origin +} + +// rootHash implements the layer interface, returning the root hash of +// corresponding state. +func (dl *diffLayer) rootHash() common.Hash { + return dl.root +} + +// stateID implements the layer interface, returning the state id of the layer. +func (dl *diffLayer) stateID() uint64 { + return dl.id +} + +// parentLayer implements the layer interface, returning the subsequent +// layer of the diff layer. +func (dl *diffLayer) parentLayer() layer { + dl.lock.RLock() + defer dl.lock.RUnlock() + + return dl.parent +} + +// node retrieves the node with provided node information. It's the internal +// version of Node function with additional accessed layer tracked. No error +// will be returned if node is not found. +func (dl *diffLayer) node(path []byte, depth int) ([]byte, error) { + // Hold the lock, ensure the parent won't be changed during the + // state accessing. + dl.lock.RLock() + defer dl.lock.RUnlock() + + // If the trie node is known locally, return it + n, ok := dl.nodes.Get(path) + if ok { + return n, nil + } + // Trie node unknown to this layer, resolve from parent + if diff, ok := dl.parent.(*diffLayer); ok { + return diff.node(path, depth+1) + } + // Failed to resolve through diff layers, fallback to disk layer + return dl.parent.Node(path) +} + +// Node implements the layer interface, retrieving the trie node blob with the +// provided node information. No error will be returned if the node is not found. +func (dl *diffLayer) Node(path []byte) ([]byte, error) { + if n := dl.cache.Get(path); n != nil { + // The query from the hash map is fastpath, + // avoiding recursive query of 128 difflayers. + diffHashCacheHitMeter.Mark(1) + diffHashCacheReadMeter.Mark(int64(len(n))) + return n, nil + } + diffHashCacheMissMeter.Mark(1) + + persistLayer := dl.originDiskLayer() + if persistLayer != nil { + blob, err := persistLayer.Node(path) + if err != nil { + // This is a bad case with a very low probability. + // r/w the difflayer cache and r/w the disklayer are not in the same lock, + // so in extreme cases, both reading the difflayer cache and reading the disklayer may fail, eg, disklayer is stale. + // In this case, fallback to the original 128-layer recursive difflayer query path. + diffHashCacheSlowPathMeter.Mark(1) + log.Debug("Retry difflayer due to query origin failed", "path", path, "hash", "error", err) + return dl.node(path, 0) + } else { // This is the fastpath. + return blob, nil + } + } + diffHashCacheSlowPathMeter.Mark(1) + log.Debug("Retry difflayer due to origin is nil", "path", path) + return dl.node(path, 0) +} + +// update implements the layer interface, creating a new layer on top of the +// existing layer tree with the specified data items. +func (dl *diffLayer) update(root common.Hash, id uint64, block uint64, nodes dbtypes.KvMap) *diffLayer { + return newDiffLayer(dl, root, id, block, nodes) +} + +// persist flushes the diff layer and all its parent layers to disk layer. +func (dl *diffLayer) persist(force bool) (layer, error) { + if parent, ok := dl.parentLayer().(*diffLayer); ok { + // Hold the lock to prevent any read operation until the new + // parent is linked correctly. + dl.lock.Lock() + + // The merging of diff layers starts at the bottom-most layer, + // therefore we recurse down here, flattening on the way up + // (diffToDisk). + result, err := parent.persist(force) + if err != nil { + dl.lock.Unlock() + return nil, err + } + dl.parent = result + dl.lock.Unlock() + } + return diffToDisk(dl, force) +} + +// diffToDisk merges a bottom-most diff into the persistent disk layer underneath +// it. The method will panic if called onto a non-bottom-most diff layer. +func diffToDisk(layer *diffLayer, force bool) (layer, error) { + disk, ok := layer.parentLayer().(*diskLayer) + if !ok { + panic(fmt.Sprintf("unknown layer type: %T", layer.parentLayer())) + } + return disk.commit(layer, force) +} diff --git a/triedb/pathdb/difflayer_test.go b/triedb/pathdb/difflayer_test.go new file mode 100644 index 000000000..a062878be --- /dev/null +++ b/triedb/pathdb/difflayer_test.go @@ -0,0 +1,170 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +// import ( +// "bytes" +// "testing" + +// "github.com/ethereum/go-ethereum/common" +// "github.com/ethereum/go-ethereum/core/rawdb" +// "github.com/ethereum/go-ethereum/trie/testutil" +// "github.com/ethereum/go-ethereum/trie/trienode" +// ) + +// func emptyLayer() *diskLayer { +// return &diskLayer{ +// db: New(rawdb.NewMemoryDatabase(), nil), +// buffer: newNodeBuffer(DefaultDirtyBufferSize, nil, 0), +// } +// } + +// // goos: darwin +// // goarch: arm64 +// // pkg: github.com/ethereum/go-ethereum/trie +// // BenchmarkSearch128Layers +// // BenchmarkSearch128Layers-8 243826 4755 ns/op +// func BenchmarkSearch128Layers(b *testing.B) { benchmarkSearch(b, 0, 128) } + +// // goos: darwin +// // goarch: arm64 +// // pkg: github.com/ethereum/go-ethereum/trie +// // BenchmarkSearch512Layers +// // BenchmarkSearch512Layers-8 49686 24256 ns/op +// func BenchmarkSearch512Layers(b *testing.B) { benchmarkSearch(b, 0, 512) } + +// // goos: darwin +// // goarch: arm64 +// // pkg: github.com/ethereum/go-ethereum/trie +// // BenchmarkSearch1Layer +// // BenchmarkSearch1Layer-8 14062725 88.40 ns/op +// func BenchmarkSearch1Layer(b *testing.B) { benchmarkSearch(b, 127, 128) } + +// func benchmarkSearch(b *testing.B, depth int, total int) { +// var ( +// npath []byte +// nhash common.Hash +// nblob []byte +// ) +// // First, we set up 128 diff layers, with 3K items each +// fill := func(parent layer, index int) *diffLayer { +// nodes := make(map[common.Hash]map[string]*trienode.Node) +// nodes[common.Hash{}] = make(map[string]*trienode.Node) +// for i := 0; i < 3000; i++ { +// var ( +// path = testutil.RandBytes(32) +// node = testutil.RandomNode() +// ) +// nodes[common.Hash{}][string(path)] = trienode.New(node.Hash, node.Blob) +// if npath == nil && depth == index { +// npath = common.CopyBytes(path) +// nblob = common.CopyBytes(node.Blob) +// nhash = node.Hash +// } +// } +// return newDiffLayer(parent, common.Hash{}, 0, 0, nodes, nil) +// } +// var layer layer +// layer = emptyLayer() +// for i := 0; i < total; i++ { +// layer = fill(layer, i) +// } +// b.ResetTimer() + +// var ( +// have []byte +// err error +// ) +// for i := 0; i < b.N; i++ { +// have, err = layer.Node(common.Hash{}, npath, nhash) +// if err != nil { +// b.Fatal(err) +// } +// } +// if !bytes.Equal(have, nblob) { +// b.Fatalf("have %x want %x", have, nblob) +// } +// } + +// // goos: darwin +// // goarch: arm64 +// // pkg: github.com/ethereum/go-ethereum/trie +// // BenchmarkPersist +// // BenchmarkPersist-8 10 111252975 ns/op +// func BenchmarkPersist(b *testing.B) { +// // First, we set up 128 diff layers, with 3K items each +// fill := func(parent layer) *diffLayer { +// nodes := make(map[common.Hash]map[string]*trienode.Node) +// nodes[common.Hash{}] = make(map[string]*trienode.Node) +// for i := 0; i < 3000; i++ { +// var ( +// path = testutil.RandBytes(32) +// node = testutil.RandomNode() +// ) +// nodes[common.Hash{}][string(path)] = trienode.New(node.Hash, node.Blob) +// } +// return newDiffLayer(parent, common.Hash{}, 0, 0, nodes, nil) +// } +// for i := 0; i < b.N; i++ { +// b.StopTimer() +// var layer layer +// layer = emptyLayer() +// for i := 1; i < 128; i++ { +// layer = fill(layer) +// } +// b.StartTimer() + +// dl, ok := layer.(*diffLayer) +// if !ok { +// break +// } +// dl.persist(false) +// } +// } + +// // BenchmarkJournal benchmarks the performance for journaling the layers. +// // +// // BenchmarkJournal +// // BenchmarkJournal-8 10 110969279 ns/op +// func BenchmarkJournal(b *testing.B) { +// b.SkipNow() + +// // First, we set up 128 diff layers, with 3K items each +// fill := func(parent layer) *diffLayer { +// nodes := make(map[common.Hash]map[string]*trienode.Node) +// nodes[common.Hash{}] = make(map[string]*trienode.Node) +// for i := 0; i < 3000; i++ { +// var ( +// path = testutil.RandBytes(32) +// node = testutil.RandomNode() +// ) +// nodes[common.Hash{}][string(path)] = trienode.New(node.Hash, node.Blob) +// } +// // TODO(rjl493456442) a non-nil state set is expected. +// return newDiffLayer(parent, common.Hash{}, 0, 0, nodes, nil) +// } +// var layer layer +// layer = emptyLayer() +// for i := 0; i < 128; i++ { +// layer = fill(layer) +// } +// b.ResetTimer() + +// for i := 0; i < b.N; i++ { +// layer.journal(new(bytes.Buffer), JournalKVType) +// } +// } diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go new file mode 100644 index 000000000..03e418ee8 --- /dev/null +++ b/triedb/pathdb/disklayer.go @@ -0,0 +1,268 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "sync" + + "github.com/VictoriaMetrics/fastcache" + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/core/rawdb" + "github.com/morph-l2/go-ethereum/ethdb" + "github.com/morph-l2/go-ethereum/log" + dbtypes "github.com/morph-l2/go-ethereum/triedb/types" +) + +// trienodebuffer is a collection of modified trie nodes to aggregate the disk +// write. The content of the trienodebuffer must be checked before diving into +// disk (since it basically is not-yet-written data). +type trienodebuffer interface { + // node retrieves the trie node with given node info. + node(path []byte) ([]byte, error) + + // commit merges the dirty nodes into the trienodebuffer. This operation won't take + // the ownership of the nodes map which belongs to the bottom-most diff layer. + // It will just hold the node references from the given map which are safe to + // copy. + commit(nodes dbtypes.KvMap) trienodebuffer + + // flush persists the in-memory dirty trie node into the disk if the configured + // memory threshold is reached. Note, all data must be written atomically. + flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64, force bool) error + + // setSize sets the buffer size to the provided number, and invokes a flush + // operation if the current memory usage exceeds the new limit. + setSize(size int, db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64) error + + // reset cleans up the disk cache. + reset() + + // empty returns an indicator if trienodebuffer contains any state transition inside. + empty() bool + + // getSize return the trienodebuffer used size. + getSize() (uint64, uint64) + + // getAllNodes return all the trie nodes are cached in trienodebuffer. + getAllNodes() dbtypes.KvMap + + // getLayers return the size of cached difflayers. + getLayers() uint64 + + // waitAndStopFlushing will block unit writing the trie nodes of trienodebuffer to disk. + waitAndStopFlushing() +} + +func NewTrieNodeBuffer(sync bool, limit int, nodes dbtypes.KvMap, layers uint64) trienodebuffer { + if sync { + log.Info("New sync node buffer", "limit", common.StorageSize(limit), "layers", layers) + return newNodeBuffer(limit, nodes, layers) + } + log.Info("New async node buffer", "limit", common.StorageSize(limit), "layers", layers) + return newAsyncNodeBuffer(limit, nodes, layers) +} + +// diskLayer is a low level persistent layer built on top of a key-value store. +type diskLayer struct { + root common.Hash // Immutable, root hash to which this layer was made for + id uint64 // Immutable, corresponding state id + db *Database // Path-based trie database + cleans *fastcache.Cache // GC friendly memory cache of clean node RLPs + buffer trienodebuffer // Node buffer to aggregate writes + stale bool // Signals that the layer became stale (state progressed) + lock sync.RWMutex // Lock used to protect stale flag +} + +// newDiskLayer creates a new disk layer based on the passing arguments. +func newDiskLayer(root common.Hash, id uint64, db *Database, cleans *fastcache.Cache, buffer trienodebuffer) *diskLayer { + // Initialize a clean cache if the memory allowance is not zero + // or reuse the provided cache if it is not nil (inherited from + // the original disk layer). + if cleans == nil && db.config.CleanCacheSize != 0 { + cleans = fastcache.New(db.config.CleanCacheSize) + } + return &diskLayer{ + root: root, + id: id, + db: db, + cleans: cleans, + buffer: buffer, + } +} + +// root implements the layer interface, returning root hash of corresponding state. +func (dl *diskLayer) rootHash() common.Hash { + return dl.root +} + +// stateID implements the layer interface, returning the state id of disk layer. +func (dl *diskLayer) stateID() uint64 { + return dl.id +} + +// parent implements the layer interface, returning nil as there's no layer +// below the disk. +func (dl *diskLayer) parentLayer() layer { + return nil +} + +// isStale return whether this layer has become stale (was flattened across) or if +// it's still live. +func (dl *diskLayer) isStale() bool { + dl.lock.RLock() + defer dl.lock.RUnlock() + + return dl.stale +} + +// markStale sets the stale flag as true. +func (dl *diskLayer) markStale() { + dl.lock.Lock() + defer dl.lock.Unlock() + + if dl.stale { + panic("triedb disk layer is stale") // we've committed into the same base from two children, boom + } + dl.stale = true +} + +// Node implements the layer interface, retrieving the trie node with the +// provided node info. No error will be returned if the node is not found. +func (dl *diskLayer) Node(path []byte) ([]byte, error) { + dl.lock.RLock() + defer dl.lock.RUnlock() + + if dl.stale { + return nil, errSnapshotStale + } + // Try to retrieve the trie node from the not-yet-written + // node buffer first. Note the buffer is lock free since + // it's impossible to mutate the buffer before tagging the + // layer as stale. + n, err := dl.buffer.node(path) + if err != nil { + return nil, err + } + if n != nil { + dirtyHitMeter.Mark(1) + dirtyReadMeter.Mark(int64(len(n))) + return n, nil + } + dirtyMissMeter.Mark(1) + + // Try to retrieve the trie node from the clean memory cache + key := path + if dl.cleans != nil { + if blob := dl.cleans.Get(nil, key); len(blob) > 0 { + cleanHitMeter.Mark(1) + cleanReadMeter.Mark(int64(len(blob))) + return blob, nil + } + cleanMissMeter.Mark(1) + } + + // Try to retrieve the trie node from the disk. + n, err = rawdb.ReadTrieNodeByKey(dl.db.diskdb, path) + if err == nil { + if dl.cleans != nil && len(n) > 0 { + dl.cleans.Set(key, n) + cleanWriteMeter.Mark(int64(len(n))) + } + return n, nil + } + return nil, err +} + +// update implements the layer interface, returning a new diff layer on top +// with the given state set. +func (dl *diskLayer) update(root common.Hash, id uint64, block uint64, nodes dbtypes.KvMap) *diffLayer { + return newDiffLayer(dl, root, id, block, nodes) +} + +// commit merges the given bottom-most diff layer into the node buffer +// and returns a newly constructed disk layer. Note the current disk +// layer must be tagged as stale first to prevent re-access. +func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) { + dl.lock.Lock() + defer dl.lock.Unlock() + + // Construct and store the state history first. If crash happens after storing + // the state history but without flushing the corresponding states(journal), + // the stored state history will be truncated from head in the next restart. + + // Mark the diskLayer as stale before applying any mutations on top. + dl.stale = true + + // Store the root->id lookup afterwards. All stored lookups are identified + // by the **unique** state root. It's impossible that in the same chain + // blocks are not adjacent but have the same root. + if dl.id == 0 { + rawdb.WriteStateID(dl.db.diskdb, dl.root, 0) + } + rawdb.WriteStateID(dl.db.diskdb, bottom.rootHash(), bottom.stateID()) + + // Construct a new disk layer by merging the nodes from the provided diff + // layer, and flush the content in disk layer if there are too many nodes + // cached. The clean cache is inherited from the original disk layer. + + ndl := newDiskLayer(bottom.root, bottom.stateID(), dl.db, dl.cleans, dl.buffer.commit(bottom.nodes)) + + if err := ndl.buffer.flush(ndl.db.diskdb, ndl.cleans, ndl.id, force); err != nil { + return nil, err + } + + // The bottom has been eaten by disklayer, releasing the hash cache of bottom difflayer. + bottom.cache.Remove(bottom) + return ndl, nil +} + +// setBufferSize sets the trie node buffer size to the provided value. +func (dl *diskLayer) setBufferSize(size int) error { + dl.lock.RLock() + defer dl.lock.RUnlock() + + if dl.stale { + return errSnapshotStale + } + return dl.buffer.setSize(size, dl.db.diskdb, dl.cleans, dl.id) +} + +// size returns the approximate size of cached nodes in the disk layer. +func (dl *diskLayer) size() (common.StorageSize, common.StorageSize) { + dl.lock.RLock() + defer dl.lock.RUnlock() + + if dl.stale { + return 0, 0 + } + dirtyNodes, dirtyimmutableNodes := dl.buffer.getSize() + return common.StorageSize(dirtyNodes), common.StorageSize(dirtyimmutableNodes) +} + +// resetCache releases the memory held by clean cache to prevent memory leak. +func (dl *diskLayer) resetCache() { + dl.lock.RLock() + defer dl.lock.RUnlock() + + // Stale disk layer loses the ownership of clean cache. + if dl.stale { + return + } + if dl.cleans != nil { + dl.cleans.Reset() + } +} diff --git a/triedb/pathdb/errors.go b/triedb/pathdb/errors.go new file mode 100644 index 000000000..beb03b678 --- /dev/null +++ b/triedb/pathdb/errors.go @@ -0,0 +1,74 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package pathdb + +import ( + "errors" + "fmt" + + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/common/hexutil" +) + +var ( + // errDatabaseReadOnly is returned if the database is opened in read only mode + // to prevent any mutation. + errDatabaseReadOnly = errors.New("read only") + + // errDatabaseWaitSync is returned if the initial state sync is not completed + // yet and database is disabled to prevent accessing state. + errDatabaseWaitSync = errors.New("waiting for sync") + + // errSnapshotStale is returned from data accessors if the underlying layer + // layer had been invalidated due to the chain progressing forward far enough + // to not maintain the layer's original state. + errSnapshotStale = errors.New("layer stale") + + // errUnexpectedHistory is returned if an unmatched state history is applied + // to the database for state rollback. + errUnexpectedHistory = errors.New("unexpected state history") + + // errStateUnrecoverable is returned if state is required to be reverted to + // a destination without associated state history available. + errStateUnrecoverable = errors.New("state is unrecoverable") + + // errUnexpectedNode is returned if the requested node with specified path is + // not hash matched with expectation. + errUnexpectedNode = errors.New("unexpected node") + + // errWriteImmutable is returned if write to background immutable nodecache + // under asyncnodebuffer + errWriteImmutable = errors.New("write immutable nodecache") + + // errFlushMutable is returned if flush the background mutable nodecache + // to disk, under asyncnodebuffer + errFlushMutable = errors.New("flush mutable nodecache") + + // errIncompatibleMerge is returned when merge node cache occurs error. + errIncompatibleMerge = errors.New("incompatible nodecache merge") + + // errRevertImmutable is returned if revert the background immutable nodecache + errRevertImmutable = errors.New("revert immutable nodecache") +) + +func newUnexpectedNodeError(loc string, expHash common.Hash, gotHash common.Hash, owner common.Hash, path []byte, blob []byte) error { + blobHex := "nil" + if len(blob) > 0 { + blobHex = hexutil.Encode(blob) + } + return fmt.Errorf("%w, loc: %s, node: (%x %v), %x!=%x, blob: %s", errUnexpectedNode, loc, owner, path, expHash, gotHash, blobHex) +} diff --git a/triedb/pathdb/journal.go b/triedb/pathdb/journal.go new file mode 100644 index 000000000..e8639ff40 --- /dev/null +++ b/triedb/pathdb/journal.go @@ -0,0 +1,535 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "io" + "io/fs" + "os" + "time" + + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/core/rawdb" + "github.com/morph-l2/go-ethereum/core/types" + "github.com/morph-l2/go-ethereum/ethdb" + "github.com/morph-l2/go-ethereum/log" + "github.com/morph-l2/go-ethereum/rlp" + + dbtypes "github.com/morph-l2/go-ethereum/triedb/types" + zkt "github.com/scroll-tech/zktrie/types" +) + +var ( + errMissJournal = errors.New("journal not found") + errMissVersion = errors.New("version not found") + errUnexpectedVersion = errors.New("unexpected journal version") + errMissDiskRoot = errors.New("disk layer root not found") + errUnmatchedJournal = errors.New("unmatched journal") +) + +const journalVersion uint64 = 0 + +// journalNode represents a trie node persisted in the journal. +type journalNode struct { + Path []byte // Path of the node in the trie + Blob []byte // RLP-encoded trie node blob, nil means the node is deleted +} + +// journalAccounts represents a list accounts belong to the layer. +type journalAccounts struct { + Addresses []common.Address + Accounts [][]byte +} + +// journalStorage represents a list of storage slots belong to an account. +type journalStorage struct { + Incomplete bool + Account common.Address + Hashes []common.Hash + Slots [][]byte +} + +type JournalWriter interface { + io.Writer + + Close() + Size() uint64 +} + +type JournalReader interface { + io.Reader + Close() +} + +type JournalFileWriter struct { + file *os.File +} + +type JournalFileReader struct { + file *os.File +} + +type JournalKVWriter struct { + journalBuf bytes.Buffer + diskdb ethdb.KeyValueStore +} + +type JournalKVReader struct { + journalBuf *bytes.Buffer +} + +// Write appends b directly to the encoder output. +func (fw *JournalFileWriter) Write(b []byte) (int, error) { + return fw.file.Write(b) +} + +func (fw *JournalFileWriter) Close() { + fw.file.Close() +} + +func (fw *JournalFileWriter) Size() uint64 { + if fw.file == nil { + return 0 + } + fileInfo, err := fw.file.Stat() + if err != nil { + log.Crit("Failed to stat journal", "err", err) + } + return uint64(fileInfo.Size()) +} + +func (kw *JournalKVWriter) Write(b []byte) (int, error) { + return kw.journalBuf.Write(b) +} + +func (kw *JournalKVWriter) Close() { + rawdb.WriteTrieJournal(kw.diskdb, kw.journalBuf.Bytes()) + kw.journalBuf.Reset() +} + +func (kw *JournalKVWriter) Size() uint64 { + return uint64(kw.journalBuf.Len()) +} + +func (fr *JournalFileReader) Read(p []byte) (n int, err error) { + return fr.file.Read(p) +} + +func (fr *JournalFileReader) Close() { + fr.file.Close() +} + +func (kr *JournalKVReader) Read(p []byte) (n int, err error) { + return kr.journalBuf.Read(p) +} + +func (kr *JournalKVReader) Close() { +} + +func newJournalWriter(file string, db ethdb.KeyValueStore, journalType JournalType) JournalWriter { + if journalType == JournalKVType { + log.Info("New journal writer for journal kv") + return &JournalKVWriter{ + diskdb: db, + } + } else { + log.Info("New journal writer for journal file", "path", file) + fd, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + return nil + } + return &JournalFileWriter{ + file: fd, + } + } +} + +func newJournalReader(file string, db ethdb.KeyValueStore, journalType JournalType) (JournalReader, error) { + if journalType == JournalKVType { + log.Info("New journal reader for journal kv") + journal := rawdb.ReadTrieJournal(db) + if len(journal) == 0 { + return nil, errMissJournal + } + return &JournalKVReader{ + journalBuf: bytes.NewBuffer(journal), + }, nil + } else { + log.Info("New journal reader for journal file", "path", file) + fd, err := os.Open(file) + if errors.Is(err, fs.ErrNotExist) { + return nil, errMissJournal + } + if err != nil { + return nil, err + } + return &JournalFileReader{ + file: fd, + }, nil + } +} + +// loadJournal tries to parse the layer journal from the disk. +func (db *Database) loadJournal(diskRoot common.Hash) (layer, error) { + start := time.Now() + journalTypeForReader := db.DetermineJournalTypeForReader() + reader, err := newJournalReader(db.config.JournalFilePath, db.diskdb, journalTypeForReader) + + if err != nil { + return nil, err + } + if reader != nil { + defer reader.Close() + } + r := rlp.NewStream(reader, 0) + + // Firstly, resolve the first element as the journal version + version, err := r.Uint64() + if err != nil { + return nil, errMissVersion + } + if version != journalVersion { + return nil, fmt.Errorf("%w want %d got %d", errUnexpectedVersion, journalVersion, version) + } + // Secondly, resolve the disk layer root, ensure it's continuous + // with disk layer. Note now we can ensure it's the layer journal + // correct version, so we expect everything can be resolved properly. + var root common.Hash + if err := r.Decode(&root); err != nil { + return nil, errMissDiskRoot + } + // The journal is not matched with persistent state, discard them. + // It can happen that geth crashes without persisting the journal. + if !bytes.Equal(root.Bytes(), diskRoot.Bytes()) { + return nil, fmt.Errorf("%w want %x got %x", errUnmatchedJournal, root, diskRoot) + } + // Load the disk layer from the journal + base, err := db.loadDiskLayer(r, journalTypeForReader) + if err != nil { + return nil, err + } + // Load all the diff layers from the journal + head, err := db.loadDiffLayer(base, r, journalTypeForReader) + if err != nil { + return nil, err + } + log.Info("Loaded layer journal", "diskroot", diskRoot, "diffhead", head.rootHash(), "elapsed", common.PrettyDuration(time.Since(start))) + return head, nil +} + +// loadLayers loads a pre-existing state layer backed by a key-value store. +func (db *Database) loadLayers() layer { + // Retrieve the root node of persistent state. + _, root := rawdb.ReadAccountTrieNode(db.diskdb, zkt.TrieRootPathKey[:]) + + // Load the layers by resolving the journal + head, err := db.loadJournal(root) + if err == nil { + return head + } + // journal is not matched(or missing) with the persistent state, discard + // it. Display log for discarding journal, but try to avoid showing + // useless information when the db is created from scratch. + if !(root == types.EmptyRootHash && errors.Is(err, errMissJournal)) { + log.Info("Failed to load journal, discard it", "err", err) + } + // Return single layer with persistent state. + return newDiskLayer(root, rawdb.ReadPersistentStateID(db.diskdb), db, nil, NewTrieNodeBuffer(db.config.SyncFlush, db.bufferSize, nil, 0)) +} + +// loadDiskLayer reads the binary blob from the layer journal, reconstructing +// a new disk layer on it. +func (db *Database) loadDiskLayer(r *rlp.Stream, journalTypeForReader JournalType) (layer, error) { + // Resolve disk layer root + var ( + root common.Hash + journalBuf *rlp.Stream + journalEncodedBuff []byte + ) + if journalTypeForReader == JournalFileType { + if err := r.Decode(&journalEncodedBuff); err != nil { + return nil, fmt.Errorf("load disk journal: %v", err) + } + journalBuf = rlp.NewStream(bytes.NewReader(journalEncodedBuff), 0) + } else { + journalBuf = r + } + + if err := journalBuf.Decode(&root); err != nil { + return nil, fmt.Errorf("load disk root: %v", err) + } + // Resolve the state id of disk layer, it can be different + // with the persistent id tracked in disk, the id distance + // is the number of transitions aggregated in disk layer. + var id uint64 + if err := journalBuf.Decode(&id); err != nil { + return nil, fmt.Errorf("load state id: %v", err) + } + stored := rawdb.ReadPersistentStateID(db.diskdb) + if stored > id { + return nil, fmt.Errorf("invalid state id: stored %d resolved %d", stored, id) + } + // Resolve nodes cached in node buffer + var encoded []journalNode + if err := journalBuf.Decode(&encoded); err != nil { + return nil, fmt.Errorf("load disk nodes: %v", err) + } + nodes := make(dbtypes.KvMap) + for _, entry := range encoded { + nodes.Put(entry.Path, entry.Blob) + } + + if journalTypeForReader == JournalFileType { + var shaSum [32]byte + if err := r.Decode(&shaSum); err != nil { + return nil, fmt.Errorf("load shasum: %v", err) + } + + expectSum := sha256.Sum256(journalEncodedBuff) + if shaSum != expectSum { + return nil, fmt.Errorf("expect shaSum: %v, real:%v", expectSum, shaSum) + } + } + + // Calculate the internal state transitions by id difference. + base := newDiskLayer(root, id, db, nil, NewTrieNodeBuffer(db.config.SyncFlush, db.bufferSize, nodes, id-stored)) + + return base, nil +} + +// loadDiffLayer reads the next sections of a layer journal, reconstructing a new +// diff and verifying that it can be linked to the requested parent. +func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream, journalTypeForReader JournalType) (layer, error) { + // Read the next diff journal entry + var ( + root common.Hash + journalBuf *rlp.Stream + journalEncodedBuff []byte + ) + if journalTypeForReader == JournalFileType { + if err := r.Decode(&journalEncodedBuff); err != nil { + // The first read may fail with EOF, marking the end of the journal + if err == io.EOF { + return parent, nil + } + return nil, fmt.Errorf("load disk journal buffer: %v", err) + } + journalBuf = rlp.NewStream(bytes.NewReader(journalEncodedBuff), 0) + } else { + journalBuf = r + } + + if err := journalBuf.Decode(&root); err != nil { + // The first read may fail with EOF, marking the end of the journal + if err == io.EOF { + return parent, nil + } + return nil, fmt.Errorf("load diff root: %v", err) + } + var block uint64 + if err := journalBuf.Decode(&block); err != nil { + return nil, fmt.Errorf("load block number: %v", err) + } + // Read in-memory trie nodes from journal + var encoded []journalNode + if err := journalBuf.Decode(&encoded); err != nil { + return nil, fmt.Errorf("load diff nodes: %v", err) + } + nodes := make(dbtypes.KvMap) + for _, entry := range encoded { + nodes.Put(entry.Path, entry.Blob) + } + + if journalTypeForReader == JournalFileType { + var shaSum [32]byte + if err := r.Decode(&shaSum); err != nil { + return nil, fmt.Errorf("load shasum: %v", err) + } + + expectSum := sha256.Sum256(journalEncodedBuff) + if shaSum != expectSum { + return nil, fmt.Errorf("expect shaSum: %v, real:%v", expectSum, shaSum) + } + } + + log.Debug("Loaded diff layer journal", "root", root, "parent", parent.rootHash(), "id", parent.stateID()+1, "block", block, "nodes", len(nodes)) + // add cache first + l := newDiffLayer(parent, root, parent.stateID()+1, block, nodes) + l.cache.Add(l) + return db.loadDiffLayer(l, r, journalTypeForReader) +} + +// journal implements the layer interface, marshaling the un-flushed trie nodes +// along with layer metadata into provided byte buffer. +func (dl *diskLayer) journal(w io.Writer, journalType JournalType) error { + dl.lock.RLock() + defer dl.lock.RUnlock() + + // Create a buffer to store encoded data + journalBuf := new(bytes.Buffer) + + // Ensure the layer didn't get stale + if dl.stale { + return errSnapshotStale + } + // Step one, write the disk root into the journal. + if err := rlp.Encode(journalBuf, dl.root); err != nil { + return err + } + // Step two, write the corresponding state id into the journal + if err := rlp.Encode(journalBuf, dl.id); err != nil { + return err + } + // Step three, write all unwritten nodes into the journal + bufferNodes := dl.buffer.getAllNodes() + nodes := make([]journalNode, 0, len(bufferNodes)) + for _, v := range bufferNodes { + entry := journalNode{Path: v.K, Blob: v.V} + nodes = append(nodes, entry) + } + if err := rlp.Encode(journalBuf, nodes); err != nil { + return err + } + + // Store the journal buf into w and calculate checksum + if journalType == JournalFileType { + shasum := sha256.Sum256(journalBuf.Bytes()) + if err := rlp.Encode(w, journalBuf.Bytes()); err != nil { + return err + } + if err := rlp.Encode(w, shasum); err != nil { + return err + } + } else { + if _, err := w.Write(journalBuf.Bytes()); err != nil { + return err + } + } + + log.Info("Journaled pathdb disk layer", "root", dl.root, "nodes", len(bufferNodes)) + return nil +} + +// journal implements the layer interface, writing the memory layer contents +// into a buffer to be stored in the database as the layer journal. +func (dl *diffLayer) journal(w io.Writer, journalType JournalType) error { + dl.lock.RLock() + defer dl.lock.RUnlock() + + // journal the parent first + if err := dl.parent.journal(w, journalType); err != nil { + return err + } + // Create a buffer to store encoded data + journalBuf := new(bytes.Buffer) + // Everything below was journaled, persist this layer too + if err := rlp.Encode(journalBuf, dl.root); err != nil { + return err + } + if err := rlp.Encode(journalBuf, dl.block); err != nil { + return err + } + // Write the accumulated trie nodes into buffer + nodes := make([]journalNode, 0, len(dl.nodes)) + for _, v := range dl.nodes { + entry := journalNode{Path: v.K, Blob: v.V} + nodes = append(nodes, entry) + } + if err := rlp.Encode(journalBuf, nodes); err != nil { + return err + } + + // Store the journal buf into w and calculate checksum + if journalType == JournalFileType { + shasum := sha256.Sum256(journalBuf.Bytes()) + if err := rlp.Encode(w, journalBuf.Bytes()); err != nil { + return err + } + if err := rlp.Encode(w, shasum); err != nil { + return err + } + } else { + if _, err := w.Write(journalBuf.Bytes()); err != nil { + return err + } + } + + log.Info("Journaled pathdb diff layer", "root", dl.root, "parent", dl.parent.rootHash(), "id", dl.stateID(), "block", dl.block, "nodes", len(dl.nodes)) + return nil +} + +// Journal commits an entire diff hierarchy to disk into a single journal entry. +// This is meant to be used during shutdown to persist the layer without +// flattening everything down (bad for reorgs). And this function will mark the +// database as read-only to prevent all following mutation to disk. +func (db *Database) Journal(root common.Hash) error { + // Run the journaling + db.lock.Lock() + defer db.lock.Unlock() + + // Retrieve the head layer to journal from. + l := db.tree.get(root) + if l == nil { + return fmt.Errorf("triedb layer [%#x] missing", root) + } + disk := db.tree.bottom() + if l, ok := l.(*diffLayer); ok { + log.Info("Persisting dirty state to disk", "head", l.block, "root", root, "layers", l.id-disk.id+disk.buffer.getLayers()) + } else { // disk layer only on noop runs (likely) or deep reorgs (unlikely) + log.Info("Persisting dirty state to disk", "root", root, "layers", disk.buffer.getLayers()) + } + start := time.Now() + + // wait and stop the flush trienodebuffer, for asyncnodebuffer need fixed diskroot + disk.buffer.waitAndStopFlushing() + // Short circuit if the database is in read only mode. + if db.readOnly { + return errDatabaseReadOnly + } + // Firstly write out the metadata of journal + db.DeleteTrieJournal(db.diskdb) + journal := newJournalWriter(db.config.JournalFilePath, db.diskdb, db.DetermineJournalTypeForWriter()) + defer journal.Close() + + if err := rlp.Encode(journal, journalVersion); err != nil { + return err + } + // The stored state in disk might be empty, convert the + // root to emptyRoot in this case. + _, diskroot := rawdb.ReadAccountTrieNode(db.diskdb, zkt.TrieRootPathKey[:]) + + // Secondly write out the state root in disk, ensure all layers + // on top are continuous with disk. + if err := rlp.Encode(journal, diskroot); err != nil { + return err + } + // Finally write out the journal of each layer in reverse order. + if err := l.journal(journal, db.DetermineJournalTypeForWriter()); err != nil { + return err + } + // Store the journal into the database and return + journalSize := journal.Size() + + // Set the db in read only mode to reject all following mutations + db.readOnly = true + log.Info("Persisted dirty state to disk", "size", common.StorageSize(journalSize), "elapsed", common.PrettyDuration(time.Since(start))) + return nil +} diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go new file mode 100644 index 000000000..68a78dac1 --- /dev/null +++ b/triedb/pathdb/layertree.go @@ -0,0 +1,301 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package pathdb + +import ( + "errors" + "fmt" + "sync" + + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/log" + dbtypes "github.com/morph-l2/go-ethereum/triedb/types" +) + +// layerTree is a group of state layers identified by the state root. +// This structure defines a few basic operations for manipulating +// state layers linked with each other in a tree structure. It's +// thread-safe to use. However, callers need to ensure the thread-safety +// of the referenced layer by themselves. +type layerTree struct { + lock sync.RWMutex + layers map[common.Hash]layer +} + +// newLayerTree constructs the layerTree with the given head layer. +func newLayerTree(head layer) *layerTree { + tree := new(layerTree) + tree.reset(head) + return tree +} + +// reset initializes the layerTree by the given head layer. +// All the ancestors will be iterated out and linked in the tree. +func (tree *layerTree) reset(head layer) { + tree.lock.Lock() + defer tree.lock.Unlock() + + for _, ly := range tree.layers { + if dl, ok := ly.(*diffLayer); ok { + // Clean up the hash cache of difflayers due to reset. + dl.cache.Remove(dl) + } + } + + var layers = make(map[common.Hash]layer) + for head != nil { + layers[head.rootHash()] = head + head = head.parentLayer() + } + tree.layers = layers +} + +// get retrieves a layer belonging to the given state root. +func (tree *layerTree) get(root common.Hash) layer { + tree.lock.RLock() + defer tree.lock.RUnlock() + + return tree.layers[root] +} + +// forEach iterates the stored layers inside and applies the +// given callback on them. +func (tree *layerTree) forEach(onLayer func(layer)) { + tree.lock.RLock() + defer tree.lock.RUnlock() + + for _, layer := range tree.layers { + onLayer(layer) + } +} + +// len returns the number of layers cached. +func (tree *layerTree) len() int { + tree.lock.RLock() + defer tree.lock.RUnlock() + + return len(tree.layers) +} + +// add inserts a new layer into the tree if it can be linked to an existing old parent. +func (tree *layerTree) add(root common.Hash, parentRoot common.Hash, block uint64, nodes dbtypes.KvMap) error { + // Reject noop updates to avoid self-loops. This is a special case that can + // happen for clique networks and proof-of-stake networks where empty blocks + // don't modify the state (0 block subsidy). + // + // Although we could silently ignore this internally, it should be the caller's + // responsibility to avoid even attempting to insert such a layer. + if root == parentRoot { + return errors.New("layer cycle") + } + if tree.get(root) != nil { + log.Info("Skip add repeated difflayer", "root", root.String(), "block_id", block) + return nil + } + parent := tree.get(parentRoot) + if parent == nil { + return fmt.Errorf("triedb parent [%#x] layer missing", parentRoot) + } + l := parent.update(root, parent.stateID()+1, block, nodes.Flatten()) + + // Before adding layertree, update the hash cache. + l.cache.Add(l) + + tree.lock.Lock() + tree.layers[l.rootHash()] = l + tree.lock.Unlock() + return nil +} + +// cap traverses downwards the diff tree until the number of allowed diff layers +// are crossed. All diffs beyond the permitted number are flattened downwards. +func (tree *layerTree) cap(root common.Hash, layers int) error { + // Retrieve the head layer to cap from + l := tree.get(root) + if l == nil { + return fmt.Errorf("triedb layer [%#x] missing", root) + } + diff, ok := l.(*diffLayer) + if !ok { + return fmt.Errorf("triedb layer [%#x] is disk layer", root) + } + tree.lock.Lock() + defer tree.lock.Unlock() + + // If full commit was requested, flatten the diffs and merge onto disk + if layers == 0 { + base, err := diff.persist(true) + if err != nil { + return err + } + for _, ly := range tree.layers { + if dl, ok := ly.(*diffLayer); ok { + dl.cache.Remove(dl) + log.Debug("Cleanup difflayer hash cache due to cap all", "diff_root", dl.root.String(), "diff_block_number", dl.block) + } + } + // Replace the entire layer tree with the flat base + tree.layers = map[common.Hash]layer{base.rootHash(): base} + log.Debug("Cap all difflayers to disklayer", "disk_root", base.rootHash().String()) + return nil + } + // Dive until we run out of layers or reach the persistent database + for i := 0; i < layers-1; i++ { + // If we still have diff layers below, continue down + if parent, ok := diff.parentLayer().(*diffLayer); ok { + diff = parent + } else { + // Diff stack too shallow, return without modifications + return nil + } + } + var persisted *diskLayer + // We're out of layers, flatten anything below, stopping if it's the disk or if + // the memory limit is not yet exceeded. + switch parent := diff.parentLayer().(type) { + case *diskLayer: + return nil + + case *diffLayer: + // Hold the lock to prevent any read operations until the new + // parent is linked correctly. + diff.lock.Lock() + + base, err := parent.persist(false) + if err != nil { + diff.lock.Unlock() + return err + } + tree.layers[base.rootHash()] = base + diff.parent = base + + diff.lock.Unlock() + persisted = base.(*diskLayer) + + default: + panic(fmt.Sprintf("unknown data layer in triedb: %T", parent)) + } + // Remove any layer that is stale or links into a stale layer + children := make(map[common.Hash][]common.Hash) + for root, layer := range tree.layers { + if dl, ok := layer.(*diffLayer); ok { + parent := dl.parentLayer().rootHash() + children[parent] = append(children[parent], root) + } + } + var remove func(root common.Hash) + remove = func(root common.Hash) { + if df, exist := tree.layers[root]; exist { + if dl, ok := df.(*diffLayer); ok { + // Clean up the hash cache of the child difflayer corresponding to the stale parent, include the re-org case. + dl.cache.Remove(dl) + log.Debug("Cleanup difflayer hash cache due to reorg", "diff_root", dl.root.String(), "diff_block_number", dl.block) + } + } + delete(tree.layers, root) + for _, child := range children[root] { + remove(child) + } + delete(children, root) + } + for root, layer := range tree.layers { + if dl, ok := layer.(*diskLayer); ok && dl.isStale() { + remove(root) + log.Debug("Remove stale the disklayer", "disk_root", dl.root.String()) + } + } + + if persisted != nil { + var updateOriginFunc func(root common.Hash) + updateOriginFunc = func(root common.Hash) { + if diff, ok := tree.layers[root].(*diffLayer); ok { + diff.lock.Lock() + diff.origin = persisted + diff.lock.Unlock() + } + for _, child := range children[root] { + updateOriginFunc(child) + } + } + updateOriginFunc(persisted.root) + } + + return nil +} + +// bottom returns the bottom-most disk layer in this tree. +func (tree *layerTree) bottom() *diskLayer { + tree.lock.RLock() + defer tree.lock.RUnlock() + + if len(tree.layers) == 0 { + return nil // Shouldn't happen, empty tree + } + // pick a random one as the entry point + var current layer + for _, layer := range tree.layers { + current = layer + break + } + for current.parentLayer() != nil { + current = current.parentLayer() + } + return current.(*diskLayer) +} + +// front return the top non-fork difflayer/disklayer root hash for rewinding. +func (tree *layerTree) front() common.Hash { + tree.lock.RLock() + defer tree.lock.RUnlock() + + chain := make(map[common.Hash][]common.Hash) + var base common.Hash + for _, layer := range tree.layers { + switch dl := layer.(type) { + case *diskLayer: + if dl.stale { + log.Info("pathdb top disklayer is stale") + return base + } + base = dl.rootHash() + case *diffLayer: + if _, ok := chain[dl.parentLayer().rootHash()]; !ok { + chain[dl.parentLayer().rootHash()] = make([]common.Hash, 0) + } + chain[dl.parentLayer().rootHash()] = append(chain[dl.parentLayer().rootHash()], dl.rootHash()) + default: + log.Crit("unsupported layer type") + } + } + if (base == common.Hash{}) { + log.Info("pathdb top difflayer is empty") + return base + } + parent := base + for { + children, ok := chain[parent] + if !ok { + log.Info("pathdb top difflayer", "root", parent) + return parent + } + if len(children) != 1 { + log.Info("pathdb top difflayer is forked", "common ancestor root", parent) + return parent + } + parent = children[0] + } +} diff --git a/triedb/pathdb/metrics.go b/triedb/pathdb/metrics.go new file mode 100644 index 000000000..1573f275a --- /dev/null +++ b/triedb/pathdb/metrics.go @@ -0,0 +1,56 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package pathdb + +import "github.com/morph-l2/go-ethereum/metrics" + +var ( + cleanHitMeter = metrics.NewRegisteredMeter("pathdb/clean/hit", nil) + cleanMissMeter = metrics.NewRegisteredMeter("pathdb/clean/miss", nil) + cleanReadMeter = metrics.NewRegisteredMeter("pathdb/clean/read", nil) + cleanWriteMeter = metrics.NewRegisteredMeter("pathdb/clean/write", nil) + + dirtyHitMeter = metrics.NewRegisteredMeter("pathdb/dirty/hit", nil) + dirtyMissMeter = metrics.NewRegisteredMeter("pathdb/dirty/miss", nil) + dirtyReadMeter = metrics.NewRegisteredMeter("pathdb/dirty/read", nil) + dirtyWriteMeter = metrics.NewRegisteredMeter("pathdb/dirty/write", nil) + dirtyNodeHitDepthHist = metrics.NewRegisteredHistogram("pathdb/dirty/depth", nil, metrics.NewExpDecaySample(1028, 0.015)) + + cleanFalseMeter = metrics.NewRegisteredMeter("pathdb/clean/false", nil) + dirtyFalseMeter = metrics.NewRegisteredMeter("pathdb/dirty/false", nil) + diskFalseMeter = metrics.NewRegisteredMeter("pathdb/disk/false", nil) + + commitTimeTimer = metrics.NewRegisteredTimer("pathdb/commit/time", nil) + commitNodesMeter = metrics.NewRegisteredMeter("pathdb/commit/nodes", nil) + commitBytesMeter = metrics.NewRegisteredMeter("pathdb/commit/bytes", nil) + + gcNodesMeter = metrics.NewRegisteredMeter("pathdb/gc/nodes", nil) + gcBytesMeter = metrics.NewRegisteredMeter("pathdb/gc/bytes", nil) + + diffLayerBytesMeter = metrics.NewRegisteredMeter("pathdb/diff/bytes", nil) + diffLayerNodesMeter = metrics.NewRegisteredMeter("pathdb/diff/nodes", nil) + + historyBuildTimeMeter = metrics.NewRegisteredTimer("pathdb/history/time", nil) + historyDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/data", nil) + historyIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/index", nil) + + diffHashCacheHitMeter = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/hit", nil) + diffHashCacheReadMeter = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/read", nil) + diffHashCacheMissMeter = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/miss", nil) + diffHashCacheSlowPathMeter = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/slowpath", nil) + diffHashCacheLengthGauge = metrics.NewRegisteredGauge("pathdb/difflayer/hashcache/size", nil) +) diff --git a/triedb/pathdb/nodebuffer.go b/triedb/pathdb/nodebuffer.go new file mode 100644 index 000000000..05c475c52 --- /dev/null +++ b/triedb/pathdb/nodebuffer.go @@ -0,0 +1,205 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "bytes" + "fmt" + "time" + + "github.com/VictoriaMetrics/fastcache" + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/core/rawdb" + "github.com/morph-l2/go-ethereum/ethdb" + "github.com/morph-l2/go-ethereum/log" + dbtypes "github.com/morph-l2/go-ethereum/triedb/types" +) + +var _ trienodebuffer = &nodebuffer{} + +// nodebuffer is a collection of modified trie nodes to aggregate the disk +// write. The content of the nodebuffer must be checked before diving into +// disk (since it basically is not-yet-written data). +type nodebuffer struct { + layers uint64 // The number of diff layers aggregated inside + size uint64 // The size of aggregated writes + limit uint64 // The maximum memory allowance in bytes + nodes dbtypes.KvMap // The dirty node set, mapped by owner and path +} + +// newNodeBuffer initializes the node buffer with the provided nodes. +func newNodeBuffer(limit int, nodes dbtypes.KvMap, layers uint64) *nodebuffer { + if nodes == nil { + nodes = make(dbtypes.KvMap) + } + var size uint64 + for _, v := range nodes { + size += uint64(len(v.K) + len(v.K)) + } + return &nodebuffer{ + layers: layers, + nodes: nodes, + size: size, + limit: uint64(limit), + } +} + +// node retrieves the trie node with given node info. +func (b *nodebuffer) node(path []byte) ([]byte, error) { + n, ok := b.nodes.Get(path) + if !ok { + return nil, nil + } + + return n, nil +} + +// commit merges the dirty nodes into the nodebuffer. This operation won't take +// the ownership of the nodes map which belongs to the bottom-most diff layer. +// It will just hold the node references from the given map which are safe to +// copy. +func (b *nodebuffer) commit(nodes dbtypes.KvMap) trienodebuffer { + var ( + delta int64 + overwrite int64 + overwriteSize int64 + ) + + for _, v := range nodes { + current, exist := b.nodes.Get(v.K) + if !exist { + b.nodes.Put(v.K, v.V) + delta += int64(len(v.K) + len(v.V)) + + continue + } + + if !bytes.Equal(current, v.V) { + delta += int64(len(v.V) - len(current)) + overwrite++ + overwriteSize += int64(len(v.V) + len(v.K)) + } + + b.nodes.Put(v.K, v.V) + } + + b.updateSize(delta) + b.layers++ + gcNodesMeter.Mark(overwrite) + gcBytesMeter.Mark(overwriteSize) + return b +} + +// updateSize updates the total cache size by the given delta. +func (b *nodebuffer) updateSize(delta int64) { + size := int64(b.size) + delta + if size >= 0 { + b.size = uint64(size) + return + } + s := b.size + b.size = 0 + log.Error("Invalid pathdb buffer size", "prev", common.StorageSize(s), "delta", common.StorageSize(delta)) +} + +// reset cleans up the disk cache. +func (b *nodebuffer) reset() { + b.layers = 0 + b.size = 0 + b.nodes = make(dbtypes.KvMap) +} + +// empty returns an indicator if nodebuffer contains any state transition inside. +func (b *nodebuffer) empty() bool { + return b.layers == 0 +} + +// setSize sets the buffer size to the provided number, and invokes a flush +// operation if the current memory usage exceeds the new limit. +func (b *nodebuffer) setSize(size int, db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64) error { + b.limit = uint64(size) + return b.flush(db, clean, id, false) +} + +// flush persists the in-memory dirty trie node into the disk if the configured +// memory threshold is reached. Note, all data must be written atomically. +func (b *nodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64, force bool) error { + if b.size <= b.limit && !force { + return nil + } + // Ensure the target state id is aligned with the internal counter. + head := rawdb.ReadPersistentStateID(db) + if head+b.layers != id { + return fmt.Errorf("buffer layers (%d) cannot be applied on top of persisted state id (%d) to reach requested state id (%d)", b.layers, head, id) + } + var ( + start = time.Now() + // Although the calculation of b.size has been as accurate as possible, + // some omissions were still found during testing and code review, but + // we are still not sure if it is completely accurate. For better protection, + // some redundancy is added here. + batch = db.NewBatchWithSize(int(float64(b.size) * DefaultBatchRedundancyRate)) + ) + nodes := writeNodes(batch, b.nodes, clean) + rawdb.WritePersistentStateID(batch, id) + + // Flush all mutations in a single batch + size := batch.ValueSize() + if err := batch.Write(); err != nil { + return err + } + commitBytesMeter.Mark(int64(size)) + commitNodesMeter.Mark(int64(nodes)) + commitTimeTimer.UpdateSince(start) + log.Debug("Persisted pathdb nodes", "nodes", len(b.nodes), "bytes", common.StorageSize(size), "elapsed", common.PrettyDuration(time.Since(start))) + b.reset() + return nil +} + +func (b *nodebuffer) waitAndStopFlushing() {} + +// writeNodes writes the trie nodes into the provided database batch. +// Note this function will also inject all the newly written nodes +// into clean cache. +func writeNodes(batch ethdb.Batch, nodes dbtypes.KvMap, clean *fastcache.Cache) (total int) { + for _, v := range nodes { + rawdb.WriteTrieNodeByKey(batch, v.K, v.V) + + if clean != nil { + clean.Set(v.K, v.V) + } + + total += 1 + } + + return total +} + +// getSize return the nodebuffer used size. +func (b *nodebuffer) getSize() (uint64, uint64) { + return b.size, 0 +} + +// getAllNodes return all the trie nodes are cached in nodebuffer. +func (b *nodebuffer) getAllNodes() dbtypes.KvMap { + return b.nodes +} + +// getLayers return the size of cached difflayers. +func (b *nodebuffer) getLayers() uint64 { + return b.layers +} diff --git a/triedb/types/database_types.go b/triedb/types/database_types.go new file mode 100644 index 000000000..93ab5b88a --- /dev/null +++ b/triedb/types/database_types.go @@ -0,0 +1,56 @@ +package types + +import ( + "bytes" + "crypto/sha256" + "errors" +) + +// ErrNotFound is used by the implementations of the interface db.Storage for +// when a key is not found in the storage +var ErrNotFound = errors.New("key not found") + +// KV contains a key (K) and a value (V) +type KV struct { + K []byte + V []byte +} + +// KvMap is a key-value map between a sha256 byte array hash, and a KV struct +type KvMap map[[sha256.Size]byte]KV + +// Get retreives the value respective to a key from the KvMap +func (m KvMap) Get(k []byte) ([]byte, bool) { + v, ok := m[sha256.Sum256(k)] + return v.V, ok +} + +// Put stores a key and a value in the KvMap +func (m KvMap) Put(k, v []byte) { + m[sha256.Sum256(k)] = KV{k, v} +} + +// Flatten returns a two-dimensional map for internal nodes. +func (m KvMap) Flatten() KvMap { + nodes := make(KvMap) + for _, v := range m { + nodes.Put(v.K, v.V) + } + return nodes +} + +// Concat concatenates arrays of bytes +func Concat(vs ...[]byte) []byte { + var b bytes.Buffer + for _, v := range vs { + b.Write(v) + } + return b.Bytes() +} + +// Clone clones a byte array into a new byte array +func Clone(b0 []byte) []byte { + b1 := make([]byte, len(b0)) + copy(b1, b0) + return b1 +} diff --git a/triedb/types/metrics.go b/triedb/types/metrics.go new file mode 100644 index 000000000..a451e99f3 --- /dev/null +++ b/triedb/types/metrics.go @@ -0,0 +1,43 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package types + +import "github.com/morph-l2/go-ethereum/metrics" + +var ( + memcacheCleanHitMeter = metrics.NewRegisteredMeter("trie/memcache/clean/hit", nil) + memcacheCleanMissMeter = metrics.NewRegisteredMeter("trie/memcache/clean/miss", nil) + memcacheCleanReadMeter = metrics.NewRegisteredMeter("trie/memcache/clean/read", nil) + memcacheCleanWriteMeter = metrics.NewRegisteredMeter("trie/memcache/clean/write", nil) + + memcacheDirtyHitMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/hit", nil) + memcacheDirtyMissMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/miss", nil) + memcacheDirtyReadMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/read", nil) + memcacheDirtyWriteMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/write", nil) + + memcacheFlushTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/flush/time", nil) + memcacheFlushNodesMeter = metrics.NewRegisteredMeter("trie/memcache/flush/nodes", nil) + memcacheFlushSizeMeter = metrics.NewRegisteredMeter("trie/memcache/flush/size", nil) + + memcacheGCTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/gc/time", nil) + memcacheGCNodesMeter = metrics.NewRegisteredMeter("trie/memcache/gc/nodes", nil) + memcacheGCSizeMeter = metrics.NewRegisteredMeter("trie/memcache/gc/size", nil) + + memcacheCommitTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/commit/time", nil) + memcacheCommitNodesMeter = metrics.NewRegisteredMeter("trie/memcache/commit/nodes", nil) + memcacheCommitSizeMeter = metrics.NewRegisteredMeter("trie/memcache/commit/size", nil) +) From 44392b79106fbfd79b2772d4ea03d988b81c0444 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 26 Nov 2024 17:59:38 +0800 Subject: [PATCH 02/61] WIP --- core/genesis.go | 1 - trie/database.go | 18 +++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/core/genesis.go b/core/genesis.go index 7e04ff0c9..ce800e65b 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -212,7 +212,6 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override head := rawdb.ReadPersistentStateID(db) var verify bool = true // new states already overide genesis states. - // if trieCfg.MorphZkTrie && (diskroot == types.EmptyRootHash1 || header.Root != diskroot) { if trieCfg.MorphZkTrie && (diskroot == types.GenesisRootHash || head > 0) { verify = false } diff --git a/trie/database.go b/trie/database.go index 7f7ea5710..1ae47c04a 100644 --- a/trie/database.go +++ b/trie/database.go @@ -330,6 +330,7 @@ type backend interface { // to disk. Report specifies whether logs will be displayed in info level. CommitState(root common.Hash, parentRoot common.Hash, blockNumber uint64, report bool) error + // Commit write custom nodes belong genesis states, only onece CommitGenesis(root common.Hash) error // Close closes the trie database backend and releases all held resources. @@ -382,10 +383,8 @@ func NewDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *Database } if db.MorphZkTrie { - log.Info("PathDB", "NewDatabaseWithConfig use morphzktrie", db.MorphZkTrie) db.backend = pathdb.New(diskdb, config.PathDB) } else { - log.Info("PathDB", "NewDatabaseWithConfig use zktrie", db.Zktrie) if db.Zktrie { db.backend = hashdb.NewZkDatabaseWithConfig(diskdb, config.HashDB) } @@ -1064,7 +1063,7 @@ func (db *Database) Journal(root common.Hash) error { if db.backend != nil { pdb, ok := db.backend.(*pathdb.Database) if !ok { - return errors.New("not supported") + return errors.New("backend [Journal] not supported") } return pdb.Journal(root) } @@ -1097,7 +1096,7 @@ func (db *Database) Get(key []byte) ([]byte, error) { if db.backend != nil { zdb, ok := db.backend.(*hashdb.ZktrieDatabase) if !ok { - return nil, errors.New("not supported") + return nil, errors.New("backend [Get] not supported") } return zdb.Get(key) } @@ -1108,11 +1107,16 @@ func (db *Database) GetFrom(root, key []byte) ([]byte, error) { if db.backend != nil { pdb, ok := db.backend.(*pathdb.Database) if !ok { - return nil, errors.New("backend GetFrom not supported") + return nil, errors.New("backend [GetFrom] not supported") + } + + r := common.BytesToHash(zkt.ReverseByteOrder(root[:])) + reader, _ := pdb.Reader(r) + if reader != nil { + return reader.Node(key) } - reader, _ := pdb.Reader(common.BytesToHash(zkt.ReverseByteOrder(root[:]))) - return reader.Node(key) + log.Info("pathdb get node reader is nill", "root", r.Hex()) } return nil, nil } From cb759c68c563844d5ce76657f76a70933c4ce878 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 26 Nov 2024 23:02:53 +0800 Subject: [PATCH 03/61] WIP --- trie/database.go | 14 ++++++++++++-- triedb/pathdb/database.go | 10 +++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/trie/database.go b/trie/database.go index 1ae47c04a..87424e1ae 100644 --- a/trie/database.go +++ b/trie/database.go @@ -36,6 +36,7 @@ import ( "github.com/morph-l2/go-ethereum/rlp" "github.com/morph-l2/go-ethereum/triedb/hashdb" "github.com/morph-l2/go-ethereum/triedb/pathdb" + zktrie "github.com/scroll-tech/zktrie/trie" zkt "github.com/scroll-tech/zktrie/types" ) @@ -1113,10 +1114,19 @@ func (db *Database) GetFrom(root, key []byte) ([]byte, error) { r := common.BytesToHash(zkt.ReverseByteOrder(root[:])) reader, _ := pdb.Reader(r) if reader != nil { - return reader.Node(key) + n, err := reader.Node(key) + if err != nil { + return nil, err + } + + if n == nil { + return nil, zktrie.ErrKeyNotFound + } + return n, nil } - log.Info("pathdb get node reader is nill", "root", r.Hex()) + log.Info("pathdb get node reader is nil", "root", r.Hex()) + return nil, errors.New("reader is nil") } return nil, nil } diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index a504755f4..715aaf92c 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -230,12 +230,20 @@ func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockN return err } + // flush persists every 3600 blocks + force := blockNumber%3600 == 0 + + layers := maxDiffLayers + if force { + layers = 0 + } + // Keep 128 diff layers in the memory, persistent layer is 129th. // - head layer is paired with HEAD state // - head-1 layer is paired with HEAD-1 state // - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state // - head-128 layer(disk layer) is paired with HEAD-128 state - return db.tree.cap(root, maxDiffLayers) + return db.tree.cap(root, layers) } // Close closes the trie database and the held freezer. From fade32fe02d89d5335acde45a4050483760719f7 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Wed, 27 Nov 2024 18:29:21 +0800 Subject: [PATCH 04/61] WIP --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e789185ea..709aa24c1 100644 --- a/go.mod +++ b/go.mod @@ -127,4 +127,4 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/scroll-tech/zktrie v0.8.4 => github.com/ryanmorphl2/zktrie v1.8.9 +replace github.com/scroll-tech/zktrie v0.8.4 => github.com/ryanmorphl2/zktrie v1.9.1 diff --git a/go.sum b/go.sum index a2c0a749f..c4ebcc210 100644 --- a/go.sum +++ b/go.sum @@ -432,8 +432,8 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/ryanmorphl2/zktrie v1.8.9 h1:Ssjldo56GNjiVDJLc8NwhHJL+ZRC2AdgkhviVrtbEts= -github.com/ryanmorphl2/zktrie v1.8.9/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk= +github.com/ryanmorphl2/zktrie v1.9.1 h1:pvxuOGppyrLmvIbxkMunUpVGdrCQi/KPnNtw4oE/KaI= +github.com/ryanmorphl2/zktrie v1.9.1/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= From 4814a62c9d40d2d5ccffda13aa045fe6b9d7a1ce Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Wed, 27 Nov 2024 22:49:00 +0800 Subject: [PATCH 05/61] WIP --- triedb/pathdb/asyncnodebuffer.go | 4 ++-- triedb/pathdb/database.go | 3 ++- triedb/pathdb/nodebuffer.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/triedb/pathdb/asyncnodebuffer.go b/triedb/pathdb/asyncnodebuffer.go index 9d256ebab..1fa9885d5 100644 --- a/triedb/pathdb/asyncnodebuffer.go +++ b/triedb/pathdb/asyncnodebuffer.go @@ -327,11 +327,11 @@ func (nc *nodecache) merge(nc1 *nodecache) (*nodecache, error) { res.limit = immutable.limit res.nodes = make(dbtypes.KvMap) for _, v := range immutable.nodes { - res.nodes.Put(v.K, v.K) + res.nodes.Put(v.K, v.V) } for _, v := range mutable.nodes { - res.nodes.Put(v.K, v.K) + res.nodes.Put(v.K, v.V) } return res, nil diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 715aaf92c..f3c233af1 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -216,7 +216,7 @@ func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockN return err } - // only 1 entry, state not change + // only 1 entity, state have no changes // some block maybe has no txns, so state do not change if root == parentRoot && len(db.dirties) == 1 { return nil @@ -231,6 +231,7 @@ func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockN } // flush persists every 3600 blocks + // @todo, use config force := blockNumber%3600 == 0 layers := maxDiffLayers diff --git a/triedb/pathdb/nodebuffer.go b/triedb/pathdb/nodebuffer.go index 05c475c52..2b640fa5b 100644 --- a/triedb/pathdb/nodebuffer.go +++ b/triedb/pathdb/nodebuffer.go @@ -48,7 +48,7 @@ func newNodeBuffer(limit int, nodes dbtypes.KvMap, layers uint64) *nodebuffer { } var size uint64 for _, v := range nodes { - size += uint64(len(v.K) + len(v.K)) + size += uint64(len(v.K) + len(v.V)) } return &nodebuffer{ layers: layers, From 0a527a0552ba0184f0f09f77befa01c274fcfbc8 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Wed, 4 Dec 2024 17:29:45 +0800 Subject: [PATCH 06/61] WIP --- cmd/geth/dbcmd.go | 14 +++++++++++++- core/blockchain_l2.go | 2 ++ core/rawdb/accessors_trie.go | 25 +++++++++++++++++++++++++ core/rawdb/database.go | 22 ++++++++++++++++++++++ trie/database.go | 3 ++- trie/hbss2pbss.go | 9 +++++++-- triedb/hashdb/zk_trie_database.go | 4 ++-- triedb/pathdb/database.go | 20 +++++++++++--------- triedb/pathdb/difflayer.go | 9 +++++++++ triedb/pathdb/layertree.go | 1 + 10 files changed, 94 insertions(+), 15 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 7bd232077..d7b6a4e5e 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -735,5 +735,17 @@ func hbss2pbss(ctx *cli.Context) error { return errors.New("too many arguments") } - return h2p.Run() + lastStateID := rawdb.ReadPersistentStateID(chaindb) + if lastStateID == 0 { + h2p.Run() + } + + // prune hbss trie node + err = rawdb.PruneHashTrieNodeInDataBase(chaindb) + if err != nil { + log.Error("Prune Hash trie node in database failed", "error", err) + return err + } + + return nil } diff --git a/core/blockchain_l2.go b/core/blockchain_l2.go index e1eb6232d..c6e8d82a8 100644 --- a/core/blockchain_l2.go +++ b/core/blockchain_l2.go @@ -121,6 +121,8 @@ func (bc *BlockChain) writeBlockStateWithoutHead(block *types.Block, receipts [] // If we're running an archive node, always flush if bc.cacheConfig.TrieDirtyDisabled { if triedb.Scheme() == rawdb.PathScheme { + // If node is running in path mode, skip explicit gc operation + // which is unnecessary in this mode. return triedb.CommitState(root, origin, current, false) } return triedb.Commit(root, false, nil) diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go index 245b306b1..df16594ea 100644 --- a/core/rawdb/accessors_trie.go +++ b/core/rawdb/accessors_trie.go @@ -17,6 +17,8 @@ package rawdb import ( + "bytes" + "github.com/morph-l2/go-ethereum/common" "github.com/morph-l2/go-ethereum/ethdb" @@ -64,3 +66,26 @@ func ReadAccountTrieNode(db ethdb.KeyValueReader, path []byte) ([]byte, common.H return data, common.BytesToHash(zkHash.Bytes()) } + +// IsLegacyTrieNode reports whether a provided database entry is a legacy trie +// node. The characteristics of legacy trie node are: +// - the key length is 32 bytes +// - the key is the hash of val +func IsLegacyTrieNode(key []byte, val []byte) bool { + if len(key) != common.HashLength { + return false + } + + n, err := zktrie.NewNodeFromBytes(val) + if err != nil { + return false + } + + zkHash, err := n.NodeHash() + if err != nil { + return false + } + + hash := common.BytesToHash(common.BitReverse(zkHash[:])) + return bytes.Equal(key[:], hash[:]) +} diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 01e339d34..fcfdbdbef 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -468,3 +468,25 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { return nil } + +func PruneHashTrieNodeInDataBase(db ethdb.Database) error { + it := db.NewIterator([]byte{}, []byte{}) + defer it.Release() + + total_num := 0 + for it.Next() { + var key = it.Key() + switch { + case IsLegacyTrieNode(key, it.Value()): + db.Delete(key) + total_num++ + if total_num%100000 == 0 { + log.Info("Pruning hash-base state trie nodes", "Complete progress: ", total_num) + } + default: + continue + } + } + log.Info("Pruning hash-base state trie nodes", "Complete progress", total_num) + return nil +} diff --git a/trie/database.go b/trie/database.go index 87424e1ae..2ec8714cb 100644 --- a/trie/database.go +++ b/trie/database.go @@ -1022,7 +1022,8 @@ func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockN } return db.backend.CommitState(root, parentRoot, blockNumber, report) } - return db.Commit(root, report, nil) + + return errors.New("not supported") } // Scheme returns the node scheme used in the database. diff --git a/trie/hbss2pbss.go b/trie/hbss2pbss.go index 2309b5730..47404833b 100644 --- a/trie/hbss2pbss.go +++ b/trie/hbss2pbss.go @@ -24,6 +24,7 @@ type Hbss2Pbss struct { datadir string trieCachePath string errChan chan error + headerBlock *types.Block } func NewHbss2Pbss(chaindb ethdb.Database, datadir, trieCachePath string) (*Hbss2Pbss, error) { @@ -33,7 +34,7 @@ func NewHbss2Pbss(chaindb ethdb.Database, datadir, trieCachePath string) (*Hbss2 headBlock := rawdb.ReadHeadBlock(chaindb) root := headBlock.Root() - log.Info("current head block", "block number", headBlock.NumberU64(), "root", root.Hex()) + log.Info("Hbss2pbss converting", "block number", headBlock.NumberU64(), "root", root.Hex(), "hash", headBlock.Header().Hash().Hex()) zkTrie, err := NewZkTrie(root, stateCache) if err != nil { return nil, err @@ -46,6 +47,7 @@ func NewHbss2Pbss(chaindb ethdb.Database, datadir, trieCachePath string) (*Hbss2 trieCachePath: trieCachePath, errChan: make(chan error), wg: &sync.WaitGroup{}, + headerBlock: headBlock, }, nil } @@ -82,6 +84,9 @@ func (h2p *Hbss2Pbss) Run() error { } log.Info("Hbss2Pbss complete", "elapsed", common.PrettyDuration(time.Since(start))) + rawdb.WritePersistentStateID(h2p.db, h2p.headerBlock.NumberU64()) + rawdb.WriteStateID(h2p.db, h2p.headerBlock.Root(), h2p.headerBlock.NumberU64()) + return nil } @@ -96,7 +101,7 @@ func (h2p *Hbss2Pbss) handleGenesis() error { } genesisRoot := genesis.Root() - log.Info(":", "genesistRoot:", genesisRoot.Hex()) + log.Info("Hbss2Pbss converting genesis", "root", genesisRoot.Hex()) h2p.concurrentTraversal(zkt.NewHashFromBytes(genesisRoot[:]), []bool{}, common.Hash{}) diff --git a/triedb/hashdb/zk_trie_database.go b/triedb/hashdb/zk_trie_database.go index a02511462..35eb7569c 100644 --- a/triedb/hashdb/zk_trie_database.go +++ b/triedb/hashdb/zk_trie_database.go @@ -136,7 +136,7 @@ func (db *ZktrieDatabase) Node(hash common.Hash) ([]byte, error) { // Put saves a key:value into the Storage func (db *ZktrieDatabase) Put(k, v []byte) error { - k = common.ReverseBytes(k) + k = common.BitReverse(k) db.lock.Lock() db.dirties.Put(k, v) @@ -152,7 +152,7 @@ func (db *ZktrieDatabase) Put(k, v []byte) error { // Get retrieves a value from a key in the Storage func (db *ZktrieDatabase) Get(key []byte) ([]byte, error) { - key = common.ReverseBytes(key[:]) + key = common.BitReverse(key[:]) db.lock.RLock() value, ok := db.dirties.Get(key) diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index f3c233af1..7a4e3ecb4 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -226,25 +226,27 @@ func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockN for k := range db.dirties { delete(db.dirties, k) } + + clearNodes := func() { + // clear tmp nodes + for k := range nodes { + delete(nodes, k) + } + } + if err := db.tree.add(root, parentRoot, blockNumber, nodes); err != nil { + clearNodes() return err } - // flush persists every 3600 blocks - // @todo, use config - force := blockNumber%3600 == 0 - - layers := maxDiffLayers - if force { - layers = 0 - } + clearNodes() // Keep 128 diff layers in the memory, persistent layer is 129th. // - head layer is paired with HEAD state // - head-1 layer is paired with HEAD-1 state // - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state // - head-128 layer(disk layer) is paired with HEAD-128 state - return db.tree.cap(root, layers) + return db.tree.cap(root, maxDiffLayers) } // Close closes the trie database and the held freezer. diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go index 1b1e6fa5a..aa9f923ce 100644 --- a/triedb/pathdb/difflayer.go +++ b/triedb/pathdb/difflayer.go @@ -303,3 +303,12 @@ func diffToDisk(layer *diffLayer, force bool) (layer, error) { } return disk.commit(layer, force) } + +func (dl *diffLayer) reset() { + // Hold the lock, ensure the parent won't be changed during the + // state accessing. + dl.lock.RLock() + defer dl.lock.RUnlock() + + dl.nodes = make(dbtypes.KvMap) +} diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go index 68a78dac1..a019b970d 100644 --- a/triedb/pathdb/layertree.go +++ b/triedb/pathdb/layertree.go @@ -145,6 +145,7 @@ func (tree *layerTree) cap(root common.Hash, layers int) error { for _, ly := range tree.layers { if dl, ok := ly.(*diffLayer); ok { dl.cache.Remove(dl) + dl.reset() log.Debug("Cleanup difflayer hash cache due to cap all", "diff_root", dl.root.String(), "diff_block_number", dl.block) } } From 01f4ce755fe6e3c8555da141df01dc6c08eb51b0 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Wed, 4 Dec 2024 21:23:41 +0800 Subject: [PATCH 07/61] WIP --- cmd/geth/dbcmd.go | 5 +++++ core/rawdb/database.go | 10 ++++++++-- trie/database.go | 1 - trie/hbss2pbss.go | 21 +++++++++++++++++++++ 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index d7b6a4e5e..96611c892 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -746,6 +746,11 @@ func hbss2pbss(ctx *cli.Context) error { log.Error("Prune Hash trie node in database failed", "error", err) return err } + err = h2p.Compact() + if err != nil { + log.Error("Compact trie node failed", "error", err) + return err + } return nil } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index fcfdbdbef..475a61183 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -473,6 +473,8 @@ func PruneHashTrieNodeInDataBase(db ethdb.Database) error { it := db.NewIterator([]byte{}, []byte{}) defer it.Release() + start := time.Now() + logged := time.Now() total_num := 0 for it.Next() { var key = it.Key() @@ -481,12 +483,16 @@ func PruneHashTrieNodeInDataBase(db ethdb.Database) error { db.Delete(key) total_num++ if total_num%100000 == 0 { - log.Info("Pruning hash-base state trie nodes", "Complete progress: ", total_num) + log.Info("Pruning hash-base state trie nodes", "Complete progress: ", total_num, "elapsed", common.PrettyDuration(time.Since(start))) } default: + if time.Since(logged) > 8*time.Second { + log.Info("Pruning hash-base state trie nodes", "Complete progress: ", total_num, "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } continue } } - log.Info("Pruning hash-base state trie nodes", "Complete progress", total_num) + log.Info("Pruning hash-base state trie nodes", "Complete progress", total_num, "elapsed", common.PrettyDuration(time.Since(start))) return nil } diff --git a/trie/database.go b/trie/database.go index 2ec8714cb..99284eb15 100644 --- a/trie/database.go +++ b/trie/database.go @@ -1126,7 +1126,6 @@ func (db *Database) GetFrom(root, key []byte) ([]byte, error) { return n, nil } - log.Info("pathdb get node reader is nil", "root", r.Hex()) return nil, errors.New("reader is nil") } return nil, nil diff --git a/trie/hbss2pbss.go b/trie/hbss2pbss.go index 47404833b..dae49ad40 100644 --- a/trie/hbss2pbss.go +++ b/trie/hbss2pbss.go @@ -108,6 +108,27 @@ func (h2p *Hbss2Pbss) handleGenesis() error { return nil } +func (h2p *Hbss2Pbss) Compact() error { + cstart := time.Now() + for b := 0x00; b <= 0xf0; b += 0x10 { + var ( + start = []byte{byte(b)} + end = []byte{byte(b + 0x10)} + ) + if b == 0xf0 { + end = nil + } + log.Info("Compacting database", "range", fmt.Sprintf("%#x-%#x", start, end), "elapsed", common.PrettyDuration(time.Since(cstart))) + if err := h2p.db.Compact(start, end); err != nil { + log.Error("Database compaction failed", "error", err) + return err + } + } + log.Info("Database compaction finished", "elapsed", common.PrettyDuration(time.Since(cstart))) + + return nil +} + func (h2p *Hbss2Pbss) writeNode(pathKey []bool, n *zktrie.Node, owner common.Hash) { if owner == (common.Hash{}) { h, _ := n.NodeHash() From 811ebabdfafef7a4ae7dac70fc217a087bee97cc Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Fri, 6 Dec 2024 15:04:00 +0800 Subject: [PATCH 08/61] WIP --- core/genesis.go | 6 ++---- core/rawdb/accessors_state.go | 8 ++++++++ core/types/block.go | 7 +++---- trie/hbss2pbss.go | 8 +++++--- triedb/pathdb/database.go | 3 +++ 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/core/genesis.go b/core/genesis.go index ce800e65b..9f7a6c489 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -38,7 +38,6 @@ import ( "github.com/morph-l2/go-ethereum/params" "github.com/morph-l2/go-ethereum/rlp" "github.com/morph-l2/go-ethereum/trie" - zkt "github.com/scroll-tech/zktrie/types" ) //go:generate gencodec -type Genesis -field-override genesisSpecMarshaling -out gen_genesis.go @@ -208,11 +207,10 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override trieCfg = &trie.Config{Zktrie: genesis.Config.Morph.ZktrieEnabled(), MorphZkTrie: genesis.Config.Morph.MorphZktrieEnabled()} } - _, diskroot := rawdb.ReadAccountTrieNode(db, zkt.TrieRootPathKey[:]) - head := rawdb.ReadPersistentStateID(db) + exist := rawdb.ExistsStateID(db, header.Root) var verify bool = true // new states already overide genesis states. - if trieCfg.MorphZkTrie && (diskroot == types.GenesisRootHash || head > 0) { + if trieCfg.MorphZkTrie && exist { verify = false } diff --git a/core/rawdb/accessors_state.go b/core/rawdb/accessors_state.go index 155a2b2bd..b06fe9898 100644 --- a/core/rawdb/accessors_state.go +++ b/core/rawdb/accessors_state.go @@ -221,3 +221,11 @@ func DeleteStateID(db ethdb.KeyValueWriter, root common.Hash) { log.Crit("Failed to delete state ID", "err", err) } } + +func ExistsStateID(db ethdb.KeyValueReader, root common.Hash) bool { + has, err := db.Has(stateIDKey(root)) + if err != nil { + return false + } + return has +} diff --git a/core/types/block.go b/core/types/block.go index 2fd4a032f..e14252dad 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -32,10 +32,9 @@ import ( ) var ( - EmptyRootHash = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") - EmptyUncleHash = rlpHash([]*Header(nil)) - EmptyAddress = common.Address{} - GenesisRootHash = common.HexToHash("09688bec5d876538664e62247c2f64fc7a02c54a3f898b42020730c7dd4933aa") + EmptyRootHash = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + EmptyUncleHash = rlpHash([]*Header(nil)) + EmptyAddress = common.Address{} ) // A BlockNonce is a 64-bit hash which proves (combined with the diff --git a/trie/hbss2pbss.go b/trie/hbss2pbss.go index dae49ad40..41f4a54ea 100644 --- a/trie/hbss2pbss.go +++ b/trie/hbss2pbss.go @@ -60,7 +60,10 @@ func (h2p *Hbss2Pbss) Run() error { } // Write genesis in new state db - h2p.handleGenesis() + err := h2p.handleGenesis() + if err != nil { + return err + } // Convert hbss state db to new state db root, err := h2p.zkTrie.Tree().Root() @@ -131,9 +134,8 @@ func (h2p *Hbss2Pbss) Compact() error { func (h2p *Hbss2Pbss) writeNode(pathKey []bool, n *zktrie.Node, owner common.Hash) { if owner == (common.Hash{}) { - h, _ := n.NodeHash() rawdb.WriteAccountTrieNode(h2p.db, zkt.PathToKey(pathKey)[:], n.CanonicalValue()) - log.Debug("WriteNodes account trie node", "type", n.Type, "path", zkt.PathToString(pathKey), "hash", h.Hex()) + log.Debug("WriteNodes account trie node", "type", n.Type, "path", zkt.PathToString(pathKey)) } else { rawdb.WriteStorageTrieNode(h2p.db, owner, zkt.PathToKey(pathKey)[:], n.CanonicalValue()) log.Debug("WriteNodes storage trie node", "owner", owner.Hex(), "type", n.Type, "path", zkt.PathToString(pathKey)) diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 7a4e3ecb4..847e77e9f 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -200,6 +200,9 @@ func (db *Database) CommitGenesis(root common.Hash) error { return err } batch.Reset() + + // Update stateID + rawdb.WriteStateID(db.diskdb, root, 0) return nil } From bcf0b6c985d6adaabe56d74f70eaaed60fd02999 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Fri, 6 Dec 2024 17:08:14 +0800 Subject: [PATCH 09/61] WIP --- triedb/pathdb/difflayer_test.go | 170 -------------------------------- 1 file changed, 170 deletions(-) delete mode 100644 triedb/pathdb/difflayer_test.go diff --git a/triedb/pathdb/difflayer_test.go b/triedb/pathdb/difflayer_test.go deleted file mode 100644 index a062878be..000000000 --- a/triedb/pathdb/difflayer_test.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package pathdb - -// import ( -// "bytes" -// "testing" - -// "github.com/ethereum/go-ethereum/common" -// "github.com/ethereum/go-ethereum/core/rawdb" -// "github.com/ethereum/go-ethereum/trie/testutil" -// "github.com/ethereum/go-ethereum/trie/trienode" -// ) - -// func emptyLayer() *diskLayer { -// return &diskLayer{ -// db: New(rawdb.NewMemoryDatabase(), nil), -// buffer: newNodeBuffer(DefaultDirtyBufferSize, nil, 0), -// } -// } - -// // goos: darwin -// // goarch: arm64 -// // pkg: github.com/ethereum/go-ethereum/trie -// // BenchmarkSearch128Layers -// // BenchmarkSearch128Layers-8 243826 4755 ns/op -// func BenchmarkSearch128Layers(b *testing.B) { benchmarkSearch(b, 0, 128) } - -// // goos: darwin -// // goarch: arm64 -// // pkg: github.com/ethereum/go-ethereum/trie -// // BenchmarkSearch512Layers -// // BenchmarkSearch512Layers-8 49686 24256 ns/op -// func BenchmarkSearch512Layers(b *testing.B) { benchmarkSearch(b, 0, 512) } - -// // goos: darwin -// // goarch: arm64 -// // pkg: github.com/ethereum/go-ethereum/trie -// // BenchmarkSearch1Layer -// // BenchmarkSearch1Layer-8 14062725 88.40 ns/op -// func BenchmarkSearch1Layer(b *testing.B) { benchmarkSearch(b, 127, 128) } - -// func benchmarkSearch(b *testing.B, depth int, total int) { -// var ( -// npath []byte -// nhash common.Hash -// nblob []byte -// ) -// // First, we set up 128 diff layers, with 3K items each -// fill := func(parent layer, index int) *diffLayer { -// nodes := make(map[common.Hash]map[string]*trienode.Node) -// nodes[common.Hash{}] = make(map[string]*trienode.Node) -// for i := 0; i < 3000; i++ { -// var ( -// path = testutil.RandBytes(32) -// node = testutil.RandomNode() -// ) -// nodes[common.Hash{}][string(path)] = trienode.New(node.Hash, node.Blob) -// if npath == nil && depth == index { -// npath = common.CopyBytes(path) -// nblob = common.CopyBytes(node.Blob) -// nhash = node.Hash -// } -// } -// return newDiffLayer(parent, common.Hash{}, 0, 0, nodes, nil) -// } -// var layer layer -// layer = emptyLayer() -// for i := 0; i < total; i++ { -// layer = fill(layer, i) -// } -// b.ResetTimer() - -// var ( -// have []byte -// err error -// ) -// for i := 0; i < b.N; i++ { -// have, err = layer.Node(common.Hash{}, npath, nhash) -// if err != nil { -// b.Fatal(err) -// } -// } -// if !bytes.Equal(have, nblob) { -// b.Fatalf("have %x want %x", have, nblob) -// } -// } - -// // goos: darwin -// // goarch: arm64 -// // pkg: github.com/ethereum/go-ethereum/trie -// // BenchmarkPersist -// // BenchmarkPersist-8 10 111252975 ns/op -// func BenchmarkPersist(b *testing.B) { -// // First, we set up 128 diff layers, with 3K items each -// fill := func(parent layer) *diffLayer { -// nodes := make(map[common.Hash]map[string]*trienode.Node) -// nodes[common.Hash{}] = make(map[string]*trienode.Node) -// for i := 0; i < 3000; i++ { -// var ( -// path = testutil.RandBytes(32) -// node = testutil.RandomNode() -// ) -// nodes[common.Hash{}][string(path)] = trienode.New(node.Hash, node.Blob) -// } -// return newDiffLayer(parent, common.Hash{}, 0, 0, nodes, nil) -// } -// for i := 0; i < b.N; i++ { -// b.StopTimer() -// var layer layer -// layer = emptyLayer() -// for i := 1; i < 128; i++ { -// layer = fill(layer) -// } -// b.StartTimer() - -// dl, ok := layer.(*diffLayer) -// if !ok { -// break -// } -// dl.persist(false) -// } -// } - -// // BenchmarkJournal benchmarks the performance for journaling the layers. -// // -// // BenchmarkJournal -// // BenchmarkJournal-8 10 110969279 ns/op -// func BenchmarkJournal(b *testing.B) { -// b.SkipNow() - -// // First, we set up 128 diff layers, with 3K items each -// fill := func(parent layer) *diffLayer { -// nodes := make(map[common.Hash]map[string]*trienode.Node) -// nodes[common.Hash{}] = make(map[string]*trienode.Node) -// for i := 0; i < 3000; i++ { -// var ( -// path = testutil.RandBytes(32) -// node = testutil.RandomNode() -// ) -// nodes[common.Hash{}][string(path)] = trienode.New(node.Hash, node.Blob) -// } -// // TODO(rjl493456442) a non-nil state set is expected. -// return newDiffLayer(parent, common.Hash{}, 0, 0, nodes, nil) -// } -// var layer layer -// layer = emptyLayer() -// for i := 0; i < 128; i++ { -// layer = fill(layer) -// } -// b.ResetTimer() - -// for i := 0; i < b.N; i++ { -// layer.journal(new(bytes.Buffer), JournalKVType) -// } -// } From 6c25201d0fd69b65b6e979aa4f257a201fcb0464 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Thu, 12 Dec 2024 19:08:59 +0800 Subject: [PATCH 10/61] compact key of storage trie node --- core/rawdb/schema.go | 18 ++++++++++++++---- core/state/state_object.go | 33 +++++++++++++++++++-------------- trie/database.go | 1 + trie/hbss2pbss.go | 3 +++ triedb/pathdb/database.go | 3 ++- 5 files changed, 39 insertions(+), 19 deletions(-) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 3032b826b..5a399f876 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -25,6 +25,7 @@ import ( leveldb "github.com/syndtr/goleveldb/leveldb/errors" "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/crypto" "github.com/morph-l2/go-ethereum/ethdb/memorydb" "github.com/morph-l2/go-ethereum/metrics" ) @@ -327,12 +328,12 @@ func accountTrieNodeKey(path []byte) []byte { return append(TrieNodeAccountPrefix, path...) } -// storageTrieNodeKey = trieNodeStoragePrefix + accountHash + nodePath. +// storageTrieNodeKey = trieNodeStoragePrefix + Keccak256Hash(accountHash + nodePath). func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte { - buf := make([]byte, len(TrieNodeStoragePrefix)+common.HashLength+len(path)) + buf := make([]byte, len(TrieNodeStoragePrefix)+common.HashLength) n := copy(buf, TrieNodeStoragePrefix) - n += copy(buf[n:], accountHash.Bytes()) - copy(buf[n:], path) + h := crypto.Keccak256Hash(append(accountHash.Bytes(), path...)).Bytes() + copy(buf[n:], h) return buf } @@ -340,3 +341,12 @@ func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte { func stateIDKey(root common.Hash) []byte { return append(stateIDPrefix, root.Bytes()...) } + +func CompactStorageTrieNodeKey(key []byte) []byte { + if key[0] == TrieNodeStoragePrefix[0] && len(key) > 33 { + h := crypto.Keccak256Hash(key[1:]).Bytes() + newKey := append(TrieNodeStoragePrefix, h...) + return newKey + } + return key +} diff --git a/core/state/state_object.go b/core/state/state_object.go index e61e8f252..489ad0672 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -68,10 +68,11 @@ func (s Storage) Copy() Storage { // Account values can be accessed and modified through the object. // Finally, call CommitTrie to write the modified storage trie into a database. type stateObject struct { - address common.Address - addrHash common.Hash // hash of ethereum address of the account - data types.StateAccount - db *StateDB + address common.Address + addrHash common.Hash // hash of ethereum address of the account + addrPoseidonHash common.Hash // hash of ethereum address of the account + data types.StateAccount + db *StateDB // DB error. // State objects are used by the consensus core and VM which are @@ -116,14 +117,20 @@ func newObject(db *StateDB, address common.Address, data types.StateAccount) *st if data.Root == (common.Hash{}) { data.Root = db.db.TrieDB().EmptyRoot() } + var addrPoseidonHash common.Hash + if db.db.TrieDB().IsMorphZk() { + addr_s, _ := zkt.ToSecureKey(address.Bytes()) + addrPoseidonHash = common.BigToHash(addr_s) + } return &stateObject{ - db: db, - address: address, - addrHash: crypto.Keccak256Hash(address[:]), - data: data, - originStorage: make(Storage), - pendingStorage: make(Storage), - dirtyStorage: make(Storage), + db: db, + address: address, + addrHash: crypto.Keccak256Hash(address[:]), + addrPoseidonHash: addrPoseidonHash, + data: data, + originStorage: make(Storage), + pendingStorage: make(Storage), + dirtyStorage: make(Storage), } } @@ -167,10 +174,8 @@ func (s *stateObject) getTrie(db Database) Trie { var err error addrHash := s.addrHash if db.TrieDB().IsMorphZk() { - addr_s, _ := zkt.ToSecureKey(s.address.Bytes()) - addrHash = common.BigToHash(addr_s) + addrHash = s.addrPoseidonHash } - s.trie, err = db.OpenStorageTrie(addrHash, s.data.Root, s.db.originalRoot) if err != nil { s.trie, _ = db.OpenStorageTrie(addrHash, common.Hash{}, s.db.originalRoot) diff --git a/trie/database.go b/trie/database.go index 99284eb15..4eba52c39 100644 --- a/trie/database.go +++ b/trie/database.go @@ -1112,6 +1112,7 @@ func (db *Database) GetFrom(root, key []byte) ([]byte, error) { return nil, errors.New("backend [GetFrom] not supported") } + key := rawdb.CompactStorageTrieNodeKey(key[:]) r := common.BytesToHash(zkt.ReverseByteOrder(root[:])) reader, _ := pdb.Reader(r) if reader != nil { diff --git a/trie/hbss2pbss.go b/trie/hbss2pbss.go index 41f4a54ea..745db1787 100644 --- a/trie/hbss2pbss.go +++ b/trie/hbss2pbss.go @@ -108,6 +108,9 @@ func (h2p *Hbss2Pbss) handleGenesis() error { h2p.concurrentTraversal(zkt.NewHashFromBytes(genesisRoot[:]), []bool{}, common.Hash{}) + // Mark genesis root state + rawdb.WriteStateID(h2p.db, genesisRoot, 0) + return nil } diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 847e77e9f..299f090b0 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -392,6 +392,7 @@ func (db *Database) Put(k, v []byte) error { db.lock.Lock() defer db.lock.Unlock() - db.dirties.Put(k, v) + key := rawdb.CompactStorageTrieNodeKey(k[:]) + db.dirties.Put(key, v) return nil } From 62f89aa55ce7cc5155919c21f6e29069b09056dc Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:32:24 +0800 Subject: [PATCH 11/61] rawdb is not found err --- core/rawdb/schema.go | 4 ++++ triedb/hashdb/zk_trie_database.go | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 5a399f876..60e319dbd 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -288,6 +288,10 @@ func isNotFoundErr(err error) bool { return errors.Is(err, leveldb.ErrNotFound) || errors.Is(err, memorydb.ErrMemorydbNotFound) } +func IsNotFoundErr(err error) bool { + return isNotFoundErr(err) +} + // SkippedTransactionKey = skippedTransactionPrefix + tx hash func SkippedTransactionKey(txHash common.Hash) []byte { return append(skippedTransactionPrefix, txHash.Bytes()...) diff --git a/triedb/hashdb/zk_trie_database.go b/triedb/hashdb/zk_trie_database.go index 35eb7569c..d7d191e97 100644 --- a/triedb/hashdb/zk_trie_database.go +++ b/triedb/hashdb/zk_trie_database.go @@ -11,7 +11,6 @@ import ( "github.com/morph-l2/go-ethereum/ethdb" "github.com/morph-l2/go-ethereum/log" "github.com/morph-l2/go-ethereum/triedb/types" - "github.com/syndtr/goleveldb/leveldb" zktrie "github.com/scroll-tech/zktrie/trie" ) @@ -170,7 +169,7 @@ func (db *ZktrieDatabase) Get(key []byte) ([]byte, error) { } v, err := db.diskdb.Get(key) - if err == leveldb.ErrNotFound { + if rawdb.IsNotFoundErr(err) { return nil, zktrie.ErrKeyNotFound } if err != nil && db.cleans != nil { From 5bf5bcb6f915ce8592edb06899d0aaac90a4db9e Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:38:27 +0800 Subject: [PATCH 12/61] remove unused --- triedb/hashdb/metrics.go | 14 ------------- triedb/pathdb/errors.go | 31 ----------------------------- triedb/pathdb/journal.go | 14 ------------- triedb/pathdb/metrics.go | 17 ++++------------ triedb/types/metrics.go | 43 ---------------------------------------- 5 files changed, 4 insertions(+), 115 deletions(-) delete mode 100644 triedb/types/metrics.go diff --git a/triedb/hashdb/metrics.go b/triedb/hashdb/metrics.go index 527c6ba75..7ccb428d8 100644 --- a/triedb/hashdb/metrics.go +++ b/triedb/hashdb/metrics.go @@ -24,20 +24,6 @@ var ( memcacheCleanReadMeter = metrics.NewRegisteredMeter("trie/memcache/clean/read", nil) memcacheCleanWriteMeter = metrics.NewRegisteredMeter("trie/memcache/clean/write", nil) - memcacheDirtyHitMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/hit", nil) - memcacheDirtyMissMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/miss", nil) - memcacheDirtyReadMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/read", nil) - memcacheDirtyWriteMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/write", nil) - - memcacheFlushTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/flush/time", nil) - memcacheFlushNodesMeter = metrics.NewRegisteredMeter("trie/memcache/flush/nodes", nil) - memcacheFlushSizeMeter = metrics.NewRegisteredMeter("trie/memcache/flush/size", nil) - - memcacheGCTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/gc/time", nil) - memcacheGCNodesMeter = metrics.NewRegisteredMeter("trie/memcache/gc/nodes", nil) - memcacheGCSizeMeter = metrics.NewRegisteredMeter("trie/memcache/gc/size", nil) - memcacheCommitTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/commit/time", nil) memcacheCommitNodesMeter = metrics.NewRegisteredMeter("trie/memcache/commit/nodes", nil) - memcacheCommitSizeMeter = metrics.NewRegisteredMeter("trie/memcache/commit/size", nil) ) diff --git a/triedb/pathdb/errors.go b/triedb/pathdb/errors.go index beb03b678..c62b6b0ab 100644 --- a/triedb/pathdb/errors.go +++ b/triedb/pathdb/errors.go @@ -18,10 +18,6 @@ package pathdb import ( "errors" - "fmt" - - "github.com/morph-l2/go-ethereum/common" - "github.com/morph-l2/go-ethereum/common/hexutil" ) var ( @@ -29,27 +25,11 @@ var ( // to prevent any mutation. errDatabaseReadOnly = errors.New("read only") - // errDatabaseWaitSync is returned if the initial state sync is not completed - // yet and database is disabled to prevent accessing state. - errDatabaseWaitSync = errors.New("waiting for sync") - // errSnapshotStale is returned from data accessors if the underlying layer // layer had been invalidated due to the chain progressing forward far enough // to not maintain the layer's original state. errSnapshotStale = errors.New("layer stale") - // errUnexpectedHistory is returned if an unmatched state history is applied - // to the database for state rollback. - errUnexpectedHistory = errors.New("unexpected state history") - - // errStateUnrecoverable is returned if state is required to be reverted to - // a destination without associated state history available. - errStateUnrecoverable = errors.New("state is unrecoverable") - - // errUnexpectedNode is returned if the requested node with specified path is - // not hash matched with expectation. - errUnexpectedNode = errors.New("unexpected node") - // errWriteImmutable is returned if write to background immutable nodecache // under asyncnodebuffer errWriteImmutable = errors.New("write immutable nodecache") @@ -60,15 +40,4 @@ var ( // errIncompatibleMerge is returned when merge node cache occurs error. errIncompatibleMerge = errors.New("incompatible nodecache merge") - - // errRevertImmutable is returned if revert the background immutable nodecache - errRevertImmutable = errors.New("revert immutable nodecache") ) - -func newUnexpectedNodeError(loc string, expHash common.Hash, gotHash common.Hash, owner common.Hash, path []byte, blob []byte) error { - blobHex := "nil" - if len(blob) > 0 { - blobHex = hexutil.Encode(blob) - } - return fmt.Errorf("%w, loc: %s, node: (%x %v), %x!=%x, blob: %s", errUnexpectedNode, loc, owner, path, expHash, gotHash, blobHex) -} diff --git a/triedb/pathdb/journal.go b/triedb/pathdb/journal.go index e8639ff40..6a8aedbd8 100644 --- a/triedb/pathdb/journal.go +++ b/triedb/pathdb/journal.go @@ -53,20 +53,6 @@ type journalNode struct { Blob []byte // RLP-encoded trie node blob, nil means the node is deleted } -// journalAccounts represents a list accounts belong to the layer. -type journalAccounts struct { - Addresses []common.Address - Accounts [][]byte -} - -// journalStorage represents a list of storage slots belong to an account. -type journalStorage struct { - Incomplete bool - Account common.Address - Hashes []common.Hash - Slots [][]byte -} - type JournalWriter interface { io.Writer diff --git a/triedb/pathdb/metrics.go b/triedb/pathdb/metrics.go index 1573f275a..90cb1c2f0 100644 --- a/triedb/pathdb/metrics.go +++ b/triedb/pathdb/metrics.go @@ -24,15 +24,10 @@ var ( cleanReadMeter = metrics.NewRegisteredMeter("pathdb/clean/read", nil) cleanWriteMeter = metrics.NewRegisteredMeter("pathdb/clean/write", nil) - dirtyHitMeter = metrics.NewRegisteredMeter("pathdb/dirty/hit", nil) - dirtyMissMeter = metrics.NewRegisteredMeter("pathdb/dirty/miss", nil) - dirtyReadMeter = metrics.NewRegisteredMeter("pathdb/dirty/read", nil) - dirtyWriteMeter = metrics.NewRegisteredMeter("pathdb/dirty/write", nil) - dirtyNodeHitDepthHist = metrics.NewRegisteredHistogram("pathdb/dirty/depth", nil, metrics.NewExpDecaySample(1028, 0.015)) - - cleanFalseMeter = metrics.NewRegisteredMeter("pathdb/clean/false", nil) - dirtyFalseMeter = metrics.NewRegisteredMeter("pathdb/dirty/false", nil) - diskFalseMeter = metrics.NewRegisteredMeter("pathdb/disk/false", nil) + dirtyHitMeter = metrics.NewRegisteredMeter("pathdb/dirty/hit", nil) + dirtyMissMeter = metrics.NewRegisteredMeter("pathdb/dirty/miss", nil) + dirtyReadMeter = metrics.NewRegisteredMeter("pathdb/dirty/read", nil) + dirtyWriteMeter = metrics.NewRegisteredMeter("pathdb/dirty/write", nil) commitTimeTimer = metrics.NewRegisteredTimer("pathdb/commit/time", nil) commitNodesMeter = metrics.NewRegisteredMeter("pathdb/commit/nodes", nil) @@ -44,10 +39,6 @@ var ( diffLayerBytesMeter = metrics.NewRegisteredMeter("pathdb/diff/bytes", nil) diffLayerNodesMeter = metrics.NewRegisteredMeter("pathdb/diff/nodes", nil) - historyBuildTimeMeter = metrics.NewRegisteredTimer("pathdb/history/time", nil) - historyDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/data", nil) - historyIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/index", nil) - diffHashCacheHitMeter = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/hit", nil) diffHashCacheReadMeter = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/read", nil) diffHashCacheMissMeter = metrics.NewRegisteredMeter("pathdb/difflayer/hashcache/miss", nil) diff --git a/triedb/types/metrics.go b/triedb/types/metrics.go deleted file mode 100644 index a451e99f3..000000000 --- a/triedb/types/metrics.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see - -package types - -import "github.com/morph-l2/go-ethereum/metrics" - -var ( - memcacheCleanHitMeter = metrics.NewRegisteredMeter("trie/memcache/clean/hit", nil) - memcacheCleanMissMeter = metrics.NewRegisteredMeter("trie/memcache/clean/miss", nil) - memcacheCleanReadMeter = metrics.NewRegisteredMeter("trie/memcache/clean/read", nil) - memcacheCleanWriteMeter = metrics.NewRegisteredMeter("trie/memcache/clean/write", nil) - - memcacheDirtyHitMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/hit", nil) - memcacheDirtyMissMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/miss", nil) - memcacheDirtyReadMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/read", nil) - memcacheDirtyWriteMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/write", nil) - - memcacheFlushTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/flush/time", nil) - memcacheFlushNodesMeter = metrics.NewRegisteredMeter("trie/memcache/flush/nodes", nil) - memcacheFlushSizeMeter = metrics.NewRegisteredMeter("trie/memcache/flush/size", nil) - - memcacheGCTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/gc/time", nil) - memcacheGCNodesMeter = metrics.NewRegisteredMeter("trie/memcache/gc/nodes", nil) - memcacheGCSizeMeter = metrics.NewRegisteredMeter("trie/memcache/gc/size", nil) - - memcacheCommitTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/commit/time", nil) - memcacheCommitNodesMeter = metrics.NewRegisteredMeter("trie/memcache/commit/nodes", nil) - memcacheCommitSizeMeter = metrics.NewRegisteredMeter("trie/memcache/commit/size", nil) -) From bafb67070ac111a29f228e2440966bf79cf60d2f Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:39:52 +0800 Subject: [PATCH 13/61] Copy nodes opt --- triedb/pathdb/database.go | 12 +----------- triedb/pathdb/layertree.go | 2 +- triedb/types/database_types.go | 4 ++-- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 299f090b0..9cf167643 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -225,25 +225,15 @@ func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockN return nil } - nodes := db.dirties.Flatten() + nodes := db.dirties.Copy() for k := range db.dirties { delete(db.dirties, k) } - clearNodes := func() { - // clear tmp nodes - for k := range nodes { - delete(nodes, k) - } - } - if err := db.tree.add(root, parentRoot, blockNumber, nodes); err != nil { - clearNodes() return err } - clearNodes() - // Keep 128 diff layers in the memory, persistent layer is 129th. // - head layer is paired with HEAD state // - head-1 layer is paired with HEAD-1 state diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go index a019b970d..cf21811dd 100644 --- a/triedb/pathdb/layertree.go +++ b/triedb/pathdb/layertree.go @@ -110,7 +110,7 @@ func (tree *layerTree) add(root common.Hash, parentRoot common.Hash, block uint6 if parent == nil { return fmt.Errorf("triedb parent [%#x] layer missing", parentRoot) } - l := parent.update(root, parent.stateID()+1, block, nodes.Flatten()) + l := parent.update(root, parent.stateID()+1, block, nodes) // Before adding layertree, update the hash cache. l.cache.Add(l) diff --git a/triedb/types/database_types.go b/triedb/types/database_types.go index 93ab5b88a..b488d9aac 100644 --- a/triedb/types/database_types.go +++ b/triedb/types/database_types.go @@ -30,8 +30,8 @@ func (m KvMap) Put(k, v []byte) { m[sha256.Sum256(k)] = KV{k, v} } -// Flatten returns a two-dimensional map for internal nodes. -func (m KvMap) Flatten() KvMap { +// Copy returns a two-dimensional map for internal nodes. +func (m KvMap) Copy() KvMap { nodes := make(KvMap) for _, v := range m { nodes.Put(v.K, v.V) From 7a2557beaa2e33713dd94e5ef5039d2bc4979a22 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 17 Dec 2024 19:14:41 +0800 Subject: [PATCH 14/61] WIP --- trie/hbss2pbss.go | 13 ------------- triedb/hashdb/zk_trie_database.go | 1 - 2 files changed, 14 deletions(-) diff --git a/trie/hbss2pbss.go b/trie/hbss2pbss.go index 745db1787..9700a7b80 100644 --- a/trie/hbss2pbss.go +++ b/trie/hbss2pbss.go @@ -167,20 +167,7 @@ func (h2p *Hbss2Pbss) concurrentTraversal(nodeHash *zkt.Hash, path []bool, owner if data.CodeSize > 0 { if !bytes.Equal(data.Root[:], common.Hash{}.Bytes()) { - // log.Info("/-------------------\\") - // log.Info("concurrentTraversal", "owner", owner.Hex(), "path", zkt.PathToString(path), "nodeHash", nodeHash.Hex(), "key", n.NodeKey.Hex(), "root", data.Root.Hex()) - - // codeHash := data.KeccakCodeHash h2p.concurrentTraversal(zkt.NewHashFromBytes(data.Root[:]), []bool{}, common.BytesToHash(n.NodeKey[:])) - // h2p.wg.Add(1) - // go func(root, o common.Hash) { - // defer h2p.wg.Done() - // log.Info("/-------------------\\") - // log.Info("concurrentTraversal", "owner", owner.Hex(), "path", zkt.PathToString(path), "nodeHash", nodeHash.Hex(), "key", n.NodeKey.Hex()) - // h2p.concurrentTraversal(zkt.NewHashFromBytes(root[:]), []bool{}, o) - // log.Info("\\-------------------/") - // }(data.Root, common.BytesToHash(n.NodeKey.Bytes())) - // log.Info("\\-------------------/") } } } diff --git a/triedb/hashdb/zk_trie_database.go b/triedb/hashdb/zk_trie_database.go index d7d191e97..aab96307c 100644 --- a/triedb/hashdb/zk_trie_database.go +++ b/triedb/hashdb/zk_trie_database.go @@ -82,7 +82,6 @@ func (db *ZktrieDatabase) CommitState(root common.Hash, parentRoot common.Hash, return err } memcacheCommitTimeTimer.Update(time.Since(start)) - // memcacheCommitBytesMeter.Mark(int64(beforeDirtySize - db.dirtiesSize)) memcacheCommitNodesMeter.Mark(int64(beforeDirtyCount - len(db.dirties))) logger := log.Debug From 5be65e2a454b6628995e033f6650fd9de5cdc149 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 17 Dec 2024 19:16:57 +0800 Subject: [PATCH 15/61] WIP --- triedb/types/database_types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/triedb/types/database_types.go b/triedb/types/database_types.go index b488d9aac..4322b2216 100644 --- a/triedb/types/database_types.go +++ b/triedb/types/database_types.go @@ -30,7 +30,7 @@ func (m KvMap) Put(k, v []byte) { m[sha256.Sum256(k)] = KV{k, v} } -// Copy returns a two-dimensional map for internal nodes. +// Copy returns a map for internal nodes. func (m KvMap) Copy() KvMap { nodes := make(KvMap) for _, v := range m { From 84eaac32b61314a8fce346f0b2212324bc3edc01 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Wed, 18 Dec 2024 10:30:20 +0800 Subject: [PATCH 16/61] copy nodes opt --- triedb/pathdb/database.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 9cf167643..19c5f161f 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -225,14 +225,11 @@ func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockN return nil } - nodes := db.dirties.Copy() - for k := range db.dirties { - delete(db.dirties, k) - } - - if err := db.tree.add(root, parentRoot, blockNumber, nodes); err != nil { + if err := db.tree.add(root, parentRoot, blockNumber, db.dirties); err != nil { + db.dirties = make(dbtypes.KvMap) return err } + db.dirties = make(dbtypes.KvMap) // Keep 128 diff layers in the memory, persistent layer is 129th. // - head layer is paired with HEAD state From 0beacac26daf84dbf4ef5c20f85d9dfcc6871bb1 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Wed, 18 Dec 2024 11:56:40 +0800 Subject: [PATCH 17/61] WIP --- triedb/pathdb/difflayer.go | 6 ++++++ triedb/pathdb/disklayer.go | 4 ++-- triedb/pathdb/journal.go | 4 ++-- triedb/pathdb/layertree.go | 4 +--- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go index aa9f923ce..c0f94bf9f 100644 --- a/triedb/pathdb/difflayer.go +++ b/triedb/pathdb/difflayer.go @@ -193,6 +193,12 @@ func (dl *diffLayer) originDiskLayer() *diskLayer { return dl.origin } +func (dl *diffLayer) updateOriginDiskLayer(persistLayer *diskLayer) { + dl.lock.Lock() + defer dl.lock.Unlock() + dl.origin = persistLayer +} + // rootHash implements the layer interface, returning the root hash of // corresponding state. func (dl *diffLayer) rootHash() common.Hash { diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go index 03e418ee8..a275029a4 100644 --- a/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -104,7 +104,7 @@ func newDiskLayer(root common.Hash, id uint64, db *Database, cleans *fastcache.C } } -// root implements the layer interface, returning root hash of corresponding state. +// rootHash implements the layer interface, returning root hash of corresponding state. func (dl *diskLayer) rootHash() common.Hash { return dl.root } @@ -114,7 +114,7 @@ func (dl *diskLayer) stateID() uint64 { return dl.id } -// parent implements the layer interface, returning nil as there's no layer +// parentLayer implements the layer interface, returning nil as there's no layer // below the disk. func (dl *diskLayer) parentLayer() layer { return nil diff --git a/triedb/pathdb/journal.go b/triedb/pathdb/journal.go index 6a8aedbd8..f140d5606 100644 --- a/triedb/pathdb/journal.go +++ b/triedb/pathdb/journal.go @@ -500,11 +500,11 @@ func (db *Database) Journal(root common.Hash) error { } // The stored state in disk might be empty, convert the // root to emptyRoot in this case. - _, diskroot := rawdb.ReadAccountTrieNode(db.diskdb, zkt.TrieRootPathKey[:]) + _, diskRoot := rawdb.ReadAccountTrieNode(db.diskdb, zkt.TrieRootPathKey[:]) // Secondly write out the state root in disk, ensure all layers // on top are continuous with disk. - if err := rlp.Encode(journal, diskroot); err != nil { + if err := rlp.Encode(journal, diskRoot); err != nil { return err } // Finally write out the journal of each layer in reverse order. diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go index cf21811dd..dcb6575a5 100644 --- a/triedb/pathdb/layertree.go +++ b/triedb/pathdb/layertree.go @@ -224,9 +224,7 @@ func (tree *layerTree) cap(root common.Hash, layers int) error { var updateOriginFunc func(root common.Hash) updateOriginFunc = func(root common.Hash) { if diff, ok := tree.layers[root].(*diffLayer); ok { - diff.lock.Lock() - diff.origin = persisted - diff.lock.Unlock() + diff.updateOriginDiskLayer(persisted) } for _, child := range children[root] { updateOriginFunc(child) From feb48bb80fa5e4db8f0546eeec8916b602f2fccd Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Wed, 18 Dec 2024 12:10:23 +0800 Subject: [PATCH 18/61] buffer: get node opt --- triedb/pathdb/asyncnodebuffer.go | 17 +++++++---------- triedb/pathdb/disklayer.go | 11 ++++------- triedb/pathdb/nodebuffer.go | 6 +++--- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/triedb/pathdb/asyncnodebuffer.go b/triedb/pathdb/asyncnodebuffer.go index 1fa9885d5..ec0f1220b 100644 --- a/triedb/pathdb/asyncnodebuffer.go +++ b/triedb/pathdb/asyncnodebuffer.go @@ -45,18 +45,15 @@ func newAsyncNodeBuffer(limit int, nodes dbtypes.KvMap, layers uint64) *asyncnod } // node retrieves the trie node with given node info. -func (a *asyncnodebuffer) node(path []byte) ([]byte, error) { +func (a *asyncnodebuffer) node(path []byte) ([]byte, bool) { a.mux.RLock() defer a.mux.RUnlock() - node, err := a.current.node(path) - if err != nil { - return nil, err - } + node := a.current.node(path) if node == nil { - return a.background.node(path) + node = a.background.node(path) } - return node, nil + return node, node != nil } // commit merges the dirty nodes into the nodebuffer. This operation won't take @@ -196,12 +193,12 @@ func newNodeCache(limit, size uint64, nodes dbtypes.KvMap, layers uint64) *nodec } } -func (nc *nodecache) node(path []byte) ([]byte, error) { +func (nc *nodecache) node(path []byte) []byte { n, ok := nc.nodes.Get(path) if ok { - return n, nil + return n } - return nil, nil + return nil } func (nc *nodecache) commit(nodes dbtypes.KvMap) error { diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go index a275029a4..64accca48 100644 --- a/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -32,7 +32,7 @@ import ( // disk (since it basically is not-yet-written data). type trienodebuffer interface { // node retrieves the trie node with given node info. - node(path []byte) ([]byte, error) + node(path []byte) ([]byte, bool) // commit merges the dirty nodes into the trienodebuffer. This operation won't take // the ownership of the nodes map which belongs to the bottom-most diff layer. @@ -153,11 +153,8 @@ func (dl *diskLayer) Node(path []byte) ([]byte, error) { // node buffer first. Note the buffer is lock free since // it's impossible to mutate the buffer before tagging the // layer as stale. - n, err := dl.buffer.node(path) - if err != nil { - return nil, err - } - if n != nil { + n, found := dl.buffer.node(path) + if found { dirtyHitMeter.Mark(1) dirtyReadMeter.Mark(int64(len(n))) return n, nil @@ -176,7 +173,7 @@ func (dl *diskLayer) Node(path []byte) ([]byte, error) { } // Try to retrieve the trie node from the disk. - n, err = rawdb.ReadTrieNodeByKey(dl.db.diskdb, path) + n, err := rawdb.ReadTrieNodeByKey(dl.db.diskdb, path) if err == nil { if dl.cleans != nil && len(n) > 0 { dl.cleans.Set(key, n) diff --git a/triedb/pathdb/nodebuffer.go b/triedb/pathdb/nodebuffer.go index 2b640fa5b..abec62bb2 100644 --- a/triedb/pathdb/nodebuffer.go +++ b/triedb/pathdb/nodebuffer.go @@ -59,13 +59,13 @@ func newNodeBuffer(limit int, nodes dbtypes.KvMap, layers uint64) *nodebuffer { } // node retrieves the trie node with given node info. -func (b *nodebuffer) node(path []byte) ([]byte, error) { +func (b *nodebuffer) node(path []byte) ([]byte, bool) { n, ok := b.nodes.Get(path) if !ok { - return nil, nil + return nil, false } - return n, nil + return n, true } // commit merges the dirty nodes into the nodebuffer. This operation won't take From a0ed014d80569d9edb8b2c74b5a58ed07f2e1887 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Thu, 19 Dec 2024 13:44:36 +0800 Subject: [PATCH 19/61] rename trie --- cmd/geth/chaincmd.go | 3 +- cmd/geth/main.go | 2 +- cmd/geth/usage.go | 2 +- cmd/utils/flags.go | 24 ++++++++---- core/blockchain.go | 17 ++++---- core/genesis.go | 10 ++--- core/state/database.go | 8 ++-- core/state/state_object.go | 4 +- core/state/state_prove.go | 2 +- core/state/statedb.go | 6 +-- eth/backend.go | 2 + eth/ethconfig/config.go | 4 ++ params/config.go | 7 ---- trie/database.go | 45 ++++++++++------------ trie/{morph_zk_trie.go => path_zk_trie.go} | 34 ++++++++-------- 15 files changed, 87 insertions(+), 83 deletions(-) rename trie/{morph_zk_trie.go => path_zk_trie.go} (87%) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 0c9f1ae4c..a575b7422 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -50,6 +50,7 @@ var ( ArgsUsage: "", Flags: []cli.Flag{ utils.DataDirFlag, + utils.PathZkTrieFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -190,7 +191,7 @@ This command dumps out the state for a given block (or latest, if none provided) // the zero'd block (i.e. genesis) or will fail hard if it can't succeed. func initGenesis(ctx *cli.Context) error { // Make sure we have a valid genesis JSON - genesisPath := ctx.Args().First() + genesisPath := ctx.GlobalString(utils.DataDirFlag.Name) if len(genesisPath) == 0 { utils.Fatalf("Must supply path to genesis JSON file") } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 6374ec10c..9c50d1397 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -158,7 +158,7 @@ var ( utils.GpoIgnoreGasPriceFlag, configFileFlag, utils.CatalystFlag, - utils.MorphZkTrieFlag, + utils.PathZkTrieFlag, utils.PathDBSyncFlag, } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 085ba294f..0ea87dd33 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -57,7 +57,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.IdentityFlag, utils.LightKDFFlag, utils.WhitelistFlag, - utils.MorphZkTrieFlag, + utils.PathZkTrieFlag, utils.PathDBSyncFlag, }, }, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 20bc8388e..8744f5c38 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -75,6 +75,7 @@ import ( "github.com/morph-l2/go-ethereum/params" "github.com/morph-l2/go-ethereum/rollup/tracing" "github.com/morph-l2/go-ethereum/rpc" + "github.com/morph-l2/go-ethereum/trie" ) func init() { @@ -852,9 +853,9 @@ var ( Usage: "Limit max fetched block range for `eth_getLogs` method", } - MorphZkTrieFlag = cli.BoolFlag{ - Name: "morphzktrie", - Usage: "Use MorphZkTrie instead of ZkTrie in state", + PathZkTrieFlag = cli.BoolFlag{ + Name: "pathzktrie", + Usage: "Use PathZkTrie instead of ZkTrie in state", } PathDBSyncFlag = cli.BoolFlag{ Name: "pathdb.sync", @@ -1730,6 +1731,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(PathDBSyncFlag.Name) { cfg.PathSyncFlush = true } + if ctx.GlobalIsSet(PathZkTrieFlag.Name) { + cfg.PathZkTrie = true + trie.Defaults.PathZkTrie = true + } + // Override any default configs for hard coded networks. switch { case ctx.GlobalBool(MainnetFlag.Name): @@ -1778,8 +1784,10 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { log.Info("Prefetch disabled") cfg.NoPrefetch = true - // use morph zktrie - cfg.Genesis.Config.Morph.MorphZkTrie = ctx.GlobalBool(MorphZkTrieFlag.Name) + // cheked for path zktrie + if cfg.PathZkTrie && !cfg.Genesis.Config.Morph.ZktrieEnabled() { + log.Crit("Must cooperate with zktrie enable to use --pathzktrie") + } case ctx.GlobalBool(MorphHoleskyFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 2810 @@ -1796,8 +1804,10 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { log.Info("Prefetch disabled") cfg.NoPrefetch = true - // use morph zktrie - cfg.Genesis.Config.Morph.MorphZkTrie = ctx.GlobalBool(MorphZkTrieFlag.Name) + // cheked for path zktrie + if cfg.PathZkTrie && !cfg.Genesis.Config.Morph.ZktrieEnabled() { + log.Crit("Must cooperate with zktrie enable to use --pathzktrie") + } case ctx.GlobalBool(DeveloperFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337 diff --git a/core/blockchain.go b/core/blockchain.go index 4130a0d46..137094064 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -139,6 +139,7 @@ type CacheConfig struct { SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it PathSyncFlush bool // Whether sync flush the trienodebuffer of pathdb to disk. + PathZkTrie bool // Use path zktrie instead of zktrie JournalFilePath string } @@ -246,7 +247,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par var hashdbConfig *hashdb.Config var pathdbConfig *pathdb.Config - if chainConfig.Morph.ZktrieEnabled() && chainConfig.Morph.MorphZktrieEnabled() { + if chainConfig.Morph.ZktrieEnabled() && cacheConfig.PathZkTrie { pathdbConfig = &pathdb.Config{ SyncFlush: cacheConfig.PathSyncFlush, CleanCacheSize: cacheConfig.TrieCleanLimit * 1024 * 1024, @@ -268,13 +269,13 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par db: db, triegc: prque.New(nil), stateCache: state.NewDatabaseWithConfig(db, &trie.Config{ - Cache: cacheConfig.TrieCleanLimit, - Journal: cacheConfig.TrieCleanJournal, - Preimages: cacheConfig.Preimages, - Zktrie: chainConfig.Morph.ZktrieEnabled(), - MorphZkTrie: chainConfig.Morph.ZktrieEnabled() && chainConfig.Morph.MorphZktrieEnabled(), - HashDB: hashdbConfig, - PathDB: pathdbConfig, + Cache: cacheConfig.TrieCleanLimit, + Journal: cacheConfig.TrieCleanJournal, + Preimages: cacheConfig.Preimages, + Zktrie: chainConfig.Morph.ZktrieEnabled(), + PathZkTrie: chainConfig.Morph.ZktrieEnabled() && cacheConfig.PathZkTrie, + HashDB: hashdbConfig, + PathDB: pathdbConfig, }), quit: make(chan struct{}), chainmu: syncx.NewClosableMutex(), diff --git a/core/genesis.go b/core/genesis.go index 9f7a6c489..06bf2538a 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -201,16 +201,16 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override if storedcfg == nil { log.Warn("Found genesis block without chain config") } else { - trieCfg = &trie.Config{Zktrie: storedcfg.Morph.ZktrieEnabled(), MorphZkTrie: storedcfg.Morph.MorphZktrieEnabled()} + trieCfg = &trie.Config{Zktrie: storedcfg.Morph.ZktrieEnabled(), PathZkTrie: trie.Defaults.PathZkTrie} } } else { - trieCfg = &trie.Config{Zktrie: genesis.Config.Morph.ZktrieEnabled(), MorphZkTrie: genesis.Config.Morph.MorphZktrieEnabled()} + trieCfg = &trie.Config{Zktrie: genesis.Config.Morph.ZktrieEnabled(), PathZkTrie: trie.Defaults.PathZkTrie} } exist := rawdb.ExistsStateID(db, header.Root) var verify bool = true // new states already overide genesis states. - if trieCfg.MorphZkTrie && exist { + if trieCfg.PathZkTrie && exist { verify = false } @@ -263,8 +263,6 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override storedcfg.Morph.MaxTxPerBlock = storedcfg.Scroll.MaxTxPerBlock storedcfg.Morph.MaxTxPayloadBytesPerBlock = storedcfg.Scroll.MaxTxPayloadBytesPerBlock storedcfg.Morph.FeeVaultAddress = storedcfg.Scroll.FeeVaultAddress - storedcfg.Morph.MorphZkTrie = storedcfg.Scroll.MorphZkTrie - } // Special case: don't change the existing config of a non-mainnet chain if no new // config is supplied. These chains would get AllProtocolChanges (and a compat error) @@ -315,7 +313,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { } var trieCfg *trie.Config if g.Config != nil { - trieCfg = &trie.Config{Zktrie: g.Config.Morph.ZktrieEnabled(), MorphZkTrie: g.Config.Morph.MorphZktrieEnabled()} + trieCfg = &trie.Config{Zktrie: g.Config.Morph.ZktrieEnabled(), PathZkTrie: trie.Defaults.PathZkTrie} } statedb, err := state.New(common.Hash{}, state.NewDatabaseWithConfig(db, trieCfg), nil) if err != nil { diff --git a/core/state/database.go b/core/state/database.go index f26f9afab..8800cba3f 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -122,7 +122,7 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { csc, _ := lru.New(codeSizeCacheSize) return &cachingDB{ zktrie: config != nil && config.Zktrie, - morphZktrie: config != nil && config.Zktrie && config.MorphZkTrie, + morphZktrie: config != nil && config.Zktrie && config.PathZkTrie, db: trie.NewDatabaseWithConfig(db, config), codeSizeCache: csc, codeCache: lru2.NewSizeConstrainedLRU(codeCacheSize), @@ -140,7 +140,7 @@ type cachingDB struct { // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { if db.morphZktrie { - tr, err := trie.NewMorphZkTrie(root, root, db.db, rawdb.TrieNodeAccountPrefix) + tr, err := trie.NewPathZkTrie(root, root, db.db, rawdb.TrieNodeAccountPrefix) if err != nil { return nil, err } @@ -165,7 +165,7 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { func (db *cachingDB) OpenStorageTrie(addrHash, root, origin common.Hash) (Trie, error) { if db.morphZktrie { prefix := append(rawdb.TrieNodeStoragePrefix, zkt.ReverseByteOrder(addrHash.Bytes())...) - tr, err := trie.NewMorphZkTrie(root, origin, db.db, prefix) + tr, err := trie.NewPathZkTrie(root, origin, db.db, prefix) if err != nil { return nil, err } @@ -193,7 +193,7 @@ func (db *cachingDB) CopyTrie(t Trie) Trie { return t.Copy() case *trie.ZkTrie: return t.Copy() - case *trie.MorphZkTrie: + case *trie.PathZkTrie: return t.Copy() default: panic(fmt.Errorf("unknown trie type %T", t)) diff --git a/core/state/state_object.go b/core/state/state_object.go index 489ad0672..21944e178 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -118,7 +118,7 @@ func newObject(db *StateDB, address common.Address, data types.StateAccount) *st data.Root = db.db.TrieDB().EmptyRoot() } var addrPoseidonHash common.Hash - if db.db.TrieDB().IsMorphZk() { + if db.db.TrieDB().IsPathZkTrie() { addr_s, _ := zkt.ToSecureKey(address.Bytes()) addrPoseidonHash = common.BigToHash(addr_s) } @@ -173,7 +173,7 @@ func (s *stateObject) getTrie(db Database) Trie { if s.trie == nil { var err error addrHash := s.addrHash - if db.TrieDB().IsMorphZk() { + if db.TrieDB().IsPathZkTrie() { addrHash = s.addrPoseidonHash } s.trie, err = db.OpenStorageTrie(addrHash, s.data.Root, s.db.originalRoot) diff --git a/core/state/state_prove.go b/core/state/state_prove.go index dec259104..1ac082fbc 100644 --- a/core/state/state_prove.go +++ b/core/state/state_prove.go @@ -54,7 +54,7 @@ func (s *StateDB) GetStorageTrieForProof(addr common.Address) (Trie, error) { stateObject := s.getStateObject(addr) addrHash := crypto.Keccak256Hash(addr[:]) - if s.IsMorphZktrie() { + if s.IsPathZktrie() { k, err := zkt.ToSecureKey(addr.Bytes()) if err != nil { return nil, fmt.Errorf("can't create storage trie on ToSecureKey %s: %v ", addr.Hex(), err) diff --git a/core/state/statedb.go b/core/state/statedb.go index b50abb22f..05f1299d4 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -194,11 +194,11 @@ func (s *StateDB) Error() error { } func (s *StateDB) IsZktrie() bool { - return s.db.TrieDB().Zktrie + return s.db.TrieDB().IsZkTrie() } -func (s *StateDB) IsMorphZktrie() bool { - return s.db.TrieDB().Zktrie && s.db.TrieDB().IsMorphZk() +func (s *StateDB) IsPathZktrie() bool { + return s.db.TrieDB().IsPathZkTrie() } func (s *StateDB) AddLog(log *types.Log) { diff --git a/eth/backend.go b/eth/backend.go index 7ba630c95..ecc08f24a 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -137,6 +137,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr } + log.Info("Initialised chain configuration", "config", chainConfig) if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, stack.ResolvePath(config.TrieCleanCacheJournal)); err != nil { @@ -192,6 +193,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { SnapshotLimit: config.SnapshotCache, Preimages: config.Preimages, PathSyncFlush: config.PathSyncFlush, + PathZkTrie: config.PathZkTrie, JournalFilePath: stack.ResolvePath(config.JournalFileName), } ) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 06c6eb9a9..11cc63917 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -92,6 +92,7 @@ var Defaults = Config{ MaxBlockRange: -1, // Default unconfigured value: no block range limit for backward compatibility JournalFileName: "trie.journal", PathSyncFlush: true, + PathZkTrie: false, } func init() { @@ -219,6 +220,9 @@ type Config struct { // State scheme used to store ethereum state and merkle trie nodes on top PathSyncFlush bool + + // Use PathZkTrie instead of ZkTrie in state + PathZkTrie bool } // CreateConsensusEngine creates a consensus engine for the given chain config. diff --git a/params/config.go b/params/config.go index 88ebbbf79..48c6755f8 100644 --- a/params/config.go +++ b/params/config.go @@ -564,9 +564,6 @@ type MorphConfig struct { // Transaction fee vault address [optional] FeeVaultAddress *common.Address `json:"feeVaultAddress,omitempty"` - - // Use MorphZkTrie instead of ZkTrie in state - MorphZkTrie bool `json:"morphZkTrie,omitempty"` } func (s MorphConfig) FeeVaultEnabled() bool { @@ -577,10 +574,6 @@ func (s MorphConfig) ZktrieEnabled() bool { return s.UseZktrie } -func (s MorphConfig) MorphZktrieEnabled() bool { - return s.MorphZkTrie -} - func (s MorphConfig) String() string { maxTxPerBlock := "" if s.MaxTxPerBlock != nil { diff --git a/trie/database.go b/trie/database.go index 4eba52c39..d6583c0cc 100644 --- a/trie/database.go +++ b/trie/database.go @@ -76,8 +76,8 @@ type Database struct { diskdb ethdb.KeyValueStore // Persistent storage for matured trie nodes // zktrie related stuff - Zktrie bool - MorphZkTrie bool + Zktrie bool + PathZkTrie bool // TODO: It's a quick&dirty implementation. FIXME later. rawDirties KvMap @@ -286,27 +286,20 @@ func expandNode(hash hashNode, n node) node { // Config defines all necessary options for database. type Config struct { - Cache int // Memory allowance (MB) to use for caching trie nodes in memory - Journal string // Journal of clean cache to survive node restarts - Preimages bool // Flag whether the preimage of trie key is recorded - Zktrie bool // use zktrie - MorphZkTrie bool // use morph zktrie + Cache int // Memory allowance (MB) to use for caching trie nodes in memory + Journal string // Journal of clean cache to survive node restarts + Preimages bool // Flag whether the preimage of trie key is recorded + Zktrie bool // use zktrie + PathZkTrie bool // use path zktrie HashDB *hashdb.Config // Configs for hash-based scheme PathDB *pathdb.Config // Configs for experimental path-based scheme } -// HashDefaults represents a config for using hash-based scheme with -// default settings. -var HashDefaults = &Config{ - Preimages: false, - HashDB: hashdb.Defaults, -} - -var ZkHashDefaults = &Config{ - Preimages: false, - HashDB: hashdb.Defaults, - Zktrie: true, +var Defaults = &Config{ + Preimages: false, + Zktrie: true, + PathZkTrie: false, } // Reader wraps the Node method of a backing trie reader. @@ -374,19 +367,21 @@ func NewDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *Database dirties: map[common.Hash]*cachedNode{{}: { children: make(map[common.Hash]uint16), }}, - rawDirties: make(KvMap), - preimages: preimage, - Zktrie: config != nil && config.Zktrie, - MorphZkTrie: config != nil && config.Zktrie && config.MorphZkTrie, + rawDirties: make(KvMap), + preimages: preimage, + Zktrie: config != nil && config.Zktrie, + PathZkTrie: config != nil && config.Zktrie && config.PathZkTrie, } if config.HashDB != nil && config.PathDB != nil { log.Crit("Both 'hash' and 'path' mode are configured") } - if db.MorphZkTrie { + if db.PathZkTrie { + log.Info("Using pathdb for zktrie backend") db.backend = pathdb.New(diskdb, config.PathDB) } else { if db.Zktrie { + log.Info("Using hashdb for zktrie backend") db.backend = hashdb.NewZkDatabaseWithConfig(diskdb, config.HashDB) } } @@ -1072,8 +1067,8 @@ func (db *Database) Journal(root common.Hash) error { return nil } -func (db *Database) IsZk() bool { return db.Zktrie } -func (db *Database) IsMorphZk() bool { return db.Zktrie && db.MorphZkTrie } +func (db *Database) IsZkTrie() bool { return db.Zktrie } +func (db *Database) IsPathZkTrie() bool { return db.Zktrie && db.PathZkTrie } // ZkTrie database interface func (db *Database) UpdatePreimage(preimage []byte, hashField *big.Int) { diff --git a/trie/morph_zk_trie.go b/trie/path_zk_trie.go similarity index 87% rename from trie/morph_zk_trie.go rename to trie/path_zk_trie.go index 940fb3d60..9c33c3599 100644 --- a/trie/morph_zk_trie.go +++ b/trie/path_zk_trie.go @@ -30,7 +30,7 @@ import ( ) // wrap zktrie for trie interface -type MorphZkTrie struct { +type PathZkTrie struct { *varienttrie.ZkTrie db *Database } @@ -39,20 +39,20 @@ func init() { zkt.InitHashScheme(poseidon.HashFixedWithDomain) } -// NewZkTrie creates a trie -// NewZkTrie bypasses all the buffer mechanism in *Database, it directly uses the +// NewPathZkTrie creates a trie +// NewPathZkTrie bypasses all the buffer mechanism in *Database, it directly uses the // underlying diskdb -func NewMorphZkTrie(root common.Hash, origin common.Hash, db *Database, prefix []byte) (*MorphZkTrie, error) { +func NewPathZkTrie(root common.Hash, origin common.Hash, db *Database, prefix []byte) (*PathZkTrie, error) { tr, err := varienttrie.NewZkTrieWithPrefix(*zkt.NewByte32FromBytes(root.Bytes()), *zkt.NewByte32FromBytes(origin.Bytes()), db, prefix) if err != nil { return nil, err } - return &MorphZkTrie{tr, db}, nil + return &PathZkTrie{tr, db}, nil } // Get returns the value for key stored in the trie. // The value bytes must not be modified by the caller. -func (t *MorphZkTrie) Get(key []byte) []byte { +func (t *PathZkTrie) Get(key []byte) []byte { sanityCheckByte32Key(key) res, err := t.TryGet(key) if err != nil { @@ -63,7 +63,7 @@ func (t *MorphZkTrie) Get(key []byte) []byte { // TryUpdateAccount will abstract the write of an account to the // secure trie. -func (t *MorphZkTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { +func (t *PathZkTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { sanityCheckByte32Key(key) value, flag := acc.MarshalFields() return t.ZkTrie.TryUpdate(key, flag, value) @@ -75,7 +75,7 @@ func (t *MorphZkTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) erro // // The value bytes must not be modified by the caller while they are // stored in the trie. -func (t *MorphZkTrie) Update(key, value []byte) { +func (t *PathZkTrie) Update(key, value []byte) { if err := t.TryUpdate(key, value); err != nil { log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) } @@ -83,13 +83,13 @@ func (t *MorphZkTrie) Update(key, value []byte) { // NOTE: value is restricted to length of bytes32. // we override the underlying zktrie's TryUpdate method -func (t *MorphZkTrie) TryUpdate(key, value []byte) error { +func (t *PathZkTrie) TryUpdate(key, value []byte) error { sanityCheckByte32Key(key) return t.ZkTrie.TryUpdate(key, 1, []zkt.Byte32{*zkt.NewByte32FromBytes(value)}) } // Delete removes any existing value for key from the trie. -func (t *MorphZkTrie) Delete(key []byte) { +func (t *PathZkTrie) Delete(key []byte) { sanityCheckByte32Key(key) if err := t.TryDelete(key); err != nil { log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) @@ -98,7 +98,7 @@ func (t *MorphZkTrie) Delete(key []byte) { // GetKey returns the preimage of a hashed key that was // previously used to store a value. -func (t *MorphZkTrie) GetKey(kHashBytes []byte) []byte { +func (t *PathZkTrie) GetKey(kHashBytes []byte) []byte { // TODO: use a kv cache in memory k, err := zkt.NewBigIntFromHashBytes(kHashBytes) if err != nil { @@ -115,7 +115,7 @@ func (t *MorphZkTrie) GetKey(kHashBytes []byte) []byte { // // Committing flushes nodes from memory. Subsequent Get calls will load nodes // from the database. -func (t *MorphZkTrie) Commit(LeafCallback) (common.Hash, int, error) { +func (t *PathZkTrie) Commit(LeafCallback) (common.Hash, int, error) { if err := t.ZkTrie.Commit(); err != nil { return common.Hash{}, 0, err } @@ -126,20 +126,20 @@ func (t *MorphZkTrie) Commit(LeafCallback) (common.Hash, int, error) { // Hash returns the root hash of SecureBinaryTrie. It does not write to the // database and can be used even if the trie doesn't have one. -func (t *MorphZkTrie) Hash() common.Hash { +func (t *PathZkTrie) Hash() common.Hash { var hash common.Hash hash.SetBytes(t.ZkTrie.Hash()) return hash } // Copy returns a copy of SecureBinaryTrie. -func (t *MorphZkTrie) Copy() *MorphZkTrie { - return &MorphZkTrie{t.ZkTrie.Copy(), t.db} +func (t *PathZkTrie) Copy() *PathZkTrie { + return &PathZkTrie{t.ZkTrie.Copy(), t.db} } // NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration // starts at the key after the given start key. -func (t *MorphZkTrie) NodeIterator(start []byte) NodeIterator { +func (t *PathZkTrie) NodeIterator(start []byte) NodeIterator { /// FIXME panic("not implemented") } @@ -168,7 +168,7 @@ func (t *MorphZkTrie) NodeIterator(start []byte) NodeIterator { // If the trie does not contain a value for key, the returned proof contains all // nodes of the longest existing prefix of the key (at least the root node), ending // with the node that proves the absence of the key. -func (t *MorphZkTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { +func (t *PathZkTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { err := t.ZkTrie.Prove(key, fromLevel, func(n *varienttrie.Node) error { nodeHash, err := n.NodeHash() if err != nil { From bba1e756f3d7b4c85fb31eb3ab13cf27183bbfce Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Thu, 19 Dec 2024 13:48:13 +0800 Subject: [PATCH 20/61] rm unused --- common/types.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/common/types.go b/common/types.go index 42e972738..7035c5941 100644 --- a/common/types.go +++ b/common/types.go @@ -65,11 +65,6 @@ func BigToHash(b *big.Int) Hash { return BytesToHash(b.Bytes()) } // If b is larger than len(h), b will be cropped from the left. func HexToHash(s string) Hash { return BytesToHash(FromHex(s)) } -// Cmp compares two hashes. -func (h Hash) Cmp(other Hash) int { - return bytes.Compare(h[:], other[:]) -} - // Bytes gets the byte representation of the underlying hash. func (h Hash) Bytes() []byte { return h[:] } From d92fee3668fdef0c954defab9b10e97d357960b0 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Thu, 19 Dec 2024 13:58:36 +0800 Subject: [PATCH 21/61] WIP --- core/state/database.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/state/database.go b/core/state/database.go index 8800cba3f..9f2fe896d 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -122,7 +122,7 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { csc, _ := lru.New(codeSizeCacheSize) return &cachingDB{ zktrie: config != nil && config.Zktrie, - morphZktrie: config != nil && config.Zktrie && config.PathZkTrie, + pathZkTrie: config != nil && config.Zktrie && config.PathZkTrie, db: trie.NewDatabaseWithConfig(db, config), codeSizeCache: csc, codeCache: lru2.NewSizeConstrainedLRU(codeCacheSize), @@ -134,12 +134,12 @@ type cachingDB struct { codeSizeCache *lru.Cache codeCache *lru2.SizeConstrainedLRU zktrie bool - morphZktrie bool + pathZkTrie bool } // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { - if db.morphZktrie { + if db.pathZkTrie { tr, err := trie.NewPathZkTrie(root, root, db.db, rawdb.TrieNodeAccountPrefix) if err != nil { return nil, err @@ -163,7 +163,7 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { // OpenStorageTrie opens the storage trie of an account. func (db *cachingDB) OpenStorageTrie(addrHash, root, origin common.Hash) (Trie, error) { - if db.morphZktrie { + if db.pathZkTrie { prefix := append(rawdb.TrieNodeStoragePrefix, zkt.ReverseByteOrder(addrHash.Bytes())...) tr, err := trie.NewPathZkTrie(root, origin, db.db, prefix) if err != nil { From b1210db4e3d535799cc9f23972d435741de3539a Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Thu, 19 Dec 2024 14:14:51 +0800 Subject: [PATCH 22/61] upgrade path zktrie --- go.mod | 2 +- go.sum | 4 ++-- trie/path_zk_trie.go | 20 ++++++++++---------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 709aa24c1..9ea337a00 100644 --- a/go.mod +++ b/go.mod @@ -127,4 +127,4 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/scroll-tech/zktrie v0.8.4 => github.com/ryanmorphl2/zktrie v1.9.1 +replace github.com/scroll-tech/zktrie v0.8.4 => github.com/ryanmorphl2/zktrie v1.9.2 diff --git a/go.sum b/go.sum index c4ebcc210..c7d0effdd 100644 --- a/go.sum +++ b/go.sum @@ -432,8 +432,8 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/ryanmorphl2/zktrie v1.9.1 h1:pvxuOGppyrLmvIbxkMunUpVGdrCQi/KPnNtw4oE/KaI= -github.com/ryanmorphl2/zktrie v1.9.1/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk= +github.com/ryanmorphl2/zktrie v1.9.2 h1:nWnmb7RaNJXVsOtnOD88/AVEUh0dlQyjPVRR64gh+KY= +github.com/ryanmorphl2/zktrie v1.9.2/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= diff --git a/trie/path_zk_trie.go b/trie/path_zk_trie.go index 9c33c3599..10e992080 100644 --- a/trie/path_zk_trie.go +++ b/trie/path_zk_trie.go @@ -19,8 +19,8 @@ package trie import ( "fmt" + "github.com/scroll-tech/zktrie/pathtrie" zkt "github.com/scroll-tech/zktrie/types" - varienttrie "github.com/scroll-tech/zktrie/varienttrie" "github.com/morph-l2/go-ethereum/common" "github.com/morph-l2/go-ethereum/core/types" @@ -31,7 +31,7 @@ import ( // wrap zktrie for trie interface type PathZkTrie struct { - *varienttrie.ZkTrie + *pathtrie.ZkTrie db *Database } @@ -43,7 +43,7 @@ func init() { // NewPathZkTrie bypasses all the buffer mechanism in *Database, it directly uses the // underlying diskdb func NewPathZkTrie(root common.Hash, origin common.Hash, db *Database, prefix []byte) (*PathZkTrie, error) { - tr, err := varienttrie.NewZkTrieWithPrefix(*zkt.NewByte32FromBytes(root.Bytes()), *zkt.NewByte32FromBytes(origin.Bytes()), db, prefix) + tr, err := pathtrie.NewZkTrieWithPrefix(*zkt.NewByte32FromBytes(root.Bytes()), *zkt.NewByte32FromBytes(origin.Bytes()), db, prefix) if err != nil { return nil, err } @@ -169,13 +169,13 @@ func (t *PathZkTrie) NodeIterator(start []byte) NodeIterator { // nodes of the longest existing prefix of the key (at least the root node), ending // with the node that proves the absence of the key. func (t *PathZkTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { - err := t.ZkTrie.Prove(key, fromLevel, func(n *varienttrie.Node) error { + err := t.ZkTrie.Prove(key, fromLevel, func(n *pathtrie.Node) error { nodeHash, err := n.NodeHash() if err != nil { return err } - if n.Type == varienttrie.NodeTypeLeaf_New { + if n.Type == pathtrie.NodeTypeLeaf_New { preImage := t.GetKey(n.NodeKey.Bytes()) if len(preImage) > 0 { n.KeyPreimage = &zkt.Byte32{} @@ -191,7 +191,7 @@ func (t *PathZkTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWri // we put this special kv pair in db so we can distinguish the type and // make suitable Proof - return proofDb.Put(magicHash, varienttrie.ProofMagicBytes()) + return proofDb.Put(magicHash, pathtrie.ProofMagicBytes()) } // VerifyProof checks merkle proofs. The given proof must contain the value for @@ -204,12 +204,12 @@ func VerifyProofSMT2(rootHash common.Hash, key []byte, proofDb ethdb.KeyValueRea return nil, err } - proof, n, err := varienttrie.BuildZkTrieProof(h, k, len(key)*8, func(key *zkt.Hash) (*varienttrie.Node, error) { + proof, n, err := pathtrie.BuildZkTrieProof(h, k, len(key)*8, func(key *zkt.Hash) (*pathtrie.Node, error) { buf, _ := proofDb.Get(key[:]) if buf == nil { - return nil, varienttrie.ErrKeyNotFound + return nil, pathtrie.ErrKeyNotFound } - n, err := varienttrie.NewNodeFromBytes(buf) + n, err := pathtrie.NewNodeFromBytes(buf) return n, err }) @@ -220,7 +220,7 @@ func VerifyProofSMT2(rootHash common.Hash, key []byte, proofDb ethdb.KeyValueRea return nil, nil } - if varienttrie.VerifyProofZkTrie(h, proof, n) { + if pathtrie.VerifyProofZkTrie(h, proof, n) { return n.Data(), nil } else { return nil, fmt.Errorf("bad proof node %v", proof) From c72970d6f5902ce608f1e22045bd91c8554ba6d0 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Thu, 19 Dec 2024 15:55:29 +0800 Subject: [PATCH 23/61] path trie proof --- trie/path_zk_trie.go | 4 +++- trie/proof.go | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/trie/path_zk_trie.go b/trie/path_zk_trie.go index 10e992080..a0981a687 100644 --- a/trie/path_zk_trie.go +++ b/trie/path_zk_trie.go @@ -29,6 +29,8 @@ import ( "github.com/morph-l2/go-ethereum/log" ) +var magicHashPathTrie []byte = []byte("THIS IS THE MAGIC INDEX FOR PATH ZKTRIE") + // wrap zktrie for trie interface type PathZkTrie struct { *pathtrie.ZkTrie @@ -191,7 +193,7 @@ func (t *PathZkTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWri // we put this special kv pair in db so we can distinguish the type and // make suitable Proof - return proofDb.Put(magicHash, pathtrie.ProofMagicBytes()) + return proofDb.Put(magicHashPathTrie, pathtrie.ProofMagicBytes()) } // VerifyProof checks merkle proofs. The given proof must contain the value for diff --git a/trie/proof.go b/trie/proof.go index b89b01e0c..60a0d854e 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -110,6 +110,11 @@ func VerifyProof(rootHash common.Hash, key []byte, proofDb ethdb.KeyValueReader) return VerifyProofSMT(rootHash, key, proofDb) } + // test the type of proof (for path trie or SMT) + if buf, _ := proofDb.Get(magicHashPathTrie); buf != nil { + return VerifyProofSMT2(rootHash, key, proofDb) + } + key = keybytesToHex(key) wantHash := rootHash for i := 0; ; i++ { From b37cd5f5c9af0ba44abcdb88c884f641dc3a77ae Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Thu, 19 Dec 2024 20:21:56 +0800 Subject: [PATCH 24/61] add flag --state.scheme --- cmd/geth/chaincmd.go | 2 +- cmd/geth/main.go | 2 +- cmd/geth/usage.go | 2 +- cmd/utils/flags.go | 45 ++++++++++++++---- core/blockchain.go | 6 +-- core/genesis.go | 6 +-- core/rawdb/accessors_trie.go | 92 ++++++++++++++++++++++++++++++++++++ eth/backend.go | 7 ++- eth/ethconfig/config.go | 4 +- trie/database.go | 6 +-- 10 files changed, 144 insertions(+), 28 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index a575b7422..bfc6ed9d1 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -50,7 +50,7 @@ var ( ArgsUsage: "", Flags: []cli.Flag{ utils.DataDirFlag, - utils.PathZkTrieFlag, + utils.StateSchemeFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 9c50d1397..a85780237 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -158,7 +158,7 @@ var ( utils.GpoIgnoreGasPriceFlag, configFileFlag, utils.CatalystFlag, - utils.PathZkTrieFlag, + utils.StateSchemeFlag, utils.PathDBSyncFlag, } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 0ea87dd33..215ff5796 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -57,7 +57,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.IdentityFlag, utils.LightKDFFlag, utils.WhitelistFlag, - utils.PathZkTrieFlag, + utils.StateSchemeFlag, utils.PathDBSyncFlag, }, }, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8744f5c38..83789d6fe 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -853,10 +853,11 @@ var ( Usage: "Limit max fetched block range for `eth_getLogs` method", } - PathZkTrieFlag = cli.BoolFlag{ - Name: "pathzktrie", - Usage: "Use PathZkTrie instead of ZkTrie in state", + StateSchemeFlag = &cli.StringFlag{ + Name: "state.scheme", + Usage: "Scheme to use for storing ethereum state ('hash' or 'path')", } + PathDBSyncFlag = cli.BoolFlag{ Name: "pathdb.sync", Usage: "Sync flush nodes cache to disk in path schema", @@ -1731,9 +1732,14 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(PathDBSyncFlag.Name) { cfg.PathSyncFlush = true } - if ctx.GlobalIsSet(PathZkTrieFlag.Name) { - cfg.PathZkTrie = true - trie.Defaults.PathZkTrie = true + + scheme, err := ParseCLIAndConfigStateScheme(ctx.String(StateSchemeFlag.Name), cfg.StateScheme) + if err != nil { + Fatalf("%v", err) + } + cfg.StateScheme = scheme + if cfg.StateScheme == rawdb.PathScheme { + trie.GenesisStateInPathTrie = true } // Override any default configs for hard coded networks. @@ -1785,8 +1791,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { cfg.NoPrefetch = true // cheked for path zktrie - if cfg.PathZkTrie && !cfg.Genesis.Config.Morph.ZktrieEnabled() { - log.Crit("Must cooperate with zktrie enable to use --pathzktrie") + if cfg.StateScheme == rawdb.PathScheme && !cfg.Genesis.Config.Morph.ZktrieEnabled() { + log.Crit("Must cooperate with zktrie enable to use --state.scheme=path") } case ctx.GlobalBool(MorphHoleskyFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { @@ -1805,8 +1811,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { cfg.NoPrefetch = true // cheked for path zktrie - if cfg.PathZkTrie && !cfg.Genesis.Config.Morph.ZktrieEnabled() { - log.Crit("Must cooperate with zktrie enable to use --pathzktrie") + if cfg.StateScheme == rawdb.PathScheme && !cfg.Genesis.Config.Morph.ZktrieEnabled() { + log.Crit("Must cooperate with zktrie enable to use --state.scheme=path") } case ctx.GlobalBool(DeveloperFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { @@ -2175,3 +2181,22 @@ func MigrateFlags(action func(ctx *cli.Context) error) func(*cli.Context) error return action(ctx) } } + +// ParseCLIAndConfigStateScheme parses state scheme in CLI and config. +func ParseCLIAndConfigStateScheme(cliScheme, cfgScheme string) (string, error) { + if cliScheme == "" { + if cfgScheme != "" { + log.Info("Use config state scheme", "config", cfgScheme) + } + return cfgScheme, nil + } + + if !rawdb.ValidateStateScheme(cliScheme) { + return "", fmt.Errorf("invalid state scheme in CLI: %s", cliScheme) + } + if cfgScheme == "" || cliScheme == cfgScheme { + log.Info("Use CLI state scheme", "CLI", cliScheme) + return cliScheme, nil + } + return "", fmt.Errorf("incompatible state scheme, CLI: %s, config: %s", cliScheme, cfgScheme) +} diff --git a/core/blockchain.go b/core/blockchain.go index 137094064..9aa874f6d 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -139,8 +139,8 @@ type CacheConfig struct { SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it PathSyncFlush bool // Whether sync flush the trienodebuffer of pathdb to disk. - PathZkTrie bool // Use path zktrie instead of zktrie JournalFilePath string + StateScheme string // Scheme used to store ethereum states and zktrit nodes on top } // defaultCacheConfig are the default caching values if none are specified by the @@ -247,7 +247,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par var hashdbConfig *hashdb.Config var pathdbConfig *pathdb.Config - if chainConfig.Morph.ZktrieEnabled() && cacheConfig.PathZkTrie { + if chainConfig.Morph.ZktrieEnabled() && cacheConfig.StateScheme == rawdb.PathScheme { pathdbConfig = &pathdb.Config{ SyncFlush: cacheConfig.PathSyncFlush, CleanCacheSize: cacheConfig.TrieCleanLimit * 1024 * 1024, @@ -273,7 +273,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par Journal: cacheConfig.TrieCleanJournal, Preimages: cacheConfig.Preimages, Zktrie: chainConfig.Morph.ZktrieEnabled(), - PathZkTrie: chainConfig.Morph.ZktrieEnabled() && cacheConfig.PathZkTrie, + PathZkTrie: chainConfig.Morph.ZktrieEnabled() && cacheConfig.StateScheme == rawdb.PathScheme, HashDB: hashdbConfig, PathDB: pathdbConfig, }), diff --git a/core/genesis.go b/core/genesis.go index 06bf2538a..37d022072 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -201,10 +201,10 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override if storedcfg == nil { log.Warn("Found genesis block without chain config") } else { - trieCfg = &trie.Config{Zktrie: storedcfg.Morph.ZktrieEnabled(), PathZkTrie: trie.Defaults.PathZkTrie} + trieCfg = &trie.Config{Zktrie: storedcfg.Morph.ZktrieEnabled(), PathZkTrie: trie.GenesisStateInPathTrie} } } else { - trieCfg = &trie.Config{Zktrie: genesis.Config.Morph.ZktrieEnabled(), PathZkTrie: trie.Defaults.PathZkTrie} + trieCfg = &trie.Config{Zktrie: genesis.Config.Morph.ZktrieEnabled(), PathZkTrie: trie.GenesisStateInPathTrie} } exist := rawdb.ExistsStateID(db, header.Root) @@ -313,7 +313,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { } var trieCfg *trie.Config if g.Config != nil { - trieCfg = &trie.Config{Zktrie: g.Config.Morph.ZktrieEnabled(), PathZkTrie: trie.Defaults.PathZkTrie} + trieCfg = &trie.Config{Zktrie: g.Config.Morph.ZktrieEnabled(), PathZkTrie: trie.GenesisStateInPathTrie} } statedb, err := state.New(common.Hash{}, state.NewDatabaseWithConfig(db, trieCfg), nil) if err != nil { diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go index df16594ea..c415e8bbb 100644 --- a/core/rawdb/accessors_trie.go +++ b/core/rawdb/accessors_trie.go @@ -18,11 +18,14 @@ package rawdb import ( "bytes" + "fmt" "github.com/morph-l2/go-ethereum/common" "github.com/morph-l2/go-ethereum/ethdb" + "github.com/morph-l2/go-ethereum/log" zktrie "github.com/scroll-tech/zktrie/trie" + zkt "github.com/scroll-tech/zktrie/types" ) // HashScheme is the legacy hash-based state scheme with which trie nodes are @@ -89,3 +92,92 @@ func IsLegacyTrieNode(key []byte, val []byte) bool { hash := common.BytesToHash(common.BitReverse(zkHash[:])) return bytes.Equal(key[:], hash[:]) } + +// HasAccountTrieNode checks the presence of the account trie node with the +// specified node path, regardless of the node hash. +func HasAccountTrieNode(db ethdb.KeyValueReader, path []byte) bool { + has, err := db.Has(accountTrieNodeKey(path)) + if err != nil { + return false + } + return has +} + +// HasLegacyTrieNode checks if the trie node with the provided hash is present in db. +func HasLegacyTrieNode(db ethdb.KeyValueReader, hash common.Hash) bool { + ok, _ := db.Has(hash.Bytes()) + return ok +} + +// ReadStateScheme reads the state scheme of persistent state, or none +// if the state is not present in database. +func ReadStateScheme(db ethdb.Database) string { + // Check if state in path-based scheme is present. + if HasAccountTrieNode(db, zkt.TrieRootPathKey[:]) { + return PathScheme + } + // The root node might be deleted during the initial snap sync, check + // the persistent state id then. + if id := ReadPersistentStateID(db); id != 0 { + return PathScheme + } + + // In a hash-based scheme, the genesis state is consistently stored + // on the disk. To assess the scheme of the persistent state, it + // suffices to inspect the scheme of the genesis state. + header := ReadHeader(db, ReadCanonicalHash(db, 0), 0) + if header == nil { + return "" // empty datadir + } + + if !HasLegacyTrieNode(db, header.Root) { + return "" // no state in disk + } + return HashScheme +} + +// ValidateStateScheme used to check state scheme whether is valid. +// Valid state scheme: hash and path. +func ValidateStateScheme(stateScheme string) bool { + if stateScheme == HashScheme || stateScheme == PathScheme { + return true + } + return false +} + +// ParseStateScheme checks if the specified state scheme is compatible with +// the stored state. +// +// - If the provided scheme is none, use the scheme consistent with persistent +// state, or fallback to path-based scheme if state is empty. +// +// - If the provided scheme is hash, use hash-based scheme or error out if not +// compatible with persistent state scheme. +// +// - If the provided scheme is path: use path-based scheme or error out if not +// compatible with persistent state scheme. +func ParseStateScheme(provided string, disk ethdb.Database) (string, error) { + // If state scheme is not specified, use the scheme consistent + // with persistent state, or fallback to hash mode if database + // is empty. + stored := ReadStateScheme(disk) + if provided == "" { + if stored == "" { + log.Info("State scheme set to default", "scheme", "hash") + return HashScheme, nil // use default scheme for empty database + } + log.Info("State scheme set to already existing disk db", "scheme", stored) + return stored, nil // reuse scheme of persistent scheme + } + // If state scheme is specified, ensure it's valid. + if provided != HashScheme && provided != PathScheme { + return "", fmt.Errorf("invalid state scheme %s", provided) + } + // If state scheme is specified, ensure it's compatible with + // persistent state. + if stored == "" || provided == stored { + log.Info("State scheme set by user", "scheme", provided) + return provided, nil + } + return "", fmt.Errorf("incompatible state scheme, stored: %s, user provided: %s", stored, provided) +} diff --git a/eth/backend.go b/eth/backend.go index ecc08f24a..dfc6874b1 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -138,6 +138,11 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { return nil, genesisErr } + config.StateScheme, err = rawdb.ParseStateScheme(config.StateScheme, chainDb) + if err != nil { + return nil, err + } + log.Info("Initialised chain configuration", "config", chainConfig) if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, stack.ResolvePath(config.TrieCleanCacheJournal)); err != nil { @@ -193,7 +198,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { SnapshotLimit: config.SnapshotCache, Preimages: config.Preimages, PathSyncFlush: config.PathSyncFlush, - PathZkTrie: config.PathZkTrie, + StateScheme: config.StateScheme, JournalFilePath: stack.ResolvePath(config.JournalFileName), } ) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 11cc63917..6f9d3d96a 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -92,7 +92,6 @@ var Defaults = Config{ MaxBlockRange: -1, // Default unconfigured value: no block range limit for backward compatibility JournalFileName: "trie.journal", PathSyncFlush: true, - PathZkTrie: false, } func init() { @@ -221,8 +220,7 @@ type Config struct { // State scheme used to store ethereum state and merkle trie nodes on top PathSyncFlush bool - // Use PathZkTrie instead of ZkTrie in state - PathZkTrie bool + StateScheme string `toml:",omitempty"` // State scheme used to store ethereum state and merkle trie nodes on top } // CreateConsensusEngine creates a consensus engine for the given chain config. diff --git a/trie/database.go b/trie/database.go index d6583c0cc..1e623a133 100644 --- a/trie/database.go +++ b/trie/database.go @@ -296,11 +296,7 @@ type Config struct { PathDB *pathdb.Config // Configs for experimental path-based scheme } -var Defaults = &Config{ - Preimages: false, - Zktrie: true, - PathZkTrie: false, -} +var GenesisStateInPathTrie = false // Reader wraps the Node method of a backing trie reader. type Reader interface { From ce557df351b953d5506a55b39fba860b4c74658c Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Thu, 19 Dec 2024 20:23:50 +0800 Subject: [PATCH 25/61] WIP --- cmd/utils/flags.go | 2 +- core/genesis.go | 6 +++--- trie/database.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 83789d6fe..fb23e7f4d 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1739,7 +1739,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } cfg.StateScheme = scheme if cfg.StateScheme == rawdb.PathScheme { - trie.GenesisStateInPathTrie = true + trie.GenesisStateInPathZkTrie = true } // Override any default configs for hard coded networks. diff --git a/core/genesis.go b/core/genesis.go index 37d022072..eddb11057 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -201,10 +201,10 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override if storedcfg == nil { log.Warn("Found genesis block without chain config") } else { - trieCfg = &trie.Config{Zktrie: storedcfg.Morph.ZktrieEnabled(), PathZkTrie: trie.GenesisStateInPathTrie} + trieCfg = &trie.Config{Zktrie: storedcfg.Morph.ZktrieEnabled(), PathZkTrie: trie.GenesisStateInPathZkTrie} } } else { - trieCfg = &trie.Config{Zktrie: genesis.Config.Morph.ZktrieEnabled(), PathZkTrie: trie.GenesisStateInPathTrie} + trieCfg = &trie.Config{Zktrie: genesis.Config.Morph.ZktrieEnabled(), PathZkTrie: trie.GenesisStateInPathZkTrie} } exist := rawdb.ExistsStateID(db, header.Root) @@ -313,7 +313,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { } var trieCfg *trie.Config if g.Config != nil { - trieCfg = &trie.Config{Zktrie: g.Config.Morph.ZktrieEnabled(), PathZkTrie: trie.GenesisStateInPathTrie} + trieCfg = &trie.Config{Zktrie: g.Config.Morph.ZktrieEnabled(), PathZkTrie: trie.GenesisStateInPathZkTrie} } statedb, err := state.New(common.Hash{}, state.NewDatabaseWithConfig(db, trieCfg), nil) if err != nil { diff --git a/trie/database.go b/trie/database.go index 1e623a133..ac93d782c 100644 --- a/trie/database.go +++ b/trie/database.go @@ -296,7 +296,7 @@ type Config struct { PathDB *pathdb.Config // Configs for experimental path-based scheme } -var GenesisStateInPathTrie = false +var GenesisStateInPathZkTrie = false // Reader wraps the Node method of a backing trie reader. type Reader interface { From ccd7590e9aa8dee0158989ca5434d9550b3161c7 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Thu, 19 Dec 2024 20:34:07 +0800 Subject: [PATCH 26/61] WIP --- cmd/utils/flags.go | 2 +- core/state/state_object.go | 4 ++-- core/state/state_prove.go | 6 +++--- core/state/statedb.go | 12 ++++++------ eth/ethconfig/config.go | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index fb23e7f4d..3692a669e 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -855,7 +855,7 @@ var ( StateSchemeFlag = &cli.StringFlag{ Name: "state.scheme", - Usage: "Scheme to use for storing ethereum state ('hash' or 'path')", + Usage: "Scheme to use for storing zktrie state ('hash' or 'path')", } PathDBSyncFlag = cli.BoolFlag{ diff --git a/core/state/state_object.go b/core/state/state_object.go index 21944e178..60a6f104b 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -119,8 +119,8 @@ func newObject(db *StateDB, address common.Address, data types.StateAccount) *st } var addrPoseidonHash common.Hash if db.db.TrieDB().IsPathZkTrie() { - addr_s, _ := zkt.ToSecureKey(address.Bytes()) - addrPoseidonHash = common.BigToHash(addr_s) + addressKey, _ := zkt.ToSecureKey(address.Bytes()) + addrPoseidonHash = common.BigToHash(addressKey) } return &stateObject{ db: db, diff --git a/core/state/state_prove.go b/core/state/state_prove.go index 1ac082fbc..06672cfe6 100644 --- a/core/state/state_prove.go +++ b/core/state/state_prove.go @@ -37,7 +37,7 @@ func (t ZktrieProofTracer) Available() bool { // NewProofTracer is not in Db interface and used explictily for reading proof in storage trie (not updated by the dirty value) func (s *StateDB) NewProofTracer(trieS Trie) ZktrieProofTracer { - if s.IsZktrie() { + if s.IsZkTrie() { zkTrie := trieS.(*zktrie.ZkTrie) if zkTrie == nil { panic("unexpected trie type for zktrie") @@ -54,7 +54,7 @@ func (s *StateDB) GetStorageTrieForProof(addr common.Address) (Trie, error) { stateObject := s.getStateObject(addr) addrHash := crypto.Keccak256Hash(addr[:]) - if s.IsPathZktrie() { + if s.IsPathZkTrie() { k, err := zkt.ToSecureKey(addr.Bytes()) if err != nil { return nil, fmt.Errorf("can't create storage trie on ToSecureKey %s: %v ", addr.Hex(), err) @@ -87,7 +87,7 @@ func (s *StateDB) GetSecureTrieProof(trieProve TrieProve, key common.Hash) ([][] var proof proofList var err error - if s.IsZktrie() { + if s.IsZkTrie() { key_s, _ := zkt.ToSecureKeyBytes(key.Bytes()) err = trieProve.Prove(key_s.Bytes(), 0, &proof) } else { diff --git a/core/state/statedb.go b/core/state/statedb.go index 05f1299d4..75cbdc18b 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -193,11 +193,11 @@ func (s *StateDB) Error() error { return s.dbErr } -func (s *StateDB) IsZktrie() bool { +func (s *StateDB) IsZkTrie() bool { return s.db.TrieDB().IsZkTrie() } -func (s *StateDB) IsPathZktrie() bool { +func (s *StateDB) IsPathZkTrie() bool { return s.db.TrieDB().IsPathZkTrie() } @@ -337,9 +337,9 @@ func (s *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash { // GetProof returns the Merkle proof for a given account. func (s *StateDB) GetProof(addr common.Address) ([][]byte, error) { - if s.IsZktrie() { - addr_s, _ := zkt.ToSecureKeyBytes(addr.Bytes()) - return s.GetProofByHash(common.BytesToHash(addr_s.Bytes())) + if s.IsZkTrie() { + addressKey, _ := zkt.ToSecureKeyBytes(addr.Bytes()) + return s.GetProofByHash(common.BytesToHash(addressKey.Bytes())) } return s.GetProofByHash(crypto.Keccak256Hash(addr.Bytes())) } @@ -615,7 +615,7 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { if len(enc) == 0 { return nil } - if s.IsZktrie() { + if s.IsZkTrie() { data, err = types.UnmarshalStateAccount(enc) } else { data = new(types.StateAccount) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 6f9d3d96a..f93483e0c 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -220,7 +220,7 @@ type Config struct { // State scheme used to store ethereum state and merkle trie nodes on top PathSyncFlush bool - StateScheme string `toml:",omitempty"` // State scheme used to store ethereum state and merkle trie nodes on top + StateScheme string `toml:",omitempty"` // State scheme used to store zktrie state nodes on top } // CreateConsensusEngine creates a consensus engine for the given chain config. From 207825a83391e880563099765a34992ddfeca4b1 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Thu, 19 Dec 2024 21:00:20 +0800 Subject: [PATCH 27/61] WIP --- core/blockchain.go | 63 +++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 9aa874f6d..8ab8cc119 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -153,6 +153,33 @@ var defaultCacheConfig = &CacheConfig{ SnapshotWait: true, } +// triedbConfig derives the configures for trie database. +func (c *CacheConfig) triedbConfig(zktrie bool) *trie.Config { + config := &trie.Config{ + Cache: c.TrieCleanLimit, + Journal: c.TrieCleanJournal, + Preimages: c.Preimages, + Zktrie: zktrie, + PathZkTrie: zktrie && c.StateScheme == rawdb.PathScheme, + } + if config.PathZkTrie { + config.PathDB = &pathdb.Config{ + SyncFlush: c.PathSyncFlush, + CleanCacheSize: c.TrieCleanLimit * 1024 * 1024, + DirtyCacheSize: c.TrieCleanLimit * 1024 * 1024, + JournalFilePath: c.JournalFilePath, + } + } else { + if config.Zktrie { + config.HashDB = &hashdb.Config{ + Cache: c.TrieCleanLimit, + Journal: c.TrieCleanJournal, + } + } + } + return config +} + // BlockChain represents the canonical chain given a database with a genesis // block. The Blockchain manages chain imports, reverts, chain reorganisations. // @@ -245,38 +272,12 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par log.Warn("Using fee vault address", "FeeVaultAddress", *chainConfig.Morph.FeeVaultAddress) } - var hashdbConfig *hashdb.Config - var pathdbConfig *pathdb.Config - if chainConfig.Morph.ZktrieEnabled() && cacheConfig.StateScheme == rawdb.PathScheme { - pathdbConfig = &pathdb.Config{ - SyncFlush: cacheConfig.PathSyncFlush, - CleanCacheSize: cacheConfig.TrieCleanLimit * 1024 * 1024, - DirtyCacheSize: cacheConfig.TrieCleanLimit * 1024 * 1024, - JournalFilePath: cacheConfig.JournalFilePath, - } - } else { - if chainConfig.Morph.ZktrieEnabled() { - hashdbConfig = &hashdb.Config{ - Cache: cacheConfig.TrieCleanLimit, - Journal: cacheConfig.TrieCleanJournal, - } - } - } - bc := &BlockChain{ - chainConfig: chainConfig, - cacheConfig: cacheConfig, - db: db, - triegc: prque.New(nil), - stateCache: state.NewDatabaseWithConfig(db, &trie.Config{ - Cache: cacheConfig.TrieCleanLimit, - Journal: cacheConfig.TrieCleanJournal, - Preimages: cacheConfig.Preimages, - Zktrie: chainConfig.Morph.ZktrieEnabled(), - PathZkTrie: chainConfig.Morph.ZktrieEnabled() && cacheConfig.StateScheme == rawdb.PathScheme, - HashDB: hashdbConfig, - PathDB: pathdbConfig, - }), + chainConfig: chainConfig, + cacheConfig: cacheConfig, + db: db, + triegc: prque.New(nil), + stateCache: state.NewDatabaseWithConfig(db, cacheConfig.triedbConfig(chainConfig.Morph.ZktrieEnabled())), quit: make(chan struct{}), chainmu: syncx.NewClosableMutex(), shouldPreserve: shouldPreserve, From bb12ec696b9379141b83af5289ff778410692af9 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Thu, 19 Dec 2024 22:20:15 +0800 Subject: [PATCH 28/61] WIP --- core/rawdb/accessors_trie.go | 2 -- triedb/hashdb/zk_trie_database.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go index c415e8bbb..fe7322a43 100644 --- a/core/rawdb/accessors_trie.go +++ b/core/rawdb/accessors_trie.go @@ -39,8 +39,6 @@ import ( // for archive node and some other tries(e.g. light trie). const HashScheme = "hash" -const ZkHashScheme = "hashZk" - // PathScheme is the new path-based state scheme with which trie nodes are stored // in the disk with node path as the database key. This scheme will only store one // version of state data in the disk, which means that the state pruning operation diff --git a/triedb/hashdb/zk_trie_database.go b/triedb/hashdb/zk_trie_database.go index aab96307c..ba9af28ee 100644 --- a/triedb/hashdb/zk_trie_database.go +++ b/triedb/hashdb/zk_trie_database.go @@ -61,7 +61,7 @@ func NewZkDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *Zktrie } } -func (db *ZktrieDatabase) Scheme() string { return rawdb.ZkHashScheme } +func (db *ZktrieDatabase) Scheme() string { return rawdb.HashScheme } func (db *ZktrieDatabase) Size() (common.StorageSize, common.StorageSize, common.StorageSize) { db.lock.RLock() From 6e33fa453e4f2e508f8e3b19c77aefd00291220a Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Fri, 20 Dec 2024 10:47:18 +0800 Subject: [PATCH 29/61] WIP --- core/genesis.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/core/genesis.go b/core/genesis.go index eddb11057..98daf7264 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -207,14 +207,13 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override trieCfg = &trie.Config{Zktrie: genesis.Config.Morph.ZktrieEnabled(), PathZkTrie: trie.GenesisStateInPathZkTrie} } - exist := rawdb.ExistsStateID(db, header.Root) - var verify bool = true + var inited bool = false // new states already overide genesis states. - if trieCfg.PathZkTrie && exist { - verify = false + if trieCfg.PathZkTrie && rawdb.ExistsStateID(db, header.Root) { + inited = true } - if verify { + if !inited { if _, err := state.New(header.Root, state.NewDatabaseWithConfig(db, trieCfg), nil); err != nil { log.Error("failed to new state in SetupGenesisBlockWithOverride", "header root", header.Root.String(), "error", err) if genesis == nil { From 570a56654831b8c9a8adfc97249d2ca814d31b23 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 24 Dec 2024 11:54:49 +0800 Subject: [PATCH 30/61] merge main --- MakefileEc2.mk | 31 +++++++++++++++++++++++++++++++ core/state_transition.go | 10 ++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/MakefileEc2.mk b/MakefileEc2.mk index 705b80ecc..fbe847cd4 100644 --- a/MakefileEc2.mk +++ b/MakefileEc2.mk @@ -21,3 +21,34 @@ build-bk-prod-morph-prod-mainnet-to-morph-nccc-geth: aws s3 cp morph-nccc-geth.tar.gz s3://morph-0582-morph-technical-department-mainnet-data/morph-setup/morph-nccc-geth.tar.gz +# build for holesky +build-bk-prod-morph-prod-testnet-to-morph-geth-holesky: + if [ ! -d dist ]; then mkdir -p dist; fi + $(GORUN) build/ci.go install ./cmd/geth + cp build/bin/geth dist/ + tar -czvf morph-geth.tar.gz dist + aws s3 cp morph-geth.tar.gz s3://morph-0582-morph-technical-department-testnet-data/testnet/holesky/morph-setup/morph-geth.tar.gz + +build-bk-prod-morph-prod-testnet-to-morph-nccc-geth-holesky: + if [ ! -d dist ]; then mkdir -p dist; fi + $(GORUN) build/ci.go install ./cmd/geth + @echo "Done building." + cp build/bin/geth dist/ + tar -czvf morph-nccc-geth.tar.gz dist + aws s3 cp morph-nccc-geth.tar.gz s3://morph-0582-morph-technical-department-testnet-data/testnet/holesky/morph-setup/morph-nccc-geth.tar.gz + +build-bk-test-morph-test-qanet-to-morph-geth-qanet: + if [ ! -d dist ]; then mkdir -p dist; fi + $(GORUN) build/ci.go install ./cmd/geth + @echo "Done building." + cp build/bin/geth dist/ + tar -czvf morph-geth.tar.gz dist + aws s3 cp morph-geth.tar.gz s3://morph-7637-morph-technical-department-qanet-data/morph-setup/morph-geth.tar.gz + +build-bk-test-morph-test-qanet-to-morph-nccc-geth-qanet: + if [ ! -d dist ]; then mkdir -p dist; fi + $(GORUN) build/ci.go install ./cmd/geth + @echo "Done building." + cp build/bin/geth dist/ + tar -czvf morph-nccc-geth.tar.gz dist + aws s3 cp morph-nccc-geth.tar.gz s3://morph-7637-morph-technical-department-qanet-data/morph-setup/morph-nccc-geth.tar.gz \ No newline at end of file diff --git a/core/state_transition.go b/core/state_transition.go index b51dcc375..60aad1d2b 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -368,12 +368,18 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { return nil, err } if st.gas < gas { - return nil, fmt.Errorf("%w: have %d, want %d", ErrIntrinsicGas, st.gas, gas) + // Allow L1 message transactions to be included in the block but fail during execution, + // instead of rejecting them outright at this point. + if st.msg.IsL1MessageTx() { + gas = st.gas + } else { + return nil, fmt.Errorf("%w: have %d, want %d", ErrIntrinsicGas, st.gas, gas) + } } st.gas -= gas // Check clause 6 - if msg.Value().Sign() > 0 && !st.evm.Context.CanTransfer(st.state, msg.From(), msg.Value()) { + if msg.Value().Sign() > 0 && !msg.IsL1MessageTx() && !st.evm.Context.CanTransfer(st.state, msg.From(), msg.Value()) { return nil, fmt.Errorf("%w: address %v", ErrInsufficientFundsForTransfer, msg.From().Hex()) } From b4f0316f638b13b585830126cbd342065c26bfee Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Thu, 26 Dec 2024 23:28:21 +0800 Subject: [PATCH 31/61] upgrage new trie --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9ea337a00..2d5211474 100644 --- a/go.mod +++ b/go.mod @@ -127,4 +127,4 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/scroll-tech/zktrie v0.8.4 => github.com/ryanmorphl2/zktrie v1.9.2 +replace github.com/scroll-tech/zktrie v0.8.4 => github.com/ryanmorphl2/zktrie v1.9.3 diff --git a/go.sum b/go.sum index c7d0effdd..835d0f221 100644 --- a/go.sum +++ b/go.sum @@ -432,8 +432,8 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/ryanmorphl2/zktrie v1.9.2 h1:nWnmb7RaNJXVsOtnOD88/AVEUh0dlQyjPVRR64gh+KY= -github.com/ryanmorphl2/zktrie v1.9.2/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk= +github.com/ryanmorphl2/zktrie v1.9.3 h1:fea9GBXT7j+r6jncsVyvBgmmnYV/XzmFNvU/g0JtxFI= +github.com/ryanmorphl2/zktrie v1.9.3/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= From 9109d84268996cb820d60d5f66f1da3c835999d4 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Thu, 2 Jan 2025 15:11:42 +0800 Subject: [PATCH 32/61] merge main --- .github/workflows/docker_release.yml | 41 ++++++++++++++++++++++++++++ rollup/tracing/tracing.go | 7 +++++ 2 files changed, 48 insertions(+) create mode 100644 .github/workflows/docker_release.yml diff --git a/.github/workflows/docker_release.yml b/.github/workflows/docker_release.yml new file mode 100644 index 000000000..36af0903a --- /dev/null +++ b/.github/workflows/docker_release.yml @@ -0,0 +1,41 @@ +name: Push Docker Image + +on: + push: + tags: + - morph-v* + +env: + IMAGE_NAME: go-ethereum + +jobs: + # Push image to GitHub Packages. + push: + runs-on: ubuntu-latest + if: github.event_name == 'push' + + steps: + - uses: actions/checkout@v4 + + - name: Build the Docker image + run: docker build . --file Dockerfile -t "${IMAGE_NAME}" + + - name: Log into registry + run: echo "${{ secrets.PACKAGE_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Push image + run: | + IMAGE_ID=ghcr.io/${{ github.repository }} + + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + # Strip "morph-v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^morph-v//') + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker tag $IMAGE_NAME $IMAGE_ID:latest + docker push $IMAGE_ID:$VERSION + docker push $IMAGE_ID:latest diff --git a/rollup/tracing/tracing.go b/rollup/tracing/tracing.go index ae7336e2e..52fccfb69 100644 --- a/rollup/tracing/tracing.go +++ b/rollup/tracing/tracing.go @@ -386,6 +386,13 @@ func (env *TraceEnv) getTxResult(state *state.StateDB, index int, block *types.B // merge required proof data proofAccounts := structLogger.UpdatedAccounts() proofAccounts[vmenv.FeeRecipient()] = struct{}{} + // add from/to address if it does not exist + if _, ok := proofAccounts[from]; !ok { + proofAccounts[from] = struct{}{} + } + if _, ok := proofAccounts[*to]; !ok { + proofAccounts[*to] = struct{}{} + } for addr := range proofAccounts { addrStr := addr.String() From d554540506c338ed83b4b01251ab7801592316c3 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Fri, 17 Jan 2025 21:19:10 +0800 Subject: [PATCH 33/61] statedb: copy with originRoot --- core/state/statedb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/state/statedb.go b/core/state/statedb.go index 75cbdc18b..cd893223c 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -732,6 +732,7 @@ func (s *StateDB) Copy() *StateDB { preimages: make(map[common.Hash][]byte, len(s.preimages)), journal: newJournal(), hasher: crypto.NewKeccakState(), + originalRoot: s.originalRoot, } // Copy the dirty states, logs, and preimages for addr := range s.journal.dirties { From 49904bf014ad728114c16b9bef9c5b67178ffeb5 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Fri, 17 Jan 2025 21:34:45 +0800 Subject: [PATCH 34/61] difflayer: remove unused cache --- triedb/pathdb/difflayer.go | 143 ++----------------------------------- triedb/pathdb/disklayer.go | 2 - triedb/pathdb/journal.go | 1 - triedb/pathdb/layertree.go | 18 ----- 4 files changed, 5 insertions(+), 159 deletions(-) diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go index c0f94bf9f..36878b4f6 100644 --- a/triedb/pathdb/difflayer.go +++ b/triedb/pathdb/difflayer.go @@ -17,7 +17,6 @@ package pathdb import ( - "crypto/sha256" "fmt" "sync" @@ -26,107 +25,6 @@ import ( dbtypes "github.com/morph-l2/go-ethereum/triedb/types" ) -type RefTrieNode struct { - refCount uint32 - blob []byte -} - -type HashNodeCache struct { - lock sync.RWMutex - cache map[[sha256.Size]byte]*RefTrieNode -} - -func (h *HashNodeCache) length() int { - if h == nil { - return 0 - } - h.lock.RLock() - defer h.lock.RUnlock() - return len(h.cache) -} - -func (h *HashNodeCache) set(key, val []byte) { - if h == nil { - return - } - h.lock.Lock() - defer h.lock.Unlock() - hash := sha256.Sum256(key) - if n, ok := h.cache[hash]; ok { - n.refCount++ - n.blob = val - } else { - h.cache[hash] = &RefTrieNode{1, val} - } -} - -func (h *HashNodeCache) Get(key []byte) []byte { - if h == nil { - return nil - } - h.lock.RLock() - defer h.lock.RUnlock() - hash := sha256.Sum256(key) - if n, ok := h.cache[hash]; ok { - return n.blob - } - return nil -} - -func (h *HashNodeCache) del(key []byte) { - if h == nil { - return - } - h.lock.Lock() - defer h.lock.Unlock() - - hash := sha256.Sum256(key) - n, ok := h.cache[hash] - if !ok { - return - } - if n.refCount > 0 { - n.refCount-- - } - if n.refCount == 0 { - delete(h.cache, hash) - } -} - -func (h *HashNodeCache) Add(ly layer) { - if h == nil { - return - } - dl, ok := ly.(*diffLayer) - if !ok { - return - } - beforeAdd := h.length() - for _, v := range dl.nodes { - h.set(v.K, v.V) - } - diffHashCacheLengthGauge.Update(int64(h.length())) - log.Debug("Add difflayer to hash map", "root", ly.rootHash(), "block_number", dl.block, "map_len", h.length(), "add_delta", h.length()-beforeAdd) -} - -func (h *HashNodeCache) Remove(ly layer) { - if h == nil { - return - } - dl, ok := ly.(*diffLayer) - if !ok { - return - } - go func() { - beforeDel := h.length() - for _, v := range dl.nodes { - h.del(v.K) - } - diffHashCacheLengthGauge.Update(int64(h.length())) - log.Debug("Remove difflayer from hash map", "root", ly.rootHash(), "block_number", dl.block, "map_len", h.length(), "del_delta", beforeDel-h.length()) - }() -} - // diffLayer represents a collection of modifications made to the in-memory tries // along with associated state changes after running a block on top. // @@ -134,12 +32,11 @@ func (h *HashNodeCache) Remove(ly layer) { // made to the state, that have not yet graduated into a semi-immutable state. type diffLayer struct { // Immutables - root common.Hash // Root hash to which this layer diff belongs to - id uint64 // Corresponding state id - block uint64 // Associated block number - nodes dbtypes.KvMap // Cached trie nodes indexed by owner and path - memory uint64 // Approximate guess as to how much memory we use - cache *HashNodeCache // trienode cache by hash key. cache is immutable, but cache's item can be add/del. + root common.Hash // Root hash to which this layer diff belongs to + id uint64 // Corresponding state id + block uint64 // Associated block number + nodes dbtypes.KvMap // Cached trie nodes indexed by owner and path + memory uint64 // Approximate guess as to how much memory we use // mutables origin *diskLayer // The current difflayer corresponds to the underlying disklayer and is updated during cap. @@ -164,12 +61,8 @@ func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes switch l := parent.(type) { case *diskLayer: dl.origin = l - dl.cache = &HashNodeCache{ - cache: make(map[[sha256.Size]byte]*RefTrieNode), - } case *diffLayer: dl.origin = l.originDiskLayer() - dl.cache = l.cache default: panic("unknown parent type") } @@ -244,32 +137,6 @@ func (dl *diffLayer) node(path []byte, depth int) ([]byte, error) { // Node implements the layer interface, retrieving the trie node blob with the // provided node information. No error will be returned if the node is not found. func (dl *diffLayer) Node(path []byte) ([]byte, error) { - if n := dl.cache.Get(path); n != nil { - // The query from the hash map is fastpath, - // avoiding recursive query of 128 difflayers. - diffHashCacheHitMeter.Mark(1) - diffHashCacheReadMeter.Mark(int64(len(n))) - return n, nil - } - diffHashCacheMissMeter.Mark(1) - - persistLayer := dl.originDiskLayer() - if persistLayer != nil { - blob, err := persistLayer.Node(path) - if err != nil { - // This is a bad case with a very low probability. - // r/w the difflayer cache and r/w the disklayer are not in the same lock, - // so in extreme cases, both reading the difflayer cache and reading the disklayer may fail, eg, disklayer is stale. - // In this case, fallback to the original 128-layer recursive difflayer query path. - diffHashCacheSlowPathMeter.Mark(1) - log.Debug("Retry difflayer due to query origin failed", "path", path, "hash", "error", err) - return dl.node(path, 0) - } else { // This is the fastpath. - return blob, nil - } - } - diffHashCacheSlowPathMeter.Mark(1) - log.Debug("Retry difflayer due to origin is nil", "path", path) return dl.node(path, 0) } diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go index 64accca48..2c600dacb 100644 --- a/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -222,8 +222,6 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) { return nil, err } - // The bottom has been eaten by disklayer, releasing the hash cache of bottom difflayer. - bottom.cache.Remove(bottom) return ndl, nil } diff --git a/triedb/pathdb/journal.go b/triedb/pathdb/journal.go index f140d5606..512ec5c6c 100644 --- a/triedb/pathdb/journal.go +++ b/triedb/pathdb/journal.go @@ -359,7 +359,6 @@ func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream, journalTypeForRea log.Debug("Loaded diff layer journal", "root", root, "parent", parent.rootHash(), "id", parent.stateID()+1, "block", block, "nodes", len(nodes)) // add cache first l := newDiffLayer(parent, root, parent.stateID()+1, block, nodes) - l.cache.Add(l) return db.loadDiffLayer(l, r, journalTypeForReader) } diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go index dcb6575a5..ef0ce9f44 100644 --- a/triedb/pathdb/layertree.go +++ b/triedb/pathdb/layertree.go @@ -49,13 +49,6 @@ func (tree *layerTree) reset(head layer) { tree.lock.Lock() defer tree.lock.Unlock() - for _, ly := range tree.layers { - if dl, ok := ly.(*diffLayer); ok { - // Clean up the hash cache of difflayers due to reset. - dl.cache.Remove(dl) - } - } - var layers = make(map[common.Hash]layer) for head != nil { layers[head.rootHash()] = head @@ -112,9 +105,6 @@ func (tree *layerTree) add(root common.Hash, parentRoot common.Hash, block uint6 } l := parent.update(root, parent.stateID()+1, block, nodes) - // Before adding layertree, update the hash cache. - l.cache.Add(l) - tree.lock.Lock() tree.layers[l.rootHash()] = l tree.lock.Unlock() @@ -144,7 +134,6 @@ func (tree *layerTree) cap(root common.Hash, layers int) error { } for _, ly := range tree.layers { if dl, ok := ly.(*diffLayer); ok { - dl.cache.Remove(dl) dl.reset() log.Debug("Cleanup difflayer hash cache due to cap all", "diff_root", dl.root.String(), "diff_block_number", dl.block) } @@ -200,13 +189,6 @@ func (tree *layerTree) cap(root common.Hash, layers int) error { } var remove func(root common.Hash) remove = func(root common.Hash) { - if df, exist := tree.layers[root]; exist { - if dl, ok := df.(*diffLayer); ok { - // Clean up the hash cache of the child difflayer corresponding to the stale parent, include the re-org case. - dl.cache.Remove(dl) - log.Debug("Cleanup difflayer hash cache due to reorg", "diff_root", dl.root.String(), "diff_block_number", dl.block) - } - } delete(tree.layers, root) for _, child := range children[root] { remove(child) From 9c0f99357a48043121a49870a2b7797af049cb52 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Fri, 17 Jan 2025 23:33:39 +0800 Subject: [PATCH 35/61] trie proof: mark todo --- rollup/tracing/tracing.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rollup/tracing/tracing.go b/rollup/tracing/tracing.go index 52fccfb69..acea543aa 100644 --- a/rollup/tracing/tracing.go +++ b/rollup/tracing/tracing.go @@ -48,6 +48,9 @@ func NewTracerWrapper() *TracerWrapper { // CreateTraceEnvAndGetBlockTrace wraps the whole block tracing logic for a block func (tw *TracerWrapper) CreateTraceEnvAndGetBlockTrace(chainConfig *params.ChainConfig, chainContext core.ChainContext, engine consensus.Engine, chaindb ethdb.Database, statedb *state.StateDB, parent *types.Block, block *types.Block, commitAfterApply bool) (*types.BlockTrace, error) { + if statedb.IsPathZkTrie() { + return nil, fmt.Errorf("@Todo, unimplement, block=%d", block.NumberU64()) + } traceEnv, err := CreateTraceEnv(chainConfig, chainContext, engine, chaindb, statedb, parent, block, commitAfterApply) if err != nil { return nil, err From 6d9401f1388bbfa2b503ce46572840e759f82fb6 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Sun, 19 Jan 2025 19:42:12 +0800 Subject: [PATCH 36/61] fix typo --- rollup/tracing/tracing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rollup/tracing/tracing.go b/rollup/tracing/tracing.go index acea543aa..e85ba1b43 100644 --- a/rollup/tracing/tracing.go +++ b/rollup/tracing/tracing.go @@ -49,7 +49,7 @@ func NewTracerWrapper() *TracerWrapper { // CreateTraceEnvAndGetBlockTrace wraps the whole block tracing logic for a block func (tw *TracerWrapper) CreateTraceEnvAndGetBlockTrace(chainConfig *params.ChainConfig, chainContext core.ChainContext, engine consensus.Engine, chaindb ethdb.Database, statedb *state.StateDB, parent *types.Block, block *types.Block, commitAfterApply bool) (*types.BlockTrace, error) { if statedb.IsPathZkTrie() { - return nil, fmt.Errorf("@Todo, unimplement, block=%d", block.NumberU64()) + return nil, fmt.Errorf("@Todo, unimplemention, block=%d", block.NumberU64()) } traceEnv, err := CreateTraceEnv(chainConfig, chainContext, engine, chaindb, statedb, parent, block, commitAfterApply) if err != nil { From 0f62c7ad1008f26d008ba3bcf8bd8a56c902c175 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Mon, 10 Feb 2025 11:25:35 +0800 Subject: [PATCH 37/61] offline prune only in hash schema --- cmd/geth/snapshot.go | 3 +++ rollup/tracing/tracing.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 3339b9660..29b14eef2 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -223,6 +223,9 @@ func pruneZKState(ctx *cli.Context) error { defer stack.Close() chaindb := utils.MakeChainDatabase(ctx, stack, false) + if rawdb.ReadStateScheme(chaindb) != rawdb.HashScheme { + log.Crit("Offline pruning is not required for path scheme") + } zkPruner, err := pruner.NewZKPruner(chaindb, ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name), stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal)) if err != nil { return err diff --git a/rollup/tracing/tracing.go b/rollup/tracing/tracing.go index e85ba1b43..fbe5e4d63 100644 --- a/rollup/tracing/tracing.go +++ b/rollup/tracing/tracing.go @@ -49,7 +49,7 @@ func NewTracerWrapper() *TracerWrapper { // CreateTraceEnvAndGetBlockTrace wraps the whole block tracing logic for a block func (tw *TracerWrapper) CreateTraceEnvAndGetBlockTrace(chainConfig *params.ChainConfig, chainContext core.ChainContext, engine consensus.Engine, chaindb ethdb.Database, statedb *state.StateDB, parent *types.Block, block *types.Block, commitAfterApply bool) (*types.BlockTrace, error) { if statedb.IsPathZkTrie() { - return nil, fmt.Errorf("@Todo, unimplemention, block=%d", block.NumberU64()) + return nil, fmt.Errorf("path trie not implemention, block=%d", block.NumberU64()) } traceEnv, err := CreateTraceEnv(chainConfig, chainContext, engine, chaindb, statedb, parent, block, commitAfterApply) if err != nil { From dea4aa8c802e0ee6249697079c9612c01d5664f0 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Mon, 10 Feb 2025 12:17:27 +0800 Subject: [PATCH 38/61] merge main --- core/state_transition.go | 7 + core/types/l2trace.go | 1 + core/vm/access_list_tracer.go | 4 + core/vm/logger.go | 98 +++++++ core/vm/logger_json.go | 4 + core/vm/runtime/runtime_test.go | 6 +- eth/tracers/api.go | 95 +++---- .../internal/tracetest/calltrace_test.go | 6 +- .../internal/tracers/4byte_tracer_legacy.js | 2 +- .../js/internal/tracers/call_tracer_legacy.js | 2 +- eth/tracers/js/tracer.go | 6 +- eth/tracers/js/tracer_test.go | 26 +- eth/tracers/native/4byte.go | 37 +-- eth/tracers/native/call.go | 239 +++++++++++++----- eth/tracers/native/gen_account_json.go | 56 ++++ eth/tracers/native/gen_callframe_json.go | 107 ++++++++ eth/tracers/native/noop.go | 8 +- eth/tracers/native/prestate.go | 73 ++++-- eth/tracers/native/tracer.go | 50 ++-- eth/tracers/tracers.go | 6 +- rollup/tracing/mux_tracer.go | 12 + rollup/tracing/tracing.go | 2 +- 22 files changed, 621 insertions(+), 226 deletions(-) create mode 100644 eth/tracers/native/gen_account_json.go create mode 100644 eth/tracers/native/gen_callframe_json.go diff --git a/core/state_transition.go b/core/state_transition.go index 60aad1d2b..d46b6f4d2 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -355,6 +355,13 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { return nil, err } + if st.evm.Config.Debug { + st.evm.Config.Tracer.CaptureTxStart(st.initialGas) + defer func() { + st.evm.Config.Tracer.CaptureTxEnd(st.gas) + }() + } + var ( msg = st.msg sender = vm.AccountRef(msg.From()) diff --git a/core/types/l2trace.go b/core/types/l2trace.go index 2ef09b588..a125a7556 100644 --- a/core/types/l2trace.go +++ b/core/types/l2trace.go @@ -85,6 +85,7 @@ type StructLogRes struct { Depth int `json:"depth"` Error string `json:"error,omitempty"` Stack []string `json:"stack,omitempty"` + ReturnData string `json:"returnData,omitempty"` Memory []string `json:"memory,omitempty"` Storage map[string]string `json:"storage,omitempty"` RefundCounter uint64 `json:"refund,omitempty"` diff --git a/core/vm/access_list_tracer.go b/core/vm/access_list_tracer.go index a5da0782a..b0f272e4b 100644 --- a/core/vm/access_list_tracer.go +++ b/core/vm/access_list_tracer.go @@ -175,6 +175,10 @@ func (*AccessListTracer) CaptureEnter(typ OpCode, from common.Address, to common func (*AccessListTracer) CaptureExit(output []byte, gasUsed uint64, err error) {} +func (t *AccessListTracer) CaptureTxStart(gasLimit uint64) {} + +func (t *AccessListTracer) CaptureTxEnd(restGas uint64) {} + // AccessList returns the current accesslist maintained by the tracer. func (a *AccessListTracer) AccessList() types.AccessList { return a.list.accessList() diff --git a/core/vm/logger.go b/core/vm/logger.go index f8b70538b..f50f634ce 100644 --- a/core/vm/logger.go +++ b/core/vm/logger.go @@ -19,10 +19,12 @@ package vm import ( "bytes" "encoding/hex" + "encoding/json" "fmt" "io" "math/big" "strings" + "sync/atomic" "time" "github.com/holiman/uint256" @@ -127,6 +129,9 @@ func (s *StructLog) ErrorString() string { // Note that reference types are actual VM data structures; make copies // if you need to retain them beyond the current call. type EVMLogger interface { + // Transaction level + CaptureTxStart(gasLimit uint64) + CaptureTxEnd(restGas uint64) CaptureStart(env *EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) CaptureState(pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, rData []byte, depth int, err error) CaptureStateAfter(pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, rData []byte, depth int, err error) @@ -162,6 +167,14 @@ type StructLogger struct { logs []*StructLog output []byte err error + + gasLimit uint64 + usedGas uint64 + + interrupt atomic.Bool // Atomic flag to signal execution interruption + reason error // Textual reason for the interruption + + ResultL1DataFee *big.Int } // NewStructLogger returns a new logger @@ -214,6 +227,11 @@ func (l *StructLogger) CaptureStart(env *EVM, from common.Address, to common.Add // // CaptureState also tracks SLOAD/SSTORE ops to track storage change. func (l *StructLogger) CaptureState(pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, rData []byte, depth int, opErr error) { + // If tracing was interrupted, set the error and stop + if l.interrupt.Load() { + return + } + memory := scope.Memory stack := scope.Stack contract := scope.Contract @@ -340,6 +358,14 @@ func (l *StructLogger) CaptureExit(output []byte, gasUsed uint64, err error) { } +func (l *StructLogger) CaptureTxStart(gasLimit uint64) { + l.gasLimit = gasLimit +} + +func (l *StructLogger) CaptureTxEnd(restGas uint64) { + l.usedGas = l.gasLimit - restGas +} + // UpdatedAccounts is used to collect all "touched" accounts func (l *StructLogger) UpdatedAccounts() map[common.Address]struct{} { return l.statesAffected @@ -367,6 +393,33 @@ func (l *StructLogger) Error() error { return l.err } // Output returns the VM return value captured by the trace. func (l *StructLogger) Output() []byte { return l.output } +func (l *StructLogger) GetResult() (json.RawMessage, error) { + // Tracing aborted + if l.reason != nil { + return nil, l.reason + } + failed := l.err != nil + returnData := common.CopyBytes(l.output) + // Return data when successful and revert reason when reverted, otherwise empty. + returnVal := fmt.Sprintf("%x", returnData) + if failed && l.err != ErrExecutionReverted { + returnVal = "" + } + return json.Marshal(&types.ExecutionResult{ + Gas: l.usedGas, + Failed: failed, + ReturnValue: returnVal, + StructLogs: formatLogs(l.StructLogs()), + L1DataFee: (*hexutil.Big)(l.ResultL1DataFee), + }) +} + +// Stop terminates execution of the tracer at the first opportune moment. +func (l *StructLogger) Stop(err error) { + l.reason = err + l.interrupt.Store(true) +} + // WriteTrace writes a formatted trace to the given writer func WriteTrace(writer io.Writer, logs []*StructLog) { for _, log := range logs { @@ -487,6 +540,10 @@ func (t *mdLogger) CaptureEnter(typ OpCode, from common.Address, to common.Addre func (t *mdLogger) CaptureExit(output []byte, gasUsed uint64, err error) {} +func (t *mdLogger) CaptureTxStart(gasLimit uint64) {} + +func (t *mdLogger) CaptureTxEnd(restGas uint64) {} + // FormatLogs formats EVM returned structured logs for json output func FormatLogs(logs []*StructLog) []*types.StructLogRes { formatted := make([]*types.StructLogRes, 0, len(logs)) @@ -511,3 +568,44 @@ func FormatLogs(logs []*StructLog) []*types.StructLogRes { } return formatted } + +// formatLogs formats EVM returned structured logs for json output +func formatLogs(logs []*StructLog) []*types.StructLogRes { + formatted := make([]*types.StructLogRes, len(logs)) + for index, trace := range logs { + formatted[index] = &types.StructLogRes{ + Pc: trace.Pc, + Op: trace.Op.String(), + Gas: trace.Gas, + GasCost: trace.GasCost, + Depth: trace.Depth, + Error: trace.ErrorString(), + RefundCounter: trace.RefundCounter, + } + if trace.Stack != nil { + stack := make([]string, len(trace.Stack)) + for i, stackValue := range trace.Stack { + stack[i] = stackValue.Hex() + } + formatted[index].Stack = stack + } + if trace.ReturnData.Len() > 0 { + formatted[index].ReturnData = hexutil.Bytes(trace.ReturnData.Bytes()).String() + } + if trace.Memory.Len() > 0 { + memory := make([]string, 0, (trace.Memory.Len()+31)/32) + for i := 0; i+32 <= trace.Memory.Len(); i += 32 { + memory = append(memory, fmt.Sprintf("%x", trace.Memory.Bytes()[i:i+32])) + } + formatted[index].Memory = memory + } + if trace.Storage != nil { + storage := make(map[string]string) + for i, storageValue := range trace.Storage { + storage[fmt.Sprintf("%x", i)] = fmt.Sprintf("%x", storageValue) + } + formatted[index].Storage = storage + } + } + return formatted +} diff --git a/core/vm/logger_json.go b/core/vm/logger_json.go index 61a3a656d..38ac2ecc6 100644 --- a/core/vm/logger_json.go +++ b/core/vm/logger_json.go @@ -98,3 +98,7 @@ func (l *JSONLogger) CaptureEnter(typ OpCode, from common.Address, to common.Add } func (l *JSONLogger) CaptureExit(output []byte, gasUsed uint64, err error) {} + +func (t *JSONLogger) CaptureTxStart(gasLimit uint64) {} + +func (t *JSONLogger) CaptureTxEnd(restGas uint64) {} diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index d327d9d97..57ca64190 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -376,7 +376,7 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) cfg.GasLimit = gas if len(tracerCode) > 0 { - tracer, err := tracers.New(tracerCode, new(tracers.Context)) + tracer, err := tracers.New(tracerCode, new(tracers.Context), nil) if err != nil { b.Fatal(err) } @@ -877,7 +877,7 @@ func TestRuntimeJSTracer(t *testing.T) { statedb.SetCode(common.HexToAddress("0xee"), calleeCode) statedb.SetCode(common.HexToAddress("0xff"), depressedCode) - tracer, err := tracers.New(jsTracer, new(tracers.Context)) + tracer, err := tracers.New(jsTracer, new(tracers.Context), nil) if err != nil { t.Fatal(err) } @@ -912,7 +912,7 @@ func TestJSTracerCreateTx(t *testing.T) { code := []byte{byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN)} statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - tracer, err := tracers.New(jsTracer, new(tracers.Context)) + tracer, err := tracers.New(jsTracer, new(tracers.Context), nil) if err != nil { t.Fatal(err) } diff --git a/eth/tracers/api.go b/eth/tracers/api.go index e5fbf27a9..ba40a0f85 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -20,6 +20,7 @@ import ( "bufio" "bytes" "context" + "encoding/json" "errors" "fmt" "io/ioutil" @@ -173,15 +174,15 @@ type TraceConfig struct { Tracer *string Timeout *string Reexec *uint64 + // Config specific to given tracer. Note struct logger + // config are historically embedded in main object. + TracerConfig json.RawMessage } // TraceCallConfig is the config for traceCall API. It holds one more // field to override the state for tracing. type TraceCallConfig struct { - *vm.LogConfig - Tracer *string - Timeout *string - Reexec *uint64 + TraceConfig StateOverrides *ethapi.StateOverride } @@ -898,12 +899,7 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc var traceConfig *TraceConfig if config != nil { - traceConfig = &TraceConfig{ - LogConfig: config.LogConfig, - Tracer: config.Tracer, - Timeout: config.Timeout, - Reexec: config.Reexec, - } + traceConfig = &config.TraceConfig } signer := types.MakeSigner(api.backend.ChainConfig(), block.Number()) @@ -921,75 +917,58 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc func (api *API) traceTx(ctx context.Context, message core.Message, txctx *Context, vmctx vm.BlockContext, statedb *state.StateDB, config *TraceConfig, l1DataFee *big.Int) (interface{}, error) { // Assemble the structured logger or the JavaScript tracer var ( - tracer vm.EVMLogger + tracer Tracer err error + timeout = defaultTraceTimeout txContext = core.NewEVMTxContext(message) ) - switch { - case config == nil: - tracer = vm.NewStructLogger(nil) - case config.Tracer != nil: - // Define a meaningful timeout of a single transaction trace - timeout := defaultTraceTimeout - if config.Timeout != nil { - if timeout, err = time.ParseDuration(*config.Timeout); err != nil { - return nil, err - } - } - if t, err := New(*config.Tracer, txctx); err != nil { + if config == nil { + config = &TraceConfig{} + } + // Default tracer is the struct logger + tracer = vm.NewStructLogger(config.LogConfig) + if config.Tracer != nil { + tracer, err = New(*config.Tracer, txctx, config.TracerConfig) + if err != nil { return nil, err - } else { - deadlineCtx, cancel := context.WithTimeout(ctx, timeout) - go func() { - <-deadlineCtx.Done() - if errors.Is(deadlineCtx.Err(), context.DeadlineExceeded) { - t.Stop(errors.New("execution timeout")) - } - }() - defer cancel() - tracer = t } - default: - tracer = vm.NewStructLogger(config.LogConfig) } // Run the transaction with tracing enabled. vmenv := vm.NewEVM(vmctx, txContext, statedb, api.backend.ChainConfig(), vm.Config{Debug: true, Tracer: tracer, NoBaseFee: true}) + // Define a meaningful timeout of a single transaction trace + if config.Timeout != nil { + if timeout, err = time.ParseDuration(*config.Timeout); err != nil { + return nil, err + } + } + deadlineCtx, cancel := context.WithTimeout(ctx, timeout) + go func() { + <-deadlineCtx.Done() + if errors.Is(deadlineCtx.Err(), context.DeadlineExceeded) { + tracer.Stop(errors.New("execution timeout")) + // Stop evm execution. Note cancellation is not necessarily immediate. + vmenv.Cancel() + } + }() + defer cancel() + // If gasPrice is 0, make sure that the account has sufficient balance to cover `l1DataFee`. if message.GasPrice().Cmp(big.NewInt(0)) == 0 { statedb.AddBalance(message.From(), l1DataFee) } - // Call Prepare to clear out the statedb access list statedb.SetTxContext(txctx.TxHash, txctx.TxIndex) - result, err := core.ApplyMessage(vmenv, message, new(core.GasPool).AddGas(message.Gas()), l1DataFee) if err != nil { return nil, fmt.Errorf("tracing failed: %w", err) } - // Depending on the tracer type, format and return the output. - switch tracer := tracer.(type) { - case *vm.StructLogger: - // If the result contains a revert reason, return it. - returnVal := fmt.Sprintf("%x", result.Return()) - if len(result.Revert()) > 0 { - returnVal = fmt.Sprintf("%x", result.Revert()) - } - return &types.ExecutionResult{ - Gas: result.UsedGas, - Failed: result.Failed(), - ReturnValue: returnVal, - StructLogs: vm.FormatLogs(tracer.StructLogs()), - L1DataFee: (*hexutil.Big)(result.L1DataFee), - }, nil - - case Tracer: - return tracer.GetResult() - - default: - panic(fmt.Sprintf("bad tracer type %T", tracer)) + l, ok := tracer.(*vm.StructLogger) + if ok { + l.ResultL1DataFee = result.L1DataFee } + return tracer.GetResult() } // APIs return the collection of RPC services the tracer package offers. diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index dccd5e629..8d9fb5e08 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -184,7 +184,7 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { } _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) ) - tracer, err := tracers.New(tracerName, new(tracers.Context)) + tracer, err := tracers.New(tracerName, new(tracers.Context), nil) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } @@ -302,7 +302,7 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - tracer, err := tracers.New(tracerName, new(tracers.Context)) + tracer, err := tracers.New(tracerName, new(tracers.Context), nil) if err != nil { b.Fatalf("failed to create call tracer: %v", err) } @@ -372,7 +372,7 @@ func TestZeroValueToNotExitCall(t *testing.T) { } _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false) // Create the tracer, the EVM environment and run it - tracer, err := tracers.New("callTracer", nil) + tracer, err := tracers.New("callTracer", nil, nil) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } diff --git a/eth/tracers/js/internal/tracers/4byte_tracer_legacy.js b/eth/tracers/js/internal/tracers/4byte_tracer_legacy.js index 462b4ad4c..e4714b8bf 100644 --- a/eth/tracers/js/internal/tracers/4byte_tracer_legacy.js +++ b/eth/tracers/js/internal/tracers/4byte_tracer_legacy.js @@ -46,7 +46,7 @@ return false; }, - // store save the given indentifier and datasize. + // store save the given identifier and datasize. store: function(id, size){ var key = "" + toHex(id) + "-" + size; this.ids[key] = this.ids[key] + 1 || 1; diff --git a/eth/tracers/js/internal/tracers/call_tracer_legacy.js b/eth/tracers/js/internal/tracers/call_tracer_legacy.js index 3ca737773..054512735 100644 --- a/eth/tracers/js/internal/tracers/call_tracer_legacy.js +++ b/eth/tracers/js/internal/tracers/call_tracer_legacy.js @@ -220,7 +220,7 @@ return this.finalize(result); }, - // finalize recreates a call object using the final desired field oder for json + // finalize recreates a call object using the final desired field order for json // serialization. This is a nicety feature to pass meaningfully ordered results // to users who don't interpret it, just display it. finalize: function(call) { diff --git a/eth/tracers/js/tracer.go b/eth/tracers/js/tracer.go index e305bf189..808852afd 100644 --- a/eth/tracers/js/tracer.go +++ b/eth/tracers/js/tracer.go @@ -425,7 +425,7 @@ type jsTracer struct { // New instantiates a new tracer instance. code specifies a Javascript snippet, // which must evaluate to an expression returning an object with 'step', 'fault' // and 'result' functions. -func newJsTracer(code string, ctx *tracers2.Context) (tracers2.Tracer, error) { +func newJsTracer(code string, ctx *tracers2.Context, cfg json.RawMessage) (tracers2.Tracer, error) { if c, ok := assetTracers[code]; ok { code = c } @@ -831,6 +831,10 @@ func (jst *jsTracer) CaptureExit(output []byte, gasUsed uint64, err error) { } } +func (t *jsTracer) CaptureTxStart(gasLimit uint64) {} + +func (t *jsTracer) CaptureTxEnd(restGas uint64) {} + // GetResult calls the Javascript 'result' function and returns its value, or any accumulated error func (jst *jsTracer) GetResult() (json.RawMessage, error) { // Transform the context into a JavaScript object and inject into the state diff --git a/eth/tracers/js/tracer_test.go b/eth/tracers/js/tracer_test.go index 571512d01..aeded0d34 100644 --- a/eth/tracers/js/tracer_test.go +++ b/eth/tracers/js/tracer_test.go @@ -80,7 +80,7 @@ func runTrace(tracer tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainCon func TestTracer(t *testing.T) { execTracer := func(code string) ([]byte, string) { t.Helper() - tracer, err := newJsTracer(code, nil) + tracer, err := newJsTracer(code, nil, nil) if err != nil { t.Fatal(err) } @@ -130,7 +130,7 @@ func TestTracer(t *testing.T) { func TestHalt(t *testing.T) { t.Skip("duktape doesn't support abortion") timeout := errors.New("stahp") - tracer, err := newJsTracer("{step: function() { while(1); }, result: function() { return null; }, fault: function(){}}", nil) + tracer, err := newJsTracer("{step: function() { while(1); }, result: function() { return null; }, fault: function(){}}", nil, nil) if err != nil { t.Fatal(err) } @@ -144,7 +144,7 @@ func TestHalt(t *testing.T) { } func TestHaltBetweenSteps(t *testing.T) { - tracer, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }}", nil) + tracer, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }}", nil, nil) if err != nil { t.Fatal(err) } @@ -168,7 +168,7 @@ func TestHaltBetweenSteps(t *testing.T) { func TestNoStepExec(t *testing.T) { execTracer := func(code string) []byte { t.Helper() - tracer, err := newJsTracer(code, nil) + tracer, err := newJsTracer(code, nil, nil) if err != nil { t.Fatal(err) } @@ -203,7 +203,7 @@ func TestIsPrecompile(t *testing.T) { chaincfg.BerlinBlock = big.NewInt(300) chaincfg.ArchimedesBlock = big.NewInt(400) txCtx := vm.TxContext{GasPrice: big.NewInt(100000)} - tracer, err := newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil) + tracer, err := newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil) if err != nil { t.Fatal(err) } @@ -217,7 +217,7 @@ func TestIsPrecompile(t *testing.T) { t.Errorf("Tracer should not consider blake2f as precompile in byzantium") } - tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil) + tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil) blockCtx = vm.BlockContext{BlockNumber: big.NewInt(250)} res, err = runTrace(tracer, &vmContext{blockCtx, txCtx}, chaincfg) if err != nil { @@ -228,7 +228,7 @@ func TestIsPrecompile(t *testing.T) { } // test sha disabled in archimedes - tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000002'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil) + tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000002'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil) blockCtx = vm.BlockContext{BlockNumber: big.NewInt(450)} res, err = runTrace(tracer, &vmContext{blockCtx, txCtx}, chaincfg) if err != nil { @@ -238,7 +238,7 @@ func TestIsPrecompile(t *testing.T) { t.Errorf("Tracer should not consider blake2f as precompile in archimedes") } - tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000003'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil) + tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000003'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil) blockCtx = vm.BlockContext{BlockNumber: big.NewInt(450)} res, err = runTrace(tracer, &vmContext{blockCtx, txCtx}, chaincfg) if err != nil { @@ -249,7 +249,7 @@ func TestIsPrecompile(t *testing.T) { } // test blake2f disabled in archimedes - tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil) + tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil) res, err = runTrace(tracer, &vmContext{blockCtx, txCtx}, chaincfg) if err != nil { t.Error(err) @@ -259,7 +259,7 @@ func TestIsPrecompile(t *testing.T) { } // test ecrecover enabled in archimedes - tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000001'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil) + tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000001'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil) res, err = runTrace(tracer, &vmContext{blockCtx, txCtx}, chaincfg) if err != nil { t.Error(err) @@ -271,14 +271,14 @@ func TestIsPrecompile(t *testing.T) { func TestEnterExit(t *testing.T) { // test that either both or none of enter() and exit() are defined - if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}}", new(tracers.Context)); err == nil { + if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}}", new(tracers.Context), nil); err == nil { t.Fatal("tracer creation should've failed without exit() definition") } - if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}, exit: function() {}}", new(tracers.Context)); err != nil { + if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}, exit: function() {}}", new(tracers.Context), nil); err != nil { t.Fatal(err) } // test that the enter and exit method are correctly invoked and the values passed - tracer, err := newJsTracer("{enters: 0, exits: 0, enterGas: 0, gasUsed: 0, step: function() {}, fault: function() {}, result: function() { return {enters: this.enters, exits: this.exits, enterGas: this.enterGas, gasUsed: this.gasUsed} }, enter: function(frame) { this.enters++; this.enterGas = frame.getGas(); }, exit: function(res) { this.exits++; this.gasUsed = res.getGasUsed(); }}", new(tracers.Context)) + tracer, err := newJsTracer("{enters: 0, exits: 0, enterGas: 0, gasUsed: 0, step: function() {}, fault: function() {}, result: function() { return {enters: this.enters, exits: this.exits, enterGas: this.enterGas, gasUsed: this.gasUsed} }, enter: function(frame) { this.enters++; this.enterGas = frame.getGas(); }, exit: function(res) { this.exits++; this.gasUsed = res.getGasUsed(); }}", new(tracers.Context), nil) if err != nil { t.Fatal(err) } diff --git a/eth/tracers/native/4byte.go b/eth/tracers/native/4byte.go index a30fdf753..d184f1980 100644 --- a/eth/tracers/native/4byte.go +++ b/eth/tracers/native/4byte.go @@ -21,7 +21,6 @@ import ( "math/big" "strconv" "sync/atomic" - "time" "github.com/morph-l2/go-ethereum/common" "github.com/morph-l2/go-ethereum/core/vm" @@ -47,20 +46,20 @@ func init() { // 0xc281d19e-0: 1 // } type fourByteTracer struct { - env *vm.EVM + noopTracer ids map[string]int // ids aggregates the 4byte ids found - interrupt uint32 // Atomic flag to signal execution interruption + interrupt atomic.Bool // Atomic flag to signal execution interruption reason error // Textual reason for the interruption activePrecompiles []common.Address // Updated on CaptureStart based on given rules } // newFourByteTracer returns a native go tracer which collects // 4 byte-identifiers of a tx, and implements vm.EVMLogger. -func newFourByteTracer() tracers.Tracer { +func newFourByteTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { t := &fourByteTracer{ ids: make(map[string]int), } - return t + return t, nil } // isPrecompiled returns whether the addr is a precompile. Logic borrowed from newJsTracer in eth/tracers/js/tracer.go @@ -81,8 +80,6 @@ func (t *fourByteTracer) store(id []byte, size int) { // CaptureStart implements the EVMLogger interface to initialize the tracing operation. func (t *fourByteTracer) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { - t.env = env - // Update list of precompiles based on current block rules := env.ChainConfig().Rules(env.Context.BlockNumber, env.Context.Time.Uint64()) t.activePrecompiles = vm.ActivePrecompiles(rules) @@ -93,19 +90,10 @@ func (t *fourByteTracer) CaptureStart(env *vm.EVM, from common.Address, to commo } } -// CaptureState implements the EVMLogger interface to trace a single step of VM execution. -func (t *fourByteTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { -} - -// CaptureStateAfter for special needs, tracks SSTORE ops and records the storage change. -func (t *fourByteTracer) CaptureStateAfter(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { -} - // CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct). func (t *fourByteTracer) CaptureEnter(op vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { // Skip if tracing was interrupted - if atomic.LoadUint32(&t.interrupt) > 0 { - t.env.Cancel() + if t.interrupt.Load() { return } if len(input) < 4 { @@ -123,19 +111,6 @@ func (t *fourByteTracer) CaptureEnter(op vm.OpCode, from common.Address, to comm t.store(input[0:4], len(input)-4) } -// CaptureExit is called when EVM exits a scope, even if the scope didn't -// execute any code. -func (t *fourByteTracer) CaptureExit(output []byte, gasUsed uint64, err error) { -} - -// CaptureFault implements the EVMLogger interface to trace an execution fault. -func (t *fourByteTracer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, depth int, err error) { -} - -// CaptureEnd is called after the call finishes to finalize the tracing. -func (t *fourByteTracer) CaptureEnd(output []byte, gasUsed uint64, _ time.Duration, err error) { -} - // GetResult returns the json-encoded nested list of call traces, and any // error arising from the encoding or forceful termination (via `Stop`). func (t *fourByteTracer) GetResult() (json.RawMessage, error) { @@ -149,5 +124,5 @@ func (t *fourByteTracer) GetResult() (json.RawMessage, error) { // Stop terminates execution of the tracer at the first opportune moment. func (t *fourByteTracer) Stop(err error) { t.reason = err - atomic.StoreUint32(&t.interrupt, 1) + t.interrupt.Store(true) } diff --git a/eth/tracers/native/call.go b/eth/tracers/native/call.go index 9fe34f524..ff480bdcd 100644 --- a/eth/tracers/native/call.go +++ b/eth/tracers/native/call.go @@ -25,100 +25,203 @@ import ( "sync/atomic" "time" + "github.com/morph-l2/go-ethereum/accounts/abi" "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/common/hexutil" "github.com/morph-l2/go-ethereum/core/vm" "github.com/morph-l2/go-ethereum/eth/tracers" + "github.com/morph-l2/go-ethereum/log" ) +//go:generate go run github.com/fjl/gencodec -type callFrame -field-override callFrameMarshaling -out gen_callframe_json.go + func init() { register("callTracer", newCallTracer) } +type callLog struct { + Address common.Address `json:"address"` + Topics []common.Hash `json:"topics"` + Data hexutil.Bytes `json:"data"` + // Position of the log relative to subcalls within the same trace + // See https://github.com/ethereum/go-ethereum/pull/28389 for details + Position hexutil.Uint `json:"position"` +} + type callFrame struct { - Type string `json:"type"` - From string `json:"from"` - To string `json:"to,omitempty"` - Value string `json:"value,omitempty"` - Gas string `json:"gas"` - GasUsed string `json:"gasUsed"` - Input string `json:"input"` - Output string `json:"output,omitempty"` - Error string `json:"error,omitempty"` - Calls []callFrame `json:"calls,omitempty"` + Type vm.OpCode `json:"-"` + From common.Address `json:"from"` + Gas uint64 `json:"gas"` + GasUsed uint64 `json:"gasUsed"` + To *common.Address `json:"to,omitempty" rlp:"optional"` + Input []byte `json:"input" rlp:"optional"` + Output []byte `json:"output,omitempty" rlp:"optional"` + Error string `json:"error,omitempty" rlp:"optional"` + RevertReason string `json:"revertReason,omitempty"` + Calls []callFrame `json:"calls,omitempty" rlp:"optional"` + Logs []callLog `json:"logs,omitempty" rlp:"optional"` + // Placed at end on purpose. The RLP will be decoded to 0 instead of + // nil if there are non-empty elements after in the struct. + Value *big.Int `json:"value,omitempty" rlp:"optional"` +} + +func (f callFrame) TypeString() string { + return f.Type.String() +} + +func (f callFrame) failed() bool { + return len(f.Error) > 0 +} + +func (f *callFrame) processOutput(output []byte, err error) { + output = common.CopyBytes(output) + if err == nil { + f.Output = output + return + } + f.Error = err.Error() + if f.Type == vm.CREATE || f.Type == vm.CREATE2 { + f.To = nil + } + if !errors.Is(err, vm.ErrExecutionReverted) || len(output) == 0 { + return + } + f.Output = output + if len(output) < 4 { + return + } + if unpacked, err := abi.UnpackRevert(output); err == nil { + f.RevertReason = unpacked + } +} + +type callFrameMarshaling struct { + TypeString string `json:"type"` + Gas hexutil.Uint64 + GasUsed hexutil.Uint64 + Value *hexutil.Big + Input hexutil.Bytes + Output hexutil.Bytes } type callTracer struct { - env *vm.EVM + noopTracer callstack []callFrame - interrupt uint32 // Atomic flag to signal execution interruption - reason error // Textual reason for the interruption + config callTracerConfig + gasLimit uint64 + interrupt atomic.Bool // Atomic flag to signal execution interruption + reason error // Textual reason for the interruption +} + +type callTracerConfig struct { + OnlyTopCall bool `json:"onlyTopCall"` // If true, call tracer won't collect any subcalls + WithLog bool `json:"withLog"` // If true, call tracer will collect event logs } // newCallTracer returns a native go tracer which tracks // call frames of a tx, and implements vm.EVMLogger. -func newCallTracer() tracers.Tracer { +func newCallTracer(ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { + var config callTracerConfig + if cfg != nil { + if err := json.Unmarshal(cfg, &config); err != nil { + return nil, err + } + } // First callframe contains tx context info // and is populated on start and end. - t := &callTracer{callstack: make([]callFrame, 1)} - return t + return &callTracer{callstack: make([]callFrame, 1), config: config}, nil } // CaptureStart implements the EVMLogger interface to initialize the tracing operation. func (t *callTracer) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { - t.env = env + toCopy := to t.callstack[0] = callFrame{ - Type: "CALL", - From: addrToHex(from), - To: addrToHex(to), - Input: bytesToHex(input), - Gas: uintToHex(gas), - Value: bigToHex(value), + Type: vm.CALL, + From: from, + To: &toCopy, + Input: common.CopyBytes(input), + Gas: t.gasLimit, + Value: value, } if create { - t.callstack[0].Type = "CREATE" + t.callstack[0].Type = vm.CREATE } } // CaptureEnd is called after the call finishes to finalize the tracing. func (t *callTracer) CaptureEnd(output []byte, gasUsed uint64, _ time.Duration, err error) { - t.callstack[0].GasUsed = uintToHex(gasUsed) - if err != nil { - t.callstack[0].Error = err.Error() - if err.Error() == "execution reverted" && len(output) > 0 { - t.callstack[0].Output = bytesToHex(output) - } - } else { - t.callstack[0].Output = bytesToHex(output) - } + t.callstack[0].processOutput(output, err) } // CaptureState implements the EVMLogger interface to trace a single step of VM execution. func (t *callTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { -} + // skip if the previous op caused an error + if err != nil { + return + } + // Only logs need to be captured via opcode processing + if !t.config.WithLog { + return + } + // Avoid processing nested calls when only caring about top call + if t.config.OnlyTopCall && depth > 1 { + return + } + // Skip if tracing was interrupted + if t.interrupt.Load() { + return + } + switch op { + case vm.LOG0, vm.LOG1, vm.LOG2, vm.LOG3, vm.LOG4: + size := int(op - vm.LOG0) -// CaptureStateAfter for special needs, tracks SSTORE ops and records the storage change. -func (t *callTracer) CaptureStateAfter(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { -} + stack := scope.Stack + stackData := stack.Data() -// CaptureFault implements the EVMLogger interface to trace an execution fault. -func (t *callTracer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, _ *vm.ScopeContext, depth int, err error) { + // Don't modify the stack + mStart := stackData[len(stackData)-1] + mSize := stackData[len(stackData)-2] + topics := make([]common.Hash, size) + for i := 0; i < size; i++ { + topic := stackData[len(stackData)-2-(i+1)] + topics[i] = common.Hash(topic.Bytes32()) + } + + data, err := tracers.GetMemoryCopyPadded(scope.Memory, int64(mStart.Uint64()), int64(mSize.Uint64())) + if err != nil { + // mSize was unrealistically large + log.Warn("failed to copy CREATE2 input", "err", err, "tracer", "callTracer", "offset", mStart, "size", mSize) + return + } + + log := callLog{ + Address: scope.Contract.Address(), + Topics: topics, + Data: hexutil.Bytes(data), + Position: hexutil.Uint(len(t.callstack[len(t.callstack)-1].Calls)), + } + t.callstack[len(t.callstack)-1].Logs = append(t.callstack[len(t.callstack)-1].Logs, log) + } } // CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct). func (t *callTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { + if t.config.OnlyTopCall { + return + } // Skip if tracing was interrupted - if atomic.LoadUint32(&t.interrupt) > 0 { - t.env.Cancel() + if t.interrupt.Load() { return } + toCopy := to call := callFrame{ - Type: typ.String(), - From: addrToHex(from), - To: addrToHex(to), - Input: bytesToHex(input), - Gas: uintToHex(gas), - Value: bigToHex(value), + Type: typ, + From: from, + To: &toCopy, + Input: common.CopyBytes(input), + Gas: gas, + Value: value, } t.callstack = append(t.callstack, call) } @@ -126,6 +229,9 @@ func (t *callTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common. // CaptureExit is called when EVM exits a scope, even if the scope didn't // execute any code. func (t *callTracer) CaptureExit(output []byte, gasUsed uint64, err error) { + if t.config.OnlyTopCall { + return + } size := len(t.callstack) if size <= 1 { return @@ -135,24 +241,30 @@ func (t *callTracer) CaptureExit(output []byte, gasUsed uint64, err error) { t.callstack = t.callstack[:size-1] size -= 1 - call.GasUsed = uintToHex(gasUsed) - if err == nil { - call.Output = bytesToHex(output) - } else { - call.Error = err.Error() - if call.Type == "CREATE" || call.Type == "CREATE2" { - call.To = "" - } - } + call.GasUsed = gasUsed + call.processOutput(output, err) t.callstack[size-1].Calls = append(t.callstack[size-1].Calls, call) } +func (t *callTracer) CaptureTxStart(gasLimit uint64) { + t.gasLimit = gasLimit +} + +func (t *callTracer) CaptureTxEnd(restGas uint64) { + t.callstack[0].GasUsed = t.gasLimit - restGas + if t.config.WithLog { + // Logs are not emitted when the call fails + clearFailedLogs(&t.callstack[0], false) + } +} + // GetResult returns the json-encoded nested list of call traces, and any // error arising from the encoding or forceful termination (via `Stop`). func (t *callTracer) GetResult() (json.RawMessage, error) { if len(t.callstack) != 1 { return nil, errors.New("incorrect number of top-level calls") } + res, err := json.Marshal(t.callstack[0]) if err != nil { return nil, err @@ -163,7 +275,20 @@ func (t *callTracer) GetResult() (json.RawMessage, error) { // Stop terminates execution of the tracer at the first opportune moment. func (t *callTracer) Stop(err error) { t.reason = err - atomic.StoreUint32(&t.interrupt, 1) + t.interrupt.Store(true) +} + +// clearFailedLogs clears the logs of a callframe and all its children +// in case of execution failure. +func clearFailedLogs(cf *callFrame, parentFailed bool) { + failed := cf.failed() || parentFailed + // Clear own logs + if failed { + cf.Logs = nil + } + for i := range cf.Calls { + clearFailedLogs(&cf.Calls[i], failed) + } } func bytesToHex(s []byte) string { diff --git a/eth/tracers/native/gen_account_json.go b/eth/tracers/native/gen_account_json.go new file mode 100644 index 000000000..602800f55 --- /dev/null +++ b/eth/tracers/native/gen_account_json.go @@ -0,0 +1,56 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package native + +import ( + "encoding/json" + "math/big" + + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/common/hexutil" +) + +var _ = (*accountMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (a account) MarshalJSON() ([]byte, error) { + type account struct { + Balance *hexutil.Big `json:"balance,omitempty"` + Code hexutil.Bytes `json:"code,omitempty"` + Nonce uint64 `json:"nonce,omitempty"` + Storage map[common.Hash]common.Hash `json:"storage,omitempty"` + } + var enc account + enc.Balance = (*hexutil.Big)(a.Balance) + enc.Code = a.Code + enc.Nonce = a.Nonce + enc.Storage = a.Storage + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (a *account) UnmarshalJSON(input []byte) error { + type account struct { + Balance *hexutil.Big `json:"balance,omitempty"` + Code *hexutil.Bytes `json:"code,omitempty"` + Nonce *uint64 `json:"nonce,omitempty"` + Storage map[common.Hash]common.Hash `json:"storage,omitempty"` + } + var dec account + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Balance != nil { + a.Balance = (*big.Int)(dec.Balance) + } + if dec.Code != nil { + a.Code = *dec.Code + } + if dec.Nonce != nil { + a.Nonce = *dec.Nonce + } + if dec.Storage != nil { + a.Storage = dec.Storage + } + return nil +} diff --git a/eth/tracers/native/gen_callframe_json.go b/eth/tracers/native/gen_callframe_json.go new file mode 100644 index 000000000..0267bf6bb --- /dev/null +++ b/eth/tracers/native/gen_callframe_json.go @@ -0,0 +1,107 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package native + +import ( + "encoding/json" + "math/big" + + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/common/hexutil" + "github.com/morph-l2/go-ethereum/core/vm" +) + +var _ = (*callFrameMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (c callFrame) MarshalJSON() ([]byte, error) { + type callFrame0 struct { + Type vm.OpCode `json:"-"` + From common.Address `json:"from"` + Gas hexutil.Uint64 `json:"gas"` + GasUsed hexutil.Uint64 `json:"gasUsed"` + To *common.Address `json:"to,omitempty" rlp:"optional"` + Input hexutil.Bytes `json:"input" rlp:"optional"` + Output hexutil.Bytes `json:"output,omitempty" rlp:"optional"` + Error string `json:"error,omitempty" rlp:"optional"` + RevertReason string `json:"revertReason,omitempty"` + Calls []callFrame `json:"calls,omitempty" rlp:"optional"` + Logs []callLog `json:"logs,omitempty" rlp:"optional"` + Value *hexutil.Big `json:"value,omitempty" rlp:"optional"` + TypeString string `json:"type"` + } + var enc callFrame0 + enc.Type = c.Type + enc.From = c.From + enc.Gas = hexutil.Uint64(c.Gas) + enc.GasUsed = hexutil.Uint64(c.GasUsed) + enc.To = c.To + enc.Input = c.Input + enc.Output = c.Output + enc.Error = c.Error + enc.RevertReason = c.RevertReason + enc.Calls = c.Calls + enc.Logs = c.Logs + enc.Value = (*hexutil.Big)(c.Value) + enc.TypeString = c.TypeString() + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (c *callFrame) UnmarshalJSON(input []byte) error { + type callFrame0 struct { + Type *vm.OpCode `json:"-"` + From *common.Address `json:"from"` + Gas *hexutil.Uint64 `json:"gas"` + GasUsed *hexutil.Uint64 `json:"gasUsed"` + To *common.Address `json:"to,omitempty" rlp:"optional"` + Input *hexutil.Bytes `json:"input" rlp:"optional"` + Output *hexutil.Bytes `json:"output,omitempty" rlp:"optional"` + Error *string `json:"error,omitempty" rlp:"optional"` + RevertReason *string `json:"revertReason,omitempty"` + Calls []callFrame `json:"calls,omitempty" rlp:"optional"` + Logs []callLog `json:"logs,omitempty" rlp:"optional"` + Value *hexutil.Big `json:"value,omitempty" rlp:"optional"` + } + var dec callFrame0 + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Type != nil { + c.Type = *dec.Type + } + if dec.From != nil { + c.From = *dec.From + } + if dec.Gas != nil { + c.Gas = uint64(*dec.Gas) + } + if dec.GasUsed != nil { + c.GasUsed = uint64(*dec.GasUsed) + } + if dec.To != nil { + c.To = dec.To + } + if dec.Input != nil { + c.Input = *dec.Input + } + if dec.Output != nil { + c.Output = *dec.Output + } + if dec.Error != nil { + c.Error = *dec.Error + } + if dec.RevertReason != nil { + c.RevertReason = *dec.RevertReason + } + if dec.Calls != nil { + c.Calls = dec.Calls + } + if dec.Logs != nil { + c.Logs = dec.Logs + } + if dec.Value != nil { + c.Value = (*big.Int)(dec.Value) + } + return nil +} diff --git a/eth/tracers/native/noop.go b/eth/tracers/native/noop.go index 48f3d2a45..33ab891cb 100644 --- a/eth/tracers/native/noop.go +++ b/eth/tracers/native/noop.go @@ -35,8 +35,8 @@ func init() { type noopTracer struct{} // newNoopTracer returns a new noop tracer. -func newNoopTracer() tracers.Tracer { - return &noopTracer{} +func newNoopTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { + return &noopTracer{}, nil } // CaptureStart implements the EVMLogger interface to initialize the tracing operation. @@ -68,6 +68,10 @@ func (t *noopTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common. func (t *noopTracer) CaptureExit(output []byte, gasUsed uint64, err error) { } +func (t *noopTracer) CaptureTxStart(gasLimit uint64) {} + +func (t *noopTracer) CaptureTxEnd(restGas uint64) {} + // GetResult returns an empty json object. func (t *noopTracer) GetResult() (json.RawMessage, error) { return json.RawMessage(`{}`), nil diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index b3981acb2..4218f0852 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -8,12 +8,15 @@ import ( "time" "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/common/hexutil" "github.com/morph-l2/go-ethereum/core/vm" "github.com/morph-l2/go-ethereum/crypto" "github.com/morph-l2/go-ethereum/eth/tracers" "github.com/morph-l2/go-ethereum/log" ) +//go:generate go run github.com/fjl/gencodec -type account -field-override accountMarshaling -out gen_account_json.go + func init() { register("prestateTracer", newPrestateTracer) } @@ -21,19 +24,10 @@ func init() { type state = map[common.Address]*account type account struct { - Balance *big.Int - Code []byte - Nonce uint64 - Storage map[common.Hash]common.Hash -} - -func (a *account) marshal() accountMarshaling { - return accountMarshaling{ - Balance: bigToHex(a.Balance), - Code: bytesToHex(a.Code), - Nonce: a.Nonce, - Storage: a.Storage, - } + Balance *big.Int `json:"balance,omitempty"` + Code []byte `json:"code,omitempty"` + Nonce uint64 `json:"nonce,omitempty"` + Storage map[common.Hash]common.Hash `json:"storage,omitempty"` } func (a *account) exists() bool { @@ -41,10 +35,8 @@ func (a *account) exists() bool { } type accountMarshaling struct { - Balance string `json:"balance,omitempty"` - Code string `json:"code,omitempty"` - Nonce uint64 `json:"nonce,omitempty"` - Storage map[common.Hash]common.Hash `json:"storage,omitempty"` + Balance *hexutil.Big + Code hexutil.Bytes } type prestateTracer struct { @@ -54,20 +46,32 @@ type prestateTracer struct { post state create bool to common.Address - gasLimit uint64 // Amount of gas bought for the whole tx + gasLimit uint64 // Amount of gas bought for the whole tx + config prestateTracerConfig interrupt atomic.Bool // Atomic flag to signal execution interruption reason error // Textual reason for the interruption created map[common.Address]bool deleted map[common.Address]bool } -func newPrestateTracer() tracers.Tracer { +type prestateTracerConfig struct { + DiffMode bool `json:"diffMode"` // If true, this tracer will return state modifications +} + +func newPrestateTracer(ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { + var config prestateTracerConfig + if cfg != nil { + if err := json.Unmarshal(cfg, &config); err != nil { + return nil, err + } + } return &prestateTracer{ pre: state{}, post: state{}, + config: config, created: make(map[common.Address]bool), deleted: make(map[common.Address]bool), - } + }, nil } // CaptureStart implements the EVMLogger interface to initialize the tracing operation. @@ -92,10 +96,18 @@ func (t *prestateTracer) CaptureStart(env *vm.EVM, from common.Address, to commo fromBal.Add(fromBal, new(big.Int).Add(value, consumedGas)) t.pre[from].Balance = fromBal t.pre[from].Nonce-- + + if create && t.config.DiffMode { + t.created[to] = true + } } // CaptureEnd is called after the call finishes to finalize the tracing. -func (t *prestateTracer) CaptureEnd(output []byte, gasUsed uint64, d time.Duration, err error) { +func (t *prestateTracer) CaptureEnd(output []byte, gasUsed uint64, _ time.Duration, err error) { + if t.config.DiffMode { + return + } + if t.create { // Keep existing account prior to contract creation at that address if s := t.pre[t.to]; s != nil && !s.exists() { @@ -157,6 +169,10 @@ func (t *prestateTracer) CaptureTxStart(gasLimit uint64) { } func (t *prestateTracer) CaptureTxEnd(restGas uint64) { + if !t.config.DiffMode { + return + } + for addr, state := range t.pre { // The deleted account's state is pruned from `post` but kept in `pre` if _, ok := t.deleted[addr]; ok { @@ -218,15 +234,20 @@ func (t *prestateTracer) CaptureTxEnd(restGas uint64) { // GetResult returns the json-encoded nested list of call traces, and any // error arising from the encoding or forceful termination (via `Stop`). func (t *prestateTracer) GetResult() (json.RawMessage, error) { - pre := make(map[string]accountMarshaling) - for addr, state := range t.pre { - pre[addrToHex(addr)] = state.marshal() + var res []byte + var err error + if t.config.DiffMode { + res, err = json.Marshal(struct { + Post state `json:"post"` + Pre state `json:"pre"` + }{t.post, t.pre}) + } else { + res, err = json.Marshal(t.pre) } - res, err := json.Marshal(pre) if err != nil { return nil, err } - return res, t.reason + return json.RawMessage(res), t.reason } // Stop terminates execution of the tracer at the first opportune moment. diff --git a/eth/tracers/native/tracer.go b/eth/tracers/native/tracer.go index 81289bb0e..857abd2e7 100644 --- a/eth/tracers/native/tracer.go +++ b/eth/tracers/native/tracer.go @@ -14,29 +14,24 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -/* -Package native is a collection of tracers written in go. - -In order to add a native tracer and have it compiled into the binary, a new -file needs to be added to this folder, containing an implementation of the -`eth.tracers.Tracer` interface. - -Aside from implementing the tracer, it also needs to register itself, using the -`register` method -- and this needs to be done in the package initialization. - -Example: - -```golang - - func init() { - register("noopTracerNative", newNoopTracer) - } - -``` -*/ +// Package native is a collection of tracers written in go. +// +// In order to add a native tracer and have it compiled into the binary, a new +// file needs to be added to this folder, containing an implementation of the +// `eth.tracers.Tracer` interface. +// +// Aside from implementing the tracer, it also needs to register itself, using the +// `register` method -- and this needs to be done in the package initialization. +// +// Example: +// +// func init() { +// register("noopTracerNative", newNoopTracer) +// } package native import ( + "encoding/json" "errors" "github.com/morph-l2/go-ethereum/eth/tracers" @@ -47,6 +42,9 @@ func init() { tracers.RegisterLookup(false, lookup) } +// ctorFn is the constructor signature of a native tracer. +type ctorFn = func(*tracers.Context, json.RawMessage) (tracers.Tracer, error) + /* ctors is a map of package-local tracer constructors. @@ -59,23 +57,23 @@ The go spec (https://golang.org/ref/spec#Package_initialization) says Hence, we cannot make the map in init, but must make it upon first use. */ -var ctors map[string]func() tracers.Tracer +var ctors map[string]ctorFn // register is used by native tracers to register their presence. -func register(name string, ctor func() tracers.Tracer) { +func register(name string, ctor ctorFn) { if ctors == nil { - ctors = make(map[string]func() tracers.Tracer) + ctors = make(map[string]ctorFn) } ctors[name] = ctor } // lookup returns a tracer, if one can be matched to the given name. -func lookup(name string, ctx *tracers.Context) (tracers.Tracer, error) { +func lookup(name string, ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { if ctors == nil { - ctors = make(map[string]func() tracers.Tracer) + ctors = make(map[string]ctorFn) } if ctor, ok := ctors[name]; ok { - return ctor(), nil + return ctor(ctx, cfg) } return nil, errors.New("no tracer found") } diff --git a/eth/tracers/tracers.go b/eth/tracers/tracers.go index de6d25b4b..d2b9a6479 100644 --- a/eth/tracers/tracers.go +++ b/eth/tracers/tracers.go @@ -43,7 +43,7 @@ type Tracer interface { Stop(err error) } -type lookupFunc func(string, *Context) (Tracer, error) +type lookupFunc func(string, *Context, json.RawMessage) (Tracer, error) var ( lookups []lookupFunc @@ -63,9 +63,9 @@ func RegisterLookup(wildcard bool, lookup lookupFunc) { // New returns a new instance of a tracer, by iterating through the // registered lookups. -func New(code string, ctx *Context) (Tracer, error) { +func New(code string, ctx *Context, cfg json.RawMessage) (Tracer, error) { for _, lookup := range lookups { - if tracer, err := lookup(code, ctx); err == nil { + if tracer, err := lookup(code, ctx, cfg); err == nil { return tracer, nil } } diff --git a/rollup/tracing/mux_tracer.go b/rollup/tracing/mux_tracer.go index 585af1c7e..612f98582 100644 --- a/rollup/tracing/mux_tracer.go +++ b/rollup/tracing/mux_tracer.go @@ -67,3 +67,15 @@ func (t *MuxTracer) CaptureEnd(output []byte, gasUsed uint64, d time.Duration, e tracer.CaptureEnd(output, gasUsed, d, err) } } + +func (t *MuxTracer) CaptureTxStart(gasLimit uint64) { + for _, tracer := range t.tracers { + tracer.CaptureTxStart(gasLimit) + } +} + +func (t *MuxTracer) CaptureTxEnd(restGas uint64) { + for _, tracer := range t.tracers { + tracer.CaptureTxEnd(restGas) + } +} diff --git a/rollup/tracing/tracing.go b/rollup/tracing/tracing.go index fbe5e4d63..5fb014324 100644 --- a/rollup/tracing/tracing.go +++ b/rollup/tracing/tracing.go @@ -313,7 +313,7 @@ func (env *TraceEnv) getTxResult(state *state.StateDB, index int, block *types.B TxIndex: index, TxHash: tx.Hash(), } - callTracer, err := tracers.New("callTracer", &tracerContext) + callTracer, err := tracers.New("callTracer", &tracerContext, nil) if err != nil { return fmt.Errorf("failed to create callTracer: %w", err) } From d8f08de39373af6fc26d90a277e9f84f7c7bf50c Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Mon, 10 Feb 2025 18:48:40 +0800 Subject: [PATCH 39/61] preimages commit --- trie/database.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trie/database.go b/trie/database.go index ac93d782c..50b2bc0da 100644 --- a/trie/database.go +++ b/trie/database.go @@ -758,6 +758,9 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H if !ok { return errors.New("not supported") } + if db.preimages != nil { + db.preimages.commit(true) + } return zdb.CommitState(node, common.Hash{}, 0, report) } From 6d086c50b08d879f6525d94efd8c46b543ab33ed Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 11 Feb 2025 14:44:52 +0800 Subject: [PATCH 40/61] update zktrie version --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2d5211474..f69daf749 100644 --- a/go.mod +++ b/go.mod @@ -127,4 +127,4 @@ require ( rsc.io/tmplfunc v0.0.3 // indirect ) -replace github.com/scroll-tech/zktrie v0.8.4 => github.com/ryanmorphl2/zktrie v1.9.3 +replace github.com/scroll-tech/zktrie v0.8.4 => github.com/morph-l2/zktrie v0.8.5-alpha diff --git a/go.sum b/go.sum index 835d0f221..cfc6cc8c6 100644 --- a/go.sum +++ b/go.sum @@ -374,6 +374,8 @@ github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqky github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/morph-l2/zktrie v0.8.5-alpha h1:RW4xGw+DugaAILjFUM1mutulLeAuAC0w7XBt7AhDyLY= +github.com/morph-l2/zktrie v0.8.5-alpha/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= @@ -432,8 +434,6 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/ryanmorphl2/zktrie v1.9.3 h1:fea9GBXT7j+r6jncsVyvBgmmnYV/XzmFNvU/g0JtxFI= -github.com/ryanmorphl2/zktrie v1.9.3/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= From eb305cb63c86c4ebf5b488ffd8c98825dc1a5884 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 4 Mar 2025 14:12:24 +0800 Subject: [PATCH 41/61] return state not available --- trie/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie/database.go b/trie/database.go index 50b2bc0da..8d380edbb 100644 --- a/trie/database.go +++ b/trie/database.go @@ -1121,7 +1121,7 @@ func (db *Database) GetFrom(root, key []byte) ([]byte, error) { return n, nil } - return nil, errors.New("reader is nil") + return nil, errors.New("state is not available") } return nil, nil } From e844e22fce8133b4ee3226f57aa74a5d97c2716c Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Wed, 12 Mar 2025 16:56:37 +0800 Subject: [PATCH 42/61] fix comment --- triedb/pathdb/journal.go | 1 - 1 file changed, 1 deletion(-) diff --git a/triedb/pathdb/journal.go b/triedb/pathdb/journal.go index 512ec5c6c..dfa654ae7 100644 --- a/triedb/pathdb/journal.go +++ b/triedb/pathdb/journal.go @@ -357,7 +357,6 @@ func (db *Database) loadDiffLayer(parent layer, r *rlp.Stream, journalTypeForRea } log.Debug("Loaded diff layer journal", "root", root, "parent", parent.rootHash(), "id", parent.stateID()+1, "block", block, "nodes", len(nodes)) - // add cache first l := newDiffLayer(parent, root, parent.stateID()+1, block, nodes) return db.loadDiffLayer(l, r, journalTypeForReader) } From 804cb3235f500b3813ab3540b0d117be5d517a62 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Fri, 14 Mar 2025 12:18:49 +0800 Subject: [PATCH 43/61] flush with time interval --- core/blockchain.go | 10 ++++++---- core/blockchain_l2.go | 21 ++++++++++++++++----- trie/database.go | 8 ++++---- triedb/hashdb/zk_trie_database.go | 4 ++-- triedb/pathdb/database.go | 13 +++++++++++-- 5 files changed, 39 insertions(+), 17 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 8ab8cc119..1e684b479 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -198,10 +198,11 @@ type BlockChain struct { chainConfig *params.ChainConfig // Chain & network configuration cacheConfig *CacheConfig // Cache configuration for pruning - db ethdb.Database // Low level persistent database to store final content in - snaps *snapshot.Tree // Snapshot tree for fast trie leaf access - triegc *prque.Prque // Priority queue mapping block numbers to tries to gc - gcproc time.Duration // Accumulates canonical block processing for trie dumping + db ethdb.Database // Low level persistent database to store final content in + snaps *snapshot.Tree // Snapshot tree for fast trie leaf access + triegc *prque.Prque // Priority queue mapping block numbers to tries to gc + gcproc time.Duration // Accumulates canonical block processing for trie dumping + flushInterval atomic.Int64 // Time interval (processing time) after which to flush a state // txLookupLimit is the maximum number of blocks from head whose tx indices // are reserved: @@ -293,6 +294,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par bc.validator = NewBlockValidator(chainConfig, bc, engine) bc.prefetcher = newStatePrefetcher(chainConfig, bc, engine) bc.processor = NewStateProcessor(chainConfig, bc, engine) + bc.flushInterval.Store(int64(cacheConfig.TrieTimeLimit)) var err error bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.insertStopped) diff --git a/core/blockchain_l2.go b/core/blockchain_l2.go index c6e8d82a8..66091d887 100644 --- a/core/blockchain_l2.go +++ b/core/blockchain_l2.go @@ -118,12 +118,24 @@ func (bc *BlockChain) writeBlockStateWithoutHead(block *types.Block, receipts [] } triedb := bc.stateCache.TrieDB() + flushInterval := time.Duration(bc.flushInterval.Load()) + // If we're running an archive node, always flush if bc.cacheConfig.TrieDirtyDisabled { if triedb.Scheme() == rawdb.PathScheme { + // If we exceeded time allowance, flush an entire trie to disk + flush := bc.gcproc > flushInterval + + commitDone := func() { + if flush { + bc.gcproc = 0 + log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", flushInterval, "gcproc", bc.gcproc) + } + } + // If node is running in path mode, skip explicit gc operation // which is unnecessary in this mode. - return triedb.CommitState(root, origin, current, false) + return triedb.CommitState(root, origin, current, false, flush, commitDone) } return triedb.Commit(root, false, nil) } @@ -145,9 +157,8 @@ func (bc *BlockChain) writeBlockStateWithoutHead(block *types.Block, receipts [] } // Find the next state trie we need to commit chosen := current - TriesInMemory - //flushInterval := time.Duration(atomic.LoadInt64(&bc.flushInterval)) // If we exceeded time allowance, flush an entire trie to disk - if bc.gcproc > bc.cacheConfig.TrieTimeLimit { + if bc.gcproc > flushInterval { // If the header is missing (canonical chain behind), we're reorging a low // diff sidechain. Suspend committing until this operation is completed. header := bc.GetHeaderByNumber(chosen) @@ -156,8 +167,8 @@ func (bc *BlockChain) writeBlockStateWithoutHead(block *types.Block, receipts [] } else { // If we're exceeding limits but haven't reached a large enough memory gap, // warn the user that the system is becoming unstable. - if chosen < lastWrite+TriesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { - log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/TriesInMemory) + if chosen < lastWrite+TriesInMemory && bc.gcproc >= 2*flushInterval { + log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", flushInterval, "optimum", float64(chosen-lastWrite)/TriesInMemory) } // Flush an entire trie and restart the counters triedb.Commit(header.Root, true, nil) diff --git a/trie/database.go b/trie/database.go index 8d380edbb..8e3ae3461 100644 --- a/trie/database.go +++ b/trie/database.go @@ -318,7 +318,7 @@ type backend interface { // Commit writes all relevant trie nodes belonging to the specified state // to disk. Report specifies whether logs will be displayed in info level. - CommitState(root common.Hash, parentRoot common.Hash, blockNumber uint64, report bool) error + CommitState(root common.Hash, parentRoot common.Hash, blockNumber uint64, report bool, flush bool, callback func()) error // Commit write custom nodes belong genesis states, only onece CommitGenesis(root common.Hash) error @@ -761,7 +761,7 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H if db.preimages != nil { db.preimages.commit(true) } - return zdb.CommitState(node, common.Hash{}, 0, report) + return zdb.CommitState(node, common.Hash{}, 0, report, false, nil) } // Create a database batch to flush persistent data out. It is important that @@ -1009,12 +1009,12 @@ func (db *Database) Reader(blockRoot common.Hash) (Reader, error) { return nil, errors.New("unknown backend") } -func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockNumber uint64, report bool) error { +func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockNumber uint64, report bool, flush bool, callback func()) error { if db.backend != nil { if db.preimages != nil { db.preimages.commit(true) } - return db.backend.CommitState(root, parentRoot, blockNumber, report) + return db.backend.CommitState(root, parentRoot, blockNumber, report, flush, callback) } return errors.New("not supported") diff --git a/triedb/hashdb/zk_trie_database.go b/triedb/hashdb/zk_trie_database.go index ba9af28ee..1f24513bc 100644 --- a/triedb/hashdb/zk_trie_database.go +++ b/triedb/hashdb/zk_trie_database.go @@ -74,7 +74,7 @@ func (db *ZktrieDatabase) Size() (common.StorageSize, common.StorageSize, common return 0, 0, db.dirtiesSize + metadataSize } -func (db *ZktrieDatabase) CommitState(root common.Hash, parentRoot common.Hash, blockNumber uint64, report bool) error { +func (db *ZktrieDatabase) CommitState(root common.Hash, parentRoot common.Hash, blockNumber uint64, report bool, flush bool, callback func()) error { beforeDirtyCount, beforeDirtySize := len(db.dirties), db.dirtiesSize start := time.Now() @@ -100,7 +100,7 @@ func (db *ZktrieDatabase) CommitState(root common.Hash, parentRoot common.Hash, } func (db *ZktrieDatabase) CommitGenesis(root common.Hash) error { - return db.CommitState(root, common.Hash{}, 0, true) + return db.CommitState(root, common.Hash{}, 0, true, false, nil) } func (db *ZktrieDatabase) commitAllDirties() error { diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 19c5f161f..0a627c271 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -209,7 +209,7 @@ func (db *Database) CommitGenesis(root common.Hash) error { // Commit traverses downwards the layer tree from a specified layer with the // provided state root and all the layers below are flattened downwards. It // can be used alone and mostly for test purposes. -func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockNumber uint64, report bool) error { +func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockNumber uint64, report bool, flush bool, callback func()) error { // Hold the lock to prevent concurrent mutations. db.lock.Lock() defer db.lock.Unlock() @@ -231,12 +231,21 @@ func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockN } db.dirties = make(dbtypes.KvMap) + layers := maxDiffLayers + if flush { + layers = 0 + } + // Keep 128 diff layers in the memory, persistent layer is 129th. // - head layer is paired with HEAD state // - head-1 layer is paired with HEAD-1 state // - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state // - head-128 layer(disk layer) is paired with HEAD-128 state - return db.tree.cap(root, maxDiffLayers) + err := db.tree.cap(root, layers) + if callback != nil { + callback() + } + return err } // Close closes the trie database and the held freezer. From 1e734d887bee652e94ad3495cdd863932bfcd274 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 18 Mar 2025 14:32:59 +0800 Subject: [PATCH 44/61] flush intercal default time --- eth/ethconfig/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index f93483e0c..fae10ab57 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -80,7 +80,7 @@ var Defaults = Config{ TrieCleanCacheJournal: "triecache", TrieCleanCacheRejournal: 60 * time.Minute, TrieDirtyCache: 256, - TrieTimeout: 60 * time.Minute, + TrieTimeout: 20 * time.Minute, SnapshotCache: 102, FilterLogCacheSize: 32, Miner: miner.DefaultConfig, From 284315c19356b12576d35e15cc15e8e6cbb0e60b Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 18 Mar 2025 15:11:56 +0800 Subject: [PATCH 45/61] decrease flush intercal default time --- eth/ethconfig/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index fae10ab57..88dd7efd7 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -80,7 +80,7 @@ var Defaults = Config{ TrieCleanCacheJournal: "triecache", TrieCleanCacheRejournal: 60 * time.Minute, TrieDirtyCache: 256, - TrieTimeout: 20 * time.Minute, + TrieTimeout: 1 * time.Minute, SnapshotCache: 102, FilterLogCacheSize: 32, Miner: miner.DefaultConfig, From c78b876918fb7319e77f15c1199bf89ac7749b5e Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 25 Mar 2025 17:17:57 +0800 Subject: [PATCH 46/61] set flush intercal default time --- eth/ethconfig/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 88dd7efd7..72ffe1af9 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -80,7 +80,7 @@ var Defaults = Config{ TrieCleanCacheJournal: "triecache", TrieCleanCacheRejournal: 60 * time.Minute, TrieDirtyCache: 256, - TrieTimeout: 1 * time.Minute, + TrieTimeout: 5 * time.Minute, SnapshotCache: 102, FilterLogCacheSize: 32, Miner: miner.DefaultConfig, From 8b62f6350948a9de38d766c8fd0c05a4a029be86 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Thu, 27 Mar 2025 15:14:04 +0800 Subject: [PATCH 47/61] add comment --- trie/database.go | 1 + 1 file changed, 1 insertion(+) diff --git a/trie/database.go b/trie/database.go index 8e3ae3461..dd8e2ef9c 100644 --- a/trie/database.go +++ b/trie/database.go @@ -526,6 +526,7 @@ func (db *Database) Reference(child common.Hash, parent common.Hash) { if db.backend != nil { zdb, ok := db.backend.(*hashdb.ZktrieDatabase) if !ok { + log.Error("Databse not support reference") return } zdb.Reference(child, parent) From 409206cbaead553a21d16a7432a19deee61265cc Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 1 Apr 2025 16:08:01 +0800 Subject: [PATCH 48/61] check scheme before using db --- eth/backend.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index dfc6874b1..40b5d0418 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -54,6 +54,7 @@ import ( "github.com/morph-l2/go-ethereum/rlp" "github.com/morph-l2/go-ethereum/rollup/batch" "github.com/morph-l2/go-ethereum/rpc" + "github.com/morph-l2/go-ethereum/trie" ) // Config contains the configuration options of the ETH protocol. @@ -133,16 +134,22 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideArrowGlacier) - if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { - return nil, genesisErr - } config.StateScheme, err = rawdb.ParseStateScheme(config.StateScheme, chainDb) if err != nil { return nil, err } + // Set path trie tag before using trie db + if config.StateScheme == rawdb.PathScheme { + trie.GenesisStateInPathZkTrie = true + } + + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideArrowGlacier) + if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { + return nil, genesisErr + } + log.Info("Initialised chain configuration", "config", chainConfig) if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, stack.ResolvePath(config.TrieCleanCacheJournal)); err != nil { From f4cd3250513ae94aea916e6cf6f6b64fed5d0885 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 1 Apr 2025 19:02:13 +0800 Subject: [PATCH 49/61] fix log, set interval time --- core/blockchain_l2.go | 2 +- eth/ethconfig/config.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/blockchain_l2.go b/core/blockchain_l2.go index 66091d887..32db6f8d7 100644 --- a/core/blockchain_l2.go +++ b/core/blockchain_l2.go @@ -129,7 +129,7 @@ func (bc *BlockChain) writeBlockStateWithoutHead(block *types.Block, receipts [] commitDone := func() { if flush { bc.gcproc = 0 - log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", flushInterval, "gcproc", bc.gcproc) + log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", flushInterval) } } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 72ffe1af9..c835f5b61 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -80,7 +80,7 @@ var Defaults = Config{ TrieCleanCacheJournal: "triecache", TrieCleanCacheRejournal: 60 * time.Minute, TrieDirtyCache: 256, - TrieTimeout: 5 * time.Minute, + TrieTimeout: 3 * time.Minute, SnapshotCache: 102, FilterLogCacheSize: 32, Miner: miner.DefaultConfig, From 8f1fd9a0d7be47a3eed5984a93d36be8b727880e Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Wed, 2 Apr 2025 12:05:16 +0800 Subject: [PATCH 50/61] fix log --- core/blockchain_l2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/blockchain_l2.go b/core/blockchain_l2.go index 32db6f8d7..d84b71442 100644 --- a/core/blockchain_l2.go +++ b/core/blockchain_l2.go @@ -128,8 +128,8 @@ func (bc *BlockChain) writeBlockStateWithoutHead(block *types.Block, receipts [] commitDone := func() { if flush { - bc.gcproc = 0 log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", flushInterval) + bc.gcproc = 0 } } From 3eb34e62a166b81b81fb5c743d87ec065d0fc38f Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 15 Apr 2025 18:58:49 +0800 Subject: [PATCH 51/61] merge main --- params/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/params/config.go b/params/config.go index 52331d58f..a4807c8c1 100644 --- a/params/config.go +++ b/params/config.go @@ -283,7 +283,7 @@ var ( ShanghaiBlock: big.NewInt(0), BernoulliBlock: big.NewInt(0), CurieBlock: big.NewInt(6330180), - Morph203Time: nil, + Morph203Time: NewUint64(1745388000), TerminalTotalDifficulty: big.NewInt(0), Morph: MorphConfig{ UseZktrie: true, @@ -313,7 +313,7 @@ var ( ShanghaiBlock: big.NewInt(0), BernoulliBlock: big.NewInt(0), CurieBlock: big.NewInt(0), - Morph203Time: nil, + Morph203Time: NewUint64(1747029600), TerminalTotalDifficulty: big.NewInt(0), Morph: MorphConfig{ UseZktrie: true, From eb9937de9b618905463991ad55244577a9a2a798 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 22 Apr 2025 16:11:19 +0800 Subject: [PATCH 52/61] modify flush time logic --- core/blockchain.go | 2 ++ core/blockchain_l2.go | 25 ++++++++++++++++++------- eth/ethconfig/config.go | 2 +- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 1e684b479..cf0d99201 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -203,6 +203,7 @@ type BlockChain struct { triegc *prque.Prque // Priority queue mapping block numbers to tries to gc gcproc time.Duration // Accumulates canonical block processing for trie dumping flushInterval atomic.Int64 // Time interval (processing time) after which to flush a state + lastFlushTime atomic.Int64 // Time of the last trie flush // txLookupLimit is the maximum number of blocks from head whose tx indices // are reserved: @@ -295,6 +296,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par bc.prefetcher = newStatePrefetcher(chainConfig, bc, engine) bc.processor = NewStateProcessor(chainConfig, bc, engine) bc.flushInterval.Store(int64(cacheConfig.TrieTimeLimit)) + bc.lastFlushTime.Store(0) var err error bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.insertStopped) diff --git a/core/blockchain_l2.go b/core/blockchain_l2.go index d84b71442..feb7b26b7 100644 --- a/core/blockchain_l2.go +++ b/core/blockchain_l2.go @@ -123,19 +123,30 @@ func (bc *BlockChain) writeBlockStateWithoutHead(block *types.Block, receipts [] // If we're running an archive node, always flush if bc.cacheConfig.TrieDirtyDisabled { if triedb.Scheme() == rawdb.PathScheme { - // If we exceeded time allowance, flush an entire trie to disk - flush := bc.gcproc > flushInterval + flush := false + if bc.lastFlushTime.Load() == 0 { + bc.lastFlushTime.Store(int64(block.Time())) + } else { + // Blocktime distance is not a good indicator of the time + // elapsed since the last commit, so we need to check + // the block time to see if we need to flush. + if int64(block.Time()) > bc.lastFlushTime.Load() { + distance := time.Duration(int64(block.Time())-bc.lastFlushTime.Load()) * time.Second - commitDone := func() { - if flush { - log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", flushInterval) - bc.gcproc = 0 + // If the distance is greater than the flush interval, we need to flush + if distance > flushInterval { + flush = true + log.Info("State in memory for too long, committing", "distance", distance, "allowance", flushInterval, "last flush time", bc.lastFlushTime.Load(), "blocktime", block.Time()) + + // Reset the last flush time + bc.lastFlushTime.Store(0) + } } } // If node is running in path mode, skip explicit gc operation // which is unnecessary in this mode. - return triedb.CommitState(root, origin, current, false, flush, commitDone) + return triedb.CommitState(root, origin, current, false, flush, nil) } return triedb.Commit(root, false, nil) } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 0ec53748d..d99652d1d 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -80,7 +80,7 @@ var Defaults = Config{ TrieCleanCacheJournal: "triecache", TrieCleanCacheRejournal: 60 * time.Minute, TrieDirtyCache: 256, - TrieTimeout: 3 * time.Minute, + TrieTimeout: 60 * time.Minute, SnapshotCache: 102, FilterLogCacheSize: 32, Miner: miner.DefaultConfig, From 034e32a8551266851f3922810ec3e72fb8cf5709 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 22 Apr 2025 21:15:37 +0800 Subject: [PATCH 53/61] default use async buffer --- eth/ethconfig/config.go | 1 - 1 file changed, 1 deletion(-) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index d99652d1d..b5ecc3c29 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -91,7 +91,6 @@ var Defaults = Config{ RPCTxFeeCap: 1, // 1 ether MaxBlockRange: -1, // Default unconfigured value: no block range limit for backward compatibility JournalFileName: "trie.journal", - PathSyncFlush: true, } func init() { From 66e494ea126b05db8387ec5b5ee2704ed7d20b78 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 22 Apr 2025 22:52:57 +0800 Subject: [PATCH 54/61] flush with timer --- triedb/pathdb/asyncnodebuffer.go | 4 ++-- triedb/pathdb/database.go | 20 ++++++++++++++------ triedb/pathdb/difflayer.go | 10 +++++----- triedb/pathdb/disklayer.go | 6 +++--- triedb/pathdb/layertree.go | 6 +++--- triedb/pathdb/nodebuffer.go | 6 +++--- 6 files changed, 30 insertions(+), 22 deletions(-) diff --git a/triedb/pathdb/asyncnodebuffer.go b/triedb/pathdb/asyncnodebuffer.go index ec0f1220b..af16e8e34 100644 --- a/triedb/pathdb/asyncnodebuffer.go +++ b/triedb/pathdb/asyncnodebuffer.go @@ -95,7 +95,7 @@ func (a *asyncnodebuffer) empty() bool { // flush persists the in-memory dirty trie node into the disk if the configured // memory threshold is reached. Note, all data must be written atomically. -func (a *asyncnodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64, force bool) error { +func (a *asyncnodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64, force, timeFlush bool) error { a.mux.Lock() defer a.mux.Unlock() @@ -115,7 +115,7 @@ func (a *asyncnodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, } } - if a.current.size < a.current.limit { + if a.current.size < a.current.limit && !timeFlush { return nil } diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 0a627c271..1b64bdda5 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -23,6 +23,7 @@ import ( "sort" "strconv" "sync" + "sync/atomic" "github.com/morph-l2/go-ethereum/common" "github.com/morph-l2/go-ethereum/core/rawdb" @@ -44,7 +45,7 @@ const ( // Too large nodebuffer will cause the system to pause for a long // time when write happens. Also, the largest batch that pebble can // support is 4GB, node will panic if batch size exceeds this limit. - MaxDirtyBufferSize = 256 * 1024 * 1024 + MaxDirtyBufferSize = 64 * 1024 * 1024 // DefaultDirtyBufferSize is the default memory allowance of node buffer // that aggregates the writes from above until it's flushed into the @@ -154,6 +155,7 @@ type Database struct { tree *layerTree // The group for all known layers lock sync.RWMutex // Lock to prevent mutations from happening at the same time dirties dbtypes.KvMap + timeFlush atomic.Bool // Flag if the node buffer is flushed to disk } // New attempts to load an already existing layer from a persistent key-value @@ -171,6 +173,8 @@ func New(diskdb ethdb.KeyValueStore, config *Config) *Database { diskdb: diskdb, dirties: make(dbtypes.KvMap), } + db.timeFlush.Store(false) + // Construct the layer tree by resolving the in-disk singleton state // and in-memory layer journal. db.tree = newLayerTree(db.loadLayers()) @@ -222,6 +226,9 @@ func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockN // only 1 entity, state have no changes // some block maybe has no txns, so state do not change if root == parentRoot && len(db.dirties) == 1 { + if flush { + db.timeFlush.Store(flush) + } return nil } @@ -231,17 +238,18 @@ func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockN } db.dirties = make(dbtypes.KvMap) - layers := maxDiffLayers - if flush { - layers = 0 + if db.timeFlush.Load() { + if !flush { + flush = true + } + db.timeFlush.Store(false) } - // Keep 128 diff layers in the memory, persistent layer is 129th. // - head layer is paired with HEAD state // - head-1 layer is paired with HEAD-1 state // - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state // - head-128 layer(disk layer) is paired with HEAD-128 state - err := db.tree.cap(root, layers) + err := db.tree.cap(root, maxDiffLayers, flush) if callback != nil { callback() } diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go index 36878b4f6..8b8f23265 100644 --- a/triedb/pathdb/difflayer.go +++ b/triedb/pathdb/difflayer.go @@ -147,7 +147,7 @@ func (dl *diffLayer) update(root common.Hash, id uint64, block uint64, nodes dbt } // persist flushes the diff layer and all its parent layers to disk layer. -func (dl *diffLayer) persist(force bool) (layer, error) { +func (dl *diffLayer) persist(force bool, timeFlush bool) (layer, error) { if parent, ok := dl.parentLayer().(*diffLayer); ok { // Hold the lock to prevent any read operation until the new // parent is linked correctly. @@ -156,7 +156,7 @@ func (dl *diffLayer) persist(force bool) (layer, error) { // The merging of diff layers starts at the bottom-most layer, // therefore we recurse down here, flattening on the way up // (diffToDisk). - result, err := parent.persist(force) + result, err := parent.persist(force, timeFlush) if err != nil { dl.lock.Unlock() return nil, err @@ -164,17 +164,17 @@ func (dl *diffLayer) persist(force bool) (layer, error) { dl.parent = result dl.lock.Unlock() } - return diffToDisk(dl, force) + return diffToDisk(dl, force, timeFlush) } // diffToDisk merges a bottom-most diff into the persistent disk layer underneath // it. The method will panic if called onto a non-bottom-most diff layer. -func diffToDisk(layer *diffLayer, force bool) (layer, error) { +func diffToDisk(layer *diffLayer, force, timeFlush bool) (layer, error) { disk, ok := layer.parentLayer().(*diskLayer) if !ok { panic(fmt.Sprintf("unknown layer type: %T", layer.parentLayer())) } - return disk.commit(layer, force) + return disk.commit(layer, force, timeFlush) } func (dl *diffLayer) reset() { diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go index 2c600dacb..ee91c43de 100644 --- a/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -42,7 +42,7 @@ type trienodebuffer interface { // flush persists the in-memory dirty trie node into the disk if the configured // memory threshold is reached. Note, all data must be written atomically. - flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64, force bool) error + flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64, force, timeFlush bool) error // setSize sets the buffer size to the provided number, and invokes a flush // operation if the current memory usage exceeds the new limit. @@ -193,7 +193,7 @@ func (dl *diskLayer) update(root common.Hash, id uint64, block uint64, nodes dbt // commit merges the given bottom-most diff layer into the node buffer // and returns a newly constructed disk layer. Note the current disk // layer must be tagged as stale first to prevent re-access. -func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) { +func (dl *diskLayer) commit(bottom *diffLayer, force, timeFlush bool) (*diskLayer, error) { dl.lock.Lock() defer dl.lock.Unlock() @@ -218,7 +218,7 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) { ndl := newDiskLayer(bottom.root, bottom.stateID(), dl.db, dl.cleans, dl.buffer.commit(bottom.nodes)) - if err := ndl.buffer.flush(ndl.db.diskdb, ndl.cleans, ndl.id, force); err != nil { + if err := ndl.buffer.flush(ndl.db.diskdb, ndl.cleans, ndl.id, force, timeFlush); err != nil { return nil, err } diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go index ef0ce9f44..845bd9b8c 100644 --- a/triedb/pathdb/layertree.go +++ b/triedb/pathdb/layertree.go @@ -113,7 +113,7 @@ func (tree *layerTree) add(root common.Hash, parentRoot common.Hash, block uint6 // cap traverses downwards the diff tree until the number of allowed diff layers // are crossed. All diffs beyond the permitted number are flattened downwards. -func (tree *layerTree) cap(root common.Hash, layers int) error { +func (tree *layerTree) cap(root common.Hash, layers int, timeFlush bool) error { // Retrieve the head layer to cap from l := tree.get(root) if l == nil { @@ -128,7 +128,7 @@ func (tree *layerTree) cap(root common.Hash, layers int) error { // If full commit was requested, flatten the diffs and merge onto disk if layers == 0 { - base, err := diff.persist(true) + base, err := diff.persist(true, false) if err != nil { return err } @@ -165,7 +165,7 @@ func (tree *layerTree) cap(root common.Hash, layers int) error { // parent is linked correctly. diff.lock.Lock() - base, err := parent.persist(false) + base, err := parent.persist(false, timeFlush) if err != nil { diff.lock.Unlock() return err diff --git a/triedb/pathdb/nodebuffer.go b/triedb/pathdb/nodebuffer.go index abec62bb2..3eb0b2180 100644 --- a/triedb/pathdb/nodebuffer.go +++ b/triedb/pathdb/nodebuffer.go @@ -132,13 +132,13 @@ func (b *nodebuffer) empty() bool { // operation if the current memory usage exceeds the new limit. func (b *nodebuffer) setSize(size int, db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64) error { b.limit = uint64(size) - return b.flush(db, clean, id, false) + return b.flush(db, clean, id, false, false) } // flush persists the in-memory dirty trie node into the disk if the configured // memory threshold is reached. Note, all data must be written atomically. -func (b *nodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64, force bool) error { - if b.size <= b.limit && !force { +func (b *nodebuffer) flush(db ethdb.KeyValueStore, clean *fastcache.Cache, id uint64, force, timeFlush bool) error { + if b.size <= b.limit && !force && !timeFlush { return nil } // Ensure the target state id is aligned with the internal counter. From 948d2a60b07c57e988e60ca13426cea7485a6e98 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 22 Apr 2025 23:12:07 +0800 Subject: [PATCH 55/61] fix commit done --- triedb/pathdb/database.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 1b64bdda5..17b8a9c10 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -229,6 +229,9 @@ func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockN if flush { db.timeFlush.Store(flush) } + if callback != nil { + callback() + } return nil } From ebf6d61bba6c7eb3f421768ef12369edf06e28fb Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Tue, 22 Apr 2025 23:19:51 +0800 Subject: [PATCH 56/61] fix commit done --- triedb/pathdb/database.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 17b8a9c10..bc77be0fe 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -218,6 +218,12 @@ func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockN db.lock.Lock() defer db.lock.Unlock() + defer func() { + if callback != nil { + callback() + } + }() + // Short circuit if the mutation is not allowed. if err := db.modifyAllowed(); err != nil { return err @@ -229,9 +235,6 @@ func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockN if flush { db.timeFlush.Store(flush) } - if callback != nil { - callback() - } return nil } @@ -252,11 +255,7 @@ func (db *Database) CommitState(root common.Hash, parentRoot common.Hash, blockN // - head-1 layer is paired with HEAD-1 state // - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state // - head-128 layer(disk layer) is paired with HEAD-128 state - err := db.tree.cap(root, maxDiffLayers, flush) - if callback != nil { - callback() - } - return err + return db.tree.cap(root, maxDiffLayers, flush) } // Close closes the trie database and the held freezer. From 3e971522cf67a104306581c631db723db8dfaf0b Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Wed, 23 Apr 2025 00:16:26 +0800 Subject: [PATCH 57/61] review pr --- triedb/pathdb/database.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index bc77be0fe..890e7fcb8 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -193,6 +193,10 @@ func (db *Database) Reader(root common.Hash) (layer, error) { func (db *Database) CommitGenesis(root common.Hash) error { log.Info("pathdb write genesis state to disk", "root", root.Hex()) + + db.lock.Lock() + defer db.lock.Unlock() + batch := db.diskdb.NewBatch() for _, v := range db.dirties { batch.Put(v.K, v.V) From 5dbe4057d8118d788130585f6a026201944a2921 Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Wed, 23 Apr 2025 13:43:48 +0800 Subject: [PATCH 58/61] set max dirtyBufferSize --- triedb/pathdb/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 890e7fcb8..f3db38afa 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -45,7 +45,7 @@ const ( // Too large nodebuffer will cause the system to pause for a long // time when write happens. Also, the largest batch that pebble can // support is 4GB, node will panic if batch size exceeds this limit. - MaxDirtyBufferSize = 64 * 1024 * 1024 + MaxDirtyBufferSize = 128 * 1024 * 1024 // DefaultDirtyBufferSize is the default memory allowance of node buffer // that aggregates the writes from above until it's flushed into the From e6e8be91ea70b55b1e3dc07b724b545e2e1d854b Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Thu, 24 Apr 2025 12:42:49 +0800 Subject: [PATCH 59/61] review pr --- triedb/pathdb/database.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index f3db38afa..1961bd101 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -184,6 +184,9 @@ func New(diskdb ethdb.KeyValueStore, config *Config) *Database { // Reader retrieves a layer belonging to the given state root. func (db *Database) Reader(root common.Hash) (layer, error) { + db.lock.Lock() + defer db.lock.Unlock() + l := db.tree.get(root) if l == nil { return nil, fmt.Errorf("state %#x is not available", root) From d2ec13b4e7758f5373f575c6d3fc87983218c5df Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Thu, 24 Apr 2025 12:45:25 +0800 Subject: [PATCH 60/61] merge from main --- common/hexutil/json.go | 45 +++++ core/types/gen_authorization.go | 75 ++++++++ core/types/transaction.go | 35 ++++ core/types/transaction_marshalling.go | 128 ++++++++++++-- core/types/tx_setcode.go | 242 ++++++++++++++++++++++++++ core/types/tx_setcode_test.go | 70 ++++++++ params/version.go | 2 +- 7 files changed, 580 insertions(+), 17 deletions(-) create mode 100644 core/types/gen_authorization.go create mode 100644 core/types/tx_setcode.go create mode 100644 core/types/tx_setcode_test.go diff --git a/common/hexutil/json.go b/common/hexutil/json.go index 50db20811..e0ac98f52 100644 --- a/common/hexutil/json.go +++ b/common/hexutil/json.go @@ -23,6 +23,8 @@ import ( "math/big" "reflect" "strconv" + + "github.com/holiman/uint256" ) var ( @@ -30,6 +32,7 @@ var ( bigT = reflect.TypeOf((*Big)(nil)) uintT = reflect.TypeOf(Uint(0)) uint64T = reflect.TypeOf(Uint64(0)) + u256T = reflect.TypeOf((*uint256.Int)(nil)) ) // Bytes marshals/unmarshals as a JSON string with 0x prefix. @@ -225,6 +228,48 @@ func (b *Big) UnmarshalGraphQL(input interface{}) error { return err } +// U256 marshals/unmarshals as a JSON string with 0x prefix. +// The zero value marshals as "0x0". +type U256 uint256.Int + +// MarshalText implements encoding.TextMarshaler +func (b U256) MarshalText() ([]byte, error) { + u256 := (*uint256.Int)(&b) + return []byte(u256.Hex()), nil +} + +// UnmarshalJSON implements json.Unmarshaler. +func (b *U256) UnmarshalJSON(input []byte) error { + // The uint256.Int.UnmarshalJSON method accepts "dec", "0xhex"; we must be + // more strict, hence we check string and invoke SetFromHex directly. + if !isString(input) { + return errNonString(u256T) + } + // The hex decoder needs to accept empty string ("") as '0', which uint256.Int + // would reject. + if len(input) == 2 { + (*uint256.Int)(b).Clear() + return nil + } + err := (*uint256.Int)(b).SetFromHex(string(input[1 : len(input)-1])) + if err != nil { + return &json.UnmarshalTypeError{Value: err.Error(), Type: u256T} + } + return nil +} + +// UnmarshalText implements encoding.TextUnmarshaler +func (b *U256) UnmarshalText(input []byte) error { + // The uint256.Int.UnmarshalText method accepts "dec", "0xhex"; we must be + // more strict, hence we check string and invoke SetFromHex directly. + return (*uint256.Int)(b).SetFromHex(string(input)) +} + +// String returns the hex encoding of b. +func (b *U256) String() string { + return (*uint256.Int)(b).Hex() +} + // Uint64 marshals/unmarshals as a JSON string with 0x prefix. // The zero value marshals as "0x0". type Uint64 uint64 diff --git a/core/types/gen_authorization.go b/core/types/gen_authorization.go new file mode 100644 index 000000000..f34c07fe5 --- /dev/null +++ b/core/types/gen_authorization.go @@ -0,0 +1,75 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package types + +import ( + "encoding/json" + "errors" + + "github.com/holiman/uint256" + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/common/hexutil" +) + +var _ = (*authorizationMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (s SetCodeAuthorization) MarshalJSON() ([]byte, error) { + type SetCodeAuthorization struct { + ChainID hexutil.U256 `json:"chainId" gencodec:"required"` + Address common.Address `json:"address" gencodec:"required"` + Nonce hexutil.Uint64 `json:"nonce" gencodec:"required"` + V hexutil.Uint64 `json:"yParity" gencodec:"required"` + R hexutil.U256 `json:"r" gencodec:"required"` + S hexutil.U256 `json:"s" gencodec:"required"` + } + var enc SetCodeAuthorization + enc.ChainID = hexutil.U256(s.ChainID) + enc.Address = s.Address + enc.Nonce = hexutil.Uint64(s.Nonce) + enc.V = hexutil.Uint64(s.V) + enc.R = hexutil.U256(s.R) + enc.S = hexutil.U256(s.S) + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (s *SetCodeAuthorization) UnmarshalJSON(input []byte) error { + type SetCodeAuthorization struct { + ChainID *hexutil.U256 `json:"chainId" gencodec:"required"` + Address *common.Address `json:"address" gencodec:"required"` + Nonce *hexutil.Uint64 `json:"nonce" gencodec:"required"` + V *hexutil.Uint64 `json:"yParity" gencodec:"required"` + R *hexutil.U256 `json:"r" gencodec:"required"` + S *hexutil.U256 `json:"s" gencodec:"required"` + } + var dec SetCodeAuthorization + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.ChainID == nil { + return errors.New("missing required field 'chainId' for SetCodeAuthorization") + } + s.ChainID = uint256.Int(*dec.ChainID) + if dec.Address == nil { + return errors.New("missing required field 'address' for SetCodeAuthorization") + } + s.Address = *dec.Address + if dec.Nonce == nil { + return errors.New("missing required field 'nonce' for SetCodeAuthorization") + } + s.Nonce = uint64(*dec.Nonce) + if dec.V == nil { + return errors.New("missing required field 'yParity' for SetCodeAuthorization") + } + s.V = uint8(*dec.V) + if dec.R == nil { + return errors.New("missing required field 'r' for SetCodeAuthorization") + } + s.R = uint256.Int(*dec.R) + if dec.S == nil { + return errors.New("missing required field 's' for SetCodeAuthorization") + } + s.S = uint256.Int(*dec.S) + return nil +} diff --git a/core/types/transaction.go b/core/types/transaction.go index 5c9c9ef6e..6dcb1cee8 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -51,6 +51,7 @@ const ( AccessListTxType = 0x01 DynamicFeeTxType = 0x02 BlobTxType = 0x03 + SetCodeTxType = 0x04 L1MessageTxType = 0x7E ) @@ -204,6 +205,8 @@ func (tx *Transaction) decodeTyped(b []byte) (TxData, error) { inner = new(BlobTx) case L1MessageTxType: inner = new(L1MessageTx) + case SetCodeTxType: + inner = new(SetCodeTx) default: return nil, ErrTxTypeNotSupported } @@ -466,6 +469,38 @@ func (tx *Transaction) WithoutBlobTxSidecar() *Transaction { return cpy } +// SetCodeAuthorizations returns the authorizations list of the transaction. +func (tx *Transaction) SetCodeAuthorizations() []SetCodeAuthorization { + setcodetx, ok := tx.inner.(*SetCodeTx) + if !ok { + return nil + } + return setcodetx.AuthList +} + +// SetCodeAuthorities returns a list of unique authorities from the +// authorization list. +func (tx *Transaction) SetCodeAuthorities() []common.Address { + setcodetx, ok := tx.inner.(*SetCodeTx) + if !ok { + return nil + } + var ( + marks = make(map[common.Address]bool) + auths = make([]common.Address, 0, len(setcodetx.AuthList)) + ) + for _, auth := range setcodetx.AuthList { + if addr, err := auth.Authority(); err == nil { + if marks[addr] { + continue + } + marks[addr] = true + auths = append(auths, addr) + } + } + return auths +} + // Hash returns the transaction hash. func (tx *Transaction) Hash() common.Hash { if hash := tx.hash.Load(); hash != nil { diff --git a/core/types/transaction_marshalling.go b/core/types/transaction_marshalling.go index 1d95cfce6..4a0d68692 100644 --- a/core/types/transaction_marshalling.go +++ b/core/types/transaction_marshalling.go @@ -32,22 +32,23 @@ import ( type txJSON struct { Type hexutil.Uint64 `json:"type"` - ChainID *hexutil.Big `json:"chainId,omitempty"` - Nonce *hexutil.Uint64 `json:"nonce"` - To *common.Address `json:"to"` - Gas *hexutil.Uint64 `json:"gas"` - GasPrice *hexutil.Big `json:"gasPrice"` - MaxPriorityFeePerGas *hexutil.Big `json:"maxPriorityFeePerGas"` - MaxFeePerGas *hexutil.Big `json:"maxFeePerGas"` - MaxFeePerBlobGas *hexutil.Big `json:"maxFeePerBlobGas,omitempty"` - Value *hexutil.Big `json:"value"` - Input *hexutil.Bytes `json:"input"` - AccessList *AccessList `json:"accessList,omitempty"` - BlobVersionedHashes []common.Hash `json:"blobVersionedHashes,omitempty"` - V *hexutil.Big `json:"v"` - R *hexutil.Big `json:"r"` - S *hexutil.Big `json:"s"` - YParity *hexutil.Uint64 `json:"yParity,omitempty"` + ChainID *hexutil.Big `json:"chainId,omitempty"` + Nonce *hexutil.Uint64 `json:"nonce"` + To *common.Address `json:"to"` + Gas *hexutil.Uint64 `json:"gas"` + GasPrice *hexutil.Big `json:"gasPrice"` + MaxPriorityFeePerGas *hexutil.Big `json:"maxPriorityFeePerGas"` + MaxFeePerGas *hexutil.Big `json:"maxFeePerGas"` + MaxFeePerBlobGas *hexutil.Big `json:"maxFeePerBlobGas,omitempty"` + Value *hexutil.Big `json:"value"` + Input *hexutil.Bytes `json:"input"` + AccessList *AccessList `json:"accessList,omitempty"` + BlobVersionedHashes []common.Hash `json:"blobVersionedHashes,omitempty"` + AuthorizationList []SetCodeAuthorization `json:"authorizationList,omitempty"` + V *hexutil.Big `json:"v"` + R *hexutil.Big `json:"r"` + S *hexutil.Big `json:"s"` + YParity *hexutil.Uint64 `json:"yParity,omitempty"` // Blob transaction sidecar encoding: Blobs []kzg4844.Blob `json:"blobs,omitempty"` @@ -166,6 +167,23 @@ func (tx *Transaction) MarshalJSON() ([]byte, error) { enc.Commitments = itx.Sidecar.Commitments enc.Proofs = itx.Sidecar.Proofs } + + case *SetCodeTx: + enc.ChainID = (*hexutil.Big)(itx.ChainID.ToBig()) + enc.Nonce = (*hexutil.Uint64)(&itx.Nonce) + enc.To = tx.To() + enc.Gas = (*hexutil.Uint64)(&itx.Gas) + enc.MaxFeePerGas = (*hexutil.Big)(itx.GasFeeCap.ToBig()) + enc.MaxPriorityFeePerGas = (*hexutil.Big)(itx.GasTipCap.ToBig()) + enc.Value = (*hexutil.Big)(itx.Value.ToBig()) + enc.Input = (*hexutil.Bytes)(&itx.Data) + enc.AccessList = &itx.AccessList + enc.AuthorizationList = itx.AuthList + enc.V = (*hexutil.Big)(itx.V.ToBig()) + enc.R = (*hexutil.Big)(itx.R.ToBig()) + enc.S = (*hexutil.Big)(itx.S.ToBig()) + yparity := itx.V.Uint64() + enc.YParity = (*hexutil.Uint64)(&yparity) } return json.Marshal(&enc) } @@ -449,6 +467,84 @@ func (tx *Transaction) UnmarshalJSON(input []byte) error { } itx.Sender = *dec.Sender + case SetCodeTxType: + var itx SetCodeTx + inner = &itx + if dec.ChainID == nil { + return errors.New("missing required field 'chainId' in transaction") + } + var overflow bool + itx.ChainID, overflow = uint256.FromBig(dec.ChainID.ToInt()) + if overflow { + return errors.New("'chainId' value overflows uint256") + } + if dec.Nonce == nil { + return errors.New("missing required field 'nonce' in transaction") + } + itx.Nonce = uint64(*dec.Nonce) + if dec.To == nil { + return errors.New("missing required field 'to' in transaction") + } + itx.To = *dec.To + if dec.Gas == nil { + return errors.New("missing required field 'gas' for txdata") + } + itx.Gas = uint64(*dec.Gas) + if dec.MaxPriorityFeePerGas == nil { + return errors.New("missing required field 'maxPriorityFeePerGas' for txdata") + } + itx.GasTipCap = uint256.MustFromBig((*big.Int)(dec.MaxPriorityFeePerGas)) + if dec.MaxFeePerGas == nil { + return errors.New("missing required field 'maxFeePerGas' for txdata") + } + itx.GasFeeCap = uint256.MustFromBig((*big.Int)(dec.MaxFeePerGas)) + if dec.Value == nil { + return errors.New("missing required field 'value' in transaction") + } + itx.Value = uint256.MustFromBig((*big.Int)(dec.Value)) + if dec.Input == nil { + return errors.New("missing required field 'input' in transaction") + } + itx.Data = *dec.Input + if dec.AccessList != nil { + itx.AccessList = *dec.AccessList + } + if dec.AuthorizationList == nil { + return errors.New("missing required field 'authorizationList' in transaction") + } + itx.AuthList = dec.AuthorizationList + + // signature R + if dec.R == nil { + return errors.New("missing required field 'r' in transaction") + } + itx.R, overflow = uint256.FromBig((*big.Int)(dec.R)) + if overflow { + return errors.New("'r' value overflows uint256") + } + // signature S + if dec.S == nil { + return errors.New("missing required field 's' in transaction") + } + itx.S, overflow = uint256.FromBig((*big.Int)(dec.S)) + if overflow { + return errors.New("'s' value overflows uint256") + } + // signature V + vbig, err := dec.yParityValue() + if err != nil { + return err + } + itx.V, overflow = uint256.FromBig(vbig) + if overflow { + return errors.New("'v' value overflows uint256") + } + if itx.V.Sign() != 0 || itx.R.Sign() != 0 || itx.S.Sign() != 0 { + if err := sanityCheckSignature(vbig, itx.R.ToBig(), itx.S.ToBig(), false); err != nil { + return err + } + } + default: return ErrTxTypeNotSupported } diff --git a/core/types/tx_setcode.go b/core/types/tx_setcode.go new file mode 100644 index 000000000..e6e990850 --- /dev/null +++ b/core/types/tx_setcode.go @@ -0,0 +1,242 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "bytes" + "crypto/ecdsa" + "errors" + "math/big" + + "github.com/holiman/uint256" + "github.com/morph-l2/go-ethereum/common" + "github.com/morph-l2/go-ethereum/common/hexutil" + "github.com/morph-l2/go-ethereum/crypto" + "github.com/morph-l2/go-ethereum/rlp" +) + +// DelegationPrefix is used by code to denote the account is delegating to +// another account. +var DelegationPrefix = []byte{0xef, 0x01, 0x00} + +// ParseDelegation tries to parse the address from a delegation slice. +func ParseDelegation(b []byte) (common.Address, bool) { + if len(b) != 23 || !bytes.HasPrefix(b, DelegationPrefix) { + return common.Address{}, false + } + return common.BytesToAddress(b[len(DelegationPrefix):]), true +} + +// AddressToDelegation adds the delegation prefix to the specified address. +func AddressToDelegation(addr common.Address) []byte { + return append(DelegationPrefix, addr.Bytes()...) +} + +// SetCodeTx implements the EIP-7702 transaction type which temporarily installs +// the code at the signer's address. +type SetCodeTx struct { + ChainID *uint256.Int + Nonce uint64 + GasTipCap *uint256.Int // a.k.a. maxPriorityFeePerGas + GasFeeCap *uint256.Int // a.k.a. maxFeePerGas + Gas uint64 + To common.Address + Value *uint256.Int + Data []byte + AccessList AccessList + AuthList []SetCodeAuthorization + + // Signature values + V *uint256.Int + R *uint256.Int + S *uint256.Int +} + +//go:generate go run github.com/fjl/gencodec -type SetCodeAuthorization -field-override authorizationMarshaling -out gen_authorization.go + +// SetCodeAuthorization is an authorization from an account to deploy code at its address. +type SetCodeAuthorization struct { + ChainID uint256.Int `json:"chainId" gencodec:"required"` + Address common.Address `json:"address" gencodec:"required"` + Nonce uint64 `json:"nonce" gencodec:"required"` + V uint8 `json:"yParity" gencodec:"required"` + R uint256.Int `json:"r" gencodec:"required"` + S uint256.Int `json:"s" gencodec:"required"` +} + +// field type overrides for gencodec +type authorizationMarshaling struct { + ChainID hexutil.U256 + Nonce hexutil.Uint64 + V hexutil.Uint64 + R hexutil.U256 + S hexutil.U256 +} + +// SignSetCode creates a signed the SetCode authorization. +func SignSetCode(prv *ecdsa.PrivateKey, auth SetCodeAuthorization) (SetCodeAuthorization, error) { + sighash := auth.sigHash() + sig, err := crypto.Sign(sighash[:], prv) + if err != nil { + return SetCodeAuthorization{}, err + } + r, s, _ := decodeSignature(sig) + return SetCodeAuthorization{ + ChainID: auth.ChainID, + Address: auth.Address, + Nonce: auth.Nonce, + V: sig[64], + R: *uint256.MustFromBig(r), + S: *uint256.MustFromBig(s), + }, nil +} + +func (a *SetCodeAuthorization) sigHash() common.Hash { + return prefixedRlpHash(0x05, []any{ + a.ChainID, + a.Address, + a.Nonce, + }) +} + +// Authority recovers the the authorizing account of an authorization. +func (a *SetCodeAuthorization) Authority() (common.Address, error) { + sighash := a.sigHash() + if !crypto.ValidateSignatureValues(a.V, a.R.ToBig(), a.S.ToBig(), true) { + return common.Address{}, ErrInvalidSig + } + // encode the signature in uncompressed format + var sig [crypto.SignatureLength]byte + a.R.WriteToSlice(sig[:32]) + a.S.WriteToSlice(sig[32:64]) + sig[64] = a.V + // recover the public key from the signature + pub, err := crypto.Ecrecover(sighash[:], sig[:]) + if err != nil { + return common.Address{}, err + } + if len(pub) == 0 || pub[0] != 4 { + return common.Address{}, errors.New("invalid public key") + } + var addr common.Address + copy(addr[:], crypto.Keccak256(pub[1:])[12:]) + return addr, nil +} + +// copy creates a deep copy of the transaction data and initializes all fields. +func (tx *SetCodeTx) copy() TxData { + cpy := &SetCodeTx{ + Nonce: tx.Nonce, + To: tx.To, + Data: common.CopyBytes(tx.Data), + Gas: tx.Gas, + // These are copied below. + AccessList: make(AccessList, len(tx.AccessList)), + AuthList: make([]SetCodeAuthorization, len(tx.AuthList)), + Value: new(uint256.Int), + ChainID: new(uint256.Int), + GasTipCap: new(uint256.Int), + GasFeeCap: new(uint256.Int), + V: new(uint256.Int), + R: new(uint256.Int), + S: new(uint256.Int), + } + copy(cpy.AccessList, tx.AccessList) + copy(cpy.AuthList, tx.AuthList) + if tx.Value != nil { + cpy.Value.Set(tx.Value) + } + if tx.ChainID != nil { + cpy.ChainID.Set(tx.ChainID) + } + if tx.GasTipCap != nil { + cpy.GasTipCap.Set(tx.GasTipCap) + } + if tx.GasFeeCap != nil { + cpy.GasFeeCap.Set(tx.GasFeeCap) + } + if tx.V != nil { + cpy.V.Set(tx.V) + } + if tx.R != nil { + cpy.R.Set(tx.R) + } + if tx.S != nil { + cpy.S.Set(tx.S) + } + return cpy +} + +// accessors for innerTx. +func (tx *SetCodeTx) txType() byte { return SetCodeTxType } +func (tx *SetCodeTx) chainID() *big.Int { return tx.ChainID.ToBig() } +func (tx *SetCodeTx) accessList() AccessList { return tx.AccessList } +func (tx *SetCodeTx) data() []byte { return tx.Data } +func (tx *SetCodeTx) gas() uint64 { return tx.Gas } +func (tx *SetCodeTx) gasFeeCap() *big.Int { return tx.GasFeeCap.ToBig() } +func (tx *SetCodeTx) gasTipCap() *big.Int { return tx.GasTipCap.ToBig() } +func (tx *SetCodeTx) gasPrice() *big.Int { return tx.GasFeeCap.ToBig() } +func (tx *SetCodeTx) value() *big.Int { return tx.Value.ToBig() } +func (tx *SetCodeTx) nonce() uint64 { return tx.Nonce } +func (tx *SetCodeTx) to() *common.Address { tmp := tx.To; return &tmp } + +func (tx *SetCodeTx) effectiveGasPrice(dst *big.Int, baseFee *big.Int) *big.Int { + if baseFee == nil { + return dst.Set(tx.GasFeeCap.ToBig()) + } + tip := dst.Sub(tx.GasFeeCap.ToBig(), baseFee) + if tip.Cmp(tx.GasTipCap.ToBig()) > 0 { + tip.Set(tx.GasTipCap.ToBig()) + } + return tip.Add(tip, baseFee) +} + +func (tx *SetCodeTx) rawSignatureValues() (v, r, s *big.Int) { + return tx.V.ToBig(), tx.R.ToBig(), tx.S.ToBig() +} + +func (tx *SetCodeTx) setSignatureValues(chainID, v, r, s *big.Int) { + tx.ChainID = uint256.MustFromBig(chainID) + tx.V.SetFromBig(v) + tx.R.SetFromBig(r) + tx.S.SetFromBig(s) +} + +func (tx *SetCodeTx) encode(b *bytes.Buffer) error { + return rlp.Encode(b, tx) +} + +func (tx *SetCodeTx) decode(input []byte) error { + return rlp.DecodeBytes(input, tx) +} + +func (tx *SetCodeTx) sigHash(chainID *big.Int) common.Hash { + return prefixedRlpHash( + SetCodeTxType, + []any{ + chainID, + tx.Nonce, + tx.GasTipCap, + tx.GasFeeCap, + tx.Gas, + tx.To, + tx.Value, + tx.Data, + tx.AccessList, + tx.AuthList, + }) +} diff --git a/core/types/tx_setcode_test.go b/core/types/tx_setcode_test.go new file mode 100644 index 000000000..06c1c82d7 --- /dev/null +++ b/core/types/tx_setcode_test.go @@ -0,0 +1,70 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "testing" + + "github.com/morph-l2/go-ethereum/common" +) + +// TestParseDelegation tests a few possible delegation designator values and +// ensures they are parsed correctly. +func TestParseDelegation(t *testing.T) { + addr := common.Address{0x42} + for _, tt := range []struct { + val []byte + want *common.Address + }{ + { // simple correct delegation + val: append(DelegationPrefix, addr.Bytes()...), + want: &addr, + }, + { // wrong address size + val: append(DelegationPrefix, addr.Bytes()[0:19]...), + }, + { // short address + val: append(DelegationPrefix, 0x42), + }, + { // long address + val: append(append(DelegationPrefix, addr.Bytes()...), 0x42), + }, + { // wrong prefix size + val: append(DelegationPrefix[:2], addr.Bytes()...), + }, + { // wrong prefix + val: append([]byte{0xef, 0x01, 0x01}, addr.Bytes()...), + }, + { // wrong prefix + val: append([]byte{0xef, 0x00, 0x00}, addr.Bytes()...), + }, + { // no prefix + val: addr.Bytes(), + }, + { // no address + val: DelegationPrefix, + }, + } { + got, ok := ParseDelegation(tt.val) + if ok && tt.want == nil { + t.Fatalf("expected fail, got %s", got.Hex()) + } + if !ok && tt.want != nil { + t.Fatalf("failed to parse, want %s", tt.want.Hex()) + } + } +} diff --git a/params/version.go b/params/version.go index b161be4fb..a602cb1b1 100644 --- a/params/version.go +++ b/params/version.go @@ -24,7 +24,7 @@ import ( const ( VersionMajor = 2 // Major version component of the current release VersionMinor = 0 // Minor version component of the current release - VersionPatch = 3 // Patch version component of the current release + VersionPatch = 5 // Patch version component of the current release VersionMeta = "mainnet" // Version metadata to append to the version string ) From a8a321cc9e3e4c2651db7d1e79b122f65e91ca4c Mon Sep 17 00:00:00 2001 From: ryanmorphl2 <163962984+ryanmorphl2@users.noreply.github.com> Date: Fri, 25 Apr 2025 20:53:09 +0800 Subject: [PATCH 61/61] fix: init with genesis path --- cmd/geth/chaincmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 2f80ec4a5..f2cc94aae 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -192,7 +192,7 @@ This command dumps out the state for a given block (or latest, if none provided) // the zero'd block (i.e. genesis) or will fail hard if it can't succeed. func initGenesis(ctx *cli.Context) error { // Make sure we have a valid genesis JSON - genesisPath := ctx.GlobalString(utils.DataDirFlag.Name) + genesisPath := ctx.Args().First() if len(genesisPath) == 0 { utils.Fatalf("Must supply path to genesis JSON file") }