pbss implemention

.github/workflows/docker_release.yml

@@ -0,0 +1,41 @@
+name: Push Docker Image
+
+on:
+  push:
+    tags: 
+      - morph-v*
+
+env:
+  IMAGE_NAME: go-ethereum
+
+jobs:
+  # Push image to GitHub Packages.
+  push:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Build the Docker image
+      run: docker build . --file Dockerfile -t "${IMAGE_NAME}"
+
+    - name: Log into registry
+      run: echo "${{ secrets.PACKAGE_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+    - name: Push image
+      run: |
+        IMAGE_ID=ghcr.io/${{ github.repository }}
+
+        # Change all uppercase to lowercase
+        IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+        # Strip git ref prefix from version
+        VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+        # Strip "morph-v" prefix from tag name
+        [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^morph-v//')
+        echo IMAGE_ID=$IMAGE_ID
+        echo VERSION=$VERSION
+        docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
+        docker tag $IMAGE_NAME $IMAGE_ID:latest
+        docker push $IMAGE_ID:$VERSION
+        docker push $IMAGE_ID:latest

MakefileEc2.mk

@@ -21,3 +21,34 @@ build-bk-prod-morph-prod-mainnet-to-morph-nccc-geth:
 	aws s3 cp morph-nccc-geth.tar.gz s3://morph-0582-morph-technical-department-mainnet-data/morph-setup/morph-nccc-geth.tar.gz
 
 
+# build for holesky
+build-bk-prod-morph-prod-testnet-to-morph-geth-holesky:
+	if [ ! -d dist ]; then mkdir -p dist; fi
+	$(GORUN) build/ci.go install ./cmd/geth
+	cp build/bin/geth dist/
+	tar -czvf morph-geth.tar.gz dist
+	aws s3 cp morph-geth.tar.gz s3://morph-0582-morph-technical-department-testnet-data/testnet/holesky/morph-setup/morph-geth.tar.gz
+
+build-bk-prod-morph-prod-testnet-to-morph-nccc-geth-holesky:
+	if [ ! -d dist ]; then mkdir -p dist; fi
+	$(GORUN) build/ci.go install ./cmd/geth
+	@echo "Done building."
+	cp build/bin/geth dist/
+	tar -czvf morph-nccc-geth.tar.gz dist
+	aws s3 cp morph-nccc-geth.tar.gz s3://morph-0582-morph-technical-department-testnet-data/testnet/holesky/morph-setup/morph-nccc-geth.tar.gz
+
+build-bk-test-morph-test-qanet-to-morph-geth-qanet:
+	if [ ! -d dist ]; then mkdir -p dist; fi
+	$(GORUN) build/ci.go install ./cmd/geth
+	@echo "Done building."
+	cp build/bin/geth dist/
+	tar -czvf morph-geth.tar.gz dist
+	aws s3 cp morph-geth.tar.gz s3://morph-7637-morph-technical-department-qanet-data/morph-setup/morph-geth.tar.gz
+
+build-bk-test-morph-test-qanet-to-morph-nccc-geth-qanet:
+	if [ ! -d dist ]; then mkdir -p dist; fi
+	$(GORUN) build/ci.go install ./cmd/geth
+	@echo "Done building."
+	cp build/bin/geth dist/
+	tar -czvf morph-nccc-geth.tar.gz dist
+	aws s3 cp morph-nccc-geth.tar.gz s3://morph-7637-morph-technical-department-qanet-data/morph-setup/morph-nccc-geth.tar.gz

cmd/geth/chaincmd.go

@@ -50,6 +50,7 @@ var (
 		ArgsUsage: "<genesisPath>",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.StateSchemeFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
@@ -190,7 +191,7 @@ This command dumps out the state for a given block (or latest, if none provided)
 // the zero'd block (i.e. genesis) or will fail hard if it can't succeed.
 func initGenesis(ctx *cli.Context) error {
 	// Make sure we have a valid genesis JSON
-	genesisPath := ctx.Args().First()
+	genesisPath := ctx.GlobalString(utils.DataDirFlag.Name)
 	if len(genesisPath) == 0 {
 		utils.Fatalf("Must supply path to genesis JSON file")
 	}

cmd/geth/dbcmd.go

@@ -70,6 +70,7 @@ Remove blockchain and state databases`,
 			dbDumpFreezerIndex,
 			dbImportCmd,
 			dbExportCmd,
+			dbHbss2PbssCmd,
 		},
 	}
 	dbInspectCmd = cli.Command{
@@ -254,6 +255,18 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		},
 		Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.",
 	}
+	dbHbss2PbssCmd = cli.Command{
+		Action:    hbss2pbss,
+		Name:      "hbss-to-pbss",
+		ArgsUsage: "<jobnum (optional)>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.AncientFlag,
+		},
+		Usage:       "Convert Hash-Base to Path-Base trie node.",
+		Description: `This command iterates the entire trie node database and convert the hash-base node to path-base node.`,
+	}
 )
 
 func removeDB(ctx *cli.Context) error {
@@ -706,3 +719,38 @@ func exportChaindata(ctx *cli.Context) error {
 	db := utils.MakeChainDatabase(ctx, stack, true)
 	return utils.ExportChaindata(ctx.Args().Get(1), kind, exporter(db), stop)
 }
+
+func hbss2pbss(ctx *cli.Context) error {
+	stack, config := makeConfigNode(ctx)
+	defer stack.Close()
+
+	chaindb := utils.MakeChainDatabase(ctx, stack, false)
+	h2p, err := trie.NewHbss2Pbss(chaindb, stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal))
+	if err != nil {
+		return err
+	}
+
+	if ctx.NArg() > 0 {
+		log.Error("Too many arguments given")
+		return errors.New("too many arguments")
+	}
+
+	lastStateID := rawdb.ReadPersistentStateID(chaindb)
+	if lastStateID == 0 {
+		h2p.Run()
+	}
+
+	// prune hbss trie node
+	err = rawdb.PruneHashTrieNodeInDataBase(chaindb)
+	if err != nil {
+		log.Error("Prune Hash trie node in database failed", "error", err)
+		return err
+	}
+	err = h2p.Compact()
+	if err != nil {
+		log.Error("Compact trie node failed", "error", err)
+		return err
+	}
+
+	return nil
+}

cmd/geth/main.go

@@ -158,6 +158,8 @@ var (
 		utils.GpoIgnoreGasPriceFlag,
 		configFileFlag,
 		utils.CatalystFlag,
+		utils.StateSchemeFlag,
+		utils.PathDBSyncFlag,
 	}
 
 	rpcFlags = []cli.Flag{

cmd/geth/snapshot.go

@@ -223,6 +223,9 @@ func pruneZKState(ctx *cli.Context) error {
 	defer stack.Close()
 
 	chaindb := utils.MakeChainDatabase(ctx, stack, false)
+	if rawdb.ReadStateScheme(chaindb) != rawdb.HashScheme {
+		log.Crit("Offline pruning is not required for path scheme")
+	}
 	zkPruner, err := pruner.NewZKPruner(chaindb, ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name), stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal))
 	if err != nil {
 		return err

cmd/geth/usage.go

@@ -57,6 +57,8 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.IdentityFlag,
 			utils.LightKDFFlag,
 			utils.WhitelistFlag,
+			utils.StateSchemeFlag,
+			utils.PathDBSyncFlag,
 		},
 	},
 	{

cmd/utils/flags.go

@@ -75,6 +75,7 @@ import (
 	"github.com/morph-l2/go-ethereum/params"
 	"github.com/morph-l2/go-ethereum/rollup/tracing"
 	"github.com/morph-l2/go-ethereum/rpc"
+	"github.com/morph-l2/go-ethereum/trie"
 )
 
 func init() {
@@ -851,6 +852,16 @@ var (
 		Name:  "rpc.getlogs.maxrange",
 		Usage: "Limit max fetched block range for `eth_getLogs` method",
 	}
+
+	StateSchemeFlag = &cli.StringFlag{
+		Name:  "state.scheme",
+		Usage: "Scheme to use for storing zktrie state ('hash' or 'path')",
+	}
+
+	PathDBSyncFlag = cli.BoolFlag{
+		Name:  "pathdb.sync",
+		Usage: "Sync flush nodes cache to disk in path schema",
+	}
 )
 
 // MakeDataDir retrieves the currently requested data directory, terminating
@@ -1718,6 +1729,19 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 			cfg.EthDiscoveryURLs = SplitAndTrim(urls)
 		}
 	}
+	if ctx.GlobalIsSet(PathDBSyncFlag.Name) {
+		cfg.PathSyncFlush = true
+	}
+
+	scheme, err := ParseCLIAndConfigStateScheme(ctx.String(StateSchemeFlag.Name), cfg.StateScheme)
+	if err != nil {
+		Fatalf("%v", err)
+	}
+	cfg.StateScheme = scheme
+	if cfg.StateScheme == rawdb.PathScheme {
+		trie.GenesisStateInPathZkTrie = true
+	}
+
 	// Override any default configs for hard coded networks.
 	switch {
 	case ctx.GlobalBool(MainnetFlag.Name):
@@ -1765,6 +1789,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		// disable prefetch
 		log.Info("Prefetch disabled")
 		cfg.NoPrefetch = true
+
+		// cheked for path zktrie
+		if cfg.StateScheme == rawdb.PathScheme && !cfg.Genesis.Config.Morph.ZktrieEnabled() {
+			log.Crit("Must cooperate with zktrie enable to use --state.scheme=path")
+		}
 	case ctx.GlobalBool(MorphHoleskyFlag.Name):
 		if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
 			cfg.NetworkId = 2810
@@ -1780,6 +1809,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		// disable prefetch
 		log.Info("Prefetch disabled")
 		cfg.NoPrefetch = true
+
+		// cheked for path zktrie
+		if cfg.StateScheme == rawdb.PathScheme && !cfg.Genesis.Config.Morph.ZktrieEnabled() {
+			log.Crit("Must cooperate with zktrie enable to use --state.scheme=path")
+		}
 	case ctx.GlobalBool(DeveloperFlag.Name):
 		if !ctx.GlobalIsSet(NetworkIdFlag.Name) {
 			cfg.NetworkId = 1337
@@ -2147,3 +2181,22 @@ func MigrateFlags(action func(ctx *cli.Context) error) func(*cli.Context) error
 		return action(ctx)
 	}
 }
+
+// ParseCLIAndConfigStateScheme parses state scheme in CLI and config.
+func ParseCLIAndConfigStateScheme(cliScheme, cfgScheme string) (string, error) {
+	if cliScheme == "" {
+		if cfgScheme != "" {
+			log.Info("Use config state scheme", "config", cfgScheme)
+		}
+		return cfgScheme, nil
+	}
+
+	if !rawdb.ValidateStateScheme(cliScheme) {
+		return "", fmt.Errorf("invalid state scheme in CLI: %s", cliScheme)
+	}
+	if cfgScheme == "" || cliScheme == cfgScheme {
+		log.Info("Use CLI state scheme", "CLI", cliScheme)
+		return cliScheme, nil
+	}
+	return "", fmt.Errorf("incompatible state scheme, CLI: %s, config: %s", cliScheme, cfgScheme)
+}

common/types.go

@@ -226,6 +226,11 @@ func IsHexAddress(s string) bool {
 	return len(s) == 2*AddressLength && isHex(s)
 }
 
+// Cmp compares two addresses.
+func (a Address) Cmp(other Address) int {
+	return bytes.Compare(a[:], other[:])
+}
+
 // Bytes gets the string representation of the underlying address.
 func (a Address) Bytes() []byte { return a[:] }
 
@@ -433,3 +438,62 @@ func (ma *MixedcaseAddress) ValidChecksum() bool {
 func (ma *MixedcaseAddress) Original() string {
 	return ma.original
 }
+
+func ReverseBytes(b []byte) []byte {
+	o := make([]byte, len(b))
+	for i := range b {
+		o[len(b)-1-i] = b[i]
+	}
+	return o
+}
+
+func bitReverseForNibble(b byte) byte {
+	switch b {
+	case 0:
+		return 0
+	case 1:
+		return 8
+	case 2:
+		return 4
+	case 3:
+		return 12
+	case 4:
+		return 2
+	case 5:
+		return 10
+	case 6:
+		return 6
+	case 7:
+		return 14
+	case 8:
+		return 1
+	case 9:
+		return 9
+	case 10:
+		return 5
+	case 11:
+		return 13
+	case 12:
+		return 3
+	case 13:
+		return 11
+	case 14:
+		return 7
+	case 15:
+		return 15
+	default:
+		panic("unexpected input")
+	}
+}
+
+func BitReverse(inp []byte) (out []byte) {
+
+	l := len(inp)
+	out = make([]byte, l)
+
+	for i, b := range inp {
+		out[l-i-1] = bitReverseForNibble(b&15)<<4 + bitReverseForNibble(b>>4)
+	}
+
+	return
+}