From a251b4713593f3b352d3683edbeeb4ff8e98a54a Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Tue, 6 May 2025 09:36:24 +0200 Subject: [PATCH] feat: add initial code --- .github/workflows/go.yml | 47 +++++ .gitignore | 7 +- .golangci.yml | 17 ++ Makefile | 45 +++++ README.md | 54 +++++- cmd/cmd.go | 76 ++++++++ cmd/export.go | 146 ++++++++++++++++ go.mod | 55 ++++++ go.sum | 214 +++++++++++++++++++++++ main.go | 21 +++ pkg/ethclientwrapper/ethclientwrapper.go | 81 +++++++++ pkg/eventfetcher/eventfetcher.go | 178 +++++++++++++++++++ pkg/filestore/filestore.go | 36 ++++ pkg/gzipstore/gzipstore.go | 37 ++++ pkg/logcache/logcache.go | 28 +++ pkg/logcache/logcache_test.go | 18 ++ 16 files changed, 1056 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/go.yml create mode 100644 .golangci.yml create mode 100644 Makefile create mode 100644 cmd/cmd.go create mode 100644 cmd/export.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 pkg/ethclientwrapper/ethclientwrapper.go create mode 100644 pkg/eventfetcher/eventfetcher.go create mode 100644 pkg/filestore/filestore.go create mode 100644 pkg/gzipstore/gzipstore.go create mode 100644 pkg/logcache/logcache.go create mode 100644 pkg/logcache/logcache_test.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..da54a5c --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,47 @@ +name: Go + +on: [pull_request] + +jobs: + build: + name: Build + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Setup Go + if: matrix.os == 'ubuntu-latest' + uses: actions/setup-go@v5 + with: + cache: false + go-version-file: go.mod + - name: Setup Go + if: matrix.os == 'macos-latest' || matrix.os == 'windows-latest' + uses: actions/setup-go@v5 + with: + cache: true + go-version-file: go.mod + - name: Set git to use LF + # make sure that line endings are not converted on windows + # as gofmt linter will report that they need to be changed + run: git config --global core.autocrlf false + - name: Lint + if: matrix.os == 'ubuntu-latest' + uses: golangci/golangci-lint-action@v6 + with: + version: v1.64.5 + args: --timeout 10m + skip-cache: false + - name: Vet + if: matrix.os == 'ubuntu-latest' + run: make vet + - name: Build + run: make build + - name: Test + run: make test + diff --git a/.gitignore b/.gitignore index 6f72f89..3726681 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,3 @@ -# If you prefer the allow list template instead of the deny list, see community template: -# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore -# # Binaries for programs and plugins *.exe *.exe~ @@ -23,3 +20,7 @@ go.work.sum # env file .env + +# Folders +dist/ +.vscode/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..f20d084 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,17 @@ +run: + timeout: 10m +linters: + enable: + - copyloopvar + - errcheck + - errname + - errorlint + - goconst + - gofmt + - gofumpt + - govet + - misspell + - nilerr + - staticcheck + - unconvert + - unused diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d5519c5 --- /dev/null +++ b/Makefile @@ -0,0 +1,45 @@ +GO ?= go +GOLANGCI_LINT ?= golangci-lint +GOLANGCI_LINT_VERSION ?= v1.64.5 + +.PHONY: binary + +binary: dist FORCE + $(GO) version +ifeq ($(OS),Windows_NT) + $(GO) build -o dist/batch-export.exe . +else + $(GO) build -o dist/batch-export . +endif + +dist: + mkdir $@ + +.PHONY: lint +lint: linter + $(GOLANGCI_LINT) run + +.PHONY: linter +linter: + which $(GOLANGCI_LINT) || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$($(GO) env GOPATH)/bin $(GOLANGCI_LINT_VERSION) + +.PHONY: vet +vet: + $(GO) vet ./... + +.PHONY: build +build: export CGO_ENABLED=0 +build: + $(GO) build -trimpath -ldflags "$(LDFLAGS)" ./... + +.PHONY: clean +clean: + $(GO) clean + rm -rf dist/ + + +.PHONY: test +test: + $(GO) test -v ./pkg/... + +FORCE: diff --git a/README.md b/README.md index d43d175..cdde46a 100644 --- a/README.md +++ b/README.md @@ -1 +1,53 @@ -# batch-export \ No newline at end of file +# batch-export + +batch-export is a tool to retrieve Ethereum event logs for specific contracts, particularly designed for Swarm's Postage Stamp contract on the Gnosis Chain. It fetches logs within a specified block range using the `export` command and saves them to a file. + +## Features + +- Retrieve event logs for a specified contract address and block range. +- Handles large block ranges by querying in smaller chunks. +- Supports rate limiting for RPC requests. +- Saves retrieved logs to a specified output file (default: `export.ndjson`) in NDJSON format. +- Graceful shutdown on interrupt signals (Ctrl+C). + +## Requirements + +- Go 1.24 or later + +## Installation + +```sh +git clone https://github.com/ethersphere/batch-export.git +cd batch-export +make binary +# The binary will be located in the dist/ folder +``` + +## Usage + +The primary command is export. + +```sh +./dist/batch-export export --help +``` + +```sh +./dist/batch-export export \ + --start 31306381 \ + --endpoint \ + --output my_logs.ndjson +``` + +## Flags + +```sh + -b, --block-range-limit uint32 Max blocks per log query (default 5) + -c, --compress Compress to GZIP + --end uint End block (optional, uses latest block if 0) (default 39810670) + -e, --endpoint string Ethereum RPC endpoint URL + -h, --help help for export + -m, --max-request int Max RPC requests/sec (default 15) + -o, --output string Output file path (NDJSON) (default "export.ndjson") + --start uint Start block (optional, uses contract start block if 0) (default 31306381) + -v, --verbosity string Log verbosity (silent, error, warn, info, debug) (default "info") +``` diff --git a/cmd/cmd.go b/cmd/cmd.go new file mode 100644 index 0000000..4ed1f11 --- /dev/null +++ b/cmd/cmd.go @@ -0,0 +1,76 @@ +package cmd + +import ( + "context" + "fmt" + "strings" + + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/spf13/cobra" +) + +type command struct { + root *cobra.Command + verbosity string + log log.Logger +} + +func (c *command) Execute(ctx context.Context) (err error) { + return c.root.ExecuteContext(ctx) +} + +func Execute(ctx context.Context) (err error) { + c, err := newCommand() + if err != nil { + return err + } + return c.Execute(ctx) +} + +func newCommand() (c *command, err error) { + c = &command{ + root: &cobra.Command{ + Use: "batch-export", + Short: "A tool to track logs of a swarm node", + Long: "A tool to track logs of a swarm node", + SilenceErrors: true, + SilenceUsage: true, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + var err error + c.log, err = newLogger(c.verbosity) + if err != nil { + return fmt.Errorf("failed to create logger: %w", err) + } + return nil + }, + }, + } + + c.root.PersistentFlags().StringVarP(&c.verbosity, "verbosity", "v", "info", "Log verbosity (silent, error, warn, info, debug)") + + if err := c.initExportCmd(); err != nil { + return nil, err + } + + return c, nil +} + +func newLogger(verbosity string) (logger log.Logger, err error) { + var level log.Level + switch strings.ToLower(verbosity) { + case "0", "silent": + level = log.VerbosityNone + case "1", "error": + level = log.VerbosityError + case "2", "warn": + level = log.VerbosityWarning + case "3", "info": + level = log.VerbosityInfo + case "4", "debug": + level = log.VerbosityDebug + default: + return nil, fmt.Errorf("invalid verbosity level: %s", verbosity) + } + + return log.NewLogger("batch-export", log.WithVerbosity(level), log.WithTimestamp()).Register(), nil +} diff --git a/cmd/export.go b/cmd/export.go new file mode 100644 index 0000000..2048a42 --- /dev/null +++ b/cmd/export.go @@ -0,0 +1,146 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + ethclient "github.com/ethersphere/batch-export/pkg/ethclientwrapper" + "github.com/ethersphere/batch-export/pkg/eventfetcher" + "github.com/ethersphere/batch-export/pkg/filestore" + "github.com/ethersphere/batch-export/pkg/gzipstore" + "github.com/ethersphere/bee/v2/pkg/config" + "github.com/ethersphere/bee/v2/pkg/util/abiutil" + "github.com/spf13/cobra" +) + +func (c *command) initExportCmd() (err error) { + var ( + startBlock uint64 + endBlock uint64 + rpcEndpoint string + maxRequest int + blockRangeLimit uint32 + outputFile string + compress bool + ) + + cmd := &cobra.Command{ + Use: "export", + Short: "Export Swarm Postage Stamp contract event logs within a block range.", + Long: `Exports event logs for the Swarm Postage Stamp contract from a specified Ethereum RPC endpoint +within a given block range (--start to --end). It handles large ranges by querying in chunks (--block-range-limit) +and respects RPC rate limits (--max-request). + +The retrieved logs are saved to the specified output file (default: 'export.ndjson') in NDJSON format. +The process can be interrupted at any time (Ctrl+C), and it will attempt to save already retrieved logs before exiting.`, + RunE: func(cmd *cobra.Command, args []string) (err error) { + ctx := cmd.Context() + + ec, err := ethclient.NewClient(ctx, rpcEndpoint, ethclient.WithRateLimit(maxRequest), ethclient.WithLogger(c.log)) + if err != nil { + return fmt.Errorf("failed to connect to the Ethereum client: %w", err) + } + defer ec.Close() + + chainID, err := ec.ChainID(ctx) + if err != nil { + return fmt.Errorf("failed to get chainID: %w", err) + } + + chainCfg, found := config.GetByChainID(chainID.Int64()) + if !found { + return fmt.Errorf("chain config not found for chain ID %d", chainID.Int64()) + } + + postageStampContractABI := abiutil.MustParseABI(chainCfg.PostageStampABI) + + client := eventfetcher.NewClient(ec, postageStampContractABI, blockRangeLimit, c.log) + + if startBlock == 0 { + startBlock = chainCfg.PostageStampStartBlock + } + + c.log.Info("Retrieving logs", "startBlock", startBlock, "endBlock", endBlock) + + logChan, errorChan := client.GetLogs(ctx, &eventfetcher.Request{ + Address: chainCfg.PostageStampAddress, + StartBlock: startBlock, + EndBlock: endBlock, + }) + + var wg sync.WaitGroup + wg.Add(1) + + ticker := time.NewTicker(15 * time.Second) + defer ticker.Stop() + + go func() { + defer wg.Done() + if err := filestore.SaveLogsAsync(ctx, logChan, outputFile); err != nil { + if errors.Is(err, context.Canceled) { + c.log.Error(err, "context canceled while saving logs") + return + } + c.log.Error(err, "error saving logs") + return + } + c.log.Info("all logs have been saved", "outputFile", outputFile) + }() + + compressFunc := func() error { + if compress { + if err := gzipstore.CompressFile(outputFile, outputFile+".gzip"); err != nil { + return fmt.Errorf("error compressing file: %w", err) + } + c.log.Info("File compressed", "outputFile", outputFile+".gzip") + } + return nil + } + + for { + select { + case err, ok := <-errorChan: + if !ok { + errorChan = nil + } else { + return fmt.Errorf("error retrieving logs: %w", err) + } + case <-ticker.C: + c.log.Info("still retrieving logs...") + case <-ctx.Done(): + c.log.Info("context canceled, waiting for logs to be saved...") + if err := compressFunc(); err != nil { + return errors.Join(fmt.Errorf("error compressing file: %w", err), ctx.Err()) + } + return ctx.Err() + } + + if errorChan == nil { + break + } + } + + wg.Wait() + if err := compressFunc(); err != nil { + return fmt.Errorf("error compressing file: %w", err) + } + + return nil + }, + } + + cmd.Flags().Uint64VarP(&startBlock, "start", "", 31306381, "Start block (optional, uses contract start block if 0)") + cmd.Flags().Uint64VarP(&endBlock, "end", "", 39810670, "End block (optional, uses latest block if 0)") + cmd.Flags().StringVarP(&rpcEndpoint, "endpoint", "e", "https://wandering-evocative-gas.xdai.quiknode.pro/0f2525676e3ba76259ab3b72243f7f60334b0000/", "Ethereum RPC endpoint URL") + cmd.Flags().IntVarP(&maxRequest, "max-request", "m", 15, "Max RPC requests/sec") + cmd.Flags().Uint32VarP(&blockRangeLimit, "block-range-limit", "b", 5, "Max blocks per log query") + cmd.Flags().StringVarP(&outputFile, "output", "o", "export.ndjson", "Output file path (NDJSON)") + cmd.Flags().BoolVarP(&compress, "compress", "c", false, "Compress to GZIP") + + c.root.AddCommand(cmd) + + return nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..00c838a --- /dev/null +++ b/go.mod @@ -0,0 +1,55 @@ +module github.com/ethersphere/batch-export + +go 1.24.0 + +require ( + github.com/ethereum/go-ethereum v1.14.5 + github.com/ethersphere/bee/v2 v2.5.0 + github.com/spf13/cobra v1.9.1 + golang.org/x/time v0.11.0 +) + +require ( + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/StackExchange/wmi v1.2.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.10.0 // indirect + github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.12.1 // indirect + github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect + github.com/deckarep/golang-set/v2 v2.6.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect + github.com/ethereum/c-kzg-4844 v1.0.0 // indirect + github.com/ethersphere/go-storage-incentives-abi v0.9.2 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.22.0 + github.com/gorilla/websocket v1.5.1 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/holiman/uint256 v1.2.4 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect + github.com/prometheus/client_golang v1.18.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect + github.com/prometheus/common v0.47.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/shirou/gopsutil v3.21.5+incompatible // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/supranational/blst v0.3.11 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect + rsc.io/tmplfunc v0.0.3 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..68ca7a3 --- /dev/null +++ b/go.sum @@ -0,0 +1,214 @@ +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= +github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88= +github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= +github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8= +github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4= +github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= +github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I= +github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= +github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= +github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= +github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= +github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= +github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= +github.com/ethereum/go-ethereum v1.14.5 h1:szuFzO1MhJmweXjoM5nSAeDvjNUH3vIQoMzzQnfvjpw= +github.com/ethereum/go-ethereum v1.14.5/go.mod h1:VEDGGhSxY7IEjn98hJRFXl/uFvpRgbIIf2PpXiyGGgc= +github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 h1:KrE8I4reeVvf7C1tm8elRjj4BdscTYzz/WAbYyf/JI4= +github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0/go.mod h1:D9AJLVXSyZQXJQVk8oh1EwjISE+sJTn2duYIZC0dy3w= +github.com/ethersphere/bee/v2 v2.5.0 h1:eqWJFw5GyT9qglXPxL+utgSN4J/77JQ9j74aXm7+ozQ= +github.com/ethersphere/bee/v2 v2.5.0/go.mod h1:hwfKQuD3wUbcLrqow3hz9HIAPSqJAxJ8pYr59qd6eXA= +github.com/ethersphere/go-storage-incentives-abi v0.9.2 h1:6Pmxuj48LBTxayzwADNYmcbiqj6ongoRWwWV4Wp1EPo= +github.com/ethersphere/go-storage-incentives-abi v0.9.2/go.mod h1:SXvJVtM4sEsaSKD0jc1ClpDLw8ErPoROZDme4Wrc/Nc= +github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= +github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= +github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= +github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.22.0 h1:k6HsTZ0sTnROkhS//R0O+55JgM8C4Bx7ia+JlgcnOao= +github.com/go-playground/validator/v10 v10.22.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= +github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 h1:X4egAf/gcS1zATw6wn4Ej8vjuVGxeHdan+bRb2ebyv4= +github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= +github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= +github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= +github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= +github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= +github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= +github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/common v0.47.0 h1:p5Cz0FNHo7SnWOmWmoRozVcjEp0bIVU8cV7OShpjL1k= +github.com/prometheus/common v0.47.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil v3.21.5+incompatible h1:OloQyEerMi7JUrXiNzy8wQ5XN+baemxSl12QgIzt0jc= +github.com/shirou/gopsutil v3.21.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= +github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= +github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= +github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/exp v0.0.0-20240213143201-ec583247a57a h1:HinSgX1tJRX3KsL//Gxynpw5CTOAIPhgL4W8PNiIpVE= +golang.org/x/exp v0.0.0-20240213143201-ec583247a57a/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= diff --git a/main.go b/main.go new file mode 100644 index 0000000..bc7be22 --- /dev/null +++ b/main.go @@ -0,0 +1,21 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/ethersphere/batch-export/cmd" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM, syscall.SIGINT) + defer stop() + + if err := cmd.Execute(ctx); err != nil { + fmt.Fprintln(os.Stderr, "Error:", err) + os.Exit(1) + } +} diff --git a/pkg/ethclientwrapper/ethclientwrapper.go b/pkg/ethclientwrapper/ethclientwrapper.go new file mode 100644 index 0000000..2e3b6e1 --- /dev/null +++ b/pkg/ethclientwrapper/ethclientwrapper.go @@ -0,0 +1,81 @@ +package ethclientwrapper + +import ( + "context" + "sync" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethersphere/bee/v2/pkg/log" + "golang.org/x/time/rate" +) + +type Client struct { + *ethclient.Client + limiter *rate.Limiter + logger log.Logger + rawURL string + mu sync.Mutex +} + +type ClientOption func(*Client) + +// WithRateLimit sets the rate limit for the Ethereum client. +func WithRateLimit(requestsPerSecond int) ClientOption { + return func(c *Client) { + c.limiter = rate.NewLimiter(rate.Limit(requestsPerSecond), requestsPerSecond) + } +} + +// WithLogger sets a logger for the Ethereum client. +func WithLogger(logger log.Logger) ClientOption { + return func(c *Client) { + c.logger = logger + } +} + +// NewClient creates a new Ethereum client with possible rate limiting. +func NewClient(ctx context.Context, rawURL string, opts ...ClientOption) (*Client, error) { + ethclient, err := ethclient.DialContext(ctx, rawURL) + if err != nil { + return nil, err + } + + c := &Client{ + Client: ethclient, + rawURL: rawURL, + limiter: nil, + logger: log.Noop, + } + + for _, option := range opts { + option(c) + } + + return c, nil +} + +// Close closes the underlying Ethereum client. +func (c *Client) Close() { + c.Client.Close() +} + +func (c *Client) FilterLogs(ctx context.Context, q ethereum.FilterQuery) ([]types.Log, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if err := c.applyRateLimit(ctx); err != nil { + return nil, err + } + + return c.Client.FilterLogs(ctx, q) +} + +// applyRateLimit checks if the limiter is set and applies the rate limit. +func (c *Client) applyRateLimit(ctx context.Context) error { + if c.limiter != nil { + return c.limiter.Wait(ctx) + } + return nil +} diff --git a/pkg/eventfetcher/eventfetcher.go b/pkg/eventfetcher/eventfetcher.go new file mode 100644 index 0000000..62b9c2c --- /dev/null +++ b/pkg/eventfetcher/eventfetcher.go @@ -0,0 +1,178 @@ +package eventfetcher + +import ( + "context" + "fmt" + "math/big" + + "github.com/ethersphere/bee/v2/pkg/log" + "github.com/go-playground/validator/v10" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethersphere/batch-export/pkg/ethclientwrapper" + "github.com/ethersphere/batch-export/pkg/logcache" +) + +type Client struct { + validate *validator.Validate + client *ethclientwrapper.Client + logger log.Logger + blockRangeLimit uint32 + logCache *logcache.Cache + + batchCreatedTopic common.Hash + batchTopUpTopic common.Hash + batchDepthIncreaseTopic common.Hash + priceUpdateTopic common.Hash + pausedTopic common.Hash +} + +func NewClient(client *ethclientwrapper.Client, postageStampContractABI abi.ABI, blockRangeLimit uint32, logger log.Logger) *Client { + return &Client{ + validate: validator.New(), + logCache: logcache.New(), + client: client, + logger: logger, + blockRangeLimit: blockRangeLimit, + batchCreatedTopic: postageStampContractABI.Events["BatchCreated"].ID, + batchTopUpTopic: postageStampContractABI.Events["BatchTopUp"].ID, + batchDepthIncreaseTopic: postageStampContractABI.Events["BatchDepthIncrease"].ID, + priceUpdateTopic: postageStampContractABI.Events["PriceUpdate"].ID, + // pausedTopic: postageStampContractABI.Events["Paused"].ID, + } +} + +type Request struct { + Address common.Address `validate:"required"` + StartBlock uint64 + EndBlock uint64 +} + +// GetLogs fetches logs and sends them to a channel +func (c *Client) GetLogs(ctx context.Context, tr *Request) (<-chan types.Log, <-chan error) { + logChan := make(chan types.Log, 100) + errorChan := make(chan error, 1) + + go func() { + defer close(logChan) + defer close(errorChan) + defer func() { + priceUpdateLog := c.logCache.Get() + if priceUpdateLog != nil { + c.logger.Info("sending last cached value", "transactionHash", priceUpdateLog.TxHash) + logChan <- *priceUpdateLog + } + }() + + if err := c.validate.Struct(tr); err != nil { + errorChan <- fmt.Errorf("error validating request: %w", err) + return + } + + var fromBlock, toBlock *big.Int + + // Determine toBlock + if tr.EndBlock == 0 { + latestBlock, err := c.client.BlockNumber(ctx) + if err != nil { + errorChan <- fmt.Errorf("failed to get latest block number: %w", err) + return + } + toBlock = new(big.Int).SetUint64(latestBlock) + } else { + toBlock = big.NewInt(int64(tr.EndBlock)) + } + + if tr.StartBlock == 0 { + fromBlock = big.NewInt(0) + } else { + fromBlock = big.NewInt(int64(tr.StartBlock)) + } + + if fromBlock.Cmp(toBlock) > 0 { + errorChan <- fmt.Errorf("start block (%s) cannot be greater than end block (%s)", fromBlock.String(), toBlock.String()) + return + } + query := c.filterQuery(tr.Address, fromBlock, toBlock) + c.fetchLogs(ctx, query, logChan, errorChan) + }() + + return logChan, errorChan +} + +// fetchLogs iterates through block ranges and fetches logs. +// It sends errors to errorChan and stops processing if an error occurs or context is cancelled. +// It is responsible for closing logsChan. +func (c *Client) fetchLogs(ctx context.Context, query ethereum.FilterQuery, logsChan chan<- types.Log, errorChan chan<- error) { + maxBlocks := uint64(c.blockRangeLimit) + startBlock := query.FromBlock.Uint64() + endBlock := query.ToBlock.Uint64() + + for start := startBlock; start <= endBlock; start += maxBlocks { + currentEnd := start + maxBlocks - 1 + if currentEnd < start { + currentEnd = endBlock + } else { + currentEnd = min(currentEnd, endBlock) + } + + chunkQuery := ethereum.FilterQuery{ + FromBlock: new(big.Int).SetUint64(start), + ToBlock: new(big.Int).SetUint64(currentEnd), + Addresses: query.Addresses, + Topics: query.Topics, + } + + c.logger.Debug("querying logs", "fromBlock", chunkQuery.FromBlock.Uint64(), "toBlock", chunkQuery.ToBlock.Uint64()) + + logs, err := c.client.FilterLogs(ctx, chunkQuery) + if err != nil { + errorChan <- fmt.Errorf("failed to retrieve logs for range %d-%d: %w", start, currentEnd, err) + return + } + + for _, log := range logs { + if log.Topics[0] == c.priceUpdateTopic { + c.logCache.Set(&log) + continue + } + select { + case logsChan <- log: + case <-ctx.Done(): + errorChan <- ctx.Err() + return // stop processing if context is cancelled + } + } + + // check cancellation between chunks + select { + case <-ctx.Done(): + errorChan <- ctx.Err() + return + default: + // continue to next chunk + } + } +} + +func (c *Client) filterQuery(postageStampContractAddress common.Address, from, to *big.Int) ethereum.FilterQuery { + return ethereum.FilterQuery{ + FromBlock: from, + ToBlock: to, + Addresses: []common.Address{ + postageStampContractAddress, + }, + Topics: [][]common.Hash{ + { + c.batchCreatedTopic, + c.batchTopUpTopic, + c.batchDepthIncreaseTopic, + c.priceUpdateTopic, + c.pausedTopic, + }, + }, + } +} diff --git a/pkg/filestore/filestore.go b/pkg/filestore/filestore.go new file mode 100644 index 0000000..9617158 --- /dev/null +++ b/pkg/filestore/filestore.go @@ -0,0 +1,36 @@ +package filestore + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/ethereum/go-ethereum/core/types" +) + +// SaveLogsAsync writes logs to a file asynchronously. +func SaveLogsAsync(ctx context.Context, logChan <-chan types.Log, filePath string) error { + file, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("error creating file: %w", err) + } + defer file.Close() + + encoder := json.NewEncoder(file) + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case logObj, ok := <-logChan: + if !ok { + return nil + } + + if err := encoder.Encode(logObj); err != nil { + return fmt.Errorf("error encoding log: %w", err) + } + } + } +} diff --git a/pkg/gzipstore/gzipstore.go b/pkg/gzipstore/gzipstore.go new file mode 100644 index 0000000..6763ad0 --- /dev/null +++ b/pkg/gzipstore/gzipstore.go @@ -0,0 +1,37 @@ +package gzipstore + +import ( + "compress/gzip" + "fmt" + "io" + "os" +) + +// CompressFile compresses the specified input file into the specified output gzip file. +func CompressFile(inputFilePath string, outputFilePath string) error { + // open the input file for reading + inputFile, err := os.Open(inputFilePath) + if err != nil { + return fmt.Errorf("failed to open input file '%s': %w", inputFilePath, err) + } + defer inputFile.Close() + + // create the output file for writing + outputFile, err := os.Create(outputFilePath) + if err != nil { + return fmt.Errorf("failed to create output file '%s': %w", outputFilePath, err) + } + defer outputFile.Close() + + // create a new gzip writer that writes to the output file + gzipWriter := gzip.NewWriter(outputFile) + defer gzipWriter.Close() + + // copy the contents from the input file to the gzip writer + _, err = io.Copy(gzipWriter, inputFile) + if err != nil { + return fmt.Errorf("failed to write compressed data to '%s': %w", outputFilePath, err) + } + + return nil +} diff --git a/pkg/logcache/logcache.go b/pkg/logcache/logcache.go new file mode 100644 index 0000000..54ed98e --- /dev/null +++ b/pkg/logcache/logcache.go @@ -0,0 +1,28 @@ +package logcache + +import ( + "sync" + + "github.com/ethereum/go-ethereum/core/types" +) + +type Cache struct { + l *types.Log + m sync.Mutex +} + +func New() *Cache { + return &Cache{} +} + +func (c *Cache) Set(l *types.Log) { + c.m.Lock() + c.l = l + c.m.Unlock() +} + +func (c *Cache) Get() *types.Log { + c.m.Lock() + defer c.m.Unlock() + return c.l +} diff --git a/pkg/logcache/logcache_test.go b/pkg/logcache/logcache_test.go new file mode 100644 index 0000000..2d0d3d5 --- /dev/null +++ b/pkg/logcache/logcache_test.go @@ -0,0 +1,18 @@ +package logcache_test + +import ( + "testing" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethersphere/batch-export/pkg/logcache" +) + +func TestBasic(t *testing.T) { + t.Parallel() + c := logcache.New() + b := &types.Log{} + c.Set(b) + if c.Get() == nil { + t.Fatal("expected block to be cached") + } +}