Skip to content

Commit

Permalink
Merge tag 'v2.58.2' into release/v2.58.2
Browse files Browse the repository at this point in the history
  • Loading branch information
mattmalec committed Mar 13, 2024
2 parents 4770e1a + 125509e commit ef51847
Show file tree
Hide file tree
Showing 483 changed files with 122,631 additions and 25,787 deletions.
78 changes: 78 additions & 0 deletions .github/workflows/qa-clean-exit-block-downloading.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
name: QA - Clean exit (block downloading)

on:
push:
branches:
- 'release/**'
pull_request:
branches:
- devel
- 'release/**'
types:
- ready_for_review

jobs:
long-running-test:
runs-on: self-hosted
env:
ERIGON_REFERENCE_DATA_DIR: /opt/erigon-release/datadir
ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir
WORKING_TIME_SECONDS: 600

steps:
- name: Check out repository
uses: actions/checkout@v2

- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: '1.21'

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.x'

#- name: Install dependencies
# run: |
# sudo apt-get update
# sudo apt-get install -y build-essential make gcc

- name: Restore Erigon Testbed Data Directory
run: |
rm -rf $ERIGON_TESTBED_DATA_DIR/chaindata
rsync -a --delete $ERIGON_REFERENCE_DATA_DIR/ $ERIGON_TESTBED_DATA_DIR/
- name: Clean Erigon Build Directory
run: |
make clean
- name: Build Erigon
run: |
make erigon
working-directory: ${{ github.workspace }}

- name: Run Erigon, send ctrl-c and check for clean exiting
run: |
# Run Erigon, send ctrl-c and check logs
python3 ${{ github.workspace }}/../../../../erigon-qa/test_system/qa-tests/clean-exit/run_and_check_clean_exit.py ${{ github.workspace }}/build/bin $ERIGON_TESTBED_DATA_DIR $WORKING_TIME_SECONDS
# Capture monitoring script exit status
monitoring_exit_status=$?

# Clean up Erigon process if it's still running
if kill -0 $ERIGON_PID 2> /dev/null; then
echo "Terminating Erigon"
kill $ERIGON_PID
wait $ERIGON_PID
else
echo "Erigon has already terminated"
fi

# Check monitoring script exit status
if [ $monitoring_exit_status -eq 0 ]; then
echo "Monitoring completed successfully"
else
echo "Error detected in Erigon logs or monitoring script exited unexpectedly"
exit 1
fi
4 changes: 0 additions & 4 deletions .github/workflows/qa-clean-exit-snapshot-downloading.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,6 @@ jobs:
make erigon
working-directory: ${{ github.workspace }}

#- name: Download Python Script for Logs Checking
# run: |
# curl -o check_erigon_exit.py 'https://gist.githubusercontent.com/mriccobene/8db4030a745de34d527f136f2caa104f/raw/3c1a860cb87d61075e78ce399e17f0ab157cacc6/check_erigon_exit.py'

- name: Run Erigon, send ctrl-c and check for clean exiting
run: |
# Run Erigon, send ctrl-c and check logs
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ jobs:
id: prepare
run: |
TAG=${GITHUB_REF#refs/tags/}
echo ::set-output name=tag_name::${TAG}
echo "tag_name=${TAG}" >> $GITHUB_OUTPUT
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
Expand Down
4 changes: 2 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# syntax = docker/dockerfile:1.2
FROM docker.io/library/golang:1.20-alpine3.17 AS builder
FROM docker.io/library/golang:1.21-alpine3.17 AS builder

RUN apk --no-cache add build-base linux-headers git bash ca-certificates libstdc++

Expand All @@ -18,7 +18,7 @@ RUN --mount=type=cache,target=/root/.cache \
make BUILD_TAGS=nosqlite,noboltdb,nosilkworm all


FROM docker.io/library/golang:1.20-alpine3.17 AS tools-builder
FROM docker.io/library/golang:1.21-alpine3.17 AS tools-builder
RUN apk --no-cache add build-base linux-headers git bash ca-certificates libstdc++
WORKDIR /app

Expand Down
4 changes: 2 additions & 2 deletions Dockerfile.debian
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# syntax = docker/dockerfile:1.2
FROM docker.io/library/golang:1.20-bullseye AS builder
FROM docker.io/library/golang:1.21-bullseye AS builder

RUN apt update
RUN apt install -y build-essential git bash ca-certificates libstdc++6
Expand All @@ -17,7 +17,7 @@ RUN --mount=type=cache,target=/root/.cache \
make all


FROM docker.io/library/golang:1.20-alpine3.17 AS tools-builder
FROM docker.io/library/golang:1.21-alpine3.17 AS tools-builder

RUN apk --no-cache add build-base linux-headers git bash ca-certificates libstdc++
WORKDIR /app
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ install:
@ls -al "$(DIST)"

PACKAGE_NAME := github.com/ledgerwatch/erigon
GOLANG_CROSS_VERSION ?= v1.20.7
GOLANG_CROSS_VERSION ?= v1.21.6

.PHONY: release-dry-run
release-dry-run: git-submodules
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ Use `--datadir` to choose where to store data.
Use `--chain=gnosis` for [Gnosis Chain](https://www.gnosis.io/), `--chain=bor-mainnet` for Polygon Mainnet,
`--chain=mumbai` for Polygon Mumbai and `--chain=amoy` for Polygon Amoy.
For Gnosis Chain you need a [Consensus Layer](#beacon-chain-consensus-layer) client alongside
Erigon (https://docs.gnosischain.com/node/guide/beacon).
Erigon (https://docs.gnosischain.com/node/manual/beacon).

Running `make help` will list and describe the convenience commands available in the [Makefile](./Makefile).

Expand Down
6 changes: 3 additions & 3 deletions cl/antiquary/antiquary.go
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ func (a *Antiquary) Loop() error {
if to-from < snaptype.Erigon2MergeLimit {
continue
}
if err := a.antiquate(a.sn.Version(), from, to); err != nil {
if err := a.antiquate(from, to); err != nil {
return err
}
case <-a.ctx.Done():
Expand All @@ -218,12 +218,12 @@ func (a *Antiquary) Loop() error {
}

// Antiquate will antiquate a specific block range (aka. retire snapshots), this should be ran in the background.
func (a *Antiquary) antiquate(version uint8, from, to uint64) error {
func (a *Antiquary) antiquate(from, to uint64) error {
if a.downloader == nil {
return nil // Just skip if we don't have a downloader
}
log.Info("[Antiquary]: Antiquating", "from", from, "to", to)
if err := freezeblocks.DumpBeaconBlocks(a.ctx, a.mainDB, a.beaconDB, version, from, to, snaptype.Erigon2MergeLimit, a.dirs.Tmp, a.dirs.Snap, 1, log.LvlDebug, a.logger); err != nil {
if err := freezeblocks.DumpBeaconBlocks(a.ctx, a.mainDB, a.beaconDB, from, to, a.dirs.Tmp, a.dirs.Snap, 1, log.LvlDebug, a.logger); err != nil {
return err
}

Expand Down
16 changes: 13 additions & 3 deletions cl/antiquary/state_antiquary.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@ func (s *Antiquary) loopStates(ctx context.Context) {
return
}

_, beforeFinalized, err := s.readHistoricalProcessingProgress(ctx)
if err != nil {
s.logger.Error("Failed to read historical processing progress", "err", err)
return
}

for {
select {
// Check if we are behind finalized
Expand All @@ -78,15 +84,16 @@ func (s *Antiquary) loopStates(ctx context.Context) {
continue
}
// Check if we are behind finalized
progress, finalized, err := s.readHistoricalProcessingProgress(ctx)
_, finalized, err := s.readHistoricalProcessingProgress(ctx)
if err != nil {
s.logger.Error("Failed to read historical processing progress", "err", err)
continue
}
// Stay behind a little bit we rely on forkchoice up until (finalized - 2*slotsPerEpoch)
if progress+s.cfg.SlotsPerEpoch/2 >= finalized {
// We wait for updated finality.
if finalized == beforeFinalized {
continue
}
beforeFinalized = finalized
if err := s.IncrementBeaconState(ctx, finalized); err != nil {
slot := uint64(0)
if s.currentState != nil {
Expand Down Expand Up @@ -427,6 +434,9 @@ func (s *Antiquary) IncrementBeaconState(ctx context.Context, to uint64) error {
if err := transition.TransitionState(s.currentState, block, blockRewardsCollector, fullValidation); err != nil {
return err
}
// if s.currentState.Slot() == 4293952 {
// s.dumpFullBeaconState()
// }
blocksProcessed++

first = false
Expand Down
71 changes: 71 additions & 0 deletions cl/beacon/beaconevents/emitter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package beaconevents

import (
"sync"

"github.com/google/uuid"
"golang.org/x/sync/errgroup"
)

type Subscription struct {
id string
topics map[string]struct{}
cb func(topic string, item any)
}

type EventName string

// Emitters creates pub/sub connection
type Emitters struct {
cbs map[string]*Subscription
mu sync.RWMutex
}

func NewEmitters() *Emitters {
return &Emitters{
cbs: map[string]*Subscription{},
}
}

// publish to all subscribers. each callback is run in a separate goroutine
func (e *Emitters) Publish(s string, a any) {
// forward gossip object
e.mu.Lock()
values := make([]*Subscription, 0, len(e.cbs))
for _, v := range e.cbs {
values = append(values, v)
}
e.mu.Unlock()

egg := errgroup.Group{}
for idx := range values {
v := values[idx]
exec := func() error { v.cb(s, a); return nil }
if _, ok := v.topics["*"]; ok {
egg.Go(exec)
} else if _, ok := v.topics[s]; ok {
egg.Go(exec)
}
}
egg.Wait()
}

// subscribe with callback. call the returned cancelfunc to unregister the callback
// publish will block until all callbacks for the message are resolved
func (e *Emitters) Subscribe(topics []string, cb func(topic string, item any)) (func(), error) {
subid := uuid.New().String()
sub := &Subscription{
id: subid,
topics: map[string]struct{}{},
cb: cb,
}
for _, v := range topics {
sub.topics[v] = struct{}{}
}
e.cbs[subid] = sub
return func() {
e.mu.Lock()
defer e.mu.Unlock()
delete(e.cbs, subid)
}, nil
}
54 changes: 54 additions & 0 deletions cl/beacon/beaconevents/emitter_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package beaconevents_test

import (
"sync/atomic"
"testing"

"github.com/ledgerwatch/erigon/cl/beacon/beaconevents"
"github.com/stretchr/testify/require"
)

func TestEmitterSet(t *testing.T) {
e := beaconevents.NewEmitters()
var called int
e.Subscribe([]string{"set"}, func(topic string, item any) {
require.EqualValues(t, "set", topic)
require.EqualValues(t, "hello", item.(string))
called = called + 1
})
e.Publish("set", "hello")
require.EqualValues(t, 1, called)
}
func TestEmitterFilters(t *testing.T) {
e := beaconevents.NewEmitters()
var a atomic.Int64
var b atomic.Int64
var ab atomic.Int64
var wild atomic.Int64
e.Subscribe([]string{"a"}, func(topic string, item any) {
require.EqualValues(t, topic, item.(string))
a.Add(1)
})
e.Subscribe([]string{"b"}, func(topic string, item any) {
require.EqualValues(t, topic, item.(string))
b.Add(1)
})
e.Subscribe([]string{"a", "b"}, func(topic string, item any) {
require.EqualValues(t, topic, item.(string))
ab.Add(1)
})
e.Subscribe([]string{"*"}, func(topic string, item any) {
require.EqualValues(t, topic, item.(string))
wild.Add(1)
})

e.Publish("a", "a")
e.Publish("b", "b")
e.Publish("b", "b")
e.Publish("c", "c")

require.EqualValues(t, 1, a.Load())
require.EqualValues(t, 2, b.Load())
require.EqualValues(t, 3, ab.Load())
require.EqualValues(t, 4, wild.Load())
}
Loading

0 comments on commit ef51847

Please sign in to comment.