Skip to content

Commit fa0c236

Browse files
ibhatt-jumptradingibhatt-jumptrading
committed
wip
2 parents d1b47b3 + e0150a6 commit fa0c236

File tree

90 files changed

+4913
-533
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

90 files changed

+4913
-533
lines changed

‎.github/workflows/backtest.yml‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,27 @@
1-
name: Replay Ledgers
1+
name: Backtest
22
on:
33
workflow_call:
44
inputs:
5-
coverage:
6-
type: boolean
7-
default: false
85
machine:
96
type: string
107
default: linux_gcc_zen2
118
extras:
129
type: string
1310
default: "handholding"
11+
script:
12+
type: string
13+
default: src/flamenco/runtime/tests/run_backtest_ci.sh
1414
workflow_dispatch:
15+
inputs:
16+
machine:
17+
type: string
18+
default: linux_gcc_zen2
19+
extras:
20+
type: string
21+
default: "handholding"
22+
script:
23+
type: string
24+
default: src/flamenco/runtime/tests/run_backtest_ci.sh
1525
jobs:
1626
backtest-ledger-replay:
1727
timeout-minutes: 15
@@ -35,11 +45,13 @@ jobs:
3545
- uses: ./.github/actions/deps
3646
with:
3747
extras: +dev
48+
3849
- uses: ./.github/actions/hugepages
3950
with:
4051
# DO NOT CHANGE THESE VALUES - increasing the hugetlbfs reservations on the CI runners too high causes other jobs to be OOM-killed
4152
count_gigantic: 398
4253
count_huge: 535
54+
4355
- uses: ./.github/actions/cpusonline
4456

4557
- name: build
@@ -50,12 +62,12 @@ jobs:
5062
run: |
5163
echo OBJDIR=$(make help | grep OBJDIR | awk '{print $4}') >> $GITHUB_ENV
5264
53-
- name: test replay ledgers
65+
- name: run backtest
5466
run: |
5567
sudo $OBJDIR/bin/firedancer-dev configure init keys
5668
sudo prlimit --pid=$$ --nofile=1048576
5769
sudo prlimit --pid=$$ --memlock=unlimited
58-
DUMP_DIR=../dump HUGE_TLBFS_ALLOW_HUGEPAGE_INCREASE=false make run-runtime-backtest
70+
DUMP_DIR=../dump HUGE_TLBFS_ALLOW_HUGEPAGE_INCREASE=false ${{ inputs.script }}
5971
6072
- name: fini
6173
if: always()

‎.github/workflows/check_seccomp.yml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: Check Generated Seccomp Files
2+
on:
3+
pull_request:
4+
5+
permissions: {}
6+
7+
jobs:
8+
check-seccomp:
9+
runs-on: ubuntu-latest
10+
timeout-minutes: 2
11+
12+
steps:
13+
- name: Checkout current commit
14+
uses: actions/checkout@v5
15+
with:
16+
fetch-depth: 0
17+
18+
- name: Setup Python environment
19+
run: |
20+
python -m venv venv && venv/bin/pip install edn_format
21+
22+
- name: Check generated seccomp files
23+
run: |
24+
set -e
25+
source venv/bin/activate
26+
make seccomp-policies
27+
if [ -n "$(git status --porcelain)" ]; then
28+
echo "Generated seccomp files are out of date. Please run 'make seccomp-policies' and commit the changes."
29+
git --no-pager diff
30+
exit 1
31+
else
32+
echo "All generated seccomp files are up to date."
33+
fi

‎.github/workflows/on_nightly.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,3 +64,8 @@ jobs:
6464
build_arm: false
6565
clang: none
6666
cid: 6
67+
backtest:
68+
uses: ./.github/workflows/backtest.yml
69+
with:
70+
script: src/flamenco/runtime/tests/run_backtest_all.sh
71+
secrets: inherit

‎NOTICE‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -545,3 +545,24 @@ https://github.com/solana-program/token/tree/program%40v8.0.0/p-token
545545
http://www.apache.org/licenses/
546546

547547
=======================================================================
548+
549+
The flatbuffers generated code in src/flamenco/runtime/tests/flatbuffers/
550+
is generated by flatcc.
551+
552+
https://github.com/dvidelabs/flatcc/
553+
554+
Copyright 2015 Mikkel F. Jørgensen, dvide.com
555+
556+
Licensed under the Apache License, Version 2.0 (the "License");
557+
you may not use this file except in compliance with the License.
558+
You may obtain a copy of the License at
559+
560+
http://www.apache.org/licenses/LICENSE-2.0
561+
562+
Unless required by applicable law or agreed to in writing, software
563+
distributed under the License is distributed on an "AS IS" BASIS,
564+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
565+
See the License for the specific language governing permissions and
566+
limitations under the License.
567+
568+
=======================================================================

‎config/extra/with-arm.mk‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ include config/extra/with-bzip2.mk
3434
include config/extra/with-lz4.mk
3535
include config/extra/with-openssl.mk
3636
include config/extra/with-rocksdb.mk
37+
include config/extra/with-flatcc.mk
3738

3839
endif
3940

‎config/extra/with-flatcc.mk‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
ifneq (,$(wildcard $(OPT)/lib/libflatccrt.a))
2+
FLATCC_LIBS:=$(OPT)/lib/libflatccrt.a
3+
FD_HAS_FLATCC:=1
4+
CPPFLAGS+=-DFD_HAS_FLATCC=1
5+
else
6+
$(info "flatcc not installed, skipping")
7+
endif

‎config/extra/with-liburing.mk‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
ifneq (,$(wildcard $(OPT)/lib/liburing.a))
2+
FD_HAS_LIBURING:=1
3+
CFLAGS+=-DFD_HAS_LIBURING=1
4+
LDFLAGS+=$(OPT)/lib/liburing.a
5+
else
6+
$(warning "liburing not installed, skipping")
7+
endif

‎config/extra/with-x86-64.mk‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,11 @@ include config/extra/with-ucontext.mk
1616
include config/extra/with-secp256k1.mk
1717
include config/extra/with-s2nbignum.mk
1818
include config/extra/with-zstd.mk
19+
include config/extra/with-liburing.mk
1920
include config/extra/with-bzip2.mk
2021
include config/extra/with-lz4.mk
2122
include config/extra/with-openssl.mk
2223
include config/extra/with-rocksdb.mk
24+
include config/extra/with-flatcc.mk
2325

2426
FD_ARCH_SUPPORTS_SANDBOX:=1

‎contrib/test/test-vectors-commit-sha.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
c8e3939ed100fda48240cc1b7a784e96415f5ffe
1+
3bded3a6e883823f246ccdbe5b4e6ba49fbf26ed

‎deps.sh‎

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ PREFIX="$(pwd)/opt"
3535

3636
DEVMODE=0
3737
MSAN=0
38+
LIBURING=0
3839
_CC="${CC:=gcc}"
3940
_CXX="${CXX:=g++}"
4041
EXTRA_CFLAGS="-g3 -fno-omit-frame-pointer"
@@ -133,15 +134,17 @@ fetch () {
133134
if [[ $MSAN == 1 ]]; then
134135
checkout_llvm
135136
fi
136-
checkout_repo zstd https://github.com/facebook/zstd "v1.5.7"
137-
checkout_repo lz4 https://github.com/lz4/lz4 "v1.10.0"
138-
checkout_repo s2n https://github.com/awslabs/s2n-bignum "" "4d2e22a"
139-
checkout_repo openssl https://github.com/openssl/openssl "openssl-3.6.0"
140-
checkout_repo secp256k1 https://github.com/bitcoin-core/secp256k1 "v0.7.0"
137+
checkout_repo zstd https://github.com/facebook/zstd "v1.5.7"
138+
checkout_repo lz4 https://github.com/lz4/lz4 "v1.10.0"
139+
checkout_repo liburing https://github.com/axboe/liburing "liburing-2.12"
140+
checkout_repo s2n https://github.com/awslabs/s2n-bignum "" "4d2e22a"
141+
checkout_repo openssl https://github.com/openssl/openssl "openssl-3.6.0"
142+
checkout_repo secp256k1 https://github.com/bitcoin-core/secp256k1 "v0.7.0"
143+
checkout_repo flatcc https://github.com/dvidelabs/flatcc.git "" "3ae5eda"
141144
if [[ $DEVMODE == 1 ]]; then
142-
checkout_repo bzip2 https://gitlab.com/bzip2/bzip2 "bzip2-1.0.8"
143-
checkout_repo rocksdb https://github.com/facebook/rocksdb "v10.5.1"
144-
checkout_repo snappy https://github.com/google/snappy "1.2.2"
145+
checkout_repo bzip2 https://gitlab.com/bzip2/bzip2 "bzip2-1.0.8"
146+
checkout_repo rocksdb https://github.com/facebook/rocksdb "v10.5.1"
147+
checkout_repo snappy https://github.com/google/snappy "1.2.2"
145148
fi
146149
}
147150

@@ -438,6 +441,16 @@ install_lz4 () {
438441
echo "[+] Successfully installed lz4"
439442
}
440443

444+
install_liburing () {
445+
cd "$PREFIX/git/liburing"
446+
447+
echo "[+] Installing liburing to $PREFIX"
448+
./configure --prefix="$PREFIX" --cc="$CC -fPIC $EXTRA_CFLAGS"
449+
"${MAKE[@]}"
450+
"${MAKE[@]}" install
451+
echo "[+] Successfully installed liburing"
452+
}
453+
441454
install_s2n () {
442455
cd "$PREFIX/git/s2n"
443456

@@ -610,6 +623,21 @@ install_snappy () {
610623
echo "[+] Successfully installed snappy"
611624
}
612625

626+
install_flatcc () {
627+
echo "[+] Installing flatcc"
628+
cd "$PREFIX/git/flatcc"
629+
cmake -B build \
630+
-DCMAKE_INSTALL_PREFIX=$PREFIX \
631+
-DCMAKE_BUILD_TYPE=Release \
632+
-DFLATCC_INSTALL=ON \
633+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
634+
-DCMAKE_CXX_FLAGS="$EXTRA_CXXFLAGS" \
635+
-DCMAKE_EXE_LINKER_FLAGS="$EXTRA_LDFLAGS"
636+
cmake --build build -j
637+
cmake --install build
638+
echo "[+] Successfully installed flatcc"
639+
}
640+
613641
install () {
614642
CC="$(command -v $_CC)"
615643
cc="$CC"
@@ -629,6 +657,11 @@ install () {
629657
fi
630658
( install_zstd )
631659
( install_lz4 )
660+
if [[ $LIBURING == 1 ]]; then
661+
if [[ "$OS" == "Linux" ]]; then
662+
( install_liburing )
663+
fi
664+
fi
632665
if [[ "$(uname -m)" == x86_64 ]]; then
633666
( install_s2n )
634667
fi
@@ -638,6 +671,7 @@ install () {
638671
( install_bzip2 )
639672
( install_snappy )
640673
( install_rocksdb )
674+
( install_flatcc )
641675
fi
642676

643677
# Merge lib64 with lib
@@ -673,6 +707,10 @@ while [[ $# -gt 0 ]]; do
673707
shift
674708
DEVMODE=1
675709
;;
710+
"+uring")
711+
shift
712+
LIBURING=1
713+
;;
676714
nuke)
677715
shift
678716
nuke

0 commit comments

Comments
 (0)