Skip to content

Continuous benchmarking in CI #7100

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Oct 13, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 33 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ on:
pull_request:
branches: [master, 11.0_release]

permissions:
# allow posting comments to pull request
pull-requests: write

concurrency:
group: ci-${{ github.ref }}-1
# Cancel previous builds for pull requests only.
Expand Down Expand Up @@ -90,8 +94,9 @@ jobs:
ocaml_compiler: ocaml-variants.5.2.0+options,ocaml-option-static
upload_binaries: true
upload_libs: true
# Build the playground compiler on the fastest runner
# Build the playground compiler and run the benchmarks on the fastest runner
build_playground: true
benchmarks: true
- os: buildjet-2vcpu-ubuntu-2204-arm # ARM
ocaml_compiler: ocaml-variants.5.2.0+options,ocaml-option-static
upload_binaries: true
Expand Down Expand Up @@ -150,7 +155,7 @@ jobs:
# matrix.ocaml_compiler may contain commas
- name: Get OPAM cache key
shell: bash
run: echo "opam_cache_key=opam-env-v3-${{ matrix.os }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('dune-project') }}" | sed 's/,/-/g' >> $GITHUB_ENV
run: echo "opam_cache_key=opam-env-v4-${{ matrix.os }}-${{ matrix.ocaml_compiler }}-${{ hashFiles('dune-project') }}" | sed 's/,/-/g' >> $GITHUB_ENV

- name: Restore OPAM environment
id: cache-opam-env
Expand Down Expand Up @@ -320,6 +325,32 @@ jobs:
if: runner.os != 'Windows'
run: make -C tests/gentype_tests/typescript-react-example clean test

- name: Run syntax benchmarks
if: matrix.benchmarks
run: ./_build/install/default/bin/syntax_benchmarks | tee tests/benchmark-output.json

- name: Download previous benchmark data
if: matrix.benchmarks
uses: actions/cache@v4
with:
path: ./tests/benchmark-cache
key: syntax-benchmark-v1

- name: Store benchmark result
# Do not run for PRs created from other repos as those won't be able to write to the pull request
if: ${{ matrix.benchmarks && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.event.repository.full_name) }}
uses: benchmark-action/github-action-benchmark@v1
with:
name: Syntax Benchmarks
tool: customSmallerIsBetter
output-file-path: tests/benchmark-output.json
external-data-json-path: ./tests/benchmark-cache/benchmark-data.json
github-token: ${{ secrets.GITHUB_TOKEN }}
alert-threshold: "150%"
comment-always: true
comment-on-alert: true
summary-always: true

- name: Build playground compiler
if: matrix.build_playground
run: |
Expand Down
4 changes: 4 additions & 0 deletions dune-project
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@
(and
:with-test
(= 0.26.2)))
(yojson
(and
:with-test
(= 2.2.2)))
(ocaml-lsp-server
(and
:with-dev-setup
Expand Down
1 change: 1 addition & 0 deletions rescript.opam
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ bug-reports: "https://github.com/rescript-lang/rescript-compiler/issues"
depends: [
"ocaml" {>= "4.10"}
"ocamlformat" {with-test & = "0.26.2"}
"yojson" {with-test & = "2.2.2"}
"ocaml-lsp-server" {with-dev-setup & = "1.19.0"}
"cppo" {= "1.6.9"}
"js_of_ocaml" {= "5.8.1"}
Expand Down
159 changes: 75 additions & 84 deletions tests/syntax_benchmarks/Benchmark.ml
Original file line number Diff line number Diff line change
Expand Up @@ -75,59 +75,31 @@ end = struct
end

module Benchmark : sig
type t
type test_result = {ms_per_run: float; allocs_per_run: int}

val make : name:string -> f:(t -> unit) -> unit -> t
val launch : t -> unit
val report : t -> unit
val run : (unit -> unit) -> num_iterations:int -> test_result
end = struct
type t = {
name: string;
mutable start: Time.t;
mutable n: int; (* current iterations count *)
mutable duration: Time.t;
bench_func: t -> unit;
mutable n: int; (* current iteration count *)
mutable total_duration: Time.t;
bench_func: unit -> unit;
mutable timer_on: bool;
(* mutable result: benchmarkResult; *)
(* The initial states *)
mutable start_allocs: float;
mutable start_bytes: float;
(* The net total of this test after being run. *)
mutable net_allocs: float;
mutable net_bytes: float;
mutable total_allocs: float;
}

let report b =
print_endline (Format.sprintf "Benchmark: %s" b.name);
print_endline (Format.sprintf "Nbr of iterations: %d" b.n);
print_endline
(Format.sprintf "Benchmark ran during: %fms" (Time.print b.duration));
print_endline
(Format.sprintf "Avg time/op: %fms"
(Time.print b.duration /. float_of_int b.n));
print_endline
(Format.sprintf "Allocs/op: %d"
(int_of_float (b.net_allocs /. float_of_int b.n)));
print_endline
(Format.sprintf "B/op: %d"
(int_of_float (b.net_bytes /. float_of_int b.n)));

(* return (float64(r.Bytes) * float64(r.N) / 1e6) / r.T.Seconds() *)
print_newline ();
()
type test_result = {ms_per_run: float; allocs_per_run: int}

let make ~name ~f () =
let make f =
{
name;
start = Time.zero;
n = 0;
bench_func = f;
duration = Time.zero;
total_duration = Time.zero;
timer_on = false;
start_allocs = 0.;
start_bytes = 0.;
net_allocs = 0.;
net_bytes = 0.;
total_allocs = 0.;
}

(* total amount of memory allocated by the program since it started in words *)
Expand All @@ -139,79 +111,74 @@ end = struct
if not b.timer_on then (
let allocated_words = mallocs () in
b.start_allocs <- allocated_words;
b.start_bytes <- allocated_words *. 8.;
b.start <- Time.now ();
b.timer_on <- true)

let stop_timer b =
if b.timer_on then (
let allocated_words = mallocs () in
let diff = Time.diff b.start (Time.now ()) in
b.duration <- Time.add b.duration diff;
b.net_allocs <- b.net_allocs +. (allocated_words -. b.start_allocs);
b.net_bytes <- b.net_bytes +. ((allocated_words *. 8.) -. b.start_bytes);
b.total_duration <- Time.add b.total_duration diff;
b.total_allocs <- b.total_allocs +. (allocated_words -. b.start_allocs);
b.timer_on <- false)

let reset_timer b =
if b.timer_on then (
let allocated_words = mallocs () in
b.start_allocs <- allocated_words;
b.net_allocs <- allocated_words *. 8.;
b.start <- Time.now ());
b.net_allocs <- 0.;
b.net_bytes <- 0.
b.start <- Time.now ())

let run_iteration b n =
Gc.full_major ();
b.n <- n;
reset_timer b;
start_timer b;
b.bench_func b;
b.bench_func ();
stop_timer b

let launch b =
(* 150 runs * all the benchmarks means around 1m of benchmark time *)
for n = 1 to 150 do
let run f ~num_iterations =
let b = make f in
for n = 1 to num_iterations do
run_iteration b n
done
done;
{
ms_per_run = Time.print b.total_duration /. float_of_int b.n;
allocs_per_run = int_of_float (b.total_allocs /. float_of_int b.n);
}
end

module Benchmarks : sig
val run : unit -> unit
end = struct
type action = Parse | Print

let string_of_action action =
match action with
| Parse -> "parser"
| Print -> "printer"

(* TODO: we could at Reason here *)
type lang = Rescript
let string_of_lang lang =
match lang with
| Rescript -> "rescript"
| Parse -> "Parse"
| Print -> "Print"

let parse_rescript src filename =
let p = Parser.make src filename in
let structure = ResParser.parse_implementation p in
assert (p.diagnostics == []);
structure

let benchmark filename lang action =
let src = IO.read_file filename in
let name =
filename ^ " " ^ string_of_lang lang ^ " " ^ string_of_action action
in
let data_dir = "tests/syntax_benchmarks/data"
let num_iterations = 150

let benchmark (filename, action) =
let path = Filename.concat data_dir filename in
let src = IO.read_file path in
let benchmark_fn =
match (lang, action) with
| Rescript, Parse ->
fun _ ->
let _ = Sys.opaque_identity (parse_rescript src filename) in
match action with
| Parse ->
fun () ->
let _ = Sys.opaque_identity (parse_rescript src path) in
()
| Rescript, Print ->
let p = Parser.make src filename in
| Print ->
let p = Parser.make src path in
let ast = ResParser.parse_implementation p in
fun _ ->
fun () ->
let _ =
Sys.opaque_identity
(let cmt_tbl = CommentTable.make () in
Expand All @@ -221,21 +188,45 @@ end = struct
in
()
in
let b = Benchmark.make ~name ~f:benchmark_fn () in
Benchmark.launch b;
Benchmark.report b
Benchmark.run benchmark_fn ~num_iterations

let specs =
[
("RedBlackTree.res", Parse);
("RedBlackTree.res", Print);
("RedBlackTreeNoComments.res", Print);
("Napkinscript.res", Parse);
("Napkinscript.res", Print);
("HeroGraphic.res", Parse);
("HeroGraphic.res", Print);
]

let run () =
let data_dir = "tests/syntax_benchmarks/data" in
benchmark (Filename.concat data_dir "RedBlackTree.res") Rescript Parse;
benchmark (Filename.concat data_dir "RedBlackTree.res") Rescript Print;
benchmark
(Filename.concat data_dir "RedBlackTreeNoComments.res")
Rescript Print;
benchmark (Filename.concat data_dir "Napkinscript.res") Rescript Parse;
benchmark (Filename.concat data_dir "Napkinscript.res") Rescript Print;
benchmark (Filename.concat data_dir "HeroGraphic.res") Rescript Parse;
benchmark (Filename.concat data_dir "HeroGraphic.res") Rescript Print
List.to_seq specs
|> Seq.flat_map (fun spec ->
let filename, action = spec in
let test_name = string_of_action action ^ " " ^ filename in
let {Benchmark.ms_per_run; allocs_per_run} = benchmark spec in
[
`Assoc
[
("name", `String (Format.sprintf "%s - time/run" test_name));
("unit", `String "ms");
("value", `Float ms_per_run);
];
`Assoc
[
("name", `String (Format.sprintf "%s - allocs/run" test_name));
("unit", `String "words");
("value", `Int allocs_per_run);
];
]
|> List.to_seq)
|> Seq.iteri (fun i json ->
print_endline (if i == 0 then "[" else ",");
print_string (Yojson.to_string json));
print_newline ();
print_endline "]"
end

let () = Benchmarks.run ()
3 changes: 2 additions & 1 deletion tests/syntax_benchmarks/dune
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
(enabled_if
(and
(<> %{profile} browser)
(>= %{ocaml_version} "4.14.0")
(or
(= %{system} macosx)
; or one of Linuxes (see https://github.com/ocaml/ocaml/issues/10613)
Expand All @@ -22,6 +23,6 @@
(foreign_stubs
(language c)
(names time))
(libraries syntax))
(libraries syntax yojson))

(data_only_dirs data)
2 changes: 1 addition & 1 deletion tests/syntax_benchmarks/time.c
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ CAMLprim value caml_mach_absolute_time(value unit) {
#elif defined(__linux__)
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
result = now.tv_sec * 1000 + now.tv_nsec / 1000000;
result = now.tv_sec * 1000000000 + now.tv_nsec;
#endif

return caml_copy_int64(result);
Expand Down