Skip to content
This repository was archived by the owner on Oct 14, 2025. It is now read-only.

Commit 8d07c3e

Browse files
andersfugmannandersfugmann
committed
Optimize decoding. Benchmarks now shows faster than protoc when using flamba
 * Assume orderd fields when reading on fast-path and revert to a slow path if/when encountering unordered fields. * Sort fields to follow standard implementations * Change constructor to take extensions last to speedup handling of extensions * Delete locate options.ml as we require ocaml >=4.08
1 parent c1f9494 commit 8d07c3e

18 files changed

Lines changed: 484 additions & 598 deletions

‎Makefile‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,26 +24,33 @@ uninstall: build ## uninstall
2424
%: %.proto
2525
protoc --experimental_allow_proto3_optional -I $(dir $<) $< -o/dev/stdout | protoc --experimental_allow_proto3_optional --decode google.protobuf.FileDescriptorSet $(GOOGLE_INCLUDE)/descriptor.proto
2626

27-
src/spec/descriptor.ml: build
28-
protoc "--plugin=protoc-gen-ocaml=_build/default/src/plugin/protoc_gen_ocaml.exe" \
27+
PLUGIN = _build/default/src/plugin/protoc_gen_ocaml.exe
28+
$(PLUGIN): force
29+
dune build src/plugin/protoc_gen_ocaml.exe
30+
31+
src/spec/descriptor.ml: $(PLUGIN)
32+
protoc "--plugin=protoc-gen-ocaml=$(PLUGIN)" \
2933
-I /usr/include \
3034
--ocaml_out=src/spec/. \
3135
$(GOOGLE_INCLUDE)/descriptor.proto
3236

33-
src/spec/plugin.ml: build
34-
protoc "--plugin=protoc-gen-ocaml=_build/default/src/plugin/protoc_gen_ocaml.exe" \
37+
src/spec/plugin.ml: $(PLUGIN)
38+
protoc "--plugin=protoc-gen-ocaml=$(PLUGIN)" \
3539
-I /usr/include \
3640
--ocaml_out=src/spec/. \
3741
$(GOOGLE_INCLUDE)/compiler/plugin.proto
3842

39-
src/spec/options.ml: build
40-
protoc "--plugin=protoc-gen-ocaml=_build/default/src/plugin/protoc_gen_ocaml.exe" \
43+
src/spec/options.ml: $(PLUGIN)
44+
protoc "--plugin=protoc-gen-ocaml=$(PLUGIN)" \
4145
-I src/spec -I /usr/include \
4246
--ocaml_out=src/spec/. \
4347
src/spec/options.proto
4448
.PHONY: bootstrap
4549
bootstrap: src/spec/descriptor.ml src/spec/plugin.ml src/spec/options.ml ## Regenerate files used for generation
4650

51+
%.ml: %.proto
52+
protoc -I $(shell pkg-config protobuf --variable=includedir) -I $(dir $<) --plugin=protoc-gen-ocaml=_build/default/src/plugin/protoc_gen_ocaml.exe \
53+
--ocaml_out=$(dir $@). $<
4754

4855

4956
.PHONY: doc
@@ -66,6 +73,8 @@ gh-pages: doc ## Publish documentation
6673
bench: ## Run benchmark to compare with ocaml-protoc
6774
dune exec bench/bench.exe
6875

76+
.PHONY: force
77+
force:
6978

7079
.PHONY: help
7180
help: ## Show this help

‎bench/bench.ml‎

Lines changed: 23 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
1+
[@@@ocaml.warning "-26"]
12
open Base
23
open Stdio
4+
5+
let meassure = Bechamel_perf.Instance.cpu_clock
6+
37
[@@@ocaml.warning "-32"]
48
module type Protoc_impl = sig
59
type m
@@ -38,14 +42,15 @@ let make_tests (type v) (module Protoc: Protoc_impl) (module Plugin: Plugin_impl
3842
let size_normal, unused_normal = verify_identity ~mode:Ocaml_protoc_plugin.Writer.Balanced v_plugin in
3943
let size_speed, unused_speed = verify_identity ~mode:Ocaml_protoc_plugin.Writer.Speed v_plugin in
4044
let size_space, unused_space = verify_identity ~mode:Ocaml_protoc_plugin.Writer.Space v_plugin in
41-
let data = Plugin.M.to_proto' (Ocaml_protoc_plugin.Writer.init ()) v_plugin |> Ocaml_protoc_plugin.Writer.contents in
42-
let v_plugin = Plugin.M.from_proto_exn (Ocaml_protoc_plugin.Reader.create data) in
43-
let v_protoc = Protoc.decode_pb_m (Pbrt.Decoder.of_string data) in
45+
let data_plugin = Plugin.M.to_proto' (Ocaml_protoc_plugin.Writer.init ()) v_plugin |> Ocaml_protoc_plugin.Writer.contents in
46+
let v_plugin' = Plugin.M.from_proto_exn (Ocaml_protoc_plugin.Reader.create data_plugin) in
47+
assert (Poly.equal v_plugin v_plugin');
48+
let v_protoc = Protoc.decode_pb_m (Pbrt.Decoder.of_string data_plugin) in
4449
let protoc_encoder = Pbrt.Encoder.create () in
4550
let () = Protoc.encode_pb_m v_protoc protoc_encoder in
4651
let data_protoc = Pbrt.Encoder.to_string protoc_encoder in
47-
let v_plugin' = Plugin.M.from_proto_exn (Ocaml_protoc_plugin.Reader.create data_protoc) in
48-
let () = match Plugin.M.equal v_plugin v_plugin' with
52+
let v_plugin'' = Plugin.M.from_proto_exn (Ocaml_protoc_plugin.Reader.create data_protoc) in
53+
let () = match Plugin.M.equal v_plugin v_plugin'' with
4954
| true -> ()
5055
| false ->
5156
eprintf "Orig: %s\n" (Plugin.M.show v_plugin);
@@ -69,8 +74,8 @@ let make_tests (type v) (module Protoc: Protoc_impl) (module Plugin: Plugin_impl
6974
let test_decode =
7075
Test.make_grouped ~name:"Decode"
7176
[
72-
Test.make ~name:"Plugin" (Staged.stage @@ fun () -> Plugin.M.from_proto_exn (Ocaml_protoc_plugin.Reader.create data |> Sys.opaque_identity));
73-
Test.make ~name:"Protoc" (Staged.stage @@ fun () -> Protoc.decode_pb_m (Pbrt.Decoder.of_string data |> Sys.opaque_identity))
77+
Test.make ~name:"Plugin" (Staged.stage @@ fun () -> Plugin.M.from_proto_exn (Ocaml_protoc_plugin.Reader.create data_plugin |> Sys.opaque_identity));
78+
Test.make ~name:"Protoc" (Staged.stage @@ fun () -> Protoc.decode_pb_m (Pbrt.Decoder.of_string data_protoc |> Sys.opaque_identity))
7479
]
7580
in
7681
Test.make_grouped ~name:(Plugin.M.name' ()) [test_encode; test_decode]
@@ -128,22 +133,22 @@ let create_test_data ~depth () =
128133

129134
let benchmark tests =
130135
let open Bechamel in
131-
let instances = Bechamel_perf.Instance.[ cpu_clock ] in
132-
let cfg = Benchmark.cfg ~limit:2000 ~quota:(Time.second 5.0) ~kde:(Some 1000) ~stabilize:true ~compaction:false () in
136+
let instances = [ meassure ] in
137+
let cfg = Benchmark.cfg ~limit:500 ~quota:(Time.second 5.0) ~kde:(Some 100) ~stabilize:true ~compaction:false () in
133138
Benchmark.all cfg instances tests
134139

135140
let analyze results =
136141
let open Bechamel in
137-
let ols = Analyze.ols ~bootstrap:10 ~r_square:false
142+
let ols = Analyze.ols ~bootstrap:5 ~r_square:false
138143
~predictors:[| Measure.run |] in
139-
let results = Analyze.all ols Bechamel_perf.Instance.cpu_clock results in
140-
Analyze.merge ols [ Bechamel_perf.Instance.cpu_clock ] [ results ]
144+
let results = Analyze.all ols meassure results in
145+
Analyze.merge ols [ meassure ] [ results ]
141146

142147
let print_bench_results results =
143148
let open Bechamel in
144149
let () = Bechamel_notty.Unit.add
145-
Bechamel_perf.Instance.cpu_clock
146-
(Measure.unit Bechamel_perf.Instance.cpu_clock)
150+
meassure
151+
(Measure.unit meassure)
147152
in
148153

149154
let img (window, results) =
@@ -162,7 +167,8 @@ let print_bench_results results =
162167

163168
let _ =
164169
let v_plugin = create_test_data ~depth:2 () |> Option.value_exn in
165-
[ make_tests (module Protoc.Bench) (module Plugin.Bench) v_plugin;
170+
[
171+
make_tests (module Protoc.Bench) (module Plugin.Bench) v_plugin;
166172
make_tests (module Protoc.Int64) (module Plugin.Int64) 27;
167173
make_tests (module Protoc.Float) (module Plugin.Float) 27.0001;
168174
make_tests (module Protoc.String) (module Plugin.String) "Benchmark";
@@ -171,8 +177,8 @@ let _ =
171177
List.init 1000 ~f:(fun i -> i) |> make_tests (module Protoc.Int64_list) (module Plugin.Int64_list);
172178
List.init 1000 ~f:(fun i -> Float.of_int i) |> make_tests (module Protoc.Float_list) (module Plugin.Float_list);
173179
List.init 1000 ~f:(fun _ -> random_string ()) |> make_tests (module Protoc.String_list) (module Plugin.String_list);
174-
(* random_list ~len:100 ~f:(fun () -> Plugin.Enum_list.Enum.ED) () |> make_tests (module Protoc.Enum_list) (module Plugin.Enum_list); *)
175-
]
180+
(* random_list ~len:100 ~f:(fun () -> Plugin.Enum_list.Enum.ED) () |> make_tests (module Protoc.Enum_list) (module Plugin.Enum_list); *)
181+
]
176182
|> List.rev |> List.iter ~f:(fun test ->
177183
test
178184
|> benchmark

0 commit comments

Comments
 (0)