Skip to content

Commit 9dc95ae

Browse files
authored
Full Abseil + RE2 support (#173)
Add RE2 support + full Abseil Related cleanups: - Simpler Docker container - Single Makefile - Support for -flto - SDK proto upgrade to v26.1 - Updated build docs - More/independent examples - Remove protobuf CI check Signed-off-by: Martijn Stevenson <[email protected]>
1 parent 6b3dc93 commit 9dc95ae

20 files changed

+17981
-17848
lines changed

Diff for: .github/workflows/cpp.yml

-23
Original file line numberDiff line numberDiff line change
@@ -60,29 +60,6 @@ jobs:
6060
export PATH=$PATH:$(go env GOPATH)/bin
6161
addlicense -ignore="**/BUILD" -check .
6262
63-
protobuf:
64-
runs-on: ubuntu-22.04
65-
66-
steps:
67-
- uses: actions/checkout@v2
68-
69-
- name: Install protobuf v3.9.1
70-
run: |
71-
git clone https://github.com/protocolbuffers/protobuf
72-
cd protobuf
73-
git checkout v3.9.1
74-
./autogen.sh
75-
./configure
76-
make
77-
sudo make install
78-
sudo ldconfig
79-
80-
- name: Re-generate and verify protobuf artifacts
81-
run: |
82-
rm *.pb.{cc,h}
83-
make protobuf
84-
git diff --exit-code -G "(^[^ /])|(^\s+[^\*])" *.pb.{cc,h}
85-
8663
build:
8764
runs-on: ubuntu-22.04
8865

Diff for: Dockerfile-sdk

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
FROM ubuntu:noble
22

3-
COPY ./sdk_container.sh /
4-
COPY ./build_wasm.sh /
3+
COPY *.sh /
54
COPY *.cc *.h *.js *.proto Makefile* /sdk/
65

76
RUN ./sdk_container.sh

Diff for: Makefile

+50-20
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,57 @@
1-
CPP_API ?= .
1+
ifdef NO_CONTEXT
2+
CPP_CONTEXT_LIB =
3+
else
4+
CPP_CONTEXT_LIB = ${PROXY_WASM_CPP_SDK}/proxy_wasm_intrinsics.cc
5+
endif
26

3-
all: proxy_wasm_intrinsics.pb.h proxy_wasm_intrinsics_lite.pb.h struct_lite.pb.h ${CPP_API}/libprotobuf.a ${CPP_API}/libprotobuf-lite.a
7+
PROTOBUF ?= none
8+
ifeq ($(PROTOBUF), full)
9+
PROTO_DEPS := protobuf
10+
PROTO_OPTS := -DPROXY_WASM_PROTOBUF_FULL=1 \
11+
${PROXY_WASM_CPP_SDK}/proxy_wasm_intrinsics.pb.cc
12+
else ifeq ($(PROTOBUF), lite)
13+
PROTO_DEPS := protobuf-lite
14+
PROTO_OPTS := -DPROXY_WASM_PROTOBUF_LITE=1 \
15+
${PROXY_WASM_CPP_SDK}/proxy_wasm_intrinsics_lite.pb.cc \
16+
${PROXY_WASM_CPP_SDK}/struct_lite.pb.cc
17+
else
18+
PROTO_DEPS :=
19+
PROTO_OPTS :=
20+
endif
421

5-
protobuf: proxy_wasm_intrinsics.pb.h proxy_wasm_intrinsics_lite.pb.h struct_lite.pb.h
22+
# Provide a list of libraries that the wasm depends on (absl_*, re2, etc).
23+
WASM_DEPS ?=
624

7-
proxy_wasm_intrinsics.pb.h: proxy_wasm_intrinsics.proto
8-
protoc --cpp_out=. proxy_wasm_intrinsics.proto
25+
# Determine dependency link options.
26+
# NOTE: Strip out -pthread which RE2 claims to need...
27+
PKG_CONFIG ?= pkg-config
28+
PKG_CONFIG_PATH = ${EMSDK}/upstream/emscripten/cache/sysroot/lib/pkgconfig
29+
WASM_LIBS = $(shell $(PKG_CONFIG) $(WASM_DEPS) $(PROTO_DEPS) \
30+
--with-path=$(PKG_CONFIG_PATH) --libs | sed -e 's/-pthread //g')
931

10-
proxy_wasm_intrinsics_lite.pb.h struct_lite.pb.h: proxy_wasm_intrinsics_lite.proto
11-
protoc --cpp_out=. -I. proxy_wasm_intrinsics_lite.proto
12-
protoc --cpp_out=. struct_lite.proto
32+
debug-deps:
33+
# WASM_DEPS : ${WASM_DEPS}
34+
# WASM_LIBS : ${WASM_LIBS}
35+
# PROTO_DEPS: ${PROTO_DEPS}
36+
# PROTO_OPTS: ${PROTO_OPTS}
1337

14-
${CPP_API}/libprotobuf.a ${CPP_API}/libprotobuf-lite.a:
15-
rm -rf protobuf-wasm \
16-
&& git clone https://github.com/protocolbuffers/protobuf protobuf-wasm \
17-
&& cd protobuf-wasm \
18-
&& git checkout v3.9.1 \
19-
&& ./autogen.sh \
20-
&& emconfigure ./configure --disable-shared CXXFLAGS="-O3 -flto" \
21-
&& emmake make \
22-
&& cd .. \
23-
&& cp protobuf-wasm/src/.libs/libprotobuf-lite.a ${CPP_API}/libprotobuf-lite.a \
24-
&& cp protobuf-wasm/src/.libs/libprotobuf.a ${CPP_API}/libprotobuf.a
38+
# TODO(mpwarres): Add Emscripten stack/heap size params in PR#174.
39+
%.wasm %.wat: %.cc
40+
em++ --no-entry -sSTANDALONE_WASM -sEXPORTED_FUNCTIONS=_malloc \
41+
--std=c++17 -O3 -flto \
42+
--js-library ${PROXY_WASM_CPP_SDK}/proxy_wasm_intrinsics.js \
43+
-I${PROXY_WASM_CPP_SDK} \
44+
${CPP_CONTEXT_LIB} \
45+
${PROTO_OPTS} \
46+
${WASM_LIBS} \
47+
$*.cc -o $*.wasm
2548

2649
clean:
27-
rm -f proxy_wasm_intrinsics.pb.h proxy_wasm_intrinsics_lite.pb.h struct_lite.pb.h ${CPP_API}/libprotobuf.a ${CPP_API}/libprotobuf-lite.a
50+
rm *.wasm
51+
52+
# NOTE: How to regenerate .pb.h and .pb.cc files for a protobuf update:
53+
# - download + extract protobuf release (currently v26.1)
54+
# - regenerate:
55+
# ./bin/protoc --cpp_out=../ -I../ -Iinclude/google/protobuf/ ../struct_lite.proto
56+
# ./bin/protoc --cpp_out=../ -I../ -Iinclude/google/protobuf/ ../proxy_wasm_intrinsics_lite.proto
57+
# ./bin/protoc --cpp_out=../ -I../ -Iinclude/google/protobuf/ ../proxy_wasm_intrinsics.proto

Diff for: Makefile.base

-11
This file was deleted.

Diff for: Makefile.base_lite

-11
This file was deleted.

Diff for: bazel/defs.bzl

+6-3
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,11 @@ load("@emsdk//emscripten_toolchain:wasm_rules.bzl", "wasm_cc_binary")
1616
load("@rules_cc//cc:defs.bzl", "cc_binary")
1717

1818
def _optimized_wasm_cc_binary_transition_impl(settings, attr):
19-
# TODO(PiotrSikora): Add -flto to copts/linkopts when fixed in emsdk.
20-
# See: https://github.com/emscripten-core/emsdk/issues/971
19+
# Define STANDALONE_WASM at compile time as well as link time (below).
20+
# This fixes Abseil by not including Emscripten JS stacktraces + symbolization.
21+
# TODO(martijneken): Remove after Abseil stops using this define.
2122
return {
22-
"//command_line_option:copt": ["-O3"],
23+
"//command_line_option:copt": ["-O3", "-flto", "-DSTANDALONE_WASM"],
2324
"//command_line_option:cxxopt": [],
2425
"//command_line_option:linkopt": [],
2526
"//command_line_option:collect_code_coverage": False,
@@ -102,6 +103,8 @@ def proxy_wasm_cc_binary(
102103
)
103104

104105
wasm_cc_binary(
106+
standalone = True,
107+
threads = "off",
105108
name = "wasm_" + name,
106109
cc_target = ":proxy_wasm_" + name.rstrip(".wasm"),
107110
tags = tags + [

Diff for: bazel/repositories.bzl

+16
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,19 @@ def proxy_wasm_cpp_sdk_repositories():
3131
strip_prefix = "protobuf-3.17.3",
3232
url = "https://github.com/protocolbuffers/protobuf/releases/download/v3.17.3/protobuf-all-3.17.3.tar.gz",
3333
)
34+
35+
maybe(
36+
http_archive,
37+
name = "com_google_absl",
38+
sha256 = "95e90be7c3643e658670e0dd3c1b27092349c34b632c6e795686355f67eca89f",
39+
strip_prefix = "abseil-cpp-20240722.0",
40+
urls = ["https://github.com/abseil/abseil-cpp/archive/20240722.0.zip"],
41+
)
42+
43+
maybe(
44+
http_archive,
45+
name = "com_google_re2",
46+
sha256 = "18cf85922e27fad3ed9c96a27733037da445f35eb1a2744c306a37c6d11e95c4",
47+
strip_prefix = "re2-2023-07-01",
48+
url = "https://github.com/google/re2/archive/2023-07-01.tar.gz",
49+
)

Diff for: build_wasm.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,4 @@
1616

1717
source /root/emsdk/emsdk_env.sh
1818
export PATH=/usr/local/bin:$PATH
19-
make
19+
make "$@"

Diff for: docs/building.md

+30-93
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
# Proxy-Wasm C++ SDK Build Instructions
22

33
The C++ SDK has dependencies on specific versions of the C++ WebAssembly
4-
toolchain [Emscripten](https://emscripten.org) and the protobuf library,
5-
therefore use of a Docker image is recommended.
4+
toolchain [Emscripten](https://emscripten.org). Use of a Docker image is
5+
recommended to achieve repeatable builds and save work.
66

77
## Docker
88

@@ -11,7 +11,7 @@ A Dockerfile for the C++ SDK is provided in [Dockerfile-sdk](../Dockerfile-sdk).
1111
It can built in this repository's root directory by:
1212

1313
```bash
14-
docker build -t wasmsdk:v2 -f Dockerfile-sdk .
14+
docker build -t wasmsdk:v3 -f Dockerfile-sdk .
1515
```
1616

1717
The docker image can be used for compiling C++ plugin code into Wasm modules.
@@ -23,12 +23,10 @@ Create a directory with your source files and a Makefile:
2323
```makefile
2424
PROXY_WASM_CPP_SDK=/sdk
2525

26-
all: myproject.wasm
27-
28-
include ${PROXY_WASM_CPP_SDK}/Makefile.base_lite
26+
include ${PROXY_WASM_CPP_SDK}/Makefile
2927
```
3028

31-
Source file (myproject.cc):
29+
Create a C++ source file (myproject.cc):
3230

3331
```c++
3432
#include <string>
@@ -57,62 +55,40 @@ void ExampleContext::onDone() { logInfo("onDone " + std::to_string(id())); }
5755

5856
### Compiling with the Docker build image
5957

60-
Run docker:
61-
62-
```bash
63-
docker run -v $PWD:/work -w /work wasmsdk:v2 /build_wasm.sh
64-
```
65-
66-
### Caching the standard libraries
67-
68-
The first time that emscripten runs it will generate the standard libraries. To
69-
cache these in the docker image, after the first successful compilation (e.g
70-
myproject.cc above), commit the image with the standard libraries:
58+
Run docker to build wasm, using a target with a .wasm suffix:
7159

7260
```bash
73-
docker commit `docker ps -l | grep wasmsdk:v2 | awk '{print $1}'` wasmsdk:v2
61+
docker run -v $PWD:/work -w /work wasmsdk:v3 /build_wasm.sh myproject.wasm
7462
```
7563

76-
This will save time on subsequent compiles.
64+
You can specify wasm dependencies via these Makefile variables:
7765

78-
### Using Abseil from the Docker image
66+
- PROTOBUF = {full, lite, none}
67+
- WASM_DEPS = list of libraries
7968

80-
Abseil (optionally) is built in /root/abseil and can be used. Note that the
81-
Abseil containers (e.g. `absl::flat_hash_set`) exercise many syscalls which are
82-
not supported. Consequentially individual files should be pulled in which are
83-
relatively self contained (e.g. `strings`). Example customized Makefile:
69+
For example:
8470

8571
```makefile
8672
PROXY_WASM_CPP_SDK=/sdk
87-
CPP_API:=${PROXY_WASM_CPP_SDK}
88-
CPP_CONTEXT_LIB = ${CPP_API}/proxy_wasm_intrinsics.cc
89-
ABSL = /root/abseil-cpp
90-
ABSL_CPP = ${ABSL}/absl/strings/str_cat.cc ${ABSL}/absl/strings/str_split.cc ${ABSL}/absl/strings/numbers.cc ${ABSL}/absl/strings/ascii.cc
9173

92-
all: plugin.wasm
74+
PROTOBUF=lite
75+
WASM_DEPS=re2 absl_strings
9376

94-
%.wasm %.wat: %.cc ${CPP_API}/proxy_wasm_intrinsics.h ${CPP_API}/proxy_wasm_enums.h ${CPP_API}/proxy_wasm_externs.h ${CPP_API}/proxy_wasm_api.h ${CPP_API}/proxy_wasm_intrinsics.js ${CPP_CONTEXT_LIB}
95-
ls /root
96-
em++ --no-entry -s EXPORTED_FUNCTIONS=['_malloc'] --std=c++17 -O3 -flto -I${CPP_API} -I${CPP_API}/google/protobuf -I/usr/local/include -I${ABSL} --js-library ${CPP_API}/proxy_wasm_intrinsics.js ${ABSL_CPP} $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf.a -o $*.wasm
77+
include ${PROXY_WASM_CPP_SDK}/Makefile
9778
```
9879

99-
Precompiled Abseil libraries are also available, so the above can also be done
100-
as:
101-
102-
```makefile
103-
PROXY_WASM_CPP_SDK=/sdk
104-
CPP_API:=${PROXY_WASM_CPP_SDK}
105-
CPP_CONTEXT_LIB = ${CPP_API}/proxy_wasm_intrinsics.cc
106-
ABSL = /root/abseil-cpp
107-
ABSL_LIBS = ${ABSL}/absl/strings/libabsl_strings.a ${ABSL}/absl/strings/libabsl_strings_internal.a ${ABSL}/absl/strings/libabsl_str_format_internal.a
80+
### Caching the standard libraries
10881

109-
all: plugin.wasm
82+
The first time that emscripten runs it will generate the standard libraries. To
83+
cache these in the docker image, after the first successful compilation (e.g
84+
myproject.cc above), commit the image with the standard libraries:
11085

111-
%.wasm %.wat: %.cc ${CPP_API}/proxy_wasm_intrinsics.h ${CPP_API}/proxy_wasm_enums.h ${CPP_API}/proxy_wasm_externs.h ${CPP_API}/proxy_wasm_api.h ${CPP_API}/proxy_wasm_intrinsics.js ${CPP_CONTEXT_LIB}
112-
ls /root
113-
em++ --no-entry -s EXPORTED_FUNCTIONS=['_malloc'] --std=c++17 -O3 -flto -I${CPP_API} -I${CPP_API}/google/protobuf -I/usr/local/include -I${ABSL} --js-library ${CPP_API}/proxy_wasm_intrinsics.js $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf.a ${ABSL_LIBS} -o $*.wasm
86+
```bash
87+
docker commit `docker ps -l | grep wasmsdk:v3 | awk '{print $1}'` wasmsdk:v3
11488
```
11589

90+
This will save time on subsequent compiles.
91+
11692
### Ownership of the resulting .wasm files
11793

11894
The compiled files may be owned by root. To chown them, add the follow lines to
@@ -124,48 +100,32 @@ PROXY_WASM_CPP_SDK=/sdk
124100
all: myproject.wasm
125101
chown ${uid}.${gid} $^
126102

127-
include ${PROXY_WASM_CPP_SDK}/Makefile.base_lite
103+
include ${PROXY_WASM_CPP_SDK}/Makefile
128104
```
129105

130106
Invocation file (e.g. build.sh):
131107

132108
```bash
133109
#!/bin/bash
134-
docker run -e uid="$(id -u)" -e gid="$(id -g)" -v $PWD:/work -w /work wasmsdk:v2 /build_wasm.sh
110+
docker run -e uid="$(id -u)" -e gid="$(id -g)" -v $PWD:/work -w /work wasmsdk:v3 /build_wasm.sh
135111
```
136112

137113
## Dependencies for building Wasm modules:
138114

139115
If you do not wish to use the Docker image, the dependencies can be installed by
140-
script (sdk\_container.sh), or by hand.
141-
142-
### protobuf v3.9.1
143-
144-
You must install the version of protobuf on your build system that matches the
145-
libprotobuf.a files (without any patches) so that the generated code matches the
146-
.a library. Currently this is based on tag v3.9.1 of
147-
https://github.com/protocolbuffers/protobuf.
148-
149-
```bash
150-
git clone https://github.com/protocolbuffers/protobuf
151-
cd protobuf
152-
git checkout v3.9.1
153-
git submodule update --init --recursive
154-
./autogen.sh
155-
./configure
156-
make
157-
make check
158-
sudo make install
159-
```
116+
script (sdk\_container.sh), or by hand. First you need Emscripten to build
117+
everything else. Then you can build other wasm-compatible libraries, such as
118+
protobuf, abseil, and RE2.
160119

161120
### Emscripten
162121

122+
Version 3.1.67 is known to work:
123+
163124
```bash
164125
git clone https://github.com/emscripten-core/emsdk.git
165126
cd emsdk
166127
git checkout 3.1.67
167-
168-
./emsdk install 3.1.67
128+
./emsdk install --shallow 3.1.67
169129
./emsdk activate 3.1.67
170130

171131
source ./emsdk_env.sh
@@ -178,26 +138,3 @@ It is possible later versions will work, e.g.
178138
./emsdk install latest
179139
./emsdk activate latest
180140
```
181-
182-
However 3.1.67 is known to work.
183-
184-
### Rebuilding the libprotobuf.a files
185-
186-
To build the protobuf static libraries for use in your proxy-wasm wasm module,
187-
if you need them, you may use Emscripten in the same way sdk\_container.sh does
188-
it:
189-
190-
```bash
191-
git clone https://github.com/protocolbuffers/protobuf protobuf-wasm
192-
cd protobuf-wasm
193-
git checkout v3.9.1
194-
git submodule update --init --recursive
195-
./autogen.sh
196-
emconfigure ./configure --disable-shared CXXFLAGS="-O3 -flto"
197-
emmake make
198-
cd ..
199-
cp protobuf-wasm/src/.libs/libprotobuf-lite.a ${CPP_API}/libprotobuf-lite.a
200-
cp protobuf-wasm/src/.libs/libprotobuf.a ${CPP_API}/libprotobuf.a
201-
```
202-
203-
Note: ensure /usr/local/bin is in your path.

0 commit comments

Comments
 (0)