Skip to content

Commit 68d274c

Browse files
kegsaykegsay
authored
Merge branch 'main' into s7evink/eventsize
2 parents b594f62 + 0c19e2d commit 68d274c

File tree

143 files changed

+5580
-1757
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

143 files changed

+5580
-1757
lines changed

‎.github/workflows/ci.yaml

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,8 @@ jobs:
1616
steps:
1717
- uses: actions/checkout@v3 # Checkout complement
1818
- uses: actions/setup-go@v4
19-
- name: "Install Complement Dependencies"
20-
run: |
21-
sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
19+
with:
20+
go-version-file: go.mod
2221
- name: "Run internal Complement tests"
2322
run: |
2423
go test ./internal/...
@@ -41,26 +40,32 @@ jobs:
4140
matrix:
4241
include:
4342
- homeserver: Synapse
44-
tags: synapse_blacklist msc3083 msc3787 msc3874 faster_joins
45-
env: "COMPLEMENT_SHARE_ENV_PREFIX=PASS_ PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite"
43+
repo: element-hq/synapse
44+
tags: synapse_blacklist
45+
packages: ./tests/msc3874 ./tests/msc3902
46+
env: "COMPLEMENT_ENABLE_DIRTY_RUNS=1 COMPLEMENT_SHARE_ENV_PREFIX=PASS_ PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite"
4647
timeout: 20m
4748

4849
- homeserver: Dendrite
50+
repo: element-hq/dendrite
4951
tags: dendrite_blacklist
50-
env: ""
52+
packages: ""
53+
env: "COMPLEMENT_ENABLE_DIRTY_RUNS=1"
5154
timeout: 10m
5255

5356
steps:
5457
- uses: actions/checkout@v3 # Checkout complement
5558

5659
- uses: actions/setup-go@v4
60+
with:
61+
go-version-file: go.mod
62+
5763
# Similar steps as dockerfiles/ComplementCIBuildkite.Dockerfile but on the host. We need
5864
# to do this so we can _be_ the host when running Complement so we can snaffle all the ports. If
5965
# we run Complement _in_ Docker then we can't -p all high numbered ports which then breaks federation
6066
# servers which listen on random high numbered ports.
6167
- name: "Install Complement Dependencies"
6268
run: |
63-
sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
6469
go install -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
6570
mkdir .gotestfmt/github -p
6671
cp .ci/complement_package.gotpl .gotestfmt/github/package.gotpl
@@ -88,7 +93,7 @@ jobs:
8893
continue
8994
;;
9095
esac
91-
(wget -O - "https://github.com/matrix-org/${{ matrix.homeserver }}/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C homeserver) && break
96+
(wget -O - "https://github.com/${{ matrix.repo }}/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C homeserver) && break
9297
done
9398
9499
# Build homeserver image
@@ -113,7 +118,7 @@ jobs:
113118

114119
- run: |
115120
set -o pipefail &&
116-
${{ matrix.env }} go test -v -json -tags "${{ matrix.tags }}" -timeout "${{ matrix.timeout }}" ./tests/... | .ci/scripts/gotestfmt
121+
${{ matrix.env }} go test -v -json -tags "${{ matrix.tags }}" -timeout "${{ matrix.timeout }}" ./tests ./tests/csapi ${{ matrix.packages }} | .ci/scripts/gotestfmt
117122
shell: bash # required for pipefail to be A Thing. pipefail is required to stop gotestfmt swallowing non-zero exit codes
118123
name: Run Complement Tests
119124
env:

‎CONTRIBUTING.md

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# Contributing to Complement
2+
3+
Thank you for taking the time to contribute to Matrix!
4+
5+
This is the repository for Complement, a black box integration testing framework for Matrix homeservers.
6+
7+
## Sign off
8+
9+
We ask that everybody who contributes to this project signs off their contributions, as explained below.
10+
11+
We follow a simple 'inbound=outbound' model for contributions: the act of submitting an 'inbound' contribution means that the contributor agrees to license their contribution under the same terms as the project's overall 'outbound' license - in our case, this is Apache Software License v2 (see [LICENSE](./LICENSE)).
12+
13+
In order to have a concrete record that your contribution is intentional and you agree to license it under the same terms as the project's license, we've adopted the same lightweight approach used by the [Linux Kernel](https://www.kernel.org/doc/html/latest/process/submitting-patches.html), [Docker](https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other projects: the [Developer Certificate of Origin](https://developercertificate.org/) (DCO). This is a simple declaration that you wrote the contribution or otherwise have the right to contribute it to Matrix:
14+
15+
```
16+
Developer Certificate of Origin
17+
Version 1.1
18+
19+
Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
20+
660 York Street, Suite 102,
21+
San Francisco, CA 94110 USA
22+
23+
Everyone is permitted to copy and distribute verbatim copies of this
24+
license document, but changing it is not allowed.
25+
26+
Developer's Certificate of Origin 1.1
27+
28+
By making a contribution to this project, I certify that:
29+
30+
(a) The contribution was created in whole or in part by me and I
31+
have the right to submit it under the open source license
32+
indicated in the file; or
33+
34+
(b) The contribution is based upon previous work that, to the best
35+
of my knowledge, is covered under an appropriate open source
36+
license and I have the right under that license to submit that
37+
work with modifications, whether created in whole or in part
38+
by me, under the same open source license (unless I am
39+
permitted to submit under a different license), as indicated
40+
in the file; or
41+
42+
(c) The contribution was provided directly to me by some other
43+
person who certified (a), (b) or (c) and I have not modified
44+
it.
45+
46+
(d) I understand and agree that this project and the contribution
47+
are public and that a record of the contribution (including all
48+
personal information I submit with it, including my sign-off) is
49+
maintained indefinitely and may be redistributed consistent with
50+
this project or the open source license(s) involved.
51+
```
52+
53+
If you agree to this for your contribution, then all that's needed is to include the line in your commit or pull request comment:
54+
55+
```
56+
Signed-off-by: Your Name <[email protected]>
57+
```
58+
59+
Git allows you to add this signoff automatically when using the `-s` flag to `git commit`, which uses the name and email set in your `user.name` and `user.email` git configs.

‎ENVIRONMENT.md

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
Complement is configured exclusively through the use of environment variables. These variables are described below.
55

66
#### `COMPLEMENT_ALWAYS_PRINT_SERVER_LOGS`
7-
If 1, always prints the Homeserver container logs even on success.
7+
If 1, always prints the Homeserver container logs even on success. When used with COMPLEMENT_ENABLE_DIRTY_RUNS, server logs are only printed once for reused deployments, at the very end of the test suite.
88
- Type: `bool`
99
- Default: 0
1010

@@ -21,6 +21,11 @@ If 1, prints out more verbose logging such as HTTP request/response bodies.
2121
- Type: `bool`
2222
- Default: 0
2323

24+
#### `COMPLEMENT_ENABLE_DIRTY_RUNS`
25+
If 1, eligible tests will be provided with reusable deployments rather than a clean deployment. Eligible tests are tests run with `Deploy(t, numHomeservers)`. If enabled, COMPLEMENT_ALWAYS_PRINT_SERVER_LOGS and COMPLEMENT_POST_TEST_SCRIPT are run exactly once, at the end of all tests in the package. The post test script is run with the test name "COMPLEMENT_ENABLE_DIRTY_RUNS", and failed=false. Enabling dirty runs can greatly speed up tests, at the cost of clear server logs and the chance of tests polluting each other. Tests using `OldDeploy` and blueprints will still have a fresh image for each test. Fresh images can still be desirable e.g user directory tests need a clean homeserver else search results can be polluted, tests which can blacklist a server over federation also need isolated deployments to stop failures impacting other tests. For these reasons, there will always be a way for a test to override this setting and get a dedicated deployment. Eventually, dirty runs will become the default running mode of Complement, with an environment variable to disable this behaviour being added later, once this has stablised.
26+
- Type: `bool`
27+
- Default: 0
28+
2429
#### `COMPLEMENT_HOSTNAME_RUNNING_COMPLEMENT`
2530
The hostname of Complement from the perspective of a Homeserver running inside a container. This can be useful for container runtimes using another hostname to access the host from a container, like Podman that uses `host.containers.internal` instead.
2631
- Type: `string`
@@ -35,7 +40,7 @@ A list of space separated blueprint names to not clean up after running. For exa
3540
- Type: `[]string`
3641

3742
#### `COMPLEMENT_POST_TEST_SCRIPT`
38-
An arbitrary script to execute after a test was executed and before the container is removed. This can be used to extract, for example, server logs or database files. The script is passed the parameters: ContainerID, TestName, TestFailed (true/false)
43+
An arbitrary script to execute after a test was executed and before the container is removed. This can be used to extract, for example, server logs or database files. The script is passed the parameters: ContainerID, TestName, TestFailed (true/false). When combined with COMPLEMENT_ENABLE_DIRTY_RUNS, the script is called exactly once at the end of the test suite, and is called with the TestName of "COMPLEMENT_ENABLE_DIRTY_RUNS" and TestFailed=false.
3944
- Type: `string`
4045
- Default: ""
4146

‎OUT-OF-REPO-TESTS.md

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
## How to run tests out-of-repo
2+
3+
- Make a new go project: `go mod init some.package.name`.
4+
- Get the latest version of Complement: `go get github.com/matrix-org/complement@main`
5+
- Add a tests directory and file: `mkdir ./tests; touch ./tests/something_test.go`
6+
7+
*something_test.go*
8+
```go
9+
package tests
10+
11+
import (
12+
"testing"
13+
14+
"github.com/matrix-org/complement"
15+
"github.com/matrix-org/complement/client"
16+
"github.com/matrix-org/complement/helpers"
17+
"github.com/matrix-org/complement/match"
18+
"github.com/matrix-org/complement/must"
19+
)
20+
21+
func TestCannotKickNonPresentUser(t *testing.T) {
22+
deployment := complement.Deploy(t, 1)
23+
defer deployment.Destroy(t)
24+
25+
alice := deployment.Register(t, "hs1", helpers.RegistrationOpts{})
26+
bob := deployment.Register(t, "hs1", helpers.RegistrationOpts{})
27+
28+
roomID := alice.MustCreateRoom(t, map[string]interface{}{
29+
"preset": "public_chat",
30+
})
31+
32+
resp := alice.Do(t, "POST", []string{"_matrix", "client", "v3", "rooms", roomID, "kick"},
33+
client.WithJSONBody(t, map[string]interface{}{
34+
"user_id": bob.UserID,
35+
"reason": "testing",
36+
}),
37+
)
38+
39+
must.MatchResponse(t, resp, match.HTTPResponse{
40+
StatusCode: 403,
41+
})
42+
}
43+
```
44+
45+
Complement needs to be bootstrapped in when running `go test`. This is doing via a `./tests/main_test.go` file:
46+
```go
47+
package tests
48+
49+
import (
50+
"testing"
51+
52+
"github.com/matrix-org/complement"
53+
)
54+
55+
func TestMain(m *testing.M) {
56+
complement.TestMain(m, "some_namespace_for_these_tests")
57+
}
58+
```
59+
If you are only running these tests and no other Complement tests (e.g the main ones in the Complement repo) *in parallel* (i.e `go test ./complement/tests ./myrepo/tests`) then the namespace value doesn't matter. If you _are_ running other tests in parallel the namespace needs to be unique for all possible Complement test packages, otherwise you will get container conflicts. So don't call it "fed" or "csapi" as they are used by the main Complement tests.
60+
61+
Now set up a `COMPLEMENT_BASE_IMAGE` and run `COMPLEMENT_BASE_IMAGE=homeserver:latest go test -v ./tests`:
62+
```
63+
2023/10/25 14:39:51 config: &{BaseImageURI:homeserver:latest DebugLoggingEnabled:false AlwaysPrintServerLogs:false EnvVarsPropagatePrefix: SpawnHSTimeout:30s KeepBlueprints:[] HostMounts:[] BaseImageURIs:map[] PackageNamespace:oor CACertificate:0x14000cea580 CAPrivateKey:0x14000cf5300 BestEffort:false HostnameRunningComplement:host.docker.internal EnableDirtyRuns:false HSPortBindingIP:127.0.0.1 PostTestScript:}
64+
=== RUN TestCannotKickNonPresentUser
65+
foo_test.go:14: Deploy times: 4.543523667s blueprints, 3.352760416s containers
66+
client.go:621: [CSAPI] POST hs1/_matrix/client/v3/register => 200 OK (16.544958ms)
67+
client.go:621: [CSAPI] POST hs1/_matrix/client/v3/register => 200 OK (13.813708ms)
68+
client.go:621: [CSAPI] POST hs1/_matrix/client/v3/createRoom => 200 OK (56.164792ms)
69+
client.go:621: [CSAPI] POST hs1/_matrix/client/v3/rooms/!ajUaasESMwLZSTpzkq:hs1/kick => 403 Forbidden (19.165125ms)
70+
--- PASS: TestCannotKickNonPresentUser (8.31s)
71+
PASS
72+
ok oor/tests 8.829s
73+
```
74+
75+
76+
NOTE: You currently cannot set up mock federation servers as that package is still internal. You can test CSAPI and deploy >1 HS though.

‎README.md

Lines changed: 13 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@
55

66
Complement is a black box integration testing framework for Matrix homeservers.
77

8+
See also [Complement Crypto](https://github.com/matrix-org/complement-crypto) for E2EE specific testing.
9+
810
## Running
911

10-
You need to have Go and Docker >= 20.10 installed, as well as `libolm3` and `libolm-dev`. Then:
12+
You need to have Go and Docker installed. Complement uses Docker API version 1.45, so your `docker version` must support that. Then:
1113

1214
```
1315
$ COMPLEMENT_BASE_IMAGE=some-matrix/homeserver-impl go test -v ./tests/...
@@ -22,23 +24,6 @@ If you need to pass environment variables to the image under test, you can:
2224
1. define a pass-through prefix with e.g. `COMPLEMENT_SHARE_ENV_PREFIX=PASS_`; then
2325
2. prefix the desired environment variables with that prefix; e.g. `PASS_SYNAPSE_COMPLEMENT_USE_WORKERS=true`.
2426

25-
### Dependencies
26-
27-
Complement supports encryption via `libolm`. You can install `libolm3` on Debian using something like:
28-
```
29-
echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sources.list.d/complement.list && apt-get update && apt-get install -y libolm3 libolm-dev/buster-backports
30-
```
31-
or on Mac:
32-
```
33-
brew install libolm
34-
```
35-
If you are on an Apple Silicon Mac then you'll need to set these environment variables too so Go can find `libolm`:
36-
```
37-
export LIBRARY_PATH=/opt/homebrew/lib
38-
export CPATH=/opt/homebrew/include
39-
export PATH=/opt/homebrew/bin:$PATH
40-
```
41-
4227
### Potential conflict with firewall software
4328

4429
The homeserver in the test image needs to be able to make requests to the mock
@@ -63,26 +48,32 @@ To do so you should:
6348
- `systemctl --user start podman.service` to start the rootless API daemon (can also be enabled).
6449
- `DOCKER_HOST=unix://$XDG_RUNTIME_DIR/podman/podman.sock BUILDAH_FORMAT=docker COMPLEMENT_HOSTNAME_RUNNING_COMPLEMENT=host.containers.internal ...`
6550

51+
If all the networking tests don't seem to pass, it might be because the default rootless network command `pasta` doesn't work in recent versions of Podman (see [this issue](https://github.com/containers/podman/issues/22653)). If that happens to you, consider changing it in Podman's configuration file located at `/etc/containers/containers.conf`:
52+
53+
```
54+
default_rootless_network_cmd = "slirp4netns"
55+
```
56+
6657
Docker image format is needed because OCI format doesn't support the HEALTHCHECK directive unfortunately.
6758

6859
### Running against Dendrite
6960

7061
For instance, for Dendrite:
7162
```
7263
# build a docker image for Dendrite...
73-
$ git clone https://github.com/matrix-org/dendrite
64+
$ git clone https://github.com/element-hq/dendrite
7465
$ (cd dendrite && docker build -t complement-dendrite -f build/scripts/Complement.Dockerfile .)
7566
# ...and test it
7667
$ COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v ./tests/...
7768
```
7869

7970
### Running against Synapse
8071

81-
If you're looking to run Complement against a local dev instance of Synapse, see [`matrix-org/synapse` -> `scripts-dev/complement.sh`](https://github.com/matrix-org/synapse/blob/develop/scripts-dev/complement.sh).
72+
If you're looking to run Complement against a local dev instance of Synapse, see [`element-hq/synapse` -> `scripts-dev/complement.sh`](https://github.com/element-hq/synapse/blob/develop/scripts-dev/complement.sh).
8273

8374
If you want to develop Complement tests while working on a local dev instance
8475
of Synapse, use the
85-
[`scripts-dev/complement.sh`](https://github.com/matrix-org/synapse/blob/develop/scripts-dev/complement.sh)
76+
[`scripts-dev/complement.sh`](https://github.com/element-hq/synapse/blob/develop/scripts-dev/complement.sh)
8677
script and set the `COMPLEMENT_DIR` environment variable to the filepath of
8778
your local Complement checkout. Arguments to `go test` can be supplied as an argument to the script, e.g.:
8879

@@ -123,7 +114,7 @@ from the host to the container. This is set via `COMPLEMENT_HOST_MOUNTS`, on the
123114
For example, for Dendrite on Linux with the default location of `$GOPATH`, do a one-time setup:
124115

125116
```shellsession
126-
$ git clone https://github.com/matrix-org/dendrite ../dendrite
117+
$ git clone https://github.com/element-hq/dendrite ../dendrite
127118
$ (cd ../dendrite && docker build -t complement-dendrite-local -f build/scripts/ComplementLocal.Dockerfile .)
128119
$ mkdir -p ../complement-go-build-cache
129120
$ export COMPLEMENT_BASE_IMAGE=complement-dendrite-local

‎b/blueprints.go

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ var KnownBlueprints = map[string]*Blueprint{
3232
BlueprintOneToOneRoom.Name: &BlueprintOneToOneRoom,
3333
BlueprintPerfManyMessages.Name: &BlueprintPerfManyMessages,
3434
BlueprintPerfManyRooms.Name: &BlueprintPerfManyRooms,
35-
BlueprintPerfE2EERoom.Name: &BlueprintPerfE2EERoom,
3635
}
3736

3837
// Blueprint represents an entire deployment to make.
@@ -64,10 +63,6 @@ type User struct {
6463
AvatarURL string
6564
AccountData []AccountData
6665
DeviceID *string
67-
// Enable end-to end encryption for this user and upload the given
68-
// amount of one-time keys. This requires the DeviceId to be set as
69-
// well.
70-
OneTimeKeys uint
7166
}
7267

7368
type AccountData struct {
@@ -84,12 +79,14 @@ type Room struct {
8479
}
8580

8681
type ApplicationService struct {
87-
ID string
88-
HSToken string
89-
ASToken string
90-
URL string
91-
SenderLocalpart string
92-
RateLimited bool
82+
ID string
83+
HSToken string
84+
ASToken string
85+
URL string
86+
SenderLocalpart string
87+
RateLimited bool
88+
SendEphemeral bool
89+
EnableEncryption bool
9390
}
9491

9592
type Event struct {

0 commit comments

Comments
 (0)