Merge branch 'main' into s7evink/eventsize

Author: kegsay

.github/workflows/ci.yaml

@@ -16,9 +16,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3 # Checkout complement
       - uses: actions/setup-go@v4
-      - name: "Install Complement Dependencies"
-        run: |
-          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
+        with:
+          go-version-file: go.mod
       - name: "Run internal Complement tests"
         run: |
           go test ./internal/...
@@ -41,26 +40,32 @@ jobs:
       matrix:
         include:
           - homeserver: Synapse
-            tags: synapse_blacklist msc3083 msc3787 msc3874 faster_joins
-            env: "COMPLEMENT_SHARE_ENV_PREFIX=PASS_ PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite"
+            repo: element-hq/synapse
+            tags: synapse_blacklist
+            packages: ./tests/msc3874 ./tests/msc3902
+            env: "COMPLEMENT_ENABLE_DIRTY_RUNS=1 COMPLEMENT_SHARE_ENV_PREFIX=PASS_ PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite"
             timeout: 20m
 
           - homeserver: Dendrite
+            repo: element-hq/dendrite
             tags: dendrite_blacklist
-            env: ""
+            packages: ""
+            env: "COMPLEMENT_ENABLE_DIRTY_RUNS=1"
             timeout: 10m
 
     steps:
       - uses: actions/checkout@v3 # Checkout complement
 
       - uses: actions/setup-go@v4
+        with:
+          go-version-file: go.mod
+
       # Similar steps as dockerfiles/ComplementCIBuildkite.Dockerfile but on the host. We need
       # to do this so we can _be_ the host when running Complement so we can snaffle all the ports. If
       # we run Complement _in_ Docker then we can't -p all high numbered ports which then breaks federation
       # servers which listen on random high numbered ports.
       - name: "Install Complement Dependencies"
         run: |
-          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
           go install -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
           mkdir .gotestfmt/github -p
           cp .ci/complement_package.gotpl .gotestfmt/github/package.gotpl
@@ -88,7 +93,7 @@ jobs:
                 continue
                 ;;
             esac
-            (wget -O - "https://github.com/matrix-org/${{ matrix.homeserver }}/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C homeserver) && break
+            (wget -O - "https://github.com/${{ matrix.repo }}/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C homeserver) && break
           done
 
       # Build homeserver image
@@ -113,7 +118,7 @@ jobs:
 
       - run: |
           set -o pipefail &&
-          ${{ matrix.env }} go test -v -json -tags "${{ matrix.tags }}" -timeout "${{ matrix.timeout }}" ./tests/... | .ci/scripts/gotestfmt
+          ${{ matrix.env }} go test -v -json -tags "${{ matrix.tags }}" -timeout "${{ matrix.timeout }}" ./tests ./tests/csapi ${{ matrix.packages }} | .ci/scripts/gotestfmt
         shell: bash # required for pipefail to be A Thing. pipefail is required to stop gotestfmt swallowing non-zero exit codes
         name: Run Complement Tests
         env:

CONTRIBUTING.md

@@ -0,0 +1,59 @@
+# Contributing to Complement
+
+Thank you for taking the time to contribute to Matrix!
+
+This is the repository for Complement, a black box integration testing framework for Matrix homeservers.
+
+## Sign off
+
+We ask that everybody who contributes to this project signs off their contributions, as explained below.
+
+We follow a simple 'inbound=outbound' model for contributions: the act of submitting an 'inbound' contribution means that the contributor agrees to license their contribution under the same terms as the project's overall 'outbound' license - in our case, this is Apache Software License v2 (see [LICENSE](./LICENSE)).
+
+In order to have a concrete record that your contribution is intentional and you agree to license it under the same terms as the project's license, we've adopted the same lightweight approach used by the [Linux Kernel](https://www.kernel.org/doc/html/latest/process/submitting-patches.html), [Docker](https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other projects: the [Developer Certificate of Origin](https://developercertificate.org/) (DCO). This is a simple declaration that you wrote the contribution or otherwise have the right to contribute it to Matrix:
+
+```
+Developer Certificate of Origin
+Version 1.1
+
+Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+660 York Street, Suite 102,
+San Francisco, CA 94110 USA
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+Developer's Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the open source license
+    indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the same open source license (unless I am
+    permitted to submit under a different license), as indicated
+    in the file; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a), (b) or (c) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the contribution
+    are public and that a record of the contribution (including all
+    personal information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed consistent with
+    this project or the open source license(s) involved.
+```
+
+If you agree to this for your contribution, then all that's needed is to include the line in your commit or pull request comment:
+
+```
+Signed-off-by: Your Name <[email protected]>
+```
+
+Git allows you to add this signoff automatically when using the `-s` flag to `git commit`, which uses the name and email set in your `user.name` and `user.email` git configs.

ENVIRONMENT.md

@@ -4,7 +4,7 @@
 Complement is configured exclusively through the use of environment variables. These variables are described below.
 
 #### `COMPLEMENT_ALWAYS_PRINT_SERVER_LOGS`
-If 1, always prints the Homeserver container logs even on success.  
+If 1, always prints the Homeserver container logs even on success. When used with COMPLEMENT_ENABLE_DIRTY_RUNS, server logs are only printed once for reused deployments, at the very end of the test suite.  
 - Type: `bool`
 - Default: 0
 
@@ -21,6 +21,11 @@ If 1, prints out more verbose logging such as HTTP request/response bodies.
 - Type: `bool`
 - Default: 0
 
+#### `COMPLEMENT_ENABLE_DIRTY_RUNS`
+If 1, eligible tests will be provided with reusable deployments rather than a clean deployment. Eligible tests are tests run with `Deploy(t, numHomeservers)`. If enabled, COMPLEMENT_ALWAYS_PRINT_SERVER_LOGS and COMPLEMENT_POST_TEST_SCRIPT are run exactly once, at the end of all tests in the package. The post test script is run with the test name "COMPLEMENT_ENABLE_DIRTY_RUNS", and failed=false.  Enabling dirty runs can greatly speed up tests, at the cost of clear server logs and the chance of tests polluting each other. Tests using `OldDeploy` and blueprints will still have a fresh image for each test. Fresh images can still be desirable e.g user directory tests need a clean homeserver else search results can be polluted, tests which can blacklist a server over federation also need isolated deployments to stop failures impacting other tests. For these reasons, there will always be a way for a test to override this setting and get a dedicated deployment.  Eventually, dirty runs will become the default running mode of Complement, with an environment variable to disable this behaviour being added later, once this has stablised.  
+- Type: `bool`
+- Default: 0
+
 #### `COMPLEMENT_HOSTNAME_RUNNING_COMPLEMENT`
 The hostname of Complement from the perspective of a Homeserver running inside a container. This can be useful for container runtimes using another hostname to access the host from a container, like Podman that uses `host.containers.internal` instead.  
 - Type: `string`
@@ -35,7 +40,7 @@ A list of space separated blueprint names to not clean up after running. For exa
 - Type: `[]string`
 
 #### `COMPLEMENT_POST_TEST_SCRIPT`
-An arbitrary script to execute after a test was executed and before the container is removed. This can be used to extract, for example, server logs or database files. The script is passed the parameters: ContainerID, TestName, TestFailed (true/false)  
+An arbitrary script to execute after a test was executed and before the container is removed. This can be used to extract, for example, server logs or database files. The script is passed the parameters: ContainerID, TestName, TestFailed (true/false). When combined with COMPLEMENT_ENABLE_DIRTY_RUNS, the script is called exactly once at the end of the test suite, and is called with the TestName of "COMPLEMENT_ENABLE_DIRTY_RUNS" and TestFailed=false.  
 - Type: `string`
 - Default: ""
 

OUT-OF-REPO-TESTS.md

@@ -0,0 +1,76 @@
+## How to run tests out-of-repo
+
+- Make a new go project: `go mod init some.package.name`.
+- Get the latest version of Complement: `go get github.com/matrix-org/complement@main`
+- Add a tests directory and file: `mkdir ./tests; touch ./tests/something_test.go`
+
+*something_test.go*
+```go
+package tests
+
+import (
+	"testing"
+
+	"github.com/matrix-org/complement"
+	"github.com/matrix-org/complement/client"
+	"github.com/matrix-org/complement/helpers"
+	"github.com/matrix-org/complement/match"
+	"github.com/matrix-org/complement/must"
+)
+
+func TestCannotKickNonPresentUser(t *testing.T) {
+	deployment := complement.Deploy(t, 1)
+	defer deployment.Destroy(t)
+
+	alice := deployment.Register(t, "hs1", helpers.RegistrationOpts{})
+	bob := deployment.Register(t, "hs1", helpers.RegistrationOpts{})
+
+	roomID := alice.MustCreateRoom(t, map[string]interface{}{
+		"preset": "public_chat",
+	})
+
+	resp := alice.Do(t, "POST", []string{"_matrix", "client", "v3", "rooms", roomID, "kick"},
+		client.WithJSONBody(t, map[string]interface{}{
+			"user_id": bob.UserID,
+			"reason":  "testing",
+		}),
+	)
+
+	must.MatchResponse(t, resp, match.HTTPResponse{
+		StatusCode: 403,
+	})
+}
+```
+
+Complement needs to be bootstrapped in when running `go test`. This is doing via a `./tests/main_test.go` file:
+```go
+package tests
+
+import (
+	"testing"
+
+	"github.com/matrix-org/complement"
+)
+
+func TestMain(m *testing.M) {
+	complement.TestMain(m, "some_namespace_for_these_tests")
+}
+```
+If you are only running these tests and no other Complement tests (e.g the main ones in the Complement repo) *in parallel* (i.e `go test ./complement/tests ./myrepo/tests`) then the namespace value doesn't matter. If you _are_ running other tests in parallel the namespace needs to be unique for all possible Complement test packages, otherwise you will get container conflicts. So don't call it "fed" or "csapi" as they are used by the main Complement tests.
+
+Now set up a `COMPLEMENT_BASE_IMAGE` and run `COMPLEMENT_BASE_IMAGE=homeserver:latest go test -v ./tests`:
+```
+2023/10/25 14:39:51 config: &{BaseImageURI:homeserver:latest DebugLoggingEnabled:false AlwaysPrintServerLogs:false EnvVarsPropagatePrefix: SpawnHSTimeout:30s KeepBlueprints:[] HostMounts:[] BaseImageURIs:map[] PackageNamespace:oor CACertificate:0x14000cea580 CAPrivateKey:0x14000cf5300 BestEffort:false HostnameRunningComplement:host.docker.internal EnableDirtyRuns:false HSPortBindingIP:127.0.0.1 PostTestScript:}
+=== RUN   TestCannotKickNonPresentUser
+    foo_test.go:14: Deploy times: 4.543523667s blueprints, 3.352760416s containers
+    client.go:621: [CSAPI] POST hs1/_matrix/client/v3/register => 200 OK (16.544958ms)
+    client.go:621: [CSAPI] POST hs1/_matrix/client/v3/register => 200 OK (13.813708ms)
+    client.go:621: [CSAPI] POST hs1/_matrix/client/v3/createRoom => 200 OK (56.164792ms)
+    client.go:621: [CSAPI] POST hs1/_matrix/client/v3/rooms/!ajUaasESMwLZSTpzkq:hs1/kick => 403 Forbidden (19.165125ms)
+--- PASS: TestCannotKickNonPresentUser (8.31s)
+PASS
+ok  	oor/tests	8.829s
+```
+
+
+NOTE: You currently cannot set up mock federation servers as that package is still internal. You can test CSAPI and deploy >1 HS though.

README.md

@@ -5,9 +5,11 @@
 
 Complement is a black box integration testing framework for Matrix homeservers.
 
+See also [Complement Crypto](https://github.com/matrix-org/complement-crypto) for E2EE specific testing.
+
 ## Running
 
-You need to have Go and Docker >= 20.10 installed, as well as `libolm3` and `libolm-dev`. Then:
+You need to have Go and Docker installed. Complement uses Docker API version 1.45, so your `docker version` must support that. Then:
 
 ```
 $ COMPLEMENT_BASE_IMAGE=some-matrix/homeserver-impl go test -v ./tests/...
@@ -22,23 +24,6 @@ If you need to pass environment variables to the image under test, you can:
 1. define a pass-through prefix with e.g. `COMPLEMENT_SHARE_ENV_PREFIX=PASS_`; then
 2. prefix the desired environment variables with that prefix; e.g. `PASS_SYNAPSE_COMPLEMENT_USE_WORKERS=true`.
 
-### Dependencies
-
-Complement supports encryption via `libolm`. You can install `libolm3` on Debian using something like:
-```
-echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sources.list.d/complement.list && apt-get update && apt-get install -y libolm3 libolm-dev/buster-backports
-```
-or on Mac:
-```
-brew install libolm
-```
-If you are on an Apple Silicon Mac then you'll need to set these environment variables too so Go can find `libolm`:
-```
-export LIBRARY_PATH=/opt/homebrew/lib
-export CPATH=/opt/homebrew/include
-export PATH=/opt/homebrew/bin:$PATH
-```
-
 ### Potential conflict with firewall software
 
 The homeserver in the test image needs to be able to make requests to the mock
@@ -63,26 +48,32 @@ To do so you should:
 - `systemctl --user start podman.service` to start the rootless API daemon (can also be enabled).
 - `DOCKER_HOST=unix://$XDG_RUNTIME_DIR/podman/podman.sock BUILDAH_FORMAT=docker COMPLEMENT_HOSTNAME_RUNNING_COMPLEMENT=host.containers.internal ...`
 
+If all the networking tests don't seem to pass, it might be because the default rootless network command `pasta` doesn't work in recent versions of Podman (see [this issue](https://github.com/containers/podman/issues/22653)). If that happens to you, consider changing it in Podman's configuration file located at `/etc/containers/containers.conf`:
+
+```
+default_rootless_network_cmd = "slirp4netns"
+```
+
 Docker image format is needed because OCI format doesn't support the HEALTHCHECK directive unfortunately.
 
 ### Running against Dendrite
 
 For instance, for Dendrite:
 ```
 # build a docker image for Dendrite...
-$ git clone https://github.com/matrix-org/dendrite
+$ git clone https://github.com/element-hq/dendrite
 $ (cd dendrite && docker build -t complement-dendrite -f build/scripts/Complement.Dockerfile .)
 # ...and test it
 $ COMPLEMENT_BASE_IMAGE=complement-dendrite:latest go test -v ./tests/...
 ```
 
 ### Running against Synapse
 
-If you're looking to run Complement against a local dev instance of Synapse, see [`matrix-org/synapse` -> `scripts-dev/complement.sh`](https://github.com/matrix-org/synapse/blob/develop/scripts-dev/complement.sh).
+If you're looking to run Complement against a local dev instance of Synapse, see [`element-hq/synapse` -> `scripts-dev/complement.sh`](https://github.com/element-hq/synapse/blob/develop/scripts-dev/complement.sh).
 
 If you want to develop Complement tests while working on a local dev instance
 of Synapse, use the
-[`scripts-dev/complement.sh`](https://github.com/matrix-org/synapse/blob/develop/scripts-dev/complement.sh)
+[`scripts-dev/complement.sh`](https://github.com/element-hq/synapse/blob/develop/scripts-dev/complement.sh)
 script and set the `COMPLEMENT_DIR` environment variable to the filepath of
 your local Complement checkout. Arguments to `go test` can be supplied as an argument to the script, e.g.:
 
@@ -123,7 +114,7 @@ from the host to the container. This is set via `COMPLEMENT_HOST_MOUNTS`, on the
 For example, for Dendrite on Linux with the default location of `$GOPATH`, do a one-time setup:
 
 ```shellsession
-$ git clone https://github.com/matrix-org/dendrite ../dendrite
+$ git clone https://github.com/element-hq/dendrite ../dendrite
 $ (cd ../dendrite && docker build -t complement-dendrite-local -f build/scripts/ComplementLocal.Dockerfile .)
 $ mkdir -p ../complement-go-build-cache
 $ export COMPLEMENT_BASE_IMAGE=complement-dendrite-local

b/blueprints.go

@@ -32,7 +32,6 @@ var KnownBlueprints = map[string]*Blueprint{
 	BlueprintOneToOneRoom.Name:                &BlueprintOneToOneRoom,
 	BlueprintPerfManyMessages.Name:            &BlueprintPerfManyMessages,
 	BlueprintPerfManyRooms.Name:               &BlueprintPerfManyRooms,
-	BlueprintPerfE2EERoom.Name:                &BlueprintPerfE2EERoom,
 }
 
 // Blueprint represents an entire deployment to make.
@@ -64,10 +63,6 @@ type User struct {
 	AvatarURL   string
 	AccountData []AccountData
 	DeviceID    *string
-	// Enable end-to end encryption for this user and upload the given
-	// amount of one-time keys. This requires the DeviceId to be set as
-	// well.
-	OneTimeKeys uint
 }
 
 type AccountData struct {
@@ -84,12 +79,14 @@ type Room struct {
 }
 
 type ApplicationService struct {
-	ID              string
-	HSToken         string
-	ASToken         string
-	URL             string
-	SenderLocalpart string
-	RateLimited     bool
+	ID               string
+	HSToken          string
+	ASToken          string
+	URL              string
+	SenderLocalpart  string
+	RateLimited      bool
+	SendEphemeral    bool
+	EnableEncryption bool
 }
 
 type Event struct {