Skip to content

Commit 298e5a2

Browse files
kegsaykegsay
authored
Don't listen on :8448 for the Complement federation server (#289)
* Don't listen on :8448 for the Complement federation server Instead, listen on a random OS-allocated high numbered port then do a switcheroo on the `ServerName` so it reads correctly e.g `host.docker.internal:56185`. This means the server name will be invalid if it is read before `Server.Listen()` is called so we now guard common access points in `Server` which rely on the server name and fail tests if the server is not yet listening when those functions are called. To further guard against misuse of server name whilst it isn't valid, turn it into a private field. * Bind to localhost * Listen on all interfaces again as we need it to listen on Docker interfaces in CI * Experimental support for Complement on ubuntu VM in GHA * Tweak * More tweaking * More tweaks * More * Guessing at this point * Maybe * Modify GOROOT * Set go version before running other complement install commands * Docstrings and BUILDKIT=1 * No need to cd complement-master * Add GOPATH * Install libolm-dev; more docs * Say we're not running under CI * Remove CI flag We don't need it anymore as CI does not run Complement inside Docker * Shadow lint * Fix shadowing
1 parent baf8565 commit 298e5a2

File tree

12 files changed

+128
-180
lines changed

12 files changed

+128
-180
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,18 +31,26 @@ jobs:
3131
tags: msc2836 dendrite_blacklist
3232
default_branch: master
3333

34-
container:
35-
image: matrixdotorg/complement # dockerfiles/ComplementCIBuildkite.Dockerfile
36-
env:
37-
CI: true
38-
DOCKER_BUILDKIT: 1
39-
ports:
40-
- 8448:8448
41-
volumes:
42-
- /var/run/docker.sock:/var/run/docker.sock
43-
4434
steps:
45-
- uses: actions/checkout@v2
35+
- uses: actions/checkout@v2 # Checkout complement
36+
37+
# Env vars are set file a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on env.
38+
# See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
39+
- name: "Set Go Version"
40+
run: |
41+
echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
42+
echo "~/go/bin" >> $GITHUB_PATH
43+
44+
# Similar steps as dockerfiles/ComplementCIBuildkite.Dockerfile but on the host. We need
45+
# to do this so we can _be_ the host when running Complement so we can snaffle all the ports. If
46+
# we run Complement _in_ Docker then we can't -p all high numbered ports which then breaks federation
47+
# servers which listen on random high numbered ports.
48+
- name: "Install Complement Dependencies"
49+
# We don't need to install Go because it is included on the Ubuntu 20.04 image:
50+
# See https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md specifically GOROOT_1_17_X64
51+
run: |
52+
sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
53+
go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
4654
4755
- name: "Checkout corresponding ${{ matrix.homeserver }} branch"
4856
# This is only done for Synapse since Dendrite's docker file pulls in
@@ -80,9 +88,15 @@ jobs:
8088
# built docker image).
8189
if: ${{ matrix.homeserver == 'Synapse' }}
8290
working-directory: homeserver
91+
env:
92+
DOCKER_BUILDKIT: 1
8393

8494
- run: docker build -t homeserver -f dockerfiles/${{ matrix.homeserver }}.Dockerfile dockerfiles/
85-
- run: set -o pipefail && go test -p 2 -v -json -tags "${{ matrix.tags }}" ./tests/... 2>&1 | gotestfmt
95+
- run: |
96+
set -o pipefail &&
97+
go test -p 2 -v -json -tags "${{ matrix.tags }}" ./tests/... 2>&1 | gotestfmt
8698
shell: bash # required for pipefail to be A Thing. pipefail is required to stop gotestfmt swallowing non-zero exit codes
99+
name: Run Complement Tests
87100
env:
88101
COMPLEMENT_BASE_IMAGE: homeserver
102+
DOCKER_BUILDKIT: 1

‎internal/docker/builder.go‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ import (
1717
"context"
1818
"fmt"
1919
"log"
20-
"os"
2120
"strings"
2221
"time"
2322

@@ -39,15 +38,6 @@ var (
3938
HostnameRunningDocker = "localhost"
4039
)
4140

42-
func init() {
43-
if os.Getenv("CI") == "true" {
44-
log.Println("Running under CI: redirecting localhost to docker host on 172.17.0.1")
45-
// this assumes we are running inside docker so they have
46-
// forwarded the docker socket to us and we're in a container.
47-
HostnameRunningDocker = "172.17.0.1"
48-
}
49-
}
50-
5141
const complementLabel = "complement_context"
5242

5343
type Builder struct {

‎internal/docker/volumes.go‎

Lines changed: 11 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,9 @@
11
package docker
22

33
import (
4-
"bufio"
54
"context"
6-
"fmt"
7-
"io/ioutil"
85
"os"
96
"path"
10-
"strings"
117

128
"github.com/docker/docker/api/types/mount"
139
"github.com/docker/docker/api/types/volume"
@@ -31,50 +27,21 @@ type VolumeCA struct {
3127
// Prepare the Certificate Authority volume. This is independent of the homeserver calling Prepare
3228
// hence the contextual string is unused.
3329
func (v *VolumeCA) Prepare(ctx context.Context, docker *client.Client, x string) error {
34-
// TODO: wrap in a lockfile
35-
if os.Getenv("CI") == "true" {
36-
// When in CI, Complement itself is a container with the CA volume mounted at /ca.
37-
// We need to mount this volume to all homeserver containers to synchronize the CA cert.
38-
// This is needed to establish trust among all containers.
39-
40-
containerID := getContainerID()
41-
if containerID == "" {
42-
return fmt.Errorf("failed to get container ID")
43-
}
44-
container, err := docker.ContainerInspect(ctx, containerID)
45-
if err != nil {
46-
return err
47-
}
48-
// Get the volume that matches the destination in our complement container
49-
for i := range container.Mounts {
50-
if container.Mounts[i].Destination == "/ca" {
51-
v.source = container.Mounts[i].Name
52-
v.typ = container.Mounts[i].Type
53-
break
54-
}
55-
}
56-
if v.source == "" {
57-
// We did not find a volume. This container might be created without a volume,
58-
// or CI=true is passed but we are not running in a container.
59-
return fmt.Errorf("CI=true but there is no /ca mounted to Complement's container")
60-
}
61-
} else {
62-
// When not in CI, our CA cert is placed in the current working dir.
63-
// We bind mount this directory to all homeserver containers.
64-
cwd, err := os.Getwd()
30+
// Our CA cert is placed in the current working dir.
31+
// We bind mount this directory to all homeserver containers.
32+
cwd, err := os.Getwd()
33+
if err != nil {
34+
return err
35+
}
36+
caCertificateDirHost := path.Join(cwd, "ca")
37+
if _, err := os.Stat(caCertificateDirHost); os.IsNotExist(err) {
38+
err = os.Mkdir(caCertificateDirHost, 0770)
6539
if err != nil {
6640
return err
6741
}
68-
caCertificateDirHost := path.Join(cwd, "ca")
69-
if _, err := os.Stat(caCertificateDirHost); os.IsNotExist(err) {
70-
err = os.Mkdir(caCertificateDirHost, 0770)
71-
if err != nil {
72-
return err
73-
}
74-
}
75-
v.source = path.Join(cwd, "ca")
76-
v.typ = mount.TypeBind
7742
}
43+
v.source = path.Join(cwd, "ca")
44+
v.typ = mount.TypeBind
7845
return nil
7946
}
8047

@@ -108,60 +75,3 @@ func (v *VolumeAppService) Mount() mount.Mount {
10875
Target: "/appservices",
10976
}
11077
}
111-
112-
func getContainerID() string {
113-
cid, err := getContainerIDViaCPUSet()
114-
if err == nil {
115-
return cid
116-
}
117-
fmt.Printf("failed to get container ID via cpuset, trying alternatives: %s\n", err)
118-
119-
cid, err = getContainerIDViaCGroups()
120-
if err == nil {
121-
return cid
122-
}
123-
124-
fmt.Printf("failed to get container ID via cgroups, out of options: %s\n", err)
125-
return ""
126-
}
127-
128-
func getContainerIDViaCGroups() (string, error) {
129-
file, err := os.Open("/proc/self/cgroup")
130-
if err != nil {
131-
return "", err
132-
}
133-
134-
scanner := bufio.NewScanner(file)
135-
defer file.Close()
136-
137-
scanner.Split(bufio.ScanLines)
138-
for scanner.Scan() {
139-
// Returns entries like this on github actions
140-
// 9:memory:/actions_job/c8d555525bad6cd896c5aa985ef68010be47b1fb321c95547761c8f1a053b86e
141-
line := scanner.Text()
142-
segments := strings.Split(line, "/")
143-
containerID := segments[len(segments)-1]
144-
if containerID == "" || len(containerID) < 64 {
145-
continue
146-
}
147-
return containerID, nil
148-
}
149-
return "", fmt.Errorf("faild to find container id in cgroups")
150-
}
151-
152-
func getContainerIDViaCPUSet() (string, error) {
153-
// /proc/1/cpuset should be /docker/<containerID>
154-
cpuset, err := ioutil.ReadFile("/proc/1/cpuset")
155-
if err != nil {
156-
return "", err
157-
}
158-
if !strings.Contains(string(cpuset), "docker") {
159-
return "", fmt.Errorf("could not identify container ID using /proc/1/cpuset - cpuset=%s", string(cpuset))
160-
}
161-
cpusetList := strings.Split(strings.TrimSpace(string(cpuset)), "/")
162-
containerID := cpusetList[len(cpusetList)-1]
163-
if len(containerID) == 0 {
164-
return "", fmt.Errorf("cpuset missing container ID")
165-
}
166-
return containerID, nil
167-
}

‎internal/federation/handle.go‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import (
1818
func MakeJoinRequestsHandler(s *Server, w http.ResponseWriter, req *http.Request) {
1919
// Check federation signature
2020
fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
21-
req, time.Now(), gomatrixserverlib.ServerName(s.ServerName), s.keyRing,
21+
req, time.Now(), gomatrixserverlib.ServerName(s.serverName), s.keyRing,
2222
)
2323
if fedReq == nil {
2424
w.WriteHeader(errResp.Code)
@@ -74,7 +74,7 @@ func MakeJoinRequestsHandler(s *Server, w http.ResponseWriter, req *http.Request
7474
// HandleMakeSendJoinRequests.
7575
func SendJoinRequestsHandler(s *Server, w http.ResponseWriter, req *http.Request) {
7676
fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
77-
req, time.Now(), gomatrixserverlib.ServerName(s.ServerName), s.keyRing,
77+
req, time.Now(), gomatrixserverlib.ServerName(s.serverName), s.keyRing,
7878
)
7979
if fedReq == nil {
8080
w.WriteHeader(errResp.Code)
@@ -106,9 +106,9 @@ func SendJoinRequestsHandler(s *Server, w http.ResponseWriter, req *http.Request
106106

107107
// return state and auth chain
108108
b, err := json.Marshal(gomatrixserverlib.RespSendJoin{
109+
Origin: gomatrixserverlib.ServerName(s.serverName),
109110
AuthEvents: authEvents,
110111
StateEvents: stateEvents,
111-
Origin: gomatrixserverlib.ServerName(s.ServerName),
112112
})
113113
if err != nil {
114114
w.WriteHeader(500)
@@ -141,7 +141,7 @@ func HandleInviteRequests(inviteCallback func(*gomatrixserverlib.Event)) func(*S
141141
// https://matrix.org/docs/spec/server_server/r0.1.4#put-matrix-federation-v2-invite-roomid-eventid
142142
s.mux.Handle("/_matrix/federation/v2/invite/{roomID}/{eventID}", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
143143
fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
144-
req, time.Now(), gomatrixserverlib.ServerName(s.ServerName), s.keyRing,
144+
req, time.Now(), gomatrixserverlib.ServerName(s.serverName), s.keyRing,
145145
)
146146
if fedReq == nil {
147147
w.WriteHeader(errResp.Code)
@@ -169,7 +169,7 @@ func HandleInviteRequests(inviteCallback func(*gomatrixserverlib.Event)) func(*S
169169
}
170170

171171
// Sign the event before we send it back
172-
signedEvent := inviteRequest.Event().Sign(s.ServerName, s.KeyID, s.Priv)
172+
signedEvent := inviteRequest.Event().Sign(s.serverName, s.KeyID, s.Priv)
173173

174174
// Send the response
175175
res := map[string]interface{}{
@@ -195,7 +195,7 @@ func HandleDirectoryLookups() func(*Server) {
195195
b, err := json.Marshal(gomatrixserverlib.RespDirectory{
196196
RoomID: roomID,
197197
Servers: []gomatrixserverlib.ServerName{
198-
gomatrixserverlib.ServerName(s.ServerName),
198+
gomatrixserverlib.ServerName(s.serverName),
199199
},
200200
})
201201
if err != nil {
@@ -235,7 +235,7 @@ func HandleEventRequests() func(*Server) {
235235
}
236236

237237
txn := gomatrixserverlib.Transaction{
238-
Origin: gomatrixserverlib.ServerName(srv.ServerName),
238+
Origin: gomatrixserverlib.ServerName(srv.serverName),
239239
OriginServerTS: gomatrixserverlib.AsTimestamp(time.Now()),
240240
PDUs: []json.RawMessage{
241241
event.JSON(),
@@ -259,7 +259,7 @@ func HandleKeyRequests() func(*Server) {
259259
keymux := srv.mux.PathPrefix("/_matrix/key/v2").Subrouter()
260260
keyFn := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
261261
k := gomatrixserverlib.ServerKeys{}
262-
k.ServerName = gomatrixserverlib.ServerName(srv.ServerName)
262+
k.ServerName = gomatrixserverlib.ServerName(srv.serverName)
263263
publicKey := srv.Priv.Public().(ed25519.PublicKey)
264264
k.VerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.VerifyKey{
265265
srv.KeyID: {
@@ -276,7 +276,7 @@ func HandleKeyRequests() func(*Server) {
276276
}
277277

278278
k.Raw, err = gomatrixserverlib.SignJSON(
279-
string(srv.ServerName), srv.KeyID, srv.Priv, toSign,
279+
string(srv.serverName), srv.KeyID, srv.Priv, toSign,
280280
)
281281
if err != nil {
282282
w.WriteHeader(500)
@@ -304,9 +304,9 @@ func HandleMediaRequests(mediaIds map[string]func(w http.ResponseWriter)) func(*
304304
origin := vars["origin"]
305305
mediaId := vars["mediaId"]
306306

307-
if origin != srv.ServerName {
307+
if origin != srv.serverName {
308308
w.WriteHeader(400)
309-
w.Write([]byte("complement: Invalid Origin; Expected " + srv.ServerName))
309+
w.Write([]byte("complement: Invalid Origin; Expected " + srv.serverName))
310310
return
311311
}
312312

@@ -338,7 +338,7 @@ func HandleTransactionRequests(pduCallback func(*gomatrixserverlib.Event), eduCa
338338

339339
// Check federation signature
340340
fedReq, errResp := gomatrixserverlib.VerifyHTTPRequest(
341-
req, time.Now(), gomatrixserverlib.ServerName(srv.ServerName), srv.keyRing,
341+
req, time.Now(), gomatrixserverlib.ServerName(srv.serverName), srv.keyRing,
342342
)
343343
if fedReq == nil {
344344
log.Printf(

0 commit comments

Comments
 (0)