diff --git a/.github/workflows/docker-famedly.yml b/.github/workflows/docker-famedly.yml new file mode 100644 index 0000000000..24a12012e0 --- /dev/null +++ b/.github/workflows/docker-famedly.yml @@ -0,0 +1,20 @@ +--- +name: Docker + +on: + push: + tags: [ 'v*.*.*_*' ] + +jobs: + docker: + uses: famedly/github-workflows/.github/workflows/docker.yml@6da23b565deec84c38ad29b0499479b86d597ce4 + with: + push: ${{ github.event_name != 'pull_request' }} # Always build, don't publish on pull requests + registry_user: famedly-ci + registry: docker-oss.nexus.famedly.de + image_name: synapse + file: docker/Dockerfile + tags: | + type=match,group=1,pattern=(v\d+.\d+.\d+)_\d+ + type=match,group=1,pattern=(v\d+.\d+.\d+_\d+) + secrets: inherit diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000..f0ee9c9709 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,101 @@ +image: alpine + +default: + tags: + - famedly + - docker + +stages: + - test + - build + +.docker-template: + image: docker:latest + stage: build + variables: + DOCKER_BUILDKIT: 1 + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + +lint+mypy+test: + stage: test + image: docker.io/python:3.9-slim + script: + - apt-get update && apt-get install -y git build-essential libffi-dev libjpeg-dev libpq-dev libssl-dev libwebp-dev libxml++2.6-dev libxslt1-dev zlib1g-dev curl + - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal --component clippy --component rustfmt + - source "$HOME/.cargo/env" + - pip install poetry + - poetry install --extras all --no-interaction --sync -vvv + - sed -i -e 's/python -m black/python -m black --check --diff/' ./scripts-dev/lint.sh + - poetry run ./scripts-dev/lint.sh + - poetry run trial -j"$(nproc)" tests + +complement: + image: deb11-docker.qcow2 + stage: test + tags: + - famedly + - libvirt + - generic + variables: + COMPLEMENT_REF: main + before_script: + - sudo bash -c "echo 'deb http://deb.debian.org/debian bullseye-backports main' > /etc/apt/sources.list.d/backports.list" + - sudo apt-get -y update --allow-releaseinfo-change + - sudo apt-get -y install libolm-dev golang-go/bullseye-backports golang-src/bullseye-backports wget g++ bash + - curl -LJO "https://gitlab-runner-downloads.s3.amazonaws.com/latest/deb/gitlab-runner_amd64.deb" + - sudo dpkg -i gitlab-runner_amd64.deb + script: + - go install gotest.tools/gotestsum@latest + - export PATH="$PATH:$HOME/go/bin" + - sed -i -e 's/,msc2716//' -e 's|go test -v|gotestsum --junitfile report.xml --format standard-verbose -- |' ./scripts-dev/complement.sh + - ./scripts-dev/complement.sh + allow_failure: true + artifacts: + when: always + reports: + junit: complement-master/report.xml + +sytest: + extends: .docker-template + stage: test + before_script: + - apk add curl perl perl-utils make perl-xml-generator + script: + - mkdir logs + - docker run -i -e SYTEST_BRANCH="master" -v $(pwd)/logs:/logs -v $(pwd):/src:ro matrixdotorg/sytest-synapse:buster + after_script: + - curl -LOJ https://raw.githubusercontent.com/matrix-org/sytest/b4f61a88af44fe5850bddac4e170ca1f4e3be79a/tap-to-junit-xml.pl + - perl tap-to-junit-xml.pl --input logs/results.tap --output report.xml --puretap + artifacts: + when: always + reports: + junit: report.xml + +docker-release: + extends: .docker-template + rules: + - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+_\d+$/' + script: + - docker build --pull -t "${CI_REGISTRY_IMAGE}:latest" -t "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}" -t "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG%_*}" -f docker/Dockerfile . + - docker push "${CI_REGISTRY_IMAGE}:latest" + - docker push "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}" + - docker push "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG%_*}" + +docker-tags: + extends: .docker-template + rules: + - if: '$CI_COMMIT_TAG && $CI_COMMIT_TAG !~ /^v\d+\.\d+\.\d+_\d+$/' + script: + - docker build --pull -t "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}" -f docker/Dockerfile . + - docker push "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}" + +docker-branches: + extends: .docker-template + rules: + - if: $CI_COMMIT_BRANCH + script: + - docker build --pull -t "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG}" -f docker/Dockerfile . + - docker push "${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG}" diff --git a/README.rst b/README.rst index d5625afe8f..3cfecb7529 100644 --- a/README.rst +++ b/README.rst @@ -27,6 +27,30 @@ SLAs. ESS can be used to support any Matrix-based frontend client. .. contents:: +Rebasing this fork +================== + +TL;DR: There's a `./make_release.sh` script which does the things below. +It currently doesn't handle rebase conflicts gracefully yet. + +This is the Famedly Fork of synapse. It applies a few patches, which need to +be rebased upon every synapse release. To do this, the following workflow is used: + +- Checkout `master` of the fork, then `fetch -a` from the upstream + +- Rebase all commits (from our master) upon `upstream/master`: `git rebase upstream/master` + +- Switch to the `release-vM.m.f` branch (comes from upstream), and merge the + master into it using `git merge --ff-only master`. Then push the `release-*` + branch to the famedly-remote. + +- The CI is configured in a way that creating a tag on the `release-`-branch + will create a new release. The tag needs to have the form `v$originalSynapseVersion_$count`, + so `v1.29.0_1`, `v1.29.0_2` and so on - as content, we suggest `v$synapseVersion - $date`. + If we change our patchset after we already released a version of synapse, we force-push to + the `release-` branch and increase the counter and push a new tag. + + 🛠️ Installing and configuration =============================== diff --git a/docker/Dockerfile b/docker/Dockerfile index 1da196b12e..c648c4c220 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -134,6 +134,17 @@ COPY --from=requirements /synapse/requirements.txt /synapse/ RUN --mount=type=cache,target=/root/.cache/pip \ pip install --prefix="/install" --no-deps --no-warn-script-location -r /synapse/requirements.txt +# Install famedly required addons + +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install setuptools \ + && pip install --prefix="/install" --no-warn-script-location synapse-token-authenticator==0.6.0 \ + && pip install --prefix="/install" --no-warn-script-location synapse-s3-storage-provider \ + && pip install --prefix="/install" --no-warn-script-location synapse-auto-accept-invite \ + && pip install --prefix="/install" --no-warn-script-location synapse-invite-checker==0.2.0 \ + && pip install --prefix="/install" --no-warn-script-location git+https://github.com/famedly/synapse-invite-policies.git@main \ + && pip install --prefix="/install" --no-warn-script-location git+https://github.com/famedly/synapse-domain-rule-checker.git@main + # Copy over the rest of the synapse source code. COPY synapse /synapse/synapse/ COPY rust /synapse/rust/ diff --git a/docs/modules/spam_checker_callbacks.md b/docs/modules/spam_checker_callbacks.md index ffdfe6082e..bbe7c0a385 100644 --- a/docs/modules/spam_checker_callbacks.md +++ b/docs/modules/spam_checker_callbacks.md @@ -12,21 +12,22 @@ The available spam checker callbacks are: _First introduced in Synapse v1.37.0_ -_Changed in Synapse v1.60.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean or a string is now deprecated._ +_Changed in Synapse v1.60.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean or a string is now deprecated._ ```python async def check_event_for_spam(event: "synapse.module_api.EventBase") -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", str, bool] ``` Called when receiving an event from a client or via federation. The callback must return one of: - - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still - decide to reject it. - - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case - of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. - - (deprecated) a non-`Codes` `str` to reject the operation and specify an error message. Note that clients - typically will not localize the error message to the user's preferred locale. - - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. - - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. + +- `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still + decide to reject it. +- `synapse.module_api.errors.Codes` to reject the operation with an error code. In case + of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. +- (deprecated) a non-`Codes` `str` to reject the operation and specify an error message. Note that clients + typically will not localize the error message to the user's preferred locale. +- (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. +- (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. If multiple modules implement this callback, they will be considered in order. If a callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one. @@ -38,7 +39,7 @@ this callback. _First introduced in Synapse v1.37.0_ -_Changed in Synapse v1.61.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ +_Changed in Synapse v1.61.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ ```python async def user_may_join_room(user: str, room: str, is_invited: bool) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool] @@ -53,12 +54,13 @@ This callback isn't called if the join is performed by a server administrator, o context of a room creation. The callback must return one of: - - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still - decide to reject it. - - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case - of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. - - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. - - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. + +- `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still + decide to reject it. +- `synapse.module_api.errors.Codes` to reject the operation with an error code. In case + of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. +- (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. +- (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. If multiple modules implement this callback, they will be considered in order. If a callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one. @@ -70,7 +72,7 @@ this callback. _First introduced in Synapse v1.37.0_ -_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ +_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ ```python async def user_may_invite(inviter: str, invitee: str, room_id: str) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool] @@ -79,15 +81,22 @@ async def user_may_invite(inviter: str, invitee: str, room_id: str) -> Union["sy Called when processing an invitation. Both inviter and invitee are represented by their Matrix user ID (e.g. `@alice:example.com`). +The callback might be invoked multiple times if it is run during room creation. +The first call will be before the room is created with an **empty** `room_id`. +The second call will be after the room is created and when the `room_id` is +already known. If an invite is rejected during the second invocation, the room +is still created, but some invites will be missing and an error returned to the +client. The callback must return one of: - - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still - decide to reject it. - - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case - of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. - - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. - - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. +- `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still + decide to reject it. +- `synapse.module_api.errors.Codes` to reject the operation with an error code. In case + of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. + +- (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. +- (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. If multiple modules implement this callback, they will be considered in order. If a callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one. @@ -95,12 +104,11 @@ The value of the first callback that does not return `synapse.module_api.NOT_SPA be used. If this happens, Synapse will not call any of the subsequent implementations of this callback. - ### `user_may_send_3pid_invite` _First introduced in Synapse v1.45.0_ -_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ +_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ ```python async def user_may_send_3pid_invite( @@ -112,7 +120,7 @@ async def user_may_send_3pid_invite( ``` Called when processing an invitation using a third-party identifier (also called a 3PID, -e.g. an email address or a phone number). +e.g. an email address or a phone number). The inviter is represented by their Matrix user ID (e.g. `@alice:example.com`), and the invitee is represented by its medium (e.g. "email") and its address @@ -135,13 +143,14 @@ await user_may_send_3pid_invite( [`user_may_invite`](#user_may_invite) will be used instead. The callback must return one of: - - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still - decide to reject it. - - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case - of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. - - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. - - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. +- `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still + decide to reject it. +- `synapse.module_api.errors.Codes` to reject the operation with an error code. In case + of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. + +- (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. +- (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. If multiple modules implement this callback, they will be considered in order. If a callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one. @@ -149,12 +158,11 @@ The value of the first callback that does not return `synapse.module_api.NOT_SPA be used. If this happens, Synapse will not call any of the subsequent implementations of this callback. - ### `user_may_create_room` _First introduced in Synapse v1.37.0_ -_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ +_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ ```python async def user_may_create_room(user_id: str) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool] @@ -163,13 +171,14 @@ async def user_may_create_room(user_id: str) -> Union["synapse.module_api.NOT_SP Called when processing a room creation request. The callback must return one of: - - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still - decide to reject it. - - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case - of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. - - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. - - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. +- `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still + decide to reject it. +- `synapse.module_api.errors.Codes` to reject the operation with an error code. In case + of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. + +- (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. +- (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. If multiple modules implement this callback, they will be considered in order. If a callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one. @@ -177,13 +186,11 @@ The value of the first callback that does not return `synapse.module_api.NOT_SPA be used. If this happens, Synapse will not call any of the subsequent implementations of this callback. - - ### `user_may_create_room_alias` _First introduced in Synapse v1.37.0_ -_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ +_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ ```python async def user_may_create_room_alias(user_id: str, room_alias: "synapse.module_api.RoomAlias") -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool] @@ -192,13 +199,14 @@ async def user_may_create_room_alias(user_id: str, room_alias: "synapse.module_a Called when trying to associate an alias with an existing room. The callback must return one of: - - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still - decide to reject it. - - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case - of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. - - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. - - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. +- `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still + decide to reject it. +- `synapse.module_api.errors.Codes` to reject the operation with an error code. In case + of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. + +- (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. +- (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. If multiple modules implement this callback, they will be considered in order. If a callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one. @@ -206,13 +214,11 @@ The value of the first callback that does not return `synapse.module_api.NOT_SPA be used. If this happens, Synapse will not call any of the subsequent implementations of this callback. - - ### `user_may_publish_room` _First introduced in Synapse v1.37.0_ -_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ +_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ ```python async def user_may_publish_room(user_id: str, room_id: str) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool] @@ -221,13 +227,14 @@ async def user_may_publish_room(user_id: str, room_id: str) -> Union["synapse.mo Called when trying to publish a room to the homeserver's public rooms directory. The callback must return one of: - - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still - decide to reject it. - - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case - of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. - - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. - - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. +- `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still + decide to reject it. +- `synapse.module_api.errors.Codes` to reject the operation with an error code. In case + of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. + +- (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. +- (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. If multiple modules implement this callback, they will be considered in order. If a callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one. @@ -235,8 +242,6 @@ The value of the first callback that does not return `synapse.module_api.NOT_SPA be used. If this happens, Synapse will not call any of the subsequent implementations of this callback. - - ### `check_username_for_spam` _First introduced in Synapse v1.37.0_ @@ -246,16 +251,16 @@ async def check_username_for_spam(user_profile: synapse.module_api.UserProfile) ``` Called when computing search results in the user directory. The module must return a -`bool` indicating whether the given user should be excluded from user directory -searches. Return `True` to indicate that the user is spammy and exclude them from +`bool` indicating whether the given user should be excluded from user directory +searches. Return `True` to indicate that the user is spammy and exclude them from search results; otherwise return `False`. The profile is represented as a dictionary with the following keys: -* `user_id: str`. The Matrix ID for this user. -* `display_name: Optional[str]`. The user's display name, or `None` if this user +- `user_id: str`. The Matrix ID for this user. +- `display_name: Optional[str]`. The user's display name, or `None` if this user has not set a display name. -* `avatar_url: Optional[str]`. The `mxc://` URL to the user's avatar, or `None` +- `avatar_url: Optional[str]`. The `mxc://` URL to the user's avatar, or `None` if this user has not set an avatar. The module is given a copy of the original dictionary, so modifying it from within the @@ -285,13 +290,13 @@ may be allowed to register but will be shadow banned. The arguments passed to this callback are: -* `email_threepid`: The email address used for registering, if any. -* `username`: The username the user would like to register. Can be `None`, meaning that +- `email_threepid`: The email address used for registering, if any. +- `username`: The username the user would like to register. Can be `None`, meaning that Synapse will generate one later. -* `request_info`: A collection of tuples, which first item is a user agent, and which +- `request_info`: A collection of tuples, which first item is a user agent, and which second item is an IP address. These user agents and IP addresses are the ones that were used during the registration process. -* `auth_provider_id`: The identifier of the SSO authentication provider, if any. +- `auth_provider_id`: The identifier of the SSO authentication provider, if any. If multiple modules implement this callback, they will be considered in order. If a callback returns `RegistrationBehaviour.ALLOW`, Synapse falls through to the next one. @@ -303,7 +308,7 @@ this callback. _First introduced in Synapse v1.37.0_ -_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ +_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ ```python async def check_media_file_for_spam( @@ -315,13 +320,14 @@ async def check_media_file_for_spam( Called when storing a local or remote file. The callback must return one of: - - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still - decide to reject it. - - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case - of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. - - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. - - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. +- `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still + decide to reject it. +- `synapse.module_api.errors.Codes` to reject the operation with an error code. In case + of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code. + +- (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`. +- (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`. If multiple modules implement this callback, they will be considered in order. If a callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one. @@ -329,7 +335,6 @@ The value of the first callback that does not return `synapse.module_api.NOT_SPA be used. If this happens, Synapse will not call any of the subsequent implementations of this callback. - ### `should_drop_federated_event` _First introduced in Synapse v1.60.0_ @@ -348,7 +353,6 @@ callback returns `False`, Synapse falls through to the next one. The value of th callback that does not return `False` will be used. If this happens, Synapse will not call any of the subsequent implementations of this callback. - ### `check_login_for_spam` _First introduced in Synapse v1.87.0_ @@ -367,13 +371,13 @@ Called when a user logs in. The arguments passed to this callback are: -* `user_id`: The user ID the user is logging in with -* `device_id`: The device ID the user is re-logging into. -* `initial_display_name`: The device display name, if any. -* `request_info`: A collection of tuples, which first item is a user agent, and which +- `user_id`: The user ID the user is logging in with +- `device_id`: The device ID the user is re-logging into. +- `initial_display_name`: The device display name, if any. +- `request_info`: A collection of tuples, which first item is a user agent, and which second item is an IP address. These user agents and IP addresses are the ones that were used during the login process. -* `auth_provider_id`: The identifier of the SSO authentication provider, if any. +- `auth_provider_id`: The identifier of the SSO authentication provider, if any. If multiple modules implement this callback, they will be considered in order. If a callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one. @@ -381,8 +385,7 @@ The value of the first callback that does not return `synapse.module_api.NOT_SPA be used. If this happens, Synapse will not call any of the subsequent implementations of this callback. -*Note:* This will not be called when a user registers. - +_Note:_ This will not be called when a user registers. ## Example diff --git a/make_release.sh b/make_release.sh new file mode 100755 index 0000000000..7142435697 --- /dev/null +++ b/make_release.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -e + +release_name=$1 +release_branch_name="release-${release_name%.*}" + +logical_cores=$([ $(uname) = 'Darwin' ] && + sysctl -n hw.logicalcpu_max || + nproc) + +if [ -z "${release_name}" ]; then + echo "Usage: $0 " + exit 1 +fi + +if [ "${release_name}" = "-h" ]; then + echo "Usage: $0 " + exit 0 +fi + +git fetch --tags --multiple origin upstream +git checkout master +git reset --hard origin/master + +echo -e "\e[34m>>>> rebasing master branch\e[0m" +git rebase upstream/master +echo -e "\e[34m>>>> running lint and tests...\e[0m" +poetry install --extras all --no-interaction --remove-untracked +poetry run ./scripts-dev/lint.sh +poetry run trial -j"${logical_cores}" tests +echo -e "\e[34m>>>> Success!\e[0m" +git push -f + +echo -e "\e[34m>>>> updating release branch\e[0m" +git checkout -B "${release_branch_name}" +git merge --ff-only master +git push -f -u origin "${release_branch_name}" + +echo -e "\e[34m>>>> updating release tag\e[0m" +git checkout "${release_name}" +git merge --ff-only master +git tag -f -s -m "${release_name}_1" "${release_name}_1" +git push -f origin "${release_name}_1" diff --git a/synapse/config/metrics.py b/synapse/config/metrics.py index 8a4ded62ef..f4f5022f35 100644 --- a/synapse/config/metrics.py +++ b/synapse/config/metrics.py @@ -54,6 +54,9 @@ def read_config(self, config: JsonDict, **kwargs: Any) -> None: self.report_stats_endpoint = config.get( "report_stats_endpoint", "https://matrix.org/report-usage-stats/push" ) + self.report_stats_exclude_alias_list = config.get( + "report_stats_exclude_alias_list", [] + ) self.metrics_port = config.get("metrics_port") self.metrics_bind_host = config.get("metrics_bind_host", "127.0.0.1") diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 2c6e672ede..59e2d3909a 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -872,6 +872,36 @@ async def create_room( except LimitExceededError: raise SynapseError(400, "Cannot invite so many users at once") + # Verify invites ahead of time. This is to prevent a partial room + # creation in case the spam checker API rejects an invite. We don't + # want a user do see a partial room in that case. + # Currently 3pid invites are not validated ahead of time since that + # invite depends on a server lookup. + for invitee in invite_list: + if not is_requester_admin: + block_invite_result = None + if self.config.server.block_non_admin_invites: + logger.info( + "Blocking invite: user is not admin and non-admin " + "invites disabled" + ) + block_invite_result = (Codes.FORBIDDEN, {}) + else: + spam_check = await self._spam_checker_module_callbacks.user_may_invite( + requester.user.to_string(), invitee, "" # intentionally blank, since no room exists yet + ) + if spam_check != self._spam_checker_module_callbacks.NOT_SPAM: + logger.info("Blocking invite due to spam checker") + block_invite_result = spam_check + + if block_invite_result is not None: + raise SynapseError( + 403, + "Invites have been disabled on this server", + errcode=block_invite_result[0], + additional_fields=block_invite_result[1], + ) + await self.event_creation_handler.assert_accepted_privacy_policy(requester) power_level_content_override = config.get("power_level_content_override") diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index 03b1e7edc4..044865bead 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -598,7 +598,19 @@ async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: if self.refreshable_access_token_lifetime is not None: access_valid_until_ms = now + self.refreshable_access_token_lifetime refresh_valid_until_ms = None - if self.refresh_token_lifetime is not None: + + custom_refresh_token_lifetime = refresh_submission.get( + "com.famedly.refresh_token_lifetime_ms" + ) + if custom_refresh_token_lifetime is not None: + if not isinstance(custom_refresh_token_lifetime, int): + raise SynapseError( + 400, + "Invalid param: com.famedly.refresh_token_lifetime_ms", + Codes.INVALID_PARAM, + ) + refresh_valid_until_ms = now + custom_refresh_token_lifetime + elif self.refresh_token_lifetime is not None: refresh_valid_until_ms = now + self.refresh_token_lifetime ( diff --git a/synapse/storage/databases/main/metrics.py b/synapse/storage/databases/main/metrics.py index 9ce1100b5c..f170c3fb67 100644 --- a/synapse/storage/databases/main/metrics.py +++ b/synapse/storage/databases/main/metrics.py @@ -241,15 +241,30 @@ def _count_users(self, txn: LoggingTransaction, time_from: int) -> int: """ Returns number of users seen in the past time_from period """ - sql = """ - SELECT COUNT(*) FROM ( - SELECT user_id FROM user_ips - WHERE last_seen > ? - GROUP BY user_id - ) u - """ - txn.execute(sql, (time_from,)) - # Mypy knows that fetchone() might return None if there are no rows. + exclude_list = [ + "@" + localpart + ":" + self.hs.config.server.server_name + for localpart in self.hs.config.metrics.report_stats_exclude_alias_list + ] + + if not exclude_list: + sql = """ + SELECT COUNT(*) FROM ( + SELECT user_id FROM user_ips + WHERE last_seen > ? + GROUP BY user_id + ) u + """ + txn.execute(sql, (time_from,)) + else: + sql = """ + SELECT COUNT(*) FROM ( + SELECT user_id FROM user_ips + WHERE last_seen > ? AND user_id NOT IN ? + GROUP BY user_id + ) u + """ + txn.execute(sql, (time_from, tuple(exclude_list))) + # We know better: "SELECT COUNT(...) FROM ..." without any GROUP BY always # returns exactly one row. (count,) = cast(Tuple[int], txn.fetchone()) diff --git a/tests/rest/client/test_auth.py b/tests/rest/client/test_auth.py index 0b5daf4bb4..16e0713181 100644 --- a/tests/rest/client/test_auth.py +++ b/tests/rest/client/test_auth.py @@ -922,6 +922,75 @@ def test_refresh_token_expiry(self) -> None: refresh_response.code, HTTPStatus.FORBIDDEN, refresh_response.result ) + @override_config( + {"refreshable_access_token_lifetime": "1m", "refresh_token_lifetime": "2m"} + ) + def test_custom_refresh_token_expiry(self) -> None: + """ + The client might override the refresh token lifetime using the custom com.famedly.refresh_token_lifetime_ms parameter. + """ + + def use_custom_refresh_token(refresh_token: str) -> FakeChannel: + """ + Helper that makes a request to use a refresh token with a custom lifetime (3 minutes). + """ + return self.make_request( + "POST", + "/_matrix/client/v3/refresh", + { + "refresh_token": refresh_token, + "com.famedly.refresh_token_lifetime_ms": 3 * 60 * 1000, + }, + ) + + body = { + "type": "m.login.password", + "user": "test", + "password": self.user_pass, + "refresh_token": True, + } + login_response = self.make_request( + "POST", + "/_matrix/client/r0/login", + body, + ) + self.assertEqual(login_response.code, HTTPStatus.OK, login_response.result) + refresh_token1 = login_response.json_body["refresh_token"] + + # refresh immediately to set custom expiry time + refresh_response = use_custom_refresh_token(refresh_token1) + self.assertEqual(refresh_response.code, HTTPStatus.OK, refresh_response.result) + self.assertIn( + "refresh_token", + refresh_response.json_body, + "No new refresh token returned after refresh.", + ) + refresh_token2 = refresh_response.json_body["refresh_token"] + + # Advance 179 seconds in the future (just shy of 3 minutes) + self.reactor.advance(179.0) + + # Refresh our session. The refresh token should still JUST be valid right now. + # By doing so, we get a new access token and a new refresh token. + refresh_response = use_custom_refresh_token(refresh_token2) + self.assertEqual(refresh_response.code, HTTPStatus.OK, refresh_response.result) + self.assertIn( + "refresh_token", + refresh_response.json_body, + "No new refresh token returned after refresh.", + ) + refresh_token3 = refresh_response.json_body["refresh_token"] + + # Advance 181 seconds in the future (just a bit more than 3 minutes) + self.reactor.advance(181.0) + + # Try to refresh our session, but instead notice that the refresh token is + # not valid (it just expired). + refresh_response = use_custom_refresh_token(refresh_token3) + self.assertEqual( + refresh_response.code, HTTPStatus.FORBIDDEN, refresh_response.result + ) + @override_config( { "refreshable_access_token_lifetime": "2m", diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index c559dfda83..1ba8e223ca 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -735,6 +735,23 @@ class RoomsCreateTestCase(RoomBase): user_id = "@sid1:red" + def _joined_rooms( + self, + ) -> List[str]: + """ + Returns the joined rooms for the user + """ + path = "/_matrix/client/v3/joined_rooms" + + channel = self.make_request( + "GET", + path, + ) + + assert channel.code == HTTPStatus.OK, channel.result + + return channel.json_body["joined_rooms"] + def test_post_room_no_keys(self) -> None: # POST with no config keys, expect new room id channel = self.make_request("POST", "/createRoom", "{}") @@ -744,6 +761,8 @@ def test_post_room_no_keys(self) -> None: assert channel.resource_usage is not None self.assertEqual(33, channel.resource_usage.db_txn_count) + assert len(self._joined_rooms()) == 1, "Expected one room to be created." + def test_post_room_initial_state(self) -> None: # POST with initial_state config key, expect new room id channel = self.make_request( @@ -757,18 +776,24 @@ def test_post_room_initial_state(self) -> None: assert channel.resource_usage is not None self.assertEqual(35, channel.resource_usage.db_txn_count) + assert len(self._joined_rooms()) == 1, "Expected one room to be created." + def test_post_room_visibility_key(self) -> None: # POST with visibility config key, expect new room id channel = self.make_request("POST", "/createRoom", b'{"visibility":"private"}') self.assertEqual(HTTPStatus.OK, channel.code) self.assertTrue("room_id" in channel.json_body) + assert len(self._joined_rooms()) == 1, "Expected one room to be created." + def test_post_room_custom_key(self) -> None: # POST with custom config keys, expect new room id channel = self.make_request("POST", "/createRoom", b'{"custom":"stuff"}') self.assertEqual(HTTPStatus.OK, channel.code) self.assertTrue("room_id" in channel.json_body) + assert len(self._joined_rooms()) == 1, "Expected one room to be created." + def test_post_room_known_and_unknown_keys(self) -> None: # POST with custom + known config keys, expect new room id channel = self.make_request( @@ -777,6 +802,8 @@ def test_post_room_known_and_unknown_keys(self) -> None: self.assertEqual(HTTPStatus.OK, channel.code) self.assertTrue("room_id" in channel.json_body) + assert len(self._joined_rooms()) == 1, "Expected one room to be created." + def test_post_room_invalid_content(self) -> None: # POST with invalid content / paths, expect 400 channel = self.make_request("POST", "/createRoom", b'{"visibili') @@ -785,6 +812,8 @@ def test_post_room_invalid_content(self) -> None: channel = self.make_request("POST", "/createRoom", b'["hello"]') self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code) + assert len(self._joined_rooms()) == 0, "Expected no room to be created." + def test_post_room_invitees_invalid_mxid(self) -> None: # POST with invalid invitee, see https://github.com/matrix-org/synapse/issues/4088 # Note the trailing space in the MXID here! @@ -793,6 +822,8 @@ def test_post_room_invitees_invalid_mxid(self) -> None: ) self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code) + assert len(self._joined_rooms()) == 0, "Expected no room to be created." + @unittest.override_config({"rc_invites": {"per_room": {"burst_count": 3}}}) def test_post_room_invitees_ratelimit(self) -> None: """Test that invites sent when creating a room are ratelimited by a RateLimiter, @@ -903,6 +934,34 @@ async def user_may_join_room_tuple( self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) self.assertEqual(join_mock.call_count, 0) + def test_spam_checker_may_invite_room(self) -> None: + """Verify that no room is created, when the spam check disallows any of the invites. + """ + + async def user_may_invite_room_codes( + inviter: str, + invitee: str, + roomid: str, + ) -> Codes: + self.assertEqual(roomid, "") + self.assertEqual(inviter, self.user_id) + return Codes.FORBIDDEN + + self.hs.get_module_api_callbacks().spam_checker._user_may_invite_callbacks.append( + user_may_invite_room_codes + ) + + channel = self.make_request( + "POST", + "/createRoom", + { + "invite": [ "@sid2:red"], + }, + ) + self.assertEqual(channel.code, HTTPStatus.FORBIDDEN, channel.json_body) + + assert len(self._joined_rooms()) == 0, "Expected no room to be created." + def _create_basic_room(self) -> Tuple[int, object]: """ Tries to create a basic room and returns the response code.