diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 6fa9e09925..2b0da445ed 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -29,7 +29,7 @@ or access the full stack on ## Running tests -To run all tests of all services, execute `run-service-tests`. TODO: Systemtests in this repo. +To run all tests of all services, execute `run-service-tests`. ## Adding a new Service @@ -171,3 +171,6 @@ a shortcut to switch to the `test` context and enter the backend shell to e.g. e Be aware that all these commands need an OpenSlides instance to be already running, meaning you have to execute `make run-dev` first. +## Helper scripts + +See [README](dev/scripts/README.md) in the scripts folder. diff --git a/Makefile b/Makefile index b2600cf8cf..d47521af9e 100644 --- a/Makefile +++ b/Makefile @@ -1,47 +1,55 @@ -run-service-tests: - git submodule foreach 'make run-tests' - -build-dev: - ./dev-commands/submodules-do.sh 'make build-dev' - make -C proxy build-dev +DC_PATH=dev/docker +SCRIPT_PATH=dev/scripts +DC=docker-compose -f $(DC_PATH)/docker-compose.dev.yml -run-dev: | build-dev - docker-compose -f docker/docker-compose.dev.yml up $(ARGS) +# Main command: start the dev server +run-dev: | build-dev + $(DC) up $(ARGS) +# Same as run-dev, but with OpenTelemetry run-dev-otel: | build-dev - docker-compose -f docker/docker-compose.dev.yml -f docker/dc.otel.dev.yml up $(ARGS) + $(DC) -f $(DC_PATH)/dc.otel.dev.yml up $(ARGS) + +# Build the docker dev images for all services in parallel +build-dev: + $(SCRIPT_PATH)/submodules-do.sh 'make build-dev' + make -C proxy build-dev + +# Run the tests of all services +run-service-tests: + git submodule foreach 'make run-tests' +# Execute while run-dev is running: Switch to the test database to execute backend tests without +# interfering with your dev database switch-to-test: - docker-compose -f docker/docker-compose.dev.yml stop postgres - docker-compose -f docker/docker-compose.test.yml up -d postgres-test - docker-compose -f docker/docker-compose.dev.yml -f docker/docker-compose.backend.yml up -d backend - docker-compose -f docker/docker-compose.dev.yml restart datastore-writer datastore-reader autoupdate vote + $(DC) stop postgres + docker-compose -f $(DC_PATH)/docker-compose.test.yml up -d postgres-test + $(DC) -f $(DC_PATH)/docker-compose.backend.yml up -d backend + $(DC) restart datastore-writer datastore-reader autoupdate vote +# Execute while run-dev is running: Switch back to your dev database switch-to-dev: - docker-compose -f docker/docker-compose.test.yml stop postgres-test - docker-compose -f docker/docker-compose.dev.yml up -d postgres backend - docker-compose -f docker/docker-compose.dev.yml restart datastore-writer datastore-reader autoupdate vote + docker-compose -f $(DC_PATH)/docker-compose.test.yml stop postgres-test + $(DC) up -d postgres backend + $(DC) restart datastore-writer datastore-reader autoupdate vote +# Shorthand to directly enter a shell in the backend after switching the databases run-backend: | switch-to-test - docker-compose -f docker/docker-compose.dev.yml exec backend ./entrypoint.sh bash --rcfile .bashrc + $(DC) exec backend ./entrypoint.sh bash --rcfile .bashrc +# Stop the dev server stop-dev: - docker-compose -f docker/docker-compose.dev.yml down --volumes --remove-orphans + $(DC) down --volumes --remove-orphans +# Stop the dev server with OpenTelemetry stop-dev-otel: - docker-compose -f docker/docker-compose.dev.yml -f docker/dc.otel.dev.yml down --volumes --remove-orphans - -copy-node-modules: - docker-compose -f docker/docker-compose.dev.yml exec client bash -c "cp -r /app/node_modules/ /app/src/" - mv openslides-client/client/src/node_modules/ openslides-client/client/ - -reload-proxy: - docker-compose -f docker/docker-compose.dev.yml exec -w /etc/caddy proxy caddy reload + $(DC) -f $(DC_PATH)/dc.otel.dev.yml down --volumes --remove-orphans +# Shorthand to execute the services-to-main script services-to-main: - ./services-to-main.sh + $(SCRIPT_PATH)/services-to-main.sh +# You may only use this one time after cloning this repository. +# Will set the upstream remote to "origin" submodules-origin-to-upstream: - # You may only use this one time after cloning this repository. - # Will set the upstream remote to "origin" git submodule foreach -q --recursive 'git remote rename origin upstream' diff --git a/db.sh b/db.sh deleted file mode 100755 index b9568604f2..0000000000 --- a/db.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -docker-compose -f docker/docker-compose.dev.yml exec datastore-writer psql -h postgres -U openslides diff --git a/dc-dev.sh b/dc-dev.sh deleted file mode 100755 index 9cb85da853..0000000000 --- a/dc-dev.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -cd "$(dirname $0)" -docker-compose -f docker/docker-compose.dev.yml $@ diff --git a/dev-commands/README.md b/dev-commands/README.md deleted file mode 100644 index 30baadd18a..0000000000 --- a/dev-commands/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# `dev-commands/import-sql.sh` - -Script to clear postgres DB and afterwards run SQL queries from a file (e.g. -created by `pg_dump`) using `import-events.sh` from the datastores cli scripts. -Run this after starting the dev setup with `make run-dev`. -To ensure consistend data output the autoupdate and depending services are -recreated after the import. -If migrations are necessary, please run \`./dc-dev.sh restart backend\` to -apply them. diff --git a/dev-commands/manage.sh b/dev-commands/manage.sh deleted file mode 100644 index 9de3eb9798..0000000000 --- a/dev-commands/manage.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -docker build openslides-manage-service/ --target manage --tag openslides-manage -docker run --network host openslides-manage $@ diff --git a/docker/build.sh b/dev/docker/build.sh similarity index 100% rename from docker/build.sh rename to dev/docker/build.sh diff --git a/docker/dc.otel.dev.yml b/dev/docker/dc.otel.dev.yml similarity index 100% rename from docker/dc.otel.dev.yml rename to dev/docker/dc.otel.dev.yml diff --git a/docker/docker-compose.backend.yml b/dev/docker/docker-compose.backend.yml similarity index 100% rename from docker/docker-compose.backend.yml rename to dev/docker/docker-compose.backend.yml diff --git a/docker/docker-compose.dev.yml b/dev/docker/docker-compose.dev.yml similarity index 76% rename from docker/docker-compose.dev.yml rename to dev/docker/docker-compose.dev.yml index b143033f00..28ad903101 100644 --- a/docker/docker-compose.dev.yml +++ b/dev/docker/docker-compose.dev.yml @@ -9,7 +9,7 @@ services: - NUM_WORKERS=8 - OPENSLIDES_DEVELOPMENT=1 volumes: - - ../openslides-datastore-service/datastore:/app/datastore + - ../../openslides-datastore-service/datastore:/app/datastore ports: - 9010:9010 - 5680:5678 @@ -21,8 +21,8 @@ services: - redis env_file: services.env volumes: - - ../openslides-datastore-service/datastore:/app/datastore - - ../openslides-datastore-service/cli:/app/cli + - ../../openslides-datastore-service/datastore:/app/datastore + - ../../openslides-datastore-service/cli:/app/cli environment: - COMMAND=create_initial_data - DATASTORE_INITIAL_DATA_FILE=https://raw.githubusercontent.com/OpenSlides/openslides-backend/main/global/data/example-data.json @@ -44,7 +44,7 @@ services: environment: - OPENSLIDES_DEVELOPMENT=1 volumes: - - ../openslides-client/client/src:/app/src + - ../../openslides-client/client/src:/app/src backend: image: openslides-backend-dev @@ -69,11 +69,11 @@ services: - EMAIL_ACCEPT_SELF_SIGNED_CERTIFICATE=false - DEFAULT_FROM_EMAIL=noreply@example.com volumes: - - ../openslides-backend/openslides_backend:/app/openslides_backend - - ../openslides-backend/tests:/app/tests - - ../openslides-backend/cli:/app/cli - - ../openslides-backend/global:/app/global - - ../openslides-backend/scripts:/app/scripts + - ../../openslides-backend/openslides_backend:/app/openslides_backend + - ../../openslides-backend/tests:/app/tests + - ../../openslides-backend/cli:/app/cli + - ../../openslides-backend/global:/app/global + - ../../openslides-backend/scripts:/app/scripts autoupdate: image: openslides-autoupdate-dev @@ -85,9 +85,9 @@ services: - OPENSLIDES_DEVELOPMENT=1 - DATASTORE_TIMEOUT=30 volumes: - - ../openslides-autoupdate-service/cmd:/root/cmd - - ../openslides-autoupdate-service/internal:/root/internal - - ../openslides-autoupdate-service/pkg:/root/pkg + - ../../openslides-autoupdate-service/cmd:/root/cmd + - ../../openslides-autoupdate-service/internal:/root/internal + - ../../openslides-autoupdate-service/pkg:/root/pkg ports: - "9012:9012" @@ -101,8 +101,8 @@ services: environment: - OPENSLIDES_DEVELOPMENT=1 volumes: - - ../openslides-icc-service/cmd:/root/cmd - - ../openslides-icc-service/internal:/root/internal + - ../../openslides-icc-service/cmd:/root/cmd + - ../../openslides-icc-service/internal:/root/internal ports: - "9007:9007" @@ -115,7 +115,7 @@ services: environment: - OPENSLIDES_DEVELOPMENT=1 volumes: - - ../openslides-auth-service/auth/src:/app/src + - ../../openslides-auth-service/auth/src:/app/src ports: - "9004:9004" @@ -128,7 +128,7 @@ services: environment: - OPENSLIDES_DEVELOPMENT=1 volumes: - - ../openslides-media-service/src:/app/src + - ../../openslides-media-service/src:/app/src manage: image: openslides-manage-dev @@ -174,7 +174,7 @@ services: - OPENSLIDES_DEVELOPMENT=1 - VOTE_DISABLE_LOG=true volumes: - - ../openslides-vote-service/cmd:/root/cmd - - ../openslides-vote-service/internal:/root/internal + - ../../openslides-vote-service/cmd:/root/cmd + - ../../openslides-vote-service/internal:/root/internal ports: - "9013:9013" diff --git a/docker/docker-compose.test.yml b/dev/docker/docker-compose.test.yml similarity index 100% rename from docker/docker-compose.test.yml rename to dev/docker/docker-compose.test.yml diff --git a/docker/otel-collector-config.yml b/dev/docker/otel-collector-config.yml similarity index 100% rename from docker/otel-collector-config.yml rename to dev/docker/otel-collector-config.yml diff --git a/docker/services.env b/dev/docker/services.env similarity index 100% rename from docker/services.env rename to dev/docker/services.env diff --git a/localprod/.gitignore b/dev/localprod/.gitignore similarity index 100% rename from localprod/.gitignore rename to dev/localprod/.gitignore diff --git a/localprod/README.md b/dev/localprod/README.md similarity index 100% rename from localprod/README.md rename to dev/localprod/README.md diff --git a/localprod/config.yml b/dev/localprod/config.yml similarity index 100% rename from localprod/config.yml rename to dev/localprod/config.yml diff --git a/localprod/setup.sh b/dev/localprod/setup.sh similarity index 100% rename from localprod/setup.sh rename to dev/localprod/setup.sh diff --git a/dev/scripts/README.md b/dev/scripts/README.md new file mode 100644 index 0000000000..b9c2466759 --- /dev/null +++ b/dev/scripts/README.md @@ -0,0 +1,53 @@ +# Scripts + +This folder gathers some useful scripts when developing OpenSlides. Most of the scripts only work in +dev mode and are neither tested nor recommended for use on productive instances. + +For ease of use, consider adding this folder to your `$PATH` to be able to call the scripts from +anywhere in the repository. + +## `clear-ds.sh` + +Shorthand to call the `truncate_db` route of the writer via curl. + +## `db.sh` + +Shorthand to get to a psql shell inside the postgres container. + +## `dc-dev.sh` + +Shorthand script to not have to type out the whole docker compose command every time one wants to +access a container. Is not a standalone program - supply docker compose commmand and options as args +as you would to `docker-compose`. + +## `export-ds.sh` + +Dumps the current content of the datastore as a JSON file into the file provided as argument +(`export.json` by default) by calling the `get_everything` route of the reader. + +## `reset-db-import-sql.sh` + +Script to clear postgres DB and afterwards run SQL queries from a file (e.g. +created by `pg_dump`) using `import-events.sh` from the datastores cli scripts. +Run this after starting the dev setup with `make run-dev`. +To ensure consistend data output the autoupdate and depending services are +recreated after the import. +If migrations are necessary, please run \`./dc-dev.sh restart backend\` to +apply them. + +## `services-to-main.sh` + +Reset the heads of all submodules to the upstream's main branch. See `--help` for details. + +## `set-ds.sh` + +Similar to `reset-db-import-sql.sh`, but accepts a JSON file as input (uses the `example-data.json` +by default). + +## `strip-meta-fieds.py` + +Helper script used by `export-ds.sh` to remove all meta fields from the output. + +## `submodules-do.sh` + +Run commands for all submodules in the repo (like `git submodule foreach`), but in parallel. diff --git a/dev-commands/clear-ds.sh b/dev/scripts/clear-ds.sh similarity index 100% rename from dev-commands/clear-ds.sh rename to dev/scripts/clear-ds.sh diff --git a/dev/scripts/db.sh b/dev/scripts/db.sh new file mode 100755 index 0000000000..0e85eae98e --- /dev/null +++ b/dev/scripts/db.sh @@ -0,0 +1,3 @@ +#!/bin/bash +cd "$(dirname $0)" +docker-compose -f $HOME/../docker/docker-compose.dev.yml exec datastore-writer psql -h postgres -U openslides diff --git a/dev/scripts/dc-dev.sh b/dev/scripts/dc-dev.sh new file mode 100755 index 0000000000..504f2c82d9 --- /dev/null +++ b/dev/scripts/dc-dev.sh @@ -0,0 +1,3 @@ +#!/bin/bash +cd "$(dirname $0)" +docker-compose -f ../docker/docker-compose.dev.yml $@ diff --git a/dev-commands/export-ds.sh b/dev/scripts/export-ds.sh similarity index 100% rename from dev-commands/export-ds.sh rename to dev/scripts/export-ds.sh diff --git a/dev-commands/reset-db-import-sql.sh b/dev/scripts/reset-db-import-sql.sh similarity index 100% rename from dev-commands/reset-db-import-sql.sh rename to dev/scripts/reset-db-import-sql.sh diff --git a/services-to-main.sh b/dev/scripts/services-to-main.sh similarity index 100% rename from services-to-main.sh rename to dev/scripts/services-to-main.sh diff --git a/dev-commands/set-ds.sh b/dev/scripts/set-ds.sh similarity index 81% rename from dev-commands/set-ds.sh rename to dev/scripts/set-ds.sh index eca1483182..239d79248d 100755 --- a/dev-commands/set-ds.sh +++ b/dev/scripts/set-ds.sh @@ -4,7 +4,7 @@ set -e cd "$(dirname $0)" # first argument is the example data -DATA=$(cat ${1:-../docs/example-data.json}) +DATA=$(cat ${1:-../../openslides-backend/global/data/example-data.json}) ./clear-ds.sh docker-compose -f ../docker/docker-compose.dev.yml exec datastore-writer \ bash -c "source export-database-variables.sh; echo '$DATA' > /data.json; export DATASTORE_INITIAL_DATA_FILE=/data.json; python cli/create_initial_data.py" diff --git a/dev-commands/strip-meta-fields.py b/dev/scripts/strip-meta-fields.py similarity index 100% rename from dev-commands/strip-meta-fields.py rename to dev/scripts/strip-meta-fields.py diff --git a/dev-commands/submodules-do.sh b/dev/scripts/submodules-do.sh similarity index 100% rename from dev-commands/submodules-do.sh rename to dev/scripts/submodules-do.sh diff --git a/docker/.env b/docker/.env deleted file mode 100644 index 3e95398464..0000000000 --- a/docker/.env +++ /dev/null @@ -1,35 +0,0 @@ -# OpenSlides instance configuration -# -# As well as environment variables for various services, this file contains -# variables used to persist custom settings for docker-compose.yml or -# docker-stack.yml. See the preamble of a docker-compose.yml.m4 or -# docker-stack.yml.m4 template for more information. -# -# Most variables are listed here only to facilitate discovery of the available -# options. Empty values cause the template's defaults to be inserted. - -# General -# ------- -INSTANCE_DOMAIN= -PROJECT_STACK_NAME= -EXTERNAL_HTTP_PORT= -DEFAULT_DOCKER_REGISTRY= - -# Docker Images -# ------------- -DOCKER_OPENSLIDES_BACKEND_NAME= -DOCKER_OPENSLIDES_BACKEND_TAG= -DOCKER_OPENSLIDES_FRONTEND_NAME= -DOCKER_OPENSLIDES_FRONTEND_TAG= - -# Configuration -# ------------- -ENABLE_ELECTRONIC_VOTING= - -# Service Replication -# ------------------- -# TODO!! -OPENSLIDES_BACKEND_SERVICE_REPLICAS= -OPENSLIDES_FRONTEND_SERVICE_REPLICAS= -REDIS_RO_SERVICE_REPLICAS= -MEDIA_SERVICE_REPLICAS= diff --git a/docker/reset-admin-password.sh b/docker/reset-admin-password.sh deleted file mode 100755 index dda528ee6f..0000000000 --- a/docker/reset-admin-password.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# This script sets password of user 1 (superadmin) to superadmin. You may also use the manage tool for this work. - -set -e - -# Hash the new password -response=$(docker-compose exec auth curl --header "Content-Type: application/json" -d '{"toHash": "superadmin"}' http://localhost:9004/internal/auth/hash) -hash=$(jq .hash <<< $response) - -# Set user/1/password to $hash -request_data_prefix='{"user_id": 1, "information": {}, "locked_fields": {}, "events": [{"type": "update", "fqid": "user/1", "fields": {"password":' -request_data="$request_data_prefix $hash}}]}" -docker-compose exec backend curl --header "Content-Type: application/json" -d "$request_data" http://datastore-writer:9011/internal/datastore/writer/write - -echo "Done" diff --git a/docker/secrets/admin.env.example b/docker/secrets/admin.env.example deleted file mode 100644 index d2c92f4e2c..0000000000 --- a/docker/secrets/admin.env.example +++ /dev/null @@ -1,3 +0,0 @@ -## secrets/adminsecret.env is sourced by the server container to set the initial -## admin user password. -# OPENSLIDES_ADMIN_PASSWORD="" diff --git a/docker/secrets/user.env.example b/docker/secrets/user.env.example deleted file mode 100644 index d4a334bc0c..0000000000 --- a/docker/secrets/user.env.example +++ /dev/null @@ -1,5 +0,0 @@ -## Example user credential configuration -# OPENSLIDES_USER_FIRSTNAME="John" -# OPENSLIDES_USER_LASTNAME="Doe" -# OPENSLIDES_USER_PASSWORD="" -# OPENSLIDES_USER_EMAIL="john@example.com" diff --git a/logo/README.rst b/logo/README.rst index fade26c8e0..f361018fb8 100644 --- a/logo/README.rst +++ b/logo/README.rst @@ -4,11 +4,11 @@ OpenSlides logo variants The OpenSlides logo is available in normal and dark layout. -The svg source files are create with Inkscape. Each svg file is -exported (with Inkscape) to png. +The `svg` source files are created with Inkscape. Each `svg` file is +exported (with Inkscape) to `png`. -All OpenSlides logo files are licensed under MIT, see LICENSE file in -root directory. +All OpenSlides logo files are licensed under MIT (see LICENSE file in +root directory). "OpenSlides Blue" color values