Kubernetes
PostgreSQL
We recommend running PostgreSQL as a cloud service such as Google Cloud SQL, Amazon RDS for PostgreSQL or Microsoft Azure PostgreSQL, and not running it in the same Kubernetes cluster as the other applications in case of node failures.
All-in-one
The entire distribution service platform can be deployed in a Kubernetes cluster with only a few commands. We provide a yaml deployment files that you can use as a base for your own deployment. Some configurations are better stored as configmaps
or secrets
. The script will create 2 replicas of every deployment and attempt to spread them across different nodes in the cluster.
Example
Copy ---
apiVersion : cloud.google.com/v1beta1
kind : BackendConfig
metadata :
name : t1c-backendconfig
spec :
timeoutSec : 300
connectionDraining :
drainingTimeoutSec : 60
---
Copy ---
apiVersion : v1
kind : Namespace
metadata :
name : t1c
---
apiVersion : v1
kind : ServiceAccount
metadata :
name : kong-serviceaccount
namespace : t1c
---
apiVersion : v1
kind : Service
metadata :
name : kong
namespace : t1c
annotations :
beta.cloud.google.com/backend-config : '{"ports": {"80":"t1c-backendconfig"}}'
spec :
type : NodePort
ports :
- name : proxy
port : 80
protocol : TCP
targetPort : 8000
- name : proxy-ssl
port : 443
protocol : TCP
targetPort : 8443
selector :
app : ingress-kong
---
apiVersion : v1
kind : Service
metadata :
name : kong-admin
namespace : t1c
annotations :
beta.cloud.google.com/backend-config : '{"ports": {"80":"t1c-backendconfig"}}'
spec :
type : NodePort
ports :
- name : admin
port : 8001
protocol : TCP
targetPort : 8001
- name : admin-ssl
port : 8444
targetPort : 8444
protocol : TCP
selector :
app : ingress-kong
---
apiVersion : v1
kind : Service
metadata :
name : kong-validation-webhook
namespace : t1c
spec :
ports :
- name : webhook
port : 443
protocol : TCP
targetPort : 8080
selector :
app : ingress-kong
---
apiVersion : apps/v1
kind : Deployment
metadata :
labels :
app : ingress-kong
name : ingress-kong
namespace : t1c
spec :
replicas : 1
selector :
matchLabels :
app : ingress-kong
template :
metadata :
annotations :
kuma.io/gateway : enabled
prometheus.io/port : "8100"
prometheus.io/scrape : "true"
traffic.sidecar.istio.io/includeInboundPorts : ""
labels :
app : ingress-kong
spec :
containers :
- env :
- name : KONG_PROXY_LISTEN
value : 0.0.0.0:8000, 0.0.0.0:8443 ssl http2
- name : KONG_ADMIN_LISTEN
value : 0.0.0.0:8001, 127.0.0.1:8444 ssl
- name : KONG_STATUS_LISTEN
value : 0.0.0.0:8100
- name : KONG_DATABASE
value : postgres
- name : KONG_PG_HOST
value : "35.205.16.113"
- name : KONG_PG_USER
valueFrom :
secretKeyRef :
name : t1cds-secrets-5h9mbf58h4
key : db-username
- name : KONG_PG_PASSWORD
valueFrom :
secretKeyRef :
name : t1cds-secrets-5h9mbf58h4
key : db-password
- name : KONG_PG_DATABASE
value : "kong"
- name : KONG_NGINX_WORKER_PROCESSES
value : "1"
- name : KONG_ADMIN_ACCESS_LOG
value : /dev/stdout
- name : KONG_ADMIN_ERROR_LOG
value : /dev/stderr
- name : KONG_PROXY_ERROR_LOG
value : /dev/stderr
- name : TZ
value : "Europe/Brussels"
image : kong:2.1
lifecycle :
preStop :
exec :
command :
- /bin/sh
- -c
- kong quit
livenessProbe :
failureThreshold : 3
httpGet :
path : /status
port : 8100
scheme : HTTP
initialDelaySeconds : 5
periodSeconds : 10
successThreshold : 1
timeoutSeconds : 1
name : proxy
ports :
- containerPort : 8000
name : proxy
protocol : TCP
- containerPort : 8443
name : proxy-ssl
protocol : TCP
- containerPort : 8100
name : metrics
protocol : TCP
- containerPort : 8001
name : admin
protocol : TCP
- containerPort : 8444
name : admin-ssl
protocol : TCP
readinessProbe :
failureThreshold : 3
httpGet :
path : /status
port : 8100
scheme : HTTP
initialDelaySeconds : 5
periodSeconds : 10
successThreshold : 1
timeoutSeconds : 1
securityContext :
runAsUser : 1000
initContainers :
- command :
- /bin/sh
- -c
- while true; do kong migrations list; if [[ 0 -eq $? ]]; then exit 0; fi;
sleep 2; done;
env :
- name : KONG_DATABASE
value : "postgres"
- name : KONG_PG_HOST
value : "35.205.16.113"
- name : KONG_PG_USER
valueFrom :
secretKeyRef :
name : t1cds-secrets-5h9mbf58h4
key : db-username
- name : KONG_PG_PASSWORD
valueFrom :
secretKeyRef :
name : t1cds-secrets-5h9mbf58h4
key : db-password
- name : KONG_PG_DATABASE
value : "kong"
image : kong:2.1
name : wait-for-migrations
serviceAccountName : kong-serviceaccount
---
apiVersion : batch/v1
kind : Job
metadata :
name : kong-migrations
namespace : t1c
spec :
template :
metadata :
name : kong-migrations
spec :
containers :
- command :
- /bin/sh
- -c
- kong migrations bootstrap
env :
- name : KONG_DATABASE
value : "postgres"
- name : KONG_PG_HOST
value : "35.205.16.113"
- name : KONG_PG_USER
valueFrom :
secretKeyRef :
name : t1cds-secrets-5h9mbf58h4
key : db-username
- name : KONG_PG_PASSWORD
valueFrom :
secretKeyRef :
name : t1cds-secrets-5h9mbf58h4
key : db-password
- name : KONG_PG_DATABASE
value : "kong"
image : kong:2.1
name : kong-migrations
restartPolicy : OnFailure
---
apiVersion : v1
kind : Service
metadata :
name : keycloak
namespace : t1c
labels :
app : keycloak
annotations :
beta.cloud.google.com/backend-config : '{"ports": {"80":"t1c-backendconfig"}}'
spec :
ports :
- name : http
port : 80
targetPort : 8080
- name : https
port : 443
targetPort : 8443
selector :
app : keycloak
type : NodePort
---
apiVersion : apps/v1
kind : Deployment
metadata :
name : keycloak
namespace : t1c
labels :
app : keycloak
spec :
replicas : 1
selector :
matchLabels :
app : keycloak
template :
metadata :
labels :
app : keycloak
spec :
restartPolicy : Always
volumes :
- configMap :
name : t1cds-jks
name : keystore
containers :
- name : keycloak
image : quay.io/keycloak/keycloak:10.0.2
env :
- name : KEYCLOAK_HOSTNAME
value : acc-ds.t1t.io
- name : KEYCLOAK_USER
valueFrom :
secretKeyRef :
name : t1cds-secrets-5h9mbf58h4
key : idp-admin-username
- name : KEYCLOAK_PASSWORD
valueFrom :
secretKeyRef :
name : t1cds-secrets-5h9mbf58h4
key : idp-admin-password
- name : PROXY_ADDRESS_FORWARDING
value : "true"
- name : DB_VENDOR
value : POSTGRES
- name : DB_ADDR
value : 35.205.16.113
- name : DB_DATABASE
value : keycloak
- name : DB_USER
valueFrom :
secretKeyRef :
name : t1cds-secrets-5h9mbf58h4
key : db-username
- name : DB_SCHEMA
value : public
- name : DB_PASSWORD
valueFrom :
secretKeyRef :
name : t1cds-secrets-5h9mbf58h4
key : db-password
- name : TZ
value : "Europe/Brussels"
volumeMounts :
- mountPath : "/mnt"
name : keystore
ports :
- name : http
containerPort : 8080
- name : https
containerPort : 8443
readinessProbe :
httpGet :
path : /auth/realms/master
port : 8080
---
apiVersion : v1
kind : Service
metadata :
labels :
app : distribution-service
name : distribution-service
namespace : t1c
annotations :
beta.cloud.google.com/backend-config : '{"ports": {"80":"t1c-backendconfig"}}'
spec :
type : NodePort
ports :
- name : http
port : 80
protocol : TCP
targetPort : 9000
selector :
app : distribution-service
---
apiVersion : apps/v1
kind : Deployment
metadata :
namespace : t1c
name : distribution-service
labels :
app : distribution-service
spec :
replicas : 1
selector :
matchLabels :
app : distribution-service
template :
metadata :
labels :
app : distribution-service
spec :
restartPolicy : Always
volumes :
- configMap :
name : t1cds-keystore
name : keystore
containers :
- name : distribution-service
image : eu.gcr.io/t1t-pre-prod/t1cds:latest
imagePullPolicy : Always
ports :
- name : http
containerPort : 9000
volumeMounts :
- mountPath : "/mnt"
name : keystore
env :
- name : DS_ALLOWED_HOST
value : ".t1t.io"
- name : DS_APP_TOKEN_VALIDITY_SECONDS
value : "600"
- name : DS_APPLICATION_ISSUER
value : t1cds-app
- name : DS_GATEWAY_ADMIN_URL
value : http://kong-admin.t1c.svc.cluster.local:8001
- name : DS_GATEWAY_BASE_PATH
value : ""
- name : DS_GATEWAY_CONSUMER_APPLICATION
value : "t1cds-app"
- name : DS_GATEWAY_CONSUMER_REGISTRATION
value : "t1cds-reg"
- name : DS_GATEWAY_CONSUMER_USER
value : "t1cds-user"
- name : DS_GATEWAY_ENABLED
value : "true"
- name : DS_GATEWAY_URL
value : https://acc-ds.t1t.io
- name : DS_IDP_ISSUER
value : https://acc-ds.t1t.io/auth/realms/trust1connector
- name : DS_KEYSTORE_ALIAS
valueFrom :
secretKeyRef :
name : t1cds-secrets-5h9mbf58h4
key : ks-alias
- name : DS_KEYSTORE_PASSWORD
valueFrom :
secretKeyRef :
name : t1cds-secrets-5h9mbf58h4
key : ks-password
- name : DS_KEYSTORE_PATH
value : /mnt/t1cds.p12
- name : DS_REG_TOKEN_VALIDITY_SECONDS
value : "600"
- name : DS_REGISTRATION_ISSUER
value : t1cds-reg
- name : DS_SECURITY_ENABLED
value : "true"
- name : DS_MAX_PAGE_SIZE
value : "100"
- name : INCLUDE_STACKTRACE
value : "true"
- name : JAVA_OPTS
value: "-Xms512m -Xmx1024m -Dpidfile.path=/dev/null -Dconfig.resource=k8s.conf -Dlogger.resource=logback-cloud.xml -Dplay.evolutions.db.default.autoApply=true"
- name : PLAY_SECRET
valueFrom :
secretKeyRef :
name : t1cds-secrets-5h9mbf58h4
key : play-secret
- name : REQUIRE_GATEWAY_HEADERS
value : "false"
- name : T1C_DOMAIN
value : "t1c.t1t.io"
- name : T1C_PORT
value : "51883"
- name : T1C_DB_URL
value : jdbc:postgresql://35.205.16.113:5432/t1c-ds
- name : T1C_DS_DB_PWD
valueFrom :
secretKeyRef :
name : t1cds-secrets-5h9mbf58h4
key : db-password
- name : T1C_DS_DB_USER
valueFrom :
secretKeyRef :
name : t1cds-secrets-5h9mbf58h4
key : db-username
- name : T1C_EVOLUTIONS_AUTO
value : "true"
- name : T1C_EVOLUTIONS_AUTO_DOWNS
value : "false"
- name : T1C_EVOLUTIONS_ENABLED
value : "true"
- name : T1C_EVOLUTIONS_SCHEMA
value : "public"
- name : RMC_LABEL
value : "rmc"
- name : TZ
value : "Europe/Brussels"
resources :
requests :
cpu : "500m"
memory : "600Mi"
readinessProbe :
httpGet :
path : /v3/system/ready
port : http
periodSeconds : 10
failureThreshold : 10
initialDelaySeconds : 20
livenessProbe :
httpGet :
path : /v3/system/alive
port : http
periodSeconds : 10
initialDelaySeconds : 20
---
apiVersion : "networking.k8s.io/v1beta1"
kind : "Ingress"
metadata :
name : "trust1connector-lb"
namespace : t1c
annotations :
kubernetes.io/ingress.global-static-ip-name : acc-trust1connector-ip
ingress.gcp.kubernetes.io/pre-shared-cert : "t1t-io-ssl-2022-02-18"
kubernetes.io/ingress.allow-http : "false"
spec :
backend :
serviceName : kong
servicePort : 80
rules :
- http :
paths :
- path : "/auth"
backend :
serviceName : "keycloak"
servicePort : 80
- path : "/auth/*"
backend :
serviceName : "keycloak"
servicePort : 80
ConfigMaps
The DS keystores can be stored as configmaps in the cluster, and be mounted as volumes in the pod containers. We require a Java keystore (jks
) file to configure the IDP, and a PKCS12 keystore (p12
) for the DS API. The contents of both keystores must be identical.
Copy kubectl create configmap t1cds-keystore --from-file=conf/t1cds.p12
kubectl create configmap t1cds-jks --from-file=conf/t1cds.jks
Secrets
Sensitive information such as usernames, passwords, and related data should be stored as secrets. Using Kustomize you can create secrets from string literals which can be set as environment variables in your deployment specs.
Copy cat <<EOF >./k8s/kustomization/kustomization.yml
secretGenerator:
- name: t1cds-secrets
literals:
- db-username={{dbUsername}}
- db-password={dbPassword}
- ks-alias={{keystoreAlias}}
- ks-password={{keystorePassword}}
- play-secret={{playSecret}}
- idp-admin-username={{idpAdminUsername}}
- idp-admin-password={{idpAdminPassword}}
EOF
kubectl apply -k ./k8s/kustomization
rm ./k8s/kustomization/kustomization.yml
GKE Guide
Database
Create a PostgreSQL 12 database instance
When creating the database instance, configure the connectivity and backups option according to your need. The database instance must be reachable from the K8s cluster.
Create the necessary databases:
Create Kubernetes Cluster
Ubuntu 18.04
PostgreSQL
1) Add PostgreSQL repository:
Copy wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" |sudo tee /etc/apt/sources.list.d/pgdg.list
2) Install PostgreSQL:
Copy sudo apt update
sudo apt -y install postgresql-12 postgresql-client-12
3) Configure PostgreSQL. The PostgreSQL server should be reachable from the DS API, Kong Gateway and Keycloak application server(s). We refer you to the documentation: https://www.postgresql.org/docs/12/
4) Create the users and the 3 databases (t1c-ds
, kong
, keycloak
):
We recommend creating different users for each database, but the same user can also be used for all databases.
Copy CREATE USER [INSERT_DATASTORE_USERNAME_HERE];
ALTER USER [INSERT_DATASTORE_USERNAME_HERE] PASSWORD '[INSERT_DATASTORE_PASSWORD_HERE]';
CREATE DATABASE [INSERT_DATABASE_NAME_HERE] OWNER [INSERT_DATASTORE_USERNAME_HERE];
Distribution Service API
1) Obtain the Distribution Service API server distributable. If you wish to build a package from source, run sbt ";clean;compile;dist"
from the project root. A zip archive containing the application will be available under the target/universal
folder
2) Install Java:
Copy wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | sudo apt-key add -
sudo add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/
# If you get a command not found error run the command below:
# sudo apt-get install -y software-properties-common
sudo apt-get update
sudo apt-get install adoptopenjdk-11-hotspot
3) Unzip to a folder of your choice. We recommend using a subdirectory of the /opt
folder.
4) Configure the Distribution Service API. See Configuration
for a detailed description of the available options.
5) Create a service. We recommend using systemctl
. Create a file in the /etc/systemd/system/
folder called t1cds.service
and configure it as follows:
Copy Description=T1C-DS API
After=syslog.target network.target
Before=httpd.service
[Service]
Environment=PLAY_SECRET=nf8dqrQM9_?XUm]JCxKu7Jyo9cMf`Eqh<VmOTlj`QWJAiKDqp?fD3J=zvOm3v9L:
ExecStart=/opt/t1cds/bin/t1c-ds
[Install]
WantedBy=multi-user.target
We strongly recommend placing sensitive information in the service definition as environment variables. See Configuration
to get a list of configuration keys.
6) Enable and start the service
Copy chmod 664 /etc/systemd/system/t1cds.service
systemctl enable /etc/systemd/system/t1cds.service
service t1cds start
Kong Gateway
We refer you to the Kong
installation guides for the platform of your choice:
The Kong
gateway should be configured to run in database mode, and the Admin API must be available on a port accessible only by the DS API.
Keycloak
We refer you to the Keycloak Installation documentation.
Docker Compose
For development and testing purposes we offer a Docker Compose image to run the platform easily. Note that you must have access to the Trust1Team Docker container registry, or import the DS API image in yours.
After executing docker-compose up, you must still bootstrap the gateway and configure the IDP keystore
Example
Copy version : "3"
services :
database :
image : "postgres:11.3"
container_name : "t1c-db"
networks :
- t1c-io
volumes :
- "./postgres:/docker-entrypoint-initdb.d"
- "t1c-data:/var/lib/postgresql/data"
command : [ "-c" , "shared_buffers=256MB" , "-c" , "max_connections=200" ]
ports :
- 5433:5432
environment :
- TZ=UTC
- PGTZ=UTC
- POSTGRES_MULTIPLE_DATABASES= "keycloak", "kong", "t1c-ds"
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=postgres
keycloak :
image : "jboss/keycloak:11.0.2"
container_name : "t1c-idp"
networks :
- t1c-io
command :
- "-Dkeycloak.profile.feature.upload_scripts=enabled"
- "-Dkeycloak.profile.feature.token_exchange=enabled"
environment :
DB_VENDOR : POSTGRES
DB_ADDR : database
DB_DATABASE : keycloak
DB_USER : postgres
DB_SCHEMA : public
DB_PASSWORD : postgres
KEYCLOAK_USER : admin
KEYCLOAK_PASSWORD : admin
volumes :
- ./conf/t1cds.jks:/mnt/t1cds.jks
ports :
- 9999:8080
depends_on :
- database
kong-migrations :
image : "kong:2.1.3"
command : kong migrations bootstrap
depends_on :
- database
environment :
KONG_DATABASE : postgres
KONG_PG_DATABASE : kong
KONG_PG_HOST : database
KONG_PG_USER : postgres
KONG_PG_PASSWORD : postgres
networks :
- t1c-io
restart : on-failure
deploy :
restart_policy :
condition : on-failure
kong-migrations-up :
image : "kong:2.1.3"
command : kong migrations up && kong migrations finish
depends_on :
- database
environment :
KONG_DATABASE : postgres
KONG_PG_DATABASE : kong
KONG_PG_HOST : database
KONG_PG_USER : postgres
KONG_PG_PASSWORD : postgres
networks :
- t1c-io
restart : on-failure
deploy :
restart_policy :
condition : on-failure
kong :
image : "kong:2.1.3"
container_name : "t1c-gtw"
depends_on :
- database
environment :
KONG_ADMIN_ACCESS_LOG : /dev/stdout
KONG_ADMIN_ERROR_LOG : /dev/stderr
KONG_ADMIN_LISTEN : 0.0.0.0:8001
KONG_PROXY_LISTEN : 0.0.0.0:8000
KONG_DATABASE : postgres
KONG_PG_DATABASE : kong
KONG_PG_HOST : database
KONG_PG_PASSWORD : postgres
KONG_PG_USER : postgres
KONG_PROXY_ACCESS_LOG : /dev/stdout
KONG_PROXY_ERROR_LOG : /dev/stderr
networks :
- t1c-io
ports :
- "8000:8000/tcp"
- "8001:8001/tcp"
healthcheck :
test : [ "CMD" , "kong" , "health" ]
interval : 10s
timeout : 10s
retries : 10
restart : always
distribution-service :
environment :
DS_ALLOWED_HOST : ".t1t.io"
DS_APP_TOKEN_VALIDITY_SECONDS : 600
DS_GATEWAY_ADMIN_URL : "http://t1c-gtw:8001"
DS_GATEWAY_CONSUMER_REGISTRATION : "t1cds-reg"
DS_GATEWAY_CONSUMER_APPLICATION : "t1cds-app"
DS_GATEWAY_CONSUMER_USER : "t1cds-user"
DS_GATEWAY_BASE_PATH : ""
DS_GATEWAY_ENABLED : "true"
DS_GATEWAY_URL : "http://localhost:8000"
DS_IDP_ISSUER : "http://localhost:9999/auth/realms/trust1connector"
DS_KEYSTORE_PATH : "/mnt/t1cds.p12"
DS_KEYSTORE_PASSWORD : "v8%j22HVvHEiC9>e"
DS_KEYSTORE_ALIAS : "t1cds"
DS_REG_TOKEN_VALIDITY_SECONDS : 600
DS_SECURITY_ENABLED : "true"
DS_MAX_PAGE_SIZE : 100
INCLUDE_STACKTRACE : "true"
PLAY_SECRET : "VHJ1c3QxQ29ubmVjdG9yIERhc2hib2FyZCBpcyB0aGUgYmVzdCBpbiBoaXMga2luZCBtYWxha2E="
REQUIRE_GATEWAY_HEADERS : "false"
T1C_EVOLUTIONS_ENABLED : "true"
T1C_EVOLUTIONS_AUTO : "true"
T1C_EVOLUTIONS_AUTO_DOWNS : "true"
T1C_DB_URL : "jdbc:postgresql://database:5432/t1c-ds"
T1C_DS_DB_USER : "postgres"
T1C_DS_DB_PWD : "postgres"
T1C_EVOLUTIONS_SCHEMA : "public"
RMC_LABEL : "rmc"
command :
- "-Dconfig.resource=k8s.conf"
- "-Dlogger.resource=logback-cloud.xml"
- "-Dplay.evolutions.db.default.autoApply=true"
image : "eu.gcr.io/t1t-pre-prod/t1cds:latest"
container_name : "t1c-ds"
volumes :
- ./conf/t1cds.p12:/mnt/t1cds.p12
networks :
- t1c-io
ports :
- 4600:9000
depends_on :
- database
networks :
t1c-io :
volumes :
t1c-data :
driver : local
You can run the docker in detached mode via the command
Copy $ docker-compose up -d