Deployment

Kubernetes

PostgreSQL

We recommend running PostgreSQL as a cloud service such as Google Cloud SQL, Amazon RDS for PostgreSQL or Microsoft Azure PostgreSQL, and not running it in the same Kubernetes cluster as the other applications in case of node failures.

All-in-one

The entire distribution service platform can be deployed in a Kubernetes cluster with only a few commands. We provide a yaml deployment files that you can use as a base for your own deployment. Some configurations are better stored as configmaps or secrets. The script will create 2 replicas of every deployment and attempt to spread them across different nodes in the cluster.

Example

---
apiVersion: cloud.google.com/v1beta1
kind: BackendConfig
metadata:
  name: t1c-backendconfig
spec:
  timeoutSec: 300
  connectionDraining:
    drainingTimeoutSec: 60
---

---
apiVersion: v1
kind: Namespace
metadata:
  name: t1c
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: kong-serviceaccount
  namespace: t1c
---
apiVersion: v1
kind: Service
metadata:
  name: kong
  namespace: t1c
  annotations:
    beta.cloud.google.com/backend-config: '{"ports": {"80":"t1c-backendconfig"}}'
spec:
  type: NodePort
  ports:
    - name: proxy
      port: 80
      protocol: TCP
      targetPort: 8000
    - name: proxy-ssl
      port: 443
      protocol: TCP
      targetPort: 8443
  selector:
    app: ingress-kong
---
apiVersion: v1
kind: Service
metadata:
  name: kong-admin
  namespace: t1c
  annotations:
    beta.cloud.google.com/backend-config: '{"ports": {"80":"t1c-backendconfig"}}'
spec:
  type: NodePort
  ports:
    - name: admin
      port: 8001
      protocol: TCP
      targetPort: 8001
    - name: admin-ssl
      port: 8444
      targetPort: 8444
      protocol: TCP
  selector:
    app: ingress-kong
---
apiVersion: v1
kind: Service
metadata:
  name: kong-validation-webhook
  namespace: t1c
spec:
  ports:
    - name: webhook
      port: 443
      protocol: TCP
      targetPort: 8080
  selector:
    app: ingress-kong
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: ingress-kong
  name: ingress-kong
  namespace: t1c
spec:
  replicas: 1
  selector:
    matchLabels:
      app: ingress-kong
  template:
    metadata:
      annotations:
        kuma.io/gateway: enabled
        prometheus.io/port: "8100"
        prometheus.io/scrape: "true"
        traffic.sidecar.istio.io/includeInboundPorts: ""
      labels:
        app: ingress-kong
    spec:
      containers:
        - env:
            - name: KONG_PROXY_LISTEN
              value: 0.0.0.0:8000, 0.0.0.0:8443 ssl http2
            - name: KONG_ADMIN_LISTEN
              value: 0.0.0.0:8001, 127.0.0.1:8444 ssl
            - name: KONG_STATUS_LISTEN
              value: 0.0.0.0:8100
            - name: KONG_DATABASE
              value: postgres
            - name: KONG_PG_HOST
              value: "35.205.16.113"
            - name: KONG_PG_USER
              valueFrom:
                secretKeyRef:
                    name: t1cds-secrets-5h9mbf58h4
                    key: db-username
            - name: KONG_PG_PASSWORD
              valueFrom:
                secretKeyRef:
                    name: t1cds-secrets-5h9mbf58h4
                    key: db-password
            - name: KONG_PG_DATABASE
              value: "kong"
            - name: KONG_NGINX_WORKER_PROCESSES
              value: "1"
            - name: KONG_ADMIN_ACCESS_LOG
              value: /dev/stdout
            - name: KONG_ADMIN_ERROR_LOG
              value: /dev/stderr
            - name: KONG_PROXY_ERROR_LOG
              value: /dev/stderr
            - name: TZ
              value: "Europe/Brussels"
          image: kong:2.1
          lifecycle:
            preStop:
              exec:
                command:
                  - /bin/sh
                  - -c
                  - kong quit
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /status
              port: 8100
              scheme: HTTP
            initialDelaySeconds: 5
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          name: proxy
          ports:
            - containerPort: 8000
              name: proxy
              protocol: TCP
            - containerPort: 8443
              name: proxy-ssl
              protocol: TCP
            - containerPort: 8100
              name: metrics
              protocol: TCP
            - containerPort: 8001
              name: admin
              protocol: TCP
            - containerPort: 8444
              name: admin-ssl
              protocol: TCP
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /status
              port: 8100
              scheme: HTTP
            initialDelaySeconds: 5
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          securityContext:
            runAsUser: 1000
      initContainers:
        - command:
            - /bin/sh
            - -c
            - while true; do kong migrations list; if [[ 0 -eq $? ]]; then exit 0; fi;
              sleep 2;  done;
          env:
            - name: KONG_DATABASE
              value: "postgres"
            - name: KONG_PG_HOST
              value: "35.205.16.113"
            - name: KONG_PG_USER
              valueFrom:
                secretKeyRef:
                    name: t1cds-secrets-5h9mbf58h4
                    key: db-username
            - name: KONG_PG_PASSWORD
              valueFrom:
                secretKeyRef:
                    name: t1cds-secrets-5h9mbf58h4
                    key: db-password
            - name: KONG_PG_DATABASE
              value: "kong"
          image: kong:2.1
          name: wait-for-migrations
      serviceAccountName: kong-serviceaccount
---
apiVersion: batch/v1
kind: Job
metadata:
  name: kong-migrations
  namespace: t1c
spec:
  template:
    metadata:
      name: kong-migrations
    spec:
      containers:
        - command:
            - /bin/sh
            - -c
            - kong migrations bootstrap
          env:
            - name: KONG_DATABASE
              value: "postgres"
            - name: KONG_PG_HOST
              value: "35.205.16.113"
            - name: KONG_PG_USER
              valueFrom:
                secretKeyRef:
                    name: t1cds-secrets-5h9mbf58h4
                    key: db-username
            - name: KONG_PG_PASSWORD
              valueFrom:
                secretKeyRef:
                    name: t1cds-secrets-5h9mbf58h4
                    key: db-password
            - name: KONG_PG_DATABASE
              value: "kong"
          image: kong:2.1
          name: kong-migrations
      restartPolicy: OnFailure
---
apiVersion: v1
kind: Service
metadata:
  name: keycloak
  namespace: t1c
  labels:
    app: keycloak
  annotations:
    beta.cloud.google.com/backend-config: '{"ports": {"80":"t1c-backendconfig"}}'
spec:
  ports:
    - name: http
      port: 80
      targetPort: 8080
    - name: https
      port: 443
      targetPort: 8443
  selector:
    app: keycloak
  type: NodePort
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: keycloak
  namespace: t1c
  labels:
    app: keycloak
spec:
  replicas: 1
  selector:
    matchLabels:
      app: keycloak
  template:
    metadata:
      labels:
        app: keycloak
    spec:
      restartPolicy: Always
      volumes:
        - configMap:
            name: t1cds-jks
          name: keystore
      containers:
        - name: keycloak
          image: quay.io/keycloak/keycloak:10.0.2
          env:
            - name: KEYCLOAK_HOSTNAME
              value: acc-ds.t1t.io
            - name: KEYCLOAK_USER
              valueFrom:
                secretKeyRef:
                    name: t1cds-secrets-5h9mbf58h4
                    key: idp-admin-username
            - name: KEYCLOAK_PASSWORD
              valueFrom:
                secretKeyRef:
                    name: t1cds-secrets-5h9mbf58h4
                    key: idp-admin-password
            - name: PROXY_ADDRESS_FORWARDING
              value: "true"
            - name: DB_VENDOR
              value: POSTGRES
            - name: DB_ADDR
              value: 35.205.16.113
            - name: DB_DATABASE
              value: keycloak
            - name: DB_USER
              valueFrom:
                secretKeyRef:
                    name: t1cds-secrets-5h9mbf58h4
                    key: db-username
            - name: DB_SCHEMA
              value: public
            - name: DB_PASSWORD
              valueFrom:
                secretKeyRef:
                    name: t1cds-secrets-5h9mbf58h4
                    key: db-password
            - name: TZ
              value: "Europe/Brussels"
          volumeMounts:
            - mountPath: "/mnt"
              name: keystore
          ports:
            - name: http
              containerPort: 8080
            - name: https
              containerPort: 8443
          readinessProbe:
            httpGet:
              path: /auth/realms/master
              port: 8080
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: distribution-service
  name: distribution-service
  namespace: t1c
  annotations:
    beta.cloud.google.com/backend-config: '{"ports": {"80":"t1c-backendconfig"}}'
spec:
  type: NodePort
  ports:
    - name: http
      port: 80
      protocol: TCP
      targetPort: 9000
  selector:
    app: distribution-service
---
apiVersion: apps/v1
kind: Deployment
metadata:
  namespace: t1c
  name: distribution-service
  labels:
    app: distribution-service
spec:
  replicas: 1
  selector:
    matchLabels:
      app: distribution-service
  template:
    metadata:
      labels:
        app: distribution-service
    spec:
      restartPolicy: Always
      volumes:
        - configMap:
            name: t1cds-keystore
          name: keystore
      containers:
        - name: distribution-service
          image: eu.gcr.io/t1t-pre-prod/t1cds:latest
          imagePullPolicy: Always
          ports:
            - name: http
              containerPort: 9000
          volumeMounts:
            - mountPath: "/mnt"
              name: keystore
          env:
            - name: DS_ALLOWED_HOST
              value: ".t1t.io"
            - name: DS_APP_TOKEN_VALIDITY_SECONDS
              value: "600"
            - name: DS_APPLICATION_ISSUER
              value: t1cds-app
            - name: DS_GATEWAY_ADMIN_URL
              value: http://kong-admin.t1c.svc.cluster.local:8001
            - name: DS_GATEWAY_BASE_PATH
              value: ""
            - name: DS_GATEWAY_CONSUMER_APPLICATION
              value: "t1cds-app"
            - name: DS_GATEWAY_CONSUMER_REGISTRATION
              value: "t1cds-reg"
            - name: DS_GATEWAY_CONSUMER_USER
              value: "t1cds-user"
            - name: DS_GATEWAY_ENABLED
              value: "true"
            - name: DS_GATEWAY_URL
              value: https://acc-ds.t1t.io
            - name: DS_IDP_ISSUER
              value: https://acc-ds.t1t.io/auth/realms/trust1connector
            - name: DS_KEYSTORE_ALIAS
              valueFrom:
                secretKeyRef:
                    name: t1cds-secrets-5h9mbf58h4
                    key: ks-alias
            - name: DS_KEYSTORE_PASSWORD
              valueFrom:
                secretKeyRef:
                     name: t1cds-secrets-5h9mbf58h4
                     key: ks-password
            - name: DS_KEYSTORE_PATH
              value: /mnt/t1cds.p12
            - name: DS_REG_TOKEN_VALIDITY_SECONDS
              value: "600"
            - name: DS_REGISTRATION_ISSUER
              value: t1cds-reg
            - name: DS_SECURITY_ENABLED
              value: "true"
            - name: DS_MAX_PAGE_SIZE
              value: "100"
            - name: INCLUDE_STACKTRACE
              value: "true"
            - name: JAVA_OPTS
              value: "-Xms512m -Xmx1024m -Dpidfile.path=/dev/null -Dconfig.resource=k8s.conf -Dlogger.resource=logback-cloud.xml -Dplay.evolutions.db.default.autoApply=true"
            - name: PLAY_SECRET
              valueFrom:
                secretKeyRef:
                    name: t1cds-secrets-5h9mbf58h4
                    key: play-secret
            - name: REQUIRE_GATEWAY_HEADERS
              value: "false"
            - name: T1C_DOMAIN
              value: "t1c.t1t.io"
            - name: T1C_PORT
              value: "51883"
            - name: T1C_DB_URL
              value: jdbc:postgresql://35.205.16.113:5432/t1c-ds
            - name: T1C_DS_DB_PWD
              valueFrom:
                secretKeyRef:
                    name: t1cds-secrets-5h9mbf58h4
                    key: db-password
            - name: T1C_DS_DB_USER
              valueFrom:
                secretKeyRef:
                    name: t1cds-secrets-5h9mbf58h4
                    key: db-username
            - name: T1C_EVOLUTIONS_AUTO
              value: "true"
            - name: T1C_EVOLUTIONS_AUTO_DOWNS
              value: "false"
            - name: T1C_EVOLUTIONS_ENABLED
              value: "true"
            - name: T1C_EVOLUTIONS_SCHEMA
              value: "public"
            - name: RMC_LABEL
              value: "rmc"
            - name: TZ
              value: "Europe/Brussels"
          resources:
            requests:
              cpu: "500m"
              memory: "600Mi"
          readinessProbe:
            httpGet:
              path: /v3/system/ready
              port: http
            periodSeconds: 10
            failureThreshold: 10
            initialDelaySeconds: 20
          livenessProbe:
            httpGet:
              path: /v3/system/alive
              port: http
            periodSeconds: 10
            initialDelaySeconds: 20
---
apiVersion: "networking.k8s.io/v1beta1"
kind: "Ingress"
metadata:
  name: "trust1connector-lb"
  namespace: t1c
  annotations:
    kubernetes.io/ingress.global-static-ip-name: acc-trust1connector-ip
    ingress.gcp.kubernetes.io/pre-shared-cert: "t1t-io-ssl-2022-02-18"
    kubernetes.io/ingress.allow-http: "false"
spec:
  backend:
    serviceName: kong
    servicePort: 80
  rules:
    - http:
        paths:
          - path: "/auth"
            backend:
              serviceName: "keycloak"
              servicePort: 80
          - path: "/auth/*"
            backend:
              serviceName: "keycloak"
              servicePort: 80

ConfigMaps

The DS keystores can be stored as configmaps in the cluster, and be mounted as volumes in the pod containers. We require a Java keystore (jks) file to configure the IDP, and a PKCS12 keystore (p12) for the DS API. The contents of both keystores must be identical.

kubectl create configmap t1cds-keystore --from-file=conf/t1cds.p12
kubectl create configmap t1cds-jks --from-file=conf/t1cds.jks

Secrets

Sensitive information such as usernames, passwords, and related data should be stored as secrets. Using Kustomize you can create secrets from string literals which can be set as environment variables in your deployment specs.

cat <<EOF >./k8s/kustomization/kustomization.yml
secretGenerator:
- name: t1cds-secrets
  literals:
  - db-username={{dbUsername}}
  - db-password={dbPassword}
  - ks-alias={{keystoreAlias}}
  - ks-password={{keystorePassword}}
  - play-secret={{playSecret}}
  - idp-admin-username={{idpAdminUsername}}
  - idp-admin-password={{idpAdminPassword}}
EOF
kubectl apply -k ./k8s/kustomization
rm ./k8s/kustomization/kustomization.yml

GKE Guide

Database

Create a PostgreSQL 12 database instance

When creating the database instance, configure the connectivity and backups option according to your need. The database instance must be reachable from the K8s cluster.

Create the necessary databases:

1. t1c-ds

2. keycloak

3. kong

Create Kubernetes Cluster

Ubuntu 18.04

PostgreSQL

1) Add PostgreSQL repository:

wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" |sudo tee  /etc/apt/sources.list.d/pgdg.list

2) Install PostgreSQL:

sudo apt update
sudo apt -y install postgresql-12 postgresql-client-12

3) Configure PostgreSQL. The PostgreSQL server should be reachable from the DS API, Kong Gateway and Keycloak application server(s). We refer you to the documentation: https://www.postgresql.org/docs/12/

4) Create the users and the 3 databases (t1c-ds, kong, keycloak):

We recommend creating different users for each database, but the same user can also be used for all databases.

CREATE USER [INSERT_DATASTORE_USERNAME_HERE];
ALTER USER [INSERT_DATASTORE_USERNAME_HERE] PASSWORD '[INSERT_DATASTORE_PASSWORD_HERE]';
CREATE DATABASE [INSERT_DATABASE_NAME_HERE] OWNER [INSERT_DATASTORE_USERNAME_HERE];

Distribution Service API

1) Obtain the Distribution Service API server distributable. If you wish to build a package from source, run sbt ";clean;compile;dist" from the project root. A zip archive containing the application will be available under the target/universal folder

2) Install Java:

wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | sudo apt-key add -
sudo add-apt-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/
# If you get a command not found error run the command below:
# sudo apt-get install -y software-properties-common
sudo apt-get update
sudo apt-get install adoptopenjdk-11-hotspot

3) Unzip to a folder of your choice. We recommend using a subdirectory of the /opt folder.

4) Configure the Distribution Service API. See Configuration for a detailed description of the available options.

5) Create a service. We recommend using systemctl. Create a file in the /etc/systemd/system/ folder called t1cds.service and configure it as follows:

Description=T1C-DS API
After=syslog.target network.target
Before=httpd.service

[Service]
Environment=PLAY_SECRET=nf8dqrQM9_?XUm]JCxKu7Jyo9cMf`Eqh<VmOTlj`QWJAiKDqp?fD3J=zvOm3v9L:
ExecStart=/opt/t1cds/bin/t1c-ds

[Install]
WantedBy=multi-user.target

We strongly recommend placing sensitive information in the service definition as environment variables. See Configuration to get a list of configuration keys.

6) Enable and start the service

chmod 664 /etc/systemd/system/t1cds.service
systemctl enable /etc/systemd/system/t1cds.service
service t1cds start

Kong Gateway

We refer you to the Kong installation guides for the platform of your choice:

Get a personalized demoKong Inc.

The Kong gateway should be configured to run in database mode, and the Admin API must be available on a port accessible only by the DS API.

Keycloak

We refer you to the Keycloak Installation documentation.

Docker Compose

For development and testing purposes we offer a Docker Compose image to run the platform easily. Note that you must have access to the Trust1Team Docker container registry, or import the DS API image in yours.

After executing docker-compose up, you must still bootstrap the gateway and configure the IDP keystore

Example

version: "3"
services:
  database:
    image: "postgres:11.3"
    container_name: "t1c-db"
    networks:
      - t1c-io
    volumes:
      - "./postgres:/docker-entrypoint-initdb.d"
      - "t1c-data:/var/lib/postgresql/data"
    command: ["-c", "shared_buffers=256MB", "-c", "max_connections=200"]
    ports:
      - 5433:5432
    environment:
      - TZ=UTC
      - PGTZ=UTC
      - POSTGRES_MULTIPLE_DATABASES= "keycloak", "kong", "t1c-ds"
      - POSTGRES_USER=postgres
      - POSTGRES_PASSWORD=postgres
  keycloak:
    image: "jboss/keycloak:11.0.2"
    container_name: "t1c-idp"
    networks:
      - t1c-io
    command:
      - "-Dkeycloak.profile.feature.upload_scripts=enabled"
      - "-Dkeycloak.profile.feature.token_exchange=enabled"
    environment:
      DB_VENDOR: POSTGRES
      DB_ADDR: database
      DB_DATABASE: keycloak
      DB_USER: postgres
      DB_SCHEMA: public
      DB_PASSWORD: postgres
      KEYCLOAK_USER: admin
      KEYCLOAK_PASSWORD: admin
    volumes:
      - ./conf/t1cds.jks:/mnt/t1cds.jks
    ports:
      - 9999:8080
    depends_on:
      - database
  kong-migrations:
    image: "kong:2.1.3"
    command: kong migrations bootstrap
    depends_on:
      - database
    environment:
      KONG_DATABASE: postgres
      KONG_PG_DATABASE: kong
      KONG_PG_HOST: database
      KONG_PG_USER: postgres
      KONG_PG_PASSWORD: postgres
    networks:
      - t1c-io
    restart: on-failure
    deploy:
      restart_policy:
        condition: on-failure
  kong-migrations-up:
    image: "kong:2.1.3"
    command: kong migrations up && kong migrations finish
    depends_on:
      - database
    environment:
      KONG_DATABASE: postgres
      KONG_PG_DATABASE: kong
      KONG_PG_HOST: database
      KONG_PG_USER: postgres
      KONG_PG_PASSWORD: postgres
    networks:
      - t1c-io
    restart: on-failure
    deploy:
      restart_policy:
        condition: on-failure
  kong:
    image: "kong:2.1.3"
    container_name: "t1c-gtw"
    depends_on:
      - database
    environment:
      KONG_ADMIN_ACCESS_LOG: /dev/stdout
      KONG_ADMIN_ERROR_LOG: /dev/stderr
      KONG_ADMIN_LISTEN: 0.0.0.0:8001
      KONG_PROXY_LISTEN: 0.0.0.0:8000
      KONG_DATABASE: postgres
      KONG_PG_DATABASE: kong
      KONG_PG_HOST: database
      KONG_PG_PASSWORD: postgres
      KONG_PG_USER: postgres
      KONG_PROXY_ACCESS_LOG: /dev/stdout
      KONG_PROXY_ERROR_LOG: /dev/stderr
    networks:
      - t1c-io
    ports:
      - "8000:8000/tcp"
      - "8001:8001/tcp"
    healthcheck:
      test: ["CMD", "kong", "health"]
      interval: 10s
      timeout: 10s
      retries: 10
    restart: always
  distribution-service:
    environment:
      DS_ALLOWED_HOST: ".t1t.io"
      DS_APP_TOKEN_VALIDITY_SECONDS: 600
      DS_GATEWAY_ADMIN_URL: "http://t1c-gtw:8001"
      DS_GATEWAY_CONSUMER_REGISTRATION: "t1cds-reg"
      DS_GATEWAY_CONSUMER_APPLICATION: "t1cds-app"
      DS_GATEWAY_CONSUMER_USER: "t1cds-user"
      DS_GATEWAY_BASE_PATH: ""
      DS_GATEWAY_ENABLED: "true"
      DS_GATEWAY_URL: "http://localhost:8000"
      DS_IDP_ISSUER: "http://localhost:9999/auth/realms/trust1connector"
      DS_KEYSTORE_PATH: "/mnt/t1cds.p12"
      DS_KEYSTORE_PASSWORD: "v8%j22HVvHEiC9>e"
      DS_KEYSTORE_ALIAS: "t1cds"
      DS_REG_TOKEN_VALIDITY_SECONDS: 600
      DS_SECURITY_ENABLED: "true"
      DS_MAX_PAGE_SIZE: 100
      INCLUDE_STACKTRACE: "true"
      PLAY_SECRET: "VHJ1c3QxQ29ubmVjdG9yIERhc2hib2FyZCBpcyB0aGUgYmVzdCBpbiBoaXMga2luZCBtYWxha2E="
      REQUIRE_GATEWAY_HEADERS: "false"
      T1C_EVOLUTIONS_ENABLED: "true"
      T1C_EVOLUTIONS_AUTO: "true"
      T1C_EVOLUTIONS_AUTO_DOWNS: "true"
      T1C_DB_URL: "jdbc:postgresql://database:5432/t1c-ds"
      T1C_DS_DB_USER: "postgres"
      T1C_DS_DB_PWD: "postgres"
      T1C_EVOLUTIONS_SCHEMA: "public"
      RMC_LABEL: "rmc"
    command:
      - "-Dconfig.resource=k8s.conf"
      - "-Dlogger.resource=logback-cloud.xml"
      - "-Dplay.evolutions.db.default.autoApply=true"
    image: "eu.gcr.io/t1t-pre-prod/t1cds:latest"
    container_name: "t1c-ds"
    volumes:
      - ./conf/t1cds.p12:/mnt/t1cds.p12
    networks:
      - t1c-io
    ports:
      - 4600:9000
    depends_on:
      - database
networks:
  t1c-io:

volumes:
  t1c-data:
    driver: local

You can run the docker in detached mode via the command

$ docker-compose up -d