Merge pull request #712 from MetaCell/release/2.2.0

Release/2.2.0

Author: filippomc

application-templates/django-app/api/test_st.py

@@ -0,0 +1,27 @@
+import os
+from pprint import pprint
+import schemathesis as st
+from schemathesis.checks import response_schema_conformance, not_a_server_error
+
+from cloudharness_test import apitest_init # include to perform default authorization
+
+app_url = os.environ.get("APP_URL", "http://samples.ch.local/api")
+
+try:
+    schema = st.from_uri(app_url + "/openapi.json")
+except:
+    # support alternative schema location
+    schema = st.from_uri(app_url.replace("/api", "") + "/openapi.json")
+
+
+@schema.parametrize(endpoint="/ping")
+def test_ping(case):
+    response = case.call()
+    pprint(response.__dict__)
+    assert response.status_code == 200, "this api errors on purpose"
+
+def test_state_machine():
+    schema.as_state_machine().run()
+# APIWorkflow = schema.as_state_machine()
+# APIWorkflow.run()
+# TestAPI = APIWorkflow.TestCase

cloudharness.png

@@ -74,11 +74,6 @@ spec:
         volumeMounts:
           - name: {{ .app.harness.database.name | quote }}
             mountPath: /data/db
-          {{- if .root.Values.backup.active }}
-          - name: "db-backups"
-            mountPath: {{ (printf "%s/%s/%s" .root.Values.backup.dir .app.harness.database.type .app.harness.database.name) | quote }}
-            readOnly: true
-          {{- end }}
           {{- if eq .app.harness.database.type "postgres" }}
           - mountPath: /dev/shm
             name: dshm  
@@ -92,11 +87,6 @@ spec:
           medium: Memory
         name: dshm     
       {{- end }}     
-      {{- if .root.Values.backup.active }}
-      - name: "db-backups"
-        persistentVolumeClaim:
-          claimName: "db-backups"
-      {{- end }}
 ---
 {{- if .root.Values.backup.active }}
 {{- include (print "deploy_utils.database." .app.harness.database.type ".backup") . }}

deployment-configuration/helm/templates/auto-database.yaml

@@ -18,8 +18,12 @@ data:
     enable-default-deny:  {{ eq (.app.harness.secured | toString) "true" }}
     listen: 0.0.0.0:8080
     enable-refresh-tokens: true
-    server-write-timeout: 180s
-    upstream-response-header-timeout: 180s
+    server-write-timeout: {{ .app.harness.proxy.timeout.send | default .root.Values.proxy.timeout.send | default 180 }}s
+    upstream-timeout: {{ .app.harness.proxy.timeout.read | default .root.Values.proxy.timeout.read | default 180 }}s
+    upstream-response-header-timeout:   {{ .app.harness.proxy.timeout.read | default .root.Values.proxy.timeout.read | default 180 }}s
+    upstream-expect-continue-timeout:  {{ .app.harness.proxy.timeout.read | default .root.Values.proxy.timeout.read | default 180 }}s
+    server-read-timeout:  {{ .app.harness.proxy.timeout.read | default .root.Values.proxy.timeout.read | default 180 }}s
+    upstream-keepalive-timeout:  {{ .app.harness.proxy.timeout.keepalive | default .root.Values.proxy.timeout.keepalive | default 180 }}s
     http-only-cookie: false
     tls-cert:
     tls-private-key:
@@ -65,7 +69,7 @@ data:
               <h2 class="message">403 Permission Denied</h2>
               <div class="error-details">
                 Sorry, you do not have access to this page, please contact your administrator.
-                If you have been assigned new authorizations try to <a href="/oauth/logout?redirect=/">login again</a>.
+                If you have been assigned new authorizations, try to refresh the page or to <a href="/oauth/logout?redirect=/">login again</a>.
               </div>
             </div>
           </div>

deployment-configuration/helm/templates/auto-gatekeepers.yaml

@@ -38,10 +38,13 @@ metadata:
     cert-manager.io/issuer: {{ printf "%s-%s" "letsencrypt" .Values.namespace }}
     {{- end }}
     nginx.ingress.kubernetes.io/ssl-redirect: {{ (and $tls .Values.ingress.ssl_redirect) | quote }}
-    nginx.ingress.kubernetes.io/proxy-body-size: '250m'
+    nginx.ingress.kubernetes.io/proxy-body-size: '{{ .Values.proxy.payload.max }}m'
     nginx.ingress.kubernetes.io/proxy-buffer-size: '128k'
     nginx.ingress.kubernetes.io/from-to-www-redirect: 'true'
     nginx.ingress.kubernetes.io/rewrite-target: /$1
+    nginx.ingress.kubernetes.io/auth-keepalive-timeout: {{ .Values.proxy.timeout.keepalive | quote }}
+    nginx.ingress.kubernetes.io/proxy-read-timeout: {{ .Values.proxy.timeout.read | quote }}
+    nginx.ingress.kubernetes.io/proxy-send-timeout: {{ .Values.proxy.timeout.send | quote }}
 spec:
   rules:
   {{- range $app := .Values.apps }}

deployment-configuration/helm/templates/ingress.yaml

@@ -67,3 +67,13 @@ backup:
       memory: "64Mi"
       # -- K8s cpu resource definition.
       cpu: "50m"
+proxy:
+  timeout:
+    # -- Timeout for proxy connections in seconds.
+    send: 60
+    # -- Timeout for proxy responses in seconds.
+    read: 60
+    keepalive: 60
+  payload:
+    # -- Maximum size of payload in MB
+    max: 250

deployment-configuration/helm/values.yaml

@@ -22,6 +22,8 @@ harness:
         - administrator
     - uri: /api/openapi.json
       white-listed: true
+    - uri: /openapi.json
+      white-listed: true
   # -- Defines reference deployment parameters. Values maps to k8s spec
   deployment:
     # -- When true, enables automatic deployment
@@ -125,3 +127,13 @@ harness:
       smoketest: true
       ignoreConsoleErrors: false
       ignoreRequestErrors: false
+  proxy:
+    timeout:
+      # -- Timeout for proxy connections in seconds.
+      send:
+      # -- Timeout for proxy responses in seconds.
+      read:
+      keepalive:
+    payload:
+      # -- Maximum size of payload in MB
+      max: 

deployment-configuration/value-template.yaml

@@ -62,7 +62,22 @@ harness:
   secured: open
 ```
 
+#### Proxy specific configurations
+Proxy configurations can be personalized in the application in the case that we want to have more restrictive values than the global ones (see [here](./ingress-domains-proxies.md#proxy-configurations) for more )
 
+```yaml
+harness:
+  proxy:
+    timeout:
+      # -- Timeout for proxy connections in seconds.
+      send:
+      # -- Timeout for proxy responses in seconds.
+      read:
+      keepalive:
+    payload:
+      # -- Maximum size of payload in MB
+      max: 
+```
 ### Secure an enpoint with OpenAPI
 
 In every api endpoint that you want to secure, add the bearerAuth security as in the example:

docs/accounts.md

@@ -111,14 +111,49 @@ Per default, database backups are disabled. However, you can overwrite backups b
 
 ```yaml
 backup:
-    active: true
+  active: true
 ```
 
+See all the default values [here](../../deployment-configuration/helm/values.yaml).
 You can find additional configuration fields for backups to overwrite in the generated `deployment/helm/values.yaml` once you deploy your applications.
 
 Backups are defined for `mongo` and `postgres` database in form of a [K8s CronJob](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/) that creates a dump of the database and stores it in a different persistent volume on the same cluster.
 
-This is done periodically according to a configurable schedule, per default once a day.
+This is done periodically according to a configurable schedule, per default every 5 minutes.
+
+A smart retention strategy is used for backups, by default:
+- all current days backups
+- one per day, last 7 days
+- one per week, last 4 weeks
+- one per month, last 6 months
+
+Implementation of backups and retention is based on https://github.com/prodrigestivill/docker-postgres-backup-local.
+
+#### How to monitor and restore backups
+
+Backups are stored in a Kubernetes volume named `db-backups`.
+
+Can mount the volume to your database pod by adding the following to your db deployment:
+
+```yaml
+...
+spec:
+  template:
+    spec:
+      containers:
+      - ...
+        volumeMounts:
+        - name: "db-backups"
+          mountPath: /backups
+          readOnly: true
+      ...
+      volumes:
+      ...
+      - name: "db-backups"
+        persistentVolumeClaim:
+          claimName: "db-backups"
+```
+
 
 
 ### MongoDB

docs/applications/databases.md

@@ -0,0 +1,69 @@
+# Ingress, domains and proxies
+
+## Default configurations for domain and subdomains
+Cloud Harness makes it very easy to configure domains and proxies, by making
+an underlying assumption:
+
+- Applications share a main base domain (say ch.org)
+- Applications can define a subdomain (say myapp)
+
+The main domain is configured in the [root values file](../deployment-configuration/values-template.yaml) and
+it is usually overridden by the `harness-deployment` command, e.g.
+
+```
+harness-deployment ... -d ch.org
+```
+
+The subdomain is defined in the application's values.yaml file in 
+harness.subdomain (see for instance the [samples application configuration](../applications/samples/deploy/values.yaml))
+
+For instance on applications/myapp/deploy/values.yaml:
+
+```yaml
+harness:
+  subdomain: myapp
+```
+
+The above configurations put together create an ingress configuration for https://myapp.ch.org and automatically configure letsencrypt to create and renew certificates.
+
+Note:
+that the tls and letsencrypt configurations are enabled by default but should usually be disabled locally with
+
+```
+harness-deployment ... -dtls -l
+```
+
+## Main application
+
+The "main" application is deployed on the base domain.
+In order to specify a main application, override the value in your `/deployment-configuration/values-template.yaml` file.
+
+Example
+```yaml
+mainapp: myapp
+```
+This creates a reverse proxy to https://ch.org pointing to myapp
+
+## Proxy configurations
+
+Ingress is a reverse proxy and as such has some configurations to take into account.
+The most common configurations are connection timeouts and payload size.
+
+To configure it, override the following values in your `deployment-configuration/values-template.yaml` file.
+
+```yaml
+proxy:
+  timeout:
+    # -- Timeout for proxy connections in seconds.
+    send: 60
+    # -- Timeout for proxy responses in seconds.
+    read: 60
+    keepalive: 60
+  payload:
+    # -- Maximum size of payload in MB
+    max: 250
+```
+
+Note that in the case that gatekeepers are enabled, the same configurations are applied
+to the gatekeepers, unless the application override them on `harness.proxy.*`.
+See also the [gatekeepers documentation](./accounts.md#secure-and-enpoint-with-the-gatekeeper).