Merge pull request #366 from MetaCell/release/0.4.0

Release/0.4.0

Author: zsinnema

LICENSE

@@ -0,0 +1,27 @@
+The MIT License (MIT)
+
+Copyright © 2021, 2022 MetaCell LLC, Ltd.
+http://metacell.us
+
+All rights reserved. This program and the accompanying materials
+are made available under the terms of the MIT License
+which accompanies this distribution, and is available at
+http://opensource.org/licenses/MIT
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -15,6 +15,23 @@ What building your cluster application with CloudHarness gives to you:
   * Log in and user management - based on Keycloak
   * Submit batch and asynchronous workflows - based on Argo
   * Orchestrate Micro-services - based on Kafka
+  
+## Why CloudHarness?
+
+The micro-service architecture is a great to get code separation and flexible development, but may not be of easy
+implementation, especially for small development teams/projects.
+In particular, these questions may rise:
+ - How do I create a deployment for my microservices?
+ - How do I orchestrate my microservices?
+ - How to create consistent api documentation?
+ - Do I need to be an experienced devops to create a micro-service based application?
+ - Wouldn't it be nice to develop a plain database/backend/frontend application without infrastructure boilerplate but still be able to configure everything I want when needed?
+ - How to run batch operations like ETL processes easily and efficiently in a cloud environment?
+ - How to manage databases without being locked to a specific vendor solution?
+ - How to perform database backups?
+ - How to manage secret data?
+ - What about having a precounfigured account management application? 
+ - Sooner rather than later I'll need an orchestration queue. Why not have that just ready to use? 
 
 ## Command line tools
 
@@ -29,6 +46,7 @@ CloudHarness provides the following command line tools to help application scaff
 
 ### Prerequisites
 
+#### Python
 Python 3.7+ must be installed.
 
 It is recommended to setup a virtual environment.
@@ -37,7 +55,19 @@ With conda:
  conda create --name ch python=3.7
  conda activate ch
  ```
+#### Docker
+[Docker](https://www.docker.com) is required to build on local terminal.
 
+#### Kubernetes command line client
+
+[kubectl](https://kubernetes.io/docs/tasks/tools/)
+
+#### Helm
+#### Skaffold
+
+Skaffold 
+
+### Install CloudHarness command line client
 Install requirements:
 
 ```bash
@@ -73,12 +103,7 @@ for instance with a Google Cloud cluster or a local Minikube.
 [--registry localhost:5000] [--tag 0.0.1]`
 1. Create the namespace `kubectl create ns ch`
 1. Create the namespace `kubectl create ns argo-workflows`
-1. (Optional) Try the helm chart with `helm install ch deployment/helm -n ch --dry-run`
-1. Install or upgrade the helm chart with `helm install ch deployment/helm  -n ch --dependency-update` on helm 3)
-
-To upgrade an already existing chart, run
-`helm upgrade ch deployment/helm -n ch --install --reset-values`
-
+1. Build images and Install or upgrade the helm chart with `skaffold deploy`
 
 Can also use Skaffold for local build and deploy (see [here](#skaffold)). 
 

applications/README.md

@@ -19,9 +19,8 @@ are automatically generated with the script `utilities/openapi-generate.py`
 After generating the codeChange the Dockerfile in order to inherit from the main Docker file:
 
 ```dockerfile
-ARG REGISTRY
-ARG TAG=latest
-FROM ${REGISTRY}cloudharness-base:${TAG}
+ARG CLOUDHARNESS_BASE
+FROM $CLOUDHARNESS_BASE
 ```
 
 The only code that should be modified shall go inside `src/[PACKAGE_NAME]/controllers`.

applications/common/server/Dockerfile

@@ -1,6 +1,4 @@
-ARG REGISTRY
-ARG TAG=latest
-ARG CLOUDHARNESS_FLASK=${REGISTRY}cloudharness-flask:${TAG}
+ARG CLOUDHARNESS_FLASK
 
 FROM $CLOUDHARNESS_FLASK
 

applications/events/deploy/resources/broker/server.properties

@@ -13,7 +13,7 @@ default.replication.factor=3
 
 min.insync.replicas=2
 
-auto.create.topics.enable=false
+auto.create.topics.enable=true
 
 # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
 # This value is recommended to be increased for installations with data dirs located in RAID array.

applications/events/deploy/templates/deployments.yml

@@ -58,12 +58,7 @@ spec:
           initialDelaySeconds: 30
           periodSeconds: 30
         resources:
-          limits:
-            cpu: 500m
-            memory: 600Mi
-          requests:
-            cpu: 50m
-            memory: 100Mi
+          {{- .Values.apps.events.kafka.resources | toYaml | nindent 10 }}
         volumeMounts:
         - mountPath: /etc/kafka
           name: config
@@ -116,7 +111,7 @@ spec:
       - ReadWriteOnce
       resources:
         requests:
-          storage: 10Gi
+          storage: {{ .Values.apps.events.kafka.storage }}
       storageClassName: standard
 ---
 apiVersion: apps/v1
@@ -181,11 +176,7 @@ spec:
           initialDelaySeconds: 12
           periodSeconds: 60
         resources:
-          limits:
-            memory: 500Mi
-          requests:
-            cpu: 10m
-            memory: 100Mi
+          {{- .Values.apps.events.pzoo.resources | toYaml | nindent 10 }}
         volumeMounts:
         - mountPath: /etc/kafka
           name: config
@@ -226,7 +217,7 @@ spec:
       - ReadWriteOnce
       resources:
         requests:
-          storage: 1Gi
+          storage: {{ .Values.apps.events.pzoo.storage }}
       storageClassName: standard
 ---
 apiVersion: apps/v1
@@ -290,12 +281,9 @@ spec:
           timeoutSeconds: 3
           initialDelaySeconds: 12
           periodSeconds: 60
+
         resources:
-          limits:
-            memory: 500Mi
-          requests:
-            cpu: 10m
-            memory: 100Mi
+          {{- .Values.apps.events.zoo.resources | toYaml | nindent 10 }}
         volumeMounts:
         - mountPath: /etc/kafka
           name: config
@@ -338,5 +326,5 @@ spec:
       - ReadWriteOnce
       resources:
         requests:
-          storage: 1Gi
+          storage: {{ .Values.apps.events.zoo.storage }}
       storageClassName: standard

applications/events/deploy/values-prod.yaml

@@ -0,0 +1,8 @@
+kafka:
+  resources:
+    limits:
+      cpu: 500m
+      memory: 4000Mi
+    requests:
+      cpu: 50m
+      memory: 100Mi

applications/events/deploy/values.yaml

@@ -10,7 +10,31 @@ harness:
     name: events
   env:
     - name: ZK_HOSTS
-      value: zookeeper:2181    
+      value: zookeeper:2181
 kafka:
   name: bootstrap
-  port: 9092
+  port: 9092
+  storage: 10Gi
+  resources:
+    limits:
+      cpu: 500m
+      memory: 600Mi
+    requests:
+      cpu: 50m
+      memory: 100Mi
+pzoo:
+  storage: 1Gi
+  resources:
+    limits:
+      memory: 500Mi
+    requests:
+      cpu: 10m
+      memory: 100Mi
+zoo:
+  storage: 1Gi
+  resources:
+    limits:
+      memory: 500Mi
+    requests:
+      cpu: 10m
+      memory: 100Mi

applications/jupyterhub/Dockerfile

@@ -1,14 +1,18 @@
+ARG CLOUDHARNESS_BASE
+FROM $CLOUDHARNESS_BASE as base
+
 FROM jupyterhub/k8s-hub:0.9.0
 USER root
 
-# COPY deploy/resources/hub/* /etc/jupyterhub/
-# COPY src/jupyterhub/jupyterhub/handlers/* /usr/local/lib/python3.6/dist-packages/jupyterhub/handlers/
-# COPY ./src/kubespawner /usr/local/lib/python3.6/dist-packages/
+COPY --from=base /libraries/cloudharness-common /libraries/cloudharness-common
+COPY --from=base /libraries/client/cloudharness_cli /libraries/client/cloudharness_cli
+
+RUN pip install -e /libraries/cloudharness-common
+RUN pip install -e /libraries/client/cloudharness_cli
+
 COPY src src
 RUN pip install ./src/harness_jupyter
 RUN pip install ./src/chauthenticator
+
 RUN chmod 777 /usr/local/lib/python3.6/dist-packages/  -R
 USER jovyan
-
-
-# CMD ["jupyterhub", "--config", "/srv/jupyterhub_config.py"]

applications/jupyterhub/deploy/resources/hub/jupyterhub_config.py

@@ -282,7 +282,7 @@ def camelCaseify(s):
         ('client_secret', None),
         ('oauth_callback_url', 'callbackUrl'),
 )
-
+print("Auth type", auth_type)
 if auth_type == 'google':
     c.JupyterHub.authenticator_class = 'oauthenticator.GoogleOAuthenticator'
     for trait, cfg_key in common_oauth_traits + (
@@ -383,6 +383,29 @@ def camelCaseify(s):
     auth_class_name = full_class_name.rsplit('.', 1)[-1]
     auth_config = c[auth_class_name]
     auth_config.update(get_config('auth.custom.config') or {})
+elif auth_type == 'keycloak':
+    from cloudharness.applications import get_configuration
+    from cloudharness.utils.config import CloudharnessConfig
+    from oauthenticator.generic import GenericOAuthenticator
+
+    accounts_app = get_configuration('accounts')
+
+    accounts_url = accounts_app.get_public_address()
+    client_id = accounts_app.conf.webclient.id
+    client_secret = accounts_app.conf.webclient.secret
+    realm = CloudharnessConfig.get_namespace()
+
+    c.JupyterHub.authenticator_class = GenericOAuthenticator
+    c.Authenticator.auto_login = True
+    c.OAuthenticator.client_id = client_id
+    c.OAuthenticator.client_secret = client_secret
+
+    c.GenericOAuthenticator.login_service = "CH"
+    c.GenericOAuthenticator.username_key = "email"
+    c.GenericOAuthenticator.authorize_url = f"{accounts_url}/auth/realms/{realm}/protocol/openid-connect/auth"
+    c.GenericOAuthenticator.token_url = f"{accounts_url}/auth/realms/{realm}/protocol/openid-connect/token"
+    c.GenericOAuthenticator.userdata_url = f"{accounts_url}/auth/realms/{realm}/protocol/openid-connect/userinfo"
+    c.GenericOAuthenticator.userdata_params = {'state': 'state'}
 else:
     raise ValueError("Unhandled auth type: %r" % auth_type)
 
@@ -518,4 +541,5 @@ def camelCaseify(s):
     exec(config_py)
 
 c.apps = get_config('apps')
-c.registry = get_config('registry')
+c.registry = get_config('registry')
+c.domain = get_config('root.domain')