Merge branch 'release/2.1.0' of github.com:MetaCell/cloud-harness into release/2.1.0

Author: filippomc

.github/workflows/trivy-analysis.yml

@@ -30,7 +30,7 @@ jobs:
 
       - name: Build an image from Dockerfile
         run: |
-          docker build -t cloudharness-base:${{ github.sha }} . -f ./infrastructure/base-images/cloudharness-base/Dockerfile
+          DOCKER_BUILDKIT=1 docker build -t cloudharness-base:${{ github.sha }} . -f ./infrastructure/base-images/cloudharness-base/Dockerfile
 
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@2a2157eb22c08c9a1fac99263430307b8d1bc7a2

deployment-configuration/helm/templates/auto-database.yaml

@@ -31,6 +31,8 @@ kind: Deployment
 metadata:
   name: {{ .app.harness.database.name | quote }}
   namespace: {{ .root.Values.namespace }}
+  labels:
+    usesvolume: {{ .app.harness.database.name }}
 spec:
   replicas: 1
   selector:
@@ -41,7 +43,18 @@ spec:
       labels:
         app: {{ .app.harness.database.name | quote }}
         service: db
+        usesvolume: {{ .app.harness.database.name }}
     spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: usesvolume
+                    operator: In
+                    values:
+                      - {{ .app.harness.database.name }}
+              topologyKey: "kubernetes.io/hostname"
       containers:
       - name: {{ .app.harness.database.name | quote }}
         imagePullPolicy: IfNotPresent
@@ -66,10 +79,19 @@ spec:
             mountPath: {{ (printf "%s/%s/%s" .root.Values.backup.dir .app.harness.database.type .app.harness.database.name) | quote }}
             readOnly: true
           {{- end }}
+          {{- if eq .app.harness.database.type "postgres" }}
+          - mountPath: /dev/shm
+            name: dshm  
+          {{- end }}  
       volumes:
       - name: {{ .app.harness.database.name | quote }}
         persistentVolumeClaim:
           claimName: {{ .app.harness.database.name | quote }}
+      {{- if eq .app.harness.database.type "postgres" }}
+      - emptyDir:
+          medium: Memory
+        name: dshm     
+      {{- end }}     
       {{- if .root.Values.backup.active }}
       - name: "db-backups"
         persistentVolumeClaim:

infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/migrations/0001_initial.py

@@ -28,7 +28,7 @@ class Migration(migrations.Migration):
             name='Member',
             fields=[
                 ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('kc_id', models.CharField(max_length=100)),
+                ('kc_id', models.CharField(max_length=100, db_index=True)),
                 ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
             ],
         ),

infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/models.py

@@ -13,7 +13,7 @@ def __str__(self):
 
 class Member(models.Model):
     user = models.OneToOneField(User, on_delete=models.CASCADE)
-    kc_id = models.CharField(max_length = 100)
+    kc_id = models.CharField(max_length = 100, db_index=True)
 
     def __str__(self):
         return f"{self.user.first_name} {self.user.last_name}"

infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/auth.py

@@ -21,8 +21,12 @@ class AuthorizationLevel(Enum):
 
 # create the auth client
 if os.path.isfile(ALLVALUES_PATH):
+    try:
     # CH values exists so running with a valid config
-    auth_client = AuthClient(os.getenv("ACCOUNTS_ADMIN_USERNAME", None), os.getenv("ACCOUNTS_ADMIN_PASSWORD", None))
+        auth_client = AuthClient(os.getenv("ACCOUNTS_ADMIN_USERNAME", None), os.getenv("ACCOUNTS_ADMIN_PASSWORD", None))
+    except:
+        log.exception("Failed to initialize auth client")
+        auth_client = None
 else:
     auth_client = None
 

infrastructure/common-images/cloudharness-django/libraries/cloudharness-django/cloudharness_django/services/user.py

@@ -3,6 +3,12 @@
 from cloudharness_django.models import Team, Member
 from cloudharness_django.services.auth import AuthorizationLevel
 
+def get_user_by_kc_id(kc_id) -> User:
+    try:
+        return Member.objects.get(kc_id=kc_id).user
+    except Member.DoesNotExist:
+        return None
+
 class UserService:
     def __init__(self, auth_service):
         self.auth_service = auth_service
@@ -28,6 +34,7 @@ def _map_kc_user(self, user, kc_user=None, is_superuser=False, delete=False):
         user.username = kc_user.get("username", kc_user["email"])
         user.first_name = kc_user.get("firstName", "")
         user.last_name = kc_user.get("lastName", "")
+        user.email = kc_user.get("email", "")
 
         user.is_active = kc_user.get("enabled", delete)
         return user
@@ -98,7 +105,7 @@ def sync_kc_user(self, kc_user, is_superuser=False, delete=False):
 
     def sync_kc_user_groups(self, kc_user):
         # Sync the user usergroups and memberships
-        user = User.objects.get(email=kc_user["email"])
+        user = User.objects.get(username=kc_user["email"])
         user_groups = []
         for kc_group in kc_user["userGroups"]:
             user_groups += [Group.objects.get(name=kc_group["name"])]

libraries/cloudharness-common/cloudharness/__init__.py

@@ -46,8 +46,9 @@ def init(appname: str):
     if not appname:
         raise NotCorrectlyInitialized
     try:
-        from cloudharness import sentry
-        sentry.init(appname)
+        from cloudharness import sentry, applications
+        if applications.get_current_configuration().is_sentry_enabled():
+            sentry.init(appname)
     except Exception as e:
         log.warning(f'Error enabling Sentry for {appname}', exc_info=True)
 

libraries/cloudharness-common/cloudharness/auth/quota.py

@@ -5,6 +5,8 @@
 from cloudharness import log
 
 # quota tree node to hold the tree quota attributes
+
+
 class QuotaNode:
     def __init__(self, name, attrs):
         self.attrs = attrs
@@ -27,14 +29,20 @@ def _filter_quota_attrs(attrs, valid_keys_map):
     return valid_attrs
 
 
+def get_group_quotas(group, application_config: ApplicationConfig):
+    base_quotas = application_config.get("harness", {}).get("quotas", {})
+    valid_keys_map = {key for key in base_quotas}
+    return _compute_quotas_from_tree(_construct_quota_tree([group], valid_keys_map))
+
+
 def _construct_quota_tree(groups, valid_keys_map) -> QuotaNode:
     root = QuotaNode("root", {})
     for group in groups:
         r = root
         paths = group["path"].split("/")[1:]
         # loop through all segements except the last segment
         # the last segment is the one we want to add the attributes to
-        for segment in paths[0 : len(paths) - 1]:
+        for segment in paths[0: len(paths) - 1]:
             for child in r.children:
                 if child.name == segment:
                     r = child
@@ -56,13 +64,13 @@ def _construct_quota_tree(groups, valid_keys_map) -> QuotaNode:
 def _compute_quotas_from_tree(node: QuotaNode):
     """Recursively traverse the tree and find the quota per level
     the lower leafs overrule parent leafs values
-    
+
     Args:
         node (QuotaNode): the quota tree of QuotaNodes of the user for the given application
-        
+
     Returns:
         dict: key/value pairs of the quotas
-        
+
     Example:
         {'quota-ws-maxcpu': 1000, 'quota-ws-open': 10, 'quota-ws-max': 8}
 
@@ -96,24 +104,23 @@ def _compute_quotas_from_tree(node: QuotaNode):
     return node.attrs
 
 
-def get_user_quotas(application_config: ApplicationConfig =None, user_id: str=None) -> dict:
+def get_user_quotas(application_config: ApplicationConfig = None, user_id: str = None) -> dict:
     """Get the user quota from Keycloak and application
-    
+
     Args:
         application_config (ApplicationConfig): the application config to use for getting the quotas
         user_id (str): the Keycloak user id or username to get the quotas for
-    
+
     Returns:
         dict: key/value pairs of the user quota
-        
+
     Example:
         {'quota-ws-maxcpu': 1000, 'quota-ws-open': 10, 'quota-ws-max': 8}
     """
     if not application_config:
         application_config = get_current_configuration()
 
-    
-    base_quotas = application_config.get("harness",{}).get("quotas", {})
+    base_quotas = application_config.get("harness", {}).get("quotas", {})
     try:
         auth_client = AuthClient()
         if not user_id:
@@ -123,11 +130,8 @@ def get_user_quotas(application_config: ApplicationConfig =None, user_id: str=No
         log.warning("Quotas not available: error retrieving user: %s", user_id)
         return base_quotas
 
-    valid_keys_map = []
-    
-    for key in base_quotas:
-        valid_keys_map.append(key)
-    
+    valid_keys_map = {key for key in base_quotas}
+
     group_quotas = _compute_quotas_from_tree(
         _construct_quota_tree(
             user["userGroups"],

libraries/cloudharness-common/cloudharness/utils/env.py

@@ -5,6 +5,7 @@
 from .. import log
 
 from .config import CloudharnessConfig as conf
+from ..applications import get_configuration
 
 TEST = 'TEST'
 PROD = 'PROD'
@@ -105,36 +106,35 @@ def get_sub_variable(*vars):
 
 
 def get_service_public_address(app_name):
-    return ".".join([get_sub_variable('CH', app_name.upper(), 'SUBDOMAIN'), get_public_domain()])
+    return get_configuration(app_name).get_public_address()
 
 
 def get_public_domain():
     return get_variable('CH_DOMAIN')
 
 
 def get_cloudharness_workflows_service_url():
-    return get_service_public_address('WORKFLOWS')
+    return get_service_public_address('workflows')
 
 def get_cloudharness_sentry_service_url():
-    return 'https://' + get_service_public_address('sentry')
+    return get_configuration('sentry').get_public_address()
 
 def get_sentry_service_cluster_address():
-    sentry_app = conf.get_application_by_filter(name='sentry')[0]
-    return f'http://{sentry_app.name}:{sentry_app.port}'
+    return get_configuration('sentry').get_service_address()
 
 def get_cloudharness_common_service_url():
-    return 'https://' + get_service_public_address('common')
+    return get_configuration('common').get_public_address()
 
 def get_common_service_cluster_address():
-    common_app = conf.get_application_by_filter(name='common')[0]
-    return f'http://{common_app.name}:{common_app.port}'
+    common_app = get_configuration('common')
+    return common_app.get_service_address()
 
 def get_auth_service_cluster_address():
-    return get_service_cluster_address('ACCOUNTS')
+    return get_configuration('accounts').get_service_address()
 
 
 def get_auth_service_url():
-    return get_service_public_address('ACCOUNTS')
+    return get_configuration('accounts').get_public_address()
 
 
 def get_auth_realm():

libraries/models/cloudharness_model/util.py

@@ -113,7 +113,8 @@ def deserialize_model(data, klass):
         instance = klass()
     except:
         raise
-    instance._raw_dict = data
+    if isinstance(data, dict):
+        instance._raw_dict = data
 
     if not hasattr(instance, "openapi_types") or isinstance(data, klass):
         return data