Refactor codefresh images caching check

Author: filippomc

tools/deployment-cli-tools/ch_cli_tools/codefresh.py

@@ -11,7 +11,7 @@
 from .models import HarnessMainConfig, ApplicationTestConfig, ApplicationHarnessConfig
 from cloudharness_utils.constants import *
 from .helm import KEY_APPS, KEY_TASK_IMAGES, KEY_TEST_IMAGES, generate_tag_from_content
-from .utils import find_dockerfiles_paths, get_app_relative_to_base_path, guess_build_dependencies_from_dockerfile, \
+from .utils import check_docker_manifest_exists, find_dockerfiles_paths, get_app_relative_to_base_path, guess_build_dependencies_from_dockerfile, \
     get_image_name, get_template, dict_merge, app_name_from_path, clean_path
 from cloudharness_utils.testing.api import get_api_filename, get_schemathesis_command, get_urls_from_api_file
 
@@ -33,7 +33,7 @@ def literal_presenter(dumper, data):
 
 yaml.add_representer(str, literal_presenter)
 
-def write_env_file(helm_values: HarnessMainConfig, filename):
+def write_env_file(helm_values: HarnessMainConfig, filename, registry_secret=None):
     env = {}
     logging.info("Create env file with image info %s", filename)
 
@@ -45,14 +45,15 @@ def check_image_exists(name, image):
         chunks = image.split(":")[0].split("/")
         registry = chunks[0] if "." in chunks[0] else "docker.io"
         image_name = "/".join(chunks[1::] if "." in chunks[0] else chunks[0::])
-        api_url = f"https://{registry}/v2/{image_name}/manifests/{tag}"
-        resp = requests.get(api_url)
-        if resp.status_code == 200:
+        exists = check_docker_manifest_exists(registry, image_name, tag, registry_secret=registry_secret)
+        if exists:
             # TODO the hash might be the same but not the parent's hash
             env[app_specific_tag_variable(name) + "_EXISTS"] = 1
         else:
             env[app_specific_tag_variable(name) + "_NEW"] = 1
 
+    
+
     for app in helm_values.apps.values():
         if app.harness and app.harness.deployment.image:
             env[app_specific_tag_variable(app.name)] = extract_tag(app.harness.deployment.image)

tools/deployment-cli-tools/ch_cli_tools/utils.py

@@ -2,6 +2,7 @@
 import socket
 import glob
 import subprocess
+import requests
 import os
 from functools import cache
 from os.path import join, dirname, isdir, basename, exists, relpath, sep, dirname as dn
@@ -33,11 +34,14 @@ def image_name_from_dockerfile_path(dockerfile_path, base_name=None):
 def app_name_from_path(dockerfile_path):
     return "-".join(p for p in dockerfile_path.split("/") if p not in NEUTRAL_PATHS)
 
+
 def clean_path(path):
     return "/".join(p for p in path.split("/") if p not in NEUTRAL_PATHS)
-    
+
+
 def get_app_relative_to_base_path(base_path, dockerfile_path):
-    return clean_path(relpath(dockerfile_path, base_path)) 
+    return clean_path(relpath(dockerfile_path, base_path))
+
 
 def get_sub_paths(base_path):
     return tuple(path for path in glob.glob(base_path + "/*") if isdir(path))
@@ -351,6 +355,7 @@ def merge_app_directories(root_paths, destination) -> None:
 def to_python_module(name):
     return name.replace('-', '_')
 
+
 @cache
 def guess_build_dependencies_from_dockerfile(filename):
     dependencies = []
@@ -367,4 +372,7 @@ def guess_build_dependencies_from_dockerfile(filename):
     return dependencies
 
 
-
+def check_docker_manifest_exists(registry, image_name, tag, registry_secret=None):
+    api_url = f"https://{registry}/v2/{image_name}/manifests/{tag}"
+    resp = requests.get(api_url)
+    return resp.status_code == 200

tools/deployment-cli-tools/tests/test_utils.py

@@ -59,3 +59,9 @@ def test_guess_build_dependencies_from_dockerfile():
 
     deps = guess_build_dependencies_from_dockerfile(os.path.join(HERE, "resources/applications/myapp/tasks/mytask"))
     assert len(deps) == 0
+
+
+def test_check_docker_manifest_exists():
+    assert check_docker_manifest_exists("gcr.io/metacellllc", "cloudharness/cloudharness-base", "latest")
+    assert not check_docker_manifest_exists("gcr.io/metacellllc", "cloudharness/cloudharness-base", "RANDOM_TAG")
+