Handle as_text correctly in cache

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

etc/scripts/utils_thirdparty.py

@@ -172,11 +172,20 @@ def fetch_wheels(
     else:
         force_pinned = False
 
-    rrp = list(get_required_remote_packages(
-        requirements_file=requirements_file,
-        force_pinned=force_pinned,
-        remote_links_url=remote_links_url,
-    ))
+    try:
+        rrp = list(get_required_remote_packages(
+            requirements_file=requirements_file,
+            force_pinned=force_pinned,
+            remote_links_url=remote_links_url,
+        ))
+    except Exception as e:
+        raise Exception(
+            dict(
+                requirements_file=requirements_file,
+                force_pinned=force_pinned,
+                remote_links_url=remote_links_url,
+            )
+        ) from e
 
     fetched_filenames = set()
     for name, version, package in rrp:
@@ -211,6 +220,7 @@ def fetch_wheels(
             print(f'Missed package {nv} in remote repo, has only:')
             for pv in rr.get_versions(n):
                 print('  ', pv)
+        raise Exception('Missed some packages in remote repo')
 
 
 def fetch_sources(
@@ -261,6 +271,8 @@ def fetch_sources(
             fetched = package.fetch_sdist(dest_dir=dest_dir)
             error = f'Failed to fetch' if not fetched else None
             yield package, error
+    if missed:
+        raise Exception(f'Missing source packages in {remote_links_url}', missed)
 
 ################################################################################
 #
@@ -693,8 +705,7 @@ def save_if_modified(location, content):
                     return False
 
             if TRACE: print(f'Saving ABOUT (and NOTICE) files for: {self}')
-            wmode = 'wb' if isinstance(content, bytes) else 'w'
-            with open(location, wmode, encoding="utf-8") as fo:
+            with open(location, 'w') as fo:
                 fo.write(content)
             return True
 
@@ -1845,7 +1856,7 @@ def get(self, path_or_url, as_text=True):
         if not os.path.exists(cached):
             content = get_file_content(path_or_url=path_or_url, as_text=as_text)
             wmode = 'w' if as_text else 'wb'
-            with open(cached, wmode, encoding="utf-8") as fo:
+            with open(cached, wmode) as fo:
                 fo.write(content)
             return content
         else:
@@ -1857,7 +1868,7 @@ def put(self, filename, content):
         """
         cached = os.path.join(self.directory, filename)
         wmode = 'wb' if isinstance(content, bytes) else 'w'
-        with open(cached, wmode, encoding="utf-8") as fo:
+        with open(cached, wmode) as fo:
             fo.write(content)
 
 
@@ -2331,7 +2342,7 @@ def get_required_remote_packages(
         repo = get_remote_repo(remote_links_url=remote_links_url)
     else:
         # a local path
-        assert os.path.exists(remote_links_url)
+        assert os.path.exists(remote_links_url), f'Path does not exist: {remote_links_url}'
         repo = get_local_repo(directory=remote_links_url)
 
     for name, version in required_name_versions:
@@ -2365,7 +2376,7 @@ def update_requirements(name, version=None, requirements_file='requirements.txt'
         updated_name_versions = sorted(updated_name_versions)
         nvs = '\n'.join(f'{name}=={version}' for name, version in updated_name_versions)
 
-        with open(requirements_file, 'w', encoding="utf-8") as fo:
+        with open(requirements_file, 'w') as fo:
             fo.write(nvs)
 
 
@@ -2383,7 +2394,7 @@ def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file='requirements.t
             raise Exception(f'Missing required package {name}=={version}')
         hashed.append(package.specifier_with_hashes)
 
-    with open(requirements_file, 'w', encoding="utf-8") as fo:
+    with open(requirements_file, 'w') as fo:
         fo.write('\n'.join(hashed))
 
 ################################################################################
@@ -2915,7 +2926,7 @@ def fetch_package_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR):
 
 def check_about(dest_dir=THIRDPARTY_DIR):
     try:
-        subprocess.check_output(f'venv/bin/about check {dest_dir}'.split())
+        subprocess.check_output(f'about check {dest_dir}'.split())
     except subprocess.CalledProcessError as cpe:
         print()
         print('Invalid ABOUT files:')
@@ -2953,7 +2964,6 @@ def find_problems(
 
     check_about(dest_dir=dest_dir)
 
-
 def compute_normalized_license_expression(declared_licenses):
     if not declared_licenses:
         return
@@ -2962,4 +2972,4 @@ def compute_normalized_license_expression(declared_licenses):
         return pypi.compute_normalized_license(declared_licenses)
     except ImportError:
         # Scancode is not installed, we join all license strings and return it
-        return ' '.join(declared_licenses)
+        return ' '.join(declared_licenses).lower()