Implement new envt. variables approach

We now load native libraries and executables from:
1. an envt. variable path
2. OR a locatin provider plugin
3. OR the PATH
or we fail

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/extractcode/libarchive2.py

@@ -37,6 +37,7 @@
 from commoncode import fileutils
 from commoncode import paths
 from commoncode import text
+from commoncode.system import on_windows
 
 import extractcode
 from extractcode import ExtractError
@@ -83,26 +84,41 @@
 """
 
 # keys for plugin-provided locations
-EXTRACTCODE_LIBARCHIVE_LIBDIR = 'extractcode.libarchive.libdir'
 EXTRACTCODE_LIBARCHIVE_DLL = 'extractcode.libarchive.dll'
 
+EXTRACTCODE_LIBARCHIVE_PATH_ENVVAR = 'EXTRACTCODE_LIBARCHIVE_PATH'
+
 
 def load_lib():
     """
-    Return the loaded libarchive shared library object from plugin-provided path.
+    Return the libarchive shared library object loaded from either:
+    - an environment variable ``EXTRACTCODE_LIBARCHIVE_PATH``
+    - a plugin-provided path,
+    - the system PATH.
+    Raise an Exception if no libarchive can be found.
     """
     from plugincode.location_provider import get_location
 
-    dll = get_location(EXTRACTCODE_LIBARCHIVE_DLL)
-    libdir = get_location(EXTRACTCODE_LIBARCHIVE_LIBDIR)
-    if not (dll and libdir) or not os.path.isfile(dll) or not os.path.isdir(libdir):
+    # try the environment first
+    dll_loc = os.environ.get(EXTRACTCODE_LIBARCHIVE_PATH_ENVVAR)
+
+    # try a plugin-provided path second
+    if not dll_loc:
+        dll_loc = get_location(EXTRACTCODE_LIBARCHIVE_DLL)
+
+    # try the PATH
+    if not dll_loc:
+        dll = 'libarchive.dll' if on_windows else 'libarchive.so'
+        dll_loc = command.find_in_path(dll)
+
+    if not dll_loc or not os.path.isfile(dll_loc):
         raise Exception(
             'CRITICAL: libarchive DLL is not installed. '
             'Unable to continue: you need to install a valid extractcode-libarchive '
-            'plugin with a valid libarchive DLL available.'
+            'plugin with a valid libarchive DLL available. '
+            f'OR set the {EXTRACTCODE_LIBARCHIVE_PATH_ENVVAR} environment variable.'
     )
-    return command.load_shared_library(dll, libdir)
-    
+    return command.load_shared_library(dll_loc)
 
 
 def set_env_with_tz():

src/extractcode/sevenzip.py

@@ -56,10 +56,11 @@
     logging.basicConfig(stream=sys.stdout)
     logger.setLevel(logging.DEBUG)
 
-# keys for plugin-provided locations
-EXTRACTCODE_7ZIP_LIBDIR = 'extractcode.sevenzip.libdir'
+# key of a plugin-provided location
 EXTRACTCODE_7ZIP_EXE = 'extractcode.sevenzip.exe'
 
+EXTRACTCODE_7ZIP_PATH_ENVVAR = 'EXTRACTCODE_7Z_PATH'
+
 sevenzip_errors = [
     ('unsupported method', 'Unsupported archive or broken archive'),
     ('wrong password', 'Password protected archive, unable to extract'),
@@ -71,22 +72,40 @@
 UNKNOWN_ERROR = 'Unknown extraction error'
 
 
-def get_bin_locations():
+def get_command_location(_cache=[]):
     """
-    Return a tuple of (lib_dir, cmd_loc) for 7zip loaded from plugin-provided path.
+    Return the location of a 7zip loaded from either:
+    - an environment variable ``EXTRACTCODE_7Z_PATH``,
+    - a plugin-provided path,
+    - the system PATH.
+    Raise an Exception if no 7Zip command can be found.
     """
+    if _cache:
+        return _cache[0]
+
     from plugincode.location_provider import get_location
 
-    cmd_loc = get_location(EXTRACTCODE_7ZIP_EXE)
-    libdir = get_location(EXTRACTCODE_7ZIP_LIBDIR)
-    if not (cmd_loc and libdir) or not os.path.isfile(cmd_loc) or not os.path.isdir(libdir):
+    # try the environment first
+    cmd_loc = os.environ.get(EXTRACTCODE_7ZIP_PATH_ENVVAR)
+
+    # try a plugin-provided path second
+    if not cmd_loc:
+        cmd_loc = get_location(EXTRACTCODE_7ZIP_EXE)
+
+    # try the PATH
+    if not cmd_loc:
+        cmd = '7z.exe' if on_windows else '7z'
+        cmd_loc = command.find_in_path(cmd)
+
+    if not cmd_loc or not os.path.isfile(cmd_loc):
         raise Exception(
             'CRITICAL: 7zip executable is not installed. '
             'Unable to continue: you need to install a valid extractcode-7z '
-            'plugin with a valid executable available.'
+            'plugin with a valid executable available. '
+            'OR set the EXTRACTCODE_7ZIP_PATH environment variable.'
     )
-
-    return libdir, cmd_loc
+    _cache.append(cmd_loc)
+    return cmd_loc
 
 
 def get_7z_errors(stdout, stderr):
@@ -315,12 +334,11 @@ def build_7z_extract_command(location, target_dir, single_entry=None, arch_type=
     if single_entry:
         args += [shlex_quote(single_entry.path)]
 
-    lib_dir, cmd_loc = get_bin_locations()
+    cmd_loc = get_command_location()
 
     ex_args = dict(
         cmd_loc=cmd_loc,
         args=args,
-        lib_dir=lib_dir,
         cwd=target_dir,
         env=timezone,
     )
@@ -387,7 +405,7 @@ def extract_file_by_file(location, target_dir, arch_type='*', skip_symlinks=True
             single_entry=entry,
             arch_type=arch_type,
         )
-        rc, stdout, stderr = command.execute2(**ex_args)
+        rc, stdout, stderr = command.execute(**ex_args)
 
         error = get_7z_errors(stdout, stderr)
         if error or rc != 0:
@@ -488,7 +506,7 @@ def list_entries(location, arch_type='*'):
         abs_location,
     ]
 
-    lib_dir, cmd_loc = get_bin_locations()
+    cmd_loc = get_command_location()
 
     rc, stdout, stderr = command.execute2(
         cmd_loc=cmd_loc,

src/extractcode/vmimage.py

@@ -38,12 +38,6 @@
 Works only if libguestfs tool guestfish is in the path.
 
 See https://libguestfs.org/
-
-On Ubuntu, you may face this issue when running guestfish:
-
-- https://bugs.launchpad.net/ubuntu/+source/linux/+bug/759725
-- https://bugs.launchpad.net/ubuntu/+source/libguestfs/+bug/1813662
-- https://unix.stackexchange.com/a/642914/185837
 """
 
 logger = logging.getLogger(__name__)
@@ -59,9 +53,16 @@
     'WARNING: guestfish executable is not installed. '
     'Unable to extract virtual machine image: you need to install the '
     'guestfish tool from libguestfs and extra FS drivers if needed. '
-    'See https://libguestfs.org/ for details.'
+    'See the ExtractCode README.rst and https://libguestfs.org/ for details.'
 )
 
+GUESTFISH_KERNEL_NOT_READABLE = (
+'''libguestfs requires the kernel executable to be readable.
+This is the case by default on most Linux distributions except on Ubuntu.
+Please follow the instructions in ExtractCode installation guide to make this happen.
+See the ExtractCode README.rst for details.
+''')
+
 EXTRACTCODE_GUESTFISH_PATH_ENVVAR = 'EXTRACTCODE_GUESTFISH_PATH'
 
 
@@ -89,26 +90,15 @@ def check_linux_kernel_is_readable():
         - https://bugzilla.redhat.com/show_bug.cgi?id=1670790
         - https://bugs.launchpad.net/ubuntu/+source/libguestfs/+bug/1813662
     """
-    error = (
-        'libguestfs requires the kernel executable to be readable. '
-        'This is the case on most Linux distribution except on Ubuntu.\n'
-        'Run this command as a temporary fix:\n'
-        '  for k in /boot/vmlinuz-*\n'
-        '    do sudo dpkg-statoverride --add --update root root 0644 /boot/vmlinuz-$(uname -r)\n'
-        '  done\n'
-        'or:\n'
-        '  sudo chmod +r /boot/vmlinuz-*\n\n',
-        'For a permanent fix see: '
-        'https://bugs.launchpad.net/ubuntu/+source/libguestfs/+bug/1813662/comments/21'
-    )
+
     if on_linux:
         kernels = list(pathlib.Path('/boot').glob('vmlinuz-*'))
         if not kernels:
-            raise ExtractErrorFailedToExtract(error)
+            raise ExtractErrorFailedToExtract(GUESTFISH_KERNEL_NOT_READABLE)
         for kern in kernels:
             if not os.access(kern, os.R_OK):
                 raise ExtractErrorFailedToExtract(
-                    f'Unable to read kernel at: {kern}.\n{error}')
+                    f'Unable to read kernel at: {kern}.\n{GUESTFISH_KERNEL_NOT_READABLE}')
 
 
 @attr.s
@@ -250,7 +240,7 @@ def run_guestfish(self, args, timeout=None):
         return as_unicode(stdout)
 
 
-def extract(location, target_dir, as_tarballs=True):
+def extract(location, target_dir, as_tarballs=False):
     """
     Extract all files from a guestfish-supported VM image archive file at
     location in the target_dir directory. Optionally only extract the
@@ -306,8 +296,7 @@ def extract(location, target_dir, as_tarballs=True):
             # we can safely extract this to a root / dir as we have only one partition
             partition, _parttype = partitions[0]
             if not as_tarballs:
-                intermediate_dir = fileutils.get_temp_dir(prefix='extractcode-vmimage')
-                tdir = intermediate_dir
+                tdir = fileutils.get_temp_dir(prefix='extractcode-vmimage')
             else:
                 tdir = target_dir
 
@@ -329,8 +318,7 @@ def extract(location, target_dir, as_tarballs=True):
                 base_name = partition.replace('/', '-')
 
                 if not as_tarballs:
-                    intermediate_dir = fileutils.get_temp_dir(prefix='extractcode-vmimage')
-                    tdir = intermediate_dir
+                    tdir = fileutils.get_temp_dir(prefix='extractcode-vmimage')
                 else:
                     tdir = target_dir
 
@@ -344,7 +332,7 @@ def extract(location, target_dir, as_tarballs=True):
                     fileutils.create_dir(partition_target_dir)
                     warns = extract_image_tarball(
                         tarball=target_tarball,
-                        target_dir=target_dir,
+                        target_dir=partition_target_dir,
                         skip_symlinks=False)
                     warnings.extend(warns)