Complete support for VM image extraction #16

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

CHANGELOG.rst

@@ -1,36 +1,13 @@
-Release notes
-=============
+Changelog
+=========
 
-vNext
------
+v (next)
+--------------
 
+ - Add support for VMDK, QCOW and VDI VM image filesystems extraction
 
-Version 21.1.21
----------------
 
-- Bump dependencies and use latest typecode and binaries. This is to fix
-  installation problems on multiple OSes.
+v20.10
+------
 
-
-Version 21.1.21
----------------
-
-- Add new [full] extra requires that install all the dependencies
-- Fix bug related to commoncode libraries loading
-- Improve the extra requirements
-- Set minimum version for dependencies
-- Improve documentation
-
-
-Version 21.1.15
----------------
-
-- Drop support for Python 2
-- Use the latest CommonCode and TypeCode libraries
-- Add azure-pipelines CI support
-
-
-Version 20.10
--------------
-
-- Initial release.
+ - Initial release as a split from ScanCode toolkit

README.rst

@@ -7,17 +7,26 @@ ExtractCode
 - keywords: archive, extraction, libarchive, 7zip, scancode-toolkit
 
 
-ExtractCode is a universal archive extractor. It uses behind the scenes 
-the Python standard library, a custom ctypes binding to libarchive and
-the 7zip command line to extract a large number of common and
-less common archives and compressed files. It tries to extract things
-in the same way on all OSes, including auto-renaming files that would
-not have valid names on certain filesystems or when there are multiple
-copies of the same path in a given archive.
+ExtractCode is a universal archive extractor. It uses behind the scenes
+multiple tools such as:
+
+- the Python standard library, 
+- a custom ctypes binding to libarchive,
+- the 7zip command line
+- optionally libguestfs on Linux
+
+With these it is possible to extract a large number of common and
+
+less common archives and compressed files. ExtractCode tries to extract things
+in the same way on all OSes, including auto-renaming files that would not have
+valid names on certain filesystems or when there are multiple copies of the same
+path in a given archive (which is possible in a tar).
+
 The extraction is driven from  a "voting" system that considers the
-file extension(s) and name, the file type and mime type (using a ctypes
+file extension(s) and name, the filetype and mimetype (using a ctypes
 binding to libmagic) to select the most appropriate extractor or
-uncompressor function. It can handle multi-level archives such as tar.gz.
+decompressor function. It can handle multi-level archives such as tar.gz and
+can extract recursively nested archives.
 
 
 
@@ -36,3 +45,44 @@ To clean up development environment::
     ./configure --clean
 
 
+To run the command line tool in the activated environment::
+
+    ./extractcode -h
+
+
+Adding support for VM images
+----------------------------
+
+Adding support for VM images requires the manual installation of libguestfs and
+it Python binding. You will need to install the libguestfs tools system package.
+On Debian and Ubuntu::
+
+    sudo apt-get install libguestfs-tools
+
+
+On Ubuntu, a manual stpe is required if the kernel executable file cannot be read.
+This is required by guestfish and libguestfs and this is an oddity there and not on Debian.
+
+Run this command as a temporary fix::
+
+    for k in /boot/vmlinuz-*
+        do sudo dpkg-statoverride --add --update root root 0644 /boot/vmlinuz-$(uname -r)
+    done
+
+or::
+
+    sudo chmod +r /boot/vmlinuz-*,
+
+
+For a permanent fix see: 
+
+    - https://bugs.launchpad.net/ubuntu/+source/libguestfs/+bug/1813662/comments/21
+
+See also for a discussion:
+
+    - https://bugs.launchpad.net/ubuntu/+source/linux/+bug/759725
+    - https://bugzilla.redhat.com/show_bug.cgi?id=1670790
+    - https://bugs.launchpad.net/ubuntu/+source/libguestfs/+bug/1813662
+
+
+

azure-pipelines.yml

@@ -21,15 +21,18 @@ jobs:
           image_name: ubuntu-18.04
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
-              all: tmp/bin/pytest -n 2 -vvs
+              all:
+                - apt-get install libguestfs-tools
+                - tmp/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-linux.yml
       parameters:
           job_name: ubuntu20_cpython
           image_name: ubuntu-20.04
           python_versions: ['3.6', '3.7', '3.8', '3.9']
           test_suites:
-              all: tmp/bin/pytest -n 2 -vvs
+              all: 
+                - tmp/bin/pytest -n 2 -vvs
 
     - template: etc/ci/azure-mac.yml
       parameters:

setup.cfg

@@ -35,6 +35,7 @@ packages = find:
 include_package_data = true
 zip_safe = false
 install_requires =
+    attrs >= 18.1, !=20.1.0
     commoncode >= 21.1.21
     plugincode >= 21.1.21
     typecode >= 21.2.23

src/extractcode/archive.py

@@ -221,10 +221,11 @@ def get_handlers(location):
                 extension_matched = exts and location.lower().endswith(exts)
 
             if TRACE_DEEP:
-                logger.debug('  get_handlers: matched type: %(type_matched)s, mime: %(mime_matched)s, ext: %(extension_matched)s' % locals())
+                print(f'  get_handlers: matched type: {type_matched}, mime: {mime_matched}, ext: {extension_matched}' % locals())
 
-            if handler.strict and not all([type_matched, mime_matched, extension_matched]):
-                logger.debug('  get_handlers: skip strict' % locals())
+            if handler.strict and not (type_matched and mime_matched and extension_matched):
+                if TRACE_DEEP:
+                    print(f'  get_handlers: skip strict: {handler.name}')
                 continue
 
             if type_matched or mime_matched or extension_matched:
@@ -1052,8 +1053,8 @@ def try_to_extract(location, target_dir, extractor):
     mimetypes=('application/octet-stream',),
     extensions=('.qcow2',),
     kind=file_system,
-    extractors=[extract_vm_image, extract_tar],
-    strict=False,
+    extractors=[extract_vm_image],
+    strict=True,
 )
 
 VMDKHandler = Handler(
@@ -1062,7 +1063,7 @@ def try_to_extract(location, target_dir, extractor):
     mimetypes=('application/octet-stream',),
     extensions=('.vmdk',),
     kind=file_system,
-    extractors=[extract_vm_image, extract_tar],
+    extractors=[extract_vm_image],
     strict=True,
 )
 
@@ -1072,7 +1073,7 @@ def try_to_extract(location, target_dir, extractor):
     mimetypes=('application/octet-stream',),
     extensions=('.vdi',),
     kind=file_system,
-    extractors=[extract_vm_image, extract_tar],
+    extractors=[extract_vm_image],
     strict=True,
 )
 

src/extractcode/extract.py

@@ -30,7 +30,7 @@
 from commoncode import fileutils
 from commoncode import ignore
 
-import extractcode
+import extractcode  # NOQA
 import extractcode.archive
 
 logger = logging.getLogger(__name__)
@@ -61,7 +61,7 @@
 
  - Symlinks may be replaced by plain file copies as if they were regular files.
    Hardlinks may be recreated as regular files, not as hardlinks to the original
-   file.
+   files.
 
  - Files and directories may be renamed when their name is a duplicate. And a
    name may be considered a duplicate ignore upper and lower case mixes even