Add new CLI option to support extracting all formats

With this all supported archive formats with be tried.

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

Author: pombredanne

src/extractcode/api.py

@@ -23,22 +23,37 @@
 """
 
 
-def extract_archives(location, recurse=True, replace_originals=False, ignore_pattern=()):
+def extract_archives(
+    location,
+    recurse=True,
+    replace_originals=False,
+    ignore_pattern=(),
+    all_formats=False,
+):
     """
     Yield ExtractEvent while extracting archive(s) and compressed files at
-    `location`. If `recurse` is True, extract nested archives-in-archives
-    recursively.
+    `location`.
+
+    If `recurse` is True, extract nested archives-in-archives recursively.
+    If `all_formats` is True, extract all supported archives formats.
+
     Archives and compressed files are extracted in a directory named
     "<file_name>-extract" created in the same directory as the archive.
+
     Note: this API is returning an iterable and NOT a sequence.
     """
+    
     from extractcode.extract import extract
     from extractcode import default_kinds
+    from extractcode import all_kinds
+
+    kinds = all_kinds if all_formats else default_kinds
+    
     for xevent in extract(
         location=location,
-        kinds=default_kinds,
+        kinds=kinds,
         recurse=recurse,
         replace_originals=replace_originals,
-        ignore_pattern=ignore_pattern
+        ignore_pattern=ignore_pattern,
     ):
         yield xevent

src/extractcode/archive.py

@@ -38,9 +38,10 @@
 from extractcode import patches
 from extractcode import special_package
 
+from extractcode import libarchive2
 from extractcode import patch
 from extractcode import sevenzip
-from extractcode import libarchive2
+
 from extractcode.uncompress import uncompress_gzip
 from extractcode.uncompress import uncompress_bzip2
 
@@ -79,8 +80,19 @@
  - http://en.wikipedia.org/wiki/List_of_file_formats#Archive_and_compressed
 """
 
-# if strict, all hanlders criteria must be matched for it to be selected
-Handler = namedtuple('Handler', ['name', 'filetypes', 'mimetypes', 'extensions', 'kind', 'extractors', 'strict'])
+# if strict, all handlers criteria must be matched for a handler to be selected
+Handler = namedtuple(
+    'Handler',
+    [
+        'name',
+        'filetypes',
+        'mimetypes',
+        'extensions',
+        'kind',
+        'extractors',
+        'strict',
+    ]
+)
 
 
 def can_extract(location):
@@ -96,13 +108,17 @@ def can_extract(location):
 
 def should_extract(location, kinds, ignore_pattern=()):
     """
-    Return True if this location should be extracted based on the provided
-    kinds
+    Return True if this location should be extracted based on the provided kinds
     """
     location = os.path.abspath(os.path.expanduser(location))
     ignore_pattern = {extension : 'User ignore: Supplied by --ignore' for extension in ignore_pattern}
     should_ignore = is_ignored(location, ignore_pattern)
-    if get_extractor(location, kinds) and not should_ignore:
+    extractor = get_extractor(location, kinds=kinds)
+
+    if TRACE_DEEP:
+        logger.debug(f'  should_extract: extractor: {extractor}, should_ignore: {should_ignore}')
+
+    if extractor and not should_ignore:
         return True
 
 
@@ -113,15 +129,19 @@ def get_extractor(location, kinds=all_kinds):
     """
     assert location
     location = os.path.abspath(os.path.expanduser(location))
-    extractors = get_extractors(location, kinds)
+    extractors = get_extractors(location, kinds=kinds)
     if not extractors:
+        if TRACE_DEEP:
+            logger.debug(f'  get_extractor: not extractors: {extractors}')
         return None
 
     if len(extractors) == 2:
         extractor1, extractor2 = extractors
-        nested_extractor = functional.partial(extract_twice,
-                                             extractor1=extractor1,
-                                             extractor2=extractor2)
+        nested_extractor = functional.partial(
+            extract_twice,
+            extractor1=extractor1,
+            extractor2=extractor2,
+        )
         return nested_extractor
     elif len(extractors) == 1:
         return extractors[0]
@@ -135,23 +155,38 @@ def get_extractors(location, kinds=all_kinds):
     location or an empty list.
     """
     handler = get_best_handler(location, kinds)
+    if TRACE_DEEP:
+        logger.debug(f'  get_extractors: handler: {handler}')
+
     return handler and handler.extractors or []
 
 
 def get_best_handler(location, kinds=all_kinds):
     """
-    Return the best handler of None for the file at location.
+    Return the best handler for the file at `location` or None .
     """
     location = os.path.abspath(os.path.expanduser(location))
     if not filetype.is_file(location):
         return
+
     handlers = list(get_handlers(location))
     if TRACE_DEEP:
-        logger.debug('get_best_handler: handlers: %(handlers)r ' % locals())
+        logger.debug(f'    get_best_handler: handlers: {handlers}')
+    if not handlers:
+        return
+
+    candidates = list(score_handlers(handlers))
+    if TRACE_DEEP:
+        logger.debug(f'    get_best_handler: candidates: {candidates}')
+    if not candidates:
+        if TRACE_DEEP:
+            logger.debug(f'    get_best_handler: candidates: {candidates}')
+        return
 
-    if handlers:
-        candidates = score_handlers(handlers)
-        return candidates and pick_best_handler(candidates, kinds)
+    picked = pick_best_handler(candidates, kinds=kinds)
+    if TRACE_DEEP:
+        logger.debug(f'    get_best_handler: picked: {picked}')
+    return picked
 
 
 def get_handlers(location):
@@ -177,6 +212,8 @@ def get_handlers(location):
 
             # default to False
             type_matched = handler.filetypes and any(t in ftype for t in handler.filetypes)
+            if TRACE_DEEP:
+                logger.debug(f'    get_handlers: handler.filetypes={handler.filetypes}')
             mime_matched = handler.mimetypes and any(m in mtype for m in handler.mimetypes)
             exts = handler.extensions
             if exts:
@@ -201,10 +238,18 @@ def score_handlers(handlers):
     Score candidate handlers. Higher score is better.
     """
     for handler, type_matched, mime_matched, extension_matched in handlers:
+        if TRACE_DEEP:
+            logger.debug(
+                f'     score_handlers: handler={handler}, '
+                f'type_matched={type_matched}, '
+                f'mime_matched={mime_matched}, '
+                f'extension_matched={extension_matched}'
+            )
         score = 0
         # increment kind value: higher kinds numerical values are more
         # specific by design
         score += handler.kind
+        if TRACE_DEEP: logger.debug(f'     score_handlers: score += handler.kind {score}')
 
         # increment score based on matched criteria
         if type_matched and mime_matched and extension_matched:
@@ -255,6 +300,10 @@ def pick_best_handler(candidates, kinds):
     """
     # sort by increasing scores
     scored = sorted(candidates, reverse=True)
+
+    if TRACE_DEEP:
+        logger.debug(f'  pick_best_handler: scored: {scored}')
+
     if not scored:
         return
 
@@ -994,7 +1043,7 @@ def try_to_extract(location, target_dir, extractor):
     strict=False
 )
 
-PatchHandler = Handler(
+`PatchHandler = Handler(
     name='Patch',
     filetypes=('diff', 'patch',),
     mimetypes=('text/x-diff',),

src/extractcode/cli.py

@@ -103,14 +103,16 @@ class ExtractCommand(cliutils.BaseCommand):
 @click.option('--shallow', is_flag=True, default=False, help='Do not extract recursively nested archives (e.g. not archives in archives).')
 @click.option('--replace-originals', is_flag=True, default=False, help='Replace extracted archives by the extracted content.')
 @click.option('--ignore', default=[], multiple=True, help='Ignore files/directories following a glob-pattern.')
+@click.option('--all-formats', is_flag=True, default=False, help='Extract archives from all known formats.')
 
 @click.help_option('-h', '--help')
 @click.option('--about', is_flag=True, is_eager=True, callback=print_about, help='Show information about ExtractCode and licensing and exit.')
 @click.option('--version', is_flag=True, is_eager=True, callback=print_version, help='Show the version and exit.')
-def extractcode(ctx, input, verbose, quiet, shallow, replace_originals, ignore, *args, **kwargs):  # NOQA
+def extractcode(ctx, input, verbose, quiet, shallow, replace_originals, ignore, all_formats, *args, **kwargs):  # NOQA
     """extract archives and compressed files found in the <input> file or directory tree.
 
     Archives found inside an extracted archive are extracted recursively.
+    Use --shallow for a shallow extraction.
     Extraction for each archive is done in-place in a new directory named
     '<archive file name>-extract' created side-by-side with an archive.
     """
@@ -125,17 +127,26 @@ def extract_event(item):
             return ''
         if not item:
             return ''
+
         source = item.source
         if not isinstance(source, str):
             source = toascii(source, translit=True).decode('utf-8', 'replace')
+
         if verbose:
             if item.done:
                 return ''
-            line = source and get_relative_path(path=source, len_base_path=len_base_path, base_is_dir=base_is_dir) or ''
+            line = source and get_relative_path(
+                path=source,
+                len_base_path=len_base_path,
+                base_is_dir=base_is_dir,
+            ) or ''
+
         else:
             line = source and fileutils.file_name(source) or ''
+
         if not isinstance(line, str):
             line = toascii(line, translit=True).decode('utf-8', 'replace')
+
         return 'Extracting: %(line)s' % locals()
 
     def display_extract_summary():
@@ -149,11 +160,19 @@ def display_extract_summary():
             has_errors = has_errors or bool(xev.errors)
             has_warnings = has_warnings or bool(xev.warnings)
             source = fileutils.as_posixpath(xev.source)
+
             if not isinstance(source, str):
                 source = toascii(source, translit=True).decode('utf-8', 'replace')
-                source = get_relative_path(path=source, len_base_path=len_base_path, base_is_dir=base_is_dir)
+
+                source = get_relative_path(
+                    path=source,
+                    len_base_path=len_base_path,
+                    base_is_dir=base_is_dir,
+                )
+
             for e in xev.errors:
                 echo_stderr('ERROR extracting: %(source)s: %(e)s' % locals(), fg='red')
+
             for warn in xev.warnings:
                 echo_stderr('WARNING extracting: %(source)s: %(warn)s' % locals(), fg='yellow')
 
@@ -174,7 +193,12 @@ def display_extract_summary():
     has_extract_errors = False
 
     extractibles = extract_archives(
-        abs_location, recurse=not shallow, replace_originals=replace_originals, ignore_pattern=ignore)
+        abs_location,
+        recurse=not shallow,
+        replace_originals=replace_originals,
+        ignore_pattern=ignore,
+        all_formats=all_formats,
+    )
 
     if not quiet:
         echo_stderr('Extracting archives...', fg='green')

src/extractcode/extract.py

@@ -123,9 +123,17 @@ def extract(
     if recurse and a nested archive is found, it is extracted to full depth
     first before resuming the file system walk.
     """
+
+    extract_events = extract_files(
+        location=location,
+        kinds=kinds,
+        recurse=recurse,
+        ignore_pattern=ignore_pattern,
+    )
+
     processed_events = []
     processed_events_append = processed_events.append
-    for event in extract_files(location, kinds, recurse, ignore_pattern):
+    for event in extract_events:
         yield event
         if replace_originals:
             processed_events_append(event)
@@ -155,7 +163,9 @@ def extract_files(
     Extract only archives of a kind listed in the `kinds` kind tuple.
 
     If `recurse` is True, extract recursively archives nested inside other
-    archives. If `recurse` is false, then do not extract further an already
+    archives.
+
+    If `recurse` is false, then do not extract further an already
     extracted archive identified by the corresponding extract suffix location.
     """
     ignored = partial(ignore.is_ignored, ignores=ignore.default_ignores, unignores={})
@@ -193,7 +203,11 @@ def extract_files(
                 logger.debug('extract:target: %(target)r' % locals())
 
             # extract proper
-            for xevent in extract_file(loc, target, kinds):
+            for xevent in extract_file(
+                location=loc,
+                target=target,
+                kinds=kinds,
+            ):
                 if TRACE:
                     logger.debug('extract:walk:extraction event: %(xevent)r' % locals())
                 yield xevent
@@ -217,6 +231,7 @@ def extract_file(
     target,
     kinds=extractcode.default_kinds,
     verbose=False,
+    all_formats=False,
 ):
     """
     Extract a single archive at `location` in the `target` directory if it is