Merge pull request #1946 from JRavi2/add-ignore-flag

Add --ignore flag to extractcode

Author: pombredanne

src/extractcode/archive.py

@@ -34,6 +34,7 @@
 from commoncode import fileutils
 from commoncode import filetype
 from commoncode import functional
+from commoncode.ignore import is_ignored
 from commoncode.system import on_linux
 from commoncode.system import py2
 
@@ -103,13 +104,15 @@ def can_extract(location):
         return True
 
 
-def should_extract(location, kinds):
+def should_extract(location, kinds, ignore_pattern=()):
     """
     Return True if this location should be extracted based on the provided
     kinds
     """
     location = os.path.abspath(os.path.expanduser(location))
-    if get_extractor(location, kinds):
+    ignore_pattern = {extension : 'User ignore: Supplied by --ignore' for extension in ignore_pattern}
+    should_ignore = is_ignored(location, ignore_pattern)
+    if get_extractor(location, kinds) and not should_ignore:
         return True
 
 

src/extractcode/extract.py

@@ -97,7 +97,7 @@
 ExtractEvent = namedtuple('ExtractEvent', 'source target done warnings errors')
 
 
-def extract(location, kinds=extractcode.default_kinds, recurse=False, replace_originals=False):
+def extract(location, kinds=extractcode.default_kinds, recurse=False, replace_originals=False, ignore_pattern=()):
     """
     Walk and extract any archives found at `location` (either a file or
     directory). Extract only archives of a kind listed in the `kinds` kind tuple.
@@ -125,7 +125,7 @@ def extract(location, kinds=extractcode.default_kinds, recurse=False, replace_or
     """
     processed_events = []
     processed_events_append = processed_events.append
-    for event in extract_files(location, kinds, recurse):
+    for event in extract_files(location, kinds, recurse, ignore_pattern):
         yield event
         if replace_originals:
             processed_events_append(event)
@@ -143,7 +143,7 @@ def extract(location, kinds=extractcode.default_kinds, recurse=False, replace_or
                 fileutils.delete(target)
 
 
-def extract_files(location, kinds=extractcode.default_kinds, recurse=False):
+def extract_files(location, kinds=extractcode.default_kinds, recurse=False, ignore_pattern=()):
     """
     Extract the files found at `location`.
 
@@ -177,7 +177,7 @@ def extract_files(location, kinds=extractcode.default_kinds, recurse=False):
                     logger.debug('extract:walk not recurse: skipped  file: %(loc)r' % locals())
                 continue
 
-            if not archive.should_extract(loc, kinds):
+            if not archive.should_extract(loc, kinds, ignore_pattern):
                 if TRACE:
                     logger.debug('extract:walk: skipped file: not should_extract: %(loc)r' % locals())
                 continue
@@ -195,7 +195,7 @@ def extract_files(location, kinds=extractcode.default_kinds, recurse=False):
             if recurse:
                 if TRACE:
                     logger.debug('extract:walk: recursing on target: %(target)r' % locals())
-                for xevent in extract(target, kinds, recurse):
+                for xevent in extract(location=target, kinds=kinds, recurse=recurse, ignore_pattern=ignore_pattern):
                     if TRACE:
                         logger.debug('extract:walk:recurse:extraction event: %(xevent)r' % locals())
                     yield xevent

tests/extractcode/data/extract/ignore/alpha.zip

@@ -856,3 +856,59 @@ def test_extract_always_returns_a_generator_and_not_a_list(self):
         test_dir = self.get_test_loc('extract/generator', copy=True)
         result = extract.extract(test_dir)
         assert isinstance(result, GeneratorType)
+    
+    def test_extract_ignore_file(self):
+        test_dir = self.get_test_loc('extract/ignore', copy=True)
+        expected = [
+            'alpha.zip',
+            'beta.tar',
+            'beta.tar-extract/a.txt',
+            'beta.tar-extract/b.txt',
+            'beta.tar-extract/c.txt',
+            'gamma/gamma.zip',
+            'gamma/gamma.zip-extract/c.txt'
+        ]
+        from extractcode import default_kinds
+        result = list(extract.extract(test_dir, recurse=True, ignore_pattern=('alpha.zip',)))
+        check_no_error(result)
+        check_files(test_dir, expected)
+
+    def test_extract_ignore_directory(self):
+        test_dir = self.get_test_loc('extract/ignore', copy=True)
+        expected = [
+            'alpha.zip',
+            'alpha.zip-extract/a.txt',
+            'alpha.zip-extract/beta.zip',
+            'alpha.zip-extract/beta.zip-extract/b.txt',
+            'alpha.zip-extract/gamma.tar',
+            'alpha.zip-extract/gamma.tar-extract/c.txt',
+            'beta.tar',
+            'beta.tar-extract/a.txt',
+            'beta.tar-extract/b.txt',
+            'beta.tar-extract/c.txt',
+            'gamma/gamma.zip',
+        ]
+        from extractcode import default_kinds
+        result = list(extract.extract(test_dir, recurse=True, ignore_pattern=('gamma',)))
+        check_no_error(result)
+        check_files(test_dir, expected)
+
+    def test_extract_ignore_pattern(self):
+        test_dir = self.get_test_loc('extract/ignore', copy=True)
+        expected = [
+            'alpha.zip',
+            'alpha.zip-extract/a.txt',
+            'alpha.zip-extract/beta.zip',
+            'alpha.zip-extract/gamma.tar',
+            'alpha.zip-extract/gamma.tar-extract/c.txt',
+            'beta.tar',
+            'beta.tar-extract/a.txt',
+            'beta.tar-extract/b.txt',
+            'beta.tar-extract/c.txt',
+            'gamma/gamma.zip',
+            'gamma/gamma.zip-extract/c.txt'
+        ]
+        from extractcode import default_kinds
+        result = list(extract.extract(test_dir, recurse=True, ignore_pattern=('b*.zip',)))
+        check_no_error(result)
+        check_files(test_dir, expected)