MAINT: Python 3 compatibility and travis testing

.gitignore

@@ -1,3 +1,5 @@
 build/
 dist/
 grin.egg-info/
+__pycache__/
+*.pyc

.travis.yml

@@ -0,0 +1,14 @@
+language: python
+
+python:
+    - 2.6
+    - 2.7
+    - 3.3
+    - 3.4
+    - 3.5
+
+install:
+    - pip install -e .
+
+script:
+    - nosetests .

examples/grinimports.py

@@ -2,10 +2,9 @@
 # -*- coding: UTF-8 -*-
 """ Transform Python files into normalized import statements for grepping.
 """
-
 import compiler
 from compiler.visitor import ASTVisitor, walk
-from cStringIO import StringIO
+from io import StringIO
 import os
 import shlex
 import sys
@@ -70,7 +69,7 @@ def normalize_file(filename, *args):
     """
     try:
         ast = compiler.parseFile(filename)
-    except Exception, e:
+    except Exception as e:
         return StringIO('')
     ip = ImportPuller()
     walk(ast, ip)

examples/grinpython.py

@@ -3,7 +3,7 @@
 """ Transform Python code by omitting strings, comments, and/or code.
 """
 
-from cStringIO import StringIO
+from io import BytesIO
 import os
 import shlex
 import string
@@ -55,7 +55,7 @@ def __call__(self, filename, mode='rb'):
         """ Open a file and convert it to a filelike object with transformed
         contents.
         """
-        g = StringIO()
+        g = BytesIO()
         f = open(filename, mode)
         try:
             gen = tokenize.generate_tokens(f.readline)

grin.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 """ grin searches text files.
 """
+from __future__ import print_function
 
 import bisect
 import fnmatch
@@ -11,9 +12,16 @@
 import shlex
 import stat
 import sys
+from io import UnsupportedOperation
 
 import argparse
 
+if sys.version_info[0] > 2:
+    to_str = lambda s : s.decode('latin1')
+    ints2bytes = bytes
+else:
+    to_str = str
+    ints2bytes = lambda ints : ''.join(map(chr, ints))
 
 #### Constants ####
 __version__ = '1.2.1'
@@ -24,8 +32,8 @@
 POST = 1
 
 # Use file(1)'s choices for what's text and what's not.
-TEXTCHARS = ''.join(map(chr, [7,8,9,10,12,13,27] + range(0x20, 0x100)))
-ALLBYTES = ''.join(map(chr, range(256)))
+TEXTCHARS = ints2bytes([7,8,9,10,12,13,27] + list(range(0x20, 0x100)))
+ALLBYTES = ints2bytes(range(256))
 
 COLOR_TABLE = ['black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan',
                'white', 'default']
@@ -35,12 +43,11 @@
         }
 
 # gzip magic header bytes.
-GZIP_MAGIC = '\037\213'
+GZIP_MAGIC = b'\037\213'
 
 # Target amount of data to read into memory at a time.
 READ_BLOCKSIZE = 16 * 1024 * 1024
 
-
 def is_binary_string(bytes):
     """ Determine if a string is classified as binary rather than text.
 
@@ -54,7 +61,8 @@ def is_binary_string(bytes):
     """
     nontext = bytes.translate(ALLBYTES, TEXTCHARS)
     return bool(nontext)
-
+
+
 def get_line_offsets(block):
     """ Compute the list of offsets in DataBlock 'block' which correspond to
     the beginnings of new lines.
@@ -80,7 +88,8 @@ def get_line_offsets(block):
             # Keep track of the count of lines within the "current block"
             if next_newline >= block.start and next_newline < block.end:
                 line_count += 1
-
+
+
 def colorize(s, fg=None, bg=None, bold=False, underline=False, reverse=False):
     """ Wraps a string with ANSI color escape sequences corresponding to the
     style parameters given.
@@ -207,7 +216,7 @@ def __init__(self, regex, options=None):
     def read_block_with_context(self, prev, fp, fp_size):
         """ Read a block of data from the file, along with some surrounding
         context.
-        
+
         Parameters
         ----------
         prev : DataBlock, or None
@@ -216,23 +225,23 @@ def read_block_with_context(self, prev, fp, fp_size):
 
         fp : filelike object
             The source of block data.
-        
+
         fp_size : int or None
             Size of the file in bytes, or None if the size could not be
             determined.
-        
+
         Returns
         -------
         A DataBlock representing the "current" block along with context.
         """
         if fp_size is None:
             target_io_size = READ_BLOCKSIZE
-            block_main = fp.read(target_io_size)
+            block_main = to_str(fp.read(target_io_size))
             is_last_block = len(block_main) < target_io_size
         else:
             remaining = max(fp_size - fp.tell(), 0)
             target_io_size = min(READ_BLOCKSIZE, remaining)
-            block_main = fp.read(target_io_size)
+            block_main = to_str(fp.read(target_io_size))
             is_last_block = target_io_size == remaining
 
         if prev is None:
@@ -271,12 +280,13 @@ def read_block_with_context(self, prev, fp, fp_size):
         before_lines = prev.data[before_start:prev.end]
         # Using readline() to force this block out to a newline boundary...
         curr_block = (prev.data[prev.end:] + block_main +
-            ('' if is_last_block else fp.readline()))
+            ('' if is_last_block else to_str(fp.readline())))
         # Read in some lines of 'after' context.
         if is_last_block:
             after_lines = ''
         else:
-            after_lines_list = [fp.readline() for i in range(self.options.after_context)]
+            after_lines_list = [to_str(fp.readline())
+                                for i in range(self.options.after_context)]
             after_lines = ''.join(after_lines_list)
 
         result = DataBlock(
@@ -308,13 +318,15 @@ def do_grep(self, fp):
             fp_size = None  # gzipped data is usually longer than the file
         else:
             try:
-                status = os.fstat(fp.fileno())
+                file_no = fp.fileno()
+            except (AttributeError, UnsupportedOperation):  # doesn't support fileno()
+                fp_size = None
+            else:
+                status = os.fstat(file_no)
                 if stat.S_ISREG(status.st_mode):
                     fp_size = status.st_size
                 else:
                     fp_size = None
-            except AttributeError:  # doesn't support fileno()
-                fp_size = None
 
         block = self.read_block_with_context(None, fp, fp_size)
         while block.end > block.start:
@@ -457,7 +469,7 @@ def report(self, context_lines, filename=None):
                         color_substring = colorize(old_substring, **style)
                         line = line[:start] + color_substring + line[end:]
                         total_offset += len(color_substring) - len(old_substring)
-                        
+
                 ns = dict(
                     lineno = i+1,
                     sep = {PRE: '-', POST: '+', MATCH: ':'}[kind],
@@ -495,8 +507,8 @@ def grep_a_file(self, filename, opener=open):
             f = sys.stdin
             filename = '<STDIN>'
         else:
-            # 'r' does the right thing for both open ('rt') and gzip.open ('rb')
-            f = opener(filename, 'r')
+            # Always open in binary mode
+            f = opener(filename, 'rb')
         try:
             unique_context = self.do_grep(f)
         finally:
@@ -587,7 +599,7 @@ def _is_binary_file(self, f):
         """
         try:
             bytes = f.read(self.binary_bytes)
-        except Exception, e:
+        except Exception as e:
             # When trying to read from something that looks like a gzipped file,
             # it may be corrupt. If we do get an error, assume that the file is binary.
             return True
@@ -1032,15 +1044,15 @@ def grin_main(argv=None):
             sys.stdout.write(report)
     except KeyboardInterrupt:
         raise SystemExit(0)
-    except IOError, e:
+    except IOError as e:
         if 'Broken pipe' in str(e):
             # The user is probably piping to a pager like less(1) and has exited
             # it. Just exit.
             raise SystemExit(0)
         raise
 
 def print_line(filename):
-    print filename
+    print(filename)
 
 def print_null(filename):
     # Note that the final filename will have a trailing NUL, just like 
@@ -1073,7 +1085,7 @@ def grind_main(argv=None):
                     output(filename)
     except KeyboardInterrupt:
         raise SystemExit(0)
-    except IOError, e:
+    except IOError as e:
         if 'Broken pipe' in str(e):
             # The user is probably piping to a pager like less(1) and has exited
             # it. Just exit.

tests/test_file_recognizer.py

@@ -1,5 +1,6 @@
 """ Test the file recognizer capabilities.
 """
+from __future__ import print_function
 
 import gzip
 import os
@@ -9,21 +10,21 @@
 
 import nose
 
-from grin import FileRecognizer
+from grin import FileRecognizer, ints2bytes, GZIP_MAGIC
 
 def empty_file(filename, open=open):
     f = open(filename, 'wb')
     f.close()
 
 def binary_file(filename, open=open):
     f = open(filename, 'wb')
-    f.write(''.join(map(chr, range(256))))
+    f.write(ints2bytes(range(255)))
     f.close()
 
 def text_file(filename, open=open):
-    lines = ['foo\n', 'bar\n'] * 100
-    lines.append('baz\n')
-    lines.extend(['foo\n', 'bar\n'] * 100)
+    lines = [b'foo\n', b'bar\n'] * 100
+    lines.append(b'baz\n')
+    lines.extend([b'foo\n', b'bar\n'] * 100)
     f = open(filename, 'wb')
     f.writelines(lines)
     f.close()
@@ -32,18 +33,17 @@ def fake_gzip_file(filename, open=open):
     """ Write out a binary file that has the gzip magic header bytes, but is not
     a gzip file.
     """
-    GZIP_MAGIC = '\037\213'
     f = open(filename, 'wb')
     f.write(GZIP_MAGIC)
-    f.write(''.join(map(chr, range(256))))
+    f.write(ints2bytes(range(255)))
     f.close()
 
 def binary_middle(filename, open=open):
     """ Write out a file that is text for the first 100 bytes, then 100 binary
     bytes, then 100 text bytes to test that the recognizer only reads some of
     the file.
     """
-    text = 'a'*100 + '\0'*100 + 'b'*100
+    text = b'a'*100 + b'\0'*100 + b'b'*100
     f = open(filename, 'wb')
     f.write(text)
     f.close()
@@ -56,25 +56,25 @@ def unreadable_file(filename):
     """ Write a file that does not have read permissions.
     """
     text_file(filename)
-    os.chmod(filename, 0200)
+    os.chmod(filename, 0o200)
 
 def unreadable_dir(filename):
     """ Make a directory that does not have read permissions.
     """
     os.mkdir(filename)
-    os.chmod(filename, 0300)
+    os.chmod(filename, 0o300)
 
 def unexecutable_dir(filename):
     """ Make a directory that does not have execute permissions.
     """
     os.mkdir(filename)
-    os.chmod(filename, 0600)
+    os.chmod(filename, 0o600)
 
 def totally_unusable_dir(filename):
     """ Make a directory that has neither read nor execute permissions.
     """
     os.mkdir(filename)
-    os.chmod(filename, 0100)
+    os.chmod(filename, 0o100)
 
 def setup():
     # Make files to test individual recognizers.
@@ -135,22 +135,14 @@ def setup():
     text_file('tree/.skip_hidden_file')
     os.mkdir('tree/unreadable_dir')
     text_file('tree/unreadable_dir/text')
-    os.chmod('tree/unreadable_dir', 0300)
+    os.chmod('tree/unreadable_dir', 0o300)
     os.mkdir('tree/unexecutable_dir')
     text_file('tree/unexecutable_dir/text')
-    os.chmod('tree/unexecutable_dir', 0600)
+    os.chmod('tree/unexecutable_dir', 0o600)
     os.mkdir('tree/totally_unusable_dir')
     text_file('tree/totally_unusable_dir/text')
-    os.chmod('tree/totally_unusable_dir', 0100)
+    os.chmod('tree/totally_unusable_dir', 0o100)
 
-def ensure_deletability(arg, dirname, fnames):
-    """ os.path.walk() callback function which will make sure every directory is
-    readable and executable so that it may be easily deleted.
-    """
-    for fn in fnames:
-        fn = os.path.join(dirname, fn)
-        if os.path.isdir(fn):
-            os.chmod(fn, 0700)
 
 def teardown():
     files_to_delete = ['empty', 'binary', 'binary_middle', 'text', 'text~',
@@ -168,10 +160,13 @@ def teardown():
                 os.unlink(filename)
             else:
                 os.rmdir(filename)
-        except Exception, e:
-            print >>sys.stderr, 'Could not delete %s: %s' % (filename, e)
+        except Exception as e:
+            print('Could not delete %s: %s' % (filename, e), file=sys.stderr)
     os.unlink('socket_test')
-    os.path.walk('tree', ensure_deletability, None)
+    for dirpath, dirnames, filenames in os.walk('tree'):
+        # Make sure every directory can be deleted
+        for dirname in dirnames:
+            os.chmod(os.path.join(dirpath, dirname), 0o700)
     shutil.rmtree('tree')