Update code to run under Python3 + Conda packaging

• Updated to work with Python 3
• Added unit tests to ensure basic functionalities of refscrub
• Updated list of platform specific dependencies to be installed when installing refscrub in dev mode (see setup.py)
• Installed and tested refscrub against Python 3.[6,7,8] using tox
• Formatted Python code to conform to the PEP8 style guide
• Created conda package for refscrub that can run on Python 3.[6,7,8]
• Updated .gitlab-ci.yml to run a linter and unit tests for Python 3.[6,7,8] in GitLab CI pipeline

refscrub/refscrub.py

@@ -13,14 +13,22 @@ August 2018
 Updates to work on both Python 2 & 3.
 Code cleanup to match PEP 8.
 Cleanup global vars.
+
+Maeva Pourpoint
+August 2020
+Updates to work under Python 3.
+Unit tests to ensure basic functionality.
+Code cleanup to conform to the PEP8 style guide.
+Directory cleanup (remove unused files introduced by Cookiecutter).
+Packaged with conda.
 """
-from __future__ import (print_function, with_statement)
-from os.path import join, basename, getsize
-import sys
+
+import argparse
 import struct
+import sys
+from os.path import basename, getsize, isfile
 
-USAGE = 'usage: %prog [options] infile1 [ infile2 ... infileN]'
-PROG_VERSION = '2018.228'
+PROG_VERSION = '2020.216'
 VERBOSE = False
 EXTRACT = False
 SUMMARY_FILE = 'scrubsum.txt'
@@ -63,21 +71,23 @@ class SNseen(dict):
         ret.look(year, day, hour)
 
     def close_fhs(self):
-        for sn in self.values():
+        for sn in list(self.values()):
             sn.close_fh()
 
     def __str__(self):
         s = "     SN: YR:DAY:HR -- YR:DAY:HR : %12s\n" % 'Good Packets'
-        for sn in self.values():
+        for sn in list(self.values()):
             s += str(sn)
         return s
 
 
 class RTPacket:
     RTstruct = struct.Struct('2c1B1B2B6B2B2B')
-    # Define the packet types as byte literals, since that is what the above structure will cause the type field
-    # do be decoded as. Without this, they will not match below.
-    packet_types = (b'AD', b'CD', b'DS', b'DT', b'EH', b'ET', b'OM', b'SH', b'SC')
+    # Define the packet types as byte literals, since that is what the above
+    # structure will cause the type field do be decoded as. Without this, they
+    # will not match below.
+    packet_types = (b'AD', b'CD', b'DS', b'DT',
+                    b'EH', b'ET', b'OM', b'SH', b'SC')
     seen = SNseen()
     goodpkts = 0
     IOErrorCount = 0
@@ -112,11 +122,13 @@ class RTPacket:
             print(e)
 
     def settimestring(self):
-        self.timestring = "%(year)0.2d:%(day)0.3d:%(hour)0.2d:%(min)0.2d:%(sec)0.2d.%(millisec)0.3d" % self.__dict__
+        self.timestring = ("%(year)0.2d:%(day)0.3d:%(hour)0.2d:%(min)0.2d:"
+                           "%(sec)0.2d.%(millisec)0.3d" % self.__dict__)
 
     def isvalid(self):
         """
-        Returns True if a valid reftek packet (headers parse well and are valid)
+        Returns True if a valid reftek packet (headers parse well and are
+        valid)
         Also populates the objects attributes SN, time, etc.
         """
 
@@ -129,11 +141,12 @@ class RTPacket:
             self.expnum = int("%0.2X" % tup[2])
             self.year = int("%0.2X" % tup[3])
             self.sn = "%0.2X%0.2X" % (tup[4], tup[5])
-            assert '9001' <= self.sn, "BAD SN"
+            assert self.sn >= '9001', "BAD SN"
             assert self.sn <= 'FFFF', "BAD SN"
             time = "%0.2X%0.2X%0.2X%0.2X%0.2X%0.2X" % tup[6:12]
             self.day, self.hour, self.min, self.sec, self.millisec = \
-                int(time[:3]), int(time[3:5]), int(time[5:7]), int(time[7:9]), int(time[9:])
+                int(time[:3]), int(time[3:5]), int(
+                    time[5:7]), int(time[7:9]), int(time[9:])
             assert self.day <= 366, "BAD TIME"
             assert self.hour <= 24, "BAD TIME"
             assert self.min <= 60, "BAD TIME"
@@ -200,8 +213,9 @@ def readfile(infile):
             infile.seek(-1023, 1)
 
     print(summary(infile))
-    # commented out as when run with many files as input like a cf dir the closed fh's need to be written to and isn't
-    # smart enough to be opened RTPacket.seen.close_fhs()
+    # commented out as when run with many files as input like a cf dir the
+    # closed fh's need to be written to and isn't smart enough to be opened
+    # RTPacket.seen.close_fhs()
 
 
 def summary(infile):
@@ -209,7 +223,8 @@ def summary(infile):
     offstring = "%6s: %12d\n"
     s = offstring % ("OFFSET", infile.tell())
     s += offstring % ("OF", FILESIZE)
-    s += "Good packets: %d = %8.2fMB\n" % (RTPacket.goodpkts, RTPacket.goodpkts / 1024.0)
+    s += "Good packets: %d = %8.2fMB\n" % (
+        RTPacket.goodpkts, RTPacket.goodpkts / 1024.0)
     s += "IOErrors: %d\n" % RTPacket.IOErrorCount
     s += str(RTPacket.seen) + '\n'
     return s
@@ -221,39 +236,55 @@ def main():
     global PREFIX
     global FILESIZE
     summaryfh = None
-    from optparse import OptionParser
-    parser = OptionParser(USAGE, version="%prog " + PROG_VERSION)
-    parser.description = "infile can be a REFTEK file or a raw disk (/dev/disk1) of a CF card."
-    parser.add_option('-v', '--verbose', dest="VERBOSE", action='store_true', default=False,
-                      help="Prints info about each packet, good or bad. This will increase runtime.")
-    parser.add_option('-e', '--extract', dest='EXTRACT', action='store_true', default=False,
-                      help='Writes good packets to files named infile.SNXX.scrub.ref OR PREFIX.SNXX.scrub.ref, '
-                           ' if given, for each Serial Number found. If output file exists it will append. Be careful '
-                           'not to duplicate data by running more than once on the same file in the same dir.')
-    parser.add_option('-p', '--prefix', dest='PREFIX',
-                      help='Prefix of output filename. Defaults to inputfilename')
-    parser.add_option('-s', '--savesum', dest='SUMMARY', action='store_true', default=False,
-                      help='Appends summary to %s' % SUMMARY_FILE)
-    options, args = parser.parse_args()
-    VERBOSE = options.VERBOSE
-    EXTRACT = options.EXTRACT
-    PREFIX = options.PREFIX
-    if options.SUMMARY:
+    parser = argparse.ArgumentParser(prog="refscrub",
+                                     usage="%(prog)s [options] infile1 "
+                                     "[ infile2 ... infileN]")
+    parser.add_argument('infile', nargs='*', metavar='infile',
+                        help="infile can be a REFTEK file or a raw disk "
+                        "(/dev/disk1) of a CF card.")
+    parser.add_argument('--version', action='version',
+                        version="%(prog)s " + PROG_VERSION)
+    parser.add_argument('-v', '--verbose', dest="VERBOSE", action='store_true',
+                        default=False, help="Prints info about each packet, "
+                        "good or bad. This will increase runtime.")
+    parser.add_argument('-e', '--extract', dest='EXTRACT', action='store_true',
+                        default=False, help="Writes good packets to files "
+                        "named infile.SNXX.scrub.ref OR "
+                        "PREFIX.SNXX.scrub.ref, if given, for each Serial "
+                        "Number found. If output file exists it will append. "
+                        "Be careful not to duplicate data by running more "
+                        "than once on the same file in the same dir.")
+    parser.add_argument('-p', '--prefix', dest='PREFIX',
+                        help="Prefix of output filename. Defaults to input"
+                        "filename")
+    parser.add_argument('-s', '--savesum', dest='SUMMARY', action='store_true',
+                        default=False, help='Appends summary to %s'
+                        % SUMMARY_FILE)
+    args = parser.parse_args()
+    if not args.infile:
+        parser.print_help()
+        sys.exit(1)
+    VERBOSE = args.VERBOSE
+    EXTRACT = args.EXTRACT
+    PREFIX = args.PREFIX
+    if args.SUMMARY:
         summaryfh = open(SUMMARY_FILE, 'a')
-    for infilename in args:
+    for infilename in args.infile:
         print("Processing: %s" % infilename)
         if not PREFIX:
             PREFIX = basename(infilename)
         print("Using prefix %s" % PREFIX)
         FILESIZE = getsize(infilename)
-        # Must open the file in binary mode or else we will have unicode issues when reading.
+        # Must open the file in binary mode or else we will have unicode issues
+        # when reading.
         with open(infilename, "rb") as infile:
             readfile(infile)
-            if options.SUMMARY:
+            if args.SUMMARY:
                 summaryfh.write(infilename + ', ' + PREFIX + '\n')
                 summaryfh.write(summary(infile))
-
         print("----------------------------------------")
+    if isfile(SUMMARY_FILE):
+        summaryfh.close()
 
 
 if __name__ == '__main__':