From 40c461f565e8c4928d388c47f77d11defe4a619d Mon Sep 17 00:00:00 2001
From: Michael Love <mlove@passcal.nmt.edu>
Date: Thu, 16 Aug 2018 14:57:26 -0600
Subject: [PATCH] Compatibility & code cleanup.
Updated for Python 2.7 & 3.6 compatibility.
Code cleanup for PEP 8 standards.
---
HISTORY.rst | 6 ++
docs/conf.py | 0
refscrub/__init__.py | 2 +-
refscrub/refscrub.py | 239 +++++++++++++++++++++++++------------------
setup.cfg | 2 +-
setup.py | 4 +-
6 files changed, 147 insertions(+), 106 deletions(-)
mode change 100755 => 100644 docs/conf.py
mode change 100755 => 100644 refscrub/refscrub.py
diff --git a/HISTORY.rst b/HISTORY.rst
index 158c12c..ffd258c 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -16,3 +16,9 @@ History
------------------
* Minor fixes of global variables
+
+2018.227 (2018-09-16)
+------------------
+
+* Python 2.7 & 3.6 compatiblity.
+* Code cleanup to match PEP 8.
diff --git a/docs/conf.py b/docs/conf.py
old mode 100755
new mode 100644
diff --git a/refscrub/__init__.py b/refscrub/__init__.py
index 384ebd4..efb2f2d 100644
--- a/refscrub/__init__.py
+++ b/refscrub/__init__.py
@@ -4,4 +4,4 @@
__author__ = """IRIS PASSCAL"""
__email__ = 'software-support@passcal.nmt.edu'
-__version__ = '2018.204'
+__version__ = '2018.227'
diff --git a/refscrub/refscrub.py b/refscrub/refscrub.py
old mode 100755
new mode 100644
index ba46a85..3d0d7b1
--- a/refscrub/refscrub.py
+++ b/refscrub/refscrub.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python
-'''
+"""
Lloyd Carothers
February 2011
Rewrite of original refscrub
@@ -7,53 +7,65 @@ Rewrite of original refscrub
Derick Hess
July 2018
Minor fixes for new build system
-'''
-from __future__ import with_statement
-USAGE = 'usage: %prog [options] infile1 [ infile2 ... infileN]'
-#USAGE = '''Scans (default) or extracts good reftek packets from a REFTEK file or CF card.'''
-PROG_VERSION = '2018.204'
+Michael Love
+August 2018
+Updates to work on both Python 2 & 3.
+Code cleanup to match PEP 8.
+"""
+from __future__ import (print_function, with_statement)
+from os.path import join, basename, getsize
+import sys
+import struct
+
+USAGE = 'usage: %prog [options] infile1 [ infile2 ... infileN]'
+PROG_VERSION = '2018.227'
VERBOSE = False
EXTRACT = False
SUMMARY_FILE = 'scrubsum.txt'
+PREFIX = ""
+INFILE = ""
+FILESIZE = 0
+TABS = "\t\t\t\t"
-tabs = "\t\t\t\t"
-from os.path import join, basename, getsize
-import sys
-import struct
-
-class SN():
- def __init__(self, SN):
+class SN:
+ def __init__(self, sn):
global EXTRACT
global PREFIX
- self.SN = SN
+ self.sn = sn
self.count = 0
self.dates = list()
if EXTRACT:
- self.outfh = open( PREFIX + '.' + SN +'.scrub.ref', 'ab')
+ self.outfh = open(PREFIX + '.' + sn + '.scrub.ref', 'ab')
else:
self.outfh = None
+
def look(self, year, day, hour):
self.dates.append((year, day, hour))
self.count += 1
+
def close_fh(self):
if self.outfh is not None:
self.outfh.close()
+
def __str__(self):
self.dates = list(set(self.dates))
self.dates.sort()
start = "%0.2d:%0.3d:%0.2d" % (self.dates[0])
- end = "%0.2d:%0.3d:%0.2d" % (self.dates[-1])
- return "%7s: %s -- %s : %12d\n" % (self.SN , start , end, self.count )
+ end = "%0.2d:%0.3d:%0.2d" % (self.dates[-1])
+ return "%7s: %s -- %s : %12d\n" % (self.sn, start, end, self.count)
+
class SNseen(dict):
def look(self, sn, year, day, hour):
- ret = self.setdefault(sn,SN(sn) )
- ret.look( year, day, hour)
+ ret = self.setdefault(sn, SN(sn))
+ ret.look(year, day, hour)
+
def close_fhs(self):
for sn in self.values():
sn.close_fh()
+
def __str__(self):
s = " SN: YR:DAY:HR -- YR:DAY:HR : %12s\n" % 'Good Packets'
for sn in self.values():
@@ -61,122 +73,142 @@ class SNseen(dict):
return s
-##class SNseen(dict):
-## def __init__(self):
-## self.outfiledict = dict()
-## def look(self, SN, year, day, hour):
-## ret = self.setdefault(SN, list())
-## #make a file for a new seen SN
-## if EXTRACT and not self.outfiledict.has_key(SN) :
-## self.outfiledict[SN] = open( PREFIX + '.' + SN +'.scrub.ref', 'ab')
-## if (year, day, hour) not in ret:
-## self[SN].append((year, day, hour))
-## def close_fhs(self):
-## for fh in self.outfiledict.values():
-## fh.close()
-## def __str__(self):
-## s = " SN: YR:DAY:HR -- YR:DAY:HR\n"
-## for sn in self.keys():
-## dates = self[sn]
-## dates.sort()
-## start = "%0.2d:%0.3d:%0.2d" % (dates[0])
-## end = "%0.2d:%0.3d:%0.2d" % (dates[-1])
-## s += "%7s: %s -- %s\n" % (sn , start , end )
-## return s
+# class SNseen(dict):
+# def __init__(self):
+# self.outfiledict = dict()
+# def look(self, SN, year, day, hour):
+# ret = self.setdefault(SN, list())
+# #make a file for a new seen SN
+# if EXTRACT and not self.outfiledict.has_key(SN) :
+# self.outfiledict[SN] = open( PREFIX + '.' + SN +'.scrub.ref', 'ab')
+# if (year, day, hour) not in ret:
+# self[SN].append((year, day, hour))
+# def close_fhs(self):
+# for fh in self.outfiledict.values():
+# fh.close()
+# def __str__(self):
+# s = " SN: YR:DAY:HR -- YR:DAY:HR\n"
+# for sn in self.keys():
+# dates = self[sn]
+# dates.sort()
+# start = "%0.2d:%0.3d:%0.2d" % (dates[0])
+# end = "%0.2d:%0.3d:%0.2d" % (dates[-1])
+# s += "%7s: %s -- %s\n" % (sn , start , end )
+# return s
class RTPacket:
RTstruct = struct.Struct('2c1B1B2B6B2B2B')
- #packetHdrFmt = '2c1B1B2B6B2B2B'
- packet_types = ( 'AD', 'CD', 'DS', 'DT', 'EH', 'ET', 'OM', 'SH', 'SC')
- seen = SNseen()
+ # packetHdrFmt = '2c1B1B2B6B2B2B'
+ packet_types = ('AD', 'CD', 'DS', 'DT', 'EH', 'ET', 'OM', 'SH', 'SC')
+ seen = SNseen()
goodpkts = 0
IOErrorCount = 0
+
def __init__(self, data):
self.data = data
self.valid = None
+ self.timestring = None
+ self.ptype = None
+ self.expnum = None
+ self.year = None
+ self.sn = None
+ self.day = None
+ self.hour = None
+ self.min = None
+ self.sec = None
+ self.millisec = None
+ self.bytecount = None
+ self.sequence = None
+
def __str__(self):
self.settimestring()
s = "%(sequence)s %(ptype)s %(sn)s %(timestring)s" % self.__dict__
return s
+
def write(self):
outfh = RTPacket.seen[self.sn].outfh
- #outfh = RTPacket.seen.outfiledict[self.sn]
+ # outfh = RTPacket.seen.outfiledict[self.sn]
try:
outfh.write(self.data)
- except Exception, e:
- print "Failed to write extracted data to %s" % outfh.filename
- print e
+ except Exception as e:
+ print("Failed to write extracted data to %s" % outfh.filename)
+ print(e)
+
def settimestring(self):
- self.timestring = "%(year)0.2d:%(day)0.3d:%(hour)0.2d:%(min)0.2d:%(sec)0.2d.%(millisec)0.3d" % self.__dict__
+ self.timestring = "%(year)0.2d:%(day)0.3d:%(hour)0.2d:%(min)0.2d:%(sec)0.2d.%(millisec)0.3d" % self.__dict__
+
def isvalid(self):
- '''
+ """
Returns True if a valid reftek packet (headers parse well and are valid)
Also populates the objects attributes SN, time, etc.
- '''
+ """
global VERBOSE
try:
tup = RTPacket.RTstruct.unpack(self.data[:16])
- #tup = unpack( self.packetHdrFmt, self.data[:16])
+ # tup = unpack( self.packetHdrFmt, self.data[:16])
self.ptype = tup[0] + tup[1]
assert self.ptype in self.packet_types, "BAD packet type"
self.expnum = int("%0.2X" % tup[2])
- self.year = int("%0.2X" %tup[3])
- self.sn = "%0.2X%0.2X" % (tup[4] , tup[5] )
+ self.year = int("%0.2X" % tup[3])
+ self.sn = "%0.2X%0.2X" % (tup[4], tup[5])
assert '9001' <= self.sn, "BAD SN"
- assert self.sn <= 'FFFF' , "BAD SN"
+ assert self.sn <= 'FFFF', "BAD SN"
time = "%0.2X%0.2X%0.2X%0.2X%0.2X%0.2X" % tup[6:12]
self.day, self.hour, self.min, self.sec, self.millisec = \
- int(time[:3]),int(time[3:5]),int(time[5:7]),int(time[7:9]),int(time[9:])
+ int(time[:3]), int(time[3:5]), int(time[5:7]), int(time[7:9]), int(time[9:])
assert self.day <= 366, "BAD TIME"
- assert self.hour <=24, "BAD TIME"
+ assert self.hour <= 24, "BAD TIME"
assert self.min <= 60, "BAD TIME"
- assert self.sec <=80, "BAD TIME"
- assert self.millisec <=1000, "BAD TIME"
+ assert self.sec <= 80, "BAD TIME"
+ assert self.millisec <= 1000, "BAD TIME"
self.bytecount = int("%0.2X%0.2X" % tup[12:14])
assert self.bytecount >= 24, "BAD byte count"
assert self.bytecount <= 1024, "BAD bytecount"
self.sequence = int("%0.2X%0.2X" % tup[14:16])
- #not a valid packet
- except Exception , e:
+ # not a valid packet
+ except Exception as e:
if VERBOSE:
- print tabs +"Not a valid REFTEK packet"
- print tabs + str(e)
+ print(TABS + "Not a valid REFTEK packet")
+ print(TABS + str(e))
self.valid = False
return False
- #parsed well :: valid packet
+ # parsed well :: valid packet
else:
self.valid = True
RTPacket.seen.look(self.sn, self.year, self.day, self.hour)
- RTPacket.goodpkts +=1
+ RTPacket.goodpkts += 1
return True
-def readfile(infile):
+
+def readfile(filename):
ticker = 1
- #speedup local vars
+ # speedup local vars
global VERBOSE
global EXTRACT
- tell = infile.tell
- read = infile.read
- seek = infile.seek
+ tell = filename.tell
+ read = filename.read
+ seek = filename.seek
while True:
ticker += 1
if VERBOSE is False and ticker == 10000:
- print summary()
+ print(summary())
ticker = 1
- if VERBOSE: print "OFFSET: %12d" % tell()
+ if VERBOSE:
+ print("OFFSET: %12d" % tell())
# often disk wont read
try:
buffer = read(1024)
except IOError:
- print tabs + "IOError on read"
- print tabs + "Skipping 1MB.."
+ print(TABS + "IOError on read")
+ print(TABS + "Skipping 1MB..")
RTPacket.IOErrorCount += 1
- seek(1024*1024, 1)
+ seek(1024 * 1024, 1)
continue
- #End of file
+ # End of file
if len(buffer) < 1024:
- if VERBOSE: print "Partial packet"
+ if VERBOSE:
+ print("Partial packet")
break
p = RTPacket(buffer)
valid = p.isvalid()
@@ -185,43 +217,48 @@ def readfile(infile):
if EXTRACT:
p.write()
if VERBOSE:
- print p
- print "<<<<<<<"
- #packet is bad
+ print(p)
+ print("<<<<<<<")
+ # packet is bad
else:
if VERBOSE:
- print tabs + "NOT VALID"
- print tabs + ">>>>>>>>"
+ print(TABS + "NOT VALID")
+ print(TABS + ">>>>>>>>")
seek(-1023, 1)
- print summary()
- #commented out as when run with many files as input like a cf dir the closed fh's need to be written to and isn't smart enough to be opened
- #RTPacket.seen.close_fhs()
+ print(summary())
+ # commented out as when run with many files as input like a cf dir the closed fh's need to be written to and isn't
+ # smart enough to be opened RTPacket.seen.close_fhs()
+
def summary():
- global infile
+ global INFILE
global FILESIZE
- OFFSTRING = "%6s: %12d\n"
- s = OFFSTRING % ( "OFFSET", infile.tell())
- s += OFFSTRING % ( "OF", FILESIZE)
+ offstring = "%6s: %12d\n"
+ s = offstring % ("OFFSET", INFILE.tell())
+ s += offstring % ("OF", FILESIZE)
s += "Good packets: %d = %8.2fMB\n" % (RTPacket.goodpkts, RTPacket.goodpkts / 1024.0)
- s+= "IOErrors: %d\n" % (RTPacket.IOErrorCount)
+ s += "IOErrors: %d\n" % RTPacket.IOErrorCount
s += str(RTPacket.seen) + '\n'
return s
+
def main():
global VERBOSE
global EXTRACT
global PREFIX
- global infile
+ global INFILE
global FILESIZE
+ summaryfh = None
from optparse import OptionParser
parser = OptionParser(USAGE, version="%prog " + PROG_VERSION)
parser.description = "infile can be a REFTEK file or a raw disk (/dev/disk1) of a CF card."
parser.add_option('-v', '--verbose', dest="VERBOSE", action='store_true', default=False,
help="Prints info about each packet, good or bad. This will increase runtime.")
parser.add_option('-e', '--extract', dest='EXTRACT', action='store_true', default=False,
- help='Writes good packets to files named infile.SNXX.scrub.ref OR PREFIX.SNXX.scrub.ref, if given, for each Serial Number found. If output file exists it will append. Be careful not to duplicate data by running more than once on the same file in the same dir.')
+ help='Writes good packets to files named infile.SNXX.scrub.ref OR PREFIX.SNXX.scrub.ref, '
+ ' if given, for each Serial Number found. If output file exists it will append. Be careful '
+ 'not to duplicate data by running more than once on the same file in the same dir.')
parser.add_option('-p', '--prefix', dest='PREFIX',
help='Prefix of output filename. Defaults to inputfilename')
parser.add_option('-s', '--savesum', dest='SUMMARY', action='store_true', default=False,
@@ -233,24 +270,22 @@ def main():
if options.SUMMARY:
summaryfh = open(SUMMARY_FILE, 'a')
for infilename in args:
- print
- "Processing: %s" % infilename
+ print("Processing: %s" % infilename)
if not PREFIX:
PREFIX = basename(infilename)
- print
- "Using prefix %s" % PREFIX
+ print("Using prefix %s" % PREFIX)
FILESIZE = getsize(infilename)
- with open(infilename) as infile:
- # with open( infile.name + 'scrub.ref')) as outfile:
+ with open(infilename) as filename:
+ # with open( filename.name + 'scrub.ref')) as outfile:
# import profile
- # profile.run('readfile(infile)')
- readfile(infile)
+ # profile.run('readfile(filename)')
+ readfile(filename)
if options.SUMMARY:
summaryfh.write(infilename + ', ' + PREFIX + '\n')
summaryfh.write(summary())
- print
- "----------------------------------------"
+ print("----------------------------------------")
+
if __name__ == '__main__':
main()
diff --git a/setup.cfg b/setup.cfg
index 01be888..5e7c599 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 2018.180
+current_version = 2018.227
commit = True
tag = True
diff --git a/setup.py b/setup.py
index 194d54d..29be5b2 100644
--- a/setup.py
+++ b/setup.py
@@ -20,7 +20,7 @@ setup(
'Intended Audience :: Developers',
'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
'Natural Language :: English',
- 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3.6',
],
description="Remove select packets from RT130 data",
entry_points={
@@ -51,6 +51,6 @@ setup(
packages=find_packages(include=['refscrub']),
test_suite='tests',
url='https://git.passcal.nmt.edu/passoft/refscrub',
- version='2018.204',
+ version='2018.227',
zip_safe=False,
)
--
GitLab