Skip to content
Snippets Groups Projects
Commit c815886b authored by Derick Hess's avatar Derick Hess
Browse files

Merge branch '3-python-3' into 'master' 

Resolve "Python 3"

Closes #3

See merge request passoft/refscrub!2
parents 2f4fc2ca a553a56a
No related branches found
No related tags found
1 merge request!2Resolve "Python 3"
Pipeline #310 passed with stage
in 1 minute and 51 seconds
Hide whitespace changes
Inline Side-by-side
HISTORY.rst
+ 6
− 0
View file @ c815886b
......@@ -16,3 +16,9 @@ History
------------------
* Minor fixes of global variables
2018.228 (2018-09-17)
------------------
* Python 2.7 & 3.6 compatiblity.
* Code cleanup to match PEP 8.
docs/conf.py 100755 → 100644
+ 0
− 0
View file @ c815886b
File mode changed from 100755 to 100644
refscrub/__init__.py
+ 1
− 1
View file @ c815886b
......@@ -4,4 +4,4 @@
__author__ = """IRIS PASSCAL"""
__email__ = 'software-support@passcal.nmt.edu'
__version__ = '2018.204'
__version__ = '2018.228'
refscrub/refscrub.py 100755 → 100644
+ 110
− 106
View file @ c815886b
#!/usr/bin/env python
'''
"""
Lloyd Carothers
February 2011
Rewrite of original refscrub
......@@ -7,53 +7,65 @@ Rewrite of original refscrub
Derick Hess
July 2018
Minor fixes for new build system
'''
from __future__ import with_statement
USAGE = 'usage: %prog [options] infile1 [ infile2 ... infileN]'
#USAGE = '''Scans (default) or extracts good reftek packets from a REFTEK file or CF card.'''
PROG_VERSION = '2018.204'
Michael Love
August 2018
Updates to work on both Python 2 & 3.
Code cleanup to match PEP 8.
Cleanup global vars.
"""
from __future__ import (print_function, with_statement)
from os.path import join, basename, getsize
import sys
import struct
USAGE = 'usage: %prog [options] infile1 [ infile2 ... infileN]'
PROG_VERSION = '2018.228'
VERBOSE = False
EXTRACT = False
SUMMARY_FILE = 'scrubsum.txt'
PREFIX = ""
FILESIZE = 0
TABS = "\t\t\t\t"
tabs = "\t\t\t\t"
from os.path import join, basename, getsize
import sys
import struct
class SN():
def __init__(self, SN):
class SN:
def __init__(self, sn):
global EXTRACT
global PREFIX
self.SN = SN
self.sn = sn
self.count = 0
self.dates = list()
if EXTRACT:
self.outfh = open( PREFIX + '.' + SN +'.scrub.ref', 'ab')
self.outfh = open(PREFIX + '.' + sn + '.scrub.ref', 'ab')
else:
self.outfh = None
def look(self, year, day, hour):
self.dates.append((year, day, hour))
self.count += 1
def close_fh(self):
if self.outfh is not None:
self.outfh.close()
def __str__(self):
self.dates = list(set(self.dates))
self.dates.sort()
start = "%0.2d:%0.3d:%0.2d" % (self.dates[0])
end = "%0.2d:%0.3d:%0.2d" % (self.dates[-1])
return "%7s: %s -- %s : %12d\n" % (self.SN , start , end, self.count )
end = "%0.2d:%0.3d:%0.2d" % (self.dates[-1])
return "%7s: %s -- %s : %12d\n" % (self.sn, start, end, self.count)
class SNseen(dict):
def look(self, sn, year, day, hour):
ret = self.setdefault(sn,SN(sn) )
ret.look( year, day, hour)
ret = self.setdefault(sn, SN(sn))
ret.look(year, day, hour)
def close_fhs(self):
for sn in self.values():
sn.close_fh()
def __str__(self):
s = " SN: YR:DAY:HR -- YR:DAY:HR : %12s\n" % 'Good Packets'
for sn in self.values():
......@@ -61,122 +73,115 @@ class SNseen(dict):
return s
##class SNseen(dict):
## def __init__(self):
## self.outfiledict = dict()
## def look(self, SN, year, day, hour):
## ret = self.setdefault(SN, list())
## #make a file for a new seen SN
## if EXTRACT and not self.outfiledict.has_key(SN) :
## self.outfiledict[SN] = open( PREFIX + '.' + SN +'.scrub.ref', 'ab')
## if (year, day, hour) not in ret:
## self[SN].append((year, day, hour))
## def close_fhs(self):
## for fh in self.outfiledict.values():
## fh.close()
## def __str__(self):
## s = " SN: YR:DAY:HR -- YR:DAY:HR\n"
## for sn in self.keys():
## dates = self[sn]
## dates.sort()
## start = "%0.2d:%0.3d:%0.2d" % (dates[0])
## end = "%0.2d:%0.3d:%0.2d" % (dates[-1])
## s += "%7s: %s -- %s\n" % (sn , start , end )
## return s
class RTPacket:
RTstruct = struct.Struct('2c1B1B2B6B2B2B')
#packetHdrFmt = '2c1B1B2B6B2B2B'
packet_types = ( 'AD', 'CD', 'DS', 'DT', 'EH', 'ET', 'OM', 'SH', 'SC')
seen = SNseen()
# Define the packet types as byte literals, since that is what the above structure will cause the type field
# do be decoded as. Without this, they will not match below.
packet_types = (b'AD', b'CD', b'DS', b'DT', b'EH', b'ET', b'OM', b'SH', b'SC')
seen = SNseen()
goodpkts = 0
IOErrorCount = 0
def __init__(self, data):
self.data = data
self.valid = None
self.timestring = None
self.ptype = None
self.expnum = None
self.year = None
self.sn = None
self.day = None
self.hour = None
self.min = None
self.sec = None
self.millisec = None
self.bytecount = None
self.sequence = None
def __str__(self):
self.settimestring()
s = "%(sequence)s %(ptype)s %(sn)s %(timestring)s" % self.__dict__
return s
def write(self):
outfh = RTPacket.seen[self.sn].outfh
#outfh = RTPacket.seen.outfiledict[self.sn]
try:
outfh.write(self.data)
except Exception, e:
print "Failed to write extracted data to %s" % outfh.filename
print e
except Exception as e:
print("Failed to write extracted data to %s" % outfh.filename)
print(e)
def settimestring(self):
self.timestring = "%(year)0.2d:%(day)0.3d:%(hour)0.2d:%(min)0.2d:%(sec)0.2d.%(millisec)0.3d" % self.__dict__
self.timestring = "%(year)0.2d:%(day)0.3d:%(hour)0.2d:%(min)0.2d:%(sec)0.2d.%(millisec)0.3d" % self.__dict__
def isvalid(self):
'''
"""
Returns True if a valid reftek packet (headers parse well and are valid)
Also populates the objects attributes SN, time, etc.
'''
"""
global VERBOSE
try:
# This tup will be a byte literal.
tup = RTPacket.RTstruct.unpack(self.data[:16])
#tup = unpack( self.packetHdrFmt, self.data[:16])
self.ptype = tup[0] + tup[1]
assert self.ptype in self.packet_types, "BAD packet type"
self.expnum = int("%0.2X" % tup[2])
self.year = int("%0.2X" %tup[3])
self.sn = "%0.2X%0.2X" % (tup[4] , tup[5] )
self.year = int("%0.2X" % tup[3])
self.sn = "%0.2X%0.2X" % (tup[4], tup[5])
assert '9001' <= self.sn, "BAD SN"
assert self.sn <= 'FFFF' , "BAD SN"
assert self.sn <= 'FFFF', "BAD SN"
time = "%0.2X%0.2X%0.2X%0.2X%0.2X%0.2X" % tup[6:12]
self.day, self.hour, self.min, self.sec, self.millisec = \
int(time[:3]),int(time[3:5]),int(time[5:7]),int(time[7:9]),int(time[9:])
int(time[:3]), int(time[3:5]), int(time[5:7]), int(time[7:9]), int(time[9:])
assert self.day <= 366, "BAD TIME"
assert self.hour <=24, "BAD TIME"
assert self.hour <= 24, "BAD TIME"
assert self.min <= 60, "BAD TIME"
assert self.sec <=80, "BAD TIME"
assert self.millisec <=1000, "BAD TIME"
assert self.sec <= 80, "BAD TIME"
assert self.millisec <= 1000, "BAD TIME"
self.bytecount = int("%0.2X%0.2X" % tup[12:14])
assert self.bytecount >= 24, "BAD byte count"
assert self.bytecount <= 1024, "BAD bytecount"
self.sequence = int("%0.2X%0.2X" % tup[14:16])
#not a valid packet
except Exception , e:
# not a valid packet
except Exception as e:
if VERBOSE:
print tabs +"Not a valid REFTEK packet"
print tabs + str(e)
print(TABS + "Not a valid REFTEK packet")
print(TABS + str(e))
self.valid = False
return False
#parsed well :: valid packet
# parsed well :: valid packet
else:
self.valid = True
RTPacket.seen.look(self.sn, self.year, self.day, self.hour)
RTPacket.goodpkts +=1
RTPacket.goodpkts += 1
return True
def readfile(infile):
ticker = 1
#speedup local vars
global VERBOSE
global EXTRACT
tell = infile.tell
read = infile.read
seek = infile.seek
while True:
ticker += 1
if VERBOSE is False and ticker == 10000:
print summary()
print(summary(infile))
ticker = 1
if VERBOSE: print "OFFSET: %12d" % tell()
if VERBOSE:
print("OFFSET: %12d" % infile.tell())
# often disk wont read
try:
buffer = read(1024)
buffer = infile.read(1024)
except IOError:
print tabs + "IOError on read"
print tabs + "Skipping 1MB.."
print(TABS + "IOError on read")
print(TABS + "Skipping 1MB..")
RTPacket.IOErrorCount += 1
seek(1024*1024, 1)
infile.seek(1024 * 1024, 1)
continue
#End of file
# End of file
if len(buffer) < 1024:
if VERBOSE: print "Partial packet"
if VERBOSE:
print("Partial packet")
break
p = RTPacket(buffer)
valid = p.isvalid()
......@@ -185,43 +190,46 @@ def readfile(infile):
if EXTRACT:
p.write()
if VERBOSE:
print p
print "<<<<<<<"
#packet is bad
print(p)
print("<<<<<<<")
# packet is bad
else:
if VERBOSE:
print tabs + "NOT VALID"
print tabs + ">>>>>>>>"
seek(-1023, 1)
print(TABS + "NOT VALID")
print(TABS + ">>>>>>>>")
infile.seek(-1023, 1)
print summary()
#commented out as when run with many files as input like a cf dir the closed fh's need to be written to and isn't smart enough to be opened
#RTPacket.seen.close_fhs()
print(summary(infile))
# commented out as when run with many files as input like a cf dir the closed fh's need to be written to and isn't
# smart enough to be opened RTPacket.seen.close_fhs()
def summary():
global infile
def summary(infile):
global FILESIZE
OFFSTRING = "%6s: %12d\n"
s = OFFSTRING % ( "OFFSET", infile.tell())
s += OFFSTRING % ( "OF", FILESIZE)
offstring = "%6s: %12d\n"
s = offstring % ("OFFSET", infile.tell())
s += offstring % ("OF", FILESIZE)
s += "Good packets: %d = %8.2fMB\n" % (RTPacket.goodpkts, RTPacket.goodpkts / 1024.0)
s+= "IOErrors: %d\n" % (RTPacket.IOErrorCount)
s += "IOErrors: %d\n" % RTPacket.IOErrorCount
s += str(RTPacket.seen) + '\n'
return s
def main():
global VERBOSE
global EXTRACT
global PREFIX
global infile
global FILESIZE
summaryfh = None
from optparse import OptionParser
parser = OptionParser(USAGE, version="%prog " + PROG_VERSION)
parser.description = "infile can be a REFTEK file or a raw disk (/dev/disk1) of a CF card."
parser.add_option('-v', '--verbose', dest="VERBOSE", action='store_true', default=False,
help="Prints info about each packet, good or bad. This will increase runtime.")
parser.add_option('-e', '--extract', dest='EXTRACT', action='store_true', default=False,
help='Writes good packets to files named infile.SNXX.scrub.ref OR PREFIX.SNXX.scrub.ref, if given, for each Serial Number found. If output file exists it will append. Be careful not to duplicate data by running more than once on the same file in the same dir.')
help='Writes good packets to files named infile.SNXX.scrub.ref OR PREFIX.SNXX.scrub.ref, '
' if given, for each Serial Number found. If output file exists it will append. Be careful '
'not to duplicate data by running more than once on the same file in the same dir.')
parser.add_option('-p', '--prefix', dest='PREFIX',
help='Prefix of output filename. Defaults to inputfilename')
parser.add_option('-s', '--savesum', dest='SUMMARY', action='store_true', default=False,
......@@ -233,24 +241,20 @@ def main():
if options.SUMMARY:
summaryfh = open(SUMMARY_FILE, 'a')
for infilename in args:
print
"Processing: %s" % infilename
print("Processing: %s" % infilename)
if not PREFIX:
PREFIX = basename(infilename)
print
"Using prefix %s" % PREFIX
print("Using prefix %s" % PREFIX)
FILESIZE = getsize(infilename)
with open(infilename) as infile:
# with open( infile.name + 'scrub.ref')) as outfile:
# import profile
# profile.run('readfile(infile)')
# Must open the file in binary mode or else we will have unicode issues when reading.
with open(infilename, "rb") as infile:
readfile(infile)
if options.SUMMARY:
summaryfh.write(infilename + ', ' + PREFIX + '\n')
summaryfh.write(summary())
summaryfh.write(summary(infile))
print("----------------------------------------")
print
"----------------------------------------"
if __name__ == '__main__':
main()
setup.cfg
+ 1
− 1
View file @ c815886b
[bumpversion]
current_version = 2018.180
current_version = 2018.228
commit = True
tag = True
......
setup.py
+ 2
− 2
View file @ c815886b
......@@ -20,7 +20,7 @@ setup(
'Intended Audience :: Developers',
'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
'Natural Language :: English',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3.6',
],
description="Remove select packets from RT130 data",
entry_points={
......@@ -51,6 +51,6 @@ setup(
packages=find_packages(include=['refscrub']),
test_suite='tests',
url='https://git.passcal.nmt.edu/passoft/refscrub',
version='2018.204',
version='2018.228',
zip_safe=False,
)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment