From 85349673305b942d7ce3714c942981a2d07d6e27 Mon Sep 17 00:00:00 2001
From: Derick Hess <dhess@passcal.nmt.edu>
Date: Tue, 11 Dec 2018 15:07:02 -0700
Subject: [PATCH] Update bline.py
---
bline/bline.py | 227 +++++++++++++++++++++++++++++++++++--------------
1 file changed, 161 insertions(+), 66 deletions(-)
diff --git a/bline/bline.py b/bline/bline.py
index 7a75f8e..0115ed9 100755
--- a/bline/bline.py
+++ b/bline/bline.py
@@ -1,14 +1,14 @@
+#! /usr/bin/env python
# BEGIN PROGRAM: BLINE
# By: Bob Greschke
# Started: 2016.105
# Baler Offloading Program
-from __future__ import print_function
from sys import argv, exit, platform, stdout
PROGSystem = platform[:3].lower()
PROG_NAME = "BLINE"
PROG_NAMELC = "bline"
-PROG_VERSION = "2018.232"
+PROG_VERSION = "2017.020"
PROG_LONGNAME = "Command Line Baler Offload Program"
PROG_SETUPSVERS = "A"
VERS_VERSURL = "http://www.passcal.nmt.edu/~bob/passoft/"
@@ -43,7 +43,7 @@ if len(argv) == 2 and argv[1] == "-i":
if len(argv) == 2 and argv[1] == "-U":
ArgUCheck = True
elif len(argv) == 2 and argv[1] == "-UD":
- ArgUDown = True
+ ArgUDown = True
else:
if len(argv) == 1 or argv[1] == "-h":
stdout.write("\n")
@@ -182,9 +182,9 @@ offloading of everything on the baler you just enter\n\
address> is the address you assigned to the baler using BaleAddr. The\n\
offloading should start. A sub-directory like 6003.sdr will be created\n\
and the data files from the baler will be in there. The file\n\
-6003log.txt will be created in the directory where you started the\n\
+6003.msg will be created in the directory where you started the\n\
program from. It will have a copy of the messages that get written to\n\
-the treminal window as the program is working. There will be a\n\
+the terminal window as the program is working. There will be a\n\
6003files.txt. This will just be a list of all of the data files that\n\
were found on the baler.\n\
\n\
@@ -266,13 +266,13 @@ Not supplying any file list is the same as using \"*\".\n"
# You don't have to supply the file(s) list.
try:
x = argv[3]
- for I in range(3, len(argv)):
+ for I in xrange(3, len(argv)):
ArgSpecFiles.append(argv[I].upper())
except IndexError:
ArgSpecFiles.append("*")
else:
IPAddr = argv[2]
- for Index in range(3, len(argv)):
+ for Index in xrange(3, len(argv)):
Arg = argv[Index]
if Arg == "-c":
ArgCheck = True
@@ -294,12 +294,12 @@ Not supplying any file list is the same as using \"*\".\n"
continue
if Arg == "-E":
ArgSkip = True
- for I in range(Index+1, len(argv)):
+ for I in xrange(Index+1, len(argv)):
ArgSkipFiles.append(argv[I].upper())
break
if Arg == "-F":
ArgSpec = True
- for I in range(Index+1, len(argv)):
+ for I in xrange(Index+1, len(argv)):
ArgSpecFiles.append(argv[I].upper())
break
stdout.write("What??: '%s'\n\n"%Arg)
@@ -332,11 +332,17 @@ from struct import unpack
# For putting Python 3 support in.
from sys import version_info
PROG_PYVERSION = "%d.%d.%d"%(version_info[0], version_info[1], version_info[2])
+if version_info[0] != 2:
+ stdout.write("%s only runs on Python 2.\n"%PROG_NAME)
+ if version_info[1] < 4:
+# The isinstance(X, *tuple*) stuff requires this.
+ stdout.write("%s only runs on Python 2.4 and above.\n"%PROG_NAME)
+ exit()
# basestring covers ASCII and Unicode str's and showed up in v2.4.
try:
- isinstance(PROG_NAME, str)
+ isinstance(PROG_NAME, basestring)
except NameError:
- str = str
+ basestring = str
# I think float_info showed up in 2.6.
try:
from sys import float_info
@@ -354,12 +360,12 @@ from inspect import stack
if PROGSystem == "dar" or PROGSystem == "lin" or PROGSystem == "sun":
from os import getuid
from os import makedirs, listdir, sep, umask
-from os.path import abspath, basename, exists, getsize
+from os.path import abspath, basename, exists, getsize, isdir
from socket import setdefaulttimeout, gethostbyname, gethostname
#setdefaulttimeout(20) for now. Seems to work better.
-#from string import rstrip, strip
+from string import rstrip, strip
from time import gmtime, localtime, sleep, strftime, time
-from urllib.request import urlopen, urlretrieve
+from urllib import urlopen, urlretrieve
# All files will be created with u=rw g=rw o=rw.
umask(0000)
@@ -401,7 +407,7 @@ def checkForUpdates(What):
stdout.write( \
"There was an error obtaining the version information from PASSCAL.\n")
return 1
- Parts = list(map(str.strip, Line.split(";")))
+ Parts = map(strip, Line.split(";"))
Parts += (VERS_PARTS-len(Parts))*[""]
if What == "check":
if PROG_VERSION < Parts[VERS_VERS]:
@@ -426,7 +432,7 @@ def checkForUpdates(What):
DLSize = 0
Buffer = Fpr.read()
Fpw.write(Buffer)
- except Exception as e:
+ except Exception, e:
# The Fp's may not exist.
try:
Fpr.close()
@@ -500,7 +506,7 @@ def fmti(Value):
else:
Offset = 0
CountDigits = 0
- for i in range(len(Value)-1, -1+Offset, -1):
+ for i in xrange(len(Value)-1, -1+Offset, -1):
NewValue = Value[i]+NewValue
CountDigits += 1
if CountDigits == 3 and i != 0:
@@ -552,7 +558,7 @@ def getBalerHtm(Msg, TagID, IPAddr):
Fp = urlopen("http://%s/baler.htm"%IPAddr)
Lines = readFileLines(Fp, True)
Fp.close()
- except IOError as e:
+ except IOError, e:
return (2, "MW", "Error getting baler.htm:\n%s\nStopped."%e, 3)
# Set by hitting the Check button when it is already checking.
if BState == STATE_STOP:
@@ -619,7 +625,7 @@ def getFilesHtm(Msg, TagID, IPAddr):
Fp = urlopen("http://%s/files.htm"%IPAddr)
Lines = readFileLines(Fp, True)
Fp.close()
- except IOError as e:
+ except IOError, e:
return (1, "MW", "Error getting files.htm:\n%s\nStopped. (%s)"%(e, \
getGMT(3)), 3)
if BState == STATE_STOP:
@@ -634,16 +640,15 @@ def getFilesHtm(Msg, TagID, IPAddr):
# things. The files.htm I got from v2.26 that looks like
# <A http="DT000200.HHZ">DT000200.HHZ</A>
# 16777216 bytes, from 2011-10-28 11:07:58 to 2011-10-30 00:54:16
-
if Line.find("<A HREF=\"") == -1:
continue
Parts = Line.split()
- if Parts[0].find("<A") == -1:
+ if Parts[0] != "<A":
continue
# The filename is the only thing surrounded by quotes.
Filename = Parts[1][Parts[1].index("\"")+1:Parts[1].rindex("\"")]
# Eliminate any IP addresses or other stuff that may get prepended to the
-# Filenames.)
+# Filenames.
Filename = basename(Filename)
if Parts[3] == "BYTES," and Parts[7] == "TO":
Size = int(Parts[2])
@@ -818,14 +823,14 @@ def listSort(InList, Index, How, Direction = ""):
if len(InList) == 0:
return InList
if How == "n":
- if isinstance(InList[0], str):
+ if isinstance(InList[0], basestring):
# Use floatt() here since we don't know if the string will be representing an
# int or a float.
Temp = [(floatt(Item), Item) for Item in InList]
- elif isinstance(InList[0], (int, float)):
+ elif isinstance(InList[0], (int, long, float)):
Temp = [(Item, Item) for Item in InList]
elif isinstance(InList[0], (list, tuple)):
- if isinstance(InList[0][Index], str):
+ if isinstance(InList[0][Index], basestring):
Temp = [(floatt(Item[Index]), Item) for Item in InList]
else:
Temp = [(Item[Index], Item) for Item in InList]
@@ -850,7 +855,7 @@ def logIt(Msg):
# also need to work with bgoff.py). Split those up. Some messages will come
# with (date time) at the end (again, for bgoff.py). Strip those off.
Lines = Msg.split("\n")
- for Index in range(0, len(Lines)):
+ for Index in xrange(0, len(Lines)):
# If we happen to go over UT midnight this won't be caught. I'll risk it.
try:
I = Lines[Index].index(" (%s "%getGMT(2))
@@ -900,8 +905,6 @@ def readFileLines(Fp, Strip = False, N = 0):
else:
RawLines = []
Lines = []
-
- RawLines= list(map(str,RawLines))
# In case the file was empty.
try:
if RawLines[0].find("\r\n") != -1:
@@ -911,9 +914,9 @@ def readFileLines(Fp, Strip = False, N = 0):
# clean off the ends of the lines.
if Count == 1:
if Strip == False:
- Lines += list(map(str.rstrip, RawLines))
+ Lines += map(rstrip, RawLines)
else:
- Lines += list(map(str.strip, RawLines))
+ Lines += map(strip, RawLines)
# If there is more than one \r\n in the string then it could mean that all of
# the file lines are all jammed together into one long string. In that case
# split the line up first then add the parts to Lines, but leave off the
@@ -924,37 +927,37 @@ def readFileLines(Fp, Strip = False, N = 0):
if Strip == False:
Lines += Parts[:-1]
else:
- Lines += list(map(str.strip, Parts[:-1]))
+ Lines += map(strip, Parts[:-1])
# Same as \r\n.
elif RawLines[0].find("\r") != -1:
Count = RawLines[0].count("\r")
if Count == 1:
if Strip == False:
- Lines += list(map(str.rstrip, RawLines))
+ Lines += map(rstrip, RawLines)
else:
- Lines += list(map(str.strip, RawLines))
+ Lines += map(strip, RawLines)
else:
for Line in RawLines:
Parts = Line.split("\r")
if Strip == False:
Lines += Parts[:-1]
else:
- Lines += list(map(str.strip, Parts[:-1]))
+ Lines += map(strip, Parts[:-1])
# Same as \r.
elif RawLines[0].find("\n") != -1:
Count = RawLines[0].count("\n")
if Count == 1:
if Strip == False:
- Lines += list(map(str.rstrip, RawLines))
+ Lines += map(rstrip, RawLines)
else:
- Lines += list(map(str.strip, RawLines))
+ Lines += map(strip, RawLines)
else:
for Line in RawLines:
Parts = Line.split("\n")
if Strip == False:
Lines += Parts[:-1]
else:
- Lines += list(map(str.strip, Parts[:-1]))
+ Lines += map(strip, Parts[:-1])
# What else can we do?
else:
Lines += RawLines
@@ -995,7 +998,7 @@ def writeFilesTxt(Msg, Filesspec, Files):
File[FH_FROM], File[FH_TO], File[FH_SIZE]))
FilesBytes += File[FH_SIZE]
Fp.close()
- except Exception as e:
+ except Exception, e:
return (1, "MW", "Error writing file list.\n%s\nStopped. (%s)"%( e, \
getGMT(3)), 3)
if BState == STATE_STOP:
@@ -1063,29 +1066,35 @@ def q330yd2md(YYYY, DOY):
##############################
# BEGIN: checkDTFile(Filespec)
-# FUNC:checkDTFile():2017.006
+# FUNC:checkDTFile():2017.020
# Reads the mini-seed file block headers to see if they are corrupted.
+# Note Ret[0] values, so the caller can keep tabs. Also the return value
+# has the file timerange added to the end.
def checkDTFile(Filespec):
Iam = stack()[0][3]
# Keep a list of station IDs that we come across. Should be just one.
StationIDs = []
# Keep a list of the channel names we find (should be just one for balers).
Channels = []
+# The caller should be smater than this...but maybe not.
+ if isdir(Filespec):
+ return (8, "RW", "File %s is a directory."%basename(Filespec), 2, "", \
+ "")
try:
Fp = open(Filespec, "rb")
- except Exception as e:
- return (1, "MW", str(e).strip(), 3)
+ except Exception, e:
+ return (1, "MW", str(e).strip(), 3, "", "")
# Determine the record size of the seed/miniseed file. This will chew up a
# little time, but different systems (like Nanometrics) uses different record
# sizes for different files for different sample rates. The balers may always
# be the same size, but you never know.
RecordSize = 0
- for i in range(0, 17):
+ for i in xrange(0, 17):
Record = Fp.read(256)
# If the file is too small to determine it's record size then just go on.
if len(Record) < 256:
Fp.close()
- return (1, "RW", "File too small.", 2)
+ return (2, "RW", "File too small.", 2, "", "")
if i == 0:
StaID = Record[8:13]
continue
@@ -1103,7 +1112,7 @@ def checkDTFile(Filespec):
break
if RecordSize == 0:
Fp.close()
- return (1, "MW", "Could not determine record size.", 3)
+ return (3, "MW", "Could not determine record size.", 3, "", "")
Fp.seek(0)
# Read the file in 10 record chunks to make it faster.
RecordsSize = RecordSize*10
@@ -1112,13 +1121,13 @@ def checkDTFile(Filespec):
LastTime = ""
# Some may use this for messages.
NoOfRecords = int(getsize(Filespec)/RecordSize)
- while True:
+ while 1:
Records = Fp.read(RecordsSize)
# We're done with this file.
if len(Records) == 0:
break
RecordsRead += 10
- for i in range(0, 10):
+ for i in xrange(0, 10):
Ptr = RecordSize*i
Record = Records[Ptr:Ptr+RecordSize]
# Need another set of Records.
@@ -1138,9 +1147,10 @@ def checkDTFile(Filespec):
Chans = list2Str(Channels)
if len(Chans) > 12:
Chans = Chans[:12]+"..."
- return (1, "YB", \
+ return (4, "YB", \
"FILE ENDS EMPTY. Times: %s to %s\n Recs: %d Stas: %s Chans: %s"% \
- (FirstTime, LastTime, RecordsRead, Stas, Chans), 2)
+ (FirstTime, LastTime, RecordsRead, Stas, Chans), 2, \
+ FirstTime, LastTime)
if ChanID not in Channels:
Channels.append(ChanID)
StaID = Record[8:13].strip()
@@ -1156,7 +1166,7 @@ def checkDTFile(Filespec):
LastTime = DateTime
Fp.close()
if RecordsRead == 0:
- return (0, "YB", "NO DATA.", 2)
+ return (5, "YB", "NO DATA.", 2, "", "")
# Same as above.
Stas = list2Str(StationIDs)
if len(Stas) > 12:
@@ -1165,17 +1175,20 @@ def checkDTFile(Filespec):
if len(Chans) > 12:
Chans = Chans[:12]+"..."
if len(StationIDs) > 1:
- return (1, "MB", \
+ return (6, "MB", \
"MULTIPLE STATIONS. Times: %s to %s\n Recs: %d Stas: %s Chans: %s"% \
- (FirstTime, LastTime, RecordsRead, Stas, Chans), 3)
+ (FirstTime, LastTime, RecordsRead, Stas, Chans), 3, \
+ FirstTime, LastTime)
elif len(Channels) > 1:
- return (1, "MB", \
+ return (7, "MB", \
"MULTIPLE CHANNELS. Times: %s to %s\n Recs: %d Stas: %s Chans: %s"% \
- (FirstTime, LastTime, RecordsRead, Stas, Chans), 3)
+ (FirstTime, LastTime, RecordsRead, Stas, Chans), 3, \
+ FirstTime, LastTime)
else:
return (0, "GB", \
"FILE OK. Times: %s to %s\n Recs: %d Stas: %s Chans: %s"% \
- (FirstTime, LastTime, RecordsRead, Stas, Chans), 1)
+ (FirstTime, LastTime, RecordsRead, Stas, Chans), 1, \
+ FirstTime, LastTime)
# END: checkDTFile
@@ -1183,7 +1196,7 @@ def checkDTFile(Filespec):
###############
# BEGIN: main()
-# FUNC:main():2017.005
+# FUNC:main():2017.020
# All of the action takes place here in one pass using if()'s.
FH_NAME = 0
FH_SIZE = 1
@@ -1220,34 +1233,116 @@ else:
exit(0)
logIt("Working directory: %s"%AbsDirspec)
logIt("Using %s"%Msgspec)
-# This just works with offloaded files so check for it first.
+# Hijack and detour the program for this.
if ArgBadBlocks == True:
if exists(DirspecSDR) == False:
logIt("Done checking. No files have been offloaded.")
logIt("")
exit(0)
- DFiles = sorted(listdir(DirspecSDR))
+ DFiles = listdir(DirspecSDR)
+ DFiles.sort()
+ FilesChecked = 0
+ FilesOK = 0
+# These get loaded with the file names depending on the Ret[0] value coming
+# from checkDTFile() for a summary.
+ FilesOpenErrors1 = []
+ FilesTooSmall2 = []
+ FilesRecSize3 = []
+ FilesEndEmpty4 = []
+ FilesNoData5 = []
+ FilesMultStas6 = []
+ FilesMultChans7 = []
+ FilesIsDir8 = []
+# For an overall baler time range.
+ FirstFilesTime = "Z"
+ LastFilesTime = ""
# Loop through all of the file(s) the user specified, then loop through DFiles
# looking for matches.
for ArgSpecFile in ArgSpecFiles:
# Go through DFiles and turn each file into "" if it is checked, so we don't
# do it more than once if it matches more than one ArgSpecFile.
Checked = 0
- for Index in range(0, len(DFiles)):
+ for Index in xrange(0, len(DFiles)):
DFile = DFiles[Index]
if len(DFile) == 0:
continue
if fnmatch(DFile, ArgSpecFile):
DFilespec = DirspecSDR+DFile
+# checkDTFile() will also catch this. We'll still look for Rec[0]==8. Non-data
+# files will probably come back with a Rec Size error.
+ if isdir(DFilespec):
+ continue
logIt("Checking %s..."%DFilespec)
+ FilesChecked += 1
Ret = checkDTFile(DFilespec)
logIt(" "+Ret[2])
+ FirstTime = Ret[4]
+ LastTime = Ret[5]
+# If something goes wrong the returned times will be "".
+ if FirstTime != "" and FirstTime < FirstFilesTime:
+ FirstFilesTime = FirstTime
+ if LastTime != "" and LastTime > LastFilesTime:
+ LastFilesTime = LastTime
+ if Ret[0] == 0:
+ FilesOK += 1
+ elif Ret[0] == 1:
+ FilesOpenErrors1.append(DFile)
+ elif Ret[0] == 2:
+ FilesTooSmall2.append(DFile)
+ elif Ret[0] == 3:
+ FilesRecSize3.append(DFile)
+ elif Ret[0] == 4:
+ FilesEndEmpty4.append(DFile)
+ elif Ret[0] == 5:
+ FilesNoData5.append(DFile)
+ elif Ret[0] == 6:
+ FilesMultStas6.append(DFile)
+ elif Ret[0] == 7:
+ FilesMultChans7.append(DFile)
+ elif Ret[0] == 8:
+ FilesIsDir8.append(DFile)
DFiles[Index] = ""
logIt("Done checking.")
logIt("")
+ logIt("Summary:")
+ logIt(" Date range: %s to %s"%(FirstFilesTime, LastFilesTime))
+ logIt("Files checked: %d"%FilesChecked)
+ logIt(" Files OK: %d"%FilesOK)
+ if len(FilesOpenErrors1) != 0:
+ logIt("File opening errors: %d"%len(FilesOpenErrors1))
+ for File in FilesOpenErrors1:
+ logIt(" "+File)
+ if len(FilesTooSmall2) != 0:
+ logIt("Files too small: %d"%len(FilesTooSmall2))
+ for File in FilesTooSmall2:
+ logIt(" "+File)
+ if len(FilesRecSize3) != 0:
+ logIt("Unknown record size: %d"%len(FilesRecSize3))
+ for File in FilesRecSize3:
+ logIt(" "+File)
+ if len(FilesEndEmpty4) != 0:
+ logIt("Ending empty: %d"%len(FilesEndEmpty4))
+ for File in FilesEndEmpty4:
+ logIt(" "+File)
+ if len(FilesNoData5) != 0:
+ logIt("No data: %d"%len(FilesNoData5))
+ for File in FilesNoData5:
+ logIt(" "+File)
+ if len(FilesMultStas6) != 0:
+ logIt("Multiple stations: %d"%len(FilesMultStas6))
+ for File in FilesMultStas6:
+ logIt(" "+File)
+ if len(FilesMultChans7) != 0:
+ logIt("Multiple channels: %d"%len(FilesMultChans7))
+ for File in FilesMultChans7:
+ logIt(" "+File)
+ elif len(FilesIsDir8) != 0:
+ logIt("Directories: %d"%len(FilesIsDir8))
+ for File in FilesIsDir8:
+ logIt(" "+File)
+ logIt("")
exit(0)
-#%%%%%%%
-
+# All other commands.
# Get the basic info and make sure we are all on the same page. Do this each
# time we do anything.
Ret = getBalerHtm("bl", TagID, IPAddr)
@@ -1308,7 +1403,7 @@ if ArgList == True:
if ArgVerify == True:
Here = 0
NotHere = 0
- for Index in range(0, len(FHFiles)):
+ for Index in xrange(0, len(FHFiles)):
Name = FHFiles[Index][FH_NAME]
Size = FHFiles[Index][FH_SIZE]
if exists(DirspecSDR+Name) and getsize(DirspecSDR+Name) == Size:
@@ -1341,7 +1436,7 @@ logIt("Offloading from baler %s @ %s"%(TagID, IPAddr))
if ArgSpec == True:
logIt("Only offloading: %s"%list2Str(ArgSpecFiles))
for ArgSpecFile in ArgSpecFiles:
- for Index in range(0, len(FHFiles)):
+ for Index in xrange(0, len(FHFiles)):
# For entries that are already [].
try:
if fnmatch(FHFiles[Index][FH_NAME], ArgSpecFile) == False:
@@ -1352,14 +1447,14 @@ if ArgSpec == True:
elif ArgSkip == True:
logIt("Excluding: %s"%list2Str(ArgSkipFiles))
for ArgSkipFile in ArgSkipFiles:
- for Index in range(0, len(FHFiles)):
+ for Index in xrange(0, len(FHFiles)):
try:
if fnmatch(FHFiles[Index][FH_NAME], ArgSkipFile):
FHFiles[Index] = []
except IndexError:
continue
# For any of the offloads always throw out files that are already here.
-for Index in range(0, len(FHFiles)):
+for Index in xrange(0, len(FHFiles)):
try:
Filename = FHFiles[Index][FH_NAME]
Size = FHFiles[Index][FH_SIZE]
@@ -1370,7 +1465,7 @@ for Index in range(0, len(FHFiles)):
continue
if ArgLSROffload == True:
logIt("Offloading low sample rate files.")
- for Index in range(0, len(FHFiles)):
+ for Index in xrange(0, len(FHFiles)):
try:
Filename = FHFiles[Index][FH_NAME]
if Filename.find(".H") != -1 or Filename.find(".S") != -1:
@@ -1421,7 +1516,7 @@ for File in BFiles:
try:
urlretrieve("http://%s/%s"%(IPAddr, Filename), "%s%s"%(DirspecSDR, \
Filename), fileBlockRetrieved)
- except Exception as e:
+ except Exception, e:
if Offloaded10 > 0:
stdout.write("\n")
logIt("ERROR: Retrieving file %s:"%Filename)
--
GitLab