bline.py

    except Exception, e:
        return (1, "MW", "Error writing file list.\n%s\nStopped. (%s)"%( e, \
                getGMT(3)), 3)
    if BState == STATE_STOP:
        return (1, "YB", "Stopped by user. (%s)"%getGMT(3), 2)
    if Msg == "bg":
        msgLn(1, "W", "Wrote file list to %s"%Filesspec)
        msgLn(0, "", "%s %s, %s %s on baler."%(fmti(FilesCount), \
                sP(FilesCount, ("file", "files")), fmti(FilesBytes), \
                sP(FilesBytes, ("byte", "bytes"))))
    elif Msg == "bl":
        logIt("Wrote file list to %s"%Filesspec)
        logIt("%s %s, %s %s on baler."%(fmti(FilesCount), sP(FilesCount, \
                ("file", "files")), fmti(FilesBytes), sP(FilesBytes, \
                ("byte", "bytes"))))
    return (0, FilesCount, FilesBytes)
# END: writeFilesTxt




# First day of the month for each non-leap year month MINUS 1. This will get
# subtracted from the DOY, so a DOY of 91, minus the first day of April 90
# (91-90) will leave the 1st of April. The 365 is the 1st of Jan of the next
# year.
PROG_FDOM = (0, 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365)

#############################
# BEGIN: q330yd2md(YYYY, DOY)
# LIB:q330yd2md():2013.023
#   Converts YYYY,DOY to Month, Day. Faster than using using ydhmst2Time().
#   Expects a 4-digit YYYY.
def q330yd2md(YYYY, DOY):
    if DOY < 1 or DOY > 366:
        return 0, 0
    if DOY < 32:
        return 1, DOY
    elif DOY < 60:
        return 2, DOY-31
    if YYYY%4 != 0:
        Leap = 0
    elif YYYY%100 != 0 or YYYY%400 == 0:
        Leap = 1
    else:
        Leap = 0
# Check for this special day.
    if Leap == 1 and DOY == 60:
        return 2, 29
# The PROG_FDOM values for Mar-Dec are set up for non-leap years. If it is a
# leap year and the date is going to be Mar-Dec (it is if we have made it this
# far), subtract Leap from the day.
    DOY -= Leap
# We start through PROG_FDOM looking for dates in March.
    Month = 3
    for FDOM in PROG_FDOM[4:]:
# See if the DOY is less than the first day of next month.
        if DOY <= FDOM:
# Subtract the DOY for the month that we are in.
            return Month, DOY-PROG_FDOM[Month]
        Month += 1
    return 0, 0
# END: q330yd2md




##############################
# BEGIN: checkDTFile(Filespec)
# FUNC:checkDTFile():2017.020
#   Reads the mini-seed file block headers to see if they are corrupted.
#   Note Ret[0] values, so the caller can keep tabs. Also the return value
#   has the file timerange added to the end.
def checkDTFile(Filespec):
    Iam = stack()[0][3]
# Keep a list of station IDs that we come across. Should be just one.
    StationIDs = []
# Keep a list of the channel names we find (should be just one for balers).
    Channels = []
# The caller should be smater than this...but maybe not.
    if isdir(Filespec):
        return (8, "RW", "File %s is a directory."%basename(Filespec), 2, "", \
                "")
    try:
        Fp = open(Filespec, "rb")
    except Exception, e:
        return (1, "MW", str(e).strip(), 3, "", "")
# Determine the record size of the seed/miniseed file. This will chew up a
# little time, but different systems (like Nanometrics) uses different record
# sizes for different files for different sample rates. The balers may always
# be the same size, but you never know.
    RecordSize = 0
    for i in xrange(0, 17):
        Record = Fp.read(256)
# If the file is too small to determine it's record size then just go on.
        if len(Record) < 256:
            Fp.close()
            return (2, "RW", "File too small.", 2, "", "")
        if i == 0:
            StaID = Record[8:13]
            continue
        if StaID == Record[8:13]:
            if i == 1:
                RecordSize = 256
            elif i == 2:
                RecordSize = 512
            elif i == 4:
                RecordSize = 1024
            elif i == 8:
                RecordSize = 2048
            elif i == 16:
                RecordSize = 4096
            break
    if RecordSize == 0:
        Fp.close()
        return (3, "MW", "Could not determine record size.", 3, "", "")
    Fp.seek(0)
# Read the file in 10 record chunks to make it faster.
    RecordsSize = RecordSize*10
    RecordsRead = 0
    FirstTime = "Z"
    LastTime = ""
# Some may use this for messages.
    NoOfRecords = int(getsize(Filespec)/RecordSize)
    while 1:
        Records = Fp.read(RecordsSize)
# We're done with this file.
        if len(Records) == 0:
            break
        RecordsRead += 10
        for i in xrange(0, 10):
            Ptr = RecordSize*i
            Record = Records[Ptr:Ptr+RecordSize]
# Need another set of Records.
            if len(Record) < RecordSize:
                break
            ChanID = Record[15:18]
# The Q330 may create an "empty" file (all 00H) and then not finish filling it
# in. The read()s keep reading, but there's nothing to process. This detects
# that and returns. This may only happen in .bms-type data.
            if ChanID == "\x00\x00\x00":
# I guess if a file is scrambled these could have 1000's of things in them.
# Both of them should only have one thing for baler files. Convert them to
# strings and then chop them off if they are long.
                Stas = list2Str(StationIDs)
                if len(Stas) > 12:
                    Stas = Stas[:12]+"..."
                Chans = list2Str(Channels)
                if len(Chans) > 12:
                    Chans = Chans[:12]+"..."
                return (4, "YB", \
        "FILE ENDS EMPTY. Times: %s to %s\n   Recs: %d  Stas: %s  Chans: %s"% \
                     (FirstTime, LastTime, RecordsRead, Stas, Chans), 2, \
                     FirstTime, LastTime)
            if ChanID not in Channels:
                Channels.append(ChanID)
            StaID = Record[8:13].strip()
            if StaID not in StationIDs:
                StationIDs.append(StaID)
            Year, Doy, Hour, Mins, Secs, Tttt= unpack(">HHBBBxH", \
                    Record[20:30])
            Month, Date = q330yd2md(Year, Doy)
            DateTime = "%d-%02d-%02d %02d:%02d"%(Year, Month, Date, Hour, Mins)
            if DateTime < FirstTime:
                FirstTime = DateTime
            if DateTime > LastTime:
                LastTime = DateTime
    Fp.close()
    if RecordsRead == 0:
        return (5, "YB", "NO DATA.", 2, "", "")
# Same as above.
    Stas = list2Str(StationIDs)
    if len(Stas) > 12:
        Stas = Stas[:12]+"..."
    Chans = list2Str(Channels)
    if len(Chans) > 12:
        Chans = Chans[:12]+"..."
    if len(StationIDs) > 1:
        return (6, "MB", \
      "MULTIPLE STATIONS. Times: %s to %s\n   Recs: %d  Stas: %s  Chans: %s"% \
                (FirstTime, LastTime, RecordsRead, Stas, Chans), 3, \
                FirstTime, LastTime)
    elif len(Channels) > 1:
        return (7, "MB", \
      "MULTIPLE CHANNELS. Times: %s to %s\n   Recs: %d  Stas: %s  Chans: %s"% \
                (FirstTime, LastTime, RecordsRead, Stas, Chans), 3, \
                FirstTime, LastTime)
    else:
        return (0, "GB", \
                "FILE OK. Times: %s to %s\n   Recs: %d  Stas: %s  Chans: %s"% \
                (FirstTime, LastTime, RecordsRead, Stas, Chans), 1, \
                FirstTime, LastTime)
# END: checkDTFile




###############
# BEGIN: main()
# FUNC:main():2017.020
#   All of the action takes place here in one pass using if()'s.
FH_NAME = 0
FH_SIZE = 1
FH_FROM = 2
FH_TO = 3
Dirspec = ".%s"%sep
AbsDirspec = abspath(Dirspec)
if AbsDirspec.endswith(sep) == False:
    AbsDirspec += sep
if ArgUCheck == True:
    Ret = checkForUpdates("check")
    exit(Ret)
if ArgUDown == True:
    Ret = checkForUpdates("get")
    exit(Ret)
DirspecSDR = ".%s%s.sdr%s"%(sep, TagID, sep)
Msgspec = Dirspec+TagID+".msg"
# Duplicating some stuff so the messages come out the way I want them to.
if exists(Msgspec) == False:
    Fp = open(Msgspec, "w")
    Fp.close()
    logIt("%s %s"%(PROG_NAME, PROG_VERSION))
    if ArgInfo == True:
        logIt("Control computer IP address (maybe): %s"% \
                gethostbyname(gethostname()))
        exit(0)
    logIt("Working directory: %s"%AbsDirspec)
    logIt("Created %s"%Msgspec)
else:
    logIt("%s %s"%(PROG_NAME, PROG_VERSION))
    if ArgInfo == True:
        logIt("Control computer IP address (maybe): %s"% \
                gethostbyname(gethostname()))
        exit(0)
    logIt("Working directory: %s"%AbsDirspec)
    logIt("Using %s"%Msgspec)
# Hijack and detour the program for this.
if ArgBadBlocks == True:
    if exists(DirspecSDR) == False:
        logIt("Done checking. No files have been offloaded.")
        logIt("")
        exit(0)
    DFiles = listdir(DirspecSDR)
    DFiles.sort()
    FilesChecked = 0
    FilesOK = 0
# These get loaded with the file names depending on the Ret[0] value coming
# from checkDTFile() for a summary.
    FilesOpenErrors1 = []
    FilesTooSmall2 = []
    FilesRecSize3 = []
    FilesEndEmpty4 = []
    FilesNoData5 = []
    FilesMultStas6 = []
    FilesMultChans7 = []
    FilesIsDir8 = []
# For an overall baler time range.
    FirstFilesTime = "Z"
    LastFilesTime = ""
# Loop through all of the file(s) the user specified, then loop through DFiles
# looking for matches.
    for ArgSpecFile in ArgSpecFiles:
# Go through DFiles and turn each file into "" if it is checked, so we don't
# do it more than once if it matches more than one ArgSpecFile.
        Checked = 0
        for Index in xrange(0, len(DFiles)):
            DFile = DFiles[Index]
            if len(DFile) == 0:
                continue
            if fnmatch(DFile, ArgSpecFile):
                DFilespec = DirspecSDR+DFile
# checkDTFile() will also catch this. We'll still look for Rec[0]==8. Non-data
# files will probably come back with a Rec Size error.
                if isdir(DFilespec):
                    continue
                logIt("Checking %s..."%DFilespec)
                FilesChecked += 1
                Ret = checkDTFile(DFilespec)
                logIt("   "+Ret[2])
                FirstTime = Ret[4]
                LastTime = Ret[5]
# If something goes wrong the returned times will be "".
                if FirstTime != "" and FirstTime < FirstFilesTime:
                    FirstFilesTime = FirstTime
                if LastTime != "" and LastTime > LastFilesTime:
                    LastFilesTime = LastTime
                if Ret[0] == 0:
                    FilesOK += 1
                elif Ret[0] == 1:
                    FilesOpenErrors1.append(DFile)
                elif Ret[0] == 2:
                    FilesTooSmall2.append(DFile)
                elif Ret[0] == 3:
                    FilesRecSize3.append(DFile)
                elif Ret[0] == 4:
                    FilesEndEmpty4.append(DFile)
                elif Ret[0] == 5:
                    FilesNoData5.append(DFile)
                elif Ret[0] == 6:
                    FilesMultStas6.append(DFile)
                elif Ret[0] == 7:
                    FilesMultChans7.append(DFile)
                elif Ret[0] == 8:
                    FilesIsDir8.append(DFile)
                DFiles[Index] = ""
    logIt("Done checking.")
    logIt("")
    logIt("Summary:")
    logIt("   Date range: %s to %s"%(FirstFilesTime, LastFilesTime))
    logIt("Files checked: %d"%FilesChecked)
    logIt("     Files OK: %d"%FilesOK)
    if len(FilesOpenErrors1) != 0:
        logIt("File opening errors: %d"%len(FilesOpenErrors1))
        for File in FilesOpenErrors1:
            logIt("    "+File)
    if len(FilesTooSmall2) != 0:
        logIt("Files too small: %d"%len(FilesTooSmall2))
        for File in FilesTooSmall2:
            logIt("    "+File)
    if len(FilesRecSize3) != 0:
        logIt("Unknown record size: %d"%len(FilesRecSize3))
        for File in FilesRecSize3:
            logIt("    "+File)
    if len(FilesEndEmpty4) != 0:
        logIt("Ending empty: %d"%len(FilesEndEmpty4))
        for File in FilesEndEmpty4:
            logIt("    "+File)
    if len(FilesNoData5) != 0:
        logIt("No data: %d"%len(FilesNoData5))
        for File in FilesNoData5:
            logIt("    "+File)
    if len(FilesMultStas6) != 0:
        logIt("Multiple stations: %d"%len(FilesMultStas6))
        for File in FilesMultStas6:
            logIt("    "+File)
    if len(FilesMultChans7) != 0:
        logIt("Multiple channels: %d"%len(FilesMultChans7))
        for File in FilesMultChans7:
            logIt("    "+File)
    elif len(FilesIsDir8) != 0:
        logIt("Directories: %d"%len(FilesIsDir8))
        for File in FilesIsDir8:
            logIt("    "+File)
    logIt("")
    exit(0)
# All other commands.
# Get the basic info and make sure we are all on the same page. Do this each
# time we do anything.
Ret = getBalerHtm("bl", TagID, IPAddr)
if Ret[0] != 0:
    logIt(Ret[2])
    logIt("")
    exit(1)
# Might as well check for this before we go to the trouble of getting the list
# of files (which could take a long time).
if ArgVerify == True:
    if exists(DirspecSDR) == False:
        logIt("Done verifying. No files have been offloaded.")
        logIt("")
        exit(0)
FWVers = Ret[1]
DSize = Ret[2]
NFiles = Ret[3]
Percent = Ret[4]
if ArgCheck == True:
    Used = (Percent/100.0)*DSize
    logIt("FWVers: %s  DiskSize: %s"%(FWVers, fmti(DSize)))
    logIt("%.1f%% of %s %s used."%(Percent, fmti(NFiles), sP(NFiles, \
            ("file", "files"))))
    logIt("Done checking.")
    logIt("")
    exit(0)
# A List of Lists of the data files on the baler according to files.htm.
FHFiles = []
# files.htm can take a long time to build (it's made and sent on-the-fly).
if Percent < 50:
    logIt("Getting files.htm...")
else:
    logIt("Getting files.htm...(could take a while)")
Ret = getFilesHtm("", TagID, IPAddr)
if Ret[0] != 0:
    logIt(Ret[2])
    logIt("")
    exit(1)
logIt("Got files.htm.")
FHFiles = Ret[1]
# Always write the whole list out to the file ./<TagID>files.htm.
Ret = writeFilesTxt("bl", Dirspec+TagID+"files.txt", FHFiles)
if Ret[0] != 0:
    logIt(Ret[2])
    logIt("")
    exit(1)
if ArgList == True:
    Count = 0
    for File in FHFiles:
        Count += 1
        stdout.write("%d. %s  %s to %s  %d\n"%(Count, File[FH_NAME], \
                File[FH_FROM], File[FH_TO], File[FH_SIZE]))
    logIt("Done listing.")
    exit(0)
# Go through FHFiles and match up by name and size the files that are already
# on the computer. Then go through the files in the directory and see if there
# are any that don't belong there.
if ArgVerify == True:
    Here = 0
    NotHere = 0
    for Index in xrange(0, len(FHFiles)):
        Name = FHFiles[Index][FH_NAME]
        Size = FHFiles[Index][FH_SIZE]
        if exists(DirspecSDR+Name) and getsize(DirspecSDR+Name) == Size:
            Here += 1
        else:
            NotHere += 1
    logIt("Already offloaded %s, not offloaded %s."%(fmti(Here), \
            fmti(NotHere)))
    DFiles = listdir(DirspecSDR)
    Extras = 0
    FHFiles2 = []
    for FHFile in FHFiles:
        if len(FHFile) == 0:
            continue
        FHFiles2.append(FHFile[FH_NAME])
    for DFile in DFiles:
        if DFile.startswith("."):
            continue
        if DFile not in FHFiles2:
            logIt("Extra file: %s"%DFile)
            Extras += 1
    if Extras > 0:
        logIt("%s non-data %s in data files directory."%(fmti(Extras), \
                sP(Extras, ("file", "files"))))
    logIt("Done verifying.")
    logIt("")
    exit(0)
logIt("Offloading from baler %s @ %s"%(TagID, IPAddr))
# Alter FHFiles as necessary and then fill up BFiles with the remains.
if ArgSpec == True:
    logIt("Only offloading: %s"%list2Str(ArgSpecFiles))
    for ArgSpecFile in ArgSpecFiles:
        for Index in xrange(0, len(FHFiles)):
# For entries that are already [].
            try:
                if fnmatch(FHFiles[Index][FH_NAME], ArgSpecFile) == False:
                    FHFiles[Index] = []
                    # Don't list them. There could be a lot.
            except IndexError:
                continue
elif ArgSkip == True:
    logIt("Excluding: %s"%list2Str(ArgSkipFiles))
    for ArgSkipFile in ArgSkipFiles:
        for Index in xrange(0, len(FHFiles)):
            try:
                if fnmatch(FHFiles[Index][FH_NAME], ArgSkipFile):
                    FHFiles[Index] = []
            except IndexError:
                continue
# For any of the offloads always throw out files that are already here.
for Index in xrange(0, len(FHFiles)):
    try:
        Filename = FHFiles[Index][FH_NAME]
        Size = FHFiles[Index][FH_SIZE]
        if exists(DirspecSDR+Filename) and \
                getsize(DirspecSDR+Filename) == Size:
            FHFiles[Index] = []
    except:
        continue
if ArgLSROffload == True:
    logIt("Offloading low sample rate files.")
    for Index in xrange(0, len(FHFiles)):
        try:
            Filename = FHFiles[Index][FH_NAME]
            if Filename.find(".H") != -1 or Filename.find(".S") != -1:
                FHFiles[Index] = []
        except IndexError:
            continue
# Now move everything left to BFiles.
BFiles = []
BFilesBytes = 0
for File in FHFiles:
    if len(File) == 0:
        continue
    BFilesBytes += File[FH_SIZE]
    BFiles.append(File)
BFilesToOff = len(BFiles)
if BFilesToOff == 0:
    logIt("There are no files left in the list to offload.")
    exit(0)
# Create this after we know if there is even going to be anything to offload.
if exists(DirspecSDR) == False:
    makedirs(DirspecSDR)
    logIt("Saving data files to %s (created)"%DirspecSDR)
else:
    logIt("Saving data files to %s (exists)"%DirspecSDR)
logIt("Offloading %s %s, %s %s..."%(fmti(BFilesToOff), sP(BFilesToOff, \
        ("file", "files")), fmti(BFilesBytes), sP(BFilesBytes, ("byte", \
        "bytes"))))
Count = 0
# Files offloaded.
OffFiles = 0
# Total bytes offloaded.
OffBytes = 0
# Bytes from the current file offloaded.
OffFileSize = 0
for File in BFiles:
    OffFileSize = 0
    Offloaded10 = 0
    if len(File) == 0:
        continue
    Count += 1
    Filename = File[FH_NAME]
    BFileSize = File[FH_SIZE]
    if BFileSize == -1:
        logIt("%d/%d. Getting %s (?)..."%(Count, BFilesToOff, Filename))
    else:
        logIt("%d/%d. Getting %s (%s)..."%(Count, BFilesToOff, Filename, \
                fmti(BFileSize)))
    try:
        urlretrieve("http://%s/%s"%(IPAddr, Filename), "%s%s"%(DirspecSDR, \
                Filename), fileBlockRetrieved)
    except Exception, e:
        if Offloaded10 > 0:
            stdout.write("\n")
        logIt("ERROR: Retrieving file %s:"%Filename)
        logIt("%s (got ~%s %s)"%(e, fmti(OffFileBytes), sP(OffFileBytes, \
                ("byte", "bytes"))))
        logIt("")
        exit(1)
    if Offloaded10 > 0:
        stdout.write("\n")
    FileSize = getsize(DirspecSDR+Filename)
    if BFileSize != -1:
        if BFileSize != FileSize:
            logIt("ERROR: Size error. %s: Baler %s, File %s"%(Filename, \
                    fmti(BFileSize), fmti(FileSize)))
            logIt("")
            exit(1)
# Same as above.
    if FileSize < 4000:
        logIt("STRANGE: File %s is only %s %s."%(Filename, fmti(FileSize), \
                sP(FileSize, ("byte", "bytes"))))
    OffFiles += 1
    OffBytes += FileSize
logIt("Offloaded %s %s, %s %s.\a"%(fmti(OffFiles), sP(OffFiles, ("file", \
        "files")), fmti(OffBytes), sP(OffBytes, ("byte", "bytes"))))
logIt("")
exit(0)
# END: main
# END PROGRAM: BLINE