bline.py

#! /usr/bin/env python
# BEGIN PROGRAM: BLINE
# By: Bob Greschke
# Started: 2016.105
#   Baler Offloading Program

from sys import argv, exit, platform, stdout
PROGSystem = platform[:3].lower()
PROG_NAME = "BLINE"
PROG_NAMELC = "bline"
PROG_VERSION = "2017.020"
PROG_LONGNAME = "Command Line Baler Offload Program"
PROG_SETUPSVERS = "A"
VERS_VERSURL = "http://www.passcal.nmt.edu/~bob/passoft/"
VERS_PARTS = 4
#VERS_NAME = 0
VERS_VERS = 1
#VERS_USIZ = 2
VERS_ZSIZ = 3
PROGKiosk = False
PROGSmallScreen = False
ArgCheck = False
ArgInfo = False
ArgList = False
ArgVerify = False
ArgBadBlocks = False
ArgOffload = False
ArgLSROffload = False
ArgSkip = False
ArgSkipFiles = []
ArgSpec = False
ArgSpecFiles = []
ArgUCheck = False
ArgUDown = False
if len(argv) == 2 and argv[1] == "-#":
    stdout.write("%s\n"%PROG_VERSION)
    exit(0)
if len(argv) == 2 and argv[1] == "-i":
    import socket
    stdout.write("Contol computer IP address (maybe): %s\n"% \
            socket.gethostbyname(socket.gethostname()))
    exit(0)
if len(argv) == 2 and argv[1] == "-U":
    ArgUCheck = True
elif len(argv) == 2 and argv[1] == "-UD":
    ArgUDown = True    
else:
    if len(argv) == 1 or argv[1] == "-h":
        stdout.write("\n")
        stdout.write( \
    "Usage: bline.py <TagID> <IP address> <options and commands>\n")
        stdout.write( \
    "       bline.py <TagID> -V [<file(s)>]\n\n\a")
        stdout.write("<TagID> = The tag ID on the front of the baler.\n")
        stdout.write( \
    "<IP address> = The IP address assigned to the baler by BaleAddr.\n")
        stdout.write( \
    "-c = Checks communication with the baler and gets basic information.\n")
        stdout.write( \
    "-i = Reports what may be the control computer's IP address.\n")
        stdout.write( \
    "-L = Saves and displays the list of files on the baler.\n")
        stdout.write( \
    "-v = Gets the baler's list of files and checks the offloaded files.\n")
        stdout.write( \
    "-V <file(s)> = Examines the specified offloaded files for bad blocks.\n")
        stdout.write( \
    "               The list of files may be omitted. In that case all of\n")
        stdout.write( \
    "               the files in the balers's .sdr directory will be\n")
        stdout.write( \
    "               examined.\n")
        stdout.write( \
    "-O = (Big O) Offloads all data files that have not been offloaded.\n")
        stdout.write( \
    "-o = (Little O) Offloads low sample rate data files that have not been\n")
        stdout.write("     offloaded.\n")
        stdout.write( \
    "-E <file(s)> = Excludes the specified file(s) during an offload.\n")
        stdout.write( \
    "-F <file(s)> = Offloads only the specified file(s).\n")
        stdout.write("-h = This help.\n")
        stdout.write("-H = More help.\n")
        stdout.write("\n")
        stdout.write("If connected to the Internet:\n")
        stdout.write("-U = Checks for a newer program version at PASSCAL.\n")
        stdout.write( \
    "-UD = Downloads most recent version from PASSCAL (try -U first).\n")
        stdout.write("\n")
        stdout.write("-c, -i, -v, -F will override the other commands.\n")
        stdout.write( \
    "-i may be the second command line argument: bline.py -i\n")
        stdout.write( \
    "-U and -UD must be the second command line argument: bline.py -U/-UD\n")
        stdout.write( \
    "-E command may be preceeded by -O or -o (default is -O).\n")
        stdout.write( \
    "-E command is, for example, for excluding \"problem\" files that may\n")
        stdout.write("   be stopping the offload process.\n")
        stdout.write("-F command overrides and does not use -O or -o.\n")
        stdout.write( \
    "-F files that are not found on the baler will be created with HTML\n")
        stdout.write( \
    "   code in them saying the file doesn't exist (~100 bytes size).\n")
        stdout.write("-E or -F commands and their file(s) must be last.\n")
        stdout.write( \
    "*, ?, [] UNIX file wildcards may be used in -E and -F file names.\n")
        stdout.write( \
    "On some systems you may need to enclose file names using wildcards\n")
        stdout.write( \
    "with quotes like  \"*.VER\" \"*.HHE\"\n")
        stdout.write("\n")
        exit(0)
    if argv[1] == "-H":
        HELPText = "USING BLINE.PY\n\
--------------\n\
BLINE.PY offloads data files from a Quantera B14 Baler. It does not\n\
set up the baler for offloading. BaleAddr is the Quanterra\n\
Windows/Wine program that you start after you've hooked up the baler\n\
to the laptop and to power. In it you set the IP address that you want\n\
the baler to be set to when the baler starts up. If, for example, the\n\
laptop IP address is something like 129.138.26.1, you would want the\n\
baler's address to end up something like 129.138.26.2. Same sub-net,\n\
etc. You set the desired IP address in BaleAddr then poke the ATTN button\n\
on the baler. When baler gets going BaleAddr handshakes and sets the IP\n\
address.\n\
\n\
You can use\n\
\n\
   bline.py -i\n\
\n\
to see what the IP address of the computer is set to. It might be\n\
correct. It won't be correct if the Wireless is on, so turn that off.\n\
Use the operating system's network setting widget to double check what\n\
the address is if the address from the -i command does not seem to be\n\
working.\n\
\n\
Depending on the operating system of the computer you are going to run\n\
BaleAddr and bline.py on, you may have to manually set an IP address\n\
for the computer. Some operating systems do not set their IP address\n\
until they are connected to a network, and in some cases there is no\n\
network until the baler wakes up from an ATTN button push. By then it\n\
is too late.\n\
\n\
When everything works you should get the messages\n\
\n\
Setting time in baler\n\
Baler OK\n\
\n\
in BaleAddr. If the 'Baler OK' message doesn't show up then the\n\
program is not really talking to the baler/the baler's IP address did\n\
not get set. It happens a lot. It seems to be a bug in the baler\n\
firmware. Quiting/Resetting BaleAddr, re-poking the ATTN button on the\n\
baler to let the baler shut down, restarting BaleAddr (if it was Quit),\n\
and poking the ATTN button again up to five times has been required to\n\
get the program and the baler to connect. It's just flakey. If you get\n\
up to maybe three cycles you can try shutting down the baler and then\n\
disconnecting the power and reconnecting, and then try again to get\n\
things going. Sometimes that helps. Make sure you have a suitable IP\n\
address in BaleAddr that is compatible with the address the computer\n\
thinks it has. That REALLY helps. :) Also make sure the Wireless is\n\
turned off like you have to do with EzBaler. None of the programs can\n\
figure out which Ethernet adapter to use when there is more than one\n\
active.\n\
\n\
bline.py is just a command line program, so once BaleAddr has\n\
connected to the baler you start a Terminal/xterm/whatever window, cd\n\
into a directory where you want a folder of data from the baler to end\n\
up, like a \"DATA\" directory, and enter\n\
\n\
   bline.py  or\n\
   <path to bline>/bline.py  or\n\
   ./bline.py\n\
\n\
It all depends on how bline.py was installed. Using no command line\n\
arguments will list all of the commands for the program. For basic\n\
offloading of everything on the baler you just enter\n\
\n\
   bline.py <TagID> <IP address> -O\n\
\n\
<TagID> is the serial number on the baler, like 6003, and the <IP\n\
address> is the address you assigned to the baler using BaleAddr. The\n\
offloading should start. A sub-directory like 6003.sdr will be created\n\
and the data files from the baler will be in there. The file\n\
6003.msg will be created in the directory where you started the\n\
program from. It will have a copy of the messages that get written to\n\
the terminal window as the program is working. There will be a\n\
6003files.txt. This will just be a list of all of the data files that\n\
were found on the baler.\n\
\n\
When you are finished with the baler and have all of the data files\n\
offloaded you can bring the baler up in BaleAddr again (assuming it's\n\
like days later and the baler has shut down) and use the Clean Baler\n\
button to erase everything and get the baler ready to go back out to\n\
the field. Please insure you have ALL of the data files, and that they\n\
are backed up, because unlike EzBaler, BaleAddr completly erases the\n\
old data.\n\
\n\
INSTALLING\n\
\n\
To install bline copy it to where ever you want. If you put it\n\
somewhere like /opt/passcal/bin/, with the other PASSCAL software, then\n\
it should be in the path set up by the PASSCAL software installation,\n\
and you will just have to use \"bline.py\" to get it going. You will\n\
probably have to use\n\
\n\
   sudo cp bline.py /opt/passcal/bin\n\
\n\
and give the login password to get it there. You will also probably\n\
have to make it executable with something like\n\
\n\
   chmod +x bline.py\n\
\n\
Depending on where you put it you may also have to use sudo before\n\
chmod. If it is not executable you will get a 'permission denied'\n\
message when you try to start it.\n\
\n\
SOME COMMAND DETAILS\n\
\n\
-v (little V)\n\
The -v command gets the list of files from the baler (files.htm\n\
if you are talking to the baler with a browser) and then looks to see\n\
if all of the files on that list are in the .sdr directory on the\n\
computer and if they are the same size as in the list. The function does\n\
not do any checksum checking, because there's no way to get the baler\n\
to compute a checksum value of the files on the baler to verify against.\n\
\n\
-V (big V)\n\
The -V command reads block-by-block through the offloaded data file(s)\n\
in the .sdr directory for the specified baler (TagID) and collects and\n\
reports:\n\
\n\
   1. The earliest block header time in the file\n\
   2. The latest block header time in the file\n\
   3. A list of all of the station IDs in the file (should normally\n\
      only be one)\n\
   4. A list of all of the channel names in a file (should only be one)\n\
   5. The number of blocks read\n\
\n\
For baler data files the normal block size is 4096 bytes and that is\n\
normally 4100 blocks per 16MB data file.\n\
\n\
Things to look for after the function runs are bad/scrambled/missing\n\
dates and times, multiple station names, multiple channel names and the\n\
program crashing. The latest block time will not normally be the same\n\
as the last sample time for each channel. For quiet and low sample rate\n\
channels the time may be quite a bit earlier than the last sample time.\n\
\n\
A space-separated list of files, a group of files, or all of the files\n\
in a .sdr directory to examine may be supplied on the command line using\n\
the standard UNIX wild card characters. On some systems you may need to\n\
enclose the file names in double quotes like  \"*\", instead of just  *.\n\
Not supplying any file list is the same as using \"*\".\n"
        stdout.write("\n")
        stdout.write(HELPText)
        stdout.write("\n")
        exit(0)
# .upper in case letters are ever in there.
    TagID = argv[1].upper()
# They are on the label, but I don't think any software uses them.
    while TagID.startswith("0"):
        TagID = TagID[1:]
# There is no IP address with the -V command. It's just looking at local files.
    if argv[2] == "-V":
        ArgBadBlocks = True
# You don't have to supply the file(s) list.
        try:
            x = argv[3]
            for I in xrange(3, len(argv)):
                ArgSpecFiles.append(argv[I].upper())
        except IndexError:
            ArgSpecFiles.append("*")
    else:
        IPAddr = argv[2]
        for Index in xrange(3, len(argv)):
            Arg = argv[Index]
            if Arg == "-c":
                ArgCheck = True
                break
            if Arg == "-i":
                ArgInfo = True
                break
            if Arg == "-L":
                ArgList = True
                break
            if Arg == "-v":
                ArgVerify = True
                break
            if Arg == "-O":
                ArgOffload = True
                continue
            if Arg == "-o":
                ArgLSROffload = True
                continue
            if Arg == "-E":
                ArgSkip = True
                for I in xrange(Index+1, len(argv)):
                    ArgSkipFiles.append(argv[I].upper())
                break
            if Arg == "-F":
                ArgSpec = True
                for I in xrange(Index+1, len(argv)):
                    ArgSpecFiles.append(argv[I].upper())
                break
            stdout.write("What??: '%s'\n\n"%Arg)
            exit(1)
# Do a little error checking.
    if ArgSpec == True:
        if len(ArgSpecFiles) == 0:
            stdout.write("No -F <file(s)> provided.\n")
            exit(0)
    if ArgSkip == True:
        if len(ArgSkipFiles) == 0:
            stdout.write("No -E <file(s)> provided.\n")
            exit(0)
    if ArgBadBlocks == True:
        if len(ArgSpecFiles) == 0:
            stdout.write("No -V <file(s)> provided.\n")
            exit(0)

from struct import unpack


########################
# BEGIN: versionChecks()
# LIB:versionChecks():2016.230
#   Checks the current version of Python and sets up a couple of things for the
#   rest of the program to use.
#   Obviously this is not a real function. It's just a collection of things for
#   all programs to check and making it look like a library function makes it
#   easy to update everywhere.
# For putting Python 3 support in.
from sys import version_info
PROG_PYVERSION = "%d.%d.%d"%(version_info[0], version_info[1], version_info[2])
if version_info[0] != 2:
    stdout.write("%s only runs on Python 2.\n"%PROG_NAME)
    if version_info[1] < 4:
# The isinstance(X, *tuple*) stuff requires this.
        stdout.write("%s only runs on Python 2.4 and above.\n"%PROG_NAME)
    exit()
# basestring covers ASCII and Unicode str's and showed up in v2.4.
try:
    isinstance(PROG_NAME, basestring)
except NameError:
    basestring = str
# I think float_info showed up in 2.6.
try:
    from sys import float_info
    minfloat = float_info.min
    maxfloat = float_info.max
except:
# These should be big enough for my programs, and not bigger than any system
# they run on can handle.
    minfloat = -1.0E100
    maxfloat = 1.0E100
# END: versionChecks

from fnmatch import fnmatch
from inspect import stack
if PROGSystem == "dar" or PROGSystem == "lin" or PROGSystem == "sun":
    from os import getuid
from os import makedirs, listdir, sep, umask
from os.path import abspath, basename, exists, getsize, isdir
from socket import setdefaulttimeout, gethostbyname, gethostname
#setdefaulttimeout(20) for now. Seems to work better.
from string import rstrip, strip
from time import gmtime, localtime, sleep, strftime, time
from urllib import urlopen, urlretrieve

# All files will be created with u=rw g=rw o=rw.
umask(0000)

# These are for some of the functions that are shared with bgoff.py. They are
# not used in bline.py.
STATE_IDLE = 0
STATE_CHECK = 1
STATE_OFFLD = 2
STATE_STOP = 99
BState = 0


##############################
# BEGIN: checkForUpdates(What)
# FUNC:checkForUpdates():2016.268
def checkForUpdates(What):
# Finds the "new"+PROG_NAMELC+".txt" file created by the program webvers at
# the URL and checks to see if the version in that file matches the version
# of this program.
    if What == "check":
        stdout.write("Checking for updates...\n")
    elif What == "get":
        stdout.write("Downloading...\n")
# Get the file that tells us about the current version on the server.
# One line:  PROG; version; original size; compressed size
    try:
        Fp = urlopen(VERS_VERSURL+"new"+PROG_NAMELC+".txt")
        Line = readFileLines(Fp)
        Fp.close()
        Line = Line[0]
# If the file doesn't exist you get something like
#     <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
# How unhandy.
        if Line.find("DOCTYPE") != -1:
            stdout.write("No update information found.\n")
            return 0
    except (IndexError, IOError):
        stdout.write( \
       "There was an error obtaining the version information from PASSCAL.\n")
        return 1
    Parts = map(strip, Line.split(";"))
    Parts += (VERS_PARTS-len(Parts))*[""]
    if What == "check":
        if PROG_VERSION < Parts[VERS_VERS]:
            stdout.write( \
                    "This is an old version of %s.\nThe current version generally available is %s.\nUse  bline.py -UD  to get the new version.\n"% \
                    (PROG_NAME, Parts[VERS_VERS]))
            return 0
        elif PROG_VERSION == Parts[VERS_VERS]:
            stdout.write("This copy of %s is up to date.\n"%PROG_NAME)
            return 0
        elif PROG_VERSION > Parts[VERS_VERS]:
            stdout.write( \
                    "Congratulations!\nThis is a newer version of %s than is generally available.\nEveryone else probably still has version %s.\n"% \
                    (PROG_NAME, Parts[VERS_VERS]))
            return 0
    elif What == "get":
        ZSize = int(Parts[VERS_ZSIZ])
        try:
            GetFile = "new%s.zip"%PROG_NAMELC
            Fpr = urlopen(VERS_VERSURL+GetFile)
            Fpw = open(Dirspec+GetFile, "wb")
            DLSize = 0
            Buffer = Fpr.read()
            Fpw.write(Buffer)
        except Exception, e:
# The Fp's may not exist.
            try:
                Fpr.close()
            except:
                pass
            try:
                Fpw.close()
            except:
                pass
            stdout.write("Error downloading new version.\n%s"%e)
            return 1
        Fpr.close()
        Fpw.close()
        stdout.write("The current update file been saved as\n\n    %s\n\n"% \
                (AbsDirspec+GetFile))
        stdout.write( \
                "Unzip the file, rename new%s.py to %s.py,\nand copy it to its final destination.\n"% \
                (PROG_NAMELC, PROG_NAMELC))
    return 0
# END: checkForUpdates




###################
# BEGIN: floatt(In)
# LIB:floatt():2009.102
#    Handles all of the annoying shortfalls of the float() function (vs C).
#    Does not handle scientific notation numbers.
def floatt(In):
    In = str(In).strip()
    if In == "":
        return 0.0
# At least let the system give it the ol' college try.
    try:
        return float(In)
    except:
        Number = ""
        for c in In:
            if c.isdigit() or c == ".":
                Number += c
            elif Number == "" and (c == "-" or c == "+"):
                Number += c
            elif c == ",":
                continue
            else:
                break
        try:
            return float(Number)
        except ValueError:
            return 0.0
# END: floatt




####################
# BEGIN: fmti(Value)
# LIB:fmti():2013.129
#   Just couldn't rely on the system to do this, although this won't handle
#   European-style numbers.
def fmti(Value):
    Value = int(Value)
    if Value > -1000 and Value < 1000:
        return str(Value)
    Value = str(Value)
    NewValue = ""
# There'll never be a + sign.
    if Value[0] == "-":
        Offset = 1
    else:
        Offset = 0
    CountDigits = 0
    for i in xrange(len(Value)-1, -1+Offset, -1):
        NewValue = Value[i]+NewValue
        CountDigits += 1
        if CountDigits == 3 and i != 0:
            NewValue = ","+NewValue
            CountDigits = 0
    if Offset != 0:
        if NewValue.startswith(","):
            NewValue = NewValue[1:]
        NewValue = Value[0]+NewValue
    return NewValue
# END: fmti




#########################################################
# BEGIN: fileBlockRetrieved(Blocks, BlockSize, TotalSize)
# FUNC:fileBlockRetrieved():2016.273
#   Gets called every 8192 bytes offloaded.
def fileBlockRetrieved(Blocks, BlockSize, TotalSize):
    global OffFileBytes
    global Offloaded10
# This will be kept updated for any offloading error message.
    OffFileBytes = Blocks*BlockSize
# Will give us 10% 20% 30%...
    Offed = int(100.0*OffFileBytes/TotalSize)//10*10
    if Offed > Offloaded10:
        if Offloaded10 == 0:
            stdout.write("  ")
        Offloaded10 = Offed
        stdout.write(" %d%%"%Offloaded10)
        stdout.flush()
    return
# END: fileBlockRetrieved




########################################
# BEGIN: getBalerHtm(Msg, TagID, IPAddr)
# LIB:getBalerHtm():2016.279
def getBalerHtm(Msg, TagID, IPAddr):
    global BState
    if Msg == "bg":
        msgLn(1, "W", "Getting baler.htm...")
    elif Msg == "bl":
        logIt("Getting baler.htm...")
    try:
        Fp = urlopen("http://%s/baler.htm"%IPAddr)
        Lines = readFileLines(Fp, True)
        Fp.close()
    except IOError, e:
        return (2, "MW", "Error getting baler.htm:\n%s\nStopped."%e, 3)
# Set by hitting the Check button when it is already checking.
    if BState == STATE_STOP:
        return (1, "YB", "Stopped by user. (%s)"%getGMT(3), 2)
    FWVers = "?"
    TagID2 = "?"
    DSize = 0
    NFiles = 0
    Percent = 0.0
    for Line in Lines:
        Line = Line.upper()
        if Line.find(">SOFTWARE VERSION:") != -1:
            Value = floatt(Line[(Line.index(">SOFTWARE VERSION:")+ \
                    len(">SOFTWARE VERSION:")):])
            FWVers = str(Value)
            continue
        if Line.find("BALER TAGID:") != -1:
            Value = intt(Line[(Line.index(">BALER TAGID:")+ \
                    len(">BALER TAGID:")):])
            TagID2 = str(Value)
            continue
        if Line.find("DISK SIZE:") != -1:
            Value = intt(Line[(Line.index("DISK SIZE:")+ \
                    len("DISK SIZE:")):])
            DSize = Value
            continue
        if Line.find("PERCENT OF") != -1:
            Parts = Line.split()
            NFiles = intt(Parts[2])
            Percent = floatt(Parts[-1])
            continue
# Talking to the wrong baler??
    if TagID != TagID2:
        return (2, "R", \
                "TagIDs do not match: Entered %s, baler %s.\nStopped. (%s)"% \
                (TagID, TagID2, getGMT(3)), 2)
    if Msg == "bg":
        msgLn(1, "", "Got baler.htm.")
    elif Msg == "bl":
        logIt("Got baler.htm.")
    return (0, FWVers, DSize, NFiles, Percent)
# END: getBalerHtm




########################################
# BEGIN: getFilesHtm(Msg, TagID, IPAddr)
# LIB:getFilesHtm():2016.278
# Used in many places to decode the List of Lists of files returned by this
# function.
FH_NAME = 0
FH_SIZE = 1
FH_FROM = 2
FH_TO = 3

def getFilesHtm(Msg, TagID, IPAddr):
    global BState
    if Msg == "bg":
        msgLn(1, "W", "Getting files.htm...")
    elif Msg == "bl":
        logIt("Getting files.htm...")
    try:
        Fp = urlopen("http://%s/files.htm"%IPAddr)
        Lines = readFileLines(Fp, True)
        Fp.close()
    except IOError, e:
        return (1, "MW", "Error getting files.htm:\n%s\nStopped. (%s)"%(e, \
                getGMT(3)), 3)
    if BState == STATE_STOP:
        return (1, "YB", "Stopped by user. (%s)"%getGMT(3), 2)
    FHFiles = []
    FHBytes = 0
    for Line in Lines:
# The line case is mixed, but the file names are always all uppercase, so just
# do that.
        Line = Line.upper()
# I don't know how often these htm files have changed, so check a bunch of
# things. The files.htm I got from v2.26 that looks like
#    <A http="DT000200.HHZ">DT000200.HHZ</A>
#            16777216 bytes, from 2011-10-28 11:07:58 to 2011-10-30 00:54:16
        if Line.find("<A HREF=\"") == -1:
            continue
        Parts = Line.split()
        if Parts[0] != "<A":
            continue
# The filename is the only thing surrounded by quotes.
        Filename = Parts[1][Parts[1].index("\"")+1:Parts[1].rindex("\"")]
# Eliminate any IP addresses or other stuff that may get prepended to the
# Filenames.
        Filename = basename(Filename)
        if Parts[3] == "BYTES," and Parts[7] == "TO":
            Size = int(Parts[2])
            From = "%s %s"%(Parts[5], Parts[6])
            To = "%s %s"%(Parts[8], Parts[9])
            FHFiles.append([Filename, Size, From, To])
            FHBytes += Size
    if len(FHFiles) == 0:
        return (1, "Y", "No files found in files.htm.\nStopped. (%s)"% \
                getGMT(3))
    if BState == STATE_STOP:
        return (1, "YB", "Stopped by user. (%s)"%getGMT(3), 2)
    FHFiles = listSort(FHFiles, FH_NAME, "a")
    if Msg == "bg":
        msgLn(1, "", "Got files.htm.")
    elif Msg == "bl":
        logIt("Got files.htm.")
    return (0, FHFiles, FHBytes)
# END: getFilesHtm




#######################
# BEGIN: getGMT(Format)
# LIB:getGMT():2015.006
#   Gets the time in various forms from the system.
def getGMT(Format):
# YYYY:DOY:HH:MM:SS (GMT)
    if Format == 0:
        return strftime("%Y:%j:%H:%M:%S", gmtime(time()))
# YYYYDOYHHMMSS (GMT)
    elif Format == 1:
        return strftime("%Y%j%H%M%S", gmtime(time()))
# YYYY-MM-DD (GMT)
    elif Format == 2:
        return strftime("%Y-%m-%d", gmtime(time()))
# YYYY-MM-DD HH:MM:SS (GMT)
    elif Format == 3:
        return strftime("%Y-%m-%d %H:%M:%S", gmtime(time()))
# YYYY, MM and DD (GMT) returned as ints
    elif Format == 4:
        GMT = gmtime(time())
        return (GMT[0], GMT[1], GMT[2])
# YYYY-Jan-01 (GMT)
    elif Format == 5:
        return strftime("%Y-%b-%d", gmtime(time()))
# YYYYMMDDHHMMSS (GMT)
    elif Format == 6:
        return strftime("%Y%m%d%H%M%S", gmtime(time()))
# Reftek Texan (year-1984) time stamp in BBBBBB format (GMT)
    elif Format == 7:
        GMT = gmtime(time())
        return pack(">BBBBBB", (GMT[0]-1984), 0, 1, GMT[3], GMT[4], GMT[5])
# Number of seconds since Jan 1, 1970 from the system.
    elif Format == 8:
        return time()
# YYYY-MM-DD/DOY HH:MM:SS (GMT)
    elif Format == 9:
        return strftime("%Y-%m-%d/%j %H:%M:%S", gmtime(time()))
# YYYY-MM-DD/DOY (GMT)
    elif Format == 10:
        return strftime("%Y-%m-%d/%j", gmtime(time()))
# YYYY, DOY, HH, MM, SS (GMT) returned as ints
    elif Format == 11:
        GMT = gmtime(time())
        return (GMT[0], GMT[7], GMT[3], GMT[4], GMT[5])
# HH:MM:SS (GMT)
    elif Format == 12:
        return strftime("%H:%M:%S", gmtime(time()))
# YYYY:DOY:HH:MM:SS (LT)
    elif Format == 13:
        return strftime("%Y:%j:%H:%M:%S", localtime(time()))
# HHMMSS (GMT)
    elif Format == 14:
        return strftime("%H%M%S", gmtime(time()))
# YYYY-MM-DD (LT)
    elif Format == 15:
        return strftime("%Y-%m-%d", localtime(time()))
# YYYY-MM-DD/DOY Day (LT)
    elif Format == 16:
        return strftime("%Y-%m-%d/%j %A", localtime(time()))
# MM-DD (LT)
    elif Format == 17:
        return strftime("%m-%d", localtime(time()))
# YYYY, MM and DD (LT) returned as ints
    elif Format == 18:
        LT = localtime(time())
        return (LT[0], LT[1], LT[2])
# YYYY-MM-DD/DOY HH:MM:SS Day (LT)
    elif Format == 19:
        return strftime("%Y-%m-%d/%j %H:%M:%S %A", localtime(time()))
# Return GMT-LT difference.
    elif Format == 20:
        Secs = time()
        LT = localtime(Secs)
        GMT = gmtime(Secs)
        return dt2Timeymddhms(-1, LT[0], -1, -1, LT[7], LT[3], LT[4], LT[5])- \
                dt2Timeymddhms(-1, GMT[0], -1, -1, GMT[7], GMT[3], GMT[4], \
                GMT[5])
# YYYY-MM-DD/DOY HH:MM:SS (LT)
    elif Format == 21:
        return strftime("%Y-%m-%d/%j %H:%M:%S", localtime(time()))
# YYYY-MM-DD HH:MM:SS (LT)
    elif Format == 22:
        return strftime("%Y-%m-%d %H:%M:%S", localtime(time()))
    return ""
# END: getGMT




#################
# BEGIN: intt(In)
# LIB:intt():2009.102
#   Handles all of the annoying shortfalls of the int() function (vs C).
def intt(In):
    In = str(In).strip()
    if In == "":
        return 0
# Let the system try it first.
    try:
        return int(In)
    except ValueError:
        Number = ""
        for c in In:
            if c.isdigit():
                Number += c
            elif Number == "" and (c == "-" or c == "+"):
                Number += c
            elif c == ",":
                continue
            else:
                break
        try:
            return int(Number)
        except ValueError:
            return 0
# END: intt




########################################################################
# BEGIN: list2Str(TheList, Delim = ", ", Sort = True, DelBlanks = False)
# LIB:list2Str():2014.343
def list2Str(TheList, Delim = ", ", Sort = True, DelBlanks = False):
    if isinstance(TheList, list) == False:
        return TheList
    if Sort == True:
        TheList.sort()
    Ret = ""
# If there is any funny-business (which has been seen).
    for Item in TheList:
        try:
            if DelBlanks == True:
                if str(Item) == "":
                    continue
            Ret += str(Item)+Delim
        except UnicodeEncodeError:
            Ret += "Error"+Delim
    return Ret[:-(len(Delim))]
# END: list2Str




#####################################################
# BEGIN: listSort(InList, Index, How, Direction = "")
# LIB:listSort():2013.218
def listSort(InList, Index, How, Direction = ""):
    if len(InList) == 0:
        return InList
    if How == "n":
        if isinstance(InList[0], basestring):
# Use floatt() here since we don't know if the string will be representing an
# int or a float.
            Temp = [(floatt(Item), Item) for Item in InList]
        elif isinstance(InList[0], (int, long, float)):
            Temp = [(Item, Item) for Item in InList]
        elif isinstance(InList[0], (list, tuple)):
            if isinstance(InList[0][Index], basestring):
                Temp = [(floatt(Item[Index]), Item) for Item in InList]
            else:
                Temp = [(Item[Index], Item) for Item in InList]
        else:
            return InList
    elif How == "a":
        Temp = [(Line[Index], Line) for Line in InList]
    Temp.sort()
    if Direction == "desc":
        Temp.reverse()
    return [Line[1] for Line in Temp]
# END: listSort




###################
# BEGIN: logIt(Msg)
# FUNC:logIt():2016.278
def logIt(Msg):
# Some messages will come with multiple lines (like from LIB functions that
# also need to work with bgoff.py). Split those up. Some messages will come
# with (date time) at the end (again, for bgoff.py). Strip those off.
    Lines = Msg.split("\n")
    for Index in xrange(0, len(Lines)):
# If we happen to go over UT midnight this won't be caught. I'll risk it.
        try:
            I = Lines[Index].index(" (%s "%getGMT(2))
            Lines[Index] = Lines[Index][0:I]
        except:
            continue
    for Line in Lines:
        if Line != "":
            stdout.write("%s  %s\n"%(getGMT(3), Line))
        else:
            stdout.write("\n")
    stdout.flush()
    Fp = open(Msgspec, "a")
    for Line in Lines:
        if Line != "":
            Fp.write("%s  %s\n"%(getGMT(3), Line))
        else:
            Fp.write("\n")
    Fp.close()
    return
# END: logIt




################################################
# BEGIN: readFileLines(Fp, Strip = False, N = 0)
# LIB:readFileLines():2015.009
#   Reads the passed file and determines how to split the lines which may end
#   differently depending on which operating system the file was written by.
#   A blob of text may also be passed as Fp and that will be split into a List
#   of lines.
def readFileLines(Fp, Strip = False, N = 0):
    if isinstance(Fp, list) == False:
        if N == 0:
            RawLines = Fp.readlines()
        else:
            RawLines = Fp.readlines(N)
    else:
# The order is important: \r\n  then  \r  then  \n.
        if Fp.find("\r\n") != -1:
            RawLines = Fp.split("\r\n")
        elif Fp.find("\r") != -1:
            RawLines = Fp.split("\r")
        elif Fp.find("\n") != -1:
            RawLines = Fp.split("\n")
        else:
            RawLines = []
    Lines = []
# In case the file was empty.
    try:
        if RawLines[0].find("\r\n") != -1:
            Count = RawLines[0].count("\r\n")
# If there is only one \r\n (presumably at the end of the line) then the OS
# must have split the file up correctly. In that case just run the map() and
# clean off the ends of the lines.
            if Count == 1:
                if Strip == False:
                    Lines += map(rstrip, RawLines)
                else:
                    Lines += map(strip, RawLines)
# If there is more than one \r\n in the string then it could mean that all of
# the file lines are all jammed together into one long string. In that case
# split the line up first then add the parts to Lines, but leave off the
# "extra" blank line ([:-1]) that the split() will produce.
            else:
                for Line in RawLines:
                    Parts = Line.split("\r\n")
                    if Strip == False:
                        Lines += Parts[:-1]
                    else:
                        Lines += map(strip, Parts[:-1])
# Same as \r\n.
        elif RawLines[0].find("\r") != -1:
            Count = RawLines[0].count("\r")
            if Count == 1:
                if Strip == False:
                    Lines += map(rstrip, RawLines)
                else:
                    Lines += map(strip, RawLines)
            else:
                for Line in RawLines:
                    Parts = Line.split("\r")
                    if Strip == False:
                        Lines += Parts[:-1]
                    else:
                        Lines += map(strip, Parts[:-1])
# Same as \r.
        elif RawLines[0].find("\n") != -1:
            Count = RawLines[0].count("\n")
            if Count == 1:
                if Strip == False:
                    Lines += map(rstrip, RawLines)
                else:
                    Lines += map(strip, RawLines)
            else:
                for Line in RawLines:
                    Parts = Line.split("\n")
                    if Strip == False:
                        Lines += Parts[:-1]
                    else:
                        Lines += map(strip, Parts[:-1])
# What else can we do?
        else:
            Lines += RawLines
    except:
        pass
    return Lines
# END: readFileLines




###########################
# BEGIN: sP(Count, Phrases)
# LIB:sP():2012.223
def sP(Count, Phrases):
    if Count == 1 or Count == -1:
        return Phrases[0]
    else:
        return Phrases[1]
# END: sP




#############################################
# BEGIN: writeFilesTxt(Msg, Filesspec, Files)
# LIB:writeFilesTxt():2016.278
def writeFilesTxt(Msg, Filesspec, Files):
    global BState
    FilesCount = 0
    FilesBytes = 0
    try:
        Fp = open(Filesspec, "w")
        FilesCount = 0
        for File in Files:
            FilesCount += 1
            Fp.write("%d. %s  %s to %s  %d\n"%(FilesCount, File[FH_NAME], \
                    File[FH_FROM], File[FH_TO], File[FH_SIZE]))
            FilesBytes += File[FH_SIZE]
        Fp.close()