diff --git a/HISTORY.rst b/HISTORY.rst
index f137e0cc63fde1b73353225cf6deb0dccf382e3a..6c2566e740cc7a53196c97b80c2e485d368dc841 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -2,88 +2,51 @@
History
=======
-2016.288 (2016-10-14)
-------------------
-* New. No changes.
-
-2016.355 (2016-12-20)
-------------------
-* Just changed some help descriptions a little.
-* Added the -H "more" help.
-
-2017.006 (2017-01-06)
-------------------
-* New -V command for examining offloaded files a bit more.
-* Added a bit more to the -H help.
-
-2017.012 (2017-01-12)
-------------------
-* Added a summary to the -V command output.
-
-2017.020 (2017-01-20)
-------------------
-* Added baler time range (all of the files) to the -V summary.
-* A couple little corrections to the help.
-
-2017.024 (2017-01-24)
-------------------
-* Added some more -H command help.
-* Made some of the status messages a little more wordy.
-* Commands like -F "DT0001*" "DT0002*" did not work. Having multiple search
- patterns would fail. "DT0002*" by itself, for example, would have worked.
- Now it all works.
-
-2017.026 (2017-01-26)
-------------------
-* Rewrote all of the command line handling stuff. There could be bugs
- all over the place.
-* Added the -m "A message" command.
-* Added the -e [<files>] command for excluding files, but then only
- offloading low sample rate files (like -o). -E offloads all files
- (like -O).
-
-2017.038 (2017-02-07)
-------------------
-* Prints out the current version when using the -U command.
-* Put in note about using "bline.py" or "bline" in the helps.
-
-2017.039 (2017-02-08)
-------------------
-* Fixed a bug in the -v command that showed up when files were missing
- or not everything had been offloaded.
-* Took the note back out about bline/bline.py. It will always be
- bline.py.
-
-2017.055 (2017-02-24)
-------------------
-* Changed the way the record size of a file is determined. It should
- always be 4096 bytes for baler files, but you never know.
-* Fixed it so you have to at least supply the TagID when using the
- -m command.
-
-2017.152 (2017-06-01)
-------------------
-* Does a simple check to see if the TagID and IPAddr are reversed.
-* Does a simple check when offloading to see if previously offloaded
- data files will be overwritten. It may report false alarms depending
- on the command.
-* Fixed a small bug in the -F command.
-
-2017.163 (2017-06-12)
-------------------
-* Lists the command line being executed so it gets into the log.
-
-2018.071 (2018-03-12)
-------------------
-* Small bug in the 'check for updates' section.
-
-2018.135 (2018-06-07)
-------------------
-* First release on new build system.
-
2019.064 (2019-03-05)
-------------------
+---------------------
* Converted for Python 2 or 3.
* Added -vl command that lists the files that have not been fully
offloaded from the baler. -v just shows how many there are.
* Does not yet contain BaleAddr replacement code.
+
+2019.259 (2019-09-16)
+---------------------
+* New version (really BLINE2) that assigns the IP address to the baler,
+ and erases data and Q330 association from the baler, thus removing
+ the need for BaleAddr.
+* The TagID of the baler to communicate with is still required on the
+ command line, but the IP address of the baler is no longer required for
+ every command on the command line.
+* The Ethernet device name may be required when assigning the IP address
+ to the baler (-b command) if BLINE says so (generally Python 2).
+* These non-regular Python modules may need to be installed to get all
+ of the new commands (-b, -X, -s) to work:
+ psutil - all OSs and Python versions
+ pexpect - Linux
+ subprocess32 - all OSs when using Python 2
+ Python 3 is recommended, since support for 2 is going away soon.
+ pip/pip3 may be used to install the required modules.
+
+2019.269 (2019-09-26)
+---------------------
+* Caught a different program having trouble checking for updates. Put
+ in some code as a possible fix.
+* Collected the imports for all of the extra required modules into one
+ place so all of the warnings come out at the same time. Use
+ bline.py 5555 -b
+ to get a list of the warnings. 5555 may be any number for this test.
+* Fixed up the long version of the Help a bit.
+
+2019.289 (2019-10-16)
+---------------------
+* Added the -A command that will combine all of the offloaded files
+ into a .ALL file.
+* Added the -G command that will combine all of the files for a
+ channel into one file. The file names are close to the miniseed
+ file names produced by sdrsplit.
+* Changed the command line switch for a couple commands.
+
+2019.297 (2019-10-24)
+---------------------
+* Enhanced the -G command and added the -GD command for concatenating
+ the offloaded channel files.
diff --git a/bline/__init__.py b/bline/__init__.py
index 711d13e65134b22cc03940982c6729341d454804..2bc3b91597f9489a758e79f5a86a2a8d9b307af0 100644
--- a/bline/__init__.py
+++ b/bline/__init__.py
@@ -4,4 +4,4 @@
__author__ = """IRIS PASSCAL"""
__email__ = 'software-support@passcal.nmt.edu'
-__version__ = '2018.135'
+__version__ = '2019.289'
diff --git a/bline/bline.py b/bline/bline.py
index ec67d11b4c0e245415cb721fd8b59b2e3b9c1c51..0a132e5056844ec7b214e0f14141938e18170691 100755
--- a/bline/bline.py
+++ b/bline/bline.py
@@ -1,4 +1,5 @@
#! /usr/bin/env python
+# -*- coding: latin-1 -*-
# BEGIN PROGRAM: BLINE
# By: Bob Greschke
# Started: 2016.105
@@ -9,12 +10,11 @@ from sys import argv, exit, platform, stdout
PROGSystem = platform[:3].lower()
PROG_NAME = "BLINE"
PROG_NAMELC = "bline"
-PROG_VERSION = "2019.064"
-PROG_LONGNAME = "Command Line Baler Offload Program"
+PROG_VERSION = "2019.297"
+PROG_LONGNAME = "Command Line Baler Control Program"
PROG_SETUPSVERS = "A"
-# These are for 'Check For Updates' stuff that will eventually be replaced
-# by just using git.
+# These are for the 'Check For Updates' stuff.
VERS_VERSURL = "https://www.passcal.nmt.edu/~bob/passoft/"
VERS_PARTS = 4
VERS_NAME = 0
@@ -24,7 +24,7 @@ VERS_ZSIZ = 3
##########################
# BEGIN: versionChecksCL()
-# LIB:versionChecksCL():2019.023
+# LIB:versionChecksCL():2019.213
# Checks the current version of Python and sets up a couple of things for the
# rest of the program to use.
# Obviously this is not a real function. It's just a collection of things for
@@ -45,12 +45,14 @@ if PROG_PYVERSION.startswith("2"):
astring = basestring
anint = (int, long)
arange = xrange
+ aninput = raw_input
elif PROG_PYVERSION.startswith("3"):
PROG_PYVERS = 3
from urllib.request import urlopen, urlretrieve
astring = str
anint = int
arange = range
+ aninput = input
else:
stdout.write("Unsupported Python version: %s\nStopping.\n"%PROG_PYVERSION)
exit(0)
@@ -61,586 +63,898 @@ maxInt = 1E100
maxFloat = 1.0E100
# END: versionChecksCL
-from struct import unpack
+from copy import deepcopy
from fnmatch import fnmatch
-from inspect import stack
-if PROGSystem == "dar" or PROGSystem == "lin" or PROGSystem == "sun":
- from os import getuid
from os import makedirs, listdir, sep
from os.path import abspath, basename, exists, getsize, isdir
-from socket import setdefaulttimeout, gethostbyname, gethostname
-#setdefaulttimeout(20) for now. Seems to work better.
from time import gmtime, localtime, sleep, strftime, time
-# These are for some of the functions that are shared with bgoff.py. They are
-# not used in bline.py.
-STATE_IDLE = 0
-STATE_CHECK = 1
-STATE_OFFLD = 2
-STATE_STOP = 99
-BState = 0
-
-PROGKiosk = False
-PROGSmallScreen = False
-ArgCheck = False
-ArgList = False
-ArgVerify = False
-ArgVerifyList = False
-ArgBadBlocks = False
-ArgLSROffload = False
-ArgSkipo = False
-ArgSkipO = False
-ArgSpec = False
-ArgSpecFiles = []
-ArgUCheck = False
-ArgUDown = False
-WriteMsg = False
-# These lovely bugs are brought to you by Microsoft. Python 3 was keeping
-# ' -#' for arguments, "" as an argument, or not passing command line
-# arguments at all. It's related to the registry value
-# HKEY_CLASSES_ROOT\Applications\python.exe\shell\open\command
-# not having a %* (no quotes around it) at the end of the Data command to
-# get Python to pass command line arguments when using 'bline.py <args>' to
-# start the program. 'python bline.py <args>' seems OK. It's a Python
-# installation problem. Don't know when all of this started. Dec 2018 was
-# fine. Feb 2019 is not. It might be an older bug that has shown back up.
-argv2 = []
-Index = 0
-for Arg in argv:
- argv[Index] = argv[Index].strip()
- if len(argv[Index]) != 0:
- argv2.append(argv[Index])
- Index += 1
-argv = argv2
-# Just always try to get these two. If they end up being needed, fine. If not,
-# that's fine too.
-try:
-# .upper in case letters are ever in there.
- TagID = argv[1].upper()
-# They are on the label, but I don't think any software uses them.
- while TagID.startswith("0"):
- TagID = TagID[1:]
-except:
- TagID = ""
-try:
- IPAddr = argv[2]
-except:
- IPAddr = ""
-# Gotta have at least one. Fool the code below.
-if len(argv) == 1:
- argv.append("-h")
-# The arguments that can be anywhere and do something and quit.
-if "-#" in argv:
- stdout.write("%s\n"%PROG_VERSION)
- exit(0)
-# LATER.
-# We don't even have to go to main() with this one.
-#if "-A" in argv:
-# stdout.write("Looking for a baler. Press the ATTN button...\n")
-# stdout.write("(Ctrl-C to stop before a baler is found.)\n")
-# Ret = address_balers()
-# stdout.write(Ret)
-# exit(0)
-if "-h" in argv:
- stdout.write( \
- "Usage: bline.py <TagID> <IP address> <Command>\n")
- stdout.write( \
- " -c = Checks communication with the baler and gets basic information.\n")
- stdout.write( \
- " -L = Saves and displays the list of files on the baler.\n")
- stdout.write("\n")
- stdout.write( \
- "bline.py <TagID> <IP address> <Command> [<file(s)>]\n")
- stdout.write( \
- " -O (Big O) = Offloads all data files that have not been offloaded.\n")
- stdout.write( \
- " -o (Little O) = Offloads low sample rate data files that have not been\n")
- stdout.write(" offloaded.\n")
- stdout.write( \
- " -v = Gets the baler's list of files and checks the offloaded files.\n")
- stdout.write( \
- " -vl = Gets the baler's list of files and checks the offloaded files.\n")
- stdout.write( \
- " This also lists the files that have not been offloaded.\n")
- stdout.write( \
- " -V = Examines the specified offloaded files for bad blocks. The\n")
- stdout.write( \
- " <IP address> may be omitted.\n")
- stdout.write("\n")
- stdout.write( \
- "bline.py <TagID> <IP address> <Command> <file(s)>\n")
- stdout.write( \
- " -e = Excludes the specified file(s) during an offload, otherwise\n")
- stdout.write( \
- " low sample rate files will be offloaded (like -o).\n")
- stdout.write( \
- " -E = Excludes the specified file(s) during an offload, otherwise\n")
- stdout.write( \
- " all files will be offloaded (like -O).\n")
- stdout.write( \
- " -F = Offloads only the specified file(s).\n")
- stdout.write("\n")
- stdout.write( \
- "bline.py <TagID> [<IP address>] <Command>\n")
- stdout.write( \
- " -m = Follow the -m with a message.\n")
- stdout.write("\n")
- stdout.write( \
- "bline.py [<TagID> <IP address>] <Command>\n")
-# LATER. Here and in the -H section.
-# stdout.write( \
-# " -A = Watch for a baler to assign an IP address to.\n")
- stdout.write( \
- " -h = This help.\n")
- stdout.write( \
- " -H = More help.\n")
- stdout.write( \
- " -i = Reports what may be the control computer's IP address and.\n")
- stdout.write(" other information.\n")
- stdout.write( \
- " -U = Checks for a newer program version at PASSCAL if connected\n")
- stdout.write( \
- " to the Internet (the program will eventually be distributed\n")
- stdout.write( \
- " using git and this will go away.\n")
- stdout.write( \
- " -UD = Downloads most recent version from PASSCAL (try -U first).\n")
- stdout.write("\n")
- stdout.write( \
- "<TagID> = The tag ID on the front of the baler.\n")
- stdout.write( \
- "<IP address> = The IP address assigned to the baler by BaleAddr.\n")
- stdout.write("\n")
- stdout.write( \
- "-E/-e commands are, for example, for excluding \"problem\" files that\n")
- stdout.write( \
- " may be stopping the offload process.\n")
- stdout.write( \
- "-F files that are not found on the baler will be created with HTML\n")
- stdout.write( \
- " code in them saying the file doesn't exist (~100 bytes size).\n")
- stdout.write( \
- "*, ?, [] UNIX file wildcards may be used in <file(s)>.\n")
- stdout.write( \
- "On most systems you will need to enclose file names using wildcards\n")
- stdout.write( \
- " with quotes like \"*.VER\" \"DT0001*\"\n")
- stdout.write( \
- "Only one command per run.\n")
- stdout.write( \
- "Always leave a space after the command line switches.\n")
- stdout.write("\n\a")
- exit(0)
-if "-H" in argv:
- HELPText = "USING BLINE.PY\n\
---------------\n\
-BLINE offloads data files from a Quantera B14 Baler. It does not\n\
-set up the baler for offloading. BaleAddr is the Quanterra\n\
-Windows/Wine program that you start after you've hooked up the baler\n\
-to the laptop and to power. In it you set the IP address that you want\n\
-the baler to be set to when the baler starts up. If, for example, the\n\
-laptop IP address is something like 129.138.26.1, you would want the\n\
-baler's address to end up something like 129.138.26.2. Same sub-net,\n\
-etc. You set the desired IP address in BaleAddr then poke the ATTN button\n\
-on the baler. When the baler gets going BaleAddr handshakes and sets the IP\n\
-address.\n\
-\n\
-You can use\n\
-\n\
- bline.py -i\n\
-\n\
-to see what the IP address of the computer is set to. It might be\n\
-correct. It won't be correct if the Wireless is on, so turn that off.\n\
-Use the operating system's network setting widget to double check what\n\
-the address is if the address from the -i command does not seem to be\n\
-working.\n\
-\n\
-Depending on the operating system of the computer you are going to run\n\
-BaleAddr and BLINE on, you may have to manually set an IP address\n\
-for the computer. Some operating systems do not set their IP address\n\
-until they are connected to a network, and in some cases there is no\n\
-network until the baler wakes up from an ATTN button push. By then it\n\
-is too late.\n\
-\n\
-When everything works you should get the messages\n\
-\n\
-Setting time in baler\n\
-Baler OK\n\
-\n\
-in BaleAddr. If the 'Baler OK' message doesn't show up then the\n\
-program is not really talking to the baler/the baler's IP address did\n\
-not get set. It happens a lot. It seems to be a bug in the baler\n\
-firmware. Quiting/Resetting BaleAddr, re-poking the ATTN button on the\n\
-baler to let the baler shut down, restarting BaleAddr (if it was Quit),\n\
-and poking the ATTN button again up to five times has been required to\n\
-get the program and the baler to connect. It's just flakey. If you get\n\
-up to maybe three cycles you can try shutting down the baler and then\n\
-disconnecting the power and reconnecting, and then try again to get\n\
-things going. Sometimes that helps. Make sure you have a suitable IP\n\
-address in BaleAddr that is compatible with the address the computer\n\
-thinks it has. That REALLY helps. :) Also make sure the Wireless is\n\
-turned off like you have to do with EzBaler. None of the programs can\n\
-figure out which Ethernet adapter to use when there is more than one\n\
-active.\n\
-\n\
-BLINE is just a command line program, so once BaleAddr has connected to\n\
-the baler you start a Terminal/xterm/whatever window, cd into the\n\
-directory where you want a folder of data from the baler to be saved to,\n\
-like a \"DATA\" directory, and enter\n\
-\n\
- bline.py\n\
-or\n\
- <path to bline>/bline.py\n\
-or\n\
- ./bline.py\n\
-\n\
-The command to start the program depends on how BLINE was installed. Using\n\
-no command line arguments will list all of the commands for the program.\n\
-For basic offloading of everything on the baler you just enter\n\
-\n\
- bline.py <TagID> <IP address> -O\n\
-\n\
-<TagID> is the serial number on the baler, like 6003.\n\
-<IP address> is the address assigned to the baler using BaleAddr.\n\
-\n\
-Before offloading starts BLINE will check to see if any files are going\n\
-to be overwritten. If so a confirmation message will appear.\n\
-\n\
-Offloading should start. A sub-directory like 6003.sdr will be created\n\
-and the data files from the baler will be in there. The file 6003.msg\n\
-will be created in the directory in which BLINE was started. It will\n\
-have a copy of the messages that get written to the terminal window as\n\
-the program is working. There will be a 6003files.txt. This will be a\n\
-list of all of the data files that were found on the baler.\n\
-\n\
-When BLINE is finished with the baler and all of the data files have been\n\
-offloaded you can bring the baler up in BaleAddr again (assuming it's\n\
-like days later and the baler has shut down) and use the Clean Baler\n\
-button to erase everything and get the baler ready to go back out to\n\
-the field. Please insure you have ALL of the data files, and that they\n\
-are backed up, because unlike EzBaler, BaleAddr completly erases the\n\
-old data.\n\
-\n\
-INSTALLING\n\
-\n\
-To install BLINE copy the file bline.py to where ever you want. If you\n\
-put it somewhere like /opt/passcal/bin/ (the preferred choice if the\n\
-PASSCAL software package has been installed), with the other PASSCAL\n\
-software, then it should be in the path set up by the PASSCAL software\n\
-installation, and just \"bline.py\" will be used to get it going.\n\
-This command will probably have to be used\n\
-\n\
- sudo cp bline.py /opt/passcal/bin\n\
-\n\
-and the login password given to get it there. It will also probably\n\
-have to made executable with something like\n\
-\n\
- chmod +x bline.py\n\
-\n\
-Depending on where it is copied sudo may also need to be used before\n\
-chmod. If it is not executable a 'permission denied' message will be\n\
-displayed when trying to start the program.\n\
-\n\
-Installing it to any other location will require ensuring that the location\n\
-is in the current path and the necessary entries have been made in files\n\
-such as .bashrc, or .bash_profile, or any other initialization files for\n\
-the command shell in use.\n\
-\n\
-COMMAND DETAILS\n\
-\n\
-BLINE is designed to be run in or started from the directory where you want\n\
-to work. You cannot use paths to directories. The current working directory\n\
-or the \"work directory\" must be set by changing to the directory of\n\
-choice using the command line with something like the 'cd' command before\n\
-starting the program. Files created by the program will be written the the\n\
-work directory and offloaded files will be written to a sub-directory of\n\
-the work directory (<TagID>.sdr). Verification commands, -v/-vl and -V,\n\
-expect the program to be started in the same directory where it was\n\
-running when the files were offloaded from the baler, if they have not\n\
-been moved since offloading them.\n\
-\n\
-All of the commands are placed on the command line:\n\
-\n\
- bline.py <TagID> <IP address> <command> <file(s)>\n\
-\n\
-See the -h help for the command line items that are required or optional\n\
-for each command.\n\
-\n\
-The <TagID> is used as part of the bookkeeping file names and the directory\n\
-name where the offloaded data files will be placed. It is also used by\n\
-BLINE to make sure it is talking to the baler the user thinks it is\n\
-talking to. The <IP address> is used for the Ethernet communications\n\
-protocol to allow the program to actually communicate with the baler. It\n\
-is the address that was set using BaleAddr.\n\
-\n\
-The -V command does not communicate with the baler, so the IP address of\n\
-the baler is not used, and does not need to be included.\n\
-\n\
-Some commands allow a space-separated list of file names with or without\n\
-standard UNIX wildcard characters to follow them to control which data\n\
-files are delt with. This may, for example, be a list of files to offload,\n\
-or it may be a list of files to examine. It depends on the command. On most\n\
-systems you will need to enclose the file names in double quotes when using\n\
-wildcard characters, like \"*\", instead of just *. In most cases not\n\
-supplying any file list is the same as using \"*\", i.e. 'all files'.\n\
-\n\
--c\n\
-This command uses the <TagID> and <IP address> to simply establish a\n\
-connection with the spcified baler at the specified IP address. Error\n\
-messages will be displayed if anything goes wrong.\n\
-\n\
--E\n\
-The -E command is basically the same as -O, except that it allows specified\n\
-files (in the <file(s)> list) to be excluded from offloading. If a baler\n\
-offload session always fails on a specific file then the -E command and\n\
-that file's name could be specified, so that it will be skipped and the\n\
-rest of the files will be offloaded.\n\
-\n\
--e\n\
--e is the same as -E, but it will only try to offload low sample rate files\n\
-that are not in the list of files to exclude.\n\
-\n\
--F\n\
-Only the file(s) specified in the [<file(s)>] argument will be offloaded.\n\
-This could be used for a different form of only offloading the low sample\n\
-rate data files. A command like\n\
-\n\
- bline.py 6003 123.123.123.4 -F \"DT0001*\"\n\
-\n\
-would offload the first file of each channel, which would also include a\n\
-little bit of the high sample rate data. For short amounts of baler data\n\
-this would be OK, but channels of low sample rate, or SOH, data would not\n\
-be offloaded if there were more than one DT-file for a channel. The -o\n\
-could be used and then this -F command example.\n\
-\n\
--h and -H\n\
-The -h command prints a summary list of possible command line arguments\n\
-to the display. The -H command prints a more detailed help document to the\n\
-display.\n\
-\n\
--i\n\
-This command asks the computer running BLINE to print its IP address.\n\
-This address may be used to help select an IP address to set the baler to\n\
-in the BaleAddr program. The address printed may or may not be correct or\n\
-useful. Some operating systems will not print the correct address until\n\
-the computer is already talking to a baler. Some operating system will\n\
-report the wrong address if both the wired and wireless Ethernet systems\n\
-are in use. The list goes on.\n\
-\n\
-The most reliable way to determine the IP address of the computer for\n\
-running BLINE is to look in the system preferences for the operating\n\
-system in use.\n\
-\n\
--L\n\
-This command simply contacts the baler, requests the list of data files\n\
-the baler thinks are available, and saves that list to the file\n\
-\n\
- <TagID>files.txt\n\
-\n\
--m\n\
-Follow the -m with a message that will be displayed, but also written to\n\
-the .msg file for the baler. The text may need to be enclosed in quotes:\n\
- -m \"Baler 5549 is station NUUK\".\n\
-\n\
--O (Big O)\n\
-This command retrieves the list of files on the baler from the baler, and\n\
-then requests all of them for offloading from the baler. The files are\n\
-placed in the sub-directory <TagID>.sdr. Additional [<file(s)>] may be\n\
-listed to control offloading a subset of the available files on the baler.\n\
-\n\
-Files in the <TagID>.sdr directory that appear to have already been\n\
-offloaded (file name and size match a file on the baler) will not be\n\
-offloaded again.\n\
-\n\
--o (small o)\n\
-This command retrieves the list of files on the baler from the baler, and\n\
-then requests all of the the files whose channel name, like .HHZ, does\n\
-not start with the letter H or S (as specified in the SEED manual as being\n\
-'high speed' channel names). The files are placed in the sub-directory\n\
-<TagID>.sdr. Additional [<file(s)>] may be listed to control offloading a\n\
-subset of the available files on the baler.\n\
-\n\
--U, -UD\n\
-If the computer is connected to the Internet running\n\
-\n\
- bline.py -U\n\
-\n\
-will check to see if there is a newer version of the program than the one\n\
-running on the computer. If there is a newer version use the -UD command to\n\
-obtain it. Unzip the file that is downloaded and install the new version\n\
-from there (see other instructions).\n\
-\n\
-Eventually updates to the program will be obtained by downloading a new\n\
-version using 'git' and -U and -UD will go away.\n\
-\n\
--v (little V) and -vl\n\
-The -v/-vl commands get the list of files from the baler (which would be\n\
-files.htm if you were talking to the baler with a browser) and then looks\n\
-to see if all of the files on that list are in the <TagID>.sdr directory\n\
-on the computer and if they are the same size as they are in the list. The\n\
-function does not do any checksum checking, because there's no way to get\n\
-the baler to compute a checksum value of the files on the baler to verify\n\
-against. A <file(s)> list may be supplied if only specified files are to\n\
-be verified. The -vl command additionally lists the files that have not\n\
-been offloaded from the baler.\n\
-\n\
--V (big V)\n\
-The -V command reads block-by-block through the offloaded data file(s)\n\
-in the <TagID>.sdr directory for the specified baler and collects and\n\
-reports:\n\
-\n\
- 1. The earliest block header time in a file\n\
- 2. The latest block header time in a file\n\
- 3. The number of blocks read, and the block size in a file\n\
- 4. A list of all of the station IDs in a file (should normally only\n\
- be one)\n\
- 5. A list of all of the channel names in a file (should normally only\n\
- be one)\n\
-\n\
-For baler data files the normal block size is 4096 bytes and that is\n\
-normally 4100 blocks per 16MB data file.\n\
-\n\
-Things to look for after the function runs are bad/scrambled/missing\n\
-dates and times, multiple station names, multiple channel names and the\n\
-program crashing. The latest block time will not normally be the same\n\
-as the last sample time for each channel. For quiet and low sample rate\n\
-channels the time may be quite a bit earlier than the last sample time.\n\
-\n\
-A summary of information is printed after all of the files have been\n\
-examined.\n"
- stdout.write("\n")
- stdout.write(HELPText)
- stdout.write("\n")
- exit(0)
-if "-i" in argv:
- import socket
- stdout.write("Contol computer IP address (maybe): %s\n"% \
- socket.gethostbyname(socket.gethostname()))
- stdout.write("Python version: %s\n"%PROG_PYVERSION)
- stdout.write("\n")
- exit(0)
-########################
-# BEGIN: args2Str(Start)
-# FUNC:args2Str():2017.163
-# Just returns a string version of the command line.
-def args2Str(Start):
- Return = ""
- for arg in argv[Start:]:
- Return += " "+arg
+
+##########################################
+# BEGIN: args2SL(SorL, Start, Delim = " ")
+# FUNC:args2SL():2017.233
+# Constructs a list or string of the command line arguments starting at the
+# argv index position if Start is an int, or after the command line argument
+# that matches Start if Start is a string.
+def args2SL(SorL, Start, Delim = " "):
+ if SorL == "s":
+ Return = ""
+ elif SorL == "l":
+ Return = []
+ if isinstance(Start, astring):
+ FoundArg = False
+ for Arg in argv:
+ if Arg == Start:
+ FoundArg = True
+ continue
+ if FoundArg == True:
+ if SorL == "s":
+ Return += "%s%s"%(Arg, Delim)
+ if SorL == "l":
+ Return.append(Arg)
+ elif isinstance(Start, anint):
+# The caller may ask for args that are not there.
+ try:
+ for Arg in argv[Start:]:
+ if SorL == "s":
+ Return += "%s%s"%(Arg, Delim)
+ if SorL == "l":
+ Return.append(Arg)
+ except:
+ pass
+ if SorL == "s" and Delim != "":
+ if Return.endswith(Delim):
+ Return = Return[:-len(Delim)]
return Return
-# END: args2Str
+# END: args2SL
+
+
+######################
+# BEGIN GROUP: bader()
+# FUNCS:bader():2019.268
+# A collection of functions from the bader.py program.
+# Gives Quanterra Baler-14 units an IP address based on the computer's IP.
+# Lloyd Carothers, IRIS/PASSCAL
+# Based on bader.py, 2019.193.
+from calendar import timegm
+from glob import glob
+import socket
+import struct
+#####################
+# BEGIN: logging(Msg)
+# FUNC:logging():2019.235
+# Instead of the logging module.
+def logging(Msg):
+ if 1 == 2:
+ print(Msg)
+ return
+# END: logging
##########################
-# BEGIN: getArgvFiles(Cmd)
-# FUNC:getArgvFiles():2019.019
-# Reads any arguments after the passed command. If they are files then fine
-# and if they are it's not my fault.
-from copy import deepcopy
+# BEGIN CLASS: CRC(object)
+# CLASS:CRC(object):2019.235
+class CRC(object):
+ TABLE = None
+ ############
+ @classmethod
+ def create_table(cls):
+ cls.TABLE = [None]*256
+ tdata = 0
+ accum = 0
+ F = 0xFFFFFFFF
+ for count in range(0, 256):
+ tdata = (count << 24)
+ accum = 0
+ for bits in range(1, 9):
+ if (tdata ^ accum) & 2**31:
+ accum = (accum << 1) ^ 1443300200
+ accum = accum & F
+ else:
+ accum = (accum << 1) & F
+ tdata = (tdata << 1) & F
+ cls.TABLE[count] = accum
+ return
+ ############
+ @classmethod
+ def compute_crc(cls, buf):
+ if not cls.TABLE:
+ cls.create_table()
+ lng = 0
+ firstByte = 0
+ F = 0xFFFFFFFF
+ dataPtr = 0
+ dataLen = len(buf)
+ while dataLen > 0:
+ if PROG_PYVERS == 2:
+ thisByte = ord(buf[dataPtr])
+ elif PROG_PYVERS == 3:
+ thisByte = buf[dataPtr]
+ newCrc = F & (lng << 8) ^ cls.TABLE[(firstByte ^ thisByte) & 255]
+ firstByte = newCrc >> 24
+ lng = newCrc
+ dataPtr += 1
+ dataLen -= 1
+ return newCrc
+# END CLASS: CRC
+
+#############################
+# BEGIN CLASS: Packet(object)
+# CLASS:Packet():2019.235
+class Packet(object):
+ header_struct = struct.Struct('!LBBHHH')
+ assert header_struct.size == 12
+ brdy_cmd_code = 0x5A
+ brdy_struct = struct.Struct('!LL2s5sBHHLQ')
+ assert brdy_struct.size == 32
+ vack_struct = struct.Struct('!5L6HQ')
+ assert vack_struct.size == 40
+ baler_response_code = 0x5C
+ baler_response_struct = struct.Struct('!HHL')
+ assert baler_response_struct.size == 8
+ seq_counter = 666
+ ####################
+ def crc_check(self):
+ if self.crc == CRC.compute_crc(self.buf[4:]):
+ logging('crc: pass')
+ return True
+ else:
+ logging('crc: fail')
+ return False
+ ######################
+ def unpack(self, buf):
+ self.buf = buf
+ self.unpack_header(buf[:self.header_struct.size])
+ self.crc_check()
+ if self.command == self.brdy_cmd_code:
+ self.unpack_brdy(buf[self.header_struct.size:])
+ return
+ #############################
+ def unpack_header(self, buf):
+ (self.crc,
+ self.command,
+ self.proto_version,
+ self.length,
+ self.seq_number,
+ self.ack_number) = self.header_struct.unpack(buf)
+ return
+ ###########################
+ def unpack_brdy(self, buf):
+ (self.q330_sn1,
+ self.q330_sn2,
+ self.net,
+ self.station_name,
+ self.flags,
+ self.model_num,
+ self.baler_version,
+ self.disk_size,
+ self.baler_sn) = self.brdy_struct.unpack(buf)
+ return
+ ###########################################
+ def pack_vack(self, ip, netmask, brdy_pkt):
+ assert isinstance(brdy_pkt, self.__class__)
+ self.q330_sn1 = 0
+ self.q330_sn2 = brdy_pkt.q330_sn2
+ self.q330_ip = 0
+ self.baler_ip = ip
+ self.baler_netmask = str(netmask)
+ self.q330_base_port = 0
+ self.q330_data_port = 0
+ self.bps = 0
+ self.flags = 0
+ self.access_timeout = 0
+ self.serial_baud_rate = 0
+ self.baler_sn = brdy_pkt.baler_sn
+ self.payload = self.vack_struct.pack(
+ self.q330_sn1,
+ self.q330_sn2,
+ self.q330_ip,
+ struct.unpack( '!I', socket.inet_aton( self.baler_ip))[0],
+ struct.unpack( '!I', socket.inet_aton( self.baler_netmask))[0],
+ self.q330_base_port,
+ self.q330_data_port,
+ self.bps,
+ self.flags,
+ self.access_timeout,
+ self.serial_baud_rate,
+ self.baler_sn)
+ # Header. C7=baler command.
+ self.command = 0xC7
+ self.proto_version = brdy_pkt.proto_version
+ self.ack_number = brdy_pkt.seq_number
+ self.pack_header()
+ return
+ ####################
+ def pack_ping(self):
+ '''
+ An empty command packet useful to ping the baler.
+ '''
+ self.baler_command = 0
+ self.sequencing_field = 0
+ self.flags = 0
+ clean_struct = struct.Struct('!2HL')
+ self.payload = clean_struct.pack(self.baler_command, \
+ self.sequencing_field, self.flags)
+ # Header. C8=baler command.
+ self.command = 0xC8
+ self.proto_version = 2
+ self.ack_number = 0
+ self.pack_header()
+ return
+ ########################################
+ def pack_set_baler_time(self, brdy_pkt):
+ base_time = timegm( (2000, 1, 1, 0, 0, 0, 0, 0, 0) )
+ now = time() - base_time
+ self.baler_command = 5
+ self.sequencing_field = 0 # used for multiple packet cmds
+ self.time = int(now)
+ time_struct = struct.Struct('!2HL')
+ self.payload = time_struct.pack(self.baler_command, \
+ self.sequencing_field, self.time)
+ # Header. C8=baler command.
+ self.command = 0xC8
+ self.proto_version = brdy_pkt.proto_version
+ self.ack_number = 0
+ self.pack_header()
+ return
+ ########################
+ def pack_shutdown(self):
+ '''
+ Turn baler off qlib calls dealocate
+ '''
+ self.baler_command = 4
+ self.sequencing_field = 0
+ self.flags = 0
+ clean_struct = struct.Struct('!H')
+ self.payload = clean_struct.pack(self.baler_command)
+ # Header. C8=baler command.
+ self.command = 0xC8
+ self.proto_version = 2
+ self.ack_number = 0
+ self.pack_header()
+ return
+ #####################
+ def pack_clean(self):
+ '''
+ Clean the baler and remove q330 association
+ '''
+ self.baler_command = 2
+ self.sequencing_field = 0
+ self.flags = 0xFF
+ clean_struct = struct.Struct('!3H')
+ self.payload = clean_struct.pack(self.baler_command, \
+ self.sequencing_field, self.flags)
+ # Header. C8=baler command.
+ self.command = 0xC8
+ self.proto_version = 2
+ self.ack_number = 0
+ self.pack_header()
+ return
+ ######################
+ def pack_header(self):
+ self.crc = 0
+ self.length = len(self.payload)
+ self.seq_counter += 1
+ header_bytes = self.header_struct.pack(
+ self.crc,
+ self.command,
+ self.proto_version,
+ self.length,
+ self.seq_counter,
+ self.ack_number)
+ qdp_packet = header_bytes + self.payload
+ # Compute and insert CRC.
+ to_crc = qdp_packet[4:]
+ self.crc = CRC.compute_crc(to_crc)
+ self.buf = struct.pack('!L', self.crc) + to_crc
+ return
+ ##################
+ def __str__(self):
+ ret = 'QDP packet:'
+ for key, value in self.__dict__.items():
+ if key in ('buf', 'payload'):
+ continue
+ ret += "\n%s:\t%s\n"%(key[:20], value)
+ return ret
+# END CLASS: Packet
+
+if not hasattr(socket, 'IP_PKTINFO'):
+ if PROGSystem == "dar":
+ socket.IP_PKTINFO = 26
+ if PROGSystem == "lin":
+ socket.IP_PKTINFO = 8
+ if PROGSystem == "win":
+ socket.IP_PKTINFO = 8
+
+#########################
+# BEGIN: socketComm(Host)
+# FUNC:socketComm():2019.255
+def socketComm(Host):
+ Sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ Sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+ Sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ Sock.settimeout(2)
+ Sock.bind((str(Host), 0))
+ return Sock
+# END: socketComm
-def getArgvFiles(Cmd):
- FileList = []
- FoundCmd = False
- for arg in argv:
- if arg == Cmd:
- FoundCmd = True
+##########################
+# BEGIN: ipFromDevice(Dev)
+# FUNC:ipFromDevice():2019.267
+def ipFromDevice(Dev):
+ try:
+ addresses = [ad for ad in net_if_addrs()[Dev]
+ if ad.family is socket.AF_INET]
+ except KeyError:
+ return (1, "RW", "Is device '%s' on this system?"%Dev, 2)
+ if len(addresses) == 0:
+ logging("Dev:%s no address."%Dev)
+ return (1, "", "'%s' has no IP address."%Dev)
+ elif len(addresses) > 1:
+ logging("Dev:%s doesn't have a single IP:%s"%(Dev, addresses))
+ return (1, "", "'%s' has %d addresses."%(Dev, len(addresses)))
+ a = addresses[0]
+ if PROG_PYVERS == 2:
+ aaddress = unicode(a.address)
+ anetmask = unicode(a.netmask)
+ net = IPv4Network((aaddress, anetmask), strict=False)
+ ip_if = ip_interface((aaddress, net.prefixlen))
+ elif PROG_PYVERS == 3:
+ net = IPv4Network((a.address, a.netmask), strict=False)
+ ip_if = ip_interface((a.address, net.prefixlen))
+ return (0, ip_if)
+# END: ipFromDevice
+
+#############################################################
+# BEGIN: addressABaler(SETspec, MSGspec, TagID, IEthDev = "")
+# FUNC:addressABaler():2019.268
+# Providing the Ethernet device overrides watching them all and cuts out
+# using recvmsg().
+def addressABaler(SETspec, MSGspec, TagID, IEthDev = ""):
+# This is about all we can check.
+ try:
+ TagID = int(TagID)
+ except:
+ return (1, "RW", "Bad TagID: '%s'"%TagID)
+# A socket for listening for baler ready packets.
+# Setting Host to "" tries on any address and any interface.
+ Host = ""
+ Port = 65535
+ Sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ Sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+ Sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+# Only needed if searching.
+ if IEthDev == "":
+ try:
+ Sock.setsockopt(socket.IPPROTO_IP, socket.IP_PKTINFO, 1)
+# It's different exceptions in different places. Just catch them all.
+ except:
+ return (1, "MW", \
+ "The Ethernet device name must be supplied on this system.", \
+ 3)
+ Sock.bind((Host, Port))
+ Tries = 0
+ while True:
+ if IEthDev == "":
+# The user may not know about this.
+ try:
+ Raw, Msgs, Flags, Addr = Sock.recvmsg(44, 128)
+ except AttributeError:
+ return (1, "MW", \
+ "The Ethernet device name must be supplied on this system.", \
+ 3)
+ logging("%s %s %s %s"%(Raw, Msgs, Flags, Addr))
+ BalerPort = Addr[1]
+# Figure out which Ethernet device the baler is on.
+ EthDev = None
+ for msg in Msgs:
+ if msg[1] == socket.IP_PKTINFO:
+ if_index = struct.unpack('I8B', msg[2])[0]
+ EthDev = socket.if_indextoname(if_index)
+ break
+ if EthDev is None:
+ return (1, "MW", "Could not determine Ethernet device.", 3)
+ elif IEthDev != "":
+ EthDev = IEthDev
+ Raw, Addr = Sock.recvfrom(128)
+ BalerPort = Addr[1]
+ logging("Received baler ready packet from %s on dev:%s"%(Addr, \
+ EthDev))
+ brdy_pkt = Packet()
+ brdy_pkt.unpack(Raw)
+ logging(brdy_pkt)
+ if TagID != brdy_pkt.q330_sn2:
+ logIt(MSGspec, "Found baler %s. Ignoring."%brdy_pkt.q330_sn2)
+ Tries += 1
+ if Tries == 3:
+ return (1, "RW", "Did not see baler %d."%TagID, 2)
continue
- if FoundCmd == True:
- FileList.append(arg.upper())
- if len(FileList) == 0:
- FileList.append("*")
- return FileList
-# END: getArgvFiles
-
-# The more complicated commands.
-CmdReady = False
-if CmdReady == False and "-V" in argv:
- ArgBadBlocks = True
- Ret = getArgvFiles("-V")
- ArgSpecFiles = deepcopy(Ret)
- CmdReady = True
-if CmdReady == False and "-U" in argv:
- ArgUCheck = True
- CmdReady = True
-if CmdReady == False and "-UD" in argv:
- ArgUDown = True
- CmdReady = True
-if CmdReady == False and "-c" in argv:
- ArgCheck = True
- CmdReady = True
-if CmdReady == False and "-L" in argv:
- ArgList = True
- CmdReady = True
-if CmdReady == False and "-v" in argv:
- ArgVerify = True
- Ret = getArgvFiles("-v")
- ArgSpecFiles = deepcopy(Ret)
- CmdReady = True
-if CmdReady == False and "-vl" in argv:
- ArgVerify = True
- ArgVerifyList = True
- Ret = getArgvFiles("-v")
- ArgSpecFiles = deepcopy(Ret)
- CmdReady = True
-if CmdReady == False and "-O" in argv:
- Ret = getArgvFiles("-O")
- ArgSpecFiles = deepcopy(Ret)
- CmdReady = True
-if CmdReady == False and "-o" in argv:
- ArgLSROffload = True
- Ret = getArgvFiles("-o")
- ArgSpecFiles = deepcopy(Ret)
- CmdReady = True
-if CmdReady == False and "-e" in argv:
- ArgSkipo = True
- Ret = getArgvFiles("-e")
- if Ret == ["*"]:
- stdout.write("No -e files specified.\n\a")
- exit(1)
- ArgSpecFiles = deepcopy(Ret)
- CmdReady = True
-if CmdReady == False and "-E" in argv:
- ArgSkipO = True
- Ret = getArgvFiles("-E")
- if Ret == ["*"]:
- stdout.write("No -E files specified.\n\a")
- exit(1)
- ArgSpecFiles = deepcopy(Ret)
- CmdReady = True
-if CmdReady == False and "-F" in argv:
- ArgSpec = True
- Ret = getArgvFiles("-F")
- if Ret == ["*"]:
- stdout.write("No -F files specified.\n\a")
- exit(1)
- ArgSpecFiles = deepcopy(Ret)
- CmdReady = True
-if CmdReady == False and "-m" in argv:
- Index = argv.index("-m")
-# You have to at least supply the TagID so we know which file to write to.
- if Index == 1:
- stdout.write("You must at least supply TagID before -m.\n\a")
- exit(1)
- TheMessage = ""
- for I in arange(Index+1, len(argv)):
- TheMessage += argv[I]+" "
- if TheMessage == "":
- stdout.write("No -m message entered.\n\a")
- exit(1)
- WriteMsg = True
- CmdReady = True
-if CmdReady == False:
- stdout.write("What??\n\n")
- exit(1)
+ logIt(MSGspec, "Baler %s found..."%brdy_pkt.q330_sn2)
+ Ret = ipFromDevice(EthDev)
+ if Ret[0] != 0:
+ return Ret
+ localhost = Ret[1]
+# Was nextFreeIP().
+ IP = 0
+# Don't go on forever looking for an address. Make them move to a quieter
+# neighborhood. Also, the baler will shutdown if it takes too long.
+ Count = 0
+ for IP in (I for I in localhost.network.hosts() if I is not Host):
+ IP = str(IP)
+ try:
+# Of course the commands are different.
+ if PROGSystem in ("dar", "lin"):
+ Cp = sb.run(["ping", "-c1", IP], timeout = .3, \
+ stdout = sb.PIPE, stderr = sb.PIPE)
+ elif PROGSystem == "win":
+ Cp = sb.run(["ping", IP, "-n", "1"], timeout = .3, \
+ stdout = sb.PIPE, stderr = sb.PIPE)
+ Count += 1
+ if Count > 100:
+ return (1, "MW", "No free address found in 100 tries.", 3)
+ except sb.TimeoutExpired:
+ break
+ if IP == 0:
+ return (1, "MW", "ERROR: Could not determine IP address.", 3)
+ BalerIP = str(IP)
+ vack = Packet()
+ vack.pack_vack(BalerIP, localhost.netmask, brdy_pkt)
+ logging(vack)
+ vack.crc_check()
+ SockComm = socketComm(localhost.ip)
+ BytesSent = SockComm.sendto(vack.buf, ('255.255.255.255', BalerPort))
+# It looks like the computer can get ahead of the baler and send commands too
+# quickly before the baler has finished doing something and is ready to
+# respond to a sent command (e.g. time and ping). This pause for sure improved
+# things on Macs. There are still random response failures, but it looks like
+# the IP setting still happens. When something like this fails with BaleAddr
+# you still can't talk to the baler.
+ sleep(1.0)
+ logging("Sent vack %d of %d"%(BytesSent, len(vack.buf)))
+ Maybe = False
+# Set the time.
+ logIt(MSGspec, "Setting time in baler...")
+ time_pkt = Packet()
+ time_pkt.pack_set_baler_time(brdy_pkt)
+ BytesSent = SockComm.sendto(time_pkt.buf, (BalerIP, BalerPort))
+ sleep(1.0)
+ try:
+ Raw, Addr = SockComm.recvfrom(14)
+ time_ack_pkt = Packet()
+ time_ack_pkt.unpack(Raw)
+ logging(time_ack_pkt)
+ logIt(MSGspec, "Baler %s time set."%brdy_pkt.q330_sn2)
+ except Exception as e:
+ logIt(MSGspec, "Did not receive time command response.")
+ logging("%s"%e)
+ Maybe = True
+# Test the connection by pinging the baler.
+ ping = Packet()
+ ping.pack_ping()
+ BytesSent = SockComm.sendto(ping.buf, (BalerIP, BalerPort))
+ logging("Sent ping %d of %d"%(BytesSent, len(ping.buf)))
+ sleep(1.0)
+ try:
+ Raw, Addr = SockComm.recvfrom(14)
+ ack_pkt = Packet()
+ ack_pkt.unpack(Raw)
+ logging(ack_pkt)
+ logIt(MSGspec, "Ping response received.")
+ logIt(MSGspec, "Baler OK.")
+ except Exception as e:
+ logIt(MSGspec, "Did not receive ping response from baler.")
+ logging("%s"%e)
+ Maybe = True
+ Ret = getSetSettings(SETspec, "set", EthDev, BalerIP, BalerPort)
+# &&&&& For now.
+ if Ret[0] != 0:
+ Maybe = True
+# Now that we know what we are talking on show the laptop stats.
+ Ret = ipFromDevice(EthDev)
+ if Ret[0] == 0:
+ CCIPMask = Ret[1]
+ CCIP, CCMask = ("%s"%CCIPMask).split("/")
+ logIt(MSGspec, "Control computer IP address: %s"%CCIP)
+ logIt(MSGspec, "Control computer netmask: %s"% \
+ bitsToNetmask(CCMask))
+ elif Ret[0] != 0:
+# Just keep going. Sometimes it all still works out.
+ logIt(MSGspec, "%s"%Ret[2])
+ Msg = "%s set to %s, port %s, on %s"%(brdy_pkt.q330_sn2, \
+ BalerIP, BalerPort, EthDev)
+ if Maybe == True:
+ Msg += " (maybe)"
+ return (0, "GB", Msg, 1)
+# END: addressABaler
+
+###########################################
+# BEGIN: cleanBaler(EDevice, baler_ip_port)
+# FUNC:cleanBaler():2019.268
+def cleanBaler(EDevice, baler_ip_port):
+ Ret = ipFromDevice(EDevice)
+ if Ret[0] != 0:
+ return Ret
+ localhost = Ret[1]
+ clean_p = Packet()
+ clean_p.pack_clean()
+ SockComm = socketComm(localhost.ip)
+ BytesSent = SockComm.sendto(clean_p.buf, baler_ip_port)
+ logging("Sent clean %d of %d"%(BytesSent, len(clean_p.buf)))
+ logIt(MSGspec, "Clean command sent...")
+ logIt(MSGspec, " Waiting...", False)
+ SockComm.settimeout(20)
+ try:
+ while 1:
+ try:
+ Raw, Addr = SockComm.recvfrom(75)
+ break
+ except socket.timeout:
+ logIt("", " Waiting...", False)
+ except KeyboardInterrupt:
+ return (1, "RW", "Stopped by user. Baler may not be finished.", 2)
+ return (0,)
+# END: cleanBaler
+
+DevFormat = "/proc/sys/net/ipv4/conf/%s/rp_filter"
+
+########################
+# BEGIN: rpFilterOK(dev)
+# FUNC:rpFilterOK(dev):2019.255
+def rpFilterOK(dev):
+# dev = DevFormat.format(dev)
+ dev = DevFormat%dev
+ with open(dev, 'r') as pf:
+ status = int(pf.read())
+ if status == 0:
+ return True
+ else:
+ logging("%s = %s"%(dev, status))
+ return False
+# END: rpFilterOK
+
+###############################
+# BEGIN: checkRPFiltersOff(Dev)
+# FUNC:checkRPFiltersOff():2019.268
+# The option value must be "off" to get baler ready broadcast packets. This
+# tests the option for the passed device, or all devices.
+# Dev is a network device, e.g. enp1s0
+# Dev=None checks all devices.
+def checkRPFiltersOff(Dev):
+# Same check as in -b command.
+ if PROGSystem != "lin":
+ logging('Platform not linux, should be fine.')
+ return (0,)
+ from pexpect import spawn
+ Devs = ["all"]
+# Add the passed Dev or get a List of all the files we'll need to check.
+ if Dev != "":
+ Devs.append(Dev)
+ else:
+ DevPaths = glob('/proc/sys/net/ipv4/conf/*/rp_filter')
+ for DevPath in DevPaths:
+ Parts = DevPath.split("/")
+ Devs.append(Parts[-2])
+ for Dev in Devs:
+ if rpFilterOK(Dev) == False:
+ stdout.write("The option filter for '%s' needs to be reset...\n"% \
+ Dev)
+ DevPath = DevFormat%Dev
+ stdout.write("Executing: sudo bash -c \"echo 0 > %s\"\n"%DevPath)
+ stdout.write(" Enter the password")
+ stdout.flush()
+ Sp = spawn("sudo bash -c \"echo 0 > %s\""%DevPath)
+ Sp.timeout = 2
+ Sp.expect("password")
+ Sp.interact()
+ if rpFilterOK(Dev) == False:
+ stdout.write("Still bad.\n")
+ else:
+ stdout.write("Fixed.\n")
+ return (0, any((rpFilterOK(Dev) for Dev in Devs)))
+# END: checkRPFiltersOff
+# END GROUP: bader
+
+
+
+
+####################################################
+# BEGIN: badBlocks(MSGspec, SDRspec, IFiles, Filter)
+# FUNC:badBlocks():2019.289
+def badBlocks(MSGspec, SDRspec, IFiles, Filter):
+ CLFiles = deepcopy(IFiles)
+ CCFiles = listdir(SDRspec)
+ CCFiles.sort()
+ FilesChecked = 0
+ FilesOK = 0
+ FilesOKSize = 0
+# These get loaded with the file names depending on the Ret[0] value coming
+# from checkDTFile() for a summary. It's teh last number in the names.
+ FilesOpenErrors1 = []
+ FilesTooSmall2 = []
+ Files256Size3 = []
+ FilesEndEmpty4 = []
+ FilesNoData5 = []
+ FilesMultStas6 = []
+ FilesMultChans7 = []
+ FilesIsDir8 = []
+# For an overall baler time range.
+ FirstFilesTime = "Z"
+ LastFilesTime = ""
+# Loop through CCFiles and then through all of the file(s) the user specified
+# looking for matches.
+ for Index in arange(0, len(CCFiles)):
+ CCFile = CCFiles[Index]
+# Probably macOS.
+ if CCFile.startswith("."):
+ continue
+# Don't process files that don't look like they came from a baler (like .ALL
+# or .<chan> files).
+ if Filter == True:
+ if basename(CCFile).find("__.") == -1:
+ continue
+ Matches = False
+ for CLFile in CLFiles:
+ if fnmatch(CCFile, CLFile) == True:
+ Matches = True
+ break
+ if Matches == False:
+ continue
+ CCFilespec = SDRspec+CCFile
+# checkDTFile() will also catch this. We'll still look for Rec[0]==8. Non-data
+# files will probably come back with a 256Size error.
+ if isdir(CCFilespec):
+ continue
+ FilesChecked += 1
+ CCFSize = getsize(CCFilespec)
+ logIt(MSGspec, " %d. Checking %s... (%s %s)"%(FilesChecked, \
+ CCFilespec, fmti(CCFSize), sP(CCFSize, ("byte", "bytes"))), \
+ False)
+ Ret = checkDTFile(CCFilespec, " ")
+ logIt(MSGspec, Ret[2], False)
+ FirstTime = Ret[4]
+ LastTime = Ret[5]
+# If something goes wrong the returned times will be "".
+ if FirstTime != "" and FirstTime < FirstFilesTime:
+ FirstFilesTime = FirstTime
+ if LastTime != "" and LastTime > LastFilesTime:
+ LastFilesTime = LastTime
+ if Ret[0] == 0:
+ FilesOK += 1
+ FilesOKSize += CCFSize
+ elif Ret[0] == 1:
+ FilesOpenErrors1.append(CCFile)
+ elif Ret[0] == 2:
+ FilesTooSmall2.append(CCFile)
+ elif Ret[0] == 3:
+ Files256Size3.append(CCFile)
+ elif Ret[0] == 4:
+ FilesEndEmpty4.append(CCFile)
+ elif Ret[0] == 5:
+ FilesNoData5.append(CCFile)
+ elif Ret[0] == 6:
+ FilesMultStas6.append(CCFile)
+ elif Ret[0] == 7:
+ FilesMultChans7.append(CCFile)
+ elif Ret[0] == 8:
+ FilesIsDir8.append(CCFile)
+ if FilesChecked == 0:
+ logIt(MSGspec, "There were no files to check.")
+ return
+ logIt(MSGspec, " Summary:", False)
+ logIt(MSGspec, " Overall date range: %s to %s"%(FirstFilesTime, \
+ LastFilesTime), False)
+ logIt(MSGspec, " Files checked: %d"%FilesChecked, False)
+ logIt(MSGspec, " Files OK: %d (%s %s)"%(FilesOK, \
+ fmti(FilesOKSize), sP(FilesOKSize, ("byte", "bytes"))), False)
+ if len(FilesOpenErrors1) != 0:
+ logIt(MSGspec, " File opening errors: %d:"%len(FilesOpenErrors1), \
+ False)
+ Count = 0
+ for File in FilesOpenErrors1:
+ Count += 1
+ logIt(MSGspec, " %d. %s"%(Count, File), False)
+ if len(FilesTooSmall2) != 0:
+ logIt(MSGspec, " Files too small: %d:"%len(FilesTooSmall2), False)
+ Count = 0
+ for File in FilesTooSmall2:
+ Count += 1
+ logIt(MSGspec, " %d. %s"%(Count, File), False)
+ if len(Files256Size3) != 0:
+ logIt(MSGspec, " Size not /256 bytes: %d:"%len(Files256Size3), False)
+ Count = 0
+ for File in Files256Size3:
+ Count += 1
+ logIt(MSGspec, " %d. %s (%s bytes)"%(Count, File, \
+ fmti(getsize(SDRspec+File))), False)
+ if len(FilesEndEmpty4) != 0:
+ logIt(MSGspec, " Ending empty: %d:"%len(FilesEndEmpty4), False)
+ Count = 0
+ for File in FilesEndEmpty4:
+ Count += 1
+ logIt(MSGspec, " %d. %s"%(Count, File), False)
+ if len(FilesNoData5) != 0:
+ logIt(MSGspec, " No data: %d:"%len(FilesNoData5), False)
+ Count = 0
+ for File in FilesNoData5:
+ Count += 1
+ logIt(MSGspec, " %d. %s"%(Count, File), False)
+ if len(FilesMultStas6) != 0:
+ logIt(MSGspec, " Multiple stations: %d:"%len(FilesMultStas6), False)
+ Count = 0
+ for File in FilesMultStas6:
+ Count += 1
+ logIt(MSGspec, " %d. %s"%(Count, File), False)
+ if len(FilesMultChans7) != 0:
+ logIt(MSGspec, " Multiple channels: %d:"%len(FilesMultChans7), False)
+ Count = 0
+ for File in FilesMultChans7:
+ Count += 1
+ logIt(MSGspec, " %d. %s"%(Count, File), False)
+ elif len(FilesIsDir8) != 0:
+ logIt(MSGspec, " Directories: %d"%len(FilesIsDir8), False)
+ Count = 0
+ for File in FilesIsDir8:
+ Count += 1
+ logIt(MSGspec, " %d. %s\n"%(Count, File), False)
+ return
+# END: badBlocks
+
+
+
+
+###############
+# BEGIN: beep()
+# FUNC:beep():2019.227
+def beep():
+ print("\a")
+# END: beep
+
+
+
+
+###################################
+# BEGIN: def bitsToNetmask(NetBits)
+# FUNC:bitsToNetmask():2019.211
+def bitsToNetmask(NetBits):
+ NetBits = int(NetBits)
+ Mask = (0xffffffff >> (32-NetBits)) << (32-NetBits)
+ return (str((0xff000000 & Mask) >> 24)+'.'+ \
+ str((0x00ff0000 & Mask) >> 16)+'.'+ \
+ str((0x0000ff00 & Mask) >> 8)+'.'+ \
+ str((0x000000ff & Mask)))
+# END: bitsToNetmask
+
+
+
+
+###########################################
+# BEGIN: checkDTFile(Filespec, Prefix = "")
+# FUNC:checkDTFile():2019.253
+# Reads the mini-seed file block headers to see if they are corrupted.
+# Note Ret[0] values, so the caller can keep tabs. Also the return value
+# has the file timerange added to the end.
+def checkDTFile(Filespec, Prefix = ""):
+# Keep a list of station IDs that we come across. Should be just one.
+ StationIDs = []
+# Keep a list of the channel names we find (should be just one for balers).
+ Channels = []
+# The caller should be smarter than this...but maybe not.
+ if isdir(Filespec):
+ return (8, "RW", "%sFile %s is a directory."%(Prefix, \
+ basename(Filespec)), 2, "", "")
+ try:
+ Fp = open(Filespec, "rb")
+ except Exception as e:
+ return (1, "MW", "%s%s"%(Prefix, str(e).strip()), 3, "", "")
+# This one is easy.
+ if getsize(Filespec)%256.0 != 0.0:
+ return (3, "MW", "%sFILE NOT MULTIPLE OF 256 BYTES."%Prefix, 3, "", "")
+# The standard record size for baler files is 4K. Read one and determine if
+# it is smaller.
+ Record = Fp.read(4096+256)
+ RecordLen = len(Record)
+ if RecordLen == 0:
+ return (0, "GB", "%sFile empty."%Prefix, 0, "", "")
+ StaID = Record[8:13].decode("latin-1")
+ if StaID == Record[8+256:13+256].decode("latin-1"):
+ RecordSize = 256
+ elif StaID == Record[8+512:13+512].decode("latin-1"):
+ RecordSize = 512
+ elif StaID == Record[8+1024:13+1024].decode("latin-1"):
+ RecordSize = 1024
+ elif StaID == Record[8+2048:13+2048].decode("latin-1"):
+ RecordSize = 2048
+ elif StaID == Record[8+4096:13+4096].decode("latin-1"):
+ RecordSize = 4096
+ else:
+ return (1, "MW", "%sCould not determine record size."%Prefix, 3, \
+ "", "")
+ Fp.seek(0)
+# Read the file in 10 record chunks to make it faster.
+ RecordsSize = RecordSize*10
+ RecordsRead = 0
+ FirstTime = "Z"
+ LastTime = ""
+ while 1:
+ Records = Fp.read(RecordsSize)
+# We're done with this file.
+ if len(Records) == 0:
+ break
+ RecordsRead += 10
+ for i in arange(0, 10):
+ Ptr = RecordSize*i
+ Record = Records[Ptr:Ptr+RecordSize]
+# Need another set of Records.
+ if len(Record) < RecordSize:
+ break
+ ChanID = Record[15:18].decode("latin-1")
+# The Q330 may create an "empty" file (all 00H) and then not finish filling it
+# in. The read()s keep reading, but there's nothing to process. This detects
+# that and returns. This may only happen in .bms-type data.
+ if ChanID == "\x00\x00\x00":
+# I guess if a file is scrambled these could have 1000's of things in them.
+# Both of them should only have one thing for baler files. Convert them to
+# strings and then chop them off if they are long.
+ Stas = list2Str(StationIDs)
+ if len(Stas) > 12:
+ Stas = Stas[:12]+"..."
+ Chans = list2Str(Channels)
+ if len(Chans) > 12:
+ Chans = Chans[:12]+"..."
+ return (4, "YB", \
+ "%sFILE ENDS EMPTY. Times: %s to %s\n%sRecs: %d(%dB) Stas: %s Chans: %s"% \
+ (Prefix, FirstTime, LastTime, Prefix, RecordsRead, \
+ RecordSize, Stas, Chans), 2, FirstTime, LastTime)
+ if ChanID not in Channels:
+ Channels.append(ChanID)
+ StaID = Record[8:13].decode("latin-1").strip()
+ if StaID not in StationIDs:
+ StationIDs.append(StaID)
+ Year, Doy, Hour, Mins, Secs, Tttt= struct.unpack(">HHBBBxH", \
+ Record[20:30])
+ Month, Date = q330yd2md(Year, Doy)
+ DateTime = "%d-%02d-%02d %02d:%02d"%(Year, Month, Date, Hour, Mins)
+ if DateTime < FirstTime:
+ FirstTime = DateTime
+ if DateTime > LastTime:
+ LastTime = DateTime
+ Fp.close()
+ if RecordsRead == 0:
+ return (5, "YB", "%sNO DATA."%Prefix, 2, "", "")
+# Same as above.
+ Stas = list2Str(StationIDs)
+ if len(Stas) > 12:
+ Stas = Stas[:12]+"..."
+ Chans = list2Str(Channels)
+ if len(Chans) > 12:
+ Chans = Chans[:12]+"..."
+ if len(StationIDs) > 1:
+ return (6, "MW", \
+ "%sMULTIPLE STATIONS. Times: %s to %s\n%sRecs: %d(%dB) Stas: %s Chans: %s"% \
+ (Prefix, FirstTime, LastTime, Prefix, RecordsRead, \
+ RecordSize, Stas, Chans), 3, FirstTime, LastTime)
+ elif len(Channels) > 1:
+ return (7, "MW", \
+ "%sMULTIPLE CHANNELS. Times: %s to %s\n%sRecs: %d(%dB) Stas: %s Chans: %s"% \
+ (Prefix, FirstTime, LastTime, Prefix, RecordsRead, \
+ RecordSize, Stas, Chans), 3, FirstTime, LastTime)
+ else:
+ return (0, "GB", \
+ "%sFILE OK. Times: %s to %s\n%sRecs: %d(%dB) Stas: %s Chans: %s"% \
+ (Prefix, FirstTime, LastTime, Prefix, RecordsRead, \
+ RecordSize, Stas, Chans), 1, FirstTime, LastTime)
+# END: checkDTFile
+
+
##############################
# BEGIN: checkForUpdates(What)
-# FUNC:checkForUpdates():2019.018
+# FUNC:checkForUpdates():2019.266
def checkForUpdates(What):
# Finds the "new"+PROG_NAMELC+".txt" file created by the program webvers at
# the URL and checks to see if the version in that file matches the version
@@ -649,6 +963,19 @@ def checkForUpdates(What):
stdout.write("Checking for updates...\n")
elif What == "get":
stdout.write("Downloading...\n")
+# This security restriction showed up in one place in a different program.
+# Don't know if it is a situation that is coming or going. It may prevent an
+# ssl CERTIFICATE_VERIFY_FAILED error if things are not right. If this fails
+# the urlopen() may too, so just go on, or maybe this system won't even need
+# this.
+ try:
+ import ssl
+ from os import environ
+ if (environ.get("PYTHONHTTPSVERIFY") == None or \
+ getattr(ssl, "_create_unverified_context", "") == ""):
+ ssl._create_default_https_context = ssl._create_unverified_context
+ except Exception as e:
+ stdout.write("SSL: %s\n"%e)
# Get the file that tells us about the current version on the server.
# One line: PROG; version; original size; compressed size
try:
@@ -662,12 +989,12 @@ def checkForUpdates(What):
# <!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
# How unhandy.
if Line.find("DOCTYPE") != -1:
- stdout.write("No update information found.\n")
+ stdout.write("No update information found.\n\n")
return 0
- except (IndexError, IOError) as e:
- stdout.write("%s\n"%e)
+ except Exception as e:
+ stdout.write("URL: %s\n"%e)
stdout.write( \
- "There was an error obtaining the version information from PASSCAL.\n")
+ "There was an error obtaining the version information from PASSCAL.\n\n")
return 1
Parts = Line.split(";")
for Index in arange(0, len(Parts)):
@@ -676,24 +1003,22 @@ def checkForUpdates(What):
if What == "check":
if PROG_VERSION < Parts[VERS_VERS]:
stdout.write( \
- "This is an old version of %s.\nThe current version generally available is %s.\nUse bline.py -UD to get the new version.\n"% \
+ "This is an old version of %s.\nThe current version generally available is %s.\nUse bline.py -UD to get the new version.\n\n"% \
(PROG_NAME, Parts[VERS_VERS]))
return 0
elif PROG_VERSION == Parts[VERS_VERS]:
- stdout.write("This copy of %s is up to date.\n"%PROG_NAME)
+ stdout.write("This copy of %s is up to date.\n\n"%PROG_NAME)
return 0
elif PROG_VERSION > Parts[VERS_VERS]:
stdout.write( \
- "Congratulations!\nThis is a newer version of %s than is generally available.\nEveryone else probably still has version %s.\n"% \
+ "Congratulations!\nThis is a newer version of %s than is generally available.\nEveryone else probably still has version %s.\n\n"% \
(PROG_NAME, Parts[VERS_VERS]))
return 0
elif What == "get":
- ZSize = int(Parts[VERS_ZSIZ])
try:
GetFile = "new%s.zip"%PROG_NAMELC
Fpr = urlopen(VERS_VERSURL+GetFile)
- Fpw = open(Dirspec+GetFile, "wb")
- DLSize = 0
+ Fpw = open(ABSCWDspec+GetFile, "wb")
Buffer = Fpr.read()
Fpw.write(Buffer)
except Exception as e:
@@ -706,14 +1031,15 @@ def checkForUpdates(What):
Fpw.close()
except:
pass
- stdout.write("Error downloading new version.\n%s"%e)
+ stdout.write("Error downloading new version.\n%s\n\n"%e)
return 1
Fpr.close()
Fpw.close()
- stdout.write("The current update file has been saved as\n\n %s\n\n"% \
- (AbsDirspec+GetFile))
stdout.write( \
- "Unzip the file, rename new%s.py to %s.py,\nand copy it to its final destination.\n"% \
+ "The current update file has been saved as\n\n %s\n\n"% \
+ (ABSCWDspec+GetFile))
+ stdout.write( \
+ "Unzip the file, rename new%s.py to %s.py,\nand copy it to its final destination.\n\n"% \
(PROG_NAMELC, PROG_NAMELC))
return 0
# END: checkForUpdates
@@ -721,6 +1047,33 @@ def checkForUpdates(What):
+#########################################################
+# BEGIN: fileBlockRetrieved(Blocks, BlockSize, TotalSize)
+# FUNC:fileBlockRetrieved():2019.234
+# Gets called every 8192 bytes offloaded and urlretrieved() passes the
+# argument values.
+def fileBlockRetrieved(Blocks, BlockSize, TotalSize):
+ global OffBytes
+ global Offloaded10
+# This will be kept updated for any offloading error message.
+ OffBytes = Blocks*BlockSize
+# Will give us 10% 20% 30%...
+ Offed = int(100.0*OffBytes/TotalSize)//10*10
+ if Offed > Offloaded10:
+ if Offloaded10 == 0:
+ stdout.write(" ")
+ Offloaded10 = Offed
+# Rounding errors on some systems can make this 110. ???
+ if Offloaded10 > 100:
+ Offloaded10 = 100
+ stdout.write(" %d%%"%Offloaded10)
+ stdout.flush()
+ return
+# END: fileBlockRetrieved
+
+
+
+
###################
# BEGIN: floatt(In)
# LIB:floatt():2018.256
@@ -786,139 +1139,150 @@ def fmti(Value):
-#########################################################
-# BEGIN: fileBlockRetrieved(Blocks, BlockSize, TotalSize)
-# FUNC:fileBlockRetrieved():2017.026
-# Gets called every 8192 bytes offloaded.
-def fileBlockRetrieved(Blocks, BlockSize, TotalSize):
- global OffFileBytes
- global Offloaded10
-# This will be kept updated for any offloading error message.
- OffFileBytes = Blocks*BlockSize
-# Will give us 10% 20% 30%...
- Offed = int(100.0*OffFileBytes/TotalSize)//10*10
- if Offed > Offloaded10:
- if Offloaded10 == 0:
- stdout.write(" ")
- Offloaded10 = Offed
-# Rounding errors on some systems can make this 110. ???
- if Offloaded10 > 100:
- Offloaded10 = 100
- stdout.write(" %d%%"%Offloaded10)
- stdout.flush()
- return
-# END: fileBlockRetrieved
+##################################
+# BEGIN: checkTagID(TagID, IPAddr)
+# LIB:checkTagID():2019.240
+# Grabs the TagID from the baler.htm page just to check that the user and
+# the program are talking about the same baler.
+def checkTagID(ITagID, IPAddr):
+ try:
+ Ret = readFileLinesRB("http://%s/baler.htm"%IPAddr, True)
+ if Ret[0] != 0:
+ return Ret
+ except IOError as e:
+ return (2, "MW", "Error getting baler.htm:\n%s"%e, 3)
+ Lines = Ret[1]
+ BTagID = ""
+ for Line in Lines:
+ Line = Line.upper()
+ if Line.find("BALER TAGID:") != -1:
+ Value = intt(Line[(Line.index(">BALER TAGID:")+ \
+ len(">BALER TAGID:")):])
+ BTagID = str(Value)
+ break
+# Talking to the wrong baler??
+ if ITagID != BTagID:
+ return (1, "RW", "TagIDs do not match: Entered %s, baler %s"% \
+ (ITagID, BTagID), 2)
+ return (0,)
+# END: checkTagID
########################################
-# BEGIN: getBalerHtm(Msg, TagID, IPAddr)
-# LIB:getBalerHtm():2019.052
-def getBalerHtm(Msg, TagID, IPAddr):
- global BState
-# Do these simple-minded checks, because you know those users, and it may be
-# easier for some programs to let this function check here, instead of figuring
-# out what was entered, for example, on the command line.
- if TagID.find(".") != -1:
- return (2, "RW", "TagID has periods in it: %s"%TagID, 2)
- if IPAddr.find(".") == -1:
- return (2, "RW", "IPAddr has no periods in it: %s"%IPAddr, 2)
- if Msg == "bg":
- msgLn(1, "W", "Getting baler.htm from %s..."%TagID)
- elif Msg == "bl":
- logIt("Getting baler.htm from %s..."%TagID)
+# BEGIN: getBalerInfoHtm(ITagID, IPAddr)
+# LIB:getBalerInfoHtm():2019.252
+# Gets the index.htm and baler.htm pages and returns basic info about the
+# baler.
+def getBalerInfoHtm(ITagID, IPAddr):
try:
- Fp = urlopen("http://%s/baler.htm"%IPAddr)
- Ret = readFileLines2(Fp, True)
+ Ret = readFileLinesRB("http://%s/index.htm"%IPAddr, True)
if Ret[0] != 0:
- return (1, Ret)
- Fp.close()
- Lines = Ret[1]
+ return Ret
+ except IOError as e:
+ return (2, "MW", "Error getting index.htm:\n%s"%e, 3)
+ Lines = Ret[1]
+ Info = {}
+ Info["NetSta"] = "?"
+ for Line in Lines:
+ Line = Line.upper()
+# The line is: <CENTER><H1>Net-Sta INDEX</H1>...
+ if Line.find("<H1>") != -1:
+ Parts = Line.split()
+ Index = Parts[0].index("<H1>")
+ Info["NetSta"] = Parts[0][Index+4:]
+ try:
+ Ret = readFileLinesRB("http://%s/baler.htm"%IPAddr, True)
+ if Ret[0] != 0:
+ return Ret
except IOError as e:
- return (2, "MW", "Error getting baler.htm:\n%s\nStopped."%e, 3)
-# Set by hitting the Check button when it is already checking.
- if BState == STATE_STOP:
- return (1, "YB", "Stopped by user. (%s)"%getGMT(3), 2)
- FWVers = "?"
- TagID2 = "?"
- DSize = 0
- NFiles = 0
- Percent = 0.0
+ return (2, "MW", "Error getting baler.htm:\n%s"%e, 3)
+ Lines = Ret[1]
+ Info["BModel"] = "?"
+ Info["FWVers"] = "?"
+ Info["MACAddr"] = "?"
+ Info["BTagID"] = "?"
+ Info["DSize"] = 0
+ Info["NFiles"] = 0
+ Info["Percent"] = 0.0
+ Info["FW2Vers"] = "?"
+ Info["SerNum"] = "?"
+ Info["Volts"] = "?"
+ Info["Temp"] = "?"
+ Info["DModel"] = "?"
+# The baler model and software version are two HTML <li> items, but are on the
+# same line, so just check each line for each possibility. All of the other
+# items are on a line by themselves.
for Line in Lines:
Line = Line.upper()
+ if Line.find(">BALER MODEL: ") != -1:
+ Info["BModel"] = Line[(Line.index(">BALER MODEL: ")+ \
+ len(">BALER MODEL: ")):Line.index("</LI>")]
if Line.find(">SOFTWARE VERSION:") != -1:
Value = floatt(Line[(Line.index(">SOFTWARE VERSION:")+ \
len(">SOFTWARE VERSION:")):])
- FWVers = str(Value)
- continue
+ Info["FWVers"] = str(Value)
+ if Line.find(">MAC ADDRESS: ") != -1:
+ Info["MACAddr"] = Line[(Line.index(">MAC ADDRESS: ")+ \
+ len(">MAC ADDRESS: ")):Line.index("</LI>")]
if Line.find("BALER TAGID:") != -1:
Value = intt(Line[(Line.index(">BALER TAGID:")+ \
len(">BALER TAGID:")):])
- TagID2 = str(Value)
- continue
+ Info["BTagID"] = str(Value)
if Line.find("DISK SIZE:") != -1:
Value = intt(Line[(Line.index("DISK SIZE:")+ \
len("DISK SIZE:")):])
- DSize = Value
- continue
+ Info["DSize"] = Value
if Line.find("PERCENT OF") != -1:
Parts = Line.split()
- NFiles = intt(Parts[2])
- Percent = floatt(Parts[-1])
- continue
+ Info["NFiles"] = intt(Parts[2])
+ Info["Percent"] = floatt(Parts[-1])
+ if Line.find(">DMU2") != -1:
+ Parts = Line.split()
+ Info["FW2Vers"] = Parts[2][Parts[2].index("=")+1: \
+ Parts[2].index(",")]
+ Info["SerNum"] = Parts[4][Parts[4].index("=")+1: \
+ Parts[4].index("</LI>")]
+ if Line.find(">SUPPLY VOLTAGE=") != -1:
+ Info["Volts"] = Line[(Line.index(">SUPPLY VOLTAGE=")+ \
+ len(">SUPPLY VOLTAGE=")):Line.index("</LI>")]
+ if Line.find(">TEMPERATURE=") != -1:
+ Info["Temp"] = Line[(Line.index(">TEMPERATURE=")+ \
+ len(">TEMPERATURE=")):Line.index("</LI>")]
+ if Line.find(">DISK MODEL: ") != -1:
+ Info["DModel"] = Line[(Line.index(">DISK MODEL: ")+ \
+ len(">DISK MODEL: ")):Line.index("</LI>")]
# Talking to the wrong baler??
- if TagID != TagID2:
- return (2, "R", \
- "TagIDs do not match: Entered %s, baler %s.\nStopped. (%s)"% \
- (TagID, TagID2, getGMT(3)), 2)
- if Msg == "bg":
- msgLn(1, "", "Got baler.htm.")
- elif Msg == "bl":
- logIt("Got baler.htm.")
- return (0, FWVers, DSize, NFiles, Percent)
-# END: getBalerHtm
+ if ITagID != Info["BTagID"]:
+ return (1, "RW", "TagIDs do not match: Entered %s, baler %s"% \
+ (ITagID, Info["BTagID"]), 2)
+ return (0, Info)
+# END: getBalerInfoHtm
-########################################
-# BEGIN: getFilesHtm(Msg, TagID, IPAddr)
-# LIB:getFilesHtm():2019.052
-# Used in many places to decode the List of Lists of files returned by this
-# function.
-FH_NAME = 0
-FH_SIZE = 1
-FH_FROM = 2
-FH_TO = 3
-
-def getFilesHtm(Msg, TagID, IPAddr):
- global BState
-# Do these simple-minded checks, because you know those users, and it may be
-# easier for some programs to let this function check here, instead of figuring
-# out what was entered, for example, on the command line.
- if TagID.find(".") != -1:
- return (2, "RW", "TagID has periods in it: %s"%TagID, 2)
- if IPAddr.find(".") == -1:
- return (2, "RW", "IPAddr has no periods in it: %s"%IPAddr, 2)
- if Msg == "bg":
- msgLn(1, "W", "Getting files.htm from %s..."%TagID)
- elif Msg == "bl":
- logIt("Getting files.htm from %s..."%TagID)
+############################################
+# BEGIN: getFilesHtm(MSGspec, TagID, IPAddr)
+# LIB:getFilesHtm():2019.235
+# Used in many places to decode the List of Lists of files returned by this
+# function.
+B_NAME = 0
+B_SIZE = 1
+B_FROM = 2
+B_TO = 3
+
+def getFilesHtm(MSGspec, TagID, IPAddr):
try:
- Fp = urlopen("http://%s/files.htm"%IPAddr)
- Ret = readFileLines2(Fp, True)
+ Ret = readFileLinesRB("http://%s/files.htm"%IPAddr, True)
if Ret[0] != 0:
- return (1, Ret)
- Fp.close()
+ return Ret
Lines = Ret[1]
except IOError as e:
- return (1, "MW", "Error getting files.htm:\n%s\nStopped. (%s)"%(e, \
- getGMT(3)), 3)
- if BState == STATE_STOP:
- return (1, "YB", "Stopped by user. (%s)"%getGMT(3), 2)
- FHFiles = []
- FHBytes = 0
+ return (1, "MW", "Error getting files.htm:\n%s"%e, 3)
+ BFiles = []
+ BBytes = 0
for Line in Lines:
# The line case is mixed, but the file names are always all uppercase, so just
# do that.
@@ -941,19 +1305,12 @@ def getFilesHtm(Msg, TagID, IPAddr):
Size = int(Parts[2])
From = "%s %s"%(Parts[5], Parts[6])
To = "%s %s"%(Parts[8], Parts[9])
- FHFiles.append([Filename, Size, From, To])
- FHBytes += Size
- if len(FHFiles) == 0:
- return (1, "Y", "No files found in files.htm.\nStopped. (%s)"% \
- getGMT(3))
- if BState == STATE_STOP:
- return (1, "YB", "Stopped by user. (%s)"%getGMT(3), 2)
- FHFiles = listSort(FHFiles, FH_NAME, "a")
- if Msg == "bg":
- msgLn(1, "", "Got files.htm.")
- elif Msg == "bl":
- logIt("Got files.htm.")
- return (0, FHFiles, FHBytes)
+ BFiles.append([Filename, Size, From, To])
+ BBytes += Size
+ if len(BFiles) == 0:
+ return (1, "Y", "No files found in files.htm.", 2)
+ BFiles = listSort(BFiles, B_NAME, "a")
+ return (0, BFiles, BBytes)
# END: getFilesHtm
@@ -1048,6 +1405,738 @@ def getGMT(Format):
+######################################################################
+# BEGIN: getSetSettings(SETspec, Cmd, Ieth = "", Iip = "", Iport = "")
+# FUNC:getSetSettings():2019.289
+# The settings file contains the Ethernet port, IP address and port number
+# for a baler. The file is usually <tagid>.set and the format is:
+# Ethernet device; IP address; port number
+# The file will be created/updated when the -b command finds a baler, or
+# when the user supplies the IP address to the -b command.
+# With the '-b <ipaddr>' command the Ethernet device and port number will
+# be blank.
+def getSetSettings(SETspec, Cmd, Ieth = "", Iip = "", Iport = ""):
+ if Cmd == "get":
+ try:
+ if exists(SETspec) == False:
+ return (1, "RW", "There is no settings file %s"% \
+ basename(SETspec))
+ Ret = readFileLinesRB(SETspec, True)
+ if Ret[0] != 0:
+ return (1, "MW", "Error reading settings file.")
+ Lines = Ret[1]
+ Parts = Lines[0].split(";")
+ Parts += [""]*(3-len(Parts))
+ for Index in arange(0, len(Parts)):
+ Parts[Index] = Parts[Index].strip()
+ except Exception as e:
+ return (1, "MW", "Error reading settings file.\n%s"%e)
+# This one has to be here and look right, so might as well check for the
+# caller.
+ if Parts[1] == "":
+ return (1, "No IP address found in file %s.set."%CLTagID)
+ if isIPV4Addr(Parts[1]) == False:
+ return (1, \
+ "The %s file IP address '%s' does not look right."% \
+ (basename(SETspec), Parts[1]))
+ return (0, Parts[0], Parts[1], Parts[2])
+ elif Cmd == "set":
+ try:
+ Fp = open(SETspec, "w")
+ Fp.write("%s;%s;%s\n"%(Ieth, Iip, Iport))
+ Fp.close()
+ except:
+ return (1, "MW", "Error writing settings file.")
+ return (0,)
+# END: getSetSettings
+
+
+
+
+###################
+# BEGIN: helpLong()
+# FUNC:helpLong():2019.297
+def helpLong():
+ HELPText = "USING BLINE.PY\n\
+--------------\n\
+BLINE is a Python command line program that offloads data files from a\n\
+Quanterra PB14F Baler on Linux, macOS and Windows. It may also be used to\n\
+set up the baler prior to offloading by assigning the baler an IP address,\n\
+and erase data and clear the Q330 association before returning the baler\n\
+to the field.\n\
+\n\
+The baler and the control computer must be on the same Ethernet sub-net\n\
+for them to communicate. Every time the baler is powered up it defaults\n\
+to an IP address and broadcasts its presence. BLINE or Quantera's BaleAddr\n\
+program must be used to set the baler's address to one compatible with\n\
+the address of the control computer.\n\
+\n\
+If using BaleAddr, and the computer IP address is something like\n\
+129.138.26.1, the baler's address should be set to something like\n\
+129.138.26.2 for the two to communicate. The desired IP address for the\n\
+baler is entered in BaleAddr's 'IP Address to Assign' field and the ATTN\n\
+button on the baler is then pressed. When the baler finishes booting\n\
+BaleAddr handshakes with the baler and sets the IP address to the entered\n\
+value.\n\
+\n\
+When using BLINE to set the IP address the baler should be connected the\n\
+same way as for BaleAddr, and then the 'bline.py <tagid> -b' command\n\
+issued. When directed by BLINE the ATTN button on the baler should be\n\
+pressed. BLINE will detect the IP address of the control computer and\n\
+select an IP address for the baler once the baler is finished booting.\n\
+Some caveats to this process are listed below.\n\
+\n\
+\n\
+IP ADDRESS CAVEATS WITH BLINE (AND BALEADDR)\n\
+--------------------------------------------\n\
+Not all versions of operating systems or versions of Python support all\n\
+of the necessary networking functions for the new BLINE to 'automatically'\n\
+determine where a baler 'is coming from' when it initially broadcasts\n\
+its 'I am here' message. In those cases BLINE must be told the name of\n\
+the Ethernet device where the baler will show up. This is related to\n\
+having to disable the WiFi system on a laptop before using BaleAddr to\n\
+set the IP address. BaleAddr can also have trouble figuring out to which\n\
+device the baler is connected. See the options for the -b comand.\n\
+\n\
+Depending on the operating system of the control computer, and sometimes\n\
+the age of the computer, the IP address of the Ethernet port may need to be\n\
+manually set using the operating system preferences utilities, and that\n\
+may not even be good enough. Some operating systems and/or computers do\n\
+not set the Ethernet port IP address until they are connected to a network.\n\
+If the baler is connected directly to the Ethernet port there is no\n\
+network until the baler wakes up with an ATTN button press. For BaleAddr\n\
+use it is then too late. You need to know the address to place into the\n\
+address field before waking up the baler.\n\
+\n\
+Some systems will set the Ethernet device IP address to a \"self-assigned\"\n\
+address once the ATTN button has been pressed. Again this is too late for\n\
+BaleAddr, but for BLINE it may work if the address is set quickly enough\n\
+while the baler is booting up.\n\
+\n\
+If the device's IP address cannot be set, or does not get set itself in\n\
+time to begin talking to a baler by the time it is ready the problem can\n\
+usually be solved by placing a hub or switch in the Ethernet line between\n\
+the control computer and the baler. Something like a Netgear GS105 may\n\
+be used, or a PASSCAL Baler Offload Box. Connecting the computer's Ethernet\n\
+port to a switch/hub should keep the port alive so it will keep a manually\n\
+assigned or be assigned a self-assigned IP address before the baler ATTN\n\
+button is pressed.\n\
+\n\
+Windows 7 and 10 control computers that were connected to the Internet were\n\
+assigned an IP address by DHCP. BLINE was started with the -b command and\n\
+the network cable was removed and one directly connected to the baler was\n\
+connected. When the baler was started BLINE assigned an IP address to the\n\
+baler that would work with the computer's DHCP address. After a couple of\n\
+minutes communication with the baler failed. The operating system or the\n\
+Ethernet hardware detected that the port was not connected to the actual\n\
+Internet (just another device). Windows decided to drop the DHCP IP\n\
+address and create a self-assigned address for the device. At that point\n\
+BLINE could not talk to the baler. Shutting down the baler and reissuing\n\
+the -b command got the computer and baler back on the same sub-net. This\n\
+may also happen with macOS.\n\
+\n\
+On Linux some files may need to be modified to allow the operating system\n\
+to detect the initial broadcast message from the baler. If modifying the\n\
+file(s) is required the password for the user account (the 'sudo password')\n\
+will be requested to allow BLINE to make those changes. The request may\n\
+be made more than once, but following that the files should not need to\n\
+be changed again until the control computer has been shutdown or rebooted.\n\
+\n\
+macOS is SO safe that it wants to approve the connection of a baler to\n\
+the Python interpreter at least once during a session when using the -b\n\
+command. When this happens you will need to click the Allow button and\n\
+the -b command will continue, but will probably fail to set the baler's IP\n\
+address. If you are quick you can reissue the -b command (use the Up-Arrow\n\
+key if using Terminal), and if the baler has not shut down (LEDs are still\n\
+green) the baler will try to connect again in about 1 minute. You do not\n\
+need to press the ATTN button to shut the baler down and then start it\n\
+back up. If you do not reissue the -b command in time the baler will shut\n\
+itself down after about 1.5 minutes.\n\
+\n\
+The reissuing trick for the -b command above after a failure, but before\n\
+the baler shuts down, may be used on all operating systems. The baler\n\
+always tries to connect again before it gives up.\n\
+\n\
+\n\
+GENERAL OPERATIONS\n\
+==================\n\
+INSTALLING AND RUNNING BLINE\n\
+----------------------------\n\
+BLINE is a single Python program file, so you need Python 2 or 3 installed\n\
+to run it. The advanced commands (-b -c -n -s -x) require packages and\n\
+modules that may not normally be installed in a standard Python\n\
+installation. These can usually be installed using 'pip' or 'pip3' (\"Pip\n\
+Installs Packages\"). Using any of the above commands will cause BLINE to\n\
+check to see if any of the modules need to be installed, so test before\n\
+going to the field where there may be no Internet.\n\
+\n\
+Using BaleAddr to set the baler IP address and using the \"old\" BLINE\n\
+commands for offloading files does not require extra Python modules.\n\
+\n\
+To install BLINE copy the file bline.py to where ever you want. If it is\n\
+placed somewhere like /opt/passcal/bin/ (the preferred choice if the\n\
+PASSCAL software package has been installed), with the other PASSCAL\n\
+software, then it should be in the path set up by the PASSCAL software\n\
+installation, and just \"bline.py\" on the command line of an xterm or\n\
+Terminal window will start execution. For that location this command will\n\
+probably have to be used\n\
+\n\
+ sudo cp bline.py /opt/passcal/bin\n\
+\n\
+and the login password given to copy it there. No matter where it is\n\
+installed it will also probably have to made executable with the command\n\
+\n\
+ chmod +x bline.py\n\
+\n\
+Depending on where it is copied \"sudo\" may also need to be added before\n\
+chmod. If it is not executable a 'permission denied' message will be\n\
+displayed when trying to execute the program.\n\
+\n\
+Installing it to any other location will require ensuring that the location\n\
+is in the current path and the necessary entries have been made in files\n\
+such as .bashrc, or .bash_profile, or any other initialization files for\n\
+the command shell in use, otherwise you will need to preface 'bline.py'\n\
+with the required path to execute it.\n\
+\n\
+After installation you can check if any additional modules need to be\n\
+installed by running the command\n\
+\n\
+ bline.py 5555 -b\n\
+\n\
+On Windows you may need \"python\" in front of the bline.py. On macOS and\n\
+Linux you probably won't. The 5555 does not have to be a real baler TagID\n\
+to run this test. The modules needing to be installed will be listed.\n\
+Remember, this is only if you are going to use BLINE to assign the baler's\n\
+IP address or clean the baler for redeployment, instead of BaleAddr.\n\
+\n\
+PIP\n\
+If you enter 'pip' on the command line and it does not exists it can be\n\
+installed using\n\
+\n\
+ python -m ensurepip --default-pip\n\
+\n\
+If, or once pip is installed the usual command for installing modules is\n\
+\n\
+ pip install <module name>\n\
+\n\
+Known extra modules that may be needed. These may not all be required on\n\
+all systems or for every command, but to use any of the commands you will\n\
+need to install all of them:\n\
+ -b -c -n -s -x commands:\n\
+ psutil\n\
+ subprocess32 (for Python2)\n\
+ ipaddress\n\
+ pexpect (for Linux)\n\
+\n\
+Using the 'ensurepip' install method may install a very old version of\n\
+pip. If pip says there is a new version you can install it using the\n\
+update instructions that will be displayed.\n\
+\n\
+\n\
+STARTING BLINE\n\
+--------------\n\
+BLINE is a command line program and to run it you start a Terminal/xterm\n\
+window in the directory where you want bookkeeping files and the offloaded\n\
+data files for a baler to be saved. The command to start the program\n\
+depends on how BLINE was installed and the operating system as described\n\
+above. Using no command line arguments will list all of the commands (the\n\
+Short Help -- you are reading the Long Help) for the program. Any of these\n\
+may be the correct way to start BLINE:\n\
+\n\
+\n\
+ bline.py\n\
+or\n\
+ <path to bline>/bline.py\n\
+or\n\
+ ./bline.py\n\
+or\n\
+ python bline.py\n\
+\n\
+\n\
+SETTING THE BALER IP ADDRESS\n\
+----------------------------\n\
+Using BaleAddr\n\
+--------------\n\
+Once the baler has been connected to the control computer and the Ethernet\n\
+IP address of the control computer has been determined (see the section\n\
+on IP address caveats above) enter the address you want the baler to be\n\
+set to into the 'IP Address to Assign' field in the program's display and\n\
+press the ATTN button. Once the baler boots up it should be assigned the\n\
+address that was entered if the address is reachable by the control\n\
+computer.\n\
+\n\
+This version of BLINE differs from the original version in that you do not\n\
+provide the baler's address on the command line for each command, however,\n\
+when using BaleAddr to set the IP address BLINE must still be told what the\n\
+baler's address was set to. This is done using the -b command\n\
+\n\
+ bline.py <tagid> -b <ipaddr>\n\
+\n\
+The entered <ipaddr> will be written to a file named <tagid>.set and\n\
+BLINE will read that file each time a command needs the address.\n\
+\n\
+Using BLINE\n\
+-----------\n\
+Provided the cavets in the section above are understood setting the IP\n\
+address of the baler using BLINE after the baler has been connected is\n\
+done using\n\
+\n\
+ bline.py <tagid> -b\n\
+\n\
+BLINE will start and wait for the ATTN button of the target baler to be\n\
+pushed and that baler to boot up. When/if the baler is seen by BLINE its\n\
+address will be set, then the time in the baler, and then the baler will\n\
+be \"pinged\" to see that it can be communicated with.\n\
+\n\
+If BLINE is being used on a system that does not fully support the\n\
+networking functions for automatically determining the correct Ethernet\n\
+device the baler is trying to connect through, BLINE can be told which\n\
+device to watch using\n\
+\n\
+ bline.py <tagid> -b <ethdev>\n\
+\n\
+It will depend on the operating system, but this will be something like\n\
+en0 or enp1s0. On Windows systems it may be something like \"Local\n\
+Area Connection\". In this case it is recommended that the network\n\
+adapter for the Ethernet device in Control Panel | Network Connections\n\
+be renamed to something sensible, like \"en0\".\n\
+\n\
+\n\
+When everything works for setting the IP address of the baler with\n\
+BaleAddr or BLINE, you should get the message\n\
+\n\
+ Baler OK\n\
+\n\
+When using either program if the 'Baler OK' message doesn't show up then\n\
+the program may not really be talking to the baler and/or the baler's IP\n\
+address did not get set. It happens a lot. Sometimes commands after this\n\
+when using BLINE may work, but most often not. Trying again after checking\n\
+the wiring and any entered IP address or Ethernet device name is the best\n\
+ideas.\n\
+\n\
+Quiting/Resetting BaleAddr, re-pressing the ATTN button on the baler to\n\
+let the baler shut down, restarting BaleAddr (if it was Quit), and pressing\n\
+the ATTN button again up to five times has been sucsessful at geting\n\
+BaleAddr and the baler to connect. It's just flakey. Make sure the Ethernet\n\
+port has or is getting set to an address as described in the caveats.\n\
+If you get up to maybe three cycles you can try shutting down the baler\n\
+and then disconnecting the power and reconnecting. Sometimes that helps.\n\
+Make sure you have a suitable IP address in BaleAddr that is compatible\n\
+with the address the control computer thinks it has. That REALLY helps.\n\
+Also make sure the WiFi network is turned off like you have to do with\n\
+EzBaler. None of the programs can figure out which Ethernet adapter to use\n\
+when there is more than one active, except BLINE when using the -b command.\n\
+In testing BLINE seems to be better.\n\
+\n\
+All of the past and current conection problems may be because of the IP\n\
+address funny business described in the caveats section above.\n\
+\n\
+\n\
+WHERE'S MY STUFF?\n\
+-----------------\n\
+All files and directories created by BLINE pertaining to a particular\n\
+baler are created in the current working directory where BLINE is started,\n\
+and they all start with the <tagid> of the baler. For example 5520.msg\n\
+contains most of the text messages displayed while using BLINE and is a\n\
+running log record of what was performed for baler 5520. Some simple\n\
+status messages and errors are not saved. Data files offloaded from the\n\
+baler will be in the directory named 5520.sdr. .sdr is a reference to\n\
+the Q330 program sdrsplit, that is used to split multiplexed data files\n\
+into files containing only one Q330 channel per file, which is how\n\
+information is recorded on the balers. The .sdr also triggers the program\n\
+QPEEK to look for one-channel-per-file data in the directory. 5520.set\n\
+is new for this BLINE version and it contains the IP information for\n\
+the baler. 5520files.txt is a listing of the data files on the baler that\n\
+is created/rewritten each time a command is executed that asks the baler\n\
+for the list of files.\n\
+\n\
+\n\
+STOPPING BLINE\n\
+--------------\n\
+As usual, Ctrl-C should stop any BLINE operation.\n\
+\n\
+\n\
+COMMAND DETAILS\n\
+===============\n\
+BLINE is designed to be run in or started from the directory where you want\n\
+to work. You cannot use paths to directories. The current working directory\n\
+or the \"work directory\" must be set by changing to the directory of\n\
+choice using the command line with something like the 'cd' command before\n\
+starting the program. Files created by the program will be written the the\n\
+work directory and offloaded files will be written to a sub-directory of\n\
+the work directory (./<tagid>.sdr). Verification commands, -v/-vl and -V,\n\
+expect the program to be started in the same directory where it was\n\
+running when the files were offloaded from the baler -- if they have not\n\
+been moved since offloading them.\n\
+\n\
+All of the commands are entered on the command line. There are several:\n\
+versions of the command line:\n\
+\n\
+ bline.py <command>\n\
+ bline.py <tagid> <command>\n\
+ bline.py <tagid> <command> [<files>]\n\
+ bline.py <tagid> <command> <files>\n\
+ bline.py <tagid> -M <message>\n\
+ bline.py <tagid> -b [<ipaddr> | <ethdev>]\n\
+\n\
+See the -h help for the command line items that are required or optional\n\
+for each command.\n\
+\n\
+The <tagid> is used as part of the bookkeeping file names and the directory\n\
+name where the offloaded data files will be placed. It is also used by\n\
+BLINE to make sure it is talking to the baler the user thinks it is\n\
+talking to.\n\
+\n\
+Some commands allow, and some require, a space-separated list of file\n\
+names with or without standard UNIX wildcard characters to follow them\n\
+to control which data files are delt with. This may, for example, be a\n\
+list of files to offload, a list of files to not offload, or it may be a\n\
+list of files to examine. It depends on the command. Depending on the\n\
+operating system the the way the command shell is set up you may need to\n\
+enclose file names on the command line in double quotes when using\n\
+wildcard characters For example \"*\", instead of just *, or \"*.BZN\",\n\
+instead of *.BZN. In most cases not supplying any file list is the same\n\
+as using \"*\", i.e. 'all files'. Again it depends on the command.\n\
+\n\
+BaleAddr and/or one of the -b command variations of BLINE must be used to\n\
+set the IP address of the baler before any of the baler-related commands\n\
+will function.\n\
+\n\
+\n\
+LIST OF COMMANDS\n\
+----------------\n\
+These are grouped according to the similarity of the command line arguments.\n\
+\n\
+bline.py <command>\n\
+------------------\n\
+-c\n\
+This command asks the computer running BLINE to print its hostname and\n\
+IP address. This address may be used to help select an IP address to set\n\
+the baler to in the BaleAddr program. The address printed may or may not\n\
+be correct or useful. Some operating systems will not print the correct\n\
+address until the computer is already talking to a baler. Some operating\n\
+systems will report the wrong address if both the wired and wireless\n\
+Ethernet systems are in use. The list goes on. An address of 127.0.0.1\n\
+or 127.0.1.1 will be displayed if the 'default' port's address is not set.\n\
+The default port used by this version of the -c command may not even be\n\
+the correct port.\n\
+\n\
+The most reliable way to determine the IP address of the computer for\n\
+running BaleAddr is to look in the system preferences or Ethernet adapter\n\
+information for the operating system in use. See the second -c command\n\
+below.\n\
+\n\
+-h and -H\n\
+The -h command prints a summary list of possible command line arguments\n\
+to the display. The -H command prints a more detailed help document to the\n\
+display.\n\
+\n\
+-n\n\
+Show the network device/interface names on the control computer. The list\n\
+can be quite long, and it may be useless in Windows, but it may help\n\
+when the Ethernet device name must be supplied for the -b command.\n\
+\n\
+-U, -UD\n\
+If the control computer is connected to the Internet running\n\
+\n\
+ bline.py -U\n\
+\n\
+will check to see if there is a newer version of the program than the one\n\
+running. If the response indicates that there is a newer version use the\n\
+-UD command to obtain it. Unzip the file that is downloaded and install\n\
+the new version.\n\
+\n\
+\n\
+bline.py <tagid> <command>\n\
+--------------------------\n\
+-A\n\
+Combines all of the offloaded data file into the file <tagid>.ALL. This\n\
+command may try to create a file larger than the operating system can\n\
+handle, so be careful.\n\
+\n\
+-b\n\
+This command, without any options after the -b, obtains the IP address of\n\
+the control computer and sets the IP address of a baler to a compatible\n\
+address after its ATTN button has been pressed and it finishes booting.\n\
+\n\
+Assigning the IP address to the baler this way records the Ethernet device\n\
+name, IP address used, and the port number to the file <tagid>.set. The\n\
+device name and port number are required for the version of the -c command\n\
+described below, as well as the -s and -x commands. All other commands\n\
+only need the IP address from the .set file (see the '-b <ipaddr>' command\n\
+description below).\n\
+\n\
+-c\n\
+This is a second version of the -c command which uses the actual Ethernet\n\
+device on the control computer that the baler was detected on and returns\n\
+the IP address of that device. The -b command must have been used to set\n\
+the address of the baler for this command to function.\n\
+\n\
+-G, -GD\n\
+These read the offloaded baler files and concatenate all of the files for\n\
+a channel into one channel file. Depending on the operating system this\n\
+may try to create a file larger than the OS can handle, but those days\n\
+are mostly gone. The original files are left in the baler's .sdr directory.\n\
+\n\
+The -G version creates the new files in the baler's .sdr directory. The\n\
+-GD version creates the new files in the directory \"./DATA/\".\n\
+\n\
+Providing a baler TagID will concatenate the files for that one baler.\n\
+Using a baler TagID of 9999 will tell BLINE to look for all of the\n\
+<baler>.sdr directories and perform the concatenation on the data files\n\
+in each baler's directory.\n\
+\n\
+-i\n\
+This command simply establishes a connection with the spcified baler and\n\
+displays the basic version and usage information from the baler.\n\
+\n\
+-l (ell)\n\
+This command requests the list of data files the baler thinks are on its\n\
+internal disk, and displays the list and saves the list to the file\n\
+\n\
+ <tagid>files.txt\n\
+\n\
+This list is also saved every time BLINE is commanded to offload data\n\
+files from a baler.\n\
+\n\
+-L\n\
+This lists the files that are in the <tagid>.sdr directory that have\n\
+presumably been offloaded from the baler.\n\
+\n\
+-s\n\
+This command shuts down the baler. It's usually easier to just use the ATTN\n\
+button. The -b command must have been used to set the baler's IP address\n\
+for this command to work.\n\
+\n\
+-x\n\
+This command cleans the baler and clears the Q330 association. You will\n\
+get one chance to cancel the operation before it starts. While the baler\n\
+is busy \"Waiting...\" will be displayed every 20 seconds. The -b command\n\
+must have been used to set the baler's IP address for this command to work.\n\
+Be careful.\n\
+\n\
+\n\
+bline.py <tagid> <command> [<files>]\n\
+------------------------------------\n\
+-O (big oh)\n\
+This command retrieves the list of files on the baler and then goes through\n\
+the list and offloads all of them. The files are placed in the directory\n\
+<tagid>.sdr. A list of space-separated files may be entered to only offload\n\
+specific files or groups of files using standard UNIX wildcards.\n\
+\n\
+Before offloading the <tagid>.sdr directory is examined and files that\n\
+appear to already be fully offloaded (not partially offloaded) will be\n\
+removed from the download list. A warning stating that some files may be\n\
+overwritten will be shown if partially offloaded files are found. This\n\
+applies to all of the file offloading commands.\n\
+\n\
+-o (small o)\n\
+This command retrieves the list of files on the baler and then offloads\n\
+the files whose channel names, like .HHZ, do not start with the letters\n\
+H or S (as specified in the SEED manual as being 'high speed' channel\n\
+names). The files are placed in the directory <tagid>.sdr. A list of\n\
+space-separated files may be entered to only offload specific files or\n\
+groups of files using standard UNIX wildcards.\n\
+\n\
+-v (little vee) and -vl (vee ell)\n\
+The -v/-vl commands get the list of files from the baler and then look\n\
+to see if all of the files on that list are in the <tagid>.sdr directory\n\
+on the computer and if they are the same size as they are in the list. The\n\
+function does not do any checksum checking, because there's no way to get\n\
+the baler to compute a checksum value of the files on the baler to verify\n\
+against. A <files> list may be supplied if only specified files are to\n\
+be verified. The -vl command additionally lists the files that have not\n\
+been offloaded from the baler.\n\
+\n\
+Both commands then go through the data files on the control computer and\n\
+list files in the <tagid>.sdr directory that are not on the list obtained\n\
+from the baler and which should not be in the .sdr directory.\n\
+\n\
+-V (big V)\n\
+The -V command reads block-by-block through the offloaded data file(s)\n\
+in the <tagid>.sdr directory for the specified baler and collects and\n\
+reports:\n\
+\n\
+ 1. The earliest block header time in a file\n\
+ 2. The latest block header time in a file\n\
+ 3. The number of blocks read, and the block size in a file\n\
+ 4. A list of all of the station IDs in a file (should normally only\n\
+ be one)\n\
+ 5. A list of all of the channel names in a file (should normally only\n\
+ be one)\n\
+\n\
+For baler data files the normal block size is 4096 bytes and there are\n\
+normally 4100 blocks per 16MB data file.\n\
+\n\
+Things to look for after the function runs are bad/scrambled/missing\n\
+dates and times, multiple station names, multiple channel names and the\n\
+program crashing. The latest block time will not normally be the same\n\
+as the last sample time for each channel, because this is only a block-\n\
+level reading. For quiet and low sample rate channels the time may be\n\
+quite a bit earlier than the last sample time.\n\
+\n\
+This function does not indicate partially offloaded files. It only checks\n\
+the integrity of the files.\n\
+\n\
+A summary of information is printed after all of the files have been\n\
+examined.\n\
+\n\
+\n\
+bline.py <tagid> <command> <files>\n\
+----------------------------------\n\
+-e\n\
+The -e command is basically the same as -e, except that it allows specified\n\
+files (in the <files> list) to be excluded from offloading. If a baler\n\
+offload session always fails on a specific file then the -e command and\n\
+that file's name could be specified, so that it will be skipped and the\n\
+rest of the files will be offloaded. The command only offloads the low\n\
+sample rate files.\n\
+\n\
+-E\n\
+Same as -e, but it will try to offload all files that are not in the list\n\
+of files to exclude.\n\
+\n\
+-F\n\
+Only the file(s) specified in the <files> argument will be offloaded.\n\
+This could be used for a different form of only offloading the low sample\n\
+rate data files. A command like\n\
+\n\
+ bline.py 6003 -F \"DT0001*\"\n\
+\n\
+would offload the first file of each channel, which would also include a\n\
+little bit of the high sample rate data, but only the first 16MB file.\n\
+\n\
+\n\
+bline.py <tagid> -M <message>\n\
+-----------------------------\n\
+-M\n\
+Follow the -M with a message that will be displayed and also written to\n\
+the .msg file for the baler. The text may need to be enclosed in quotes:\n\
+\n\
+ bline.py -M \"Baler 5549 is station NUUK\"\n\
+\n\
+on some systems.\n\
+\n\
+\n\
+bline.py <tagid> -b [<ipaddr> | <ethdev>]\n\
+-----------------------------------------\n\
+This is the second form of the -b command that must be used if the baler\n\
+IP address is set using BaleAddr by supplying the <ipaddr> set by BaleAddr.\n\
+This will write the supplied IP address to the file <tagid>.set. This must\n\
+be done before any of the baler-related commands will function.\n\
+\n\
+This form of the -b command may also be used to supply the name of the\n\
+Ethernet device that BLINE should watch when setting the IP address. This\n\
+will be something like en0 or enp1s0. Systems that fully support the\n\
+networking functions BLINE uses will determine this value automatically.\n\
+\n\
+\n\
+END\n"
+ logIt("", "", False)
+ logIt("", HELPText, False)
+ return
+# END: helpLong
+
+
+
+
+####################
+# BEGIN: helpShort()
+# FUNC:helpShort():2019.297
+def helpShort():
+ HELPshort = "bline.py <command>\n\
+ -h = This help.\n\
+ -H = More help.\n\
+ -c = Reports what may be the control computer's IP address and\n\
+ other information.\n\
+ -n = Lists the Ethernet devices on the control computer. May help\n\
+ with the -b command if the device name must be supplied.\n\
+ -U = Checks for a newer program version at PASSCAL if connected\n\
+ to the Internet (the program will eventually be distributed\n\
+ using git and this will go away.\n\
+ -UD = Downloads most recent version from PASSCAL (try -U first).\n\
+\n\
+bline.py <tagid> <command>\n\
+ -A = Copies all of the offloaded files into <tagid>.ALL\n\
+ -c = This is a second version of -c which uses the actual Ethernet\n\
+ device that the -b command found for the <tagid> baler.\n\
+ -G = Copies all of the offloaded files for each channel into a single\n\
+ file in the baler's .sdr directory.\n\
+ -GD = Copies all of the offloaded files for each channel into a single\n\
+ file in the ./DATA directory.\n\
+ -i = Gets basic information from the baler.\n\
+ -l = (ell) Displays and saves the list of files on the baler.\n\
+ -L = Displays the list of files in the baler's .sdr directory.\n\
+ -s = Shuts down the baler.\n\
+ -x = Cleans the baler and clears any Q330 association.\n\
+\n\
+bline.py <tagid> <command> [<files>]\n\
+ -O = (big oh) Offloads all data files that have not been offloaded.\n\
+ -o = (little oh) Offloads low sample rate data files that have not been\n\
+ offloaded.\n\
+ -v = Gets the baler's list of files and checks the offloaded files\n\
+ to see how many are missing or have only been partially\n\
+ offloaded.\n\
+ -vl = (vee ell) Same as -v, except this also lists the files that have\n\
+ not been offloaded.\n\
+ -V = Examines offloaded files for bad blocks.\n\
+\n\
+bline.py <tagid> <command> <files>\n\
+ -e = Excludes the specified file(s) during an offload, otherwise\n\
+ low sample rate files will be offloaded (like -o).\n\
+ -E = Excludes the specified file(s) during an offload, otherwise\n\
+ all files will be offloaded (like -O).\n\
+ -F = Offloads only the specified file(s).\n\
+\n\
+bline.py <tagid> <command> <message>\n\
+ -M = Follow the -M with a text message.\n\
+\n\
+bline.py <tagid> <command> [<ipaddr> | <ethdev>]\n\
+ -b = Watch for the specified baler and assign it an IP address. To\n\
+ make BLINE assign the address leave off the <ipaddr>. If BaleAddr\n\
+ is used enter the <ipaddr> that BaleAddr set the baler to\n\
+ before running other commands. Supply the <ethdev> if the\n\
+ operating system cannot support automatically looking through\n\
+ all Ethernet devices and assigning the IP address to the baler.\n\
+\n\
+<tagid> = The tag ID on the front of the baler.\n\
+<ipaddr> = The IP address assigned to the baler by BLINE or BaleAddr.\n\
+<files> = A list of space-separated file names (can have wildcards).\n\
+\n\
+-E/-e commands are, for example, for excluding \"problem\" files that\n\
+ may be stopping the offload process.\n\
+*, ?, [] UNIX file wildcards may be used in <files>.\n\
+On some systems you may need to enclose file names using wildcards with\n\
+ quotes like \"*.VER\" \"DT0001*\"\n\
+Only one command may be executed per run.\n\
+Always leave a space after the command line switches:\n\
+ -E file, not -Efile\n"
+ logIt("", HELPshort, False)
+ return
+# END: helpShort
+
+
+
+
+#####################
+# BEGIN: helpVShort()
+# FUNC:helpVShort():2019.297
+def helpVShort():
+ logIt("", "%s %s\nPython %s"%(PROG_NAME, PROG_VERSION, PROG_PYVERSION), \
+ False)
+ HELPVshort = "bline.py [ -h | -H | -c | -n | -U | -UD ]\n\
+bline.py <tagid> [ -A | -c | -G | -GD | -i | -l | -L | -s | -x ]\n\
+bline.py <tagid> [ -o | -O | -v | -vl | -V ] [<files>]\n\
+bline.py <tagid> [ -e | -E | -F ] <files>\n\
+bline.py <tagid> -M <message>\n\
+bline.py <tagid> -b [<ipaddr> | <ethdev>]\n"
+ logIt("", HELPVshort, False)
+ return
+# END: helpVShort
+
+
+
+
#################
# BEGIN: intt(In)
# LIB:intt():2018.257
@@ -1079,6 +2168,45 @@ def intt(In):
+#####################
+# BEGIN: isHigh(File)
+# FUNC:isHigh(File):2019.288
+# Returns True if the channel name says the file is a high sample rate file.
+# This is according to the PASSCAL channel naming rules.
+def isHigh(File):
+ if File.find("_.F") == -1 and File.find("_.G") == -1 and \
+ File.find("_.C") == -1 and File.find("_.D") == -1 and \
+ File.find("_.H") == -1 and File.find("_.E") == -1 and \
+ File.find("_.B") == -1 and File.find("_.S") == -1:
+ return False
+ return True
+# END: isHigh
+
+
+
+
+###########################
+# BEGIN: isIPV4Addr(IPAddr)
+# FUNC:isIPAddr(IPV4Addr):2019.224
+# Looks over the passed IPAddr to see if it might be an IPV4 address.
+# A : in the first section and last is also allowed for dev:1.1.1.1:port.
+def isIPV4Addr(IPAddr):
+ Parts = IPAddr.split(".")
+ if len(Parts) != 4:
+ return False
+ for P in [0, 3]:
+ if Parts[P].find(":") != -1:
+ N = Parts[P].split(":")[1]
+ Parts[P] = N
+ for P in arange(0, 3):
+ if len(Parts[P].strip()) == 0:
+ return False
+ return True
+# END: isIPV4Addr
+
+
+
+
########################################################################
# BEGIN: list2Str(TheList, Delim = ", ", Sort = True, DelBlanks = False)
# LIB:list2Str():2018.235
@@ -1136,13 +2264,19 @@ def listSort(InList, Index, How, Direction = ""):
-###################
-# BEGIN: logIt(Msg)
-# FUNC:logIt():2019.018
-def logIt(Msg):
-# Some messages will come with multiple lines (like from LIB functions that
-# also need to work with bgoff.py). Split those up. Some messages will come
-# with (date time) at the end (again, for bgoff.py). Strip those off.
+#########################################
+# BEGIN: logIt(MSGspec, Msg, Time = True)
+# FUNC:logIt():2019.252
+# Set MSGspec to "" to just get message to stdout.
+# Time = Control timestamp at start of message.
+def logIt(MSGspec, Msg, Time = 1):
+ if MSGspec != "":
+ if exists(MSGspec) == False:
+ Fp = open(MSGspec, "w")
+ Fp.close()
+# Some messages will come with multiple lines (like from LIB functions). Split
+# those up. Some messages may come with (date time) at the end. Strip those
+# off.
Lines = Msg.split("\n")
for Index in arange(0, len(Lines)):
# If we happen to go over UT midnight this won't be caught. I'll risk it.
@@ -1151,87 +2285,174 @@ def logIt(Msg):
Lines[Index] = Lines[Index][0:I]
except:
continue
+# Do the stdout stuff and then the file stuff.
for Line in Lines:
if Line != "":
- stdout.write("%s %s\n"%(getGMT(3), Line))
+ if Time == False:
+ stdout.write("%s\n"%Line)
+ else:
+ stdout.write("%s %s\n"%(getGMT(3), Line))
else:
stdout.write("\n")
stdout.flush()
- Fp = open(Msgspec, "a")
- for Line in Lines:
- if Line != "":
- Fp.write("%s %s\n"%(getGMT(3), Line))
- else:
- Fp.write("\n")
- Fp.close()
+ if MSGspec != "":
+# Don't write a bunch of trailing blank lines to the log. They may be in the
+# message to get a bit of space between the message and the command line
+# prompt. Lines may be empty, so try.
+ try:
+ while len(Lines[-1]) == 0:
+ Lines = Lines[:-1]
+ Fp = open(MSGspec, "a")
+ for Line in Lines:
+# Also no beeping in the log.
+ Line = Line.replace("\a", "")
+ if Line != "":
+ if Time == False:
+ Fp.write("%s\n"%Line)
+ else:
+ Fp.write("%s %s\n"%(getGMT(3), Line))
+ else:
+ Fp.write("\n")
+ Fp.close()
+ except IndexError:
+ pass
return
# END: logIt
-##########################################
-# BEGIN: readFileLines2(Fp, Strip = False)
-# LIB:readFileLines2():2019.052
-# This is like reafFileLines() in that the file has to be opened for it, but
-# is also like readFileLinesRB() in the way it splits the lines. It is
-# for things like URL openings and such that may or may not be allowed to
-# be opened "rb".
-# Reads the passed file and returns a List of lines after determining how to
-# split the lines which may end differently depending on which operating
-# system the file was written by.
-# A List of continuous text may also be passed as Fp and that will be split
-# into a List of lines.
-# Since this does a split on the delimiters the lines returned will not have
-# \r\n or \r or \n at the end.
-# It also removes trailing whitespace from the lines.
-# The file should be opened with "r", not "rb", by the caller.
-def readFileLines2(Fp, Strip = False):
+# First day of the month for each non-leap year month MINUS 1. This will get
+# subtracted from the DOY, so a DOY of 91, minus the first day of April 90
+# (91-90) will leave the 1st of April. The 365 is the 1st of Jan of the next
+# year.
+PROG_FDOM = (0, 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365)
+
+#############################
+# BEGIN: q330yd2md(YYYY, DOY)
+# LIB:q330yd2md():2013.023
+# Converts YYYY,DOY to Month, Day. Faster than using using ydhmst2Time().
+# Expects a 4-digit YYYY.
+def q330yd2md(YYYY, DOY):
+ if DOY < 1 or DOY > 366:
+ return 0, 0
+ if DOY < 32:
+ return 1, DOY
+ elif DOY < 60:
+ return 2, DOY-31
+ if YYYY%4 != 0:
+ Leap = 0
+ elif YYYY%100 != 0 or YYYY%400 == 0:
+ Leap = 1
+ else:
+ Leap = 0
+# Check for this special day.
+ if Leap == 1 and DOY == 60:
+ return 2, 29
+# The PROG_FDOM values for Mar-Dec are set up for non-leap years. If it is a
+# leap year and the date is going to be Mar-Dec (it is if we have made it this
+# far), subtract Leap from the day.
+ DOY -= Leap
+# We start through PROG_FDOM looking for dates in March.
+ Month = 3
+ for FDOM in PROG_FDOM[4:]:
+# See if the DOY is less than the first day of next month.
+ if DOY <= FDOM:
+# Subtract the DOY for the month that we are in.
+ return Month, DOY-PROG_FDOM[Month]
+ Month += 1
+ return 0, 0
+# END: q330yd2md
+
+
+
+
+###########################################################
+# BEGIN: readFileLinesRB(Filespec, Strip = False, Bees = 0)
+# LIB:readFileLinesRB():2019.239
+# This is the same idea as readFileLines(), but the Filespec is passed and
+# the file is treated as a 'hostile text file' that may be corrupted. This
+# can be used any time, but was developed for reading Reftek LOG files which
+# can be corrupted, or just have a lot of extra junk added by processing
+# programs.
+# This is based on the method used in rt72130ExtractLogData().
+# The return value is (0, [lines]) if things go OK, or a standard error
+# message if not, except the "e" of the exception also will be returned
+# after the passed Filespec, so the caller can construct their own error
+# message if needed.
+# If Bees is not 0 then that many bytes of the file will be returned and
+# converted to lines. If Bees is less than the size of the file the last
+# line will be discarded since it's a good bet that it will be a partial
+# line.
+# Weird little kludge: Setting Bees to -42 tells the function that Filespec
+# contains a bunch of text and that it should be split up into lines and
+# returned just as if the text had come from reading a file.
+# If Filespec has "http:" or "https:" in it then urlopen() will be used.
+def readFileLinesRB(Filespec, Strip = False, Bees = 0):
Lines = []
- if isinstance(Fp, list) == False and isinstance(Fp, astring) == False:
- if PROG_PYVERS == 2:
- Raw = Fp.read()
- elif PROG_PYVERS == 3:
-# To avoid the b"".
- Raw = Fp.read().decode("latin-1")
-# A blob of loaded by the caller.
- elif isinstance(Fp, astring):
- Raw = Fp
- if isinstance(Fp, list) == False and isinstance(Fp, astring) == False:
+ if Bees != -42:
+ try:
+ if Filespec.find("http:") == -1 and Filespec.find("https:") == -1:
+# These should be text files, but there's no way to know if they are ASCII or
+# Unicode or garbage, especially if they are corrupted, so open binarially.
+ Fp = open(Filespec, "rb")
+ else:
+ Fp = urlopen(Filespec)
+# This will be trouble if the file is huge, but that should be rare. Bees can
+# be used if the file is known to be yuge. This should result in one long
+# string. This and the "rb" above seems to work on Py2 and 3.
+ if Bees == 0:
+ Raw = Fp.read().decode("latin-1")
+ else:
+ Raw = Fp.read(Bees).decode("latin-1")
+ Fp.close()
+ if len(Raw) == 0:
+ return (0, Lines)
+ except Exception as e:
+ try:
+ Fp.close()
+ except:
+ pass
+ return (1, "MW", "%s: Error opening/reading file.\n%s"% \
+ (basename(Filespec), e), 3, Filespec, e)
+ else:
+ Raw = Filespec
+ Filespec = "PassedLines"
# Yes, this is weird. These should be "text" files and in a non-corrupted file
# there should be either all \n or all \r or the same number of \r\n and \n
# and \r. Try and split the file up based on these results.
- RN = Raw.count("\r\n")
- N = Raw.count("\n")
- R = Raw.count("\r")
+ RN = Raw.count("\r\n")
+ N = Raw.count("\n")
+ R = Raw.count("\r")
# Just one line by itself with no delimiter? OK.
- if RN == 0 and N == 0 and R == 0:
- return (0, [Raw])
+ if RN == 0 and N == 0 and R == 0:
+ return (0, [Raw])
# Perfect \n. May be the most popular, so we'll check for it first.
- if N != 0 and R == 0 and RN == 0:
- RawLines = Raw.split("\n")
+ if N != 0 and R == 0 and RN == 0:
+ RawLines = Raw.split("\n")
# Perfect \r\n file. We checked for RN=0 above.
- elif RN == N and RN == R:
- RawLines = Raw.split("\r\n")
+ elif RN == N and RN == R:
+ RawLines = Raw.split("\r\n")
# Perfect \r.
- elif R != 0 and N == 0 and RN == 0:
- RawLines = Raw.split("\r")
- else:
+ elif R != 0 and N == 0 and RN == 0:
+ RawLines = Raw.split("\r")
+ else:
# There was something in the file, so make a best guess based on the largest
# number. It might be complete crap, but what else can we do?
- if N >= RN and N >= R:
- RawLines = Raw.split("\n")
- elif N >= RN and N >= R:
- RawLines = Raw.split("\r\n")
- elif R >= N and R >= RN:
- RawLines = Raw.split("\n")
+ if N >= RN and N >= R:
+ RawLines = Raw.split("\n")
+ elif N >= RN and N >= R:
+ RawLines = Raw.split("\r\n")
+ elif R >= N and R >= RN:
+ RawLines = Raw.split("\n")
# If all of those if's couldn't figure it out.
- else:
- return (1, "RW", "%s: Unrecognized file format."% \
- basename(Filespec), 2, Filespec)
-# The caller must have loaded the individual lines and passed them.
- elif isinstance(Fp, list):
- RawLines = Fp
+ else:
+ return (1, "RW", "%s: Unrecognized file format."% \
+ basename(Filespec), 2, Filespec)
+# If Bees is not 0 then throw away the last line if the file is larger than
+# the number of bytes requested.
+ if Bees != 0 and Bees < getsize(Filespec):
+ RawLines = RawLines[:-1]
# Get rid of trailing empty lines. They can sneak in from various places
# depending on who wrote the file. Do the strip in case there are something
# like leftover \r's when \n was used for splitting.
@@ -1241,7 +2462,7 @@ def readFileLines2(Fp, Strip = False):
if len(RawLines) == 0:
return (0, Lines)
# If the caller doesn't want anything else then just go through and get rid of
-# any trailing whitespace, else get rid of all the whitespace.
+# any trailing spaces, else get rid of all the spaces.
if Strip == False:
for Line in RawLines:
Lines.append(Line.rstrip())
@@ -1249,7 +2470,7 @@ def readFileLines2(Fp, Strip = False):
for Line in RawLines:
Lines.append(Line.strip())
return (0, Lines)
-# END: readFileLines2
+# END: readFileLinesRB
@@ -1267,645 +2488,930 @@ def sP(Count, Phrases):
-#############################################
-# BEGIN: writeFilesTxt(Msg, Filesspec, Files)
-# LIB:writeFilesTxt():2019.018
-def writeFilesTxt(Msg, Filesspec, Files):
- global BState
+########################################################
+# BEGIN: writeFilesTxt(MSGspec, TagID, Filesspec, Files)
+# LIB:writeFilesTxt():2019.233
+# Writes the list of files obtained by getFilesHtm() to the passed Filespec.
+def writeFilesTxt(MSGspec, TagID, Filesspec, Files):
FilesCount = 0
FilesBytes = 0
try:
Fp = open(Filesspec, "w")
FilesCount = 0
+ Fp.write("Files on baler %s at %s:\n"%(TagID, getGMT(3)))
for File in Files:
FilesCount += 1
- Fp.write("%d. %s %s to %s %d\n"%(FilesCount, File[FH_NAME], \
- File[FH_FROM], File[FH_TO], File[FH_SIZE]))
- FilesBytes += File[FH_SIZE]
+ Fp.write("%d. %s %s to %s %d\n"%(FilesCount, File[B_NAME], \
+ File[B_FROM], File[B_TO], File[B_SIZE]))
+ FilesBytes += File[B_SIZE]
Fp.close()
except Exception as e:
- return (1, "MW", "Error writing file list.\n%s\nStopped. (%s)"%( e, \
- getGMT(3)), 3)
- if BState == STATE_STOP:
- return (1, "YB", "Stopped by user. (%s)"%getGMT(3), 2)
- if Msg == "bg":
- msgLn(1, "W", "Wrote file list to %s"%Filesspec)
- msgLn(0, "", "%s %s, %s %s on baler."%(fmti(FilesCount), \
- sP(FilesCount, ("file", "files")), fmti(FilesBytes), \
- sP(FilesBytes, ("byte", "bytes"))))
- elif Msg == "bl":
- logIt("Wrote file list to %s"%Filesspec)
- logIt("%s %s, %s %s on baler."%(fmti(FilesCount), sP(FilesCount, \
- ("file", "files")), fmti(FilesBytes), sP(FilesBytes, \
- ("byte", "bytes"))))
+ return (1, "MW", "Error writing file list.\n%s"%e, 3)
+ logIt(MSGspec, "Wrote file list to: %s"%Filesspec)
+ logIt(MSGspec, "%s %s, %s %s on baler."%(fmti(FilesCount), sP(FilesCount, \
+ ("file", "files")), fmti(FilesBytes), sP(FilesBytes, ("byte", \
+ "bytes"))))
return (0, FilesCount, FilesBytes)
# END: writeFilesTxt
-# First day of the month for each non-leap year month MINUS 1. This will get
-# subtracted from the DOY, so a DOY of 91, minus the first day of April 90
-# (91-90) will leave the 1st of April. The 365 is the 1st of Jan of the next
-# year.
-PROG_FDOM = (0, 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365)
-
-#############################
-# BEGIN: q330yd2md(YYYY, DOY)
-# LIB:q330yd2md():2013.023
-# Converts YYYY,DOY to Month, Day. Faster than using using ydhmst2Time().
-# Expects a 4-digit YYYY.
-def q330yd2md(YYYY, DOY):
- if DOY < 1 or DOY > 366:
- return 0, 0
- if DOY < 32:
- return 1, DOY
- elif DOY < 60:
- return 2, DOY-31
- if YYYY%4 != 0:
- Leap = 0
- elif YYYY%100 != 0 or YYYY%400 == 0:
- Leap = 1
- else:
- Leap = 0
-# Check for this special day.
- if Leap == 1 and DOY == 60:
- return 2, 29
-# The PROG_FDOM values for Mar-Dec are set up for non-leap years. If it is a
-# leap year and the date is going to be Mar-Dec (it is if we have made it this
-# far), subtract Leap from the day.
- DOY -= Leap
-# We start through PROG_FDOM looking for dates in March.
- Month = 3
- for FDOM in PROG_FDOM[4:]:
-# See if the DOY is less than the first day of next month.
- if DOY <= FDOM:
-# Subtract the DOY for the month that we are in.
- return Month, DOY-PROG_FDOM[Month]
- Month += 1
- return 0, 0
-# END: q330yd2md
+###############################################
+# BEGIN: logHeader(MSGspec, Which, Time = True)
+# FUNC:logHeader():2019.254
+def logHeader(MSGspec, Which, Time = True):
+ logIt(MSGspec, " ", False)
+ logIt(MSGspec, "Command: %s"%args2SL("s", 0), Time)
+ if "1" in Which:
+ logIt(MSGspec, "%s %s"%(PROG_NAME, PROG_VERSION), Time)
+ if "2" in Which:
+ logIt(MSGspec, "Python version: %s"%PROG_PYVERSION, Time)
+ if "3" in Which:
+ logIt(MSGspec, "Working directory: %s"%ABSCWDspec, Time)
+ logIt(MSGspec, "Writing messages to: %s"%MSGspec, Time)
+ return
+# END: logHeader
-##############################
-# BEGIN: checkDTFile(Filespec)
-# FUNC:checkDTFile():2019.018
-# Reads the mini-seed file block headers to see if they are corrupted.
-# Note Ret[0] values, so the caller can keep tabs. Also the return value
-# has the file timerange added to the end.
-def checkDTFile(Filespec):
- Iam = stack()[0][3]
-# Keep a list of station IDs that we come across. Should be just one.
- StationIDs = []
-# Keep a list of the channel names we find (should be just one for balers).
- Channels = []
-# The caller should be smarter than this...but maybe not.
- if isdir(Filespec):
- return (8, "RW", "File %s is a directory."%basename(Filespec), 2, "", \
- "")
- try:
- Fp = open(Filespec, "rb")
- except Exception as e:
- return (1, "MW", str(e).strip(), 3, "", "")
-# The standard record size for baler files is 4K. Read one and determine if
-# it is smaller.
- Record = Fp.read(4096+256)
- RecordLen = len(Record)
- if RecordLen == 0:
- return (0, "GB", "File empty.", 0, "", "")
- StaID = Record[8:13].decode("latin-1")
- if StaID == Record[8+256:13+256].decode("latin-1"):
- RecordSize = 256
- elif StaID == Record[8+512:13+512].decode("latin-1"):
- RecordSize = 512
- elif StaID == Record[8+1024:13+1024].decode("latin-1"):
- RecordSize = 1024
- elif StaID == Record[8+2048:13+2048].decode("latin-1"):
- RecordSize = 2048
- elif StaID == Record[8+4096:13+4096].decode("latin-1"):
- RecordSize = 4096
- else:
- return (1, "MW", "Could not determine record size.", 3, "", "")
- Fp.seek(0)
-# Read the file in 10 record chunks to make it faster.
- RecordsSize = RecordSize*10
- RecordsRead = 0
- FirstTime = "Z"
- LastTime = ""
-# Some may use this for messages.
- NoOfRecords = int(getsize(Filespec)/RecordSize)
- while 1:
- Records = Fp.read(RecordsSize)
-# We're done with this file.
- if len(Records) == 0:
- break
- RecordsRead += 10
- for i in arange(0, 10):
- Ptr = RecordSize*i
- Record = Records[Ptr:Ptr+RecordSize]
-# Need another set of Records.
- if len(Record) < RecordSize:
- break
- ChanID = Record[15:18].decode("latin-1")
-# The Q330 may create an "empty" file (all 00H) and then not finish filling it
-# in. The read()s keep reading, but there's nothing to process. This detects
-# that and returns. This may only happen in .bms-type data.
- if ChanID == "\x00\x00\x00":
-# I guess if a file is scrambled these could have 1000's of things in them.
-# Both of them should only have one thing for baler files. Convert them to
-# strings and then chop them off if they are long.
- Stas = list2Str(StationIDs)
- if len(Stas) > 12:
- Stas = Stas[:12]+"..."
- Chans = list2Str(Channels)
- if len(Chans) > 12:
- Chans = Chans[:12]+"..."
- return (4, "YB", \
- "FILE ENDS EMPTY. Times: %s to %s\n Recs: %d(%d) Stas: %s Chans: %s"% \
- (FirstTime, LastTime, RecordsRead, RecordSize, Stas, \
- Chans), 2, FirstTime, LastTime)
- if ChanID not in Channels:
- Channels.append(ChanID)
- StaID = Record[8:13].decode("latin-1").strip()
- if StaID not in StationIDs:
- StationIDs.append(StaID)
- Year, Doy, Hour, Mins, Secs, Tttt= unpack(">HHBBBxH", \
- Record[20:30])
- Month, Date = q330yd2md(Year, Doy)
- DateTime = "%d-%02d-%02d %02d:%02d"%(Year, Month, Date, Hour, Mins)
- if DateTime < FirstTime:
- FirstTime = DateTime
- if DateTime > LastTime:
- LastTime = DateTime
- Fp.close()
- if RecordsRead == 0:
- return (5, "YB", "NO DATA.", 2, "", "")
-# Same as above.
- Stas = list2Str(StationIDs)
- if len(Stas) > 12:
- Stas = Stas[:12]+"..."
- Chans = list2Str(Channels)
- if len(Chans) > 12:
- Chans = Chans[:12]+"..."
- if len(StationIDs) > 1:
- return (6, "MB", \
- "MULTIPLE STATIONS. Times: %s to %s\n Recs: %d(%d) Stas: %s Chans: %s"% \
- (FirstTime, LastTime, RecordsRead, RecordSize, Stas, Chans), \
- 3, FirstTime, LastTime)
- elif len(Channels) > 1:
- return (7, "MB", \
- "MULTIPLE CHANNELS. Times: %s to %s\n Recs: %d(%d) Stas: %s Chans: %s"% \
- (FirstTime, LastTime, RecordsRead, RecordSize, Stas, Chans), \
- 3, FirstTime, LastTime)
- else:
- return (0, "GB", \
- "FILE OK. Times: %s to %s\n Recs: %d(%d) Stas: %s Chans: %s"% \
- (FirstTime, LastTime, RecordsRead, RecordSize, Stas, Chans), 1, \
- FirstTime, LastTime)
-# END: checkDTFile
+###############
+# BEGIN: main()
+# FUNC:main():2019.297
+# These lovely items are brought to you by Microsoft. Python 3 was keeping
+# ' -#' for arguments, "" as an argument, or not passing command line
+# arguments at all. It's related to the registry value
+# HKEY_CLASSES_ROOT\Applications\python.exe\shell\open\command
+# and not having a %* (no quotes around it) at the end of the Data command
+# to get Python to pass command line arguments when using 'bline.py <args>'
+# to start the program. 'python bline.py <args>' seems OK. It's a Python
+# installation problem. Don't know when all of this started. It was fine Dec
+# 2018. Feb 2019 was not. It might be an older bug that has shown back up.
+# This goes through the arguments and cleans them up.
+argv2 = []
+Index = 0
+for Arg in argv:
+ argv[Index] = argv[Index].strip()
+ if len(argv[Index]) != 0:
+ argv2.append(argv[Index])
+ Index += 1
+argv = argv2
+
+CWDspec = ".%s"%sep
+ABSCWDspec = abspath(CWDspec)
+if ABSCWDspec.endswith(sep) == False:
+ ABSCWDspec += sep
+SDRspec = ""
+MSGspec = ""
+SETspec = ""
+FDev = ""
+FIPAddr = ""
+FPNumber = 0
+CLTagID = ""
+
+# At least say something!
+if len(argv) == 1:
+ helpVShort()
+ exit(0)
+# Import Central. The "new" commands need non-standard Python modules. If any
+# of these commands are used I want the program to list all of the modules
+# that are not installed for any of these commands, or just go ahead and
+# import them. The 'IP supplied' version of the -b command, and the -c command
+# throw a bit of a monkey wrench into keeping this simple.
+if "-b" in argv or "-c" in argv or "-s" in argv or "-x" in argv:
+ if len(argv) == 4 and argv[2] == "-b" and isIPV4Addr(argv[3]) == True:
+ pass
+ elif len(argv) == 2 and argv[1] == "-c":
+ pass
+ else:
+ OK = True
+ try:
+ from psutil import net_if_addrs
+ except ImportError:
+ logIt("", \
+ "The module 'psutil' will need to be installed on this system.", \
+ False)
+ OK = False
+ try:
+ from ipaddress import IPv4Network, ip_interface
+ except ImportError:
+ logIt("", \
+ "The module 'ipaddress' will need to be installed on this system.", \
+ False)
+ OK = False
+ if PROG_PYVERS == 2:
+ try:
+ import subprocess32 as sb
+ except ImportError:
+ logIt("", \
+ "The module 'subprocess32' needs to be installed on this system.", \
+ False)
+ OK = False
+ elif PROG_PYVERS == 3:
+ import subprocess as sb
+# This is the only OS that will make use of this.
+ if PROGSystem == "lin":
+ try:
+ from pexpect import spawn
+ except ImportError:
+ logIt("", \
+ "The module 'pexpect' will need to be installed on this system.", \
+ False)
+ OK = False
+ if OK == False:
+ logIt("", "", False)
+ exit(1)
+# I'm repeating this check for -n, because this may be useful without using
+# -b,-c,-s,-x in the BaleAddr mode. There's no need to make the user install
+# all of those other modules.
+if "-n" in argv:
+ try:
+ from psutil import net_if_addrs
+ except ImportError:
+ logIt("", \
+ "The module 'psutil' will need to be installed on this system.\n", \
+ False)
+ exit(1)
+#==========
+# The simple bline.py <command> commands.
+#==========
+#===== bline.py -# =====#
+#===== bline.py -a =====#
+#===== bline.py -h =====#
+#===== bline.py -H =====#
+#===== bline.py -U =====#
+#===== bline.py -UD =====#
+#===== bline.py -c =====#
-###############
-# BEGIN: main()
-# FUNC:main():2019.064
-# All of the action takes place here in one pass using if()'s.
-FH_NAME = 0
-FH_SIZE = 1
-FH_FROM = 2
-FH_TO = 3
-Dirspec = ".%s"%sep
-AbsDirspec = abspath(Dirspec)
-# LATER.
-#if ArgBader == True:
-# Ret = address_balers()
-# logIt(Ret)
-# exit(0)
-if AbsDirspec.endswith(sep) == False:
- AbsDirspec += sep
-if ArgUCheck == True:
- stdout.write("%s %s\n"%(PROG_NAME, PROG_VERSION))
+# These commands are pretty straightforward and don't require any current
+# directory, file or address information.
+if "-#" in argv:
+ logIt("", "%s"%PROG_VERSION, False)
+ exit(0)
+if argv[1] == "-a":
+ logIt("", "", False)
+ logIt("", PROG_LONGNAME, False)
+ logIt("", "Version %s"%PROG_VERSION, False)
+ logIt("", "PASSCAL Instrument Center", False)
+ logIt("", "Socorro, New Mexico USA", False)
+ logIt("", "", False)
+ logIt("", "Email: passcal@passcal.nmt.edu", False)
+ logIt("", "Phone: 575-835-5070", False)
+ logIt("", "", False)
+ logIt("", "Python: %s\n"%PROG_PYVERSION, False)
+ exit(0)
+if argv[1] == "-c":
+ logHeader("", "1")
+ logIt("", "Control computer host name: %s"%socket.gethostname(), False)
+ logIt("", "Contol computer IP address: %s (maybe)\n"% \
+ socket.gethostbyname(socket.gethostname()), False)
+ exit(0)
+if "-h" in argv:
+ helpShort()
+ exit(0)
+if "-H" in argv:
+ helpLong()
+ exit(0)
+if "-n" in argv:
+ logIt("", "\nEnabled network device/interface names on this system:", \
+ False)
+ Nis = net_if_addrs()
+ Keys = list(Nis.keys())
+ Keys.sort()
+ for Ni in Keys:
+ logIt("", " %s"%Ni, False)
+ logIt("", "", False)
+ exit(0)
+if "-U" in argv:
+ logHeader("", "1", False)
Ret = checkForUpdates("check")
exit(Ret)
-if ArgUDown == True:
+if "-UD" in argv:
+ logHeader("", "1", False)
Ret = checkForUpdates("get")
exit(Ret)
-DirspecSDR = ".%s%s.sdr%s"%(sep, TagID, sep)
-Msgspec = Dirspec+TagID+".msg"
-# Duplicating some stuff so the messages come out the way I want them to.
-if exists(Msgspec) == False:
- Fp = open(Msgspec, "w")
- Fp.close()
- if WriteMsg == True:
- logIt(TheMessage)
+# There are no more short 'bline -?' commands beyond this point.
+if len(argv) < 3:
+ logIt("", "What??\a\n", False)
+ exit(1)
+
+# Everything from here on uses the command line TagID in the first position, so
+# make sure the user has entered what looks like a valid TagID.
+CLTagID = argv[1]
+# Zeros are on the baler tags, but I don't think any software uses them.
+while CLTagID.startswith("0"):
+ CLTagID = CLTagID[1:]
+if len(CLTagID) == 0:
+ logIt("", "TagID was just zeros?\a\n", False)
+ exit(1)
+try:
+ Value = int(CLTagID)
+except:
+ logIt("", "TagID '%s' must be just numbers.\a\n"%CLTagID, False)
+ exit(1)
+# Now make these. Some commands will use some of these and some won't.
+SDRspec = ".%s%s.sdr%s"%(sep, CLTagID, sep)
+MSGspec = CWDspec+CLTagID+".msg"
+SETspec = CWDspec+CLTagID+".set"
+TXTFspec = CWDspec+CLTagID+"files.txt"
+
+# These commands just take care of themselves and then quit.
+#===== bline.py <tagid> -A =====#
+#===== bline.py <tagid> -c =====#
+#===== bline.py <tagid> -G =====#
+#===== bline.py <tagid> -GD =====#
+#===== bline.py <tagid> -M <message> =====#
+
+if argv[2] == "-A":
+ logHeader(MSGspec, "")
+ Files = glob("%s*__.*"%SDRspec)
+ if len(Files) == 0:
+ logIt(MSGspec, "No offloaded files found.")
+ logIt(MSGspec, "Are you in the directory above the .sdr directory?\n")
exit(0)
- logIt("%s %s"%(PROG_NAME, PROG_VERSION))
- logIt("Command:%s"%args2Str(0))
- logIt("Working directory: %s"%AbsDirspec)
- logIt("Created %s"%Msgspec)
-else:
- if WriteMsg == True:
- logIt(TheMessage)
+ logIt(MSGspec, "Files to copy: %d"%len(Files))
+# Get the list of channels from the offloded files.
+ Chans = []
+ for File in Files:
+ Chan = File[-4:]
+ if Chan not in Chans:
+ Chans.append(Chan)
+ Chans.sort()
+ Files.sort()
+ from shutil import copyfileobj
+ Count = 0
+ try:
+ OutFile = "%s%s.ALL"%(SDRspec, CLTagID)
+ FpA = open(OutFile, "wb")
+ logIt(MSGspec, "Copying files to %s..."%OutFile)
+ for Chan in Chans:
+ for File in Files:
+ if File.endswith(Chan):
+ Count += 1
+ logIt(MSGspec, " %d. Copying file %s..."%(Count, \
+ basename(File)), False)
+ Fp = open(File, "rb")
+ copyfileobj(Fp, FpA, -1)
+ Fp.close()
+ FpA.close()
+ except KeyboardInterrupt:
+ try:
+ Fp.close()
+ except:
+ pass
+ try:
+ FpA.close()
+ except:
+ pass
+ logIt(MSGspec, "Stopped by user.")
exit(0)
- logIt("%s %s"%(PROG_NAME, PROG_VERSION))
- logIt("Command:%s"%args2Str(0))
- logIt("Working directory: %s"%AbsDirspec)
- logIt("Using %s"%Msgspec)
-# Hijack and detour the program for this.
-if ArgBadBlocks == True:
- if exists(DirspecSDR) == False:
- logIt("Done checking. No files have been offloaded.")
- logIt("")
+ logIt(MSGspec, "Finished.")
+ exit(0)
+
+
+# This is a second form of the -c command that will use the actual Ethernet
+# device for the entered baler if BLINE's -b command was used to set the baler
+# address.
+if argv[2] == "-c":
+ logHeader(MSGspec, "")
+# It has to call this here, because the community call is below this.
+ Ret = getSetSettings(SETspec, "get")
+ if Ret[0] != 0:
+ logIt(MSGspec, "%s\n"%Ret[2])
+ exit(1)
+ FDev = Ret[1]
+ if FDev == "":
+ logIt(MSGspec, "%s.set file does not contain the Ethernet device.\n"% \
+ CLTagID)
exit(0)
- DFiles = listdir(DirspecSDR)
- DFiles.sort()
- FilesChecked = 0
- FilesOK = 0
- FilesOKSize = 0
-# These get loaded with the file names depending on the Ret[0] value coming
-# from checkDTFile() for a summary.
- FilesOpenErrors1 = []
- FilesTooSmall2 = []
- FilesRecSize3 = []
- FilesEndEmpty4 = []
- FilesNoData5 = []
- FilesMultStas6 = []
- FilesMultChans7 = []
- FilesIsDir8 = []
-# For an overall baler time range.
- FirstFilesTime = "Z"
- LastFilesTime = ""
-# Loop through DFiles and then through all of the file(s) the user specified
-# looking for matches.
- for Index in arange(0, len(DFiles)):
- DFile = DFiles[Index]
- Matches = False
- for ArgSpecFile in ArgSpecFiles:
- if fnmatch(DFile, ArgSpecFile) == True:
- Matches = True
- break
- if Matches == False:
- continue
- DFilespec = DirspecSDR+DFile
-# checkDTFile() will also catch this. We'll still look for Rec[0]==8. Non-data
-# files will probably come back with a Rec Size error.
- if isdir(DFilespec):
- continue
- FilesChecked += 1
- logIt("%d. Checking %s..."%(FilesChecked, DFilespec))
- Ret = checkDTFile(DFilespec)
- logIt(" "+Ret[2])
- FirstTime = Ret[4]
- LastTime = Ret[5]
-# If something goes wrong the returned times will be "".
- if FirstTime != "" and FirstTime < FirstFilesTime:
- FirstFilesTime = FirstTime
- if LastTime != "" and LastTime > LastFilesTime:
- LastFilesTime = LastTime
- if Ret[0] == 0:
- FilesOK += 1
- FilesOKSize += getsize(DFilespec)
- elif Ret[0] == 1:
- FilesOpenErrors1.append(DFile)
- elif Ret[0] == 2:
- FilesTooSmall2.append(DFile)
- elif Ret[0] == 3:
- FilesRecSize3.append(DFile)
- elif Ret[0] == 4:
- FilesEndEmpty4.append(DFile)
- elif Ret[0] == 5:
- FilesNoData5.append(DFile)
- elif Ret[0] == 6:
- FilesMultStas6.append(DFile)
- elif Ret[0] == 7:
- FilesMultChans7.append(DFile)
- elif Ret[0] == 8:
- FilesIsDir8.append(DFile)
- logIt("Done checking.")
- logIt("Summary:")
- logIt("Overall date range: %s to %s"%(FirstFilesTime, LastFilesTime))
- logIt(" Files checked: %d"%FilesChecked)
- logIt(" Files OK: %d (%s %s)"%(FilesOK, fmti(FilesOKSize), \
- sP(FilesOKSize, ("byte", "bytes"))))
- if len(FilesOpenErrors1) != 0:
- logIt("File opening errors: %d"%len(FilesOpenErrors1))
- Count = 0
- for File in FilesOpenErrors1:
- Count += 1
- logIt(" %d. %s"%(Count, File))
- if len(FilesTooSmall2) != 0:
- logIt("Files too small: %d"%len(FilesTooSmall2))
- Count = 0
- for File in FilesTooSmall2:
- Count += 1
- logIt(" %d. %s"%(Count, File))
- if len(FilesRecSize3) != 0:
- logIt("Unknown record size: %d"%len(FilesRecSize3))
- Count = 0
- for File in FilesRecSize3:
- Count += 1
- logIt(" %d. %s"%(Count, File))
- if len(FilesEndEmpty4) != 0:
- logIt("Ending empty: %d"%len(FilesEndEmpty4))
- Count = 0
- for File in FilesEndEmpty4:
- Count += 1
- logIt(" %d. %s"%(Count, File))
- if len(FilesNoData5) != 0:
- logIt("No data: %d"%len(FilesNoData5))
- Count = 0
- for File in FilesNoData5:
- Count += 1
- logIt(" %d. %s"%(Count, File))
- if len(FilesMultStas6) != 0:
- logIt("Multiple stations: %d"%len(FilesMultStas6))
- Count = 0
- for File in FilesMultStas6:
- Count += 1
- logIt(" %d. %s"%(Count, File))
- if len(FilesMultChans7) != 0:
- logIt("Multiple channels: %d"%len(FilesMultChans7))
- Count = 0
- for File in FilesMultChans7:
- Count += 1
- logIt(" %d. %s"%(Count, File))
- elif len(FilesIsDir8) != 0:
- logIt("Directories: %d"%len(FilesIsDir8))
+ Ret = ipFromDevice(FDev)
+ if Ret[0] != 0:
+ logIt(MSGspec, "%s\n"%Ret[2])
+ exit(0)
+ IPMask = Ret[1]
+ IP, Mask = ("%s"%IPMask).split("/")
+ logIt(MSGspec, "Control computer Ethernet device: %s"%FDev)
+ logIt(MSGspec, "Control computer IP address: %s"%IP)
+ logIt(MSGspec, "Control computer netmask: %s\n"%bitsToNetmask(Mask))
+ exit(0)
+
+if argv[2] == "-G" or argv[2] == "-GD":
+ from shutil import copyfileobj
+ from struct import unpack
+ logHeader(MSGspec, "")
+ if CLTagID != "9999":
+# Make a TagIDDirs entry to fool the code below.
+ TagIDDirs = [".%s%s.sdr"%(sep, CLTagID)]
+ elif CLTagID == "9999":
+ TagIDDirs = glob(".%s*.sdr"%sep)
+ if len(TagIDDirs) == 0:
+# ID 9999.msg will be fine.
+ logIt(MSGspec, "No offloaded files found.")
+ logIt(MSGspec, \
+ "Are you in the directory above the .sdr directories?\n")
+ exit(0)
+ for TagID in TagIDDirs:
+ TagID = TagID.split(".sdr")[0]
+ TagID = TagID[2:]
+# Rebuild these for each TagID. Won't be necessary for -GD, but...
+ SDRspec = ".%s%s.sdr%s"%(sep, TagID, sep)
+ if argv[2] == "-G":
+ DATAspec = SDRspec
+ elif argv[2] == "-GD":
+ DATAspec = ".%sDATA%s"%(sep, sep)
+ if exists(DATAspec) == False:
+ makedirs(DATAspec)
+# We'll be kinda specific so we don't have to keep filtering below.
+ Files = glob("%s*__.*"%SDRspec)
+ if len(Files) == 0:
+ logIt(MSGspec, "Looking in %s"%SDRspec)
+ logIt(MSGspec, "No offloaded files found.")
+ logIt(MSGspec, \
+ "Are you in the directory above the .sdr directory?\n")
+ exit(0)
+# Get the list of channels from the offloded files.
+ Chans = []
+ for File in Files:
+ Chan = File[-4:]
+ if Chan not in Chans:
+ Chans.append(Chan)
+ logIt(MSGspec, "Channels to group: %d"%len(Chans))
+ Chans.sort()
+ Files.sort()
Count = 0
- for File in FilesIsDir8:
- Count += 1
- logIt(" %d. %s"%(Count, File))
- logIt("")
+ try:
+ for Chan in Chans:
+# We need to go through the channel files, find the first one, open it and
+# extract the station name, net code, etc. for the file name. If this set of
+# data files has multiple of any of the file name items then all bets are off.
+# This only reads the first header.
+ for File in Files:
+ if File.endswith(Chan):
+ Fp = open(File, "rb")
+ Header = Fp.read(128)
+ Fp.close()
+ if PROG_PYVERS == 2:
+ Qual = Header[6]
+ elif PROG_PYVERS == 3:
+ Qual = chr(Header[6])
+ StaID = Header[8:13].strip().decode("latin-1")
+ LocID = Header[13:15].strip().decode("latin-1")
+ ChanID = Header[15:18].strip().decode("latin-1")
+ NetCode = Header[18:20].strip().decode("latin-1")
+ Year, Doy, Hour, Mins, Secs, \
+ Tttt= unpack(">HHBBBxH", Header[20:30])
+ STime = "%s.%03d.%02d%02d%02d"%(Year, Doy, Hour, \
+ Mins, Secs)
+# This is [another] one of many versions of file names.
+ OutFile = "%s%s.%s.%s.%s.%s.%s"%(DATAspec, StaID, \
+ NetCode, LocID, ChanID, Qual, STime)
+ FpG = open(OutFile, "wb")
+ break
+ Count += 1
+ logIt(MSGspec, " %d. Copying files to %s"%(Count, OutFile), \
+ False)
+ for File in Files:
+ if File.endswith(Chan):
+ Fp = open(File, "rb")
+ copyfileobj(Fp, FpG, -1)
+ Fp.close()
+ FpG.close()
+ except KeyboardInterrupt:
+ try:
+ Fp.close()
+ except:
+ pass
+ try:
+ FpG.close()
+ except:
+ pass
+ logIt(MSGspec, "Stopped by user.")
+ exit(0)
+ logIt(MSGspec, "Finished.")
+ exit(0)
+
+# The user may want to put a message into the .msg file for the baler before
+# doing anything else (service run info, site ID, etc.), so check for this
+# command before the exists() checks below. This will get the .msg file
+# created if it does not exist.
+if argv[2] == "-M":
+ CLMessage = args2SL("s", 3)
+ if CLMessage == "":
+ logIt("", "No -M message entered.\a\n", False)
+ beep()
+ exit(1)
+ logIt(MSGspec, "\n%s\n"%CLMessage)
exit(0)
-# All other commands.
-# Get the basic info and make sure we are all on the same page. Do this each
-# time we do anything.
-Ret = getBalerHtm("bl", TagID, IPAddr)
-if Ret[0] != 0:
- logIt(Ret[2])
- logIt("")
+
+#========== Now do the rest of the setups ==========#
+
+# The -b command is used to find balers and save IP addresses, so skip this
+# check for that command.
+if argv[2] != "-b" and exists(SDRspec) == False and \
+ exists(MSGspec) == False and exists(SETspec) == False:
+ logIt(MSGspec, "TagID '%s' is unknown.\a\n"%CLTagID)
exit(1)
-# Might as well check for this before we go to the trouble of getting the list
-# of files (which could take a long time).
-if ArgVerify == True:
- if exists(DirspecSDR) == False:
- logIt("Done verifying. No files have been offloaded.")
- logIt("")
+
+# Commands that expect there to be offloaded files. There may still be a
+# directory, but no files. The commands will have to handle that.
+if argv[2] in ("-v", "-vl", "-V"):
+ if exists(SDRspec) == False:
+ logIt("", "No baler files have been offloaded.\n", False)
exit(0)
-FWVers = Ret[1]
-DSize = Ret[2]
-NFiles = Ret[3]
-Percent = Ret[4]
-if ArgCheck == True:
- Used = (Percent/100.0)*DSize
- logIt("FWVers: %s DiskSize: %s"%(FWVers, fmti(DSize)))
- logIt("%.1f%% of %s %s used."%(Percent, fmti(NFiles), sP(NFiles, \
- ("file", "files"))))
- logIt("Done checking.")
- logIt("")
- exit(0)
-# A List of Lists of the data files on the baler according to files.htm.
-FHFiles = []
+
+# The commands that need a simple header before the other prep stuff below.
+if argv[2] in ("-L", "-s", "-x"):
+ logHeader(MSGspec, "")
+
+# The commands that need a little more header info.
+if argv[2] in ("-e", "-E", "-F", "-i", "-l", "-o", "-O", "-v", "-vl", "-V"):
+ logHeader(MSGspec, "3")
+
+# Get what should be the baler's comm info for these commands.
+if argv[2] in ("-e", "-E", "-F", "-i", "-l", "-o", "-O", "-s", "-v", "-vl", \
+ "-x"):
+ Ret = getSetSettings(SETspec, "get")
+ if Ret[0] != 0:
+ logIt(MSGspec, "%s\n"%Ret[2])
+ exit(1)
+ FDev = Ret[1]
+ FIPAddr = Ret[2]
+ FPNumber = intt(Ret[3])
+
+# Throw this check in here, since there will be no reason to continue.
+ if argv[2] in ("-s", "-x"):
+ if FDev == "" or FPNumber == 0:
+ logIt(MSGspec, \
+ "%s.set file does not contain the Ethernet device/port"% \
+ CLTagID)
+ logIt(MSGspec, "information required for this command.\n")
+ exit(0)
+
+# The commands that might use a list of command line files even just *.
+if argv[2] in ("-o", "-O", "-v", "-vl", "-V"):
+ IFiles = args2SL("l", argv[2])
+ if len(IFiles) == 0:
+ IFiles.append("*")
+
+# Commands that have to have some command line files specified and not just *.
+if argv[2] in ("-e", "-E", "-F"):
+ IFiles = args2SL("l", argv[2])
+ if len(IFiles) == 0:
+ logIt(MSGspec, "No %s files specified.\n"%argv[2])
+ exit(1)
+ if "*" in IFiles:
+ logIt(MSGspec, "The wildcard * is not allowed for this command.")
+ exit(1)
+
+# Check that the entered TagID and IP address go together and try to get the
+# general info from the baler.
+if argv[2] in ("-e", "-E", "-F", "-i", "-l", "-o", "-O", "-v", "-vl"):
+ logIt(MSGspec, "Getting baler info from %s at %s..."%(CLTagID, FIPAddr))
+ try:
+ Ret = getBalerInfoHtm(CLTagID, FIPAddr)
+ except KeyboardInterrupt:
+ logIt(MSGspec, "Stopped by user.")
+ exit(0)
+ if Ret[0] != 0:
+ logIt(MSGspec, "%s\n"%Ret[2])
+ exit(1)
+ BInfo = Ret[1]
+ logIt(MSGspec, "Got baler info.")
+
+# A simpler verification check than above. This also gets done by using
+# getBalerInfoHtm() above, so you don't need to call this too.
+if argv[2] in ("-s", "-x"):
+ try:
+ Ret = checkTagID(CLTagID, FIPAddr)
+ if Ret[0] != 0:
+ logIt(MSGspec, "%s\n"%Ret[2])
+ exit(1)
+ except KeyboardInterrupt:
+ logIt(MSGspec, "Stopped by user.")
+ exit(0)
+ if Ret[0] != 0:
+ logIt(MSGspec, "%s\n"%Ret[2])
+ exit(1)
+
+# The commands that use the list of files from the baler.
+if argv[2] in ("-F", "-e", "-E", "-l", "-o", "-O", "-v", "-vl"):
+ BFiles = []
# files.htm can take a long time to build (it's made and sent on-the-fly).
-if Percent < 50:
- logIt("Getting files.htm...")
-else:
- logIt("Getting files.htm...(could take a while)")
-Ret = getFilesHtm("", TagID, IPAddr)
-if Ret[0] != 0:
- logIt(Ret[2])
- logIt("")
- exit(1)
-logIt("Got files.htm.")
-FHFiles = Ret[1]
-# Always write the whole list out to the file ./<TagID>files.htm.
-Ret = writeFilesTxt("bl", Dirspec+TagID+"files.txt", FHFiles)
-if Ret[0] != 0:
- logIt(Ret[2])
- logIt("")
- exit(1)
-if ArgList == True:
+ if BInfo["Percent"] < 50:
+ logIt(MSGspec, "Getting files.htm from %s..."%CLTagID)
+ else:
+ logIt(MSGspec, "Getting files.htm from %s...(could take a while)"% \
+ CLTagID)
+ try:
+ Ret = getFilesHtm(MSGspec, CLTagID, FIPAddr)
+ except KeyboardInterrupt:
+ logIt(MSGspec, "Stopped by user.")
+ exit(0)
+ if Ret[0] != 0:
+ logIt(MSGspec, "%s\n"%Ret[2])
+ exit(1)
+ BFiles = Ret[1]
+ logIt(MSGspec, "Got files.htm.")
+# Always write the whole list out since we were able to get it.
+ Ret = writeFilesTxt(MSGspec, CLTagID, TXTFspec, BFiles)
+ if Ret[0] != 0:
+ logIt(MSGspec, "%s\n"%Ret[2])
+ exit(1)
+
+# Commands that want the BFiles list (obtained with getFilesHtm()) pruned down
+# to just the lower sample rate files.
+if argv[2] in ("-e", "-o"):
+ for Index in arange(0, len(BFiles)):
+ try:
+ BName = BFiles[Index][B_NAME]
+ if isHigh(BName) == True:
+# print("Removed from download list: %s"%BName)
+ BFiles[Index] = []
+ except IndexError:
+ continue
+
+# Commands that want the BFiles list to only have matching entries with the
+# command line specified files.
+if argv[2] in ("-F", "-o", "-O", "-V"):
+# If this is in there just leave BFiles the way it is.
+ if "*" not in IFiles:
+ for Index in arange(0, len(BFiles)):
+ if len(BFiles[Index]) == 0:
+ continue
+ Match = False
+ for IFile in IFiles:
+ if fnmatch(BFiles[Index][B_NAME], IFile) == True:
+ Match = True
+ break
+ if Match == False:
+ BFiles[Index] = []
+
+# Commands that DON'T want matching command line specified files in the BFiles
+# list.
+if argv[2] in ("-e", "-E"):
+ for Index in arange(0, len(BFiles)):
+ if len(BFiles[Index]) == 0:
+ continue
+ Match = False
+ for IFile in IFiles:
+ if fnmatch(BFiles[Index][B_NAME], IFile) == True:
+ Match = True
+ break
+ if Match == True:
+ BFiles[Index] = []
+
+#===== bline.py <tagid> -b [<ipaddr> | <ethdev>] =====#
+
+# Assign an IP address to the baler <tagid>.
+# Create the <tagid>.set file with the <ipaddr> in it.
+# Watch for the baler on the specified <ethdev>.
+# If nothing is supplied the program will watch for a baler on all Ethernet
+# devices...if it can.
+# If the <ipaddr> is provided this will create/update the <tagid>.set file
+# with the provided IP address for the rest of the HTML-based functions to
+# use. There will be no Ethernet device or port number with the address, so
+# the commands that need those (like -x) will not function. Use this if the
+# IP address was set using something like BaleAddr.
+# If the <ethdev> is supplied the address-assigning routine will be run, but
+# only watching that device. Python 2, Windows, etc. do not have the
+# socket.recvmsg() function, so they need to be told the device name.
+if argv[2] == "-b":
+ if len(argv) == 3:
+ Ret = checkRPFiltersOff("")
+ if Ret[0] != 0:
+ logIt("", "%s\n"%Ret[1], False)
+ exit(1)
+ logHeader(MSGspec, "13")
+ logIt(MSGspec, "Watching for baler %s."%CLTagID)
+ logIt(MSGspec, "(Ctrl-C to stop before the baler is found.)")
+ logIt(MSGspec, "Press the ATTN button...")
+ try:
+ Ret = addressABaler(SETspec, MSGspec, CLTagID)
+ except KeyboardInterrupt:
+ logIt(MSGspec, "Stopped by user.")
+ exit(0)
+ logIt(MSGspec, Ret[2])
+ logIt(MSGspec, "")
+ exit(0)
+ elif len(argv) == 4:
+# If the passed item does not look like an IP address ass/u/me it is an
+# Ethernet device, or that the user doesn't know what they are doing.
+ if isIPV4Addr(argv[3]) == True:
+ Ret = getSetSettings(SETspec, "set", "", argv[3], "")
+ if Ret[0] != 0:
+ logIt("", "%s\n"%Ret[2], False)
+ exit(1)
+ logHeader(MSGspec, "13")
+ logIt(MSGspec, "IP Address %s saved to file %s.set\n"%(argv[3], \
+ CLTagID))
+ exit(0)
+ else:
+# This is almost the same as the len=3 section above.
+ Ret = checkRPFiltersOff("")
+ if Ret[0] != 0:
+ logIt("", "%s\n"%Ret[1], False)
+ exit(1)
+ logHeader(MSGspec, "13")
+ logIt(MSGspec, "Watching for baler %s on %s."%(CLTagID, argv[3]))
+ logIt(MSGspec, "(Ctrl-C to stop before the baler is found.)")
+ logIt(MSGspec, "Press the ATTN button...")
+ try:
+ Ret = addressABaler(SETspec, MSGspec, CLTagID, argv[3])
+ except KeyboardInterrupt:
+ logIt(MSGspec, "Stopped by user.")
+ exit(0)
+ logIt(MSGspec, Ret[2])
+ logIt(MSGspec, "")
+ exit(0)
+
+#===== bline.py <tagid> -x =====#
+
+# Sends the clean baler command.
+if argv[2] == "-x":
+ logIt("", ">>> -x command will erase all data on the baler. <<<", False)
+# Keep the original 'yes' answer for the log entry.
+ OAnswer = aninput("Continue to clean? (yes/no) ")
+ Answer = OAnswer.lower()
+ if Answer.startswith("y") == False:
+ logIt(MSGspec, "Nothing done.\n")
+ exit(0)
+ logIt(MSGspec, "Continue to clean? (yes/no) %s"%OAnswer)
+ Ret = cleanBaler(FDev, (FIPAddr, FPNumber))
+ if Ret[0] != 0:
+ logIt(MSGspec, Ret[2])
+ exit(1)
+ logIt(MSGspec, "Finished cleaning.\n")
+ exit(0)
+
+#===== bline.py <tagid> -i =====#
+
+if argv[2] == "-i":
+ if BInfo["NetSta"] == "":
+ BInfo["NetSta"] = "(none)"
+ logIt(MSGspec, "NetCode-StaID: %s"%BInfo["NetSta"])
+ logIt(MSGspec, "Model: %s FWVersion: %s"%(BInfo["BModel"], \
+ BInfo["FWVers"]))
+ logIt(MSGspec, "Disk Size: %s"%fmti(BInfo["DSize"]))
+ logIt(MSGspec, "%.1f%% of %s %s used."%(BInfo["Percent"], \
+ fmti(BInfo["NFiles"]), sP(BInfo["NFiles"], ("file", "files"))))
+ logIt(MSGspec, "Temperature: %s"%BInfo["Temp"])
+ logIt(MSGspec, "Supply voltage: %s"%BInfo["Volts"])
+ logIt(MSGspec, "MAC address: %s"%BInfo["MACAddr"])
+ logIt(MSGspec, "Disk model: %s"%BInfo["DModel"])
+ logIt(MSGspec, "DMU2 FWVersion: %s"%BInfo["FW2Vers"])
+ logIt(MSGspec, "Serial Number: %s\n"%BInfo["SerNum"])
+ exit(0)
+
+#===== bline.py <tagid> -l =====#
+
+# Just lists the files on the baler according to files.htm.
+if argv[2] == "-l":
+ logIt(MSGspec, "Baler %s file listing:"%CLTagID, False)
+ Count = 0
+ for File in BFiles:
+ Count += 1
+ logIt(MSGspec, " %d. %s %s to %s %d"%(Count, File[B_NAME], \
+ File[B_FROM], File[B_TO], File[B_SIZE]), False)
+ logIt(MSGspec, "")
+ exit(0)
+
+#===== bline.py <tagid> -L =====#
+
+# Just lists in the <tagid>.sdr directory.
+if argv[2] == "-L":
+ logIt(MSGspec, "Offloaded files in directory %s:"%SDRspec, False)
+ if exists(SDRspec) == False:
+ logIt(MSGspec, " Directory %s does not exist."%SDRspec, False)
+ exit(0)
+ Files = listdir(SDRspec)
+ Files.sort()
+ Count = 0
+ for File in Files:
+ if File.startswith("."):
+ continue
+ Count += 1
+ Size = getsize(SDRspec+File)
+ logIt(MSGspec, " %d. %s (%s %s)"%(Count, File, fmti(Size), \
+ sP(Size, ("byte", "bytes"))), False)
+ if Count == 0:
+ logIt(MSGspec, " None", False)
+ logIt(MSGspec, "")
+ exit(0)
+
+#===== bline.py <tagid> -s =====#
+
+# Sends the shudown command to the baler.
+if argv[2] == "-s":
+ Shutdown = Packet()
+ Shutdown.pack_shutdown()
+ Ret = ipFromDevice(FDev)
+ if Ret[0] != 0:
+ logIt(MSGspec, "%s\n"%Ret[2])
+ exit(0)
+ localhost = Ret[1]
+ SockComm = socketComm(localhost.ip)
+ SockComm.sendto(Shutdown.buf, (FIPAddr, FPNumber))
+ logIt(MSGspec, "Shutdown command sent. Watch the baler.\n")
+ exit(0)
+
+#==== bline.py <tagid> -V =====#
+
+# Goes through the offloaded files and checks to see that at least the
+# mini-seed headers make sense...or don't. Does not need a baler connected.
+if argv[2] == "-V":
+ Files = args2SL("l", "-V")
+ if len(Files) == 0:
+ Files.append("*")
+ badBlocks(MSGspec, SDRspec, Files, True)
+ logIt(MSGspec, "")
+ exit(0)
+
+#===== bline.py <tagid> -e -E -F -o -O [<files>] =====#
+
+# Offload the low sample rate files or all files that have not already been
+# fully offloaded. BFiles has been trimmed down to the lower sample rate files
+# in a previous section, and any matching or non-matching files depending on
+# the command. So this just processes what's left in BFiles.
+if argv[2] in ("-e", "-E", "-F", "-o", "-O"):
+# Count the number of files we are really going to offload. The baler may be
+# empty.
+ BToOffload = 0
+ for BFile in BFiles:
+# This file was a high sample rate file or not-requested file removed above.
+ if len(BFile) == 0:
+ continue
+ BToOffload += 1
+ if BToOffload == 0:
+ logIt(MSGspec, "There are no requested baler files to offload.\n")
+ exit(0)
+# Just make sure before we start checking for offloaded files.
+ if exists(SDRspec) == False:
+ makedirs(SDRspec)
+# Now further reduce the number of files in BFiles by checking to see how many
+# of them have already been fully offloaded.
+ BToOffload = 0
+ BTotalSize = 0
+# How many already exist, but are not the right size.
+ OffExist = 0
+ for Index in arange(0, len(BFiles)):
+ if len(BFiles[Index]) == 0:
+ continue
+ BName = BFiles[Index][B_NAME]
+ BSize = BFiles[Index][B_SIZE]
+ if exists(SDRspec+BName) == True:
+ if getsize(SDRspec+BName) == BSize:
+ BFiles[Index] = []
+ continue
+ OffExist += 1
+ BToOffload += 1
+ BTotalSize += BSize
+ if BToOffload == 0:
+ logIt(MSGspec, \
+ "All requested baler files have already been offloaded.\n")
+ exit(0)
+# I'm just sayin'...
+ if OffExist != 0:
+ logIt("", " %d previously offloaded %s may be overwritten."% \
+ (OffExist, sP(OffExist, ("file", "files"))), False)
+ Answer = aninput(" Is this OK? (yes/no): ")
+ Answer = Answer.strip().lower()
+ if Answer.startswith("y") == False:
+ logIt("", " Nothing done.", False)
+ exit(0)
+ logIt(MSGspec, "Saving data files to: %s"%SDRspec)
+# MacBook was 40.2s, Linux Dell was 41.9s, old Win7 42.0s...
+ TTO = intt(BTotalSize/16777216.0*41.0/60.0)
+ if TTO == 0:
+ TTO = 1
+ logIt(MSGspec, "Offloading %s %s, %s %s (~%s%s)..."% \
+ (fmti(BToOffload), sP(BToOffload, ("file", "files")), \
+ fmti(BTotalSize), sP(BTotalSize, ("byte", "bytes")), fmti(TTO), \
+ sP(TTO, ("min", "mins"))))
Count = 0
- for File in FHFiles:
+# Files offloaded.
+ Offloaded = 0
+# Total bytes offloaded.
+ OffTotalSize = 0
+# Bytes from the current file offloaded. Gets updated by fileBlockRetrieved().
+ OffBytes = 0
+ for BFile in BFiles:
+ if len(BFile) == 0:
+ continue
+ OffBytes = 0
+ Offloaded10 = 0
Count += 1
- stdout.write("%d. %s %s to %s %d\n"%(Count, File[FH_NAME], \
- File[FH_FROM], File[FH_TO], File[FH_SIZE]))
- logIt("Done listing.")
+ BName = BFile[B_NAME]
+ BSize = BFile[B_SIZE]
+ if BSize == -1:
+ logIt(MSGspec, "%d/%d. Getting %s (?)..."%(Count, BToOffload, \
+ BName))
+ else:
+ logIt(MSGspec, "%d/%d. Getting %s (%s)..."%(Count, BToOffload, \
+ BName, fmti(BSize)))
+ try:
+ try:
+ urlretrieve("http://%s/%s"%(FIPAddr, BName), "%s%s"%(SDRspec, \
+ BName), fileBlockRetrieved)
+ except KeyboardInterrupt:
+ if Offloaded10 > 0:
+ stdout.write("\n")
+ logIt(MSGspec, "Stopped by user.\n")
+ exit(0)
+ except Exception as e:
+ if Offloaded10 > 0:
+ stdout.write("\n")
+ logIt(MSGspec, "ERROR: Retrieving file %s:"%BName)
+ logIt(MSGspec, "%s (got ~%s %s)\n"%(e, fmti(OffBytes), \
+ sP(OffBytes, ("byte", "bytes"))))
+ exit(1)
+ OffSize = getsize(SDRspec+BName)
+ OffTotalSize += OffSize
+ if Offloaded10 > 0:
+ stdout.write(" (%.0f%%)\n"%(float(OffTotalSize)/BTotalSize*100))
+ if BSize != -1:
+ if OffSize != BSize:
+ logIt(MSGspec, "ERROR: Size error. %s: Baler %s, File %s\n"% \
+ (BName, fmti(BSize), fmti(OffSize)))
+ exit(1)
+ if OffSize < 4000:
+ logIt(MSGspec, "STRANGE: File %s is only %s %s."%(BName, \
+ fmti(OffSize), sP(OffSize, ("byte", "bytes"))))
+ Offloaded += 1
+ logIt(MSGspec, "Offloaded %s %s, %s %s.\n"%(fmti(Offloaded), \
+ sP(Offloaded, ("file", "files")), fmti(OffTotalSize), \
+ sP(OffTotalSize, ("byte", "bytes"))))
exit(0)
-# Go through FHFiles and match up by name and size the files that are already
-# on the computer. Then go through the files in the directory and see if there
+
+#===== bline.py <tagid> -v | -vl [<files>] =====#
+
+# Go through BFiles and match up by name and size the files that are already
+# on the computer, then go through the files in the directory and see if there
# are any that don't belong there.
-if ArgVerify == True:
+if argv[2] in ("-v", "-vl"):
Here = 0
HereSize = 0
NotHere = 0
- for Index in arange(0, len(FHFiles)):
- Name = FHFiles[Index][FH_NAME]
- Size = FHFiles[Index][FH_SIZE]
- if exists(DirspecSDR+Name):
- SDRSize = getsize(DirspecSDR+Name)
- if SDRSize == Size:
+# Make this copy so we can be destructive in case the command is -vl.
+ BFiles2 = deepcopy(BFiles)
+ for Index in arange(0, len(BFiles2)):
+ BName = BFiles2[Index][B_NAME]
+ BSize = BFiles2[Index][B_SIZE]
+ if exists(SDRspec+BName):
+ SDRSize = getsize(SDRspec+BName)
+ if SDRSize == BSize:
Here += 1
HereSize += SDRSize
+# Only keep the files in the list that are not here.
+ BFiles2[Index] = []
else:
NotHere += 1
else:
NotHere += 1
- logIt(" Data files fully offloaded: %s (%s %s)"%(fmti(Here), \
- fmti(HereSize), sP(HereSize, ("byte", "bytes"))))
- logIt("Data files not fully offloaded: %s"%fmti(NotHere))
+ logIt(MSGspec, " Data files fully offloaded: %s (%s %s)"% \
+ (fmti(Here), fmti(HereSize), sP(HereSize, ("byte", "bytes"))), \
+ False)
+ logIt(MSGspec, " Data files not fully offloaded: %s"%fmti(NotHere), \
+ False)
# List the files that have not been offloaded with (-vl command).
- if ArgVerifyList == True:
- logIt("Files not fully offloaded:")
- NotHere = 0
- for Index in arange(0, len(FHFiles)):
- Name = FHFiles[Index][FH_NAME]
- Size = FHFiles[Index][FH_SIZE]
- if exists(DirspecSDR+Name):
- SDRSize = getsize(DirspecSDR+Name)
- if SDRSize == Size:
- pass
- else:
- NotHere += 1
- logIt("%d. %s (%s %s, short)"%(NotHere, Name, \
- fmti(SDRSize), sP(SDRSize, ("byte", "bytes"))))
- else:
- NotHere += 1
- logIt("%d. %s"%(NotHere, Name))
- DFiles = listdir(DirspecSDR)
+ if argv[2] == "-vl":
+ if NotHere > 0:
+ Count = 0
+ for Index in arange(0, len(BFiles2)):
+ if len(BFiles2[Index]) > 0:
+ BName = BFiles[Index][B_NAME]
+ BSize = BFiles[Index][B_SIZE]
+ Count += 1
+ logIt(MSGspec, " %d. %s (%s %s)"%(Count, BName, \
+ fmti(BSize), sP(BSize, ("byte", "bytes"))), False)
+ elif NotHere == 0:
+ logIt(MSGspec, " All files fully offloaded.", False)
+# Now the reverse check for files that don't belong in the .sdr.
+ DFiles = listdir(SDRspec)
+ logIt(MSGspec, " Files that do not belong in %s:"%SDRspec, False)
Extras = 0
- FHFiles2 = []
- for FHFile in FHFiles:
- if len(FHFile) == 0:
- continue
- FHFiles2.append(FHFile[FH_NAME])
for DFile in DFiles:
- if DFile.startswith("."):
+ if len(DFile) == 0 or DFile.startswith("."):
continue
- if DFile not in FHFiles2:
- if isdir(DirspecSDR+DFile):
- logIt(" Directory in .%s%s.sdr: %s"%(sep, TagID, DFile))
- else:
- Size = getsize(DirspecSDR+DFile)
- logIt("Extra file in .%s%s.sdr: %s (%s %s)"%(sep, TagID, \
- DFile, fmti(Size), sP(Size, ("byte", "bytes"))))
- Extras += 1
- if Extras > 0:
- logIt("%s non-data file %s in .%s%s.sdr."%(fmti(Extras), \
- sP(Extras, ("item", "items")), sep, TagID))
- logIt("Done verifying.")
- logIt("")
- exit(0)
-logIt("Offloading from baler %s @ %s"%(TagID, IPAddr))
-# Alter FHFiles as necessary and then fill up BFiles with the remains.
-if ArgSpec == True:
- logIt("Only offloading: %s"%list2Str(ArgSpecFiles))
-# Check each file against the command line argument(s).
- for Index in arange(0, len(FHFiles)):
- Matches = False
- for ArgSpecFile in ArgSpecFiles:
-# For entries that are already [].
- try:
- if fnmatch(FHFiles[Index][FH_NAME], ArgSpecFile) == True:
- Matches = True
- # Don't list them. There could be a lot.
- break
- except IndexError:
- continue
- if Matches == False:
- FHFiles[Index] = []
-elif ArgSkipO == True or ArgSkipo == True:
- logIt("Excluding: %s"%list2Str(ArgSpecFiles))
- for Index in arange(0, len(FHFiles)):
- Matches = False
- for ArgSpecFile in ArgSpecFiles:
- try:
- if fnmatch(FHFiles[Index][FH_NAME], ArgSpecFile) == True:
- Matches = True
- break
- except IndexError:
- continue
- if Matches == True:
- FHFiles[Index] = []
-# Because users and PASSCAL people are just too stupid.
-for Index in arange(0, len(FHFiles)):
-# FHFIles[Index] may be [], so try.
- try:
- Filename = FHFiles[Index][FH_NAME]
- if len(Filename) != 0:
- if exists(DirspecSDR+Filename):
- logIt("Some offloaded files may overwrite existing files.")
- logIt("Is this OK?")
- if PROG_PYVERS == 2:
- Answer = raw_input("Answer (yes/no): ")
- elif PROG_PYVERS == 3:
- Answer = input("Answer (yes/no): ")
- Answer = Answer.strip().lower()
- if Answer.startswith("n"):
- logIt("Answer: N")
- exit(1)
- elif Answer.startswith("y"):
- logIt("Answer: Y")
- else:
- logIt("Wrong answer.")
- exit(1)
+ Found = False
+ for BFile in BFiles:
+ if DFile == BFile[B_NAME]:
+ Found = True
break
- except:
- continue
-# For any of the offloads always throw out files that are already here and
-# that are the same size.
-for Index in arange(0, len(FHFiles)):
- try:
- Filename = FHFiles[Index][FH_NAME]
- Size = FHFiles[Index][FH_SIZE]
- if exists(DirspecSDR+Filename) and \
- getsize(DirspecSDR+Filename) == Size:
- FHFiles[Index] = []
- except:
- continue
-if ArgLSROffload == True or ArgSkipo == True:
- logIt("Offloading low sample rate files.")
- for Index in arange(0, len(FHFiles)):
- try:
- Filename = FHFiles[Index][FH_NAME]
- if Filename.find(".H") != -1 or Filename.find(".B") != -1 or \
- Filename.find(".S") != -1:
- FHFiles[Index] = []
- except IndexError:
- continue
-# Now move everything left to BFiles.
-BFiles = []
-BFilesBytes = 0
-for File in FHFiles:
- if len(File) == 0:
- continue
- BFilesBytes += File[FH_SIZE]
- BFiles.append(File)
-BFilesToOff = len(BFiles)
-if BFilesToOff == 0:
- logIt("All files appear to have been offloaded.")
+ if Found == False:
+ DSize = getsize(SDRspec+DFile)
+ Extras += 1
+ logIt(MSGspec, " %d. %s (%s %s)"%(Extras, DFile, \
+ fmti(DSize), sP(DSize, ("byte", "bytes"))), False)
+ if Extras == 0:
+ logIt(MSGspec, " None.", False)
+ logIt(MSGspec, "", False)
exit(0)
-# Create this after we know if there is even going to be anything to offload.
-if exists(DirspecSDR) == False:
- makedirs(DirspecSDR)
- logIt("Saving data files to %s (created)"%DirspecSDR)
-else:
- logIt("Saving data files to %s (exists)"%DirspecSDR)
-logIt("Offloading %s %s, %s %s..."%(fmti(BFilesToOff), sP(BFilesToOff, \
- ("file", "files")), fmti(BFilesBytes), sP(BFilesBytes, ("byte", \
- "bytes"))))
-Count = 0
-# Files offloaded.
-OffFiles = 0
-# Total bytes offloaded.
-OffBytes = 0
-# Bytes from the current file offloaded.
-OffFileSize = 0
-for File in BFiles:
- OffFileSize = 0
- Offloaded10 = 0
- if len(File) == 0:
- continue
- Count += 1
- Filename = File[FH_NAME]
- BFileSize = File[FH_SIZE]
- if BFileSize == -1:
- logIt("%d/%d. Getting %s (?)..."%(Count, BFilesToOff, Filename))
- else:
- logIt("%d/%d. Getting %s (%s)..."%(Count, BFilesToOff, Filename, \
- fmti(BFileSize)))
- try:
- urlretrieve("http://%s/%s"%(IPAddr, Filename), "%s%s"%(DirspecSDR, \
- Filename), fileBlockRetrieved)
- except Exception as e:
- if Offloaded10 > 0:
- stdout.write("\n")
- logIt("ERROR: Retrieving file %s:"%Filename)
- logIt("%s (got ~%s %s)"%(e, fmti(OffFileBytes), sP(OffFileBytes, \
- ("byte", "bytes"))))
- logIt("")
- exit(1)
- if Offloaded10 > 0:
- stdout.write("\n")
- FileSize = getsize(DirspecSDR+Filename)
- if BFileSize != -1:
- if BFileSize != FileSize:
- logIt("ERROR: Size error. %s: Baler %s, File %s"%(Filename, \
- fmti(BFileSize), fmti(FileSize)))
- logIt("")
- exit(1)
-# Same as above.
- if FileSize < 4000:
- logIt("STRANGE: File %s is only %s %s."%(Filename, fmti(FileSize), \
- sP(FileSize, ("byte", "bytes"))))
- OffFiles += 1
- OffBytes += FileSize
-logIt("Offloaded %s %s, %s %s.\a"%(fmti(OffFiles), sP(OffFiles, ("file", \
- "files")), fmti(OffBytes), sP(OffBytes, ("byte", "bytes"))))
-logIt("")
-exit(0)
+
+logIt("", "What??\a\n", False)
+exit(1)
# END: main
# END PROGRAM: BLINE
diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml
index c7030b4cfa359432f534ab34d7b314606c3e3c97..3014c78d8d2f4c6f0ef01383f7d2fcc6d75e0eac 100644
--- a/conda.recipe/meta.yaml
+++ b/conda.recipe/meta.yaml
@@ -1,6 +1,6 @@
package:
name: bline
- version: 2018.135
+ version: 2019.297
source:
path: ..
diff --git a/setup.cfg b/setup.cfg
index 3c6e531150750912196df1fd45072e44980e23e3..d2d5bd00c826c99d8f8ea34a18dca27946a85a5a 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 2019.064
+current_version = 2019.297
commit = True
tag = True
diff --git a/setup.py b/setup.py
index 3752eaa21ccd12c0489c2848889fd7bf4640ab7f..cf074bca4ca7edf1cf8366e0f638ea55510275a0 100644
--- a/setup.py
+++ b/setup.py
@@ -11,7 +11,6 @@ with open('README.rst') as readme_file:
with open('HISTORY.rst') as history_file:
history = history_file.read()
-
setup(
author="IRIS PASSCAL",
author_email='software-support@passcal.nmt.edu',
@@ -28,7 +27,10 @@ setup(
'bline=bline.bline:main',
],
},
- install_requires=[],
+ install_requires=['psutil',
+ 'ipaddress',
+ 'pexpect',
+ 'subprocess32'],
setup_requires = [],
extras_require={
'dev': [
@@ -51,6 +53,6 @@ setup(
packages=find_packages(include=['bline']),
test_suite='tests',
url='https://git.passcal.nmt.edu/passoft/bline',
- version='2019.064',
+ version='2019.297',
zip_safe=False,
)