From 51887f107504c30b7ef64f91c71ecd521c4a7350 Mon Sep 17 00:00:00 2001
From: Robert Greschke <bob@passcal.nmt.edu>
Date: Fri, 25 Oct 2019 08:42:10 -0600
Subject: [PATCH] Enhanced -G concatenation routine and added -GD command.
---
HISTORY.rst | 8 +-
bline/bline.py | 175 ++++++++++++++++++++++++-----------------
conda.recipe/meta.yaml | 2 +-
setup.cfg | 2 +-
setup.py | 2 +-
5 files changed, 115 insertions(+), 74 deletions(-)
diff --git a/HISTORY.rst b/HISTORY.rst
index 6047cf9..6c2566e 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -3,7 +3,7 @@ History
=======
2019.064 (2019-03-05)
-------------------
+---------------------
* Converted for Python 2 or 3.
* Added -vl command that lists the files that have not been fully
offloaded from the baler. -v just shows how many there are.
@@ -38,9 +38,15 @@ History
* Fixed up the long version of the Help a bit.
2019.289 (2019-10-16)
+---------------------
* Added the -A command that will combine all of the offloaded files
into a .ALL file.
* Added the -G command that will combine all of the files for a
channel into one file. The file names are close to the miniseed
file names produced by sdrsplit.
* Changed the command line switch for a couple commands.
+
+2019.297 (2019-10-24)
+---------------------
+* Enhanced the -G command and added the -GD command for concatenating
+ the offloaded channel files.
diff --git a/bline/bline.py b/bline/bline.py
index 3c9ca43..0a132e5 100755
--- a/bline/bline.py
+++ b/bline/bline.py
@@ -10,7 +10,7 @@ from sys import argv, exit, platform, stdout
PROGSystem = platform[:3].lower()
PROG_NAME = "BLINE"
PROG_NAMELC = "bline"
-PROG_VERSION = "2019.289"
+PROG_VERSION = "2019.297"
PROG_LONGNAME = "Command Line Baler Control Program"
PROG_SETUPSVERS = "A"
@@ -1455,7 +1455,7 @@ def getSetSettings(SETspec, Cmd, Ieth = "", Iip = "", Iport = ""):
###################
# BEGIN: helpLong()
-# FUNC:helpLong():2019.289
+# FUNC:helpLong():2019.297
def helpLong():
HELPText = "USING BLINE.PY\n\
--------------\n\
@@ -1872,11 +1872,19 @@ device on the control computer that the baler was detected on and returns\n\
the IP address of that device. The -b command must have been used to set\n\
the address of the baler for this command to function.\n\
\n\
--G\n\
-This reads the offloaded baler files and concatenates all of the file for\n\
+-G, -GD\n\
+These read the offloaded baler files and concatenate all of the files for\n\
a channel into one channel file. Depending on the operating system this\n\
may try to create a file larger than the OS can handle, but those days\n\
-are mostly gone. The original files are left in the .sdr directory.\n\
+are mostly gone. The original files are left in the baler's .sdr directory.\n\
+\n\
+The -G version creates the new files in the baler's .sdr directory. The\n\
+-GD version creates the new files in the directory \"./DATA/\".\n\
+\n\
+Providing a baler TagID will concatenate the files for that one baler.\n\
+Using a baler TagID of 9999 will tell BLINE to look for all of the\n\
+<baler>.sdr directories and perform the concatenation on the data files\n\
+in each baler's directory.\n\
\n\
-i\n\
This command simply establishes a connection with the spcified baler and\n\
@@ -2034,7 +2042,7 @@ END\n"
####################
# BEGIN: helpShort()
-# FUNC:helpShort():2019.289
+# FUNC:helpShort():2019.297
def helpShort():
HELPshort = "bline.py <command>\n\
-h = This help.\n\
@@ -2052,8 +2060,10 @@ bline.py <tagid> <command>\n\
-A = Copies all of the offloaded files into <tagid>.ALL\n\
-c = This is a second version of -c which uses the actual Ethernet\n\
device that the -b command found for the <tagid> baler.\n\
- -G = Copies all of the offloaded files for each channel into a file\n\
- <tagid>.<chan>.\n\
+ -G = Copies all of the offloaded files for each channel into a single\n\
+ file in the baler's .sdr directory.\n\
+ -GD = Copies all of the offloaded files for each channel into a single\n\
+ file in the ./DATA directory.\n\
-i = Gets basic information from the baler.\n\
-l = (ell) Displays and saves the list of files on the baler.\n\
-L = Displays the list of files in the baler's .sdr directory.\n\
@@ -2110,12 +2120,12 @@ Always leave a space after the command line switches:\n\
#####################
# BEGIN: helpVShort()
-# FUNC:helpVShort():2019.289
+# FUNC:helpVShort():2019.297
def helpVShort():
logIt("", "%s %s\nPython %s"%(PROG_NAME, PROG_VERSION, PROG_PYVERSION), \
False)
HELPVshort = "bline.py [ -h | -H | -c | -n | -U | -UD ]\n\
-bline.py <tagid> [ -A | -c | -G | -i | -l | -L | -s | -x ]\n\
+bline.py <tagid> [ -A | -c | -G | -GD | -i | -l | -L | -s | -x ]\n\
bline.py <tagid> [ -o | -O | -v | -vl | -V ] [<files>]\n\
bline.py <tagid> [ -e | -E | -F ] <files>\n\
bline.py <tagid> -M <message>\n\
@@ -2528,7 +2538,7 @@ def logHeader(MSGspec, Which, Time = True):
###############
# BEGIN: main()
-# FUNC:main():2019.288
+# FUNC:main():2019.297
# These lovely items are brought to you by Microsoft. Python 3 was keeping
# ' -#' for arguments, "" as an argument, or not passing command line
# arguments at all. It's related to the registry value
@@ -2713,6 +2723,7 @@ TXTFspec = CWDspec+CLTagID+"files.txt"
#===== bline.py <tagid> -A =====#
#===== bline.py <tagid> -c =====#
#===== bline.py <tagid> -G =====#
+#===== bline.py <tagid> -GD =====#
#===== bline.py <tagid> -M <message> =====#
if argv[2] == "-A":
@@ -2788,74 +2799,98 @@ if argv[2] == "-c":
logIt(MSGspec, "Control computer netmask: %s\n"%bitsToNetmask(Mask))
exit(0)
-if argv[2] == "-G":
+if argv[2] == "-G" or argv[2] == "-GD":
+ from shutil import copyfileobj
+ from struct import unpack
logHeader(MSGspec, "")
+ if CLTagID != "9999":
+# Make a TagIDDirs entry to fool the code below.
+ TagIDDirs = [".%s%s.sdr"%(sep, CLTagID)]
+ elif CLTagID == "9999":
+ TagIDDirs = glob(".%s*.sdr"%sep)
+ if len(TagIDDirs) == 0:
+# ID 9999.msg will be fine.
+ logIt(MSGspec, "No offloaded files found.")
+ logIt(MSGspec, \
+ "Are you in the directory above the .sdr directories?\n")
+ exit(0)
+ for TagID in TagIDDirs:
+ TagID = TagID.split(".sdr")[0]
+ TagID = TagID[2:]
+# Rebuild these for each TagID. Won't be necessary for -GD, but...
+ SDRspec = ".%s%s.sdr%s"%(sep, TagID, sep)
+ if argv[2] == "-G":
+ DATAspec = SDRspec
+ elif argv[2] == "-GD":
+ DATAspec = ".%sDATA%s"%(sep, sep)
+ if exists(DATAspec) == False:
+ makedirs(DATAspec)
# We'll be kinda specific so we don't have to keep filtering below.
- Files = glob("%s*__.*"%SDRspec)
- if len(Files) == 0:
- logIt(MSGspec, "No offloaded files found.")
- logIt(MSGspec, "Are you in the directory above the .sdr directory?\n")
- exit(0)
+ Files = glob("%s*__.*"%SDRspec)
+ if len(Files) == 0:
+ logIt(MSGspec, "Looking in %s"%SDRspec)
+ logIt(MSGspec, "No offloaded files found.")
+ logIt(MSGspec, \
+ "Are you in the directory above the .sdr directory?\n")
+ exit(0)
# Get the list of channels from the offloded files.
- Chans = []
- for File in Files:
- Chan = File[-4:]
- if Chan not in Chans:
- Chans.append(Chan)
- logIt(MSGspec, "Channels to group: %d"%len(Chans))
- Chans.sort()
- Files.sort()
- from shutil import copyfileobj
- from struct import unpack
- Count = 0
- try:
- for Chan in Chans:
+ Chans = []
+ for File in Files:
+ Chan = File[-4:]
+ if Chan not in Chans:
+ Chans.append(Chan)
+ logIt(MSGspec, "Channels to group: %d"%len(Chans))
+ Chans.sort()
+ Files.sort()
+ Count = 0
+ try:
+ for Chan in Chans:
# We need to go through the channel files, find the first one, open it and
# extract the station name, net code, etc. for the file name. If this set of
# data files has multiple of any of the file name items then all bets are off.
# This only reads the first header.
- for File in Files:
- if File.endswith(Chan):
- Fp = open(File, "rb")
- Header = Fp.read(128)
- Fp.close()
- if PROG_PYVERS == 2:
- Qual = Header[6]
- elif PROG_PYVERS == 3:
- Qual = chr(Header[6])
- StaID = Header[8:13].strip().decode("latin-1")
- LocID = Header[13:15].strip().decode("latin-1")
- ChanID = Header[15:18].strip().decode("latin-1")
- NetCode = Header[18:20].strip().decode("latin-1")
- Year, Doy, Hour, Mins, Secs, Tttt= unpack(">HHBBBxH", \
- Header[20:30])
- STime = "%s.%03d.%02d%02d%02d"%(Year, Doy, Hour, Mins, \
- Secs)
+ for File in Files:
+ if File.endswith(Chan):
+ Fp = open(File, "rb")
+ Header = Fp.read(128)
+ Fp.close()
+ if PROG_PYVERS == 2:
+ Qual = Header[6]
+ elif PROG_PYVERS == 3:
+ Qual = chr(Header[6])
+ StaID = Header[8:13].strip().decode("latin-1")
+ LocID = Header[13:15].strip().decode("latin-1")
+ ChanID = Header[15:18].strip().decode("latin-1")
+ NetCode = Header[18:20].strip().decode("latin-1")
+ Year, Doy, Hour, Mins, Secs, \
+ Tttt= unpack(">HHBBBxH", Header[20:30])
+ STime = "%s.%03d.%02d%02d%02d"%(Year, Doy, Hour, \
+ Mins, Secs)
# This is [another] one of many versions of file names.
- OutFile = "%s%s.%s.%s.%s.%s.%s"%(SDRspec, StaID, NetCode, \
- LocID, ChanID, Qual, STime)
- FpG = open(OutFile, "wb")
- break
- Count += 1
- logIt(MSGspec, " %d. Copying files to %s"%(Count, OutFile), \
- False)
- for File in Files:
- if File.endswith(Chan):
- Fp = open(File, "rb")
- copyfileobj(Fp, FpG, -1)
- Fp.close()
- FpG.close()
- except KeyboardInterrupt:
- try:
- Fp.close()
- except:
- pass
- try:
+ OutFile = "%s%s.%s.%s.%s.%s.%s"%(DATAspec, StaID, \
+ NetCode, LocID, ChanID, Qual, STime)
+ FpG = open(OutFile, "wb")
+ break
+ Count += 1
+ logIt(MSGspec, " %d. Copying files to %s"%(Count, OutFile), \
+ False)
+ for File in Files:
+ if File.endswith(Chan):
+ Fp = open(File, "rb")
+ copyfileobj(Fp, FpG, -1)
+ Fp.close()
FpG.close()
- except:
- pass
- logIt(MSGspec, "Stopped by user.")
- exit(0)
+ except KeyboardInterrupt:
+ try:
+ Fp.close()
+ except:
+ pass
+ try:
+ FpG.close()
+ except:
+ pass
+ logIt(MSGspec, "Stopped by user.")
+ exit(0)
logIt(MSGspec, "Finished.")
exit(0)
diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml
index f9acfd2..3014c78 100644
--- a/conda.recipe/meta.yaml
+++ b/conda.recipe/meta.yaml
@@ -1,6 +1,6 @@
package:
name: bline
- version: 2019.289
+ version: 2019.297
source:
path: ..
diff --git a/setup.cfg b/setup.cfg
index c0a890e..d2d5bd0 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 2019.289
+current_version = 2019.297
commit = True
tag = True
diff --git a/setup.py b/setup.py
index ea0c35d..cf074bc 100644
--- a/setup.py
+++ b/setup.py
@@ -53,6 +53,6 @@ setup(
packages=find_packages(include=['bline']),
test_suite='tests',
url='https://git.passcal.nmt.edu/passoft/bline',
- version='2019.289',
+ version='2019.297',
zip_safe=False,
)
--
GitLab