Read mseed and log data

sohstationviewer/controller/core/utils_tmp.py

+#################
+# BEGIN: intt(In)
+# LIB:intt():2018.257
+#   Handles all of the annoying shortfalls of the int() function (vs C).
+def intt(In):
+    In = str(In).strip()
+    if len(In) == 0:
+        return 0
+    # Let the system try it first.
+    try:
+        return int(In)
+    except ValueError:
+        Number = ""
+        for c in In:
+            if c.isdigit():
+                Number += c
+            elif (c == "-" or c == "+") and len(Number) == 0:
+                Number += c
+            elif c == ",":
+                continue
+            else:
+                break
+        try:
+            return int(Number)
+        except ValueError:
+            return 0
+# END: intt
+
+###################
+# BEGIN: floatt(In)
+# LIB:floatt():2018.256
+#    Handles all of the annoying shortfalls of the float() function (vs C).
+#    Does not handle scientific notation numbers.
+def floatt(In):
+    In = str(In).strip()
+    if len(In) == 0:
+        return 0.0
+    # At least let the system give it the ol' college try.
+    try:
+        return float(In)
+    except Exception:
+        Number = ""
+        for c in In:
+            if c.isdigit() or c == ".":
+                Number += c
+            elif (c == "-" or c == "+") and len(Number) == 0:
+                Number += c
+            elif c == ",":
+                continue
+            else:
+                break
+        try:
+            return float(Number)
+        except ValueError:
+            return 0.0
+# END: floatt
+
+######################################
+# BEGIN: rtnPattern(In, Upper = False)
+# this may need to be moved to another place
+# LIB:rtnPattern():2006.114
+def rtnPattern(In, Upper=False):
+    Rtn = ""
+    for c in In:
+        if c.isdigit():
+            Rtn += "0"
+        elif c.isupper():
+            Rtn += "A"
+        elif c.islower():
+            Rtn += "a"
+        else:
+            Rtn += c
+    # So the A/a chars will always be A, so the caller knows what to look for.
+    if Upper is True:
+        return Rtn.upper()
+    return Rtn
+# END: rtnPattern
\ No newline at end of file

sohstationviewer/controller/processing.py

+from sohstationviewer.model.mseed_text import MSeed_Text
+
+
+def loadData(parent, listOfDir, reqInfoChans):
+    """
+    Go through root dir and read all files in that dir and its subdirs
+    """
+    dataObject = None
+    for d in listOfDir:
+        if dataObject is None:
+            # dataObject = Reftek.Reftek(parent, d)
+            # if dataObject.hasData():
+            #    continue
+            dataObject = MSeed_Text(parent, d, reqInfoChans)
+            if dataObject.hasData():
+                continue
+            # If no data can be read from the first dir, throw exception
+            raise Exception("No data can be read from ", d)
+        else:
+            dataObject.readDir(d)
+
+    return dataObject.plottingData

sohstationviewer/model/core/dataType.py

+import os
+
+
+class WrongDataTypeError(Exception):
+    def __init__(self, *args, **kwargs):
+        self.args = (args, kwargs)
+
+
+class DataType:
+    def __init__(self, parent, dir, reqInfoChans=[]):
+        self.parent = parent
+        self.dir = dir
+        self.reqInfoChans = reqInfoChans
+        self.noneReqChans = set()
+        self.curNetStatLoc = ('_', '_', '_')
+        self.processingLog = []     # [(message, type)]
+        self.logData = {}
+        self.plottingData = {}
+        self.streams = {}
+
+        self.readDir(dir)
+
+    def readFile(self, pathToFile):
+        pass
+
+    def combineData(self):
+        """
+        Merge channels in each stream
+        """
+        pass
+
+    def readDir(self, dir):
+        count = 0
+        for path, subdirs, files in os.walk(dir):
+
+            for fileName in files:
+                if fileName.startswith('._'):
+                    # skip mac's info file
+                    continue
+                if not os.path.isfile(os.path.join(path, fileName)):
+                    continue
+                self.readFile(path, fileName)
+                count += 1
+                if count % 50 == 0:
+                    self.displayTrackingInfo("Reading file %s" % count,
+                                             'info')
+        self.combineData()
+
+    def hasData(self):
+        if len(self.logData) == 0 and len(self.plottingData) == 0:
+            return False
+        return True
+
+    def displayTrackingInfo(self, text, type):
+        """
+        :param text: text to be displayed
+        :param type: info/warning/error
+        """
+        print("displayTrackingInfo:", text)
+        self.parent.displayTrackingInfo(text, type)
+
+    def trackInfo(self, text, type):
+        self.displayTrackingInfo(text, type)
+        self.processingLog.append((text, type))

sohstationviewer/model/mseed_text.py

+import os
+
+from obspy.core import Stream, read as read_ms
+from obspy import UTCDateTime
+from struct import unpack
+
+from sohstationviewer.model.core.dataType import DataType
+
+
+class MSeed_Text(DataType):
+    def __init__(self, *kwarg):
+        self.readBlkt = {500: self.readBlkt500}
+        super().__init__(*kwarg)
+
+    def readFile(self, path, fileName):
+        if not self.readMiniseed(path, fileName):
+            self.readText(path, fileName)
+
+    def combineData(self):
+        for k in self.streams:           # k=netID, statID, locID
+            stream = self.streams[k]
+            stream.merge()
+            # gaps is list of [network, station, location, channel, starttime,
+            # endtime, duration, number of missing samples]
+            gaps = stream.get_gaps()
+            # stream.print_gaps()
+            gaps = self.squashGaps(gaps)
+            self.plottingData[k] = {'gaps': gaps,
+                                    'channels': {}}
+            maxEndtime = UTCDateTime(0)
+            minStarttime = UTCDateTime(20E10)
+            for trace in stream:
+                startTm, endTm = self.readTrace(
+                    trace, self.plottingData[k]['channels'])
+                if startTm < minStarttime:
+                    minStarttime = startTm
+                if endTm > maxEndtime:
+                    maxEndtime = endTm
+            self.plottingData[k]['earliestUTC'] = minStarttime
+            self.plottingData[k]['latestUTC'] = maxEndtime
+
+    def addLog(self, chan, logText):
+        if chan not in self.logData[self.curNetStatLoc].keys():
+            self.logData[self.curNetStatLoc][chan] = []
+        self.logData[self.curNetStatLoc][chan].append(logText)
+
+    def readText(self, path, fileName):
+        """
+        Read log file and add to logData under channel TEXT
+        """
+        with open(os.path.join(path, fileName), 'r') as file:
+            try:
+                content = file.read()
+            except UnicodeDecodeError:
+                self.trackInfo("Can't process file: %s" % fileName, 'error')
+                return
+            logText = "\n\n** STATE OF HEALTH: %s\n" % fileName
+            logText += content
+            self.addLog('TEXT', logText)
+
+    def readMiniseed(self, path, fileName):
+        """
+        Read ms and add trace in self.streams to merge later
+        Or read log wrap in ms and add to logData under channel in ms header
+        """
+        try:
+            stream = read_ms(os.path.join(path, fileName))
+        except TypeError:
+            return False
+
+        file = None
+        for trace in stream:
+            chanID = trace.stats['channel']
+            if chanID not in self.reqInfoChans:
+                self.noneReqChans.add(chanID)
+                continue
+            netID = trace.stats['network']
+            statID = trace.stats['station']
+            locID = trace.stats['location']
+            self.curNetStatLoc = k = (netID, statID, locID)
+            if trace.stats.mseed['encoding'] == 'ASCII':
+                file = self.readASCII(path, fileName, file, trace, k)
+            else:
+                if k not in self.streams.keys():
+                    self.streams[k] = Stream()
+                self.streams[k].append(trace)
+        if file is not None:
+            file.close()
+        return True
+
+    def readASCII(self, path, fileName, file, trace, k):
+        byteorder = trace.stats.mseed['byteorder']
+        h = trace.stats
+        logText = "\n\n**** STATE OF HEALTH: "
+        logText += ("From:%s  To:%s\n" % (h.starttime, h.endtime))
+        textFromData = trace.data.tobytes().decode()
+        if textFromData != '':
+            logText += textFromData
+        else:
+            recLen = h.mseed['record_length']
+            if file is None:
+                file = open(os.path.join(path, fileName), 'rb')
+            databytes = file.read(recLen)
+            followingBlktNum = unpack('%s%s' % (byteorder, 'B'),
+                                      databytes[39:40])[0]
+            if followingBlktNum > 1:
+                # skip blockette 1000
+                nextBlocketteType = unpack('%s%s' % (byteorder, 'H'),
+                                           databytes[56:58])[0]
+                try:
+                    logText += self.readBlkt[nextBlocketteType](databytes,
+                                                                byteorder)
+                except KeyError:
+                    msg = ("Function to read blockette %s isn't implemented "
+                           "yet. Implementer needs to write and add the "
+                           "function to the dict self.readBlkt in Mseed_Text."
+                           "__init__()" % nextBlocketteType)
+                    self.trackInfo(msg, 'error')
+                    return file
+        if k not in self.logData:
+            self.logData[k] = {}
+        if h.channel not in self.logData[k]:
+            self.logData[k][h.channel] = []
+        self.logData[k][h.channel].append(logText)
+        return file
+
+    def readBlkt500(self, databytes, byteorder):
+        t = {}
+        (vcocorr, t['year'], t['doy'], t['hour'], t['min'], t['sec'],
+         junk, t['micro'], t['micro2'], qual, cnt, type, mod, stat
+         ) = unpack('%s%s' % (byteorder, 'fHHBBBBHBBI16s32s128s'),
+                    databytes[60:256])
+        logText = "\nVCO Correction: %s" % vcocorr
+        logText += ("\nTime of exception: %(year)s:%(doy)s:%(hour)s:%(min)s:"
+                    "%(sec)s:%(micro)s:%(micro2)s" % t)
+        logText += "\nReception Quality: %s" % qual
+        logText += "\nException Count: %s" % cnt
+        logText += "\nException Type: %s" % type.strip()
+        logText += "\nClock Model: %s" % mod.strip()
+        logText += "\nClock Status: %s" % stat.strip()
+        return logText
+
+    def readTrace(self, trace, channels):
+        chan = {}
+        chan['netID'] = trace.stats['network']
+        chan['statID'] = trace.stats['station']
+        chan['locID'] = trace.stats['location']
+        chanID = chan['chanID'] = trace.stats['channel']
+        chan['samplerate'] = trace.stats['sampling_rate']
+        chan['startTmEpoch'] = trace.stats['starttime'].timestamp
+        chan['endTmEpoch'] = trace.stats['endtime'].timestamp
+        """
+        trace time start with 0 => need to add with epoch starttime
+        """
+        chan['times'] = trace.times() + trace.stats['starttime'].timestamp
+        chan['data'] = trace.data
+        if chanID not in channels.keys():
+            channels[chanID] = chan
+        else:
+            msg = ("Something wrong with code logic."
+                   "After stream is merged there should be only one trace "
+                   "for channel %s: %s" % (chanID, trace))
+            self.trackInfo(msg, "error")
+        return trace.stats['starttime'], trace.stats['endtime']
+
+    def squashGaps(self, gaps):
+        """
+        Squash different lists of gaps for all channels to be one list of
+        (minStarttime, maxEndtime) gaps
+        : param gaps: list of [network, station, location, channel, starttime,
+        endtime, duration, number of missing samples]
+        : return squashedGaps
+        """
+        if len(gaps) == 0:
+            return gaps
+
+        # create dict consist of list of gaps for each channel
+        gaps_dict = {}
+        for g in gaps:
+            if g[3] not in gaps_dict:
+                gaps_dict[g[3]] = []
+            gaps_dict[g[3]].append((g[4].timestamp, g[5].timestamp))
+
+        firstLen = len(gaps_dict[gaps[0][3]])
+        diffLenList = [len(gaps_dict[k]) for k in gaps_dict.keys()
+                       if len(gaps_dict[k]) != firstLen]
+        if len(diffLenList) > 0:
+            msg = "Number of Gaps for different channel are not equal.\n"
+            for k in gaps_dict.keys():
+                msg += "%s: %s\n" % (k, len(gaps_dict[k]))
+            self.displayTrackingInfo(msg, 'error')
+            return []
+        squashedGaps = []
+        for i in range(firstLen):
+            maxEndtime = 0
+            minStarttime = 1E100
+            for val in gaps_dict.values():
+                if val[i][0] < minStarttime:
+                    minStarttime = val[i][0]
+                if val[i][1] > maxEndtime:
+                    maxEndtime = val[i][1]
+            squashedGaps.append((minStarttime, maxEndtime))
+        return squashedGaps

sohstationviewer/view/mainwindow.py

@@ -64,12 +64,12 @@ class MainWindow(QtWidgets.QMainWindow, Ui_MainWindow):
 
         # Connect slots to change the date format displayed
         # by the QDateEdit widgets
-        # self.showYYYYDOYDates.triggered.connect(lambda:
-        #                                         self.setDateFormat('yyyy:D'))
-        self.showYYYY_MM_DDDates.triggered.connect(lambda:
-                                                   self.setDateFormat('yyyy-MM-dd'))
-        self.showYYYYMMMDDDates.triggered.connect(lambda:
-                                                  self.setDateFormat('yyyyMMMdd'))
+        # self.showYYYYDOYDates.triggered.connect(
+        # lambda: self.setDateFormat('yyyy:D'))
+        self.showYYYY_MM_DDDates.triggered.connect(
+            lambda: self.setDateFormat('yyyy-MM-dd'))
+        self.showYYYYMMMDDDates.triggered.connect(
+            lambda: self.setDateFormat('yyyyMMMdd'))
 
         self.timeToDateEdit.setCalendarWidget(CalendarWidget(self))
         self.timeToDateEdit.setDate(QtCore.QDate.currentDate())
@@ -80,7 +80,8 @@ class MainWindow(QtWidgets.QMainWindow, Ui_MainWindow):
         # self.showYYYYDOYDates.triggered.emit()
         self.showYYYY_MM_DDDates.triggered.emit()
 
-        self.openFilesList.itemDoubleClicked.connect(self.openFilesListItemDoubleClicked)
+        self.openFilesList.itemDoubleClicked.connect(
+            self.openFilesListItemDoubleClicked)
 
         pal = self.openFilesList.palette()
         pal.setColor(QtGui.QPalette.Highlight, QtGui.QColor(128, 255, 128))