documentation/01 _ Table of Contents.help.md

@@ -4,6 +4,8 @@ Welcome to the SOH Station Viewer documentation. Here you will find usage guides
 
 On the left-hand side you will find a list of currently available help topics.
 
+If the links of the Table of Contents are broken, click on Recreate Table of Content <img src='recreate_table_contents.png' height=30 style='margin: 3px 0px 0px 0px;'/> to rebuild it.
+
 The home button can be used to return to this page at any time.
 
 # Table of Contents
@@ -14,19 +16,23 @@ The home button can be used to return to this page at any time.
 
 + [How to Use Help](03%20_%20How%20to%20Use%20Help.help.md)
 
-+ [Search SOH n LOG](04%20_%20Search%20SOH%20n%20LOG.help.md)
++ [Search List of Directories](04%20_%20Search%20List%20of%20Directories.help.md)
+
++ [Read from Data Card](05%20_%20Read%20from%20Data%20Card.help.md)
+
++ [Select SOH](06%20_%20Select%20SOH.help.md)
 
-+ [Search List of Directories](05%20_%20Search%20List%20of%20Directories.help.md)
++ [Select Mass Position](07%20_%20Select%20Mass%20Position.help.md)
 
-+ [Read from Data Card](06%20_%20Read%20from%20Data%20Card.help.md)
++ [Select Waveforms](08%20_%20Select%20Waveforms.help.md)
 
-+ [Select SOH](07%20_%20Select%20SOH.help.md)
++ [Gap Display](09%20_%20Gap%20Display.help.md)
 
-+ [Select Mass Position](08%20_%20Select%20Mass%20Position.help.md)
++ [Change TPS Color Range](10%20_%20Change%20TPS%20Color%20Range.help.md)
 
-+ [Select Waveforms](09%20_%20Select%20Waveforms.help.md)
++ [Save Plots](11%20_%20Save%20Plots.help.md)
 
-+ [Gap Display](10%20_%20Gap%20Display.help.md)
++ [Search SOH n LOG](12%20_%20Search%20SOH%20n%20LOG.help.md)
 
 + [GPS Dialog](20%20_%20GPS%20Dialog.help.md)
 

documentation/09 _ Select Waveforms.help.md → documentation/08 _ Select Waveforms.help.md

@@ -44,10 +44,12 @@ checked,  a warning will be created, "Checked data streams will be ignored
 for RT130 data type."
 
 ## Displaying waveform channels
-If one of TPS or RAW checkboxes aren't checked which means no data need to be
-displayed,  all the waveform selected will be ignored.
-
-To display waveform channels,  user need to check:
+TPS needs to be checked to display Time-Power-Squared of waveform.
+RAW needs to be checked to display actual signal of waveform.
 + <img alt="TPS" src="images/select_waveform/select_TPS.png" height="30" />: to diplay Time-Power-Squared of the selected waveform data 
 + <img alt="RAW" src="images/select_waveform/select_RAW.png" height="30" />: and check RAW to display the actual selected waveform data.
-<br />
\ No newline at end of file
+<br />
+
+If any of waveform is checked but no TPS or RAW is checked,
++ For RT130, the program will read event of the selected data stream.
++ For MSeed, the program will pop up message request user to clear waveform selection or select either TPS or RAW.
\ No newline at end of file

documentation/11 _ Save Plots.help.md

+# Save Plots
+
+---------------------------
+---------------------------
+
+## Step 1: click 'Save Plot'
+In Main Window,  Raw Data Plot and TPS Plot there are buttons labeled 'Save Plot'.
+
+User need to click those button to save plots in each window.
+
+* Saving State-of-Health plots
+<br />
+<img alt="Save SOH" src="images/save_plots/save_button_soh.png" height="30" />
+<br />
+* Saving Raw data plots
+<br />
+<img alt="Save Waveform" src="images/save_plots/save_button_wf.png" height="60" />
+<br />
+* Saving Time-power-square plots
+<br />
+<img alt="Save TPS" src="images/save_plots/save_button_tps.png" height="80" />
+<br />
+<br />
+<br />
+
+If the current color mode is black, user will be asked to continue or cancel 
+to change mode before saving the image.
+
+<br />
+<br />
+<img alt="Want to change color mode?" src="images/save_plots/question_on_changing_black_mode.png" height="150" />
+<br />
+
+* If user click 'Cancel'. The process of saving plots will be canceled for user 
+to change mode before restarting saving plots again.
+* If user click 'Continue'. The process of saving plots will be continue and the 
+image will be saved in black mode.
+<br />
+
+---------------------------
+## Step 2: Edit file path and select image's format
+Once clicking on 'Save Plot' button,  the 'Save Plot' dialog will pop up.
+
+<br />
+<br />
+<img alt="Select Image Format dialog" src="images/save_plots/save_file_dialog.png" height="200" />
+<br />
+
++ The default path to save the image file is preset in (1) text box.  If user 
+wants to change the path,  click on 'Save Directory button' to open file dialog 
+for changing path.
++ The default filename to save the image is preset in (2) text box.  User can 
+change the name in this box.
++ In side oval (3) are the radio buttons to select image format to save 
+file.
++ For 'PNG' format,  user can change DPI which is the resolution of the 
+image.  Other formats are vector formats which don't require resolution.
+
+Then user can click 'CANCEL' to cancel saving plot or click 'SAVE PLOT' to save
+the current plots to file.
\ No newline at end of file

documentation/99 _ test.md

@@ -39,7 +39,7 @@ printf("%s\n", syntaxHighlighting.doesItWork ? "Success!" : "Oof.");
 ^ This is a horizontal line
 
 v This is an image
-![An Image?](images/image.jpg)
+![An Image?](recreate_table_contents.png)
 
 ---
 Another horizontal line

sohstationviewer/conf/constants.py

@@ -50,8 +50,11 @@ TABLE_CONTENTS = "01 _ Table of Contents.help.md"
 SEARCH_RESULTS = "Search Results.md"
 
 # the list of all color modes
-ALL_COLOR_MODES = {'B', 'W'}
+ALL_COLOR_MODES = {'B': 'black', 'W': 'white'}
 
+# List of image formats. Have to put PNG at the beginning to go with
+# dpi in dialog
+IMG_FORMAT = ['PNG', 'PDF', 'EPS', 'SVG']
 # ================================================================= #
 #                      PLOTTING CONSTANT
 # ================================================================= #

sohstationviewer/controller/processing.py

@@ -6,18 +6,22 @@ channels, datatype
 import os
 import json
 import re
-import traceback
 from pathlib import Path
 from typing import List, Set, Optional, Dict, Tuple
 
 from PySide2.QtCore import QEventLoop, Qt
 from PySide2.QtGui import QCursor
 from PySide2.QtWidgets import QTextBrowser, QApplication
-from obspy.core import read as read_ms
-from obspy.io.reftek.core import Reftek130Exception
+from obspy.io import reftek
+
+from sohstationviewer.model.mseed_data.record_reader import RecordReader \
+    as MSeedRecordReader
+from sohstationviewer.model.mseed_data.record_reader_helper import \
+    MSeedReadError
+from sohstationviewer.model.mseed_data.mseed_reader import \
+    move_to_next_record
 
 from sohstationviewer.database.extract_data import get_signature_channels
-from sohstationviewer.model.data_type_model import DataTypeModel
 from sohstationviewer.model.handling_data import (
     read_mseed_chanids_from_headers)
 
@@ -28,69 +32,6 @@ from sohstationviewer.controller.util import (
 from sohstationviewer.view.util.enums import LogType
 
 
-def load_data(data_type: str, tracking_box: QTextBrowser, dir_list: List[str],
-              list_of_rt130_paths: List[Path],
-              req_wf_chans: List[str] = [], req_soh_chans: List[str] = [],
-              read_start: Optional[float] = None,
-              read_end: Optional[float] = None) -> DataTypeModel:
-    """
-    Load the data stored in list_of_dir and store it in a DataTypeModel object.
-    The concrete class of the data object is based on dataType. Run on the same
-    thread as its caller, and so will block the GUI if called on the main
-    thread. It is advisable to use model.data_loader.DataLoader to load data
-    unless it is necessary to load data in the main thread (e.g. if there is
-    a need to access the call stack).
-
-    :param data_type: type of data read
-    :param tracking_box: widget to display tracking info
-    :param dir_list: list of directories selected by users
-    :param list_of_rt130_paths: list of rt130 directories selected by users
-    :param req_wf_chans: requested waveform channel list
-    :param req_soh_chans: requested soh channel list
-    :param read_start: start time of read data
-    :param read_end: finish time of read data
-    :return data_object: object that keep the data read from
-        list_of_dir
-    """
-    data_object = None
-    if list_of_rt130_paths == []:
-        for d in dir_list:
-            if data_object is None:
-                try:
-                    data_object = DataTypeModel.create_data_object(
-                        data_type, tracking_box, d, [],
-                        req_wf_chans=req_wf_chans, req_soh_chans=req_soh_chans,
-                        read_start=read_start, read_end=read_end)
-                except Exception:
-                    fmt = traceback.format_exc()
-                    msg = f"Dir {d} can't be read due to error: {str(fmt)}"
-                    display_tracking_info(tracking_box, msg, LogType.WARNING)
-
-                # if data_object.has_data():
-                #     continue
-                # If no data can be read from the first dir, throw exception
-                # raise Exception("No data can be read from ", d)
-            # TODO: will work with select more than one dir later
-            # else:
-            #     data_object.readDir(d)
-
-    else:
-        try:
-            data_object = DataTypeModel.create_data_object(
-                data_type, tracking_box, [''], list_of_rt130_paths,
-                req_wf_chans=req_wf_chans, req_soh_chans=req_soh_chans,
-                read_start=read_start, read_end=read_end)
-        except Exception:
-            fmt = traceback.format_exc()
-            msg = f"RT130 selected can't be read due to error: {str(fmt)}"
-            display_tracking_info(tracking_box, msg, LogType.WARNING)
-
-    if data_object is None:
-        msg = "No data object created. Check with implementer"
-        display_tracking_info(tracking_box, msg, LogType.WARNING)
-    return data_object
-
-
 def read_mseed_channels(tracking_box: QTextBrowser, list_of_dir: List[str],
                         on_unittest: bool = False
                         ) -> Set[str]:
@@ -139,7 +80,7 @@ def read_mseed_channels(tracking_box: QTextBrowser, list_of_dir: List[str],
                 spr_gr_1_chan_ids.update(ret[3])
     if not on_unittest:
         QApplication.restoreOverrideCursor()
-    return sorted(list(soh_chan_ids)), sorted(list(mass_pos_chan_ids)),\
+    return sorted(list(soh_chan_ids)), sorted(list(mass_pos_chan_ids)), \
         sorted(list(wf_chan_ids)), sorted(list(spr_gr_1_chan_ids))
 
 
@@ -157,6 +98,7 @@ def detect_data_type(list_of_dir: List[str]) -> Optional[str]:
     sign_chan_data_type_dict = get_signature_channels()
 
     dir_data_type_dict = {}
+    is_multiplex = None
     for d in list_of_dir:
         data_type = "Unknown"
         for path, subdirs, files in os.walk(d):
@@ -165,17 +107,24 @@ def detect_data_type(list_of_dir: List[str]) -> Optional[str]:
                 if not validate_file(path2file, file_name):
                     continue
                 ret = get_data_type_from_file(path2file,
-                                              sign_chan_data_type_dict)
+                                              sign_chan_data_type_dict,
+                                              is_multiplex)
                 if ret is not None:
-                    data_type, chan = ret
-                    break
+                    d_type, is_multiplex = ret
+                    if d_type is not None:
+                        data_type = d_type
+                        break
             if data_type != "Unknown":
                 break
+
+        if is_multiplex is None:
+            raise Exception("No channel found for the data set")
+
         if data_type == "Unknown":
-            dir_data_type_dict[d] = ("Unknown", '_')
+            dir_data_type_dict[d] = "Unknown"
         else:
-            dir_data_type_dict[d] = (data_type, chan)
-    data_type_list = {d[0] for d in dir_data_type_dict.values()}
+            dir_data_type_dict[d] = data_type
+    data_type_list = list(set(dir_data_type_dict.values()))
     if len(data_type_list) > 1:
         dir_data_type_str = json.dumps(dir_data_type_dict)
         dir_data_type_str = re.sub(r'\{|\}|"', '', dir_data_type_str)
@@ -185,38 +134,78 @@ def detect_data_type(list_of_dir: List[str]) -> Optional[str]:
                f"Please have only data that related to each other.")
         raise Exception(msg)
 
-    elif data_type_list == {'Unknown'}:
-        msg = ("There are no known data detected.\n"
-               "Please select different folder(s).")
+    elif data_type_list == ['Unknown']:
+        msg = ("There are no known data detected.\n\n"
+               "Do you want to cancel to select different folder(s)\n"
+               "Or continue to read any available mseed file?")
         raise Exception(msg)
-
-    return list(dir_data_type_dict.values())[0][0]
+    return data_type_list[0], is_multiplex
 
 
 def get_data_type_from_file(
         path2file: Path,
-        sign_chan_data_type_dict: Dict[str, str]
-) -> Optional[Tuple[str, str]]:
+        sign_chan_data_type_dict: Dict[str, str],
+        is_multiplex: bool = None
+) -> Optional[Tuple[Optional[str], bool]]:
     """
-    + Try to read mseed data from given file
-        if catch TypeError: no data type detected => return None
-        if catch Reftek130Exception: data type => return data type RT130
-        otherwise data type is mseed which includes: q330, pegasus, centaur
-    + Continue to identify data type for a file by checking if the channel
-    in that file is a unique channel of a data type.
+    + Exclude files for waveform data to improve performance
+    + Loop through each record for file
+        If MSeedRecordReader gives Error; check if the file is RT130, report
+            data_type is RT130 or else, return to continue checking on another
+            file.
+        If there're more than one channels in a file, this file is multiplex.
+        If found signature channel, report the data_type of the file.
     :param path2file: absolute path to processed file
     :param sign_chan_data_type_dict: dict of unique chan for data
         type
+    :param is_multiplex: if the file is multiplex
     :return: detected data type, channel from which data type is detected
     """
-    try:
-        stream = read_ms(path2file)
-    except TypeError:
-        return
-    except Reftek130Exception:
-        return 'RT130', '_'
-
-    for trace in stream:
-        chan = trace.stats['channel']
+    wf_chan_posibilities = ['FH', 'FN',  # ≥ 1000 to < 5000
+                            'GH', 'GL',  # ≥ 1000 to < 5000
+                            'DH', 'DL',  # ≥ 250 to < 1000
+                            'CH', 'CN',  # ≥ 250 to < 1000
+                            'EH', 'EL', 'EP',  # ≥ 80
+                            'SH', 'SL', 'SP',  # ≥ 10 to < 80
+                            'HH', 'HN',  # ≥ 80
+                            'BH', 'BN',  # ≥ 10 to < 80
+                            'MH', 'MN', 'MP', 'ML',
+                            'LH', 'LL', 'LP', 'LN',
+                            'VP', 'VL', 'VL', 'VH',
+                            'UN', 'UP', 'UL', 'UH']
+
+    if any(x in path2file.name for x in wf_chan_posibilities):
+        # Skip checking waveform files which aren't signature channels
+        return None, False
+
+    file = open(path2file, 'rb')
+    chans_in_stream = set()
+    data_type = None
+    while 1:
+        is_eof = (file.read(1) == b'')
+        if is_eof:
+            break
+        file.seek(-1, 1)
+        current_record_start = file.tell()
+        try:
+            record = MSeedRecordReader(file)
+        except MSeedReadError:
+            file.close()
+            if reftek.core._is_reftek130(path2file):
+                return 'RT130', False
+            return
+
+        chan = record.record_metadata.channel
+        if is_multiplex is None:
+            chans_in_stream.add(chan)
+            if len(chans_in_stream) > 1:
+                is_multiplex = True
         if chan in sign_chan_data_type_dict.keys():
-            return sign_chan_data_type_dict[chan], chan
+            data_type = sign_chan_data_type_dict[chan]
+            if is_multiplex:
+                file.close()
+                return data_type, is_multiplex
+        move_to_next_record(file, current_record_start, record)
+    file.close()
+    is_multiplex = True if len(chans_in_stream) > 1 else False
+    return data_type, is_multiplex

sohstationviewer/controller/util.py

@@ -66,19 +66,20 @@ def display_tracking_info(tracking_box: QTextBrowser, text: str,
     msg = {'text': text}
     if type == LogType.ERROR:
         msg['color'] = 'white'
-        msg['bgcolor'] = '#e46269'
+        msg['bgcolor'] = '#c45259'
     elif type == LogType.WARNING:
-        msg['color'] = '#ffd966'
-        msg['bgcolor'] = 'orange'
+        msg['color'] = 'white'
+        msg['bgcolor'] = '#c4a347'
     else:
         msg['color'] = 'blue'
         msg['bgcolor'] = 'white'
     html_text = """<body>
-        <div style='color:%(color)s; background-color:%(bgcolor)s'>
-            %(text)s
+        <div style='color:%(color)s'>
+            <strong>%(text)s</strong>
         </div>
         </body>"""
     tracking_box.setHtml(html_text % msg)
+    tracking_box.setStyleSheet(f"background-color: {msg['bgcolor']}")
     # parent.update()
     tracking_box.repaint()