diff --git a/sohstationviewer/controller/processing.py b/sohstationviewer/controller/processing.py
index 83cc1d96b99daf6428a31f620234f33c645623d0..34da617fed44d4d248d0854a2698c043da0e8ead 100644
--- a/sohstationviewer/controller/processing.py
+++ b/sohstationviewer/controller/processing.py
@@ -14,16 +14,16 @@ from PySide2.QtGui import QCursor
from PySide2.QtWidgets import QTextBrowser, QApplication
from obspy.io import reftek
+from sohstationviewer.model.mseed_data.mseed_reader import \
+ move_to_next_record
+
from sohstationviewer.model.mseed_data.record_reader import RecordReader \
as MSeedRecordReader
from sohstationviewer.model.mseed_data.record_reader_helper import \
MSeedReadError
-from sohstationviewer.model.mseed_data.mseed_reader import \
- move_to_next_record
-
from sohstationviewer.database.extract_data import get_signature_channels
from sohstationviewer.model.handling_data import (
- read_mseed_chanids_from_headers)
+ check_chan)
from sohstationviewer.controller.util import (
validate_file, display_tracking_info
@@ -32,7 +32,61 @@ from sohstationviewer.controller.util import (
from sohstationviewer.view.util.enums import LogType
+def _read_mseed_chanids(path2file: Path, is_multiplex) \
+ -> Tuple[Set[str], Set[str], Set[str], Set[str]]:
+ """
+ from the given file get set of channel ids for soh, mass position,
+ waveform, soh with sample rate greater than 1 which are
+ calibration signals .
+ :param path2file: path to file
+ :param is_multiplex: flag that tell if data has more than one channel in
+ a file
+ :return soh_chan_ids: list of chan_ids for SOH
+ :return mass_pos_chan_ids: list of chan_ids for mass position
+ :return wf_chan_ids: list of chan_ids for waveform
+ :return spr_gr_1_chan_ids: list of chan_ids for SOH with sample rate > 1
+ """
+ soh_chan_ids = set()
+ mass_pos_chan_ids = set()
+ wf_chan_ids = set()
+ spr_gr_1_chan_ids = set()
+ file = open(path2file, 'rb')
+ while 1:
+ is_eof = (file.read(1) == b'')
+ if is_eof:
+ break
+ file.seek(-1, 1)
+ current_record_start = file.tell()
+ try:
+ record = MSeedRecordReader(file)
+ except MSeedReadError:
+ file.close()
+ raise Exception(f"File{path2file}: Data type isn't MSeed.")
+ chan_id = record.record_metadata.channel
+ chan_type = check_chan(chan_id, [], ['*'], True, True)
+ if chan_type is False:
+ if not is_multiplex:
+ break
+ continue
+ if chan_type == 'MP':
+ mass_pos_chan_ids.add(chan_id)
+ elif chan_type == 'SOH':
+ sample_rate = record.record_metadata.sample_rate
+ if sample_rate <= 1:
+ soh_chan_ids.add(chan_id)
+ else:
+ spr_gr_1_chan_ids.add(chan_id)
+ else:
+ wf_chan_ids.add(chan_id)
+ if not is_multiplex:
+ break
+ move_to_next_record(file, current_record_start, record)
+ file.close()
+ return soh_chan_ids, mass_pos_chan_ids, wf_chan_ids, spr_gr_1_chan_ids
+
+
def read_mseed_channels(tracking_box: QTextBrowser, list_of_dir: List[str],
+ is_multiplex: bool,
on_unittest: bool = False
) -> Set[str]:
"""
@@ -43,6 +97,7 @@ def read_mseed_channels(tracking_box: QTextBrowser, list_of_dir: List[str],
include_mp123zne and include_mp456uvw for MSeed are False
:param tracking_box: widget to display tracking info
:param list_of_dir: list of directories selected by users
+ :param is_multiplex: flag that tell if data is multiplex
:param on_unittest: flag to avoid cursor code to display tracking_info
:return data_object.channels: set of channels present in listofDir
"""
@@ -70,14 +125,30 @@ def read_mseed_channels(tracking_box: QTextBrowser, list_of_dir: List[str],
count += 1
if count % 10 == 0:
display_tracking_info(
- tracking_box, f'Read {count} file headers/ SOH files',
+ tracking_box, f'Read {count} file headers',
LogType.INFO)
-
- ret = read_mseed_chanids_from_headers(path2file, file_name)
- soh_chan_ids.update(ret[0])
- mass_pos_chan_ids.update(ret[1])
- wf_chan_ids.update(ret[2])
- spr_gr_1_chan_ids.update(ret[3])
+ if reftek.core._is_reftek130(path2file):
+ # if data_type is RT130, this function shouldn't be called
+ display_tracking_info(
+ tracking_box, "There is something wrong with the "
+ "work-flow. read_mseed_channels shouldn't be fed an "
+ f"RT130 file: {path2file}",
+ LogType.ERROR
+ )
+ return [], [], [], []
+ try:
+ # skip when data type isn't mseed
+ ret = _read_mseed_chanids(path2file, is_multiplex)
+ soh_chan_ids.update(ret[0])
+ mass_pos_chan_ids.update(ret[1])
+ wf_chan_ids.update(ret[2])
+ spr_gr_1_chan_ids.update(ret[3])
+ except Exception:
+ display_tracking_info(
+ tracking_box, f'Skip non-mseed file: {file_name}',
+ LogType.WARNING
+ )
+ pass
if not on_unittest:
QApplication.restoreOverrideCursor()
return sorted(list(soh_chan_ids)), sorted(list(mass_pos_chan_ids)), \
diff --git a/sohstationviewer/database/soh.db b/sohstationviewer/database/soh.db
index cbcf2ad15f3ee0094460b02a42446a03dc49f09e..4b0ea0a9262a978f1b1e72bcb45b4774afb180d5 100755
Binary files a/sohstationviewer/database/soh.db and b/sohstationviewer/database/soh.db differ
diff --git a/sohstationviewer/model/general_data/general_data.py b/sohstationviewer/model/general_data/general_data.py
index e1dc2c23880f15848b067f5b29a6228b5796fc4b..4e8924e0e9a4ec239e5fd10ed8e61f7c9ceb2b5a 100644
--- a/sohstationviewer/model/general_data/general_data.py
+++ b/sohstationviewer/model/general_data/general_data.py
@@ -318,20 +318,6 @@ class GeneralData():
execute_db(f'UPDATE PersistentData SET FieldValue="{self.tmp_dir}" '
f'WHERE FieldName="tempDataDirectory"')
- def check_not_found_soh_channels(self):
- # FROM data_type_model.Data_Type_Model.check_not_found_soh_channels
- all_chans_meet_req = (
- list(self.soh_data[self.selected_key].keys()) +
- list(self.mass_pos_data[self.selected_key].keys()) +
- list(self.log_data[self.selected_key].keys()))
-
- not_found_chans = [c for c in self.req_soh_chans
- if c not in all_chans_meet_req]
- if not_found_chans != []:
- msg = (f"No data found for the following channels: "
- f"{', '.join( not_found_chans)}")
- self.processing_log.append((msg, LogType.WARNING))
-
def sort_all_data(self):
"""
FROM data_type_model.Data_Type_Model.sort_all_data
@@ -341,51 +327,51 @@ class GeneralData():
because it is created from log data which is sorted in
prepare_soh_data_from_log_data()
"""
- sort_data(self.waveform_data[self.selected_key])
- sort_data(self.mass_pos_data[self.selected_key])
- try:
- sort_data(self.soh_data[self.selected_key])
- except KeyError:
- # Reftek's SOH trace doesn't have startTmEpoch and
- # actually soh_data consists of only one trace
- pass
+ for key in self.keys:
+ sort_data(self.waveform_data[key])
+ sort_data(self.mass_pos_data[key])
+ try:
+ sort_data(self.soh_data[key])
+ except KeyError:
+ # Reftek's SOH trace doesn't have startTmEpoch and
+ # actually soh_data consists of only one trace
+ pass
def combine_all_data(self):
- combine_data(self.selected_key, self.waveform_data, self.gap_minimum)
- combine_data(self.selected_key, self.mass_pos_data, self.gap_minimum)
- try:
- combine_data(self.selected_key, self.soh_data, self.gap_minimum)
- except KeyError:
- # Reftek's SOH trace doesn't have startTmEpoch and
- # actually soh_data consists of only one trace
- pass
+ for key in self.keys:
+ combine_data(key, self.waveform_data, self.gap_minimum)
+ combine_data(key, self.mass_pos_data, self.gap_minimum)
+ try:
+ combine_data(key, self.soh_data, self.gap_minimum)
+ except KeyError:
+ # Reftek's SOH trace doesn't have startTmEpoch and
+ # actually soh_data consists of only one trace
+ pass
def retrieve_gaps_from_data_dicts(self):
"""
Getting gaps from each data_dicts then squash all related gaps
"""
- self.gaps[self.selected_key] = []
- retrieve_gaps_from_data_dict(
- self.selected_key, self.soh_data, self.gaps)
- retrieve_gaps_from_data_dict(
- self.selected_key, self.mass_pos_data, self.gaps)
- retrieve_gaps_from_data_dict(
- self.selected_key, self.waveform_data, self.gaps)
+ for key in self.keys:
+ self.gaps[key] = []
+ retrieve_gaps_from_data_dict(key, self.soh_data, self.gaps)
+ retrieve_gaps_from_data_dict(key, self.mass_pos_data, self.gaps)
+ retrieve_gaps_from_data_dict(key, self.waveform_data, self.gaps)
- self.gaps[self.selected_key] = squash_gaps(
- self.gaps[self.selected_key])
+ self.gaps[key] = squash_gaps(self.gaps[key])
def retrieve_data_time_from_data_dicts(self):
"""
Going through each data_dict to update the data_time to be
[min of startTimeEpoch, max of endTimeEpoch] for each station.
"""
- retrieve_data_time_from_data_dict(
- self.selected_key, self.soh_data, self.data_time)
- retrieve_data_time_from_data_dict(
- self.selected_key, self.mass_pos_data, self.data_time)
- retrieve_data_time_from_data_dict(
- self.selected_key, self.waveform_data, self.data_time)
+ for key in self.keys:
+ retrieve_data_time_from_data_dict(
+ key, self.soh_data, self.data_time)
+ retrieve_data_time_from_data_dict(
+ key, self.mass_pos_data, self.data_time)
+ retrieve_data_time_from_data_dict(
+ key, self.waveform_data, self.data_time)
def fill_empty_data(self):
"""
@@ -406,12 +392,13 @@ class GeneralData():
Applying convert_factor to avoid using flags to prevent double
applying convert factor when plotting
"""
- apply_convert_factor_to_data_dict(
- self.selected_key, self.soh_data, self.data_type)
- apply_convert_factor_to_data_dict(
- self.selected_key, self.mass_pos_data, self.data_type)
- apply_convert_factor_to_data_dict(
- self.selected_key, self.waveform_data, self.data_type)
+ for key in self.keys:
+ apply_convert_factor_to_data_dict(
+ key, self.soh_data, self.data_type)
+ apply_convert_factor_to_data_dict(
+ key, self.mass_pos_data, self.data_type)
+ apply_convert_factor_to_data_dict(
+ key, self.waveform_data, self.data_type)
def reset_all_selected_data(self):
"""
diff --git a/sohstationviewer/view/channel_prefer_dialog.py b/sohstationviewer/view/channel_prefer_dialog.py
index 0e08d7f670b19bea22c206d759e55cafd1e0b774..f1166c55fbbe270de0ac941f9574abe7d13bac6f 100755
--- a/sohstationviewer/view/channel_prefer_dialog.py
+++ b/sohstationviewer/view/channel_prefer_dialog.py
@@ -406,7 +406,7 @@ class ChannelPreferDialog(OneWindowAtATimeDialog):
data_type = 'RT130'
else:
try:
- data_type = detect_data_type(self.dir_names)
+ data_type, is_multiplex = detect_data_type(self.dir_names)
except Exception as e:
QtWidgets.QMessageBox.warning(self, "Scan Channels", str(e))
return
@@ -428,7 +428,7 @@ class ChannelPreferDialog(OneWindowAtATimeDialog):
else:
self.scan_chan_btn.setEnabled(True)
ret = read_mseed_channels(self.tracking_info_text_browser,
- self.dir_names)
+ self.dir_names, is_multiplex)
if ret == ([], [], [], []):
msg = "No data can be read from " + ', '.join(self.dir_names)
return QtWidgets.QMessageBox.warning(self, "No data", msg)
diff --git a/sohstationviewer/view/plotting/plotting_widget/multi_threaded_plotting_widget.py b/sohstationviewer/view/plotting/plotting_widget/multi_threaded_plotting_widget.py
index 17f0520db3f126a878bcb31550209cebded83c4d..7bf3f9212dbcfcc81bc86995bc164e07b94375a7 100644
--- a/sohstationviewer/view/plotting/plotting_widget/multi_threaded_plotting_widget.py
+++ b/sohstationviewer/view/plotting/plotting_widget/multi_threaded_plotting_widget.py
@@ -144,11 +144,6 @@ class MultiThreadedPlottingWidget(PlottingWidget):
# instruction here
self.processing_log.append((msg, LogType.WARNING))
- # These channels contain GPS data which are only shown in
- # the GPS dialog.
- if chan_id in {'GLO', 'VLO', 'VLA', 'GLA', 'VEL', 'VNS', 'GNS',
- 'GEL'}:
- continue
plotting_data[chan_id]['chan_db_info'] = chan_db_info
if not_plot_chans != []:
msg = (f"The database settings 'plotType' or 'height' show not to "
diff --git a/tests/controller/test_processing.py b/tests/controller/test_processing.py
index a4cdf4a0f3be4b01ad5d3f0d2b109827bd842328..fb377d015d60633f8ab05c3ae8051c7f98d4dbd9 100644
--- a/tests/controller/test_processing.py
+++ b/tests/controller/test_processing.py
@@ -21,8 +21,8 @@ pegasus_dir = TEST_DATA_DIR.joinpath('Pegasus-sample/Pegasus_SVC4/soh')
multiplex_dir = TEST_DATA_DIR.joinpath('Q330_multiplex')
-class TestReadChannels(TestCase):
- """Test suite for load_data and read_mseed_channels."""
+class TestReadMSeedChannel(TestCase):
+ """Test suite for read_mseed_channels."""
def setUp(self) -> None:
"""Set up test fixtures."""
@@ -40,61 +40,95 @@ class TestReadChannels(TestCase):
Test basic functionality of load_data - the given directory contains
MSeed data.
"""
- q330_soh_channels = sorted(['LOG', 'VKI'])
- q330_mass_pos_channels = ['VM1']
- q330_wf_channels = ['HHE', 'LHE']
- q330_spr_gt_1 = []
- ret = read_mseed_channels(self.widget_stub, [q330_dir], True)
- self.assertListEqual(ret[0], q330_soh_channels)
- self.assertListEqual(ret[1], q330_mass_pos_channels)
- self.assertListEqual(ret[2], q330_wf_channels)
- self.assertListEqual(ret[3], q330_spr_gt_1)
-
- centaur_soh_channels = sorted(
- ['VDT', 'EX3', 'GEL', 'VEC', 'EX2', 'LCE', 'EX1', 'GLA', 'LCQ',
- 'GPL', 'GNS', 'GST', 'VCO', 'GAN', 'GLO', 'VPB', 'VEI'])
- centaur_mass_pos_channels = sorted(['VM1', 'VM2', 'VM3'])
- centaur_wf_channels = []
- centaur_spr_gt_1 = []
- ret = read_mseed_channels(self.widget_stub, [centaur_dir], True)
- self.assertListEqual(ret[0], centaur_soh_channels)
- self.assertListEqual(ret[1], centaur_mass_pos_channels)
- self.assertListEqual(ret[2], centaur_wf_channels)
- self.assertListEqual(ret[3], centaur_spr_gt_1)
-
- pegasus_soh_channels = sorted(['VDT', 'VE1'])
- pegasus_mass_pos_channels = sorted(['VM1'])
- pegasus_wf_channels = []
- pegasus_spr_gt_1 = []
- ret = read_mseed_channels(self.widget_stub, [pegasus_dir], True)
- self.assertListEqual(ret[0], pegasus_soh_channels)
- self.assertListEqual(ret[1], pegasus_mass_pos_channels)
- self.assertListEqual(ret[2], pegasus_wf_channels)
- self.assertListEqual(ret[3], pegasus_spr_gt_1)
-
- multiplex_soh_channels = ['LOG']
- multiplex_mass_pos_channels = []
- multiplex_wf_channels = sorted(
- ['BH1', 'BH2', 'BH3', 'BH4', 'BH5', 'BH6',
- 'EL1', 'EL2', 'EL4', 'EL5', 'EL6', 'ELZ'])
- multiplex_spr_gt_1 = sorted(
- ['BS1', 'BS2', 'BS3', 'BS4', 'BS5', 'BS6',
- 'ES1', 'ES2', 'ES3', 'ES4', 'ES5', 'ES6',
- 'LS1', 'LS2', 'LS3', 'LS4', 'LS5', 'LS6',
- 'SS1', 'SS2', 'SS3', 'SS4', 'SS5', 'SS6'])
- ret = read_mseed_channels(self.widget_stub, [multiplex_dir], True)
- self.assertListEqual(ret[0], multiplex_soh_channels)
- self.assertListEqual(ret[1], multiplex_mass_pos_channels)
- self.assertListEqual(ret[2], multiplex_wf_channels)
- self.assertListEqual(ret[3], multiplex_spr_gt_1)
+ with self.subTest("q330 - non multiplex"):
+ q330_soh_channels = sorted(['LOG', 'VKI'])
+ q330_mass_pos_channels = ['VM1']
+ q330_wf_channels = ['HHE', 'LHE']
+ q330_spr_gt_1 = []
+ ret = read_mseed_channels(self.widget_stub, [q330_dir],
+ False, True)
+ self.assertListEqual(ret[0], q330_soh_channels)
+ self.assertListEqual(ret[1], q330_mass_pos_channels)
+ self.assertListEqual(ret[2], q330_wf_channels)
+ self.assertListEqual(ret[3], q330_spr_gt_1)
+
+ with self.subTest(("centaur - multiplex - is_multiplex=True")):
+ centaur_soh_channels = sorted(
+ ['VDT', 'EX3', 'GEL', 'VEC', 'EX2', 'LCE', 'EX1', 'GLA', 'LCQ',
+ 'GPL', 'GNS', 'GST', 'VCO', 'GAN', 'GLO', 'VPB', 'VEI'])
+ centaur_mass_pos_channels = sorted(['VM1', 'VM2', 'VM3'])
+ centaur_wf_channels = []
+ centaur_spr_gt_1 = []
+ ret = read_mseed_channels(self.widget_stub, [centaur_dir],
+ True, True)
+ self.assertListEqual(ret[0], centaur_soh_channels)
+ self.assertListEqual(ret[1], centaur_mass_pos_channels)
+ self.assertListEqual(ret[2], centaur_wf_channels)
+ self.assertListEqual(ret[3], centaur_spr_gt_1)
+
+ with self.subTest("centaur - multiplex - is_multiplex=False"):
+ # not all channels detected if is_multiplex isn't set
+ centaur_soh_channels = sorted(['GEL'])
+ centaur_mass_pos_channels = sorted([])
+ centaur_wf_channels = []
+ centaur_spr_gt_1 = []
+ ret = read_mseed_channels(self.widget_stub, [centaur_dir],
+ False, True)
+ self.assertListEqual(ret[0], centaur_soh_channels)
+ self.assertListEqual(ret[1], centaur_mass_pos_channels)
+ self.assertListEqual(ret[2], centaur_wf_channels)
+ self.assertListEqual(ret[3], centaur_spr_gt_1)
+
+ with self.subTest("pegasus - non multiplex"):
+ pegasus_soh_channels = sorted(['VDT', 'VE1'])
+ pegasus_mass_pos_channels = sorted(['VM1'])
+ pegasus_wf_channels = []
+ pegasus_spr_gt_1 = []
+ ret = read_mseed_channels(self.widget_stub, [pegasus_dir],
+ False, True)
+ self.assertListEqual(ret[0], pegasus_soh_channels)
+ self.assertListEqual(ret[1], pegasus_mass_pos_channels)
+ self.assertListEqual(ret[2], pegasus_wf_channels)
+ self.assertListEqual(ret[3], pegasus_spr_gt_1)
+
+ with self.subTest("q330 - multiplex - is_multiplex=True"):
+ multiplex_soh_channels = ['LOG']
+ multiplex_mass_pos_channels = []
+ multiplex_wf_channels = sorted(
+ ['BH1', 'BH2', 'BH3', 'BH4', 'BH5', 'BH6',
+ 'EL1', 'EL2', 'EL4', 'EL5', 'EL6', 'ELZ'])
+ multiplex_spr_gt_1 = sorted(
+ ['BS1', 'BS2', 'BS3', 'BS4', 'BS5', 'BS6',
+ 'ES1', 'ES2', 'ES3', 'ES4', 'ES5', 'ES6',
+ 'LS1', 'LS2', 'LS3', 'LS4', 'LS5', 'LS6',
+ 'SS1', 'SS2', 'SS3', 'SS4', 'SS5', 'SS6'])
+ ret = read_mseed_channels(self.widget_stub, [multiplex_dir],
+ True, True)
+ self.assertListEqual(ret[0], multiplex_soh_channels)
+ self.assertListEqual(ret[1], multiplex_mass_pos_channels)
+ self.assertListEqual(ret[2], multiplex_wf_channels)
+ self.assertListEqual(ret[3], multiplex_spr_gt_1)
+
+ with self.subTest("q330 - multiplex - is_multiplex=False"):
+ # not all channels detected if is_multiplex isn't set
+ multiplex_soh_channels = []
+ multiplex_mass_pos_channels = []
+ multiplex_wf_channels = sorted(['EL1'])
+ multiplex_spr_gt_1 = sorted([])
+ ret = read_mseed_channels(self.widget_stub, [multiplex_dir],
+ False, True)
+ self.assertListEqual(ret[0], multiplex_soh_channels)
+ self.assertListEqual(ret[1], multiplex_mass_pos_channels)
+ self.assertListEqual(ret[2], multiplex_wf_channels)
+ self.assertListEqual(ret[3], multiplex_spr_gt_1)
def test_read_channels_rt130_dir(self):
"""
Test basic functionality of load_data - the given directory contains
RT130 data.
"""
- with self.assertRaises(Exception):
- read_mseed_channels(self.widget_stub, [rt130_dir], True)
+ ret = read_mseed_channels(self.widget_stub, [rt130_dir], True, True)
+ self.assertEqual(ret, ([], [], [], []))
def test_read_mseed_channels_no_dir(self):
"""
@@ -102,7 +136,7 @@ class TestReadChannels(TestCase):
given.
"""
no_dir = []
- ret = read_mseed_channels(self.widget_stub, no_dir, True)
+ ret = read_mseed_channels(self.widget_stub, no_dir, True, True)
self.assertEqual(ret, ([], [], [], []))
def test_read_mseed_channels_dir_does_not_exist(self):
@@ -111,11 +145,12 @@ class TestReadChannels(TestCase):
does not exist.
"""
empty_name_dir = ['']
- ret = read_mseed_channels(self.widget_stub, empty_name_dir, True)
+ ret = read_mseed_channels(self.widget_stub, empty_name_dir, True, True)
self.assertEqual(ret, ([], [], [], []))
non_existent_dir = ['non_existent_dir']
- ret = read_mseed_channels(self.widget_stub, non_existent_dir, True)
+ ret = read_mseed_channels(self.widget_stub, non_existent_dir,
+ True, True)
self.assertEqual(ret, ([], [], [], []))
def test_read_mseed_channels_empty_dir(self):
@@ -124,7 +159,8 @@ class TestReadChannels(TestCase):
is empty.
"""
with TemporaryDirectory() as empty_dir:
- ret = read_mseed_channels(self.widget_stub, [empty_dir], True)
+ ret = read_mseed_channels(self.widget_stub, [empty_dir],
+ True, True)
self.assertEqual(ret, ([], [], [], []))
def test_read_mseed_channels_empty_data_dir(self):
@@ -134,7 +170,8 @@ class TestReadChannels(TestCase):
"""
with TemporaryDirectory() as outer_dir:
with TemporaryDirectory(dir=outer_dir):
- ret = read_mseed_channels(self.widget_stub, [outer_dir], True)
+ ret = read_mseed_channels(self.widget_stub, [outer_dir],
+ True, True)
self.assertEqual(ret, ([], [], [], []))