Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • software_public/passoft/sohstationviewer
1 result
Show changes
Commits on Source (20)
Showing
with 696 additions and 346 deletions
<img alt="Whole Section" src="images/select_soh/whole_section.png" width="250" />
<br />
# Selecting SOH Channels
---------------------------
---------------------------
## Selecting all SOH channels
To read all SOH channels in the selected data set, check the checkbox "All SOH".
<br />
<img alt="Select all SOH channels" src="images/select_soh/all_soh_checkbox.png" height="30" />
<br />
If this checkbox is unchecked, the latest Preferred SOH list will be used when reading the data set.
<br />
<img alt="Current preferred SOH list" src="images/select_soh/current_soh_list.png" height="30" />
<br />
## Get a preferred SOH channel list to read
Click button "Pref" to open "SOH Channel Preferences" to create/edit/select a list of SOH channel to read.
<br />
<img alt="SOH Channel Preferences Dialog" src="images/select_soh/soh_channel_preferences.png" height="300" />
<br />
### SOH channel list
Each row in the table at the center of "SOH Channel Preferences" window represents an SOH list.
<br />
<img alt="A Row of SOH List" src="images/select_soh/full_row.png" height="75" />
<br />
### Select an SOH channel list
Check on the radio button (1) at the beginning of each row to select the list.
The radio button can also be checked automatically when user try to edit the row.
### Create a new list
+ Add name of the list to the Name box (2)
+ Select a Data Type (3)
+ Add preferred channels to Preferred SOH List box (4). Channels are separated by commas.
+ The Preferred SOH List can be edited directly in box 4, but user can also click on button EDIT (5) to open a separated dialog for editing easier.
+ Button CLR (6) is for quickly empty the Preferred SOH List box.
### Bottom buttons
<br />
<img alt="Buttons" src="images/select_soh/buttons.png" height="30" />
<br />
**Beside adding preferred channels manually, users have two other options to start the list:**
+ "Add DB channels". All SOH channels for the selected Data Type in the database will be added to Preferred SOH List box.
+ "Scan Channel from Data Source". SOHView will scan for all available SOH channels in the data set.
**For saving Preferred SOH List**
+ "Save": Save any changes to database
+ "Save - Add to Main": Save any changes to database and add the selected Preferred SOH's name to Main Window so that its SOH list will be included when reading the data set.
\ No newline at end of file
documentation/images/select_soh/all_soh_checkbox.png

7.21 KiB

documentation/images/select_soh/buttons.png

19.4 KiB

documentation/images/select_soh/current_soh_list.png

4.59 KiB

documentation/images/select_soh/full_row.png

38.1 KiB

documentation/images/select_soh/soh_channel_preferences.png

319 KiB

documentation/images/select_soh/whole_section.png

10.2 KiB

......@@ -92,14 +92,14 @@ def format_time(time: Union[UTCDateTime, float], date_mode: str,
format = ''
if date_mode == 'YYYY-MM-DD':
format = '%Y-%m-%d'
elif date_mode == 'YYYYMMDD':
format = '%Y%m%d'
elif date_mode == 'YYYYMMMDD':
format = '%Y%b%d'
elif date_mode == 'YYYY:DOY':
format = '%Y:%j'
if time_mode == 'HH:MM:SS':
format += " %H:%M:%S"
ret = t.strftime(format)
ret = t.strftime(format).upper()
return ret
......
......@@ -369,3 +369,12 @@ class DataTypeModel():
def save_temp_data_folder_to_database(self):
execute_db(f'UPDATE PersistentData SET FieldValue="{self.tmp_dir}" '
f'WHERE FieldName="tempDataDirectory"')
def check_not_found_soh_chans(self):
not_found_soh_chans = [
c for c in self.req_soh_chans
if c not in self.soh_data[self.selected_key].keys()]
if not_found_soh_chans != []:
msg = (f"No data found for soh channels: "
f"{', '.join( not_found_soh_chans)}")
self.processing_log.append((msg, LogType.WARNING))
......@@ -173,7 +173,7 @@ def read_waveform_trace(trace: Trace, sta_id: Union[Tuple[str, str], str],
return tr
def read_waveform_mseed(path2file: str, file_name: str,
def read_waveform_mseed(path2file: str,
sta_id: str, chan_id: str,
traces_info: List, data_time: List[float], tmp_dir: str
) -> None:
......@@ -181,7 +181,6 @@ def read_waveform_mseed(path2file: str, file_name: str,
Read traces from waveform mseed file to append to tracesInfo.
data_time is update for new min and max time.
:param path2file: absolute path to waveform mseed file
:param file_name: name of waveform mseed file
:param sta_id: station ID from indexing
:param chan_id: channel ID from indexing
:param traces_info: holder of traces_info, refer
......@@ -636,107 +635,6 @@ def get_each_day_5_min_list(start_tm: float, end_tm: float) -> np.ndarray:
return every_day_5_min_list
def get_trim_tps_data(chan: Dict, start_tm: float, end_tm: float,
every_day_5_min_list: List[List[float]]
) -> Optional[bool]:
"""
Different with soh_data where times and data are each in one np.array,
in waveform_data, times and data are each kept in a list of np.memmap
files along with startTmEpoch and endTmEpoch.
chan['startIdx'] and chan['endIdx'] will be identify to exclude np.memmap
files that aren't in the zoom time range (startTm, endTm).
Data in np.memmap will be trimmed according to times then time-power-square
value for each 5 minutes will be calculated and saved in
chan['tps-data']: np.mean(np.square(5m data))
:param chan: dict - chan dictionary, refer to
DataTypeModel.__init__.waveform_data[key]['readData'][chan_id]
:param start_tm: float - start time of zoomed section
:param end_tm: float - end time of zoomed section
:param every_day_5_min_list: [[288 of floats], ] - the list of all start
of five minutes for every day in which each day has 288 of 5 minutes.
"""
# preset all 0 for all 5 minutes for each day
tps_data = np.zeros((len(every_day_5_min_list), const.NO_5M_DAY))
# zoom in to the given range
chan['startIdx'] = 0
chan['endIdx'] = len(chan['tracesInfo'])
if ((start_tm > chan['tracesInfo'][-1]['endTmEpoch']) or
(end_tm < chan['tracesInfo'][0]['startTmEpoch'])):
return False
indexes = [index for index, tr in enumerate(chan['tracesInfo'])
if tr['startTmEpoch'] > start_tm]
if indexes != []:
chan['startIdx'] = indexes[0]
if chan['startIdx'] > 0:
chan['startIdx'] -= 1 # startTm in middle of trace
else:
chan['startIdx'] = 0
indexes = [idx for (idx, tr) in enumerate(chan['tracesInfo'])
if tr['endTmEpoch'] <= end_tm]
if indexes != []:
chan['endIdx'] = indexes[-1]
if chan['endIdx'] < len(chan['tracesInfo']) - 1:
chan['endIdx'] += 1 # endTm in middle of trace
else:
chan['endIdx'] = 0
chan['endIdx'] += 1 # a[x:y+1] = [a[x], ...a[y]
z_traces_info = chan['tracesInfo'][chan['startIdx']:chan['endIdx']]
spr = chan['samplerate']
chan['tps_data'] = []
start_tps_tm = 0
acc_data_list = []
for tr_idx, tr in enumerate(z_traces_info):
times = np.memmap(tr['times_f'],
dtype='int64', mode='r',
shape=tr['size'])
data = np.memmap(tr['data_f'],
dtype='int64', mode='r',
shape=tr['size'])
start_index = 0
if tr_idx == 0:
# get index of times with closet value to startTm
start_index = np.abs(times - start_tm).argmin()
start_tps_tm = times[start_index]
# identify index in case of overlaps or gaps
index = np.where((every_day_5_min_list <= times[start_index]) &
(every_day_5_min_list + const.SEC_5M >
times[start_index]))
curr_row = index[0][0]
curr_col = index[1][0]
next_tps_tm = start_tps_tm + const.SEC_5M
while end_tm >= next_tps_tm:
next_index = int(start_index + spr * const.SEC_5M)
if next_index >= tr['size']:
acc_data_list.append(data[start_index:tr['size']])
break
else:
acc_data_list.append(np.square(data[start_index:next_index]))
acc_data = np.hstack(acc_data_list)
if acc_data.size == 0:
tps_data[curr_row, curr_col] = 0
else:
tps_data[curr_row, curr_col] = np.mean(acc_data)
start_index = next_index
curr_col += 1
acc_data_list = []
if curr_col == const.NO_5M_DAY:
curr_col = 0
curr_row += 1
next_tps_tm += const.SEC_5M
chan['tps_data'] = tps_data
def find_tps_tm(given_tm: float, each_day_5_min_list: List[List[float]]
) -> Tuple[float, float]:
"""
......
......@@ -58,6 +58,7 @@ class MSeed(DataTypeModel):
if len(self.req_wf_chans) != 0:
self.read_wf_files(self.selected_key)
self.check_not_found_soh_chans()
self.get_gps_data_q330()
def read_soh_and_index_waveform(self, folder: str):
......@@ -280,6 +281,8 @@ class MSeed(DataTypeModel):
"""
count = 0
if sta_id not in self.waveform_data.keys():
return
for chan_id in self.waveform_data[sta_id]['filesInfo']:
traces_info = self.waveform_data[sta_id][
'readData'][chan_id]['tracesInfo']
......@@ -301,7 +304,6 @@ class MSeed(DataTypeModel):
if not has_data:
continue
read_waveform_mseed(file_info['path2file'],
file_info['fileName'],
sta_id, chan_id, traces_info,
self.data_time[sta_id], self.tmp_dir)
file_info['read'] = True
......
......@@ -15,7 +15,7 @@ class LogInfo():
def __init__(self, parent: RT130,
track_info: Callable[[str, LogType], None], log_text: str,
key: Tuple[str, LogType], req_data_streams: List[int],
is_log_file: bool = False):
req_soh_chans: List[str], is_log_file: bool = False):
"""
Help to extract channel data from LogText which include SOH Message and
Event message.
......@@ -23,7 +23,8 @@ class LogInfo():
:param track_info: to track data processing
:param log_text: SOH and Event messages in time order
:param key: ID of the data set including unit_id and exp_no
:param req_data_streams: requested data stream ID
:param req_data_streams: requested data stream ID,
:param req_soh_chans: requested data stream ID
:param is_log_file: flag indicate if this is a log file
"""
self.parent = parent
......@@ -32,6 +33,7 @@ class LogInfo():
self.key = key
self.unit_id, self.exp_no = key
self.req_data_streams = req_data_streams
self.req_soh_chans = req_soh_chans
self.is_log_file = is_log_file
"""
track_year to add year to time since year time not include year after
......@@ -311,6 +313,8 @@ class LogInfo():
:param idx: int - index of SOH message line
"""
if self.req_soh_chans and chan_id not in self.req_soh_chans:
return
if chan_id not in self.chans:
self.chans[chan_id] = {}
self.chans[chan_id]['orgTrace'] = {
......
......@@ -57,6 +57,8 @@ class RT130(DataTypeModel):
self.processing_log.append((msg, LogType.WARNING))
self.get_gps_data()
self.check_not_found_soh_chans()
def read_soh_index_waveform(self, folder: str) -> None:
"""
Loop all files in dir to read for soh data, mass position data and
......@@ -324,7 +326,8 @@ class RT130(DataTypeModel):
log_str = ''.join(logs)
self.log_data[k] = {'SOH': [log_str]}
log_obj = LogInfo(
self, self.track_info, log_str, k, self.req_data_streams)
self, self.track_info, log_str, k,
self.req_data_streams, self.req_soh_chans)
self.data_time[k][0] = min(log_obj.min_epoch, self.data_time[k][0])
self.data_time[k][1] = max(log_obj.max_epoch, self.data_time[k][1])
for c_name in self.soh_data[k]:
......
......@@ -8,7 +8,6 @@ class CalendarWidget(QtWidgets.QCalendarWidget):
"""
def __init__(self, parent):
super().__init__(parent)
self.setup_ui()
"""
toggle_day_of_year: QCheckBox: for user to choose to show day of year
or not
......@@ -20,6 +19,7 @@ class CalendarWidget(QtWidgets.QCalendarWidget):
"""
self._show_day_of_year = getattr(
self, 'toggle_day_of_year', None) is None
self.setup_ui()
def setup_ui(self):
"""
......@@ -48,6 +48,8 @@ class CalendarWidget(QtWidgets.QCalendarWidget):
self.toggle_day_of_year.show()
self.set_show_day_of_year(False)
def show_day_of_year(self):
"""
Get the value of self._show_day_of_year
......
......@@ -3,10 +3,9 @@ import pathlib
import shutil
import traceback
from datetime import datetime
from typing import Union
from typing import List, Tuple, Union
from copy import deepcopy
from pathlib import Path
from typing import List, Tuple
from PySide2 import QtCore, QtWidgets, QtGui
......@@ -26,6 +25,7 @@ from sohstationviewer.view.plotting.waveform_dialog import WaveformDialog
from sohstationviewer.view.search_message.search_message_dialog import (
SearchMessageDialog
)
from sohstationviewer.view.plotting.state_of_health_widget import SOHWidget
from sohstationviewer.view.help_view import HelpBrowser
from sohstationviewer.view.ui.main_ui import UIMainWindow
from sohstationviewer.view.util.enums import LogType
......@@ -47,15 +47,15 @@ class MainWindow(QtWidgets.QMainWindow, UIMainWindow):
super().__init__(parent)
self.setup_ui(self)
"""
dir_names: [PosixPath,] - list of absolute path of data set directory
dir_names: list of absolute path of data set directory
"""
self.dir_names = []
self.dir_names: List[Path] = []
"""
data_type: str - type of data set
"""
self.data_type = 'Unknown'
self.data_type: str = 'Unknown'
self.data_loader = DataLoader()
self.data_loader: DataLoader = DataLoader()
self.data_loader.finished.connect(self.replot_loaded_data)
"""
......@@ -67,90 +67,84 @@ class MainWindow(QtWidgets.QMainWindow, UIMainWindow):
"""
self.forms_in_forms_menu: List[QtWidgets.QWidget] = []
"""
req_soh_chans: [str,] - list of State-Of-Health channels to read data
req_soh_chans: list of State-Of-Health channels to read data
from. For Reftek, the list of channels is fixed => may not need
"""
self.req_soh_chans = []
self.req_soh_chans: List[str] = []
"""
req_wf_chans: [str,] - list of waveform channels to read data from.
req_wf_chans: list of waveform channels to read data from.
For Reftek, it is [int,] which is list of index of data stream
to read data from.
"""
self.req_wf_chans = []
self.req_wf_chans: List[Union[str, int]] = []
"""
start_tm: float - start time of data to read
end_tm: float - end time of data to read
start_tm: start time of data to read
end_tm: end time of data to read
"""
self.start_tm = 0
self.end_tm = 0
self.start_tm: float = 0
self.end_tm: float = 0
"""
data_object: DataTypeModel - Object that keep data read from data set
for plotting
data_object: Object that keep data read from data set for plotting
"""
self.data_object = None
self.data_object: Union[DataTypeModel, None] = None
"""
min_gap: float - minimum minutes of gap length to be display on gap bar
min_gap: minimum minutes of gap length to be display on gap bar
"""
self.min_gap = None
self.min_gap: Union[float, None] = None
"""
ids_name: str - name of selected preferred channels list
pref_soh_list_name: name of selected preferred channels list
"""
self.ids_name = ''
self.pref_soh_list_name: str = ''
"""
ids: [str,] - selected preferred channels
pref_soh_list: selected preferred channels
"""
self.ids = []
self.pref_soh_list: List[str] = []
"""
ids_data_type: str - data type of the preferred channels list
pref_soh_list_data_type: data type of the preferred channels list
"""
self.ids_data_type = 'Unknown'
self.pref_soh_list_data_type: str = 'Unknown'
# Options
"""
date_format: str - format for date
date_format: format for date
"""
self.date_format = 'YYYY-MM-DD'
self.date_format: str = 'YYYY-MM-DD'
"""
mass_pos_volt_range_opt: str - option for map value/color of
mass position
mass_pos_volt_range_opt: option for map value/color of mass position
"""
self.mass_pos_volt_range_opt = 'regular'
self.mass_pos_volt_range_opt: str = 'regular'
"""
bit_weight_opt: str - option for bitweight
bit_weight_opt: option for bitweight
"""
self.bit_weight_opt = '' # currently only need one option
self.bit_weight_opt: str = '' # currently only need one option
self.get_channel_prefer()
self.yyyy_mm_dd_action.triggered.emit()
"""
waveform_dlg: PlottingWidget - widget to display waveform channels'
plotting
waveform_dlg: widget to display waveform channels' plotting
"""
self.waveform_dlg = WaveformDialog(self)
self.waveform_dlg: SOHWidget = WaveformDialog(self)
"""
tps_dlg: PlottingWidget - widget to display time-power-squared of
waveform channels
tps_dlg: dialog to display time-power-squared of waveform channels
"""
self.tps_dlg = TimePowerSquaredDialog(self)
self.tps_dlg: TimePowerSquaredDialog = TimePowerSquaredDialog(self)
"""
help_browser: HelpBrowser - Display help documents with searching
feature.
help_browser: Display help documents with searching feature.
"""
self.help_browser = HelpBrowser()
self.help_browser: HelpBrowser = HelpBrowser()
"""
search_message_dialog: SearchMessageDialog - Display log, soh message
with searching feature.
search_message_dialog: Display log, soh message with searching feature.
"""
self.search_message_dialog = SearchMessageDialog()
self.search_message_dialog: SearchMessageDialog = SearchMessageDialog()
self.pull_current_directory_from_db()
self.delete_old_temp_data_folder()
self.has_problem = False
self.is_loading_data = False
self.is_plotting_soh = False
self.is_plotting_waveform = False
self.is_plotting_tps = False
self.is_stopping = False
self.has_problem: bool = False
self.is_loading_data: bool = False
self.is_plotting_soh: bool = False
self.is_plotting_waveform: bool = False
self.is_plotting_tps: bool = False
self.is_stopping: bool = False
@QtCore.Slot()
def open_data_type(self):
......@@ -246,12 +240,13 @@ class MainWindow(QtWidgets.QMainWindow, UIMainWindow):
channels list selected. If no list selected, re-check "All SOH"
"""
if not self.all_soh_chans_check_box.isChecked():
if self.ids == []:
if self.pref_soh_list == []:
self.all_soh_chans_check_box.setChecked(True)
else:
self.curr_soh_ids_name_line_edit.setText(self.ids_name)
self.curr_pref_soh_list_name_txtbox.setText(
self.pref_soh_list_name)
else:
self.curr_soh_ids_name_line_edit.setText('')
self.curr_pref_soh_list_name_txtbox.setText('')
@QtCore.Slot()
def set_date_format(self, display_format: str):
......@@ -274,9 +269,7 @@ class MainWindow(QtWidgets.QMainWindow, UIMainWindow):
which was clicked.
"""
print(f'Opening {item.text()}')
# TODO: Do something with the Path object,
# i.e., path.open(), or path.iterdir() ...
self.read_selected_files()
@QtCore.Slot()
def change_current_directory(self):
......@@ -289,8 +282,11 @@ class MainWindow(QtWidgets.QMainWindow, UIMainWindow):
fd.setFileMode(QtWidgets.QFileDialog.Directory)
fd.setDirectory(self.curr_dir_line_edit.text())
fd.exec_()
new_path = fd.selectedFiles()[0]
self.set_current_directory(new_path)
try:
new_path = fd.selectedFiles()[0]
self.set_current_directory(new_path)
except IndexError:
pass
def get_requested_wf_chans(self) -> List[Union[str, int]]:
"""
......@@ -334,11 +330,12 @@ class MainWindow(QtWidgets.QMainWindow, UIMainWindow):
:rtype: List[str]
"""
if (not self.all_soh_chans_check_box.isChecked() and
self.data_type != self.ids_data_type):
msg = (f"DataType detected for the selected data set is "
f"{self.data_type} which is different to IDs' "
f"{self.ids_data_type}.\n"
f"SOHStationViewer will read all data available.\n"
self.data_type != self.pref_soh_list_data_type):
msg = (f"Data Type detected for the selected data set is "
f"{self.data_type} which is different from Channel "
f"Preferences {self.pref_soh_list_name}'s Data Type: "
f"{self.pref_soh_list_data_type}.\n\n"
f"SOHStationViewer will read all data available.\n\n"
f"Do you want to continue?")
result = QtWidgets.QMessageBox.question(
self, "Confirmation", msg,
......@@ -346,9 +343,9 @@ class MainWindow(QtWidgets.QMainWindow, UIMainWindow):
if result == QtWidgets.QMessageBox.No:
return
self.all_soh_chans_check_box.setChecked(True)
self.curr_soh_ids_name_line_edit.setText('')
self.curr_pref_soh_list_name_txtbox.setText('')
return []
return (self.ids
return (self.pref_soh_list
if not self.all_soh_chans_check_box.isChecked() else [])
@QtCore.Slot()
......@@ -377,6 +374,10 @@ class MainWindow(QtWidgets.QMainWindow, UIMainWindow):
except AttributeError:
pass
self.req_soh_chans = (self.pref_soh_list
if not self.all_soh_chans_check_box.isChecked()
else [])
self.dir_names = [
Path(self.curr_dir_line_edit.text()).joinpath(item.text())
for item in self.open_files_list.selectedItems()]
......@@ -391,6 +392,7 @@ class MainWindow(QtWidgets.QMainWindow, UIMainWindow):
self.req_soh_chans = self.get_requested_soh_chan()
self.req_wf_chans = self.get_requested_wf_chans()
start_tm_str = self.time_from_date_edit.date().toString(
QtCore.Qt.ISODate)
end_tm_str = self.time_to_date_edit.date().toString(QtCore.Qt.ISODate)
......@@ -448,15 +450,29 @@ class MainWindow(QtWidgets.QMainWindow, UIMainWindow):
self.stop_load_data()
if self.is_plotting_soh:
display_tracking_info(self.tracking_info_text_browser,
'Stopping SOH plot...', 'info')
'Stopping SOH plot...')
if self.is_plotting_waveform:
display_tracking_info(self.waveform_dlg.info_text_browser,
'Stopping waveform plot...', 'info')
'Stopping waveform plot...')
waveform_widget = self.waveform_dlg.plotting_widget
running_processor = waveform_widget.data_processors[0]
running_processor.stopped.connect(self.reset_flags)
waveform_widget.request_stop()
self.waveform_dlg.plotting_widget.request_stop()
if self.is_plotting_tps:
display_tracking_info(self.tps_dlg.info_text_browser,
'Stopping TPS plot...')
tps_widget = self.tps_dlg.plotting_widget
tps_widget.request_stop()
def check_if_all_stopped(self):
"""
Check if everything has been stopped. If true, reset the is_stopping
flag.
"""
not_all_stopped = (self.is_loading_data or self.is_plotting_soh or
self.is_plotting_waveform or self.is_plotting_tps)
if not not_all_stopped:
self.is_stopping = False
@QtCore.Slot()
def data_loaded(self, data_obj: DataTypeModel):
......@@ -489,7 +505,10 @@ class MainWindow(QtWidgets.QMainWindow, UIMainWindow):
soh_chans = list(soh_data.keys())
mp_data = deepcopy(do.mass_pos_data[sel_key])
if len(self.req_wf_chans) != 0:
wf_data = deepcopy(do.waveform_data[sel_key]['readData'])
try:
wf_data = deepcopy(do.waveform_data[sel_key]['readData'])
except KeyError:
wf_data = {}
else:
wf_data = {}
try:
......@@ -509,10 +528,18 @@ class MainWindow(QtWidgets.QMainWindow, UIMainWindow):
peer_plotting_widgets = [self.plotting_widget]
if self.tps_check_box.isChecked():
self.is_plotting_tps = True
peer_plotting_widgets.append(self.tps_dlg.plotting_widget)
self.tps_dlg.set_data(
self.data_type, ','.join([str(d) for d in self.dir_names]))
self.tps_dlg.show()
# The waveform and TPS plots is being stopped at the same time, so
# we can't simply reset all flags. Instead, we use an intermediate
# method that check whether all plots have been stopped before
# resetting the is_stopping flag.
tps_widget = self.tps_dlg.plotting_widget
tps_widget.stopped.connect(self.reset_is_plotting_tps)
tps_widget.stopped.connect(self.check_if_all_stopped)
self.tps_dlg.plotting_widget.plot_channels(
self.start_tm, self.end_tm, sel_key,
do.data_time[sel_key],
......@@ -528,6 +555,9 @@ class MainWindow(QtWidgets.QMainWindow, UIMainWindow):
self.waveform_dlg.set_data(
self.data_type, ','.join([str(d) for d in self.dir_names]))
self.waveform_dlg.show()
waveform_widget = self.waveform_dlg.plotting_widget
waveform_widget.stopped.connect(self.reset_is_plotting_waveform)
waveform_widget.stopped.connect(self.check_if_all_stopped)
self.waveform_dlg.plotting_widget.plot_channels(
self.start_tm, self.end_tm, sel_key,
do.data_time[sel_key], time_tick_total,
......@@ -564,6 +594,22 @@ class MainWindow(QtWidgets.QMainWindow, UIMainWindow):
self.is_plotting_tps = False
self.is_stopping = False
@QtCore.Slot()
def reset_is_plotting_waveform(self):
"""
Reset the is_plotting_waveform flag. Used because lambda does not allow
assignment.
"""
self.is_plotting_waveform = False
@QtCore.Slot()
def reset_is_plotting_tps(self):
"""
Reset the is_plotting_tps flag. Used because lambda does not allow
assignment.
"""
self.is_plotting_tps = False
def set_current_directory(self, path: str = '') -> None:
"""
Update currentDirectory with path in DB table PersistentData.
......@@ -589,18 +635,18 @@ class MainWindow(QtWidgets.QMainWindow, UIMainWindow):
def get_channel_prefer(self):
"""
Read the current preferred channel list from database to set ids_name,
ids, self.ids_data_type
Read the current preferred channel list from database to set
pref_soh_list_name, pref_soh_list, self.pref_soh_list_data_type
"""
self.ids_name = ''
self.ids = []
self.pref_soh_list_name = ''
self.pref_soh_list = []
self.data_type = 'Unknown'
rows = execute_db_dict('SELECT name, IDs, dataType FROM ChannelPrefer '
'WHERE current=1')
if len(rows) > 0:
self.ids_name = rows[0]['name']
self.ids = [t.strip() for t in rows[0]['IDs'].split(',')]
self.ids_data_type = rows[0]['dataType']
self.pref_soh_list_name = rows[0]['name']
self.pref_soh_list = [t.strip() for t in rows[0]['IDs'].split(',')]
self.pref_soh_list_data_type = rows[0]['dataType']
def resizeEvent(self, event):
"""
......@@ -637,6 +683,10 @@ class MainWindow(QtWidgets.QMainWindow, UIMainWindow):
self.data_loader.thread.quit()
self.data_loader.thread.wait()
# close all remaining windows
for window in QtWidgets.QApplication.topLevelWidgets():
window.close()
def delete_old_temp_data_folder(self) -> None:
"""
Delete temp_data_folder which is used for keeping memmap files in case
......
......@@ -5,8 +5,7 @@ from sohstationviewer.view.util.plot_func_names import plot_functions
from sohstationviewer.view.plotting.plotting_widget import plotting_widget
from sohstationviewer.controller.plotting_data import get_title
from sohstationviewer.controller.util import (
display_tracking_info, apply_convert_factor)
from sohstationviewer.controller.util import apply_convert_factor
from sohstationviewer.conf import constants
......@@ -73,10 +72,13 @@ class SOHWidget(plotting_widget.PlottingWidget):
if chan_db_info['height'] == 0:
# not draw
continue
if chan_db_info['channel'] == 'DEFAULT':
if 'DEFAULT' in chan_db_info['channel']:
msg = (f"Channel {chan_id}'s "
f"definition can't be found database.")
display_tracking_info(self.tracking_box, msg, LogType.WARNING)
f"definition can't be found in database. It will"
f"be displayed in DEFAULT style.")
# TODO: give user a way to add it to DB and leave instruction
# here
self.processing_log.append((msg, LogType.WARNING))
if chan_db_info['plotType'] == '':
continue
......
# Display time-power-squared values for waveform data
from math import sqrt
import numpy as np
from typing import List
import numpy as np
from PySide2 import QtWidgets, QtCore
from sohstationviewer.view.plotting.plotting_widget import plotting_widget
from sohstationviewer.view.util.color import clr
from sohstationviewer.conf import constants as const
from sohstationviewer.controller.plotting_data import (
get_title, get_day_ticks, format_time)
get_title, get_day_ticks, format_time,
)
from sohstationviewer.controller.util import (
display_tracking_info, add_thousand_separator
display_tracking_info, add_thousand_separator,
)
from sohstationviewer.model.handling_data import (
get_trim_tps_data, get_each_day_5_min_list, find_tps_tm)
from sohstationviewer.database.extract_data import (
get_color_def, get_color_ranges, get_chan_label)
from sohstationviewer.conf import constants as const
get_color_def, get_color_ranges, get_chan_label,
)
from sohstationviewer.model.handling_data import (
get_each_day_5_min_list, find_tps_tm,
)
from sohstationviewer.view.plotting.plotting_widget import plotting_widget
from sohstationviewer.view.plotting.time_power_squared_processor import (
TimePowerSquaredProcessor,
)
from sohstationviewer.view.util.color import clr
class TimePowerSquaredWidget(plotting_widget.PlottingWidget):
stopped = QtCore.Signal()
"""
Widget to display time power square data for waveform channels
"""
......@@ -56,6 +60,17 @@ class TimePowerSquaredWidget(plotting_widget.PlottingWidget):
"""
self.tps_t = 0
self.tps_processors: List[TimePowerSquaredProcessor] = []
# The list of all channels that are processed.
self.channels = []
# The list of channels that have been processed.
self.processed_channels = []
# The post-processing step does not take too much time so there is no
# need to limit the number of threads that can run at once.
self.thread_pool = QtCore.QThreadPool()
self.finished_lock = QtCore.QMutex()
super().__init__(*args, **kwarg)
def plot_channels(self, start_tm, end_tm, key,
......@@ -71,6 +86,12 @@ class TimePowerSquaredWidget(plotting_widget.PlottingWidget):
:param waveform_data: dict - read waveform data of selected data set,
refer to DataTypeModel.__init__.waveform_data[key]['read_data']
"""
self.processed_channels = []
self.channels = []
self.tps_processors = []
start_msg = 'Plotting TPS data...'
display_tracking_info(self.tracking_box, start_msg)
self.processing_log = [] # [(message, type)]
self.gap_bar = None
self.min_x = max(data_time[0], start_tm)
......@@ -78,6 +99,7 @@ class TimePowerSquaredWidget(plotting_widget.PlottingWidget):
if self.axes:
self.plotting_axes.fig.clear()
self.draw()
self.date_mode = self.parent.date_format.upper()
if waveform_data == {}:
title = "NO WAVEFORM DATA TO DISPLAY."
......@@ -104,8 +126,60 @@ class TimePowerSquaredWidget(plotting_widget.PlottingWidget):
self.each_day5_min_list = get_each_day_5_min_list(self.min_x,
self.max_x)
for chan_id in self.plotting_data1:
ax = self.get_plot_data(self.plotting_data1[chan_id], chan_id)
c_data = self.plotting_data1[chan_id]
if 'tps_data' not in c_data:
self.channels.append(chan_id)
channel_processor = TimePowerSquaredProcessor(
chan_id, c_data, self.min_x, self.max_x,
self.each_day5_min_list
)
channel_processor.signals.finished.connect(self.channel_done)
channel_processor.signals.stopped.connect(self.channel_done)
self.tps_processors.append(channel_processor)
# Because the widget determine if processing is done by comparing the
# lists of scheduled and finished channels, if a channel runs fast
# enough that it finishes before any other channel can be scheduled,
# it will be the only channel executed. To prevent this, we tell the
# threadpool to only start running the processors once all channels
# have been scheduled.
for processor in self.tps_processors:
self.thread_pool.start(processor)
@QtCore.Slot()
def channel_done(self, chan_id: str):
"""
Slot called when a TPS processor is finished. Plot the TPS data of
channel chan_id if chan_id is not an empty string and add chan_id to
the list of processed of channels. If the list of processed channels
is the same as the list of all channels, notify the user that the
plotting is finished and add finishing touches to the plot.
If chan_id is the empty string, notify the user that the plotting has
been stopped.
:param chan_id: the name of the channel whose TPS data was processed.
If the TPS plot is stopped before it is finished, this will be the
empty string
"""
self.finished_lock.lock()
if chan_id != '':
ax = self.plot_channel(self.plotting_data1[chan_id], chan_id)
self.axes.append(ax)
self.processed_channels.append(chan_id)
if len(self.processed_channels) == len(self.channels):
if chan_id == '':
stopped_msg = 'TPS plot stopped.'
display_tracking_info(self.tracking_box, stopped_msg)
else:
finished_msg = 'TPS plot finished.'
display_tracking_info(self.tracking_box, finished_msg)
self.done()
self.stopped.emit()
self.finished_lock.unlock()
def done(self):
"""Add finishing touches to the plot and display it on the screen."""
self.set_legend()
# Set view size fit with the given data
if self.main_widget.geometry().height() < self.plotting_bot_pixel:
......@@ -113,7 +187,7 @@ class TimePowerSquaredWidget(plotting_widget.PlottingWidget):
self.set_lim_markers()
self.draw()
def get_plot_data(self, c_data, chan_id):
def plot_channel(self, c_data, chan_id):
"""
TPS is plotted in lines of small rectangular, so called bars.
Each line is a day so - y value is the order of days
......@@ -125,10 +199,8 @@ class TimePowerSquaredWidget(plotting_widget.PlottingWidget):
based on mapping between tps value of the five minutes against
the selected color range.
This function trim data to minx, max_x and calculate time-power-square
for each 5 minute into c_data['tps_data'] then draw each 5 minute
with the color corresponding to value.
Create ruler, zoom_marker1, zoom_marker2 for the channel.
This function draws each 5 minute with the color corresponding to
value and create ruler, zoom_marker1, and zoom_marker2 for the channel.
:param c_data: dict - data of the channel which includes down-sampled
data in keys 'times' and 'data'. Refer to
......@@ -136,10 +208,7 @@ class TimePowerSquaredWidget(plotting_widget.PlottingWidget):
:param chan_id: str - name of channel
:return ax: matplotlib.axes.Axes - axes of the channel
"""
if 'tps_data' not in c_data:
# get new minX, maxX according to exact start time of days
get_trim_tps_data(c_data, self.min_x, self.max_x,
self.each_day5_min_list)
total_days = c_data['tps_data'].shape[0]
plot_h = self.plotting_axes.get_height(
1.5 * total_days, bw_plots_distance=0.003)
......@@ -280,6 +349,10 @@ class TimePowerSquaredWidget(plotting_widget.PlottingWidget):
:param event: pick event - event when object of canvas is selected.
The event happens before button_press_event.
"""
if event.mouseevent.name == 'scroll_event':
return
if event.mouseevent.button in ('up', 'down'):
return
info_str = ""
if event.artist in self.axes:
xdata = event.mouseevent.xdata
......@@ -293,7 +366,7 @@ class TimePowerSquaredWidget(plotting_widget.PlottingWidget):
xdata = 287
ydata = round(event.mouseevent.ydata) # y value on the plot
# refer to description in get_plot_data to understand x,y vs
# refer to description in plot_channel to understand x,y vs
# day_index, five_min_index
day_index = - ydata
five_min_index = xdata
......@@ -370,6 +443,11 @@ class TimePowerSquaredWidget(plotting_widget.PlottingWidget):
for zm2 in self.zoom_marker2s:
zm2.set_data(x_idx, y_idx)
def request_stop(self):
"""Request all running channel processors to stop."""
for processor in self.tps_processors:
processor.request_stop()
class TimePowerSquaredDialog(QtWidgets.QWidget):
def __init__(self, parent):
......@@ -400,7 +478,7 @@ class TimePowerSquaredDialog(QtWidgets.QWidget):
main_layout.setSpacing(0)
"""
trackingInfoTextBrowser: QTextBrowser - to display info text
tracking_info_text_browser: QTextBrowser - to display info text
"""
self.info_text_browser = QtWidgets.QTextBrowser(self)
"""
......
from typing import Dict, Optional, List
import numpy as np
from PySide2 import QtCore
from sohstationviewer.conf import constants as const
class TimePowerSquaredProcessorSignal(QtCore.QObject):
finished = QtCore.Signal(str)
stopped = QtCore.Signal(str)
class TimePowerSquaredProcessor(QtCore.QRunnable):
def __init__(self, channel_id: str, channel_data: dict, start_time: float,
end_time: float, each_day_5_mins_list: np.ndarray):
super().__init__()
self.channel_id = channel_id
self.channel_data = channel_data
self.start_time = start_time
self.end_time = end_time
self.each_day_5_mins_list = each_day_5_mins_list
self.signals = TimePowerSquaredProcessorSignal()
# Flag to indicate whether the processor should stop running and clean
# up.
self.stop = False
self.stop_lock = QtCore.QMutex()
def trim_waveform_data(self) -> List[Dict]:
"""
Trim off waveform traces whose times do not intersect the closed
interval [self.start_time, self.end_time]. Store the traces that are
not removed in self.trimmed_trace_list.
"""
data_start_time = self.channel_data['tracesInfo'][0]['startTmEpoch']
data_end_time = self.channel_data['tracesInfo'][-1]['endTmEpoch']
if (self.start_time > data_end_time
or self.end_time < data_start_time):
return []
good_start_indices = [index
for index, tr
in enumerate(self.channel_data['tracesInfo'])
if tr['startTmEpoch'] > self.start_time]
if good_start_indices:
start_idx = good_start_indices[0]
if start_idx > 0:
start_idx -= 1 # start_time in middle of trace
else:
start_idx = 0
good_end_indices = [idx
for idx, tr
in enumerate(self.channel_data['tracesInfo'])
if tr['endTmEpoch'] <= self.end_time]
if good_end_indices:
end_idx = good_end_indices[-1]
if end_idx < len(self.channel_data['tracesInfo']) - 1:
end_idx += 1 # end_time in middle of trace
else:
end_idx = 0
end_idx += 1 # a[x:y+1] = [a[x], ...a[y]]
good_indices = slice(start_idx, end_idx)
return self.channel_data['tracesInfo'][good_indices]
def run(self) -> Optional[bool]:
"""
Different from soh_data where times and data are each in one np.array,
in waveform_data, times and data are each kept in a list of np.memmap
files along with startTmEpoch and endTmEpoch.
self.channel_data['startIdx'] and self.channel_data['endIdx'] will be
used to exclude np.memmap files that aren't in the zoom time range
(startTm, endTm). Data in np.memmap will be trimmed according to times
then time-power-square value for each 5 minutes will be calculated and
saved in channel_data['tps-data']: np.mean(np.square(5m data))
"""
trimmed_traces_list = self.trim_waveform_data()
# preset all 0 for all 5 minutes for each day
tps_data = np.zeros((len(self.each_day_5_mins_list), const.NO_5M_DAY))
spr = self.channel_data['samplerate']
self.channel_data['tps_data'] = []
start_tps_tm = 0
acc_data_list = []
for tr_idx, tr in enumerate(trimmed_traces_list):
self.stop_lock.lock()
if self.stop:
self.stop_lock.unlock()
return self.signals.stopped.emit('')
self.stop_lock.unlock()
times = np.memmap(tr['times_f'],
dtype='int64', mode='r',
shape=tr['size'])
data = np.memmap(tr['data_f'],
dtype='int64', mode='r',
shape=tr['size'])
start_index = 0
if tr_idx == 0:
# get index of times with closet value to startTm
start_index = np.abs(times - self.start_time).argmin()
start_tps_tm = times[start_index]
# identify index in case of overlaps or gaps
index = np.where(
(self.each_day_5_mins_list <= times[start_index]) &
(self.each_day_5_mins_list + const.SEC_5M > times[start_index])
)
curr_row = index[0][0]
curr_col = index[1][0]
next_tps_tm = start_tps_tm + const.SEC_5M
while self.end_time >= next_tps_tm:
self.stop_lock.lock()
if self.stop:
self.stop_lock.unlock()
return self.signals.stopped.emit('')
self.stop_lock.unlock()
next_index = int(start_index + spr * const.SEC_5M)
if next_index >= tr['size']:
acc_data_list.append(data[start_index:tr['size']])
break
else:
acc_data_list.append(
np.square(data[start_index:next_index]))
acc_data = np.hstack(acc_data_list)
if acc_data.size == 0:
tps_data[curr_row, curr_col] = 0
else:
tps_data[curr_row, curr_col] = np.mean(acc_data)
start_index = next_index
curr_col += 1
acc_data_list = []
if curr_col == const.NO_5M_DAY:
curr_col = 0
curr_row += 1
next_tps_tm += const.SEC_5M
self.channel_data['tps_data'] = tps_data
self.signals.finished.emit(self.channel_id)
def request_stop(self):
"""Request that the processor stops by setting the stop flag."""
self.stop_lock.lock()
self.stop = True
self.stop_lock.unlock()
......@@ -25,6 +25,7 @@ class WaveformWidget(plotting_widget.PlottingWidget):
"""
finished = QtCore.Signal()
stopped = QtCore.Signal()
def __init__(self, parent, tracking_box, name):
super().__init__(parent, tracking_box, name)
......@@ -42,6 +43,7 @@ class WaveformWidget(plotting_widget.PlottingWidget):
# Used to ensure that the user cannot read a new data set when we are
# zooming in on the waveform plot.
self.is_working = False
self.finished.connect(self.stopped)
def reset_widget(self):
"""
......@@ -181,7 +183,7 @@ class WaveformWidget(plotting_widget.PlottingWidget):
)
self.data_processors.append(channel_processor)
channel_processor.finished.connect(self.process_channel)
channel_processor.stopped.connect(self.stopped)
channel_processor.stopped.connect(self.has_stopped)
def plot_mass_pos_channels(self):
"""
......@@ -308,7 +310,7 @@ class WaveformWidget(plotting_widget.PlottingWidget):
self.data_processors = [self.data_processors[0]]
@QtCore.Slot()
def stopped(self):
def has_stopped(self):
"""
The slot that is called when the last channel processor has terminated
all running background threads.
......@@ -316,6 +318,7 @@ class WaveformWidget(plotting_widget.PlottingWidget):
display_tracking_info(self.tracking_box,
'Waveform plot stopped', 'info')
self.is_working = False
self.stopped.emit()
class WaveformDialog(QtWidgets.QWidget):
......