Skip to content
Snippets Groups Projects

allow to select multi folder to load

Merged Lan Dam requested to merge i74_load_multi_folder into master
1 file
+ 13
61
Compare changes
  • Side-by-side
  • Inline
from __future__ import annotations
from __future__ import annotations
import os
from pathlib import Path
from pathlib import Path
from tempfile import TemporaryDirectory
from tempfile import TemporaryDirectory
from typing import Optional, Union, List, Tuple, Dict
from typing import Optional, Union, List, Tuple, Dict
@@ -11,8 +10,7 @@ from obspy.core import Stream
@@ -11,8 +10,7 @@ from obspy.core import Stream
from PySide2 import QtCore
from PySide2 import QtCore
from PySide2 import QtWidgets
from PySide2 import QtWidgets
from sohstationviewer.controller.util import (
from sohstationviewer.controller.util import display_tracking_info
display_tracking_info, validate_file, validate_dir, get_total_files)
from sohstationviewer.conf import constants
from sohstationviewer.conf import constants
from sohstationviewer.view.plotting.gps_plot.gps_point import GPSPoint
from sohstationviewer.view.plotting.gps_plot.gps_point import GPSPoint
from sohstationviewer.view.util.enums import LogType
from sohstationviewer.view.util.enums import LogType
@@ -38,7 +36,7 @@ class ThreadStopped(Exception):
@@ -38,7 +36,7 @@ class ThreadStopped(Exception):
class DataTypeModel():
class DataTypeModel():
def __init__(self, data_type, tracking_box: QtWidgets.QTextBrowser,
def __init__(self, data_type, tracking_box: QtWidgets.QTextBrowser,
list_of_dir: List[str], list_of_rt130_paths: List[Path] = [],
folder: str, list_of_rt130_paths: List[Path] = [],
req_wf_chans: Union[List[str], List[int]] = [],
req_wf_chans: Union[List[str], List[int]] = [],
req_soh_chans: List[str] = [],
req_soh_chans: List[str] = [],
read_start: Optional[float] = UTCDateTime(0).timestamp,
read_start: Optional[float] = UTCDateTime(0).timestamp,
@@ -55,7 +53,7 @@ class DataTypeModel():
@@ -55,7 +53,7 @@ class DataTypeModel():
:param data_type: type of the object
:param data_type: type of the object
:param tracking_box: widget to display tracking info
:param tracking_box: widget to display tracking info
:param list_of_dir: list of paths to the folders of data
:param folder: path to the folder of data
:param list_of_rt130_paths: path to the folders of RT130 data
:param list_of_rt130_paths: path to the folders of RT130 data
:param req_wf_chans: requested waveform channel list
:param req_wf_chans: requested waveform channel list
:param req_soh_chans: requested SOH channel list
:param req_soh_chans: requested SOH channel list
@@ -74,7 +72,7 @@ class DataTypeModel():
@@ -74,7 +72,7 @@ class DataTypeModel():
"""
"""
self.data_type = data_type
self.data_type = data_type
self.tracking_box = tracking_box
self.tracking_box = tracking_box
self.list_of_dir = list_of_dir
self.dir = folder
self.list_of_rt130_paths = list_of_rt130_paths
self.list_of_rt130_paths = list_of_rt130_paths
self.req_soh_chans = req_soh_chans
self.req_soh_chans = req_soh_chans
self.req_wf_chans = req_wf_chans
self.req_wf_chans = req_wf_chans
@@ -262,6 +260,14 @@ class DataTypeModel():
@@ -262,6 +260,14 @@ class DataTypeModel():
self.gps_points: List[GPSPoint] = []
self.gps_points: List[GPSPoint] = []
 
def read_folder(self, folder: str) -> Tuple[Dict]:
 
"""
 
Read data from given folder
 
:param folder: path to folder to read data
 
:return: Tuple of different data dicts
 
"""
 
pass
 
def select_key(self) -> Union[str, Tuple[str, str]]:
def select_key(self) -> Union[str, Tuple[str, str]]:
"""
"""
Get the key for the data set to process.
Get the key for the data set to process.
@@ -272,7 +278,7 @@ class DataTypeModel():
@@ -272,7 +278,7 @@ class DataTypeModel():
def processing_data(self):
def processing_data(self):
if self.creator_thread.isInterruptionRequested():
if self.creator_thread.isInterruptionRequested():
raise ThreadStopped()
raise ThreadStopped()
self.read_folders()
self.read_folder(self.dir)
if self.creator_thread.isInterruptionRequested():
if self.creator_thread.isInterruptionRequested():
raise ThreadStopped()
raise ThreadStopped()
@@ -284,60 +290,6 @@ class DataTypeModel():
@@ -284,60 +290,6 @@ class DataTypeModel():
raise ThreadStopped()
raise ThreadStopped()
self.finalize_data()
self.finalize_data()
def read_folders(self, folders) -> None:
"""
Read data from given folders to create data dicts which are
attributes of current class
"""
count = 0
total = get_total_files(folders)
for folder in folders:
self.read_folder(folder, total, count)
def read_folder(self, folder: str, total: int, count: int) -> int:
"""
Read data from current folder.
:param folder: folder to read data from
:param total: total of all valid files
:param count: total of files that have been processed before this
folder to keep track of progress
:return count: total of files that have been processed after this
folder to keep track of progress
"""
for path, sub_dirs, files in os.walk(folder):
try:
validate_dir(path)
except Exception as e:
# skip Information folder
self.track_info(str(e), LogType.WARNING)
continue
for file_name in files:
if self.creator_thread.isInterruptionRequested():
raise ThreadStopped()
path2file = Path(path).joinpath(file_name)
if not validate_file(path2file, file_name):
continue
self.read_data_file(path2file, file_name)
count += 1
if count % 10 == 0:
self.track_info(
f'Read {count} files/{total}', LogType.INFO)
return count
def read_data_file(self, path2file: str, file_name: str) -> None:
"""
Read data from path <path2ffile>, with name <file_name>
:param path2file: absolute path to data file
:param file_name: name of data file
"""
pass
def finalize_data(self):
def finalize_data(self):
"""
"""
This function should be called after all folders finish reading to
This function should be called after all folders finish reading to
Loading