Skip to content
Snippets Groups Projects

allow to select multi folder to load

Merged Lan Dam requested to merge i74_load_multi_folder into master
from __future__ import annotations
import os
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Optional, Union, List, Tuple, Dict
@@ -11,8 +10,7 @@ from obspy.core import Stream
from PySide2 import QtCore
from PySide2 import QtWidgets
from sohstationviewer.controller.util import (
display_tracking_info, validate_file, validate_dir, get_total_files)
from sohstationviewer.controller.util import display_tracking_info
from sohstationviewer.conf import constants
from sohstationviewer.view.plotting.gps_plot.gps_point import GPSPoint
from sohstationviewer.view.util.enums import LogType
@@ -38,7 +36,7 @@ class ThreadStopped(Exception):
class DataTypeModel():
def __init__(self, data_type, tracking_box: QtWidgets.QTextBrowser,
list_of_dir: List[str], list_of_rt130_paths: List[Path] = [],
folder: str, list_of_rt130_paths: List[Path] = [],
req_wf_chans: Union[List[str], List[int]] = [],
req_soh_chans: List[str] = [],
read_start: Optional[float] = UTCDateTime(0).timestamp,
@@ -55,7 +53,7 @@ class DataTypeModel():
:param data_type: type of the object
:param tracking_box: widget to display tracking info
:param list_of_dir: list of paths to the folders of data
:param folder: path to the folder of data
:param list_of_rt130_paths: path to the folders of RT130 data
:param req_wf_chans: requested waveform channel list
:param req_soh_chans: requested SOH channel list
@@ -74,7 +72,7 @@ class DataTypeModel():
"""
self.data_type = data_type
self.tracking_box = tracking_box
self.list_of_dir = list_of_dir
self.dir = folder
self.list_of_rt130_paths = list_of_rt130_paths
self.req_soh_chans = req_soh_chans
self.req_wf_chans = req_wf_chans
@@ -262,6 +260,14 @@ class DataTypeModel():
self.gps_points: List[GPSPoint] = []
def read_folder(self, folder: str) -> Tuple[Dict]:
"""
Read data from given folder
:param folder: path to folder to read data
:return: Tuple of different data dicts
"""
pass
def select_key(self) -> Union[str, Tuple[str, str]]:
"""
Get the key for the data set to process.
@@ -272,7 +278,7 @@ class DataTypeModel():
def processing_data(self):
if self.creator_thread.isInterruptionRequested():
raise ThreadStopped()
self.read_folders()
self.read_folder(self.dir)
if self.creator_thread.isInterruptionRequested():
raise ThreadStopped()
@@ -284,60 +290,6 @@ class DataTypeModel():
raise ThreadStopped()
self.finalize_data()
def read_folders(self, folders) -> None:
"""
Read data from given folders to create data dicts which are
attributes of current class
"""
count = 0
total = get_total_files(folders)
for folder in folders:
self.read_folder(folder, total, count)
def read_folder(self, folder: str, total: int, count: int) -> int:
"""
Read data from current folder.
:param folder: folder to read data from
:param total: total of all valid files
:param count: total of files that have been processed before this
folder to keep track of progress
:return count: total of files that have been processed after this
folder to keep track of progress
"""
for path, sub_dirs, files in os.walk(folder):
try:
validate_dir(path)
except Exception as e:
# skip Information folder
self.track_info(str(e), LogType.WARNING)
continue
for file_name in files:
if self.creator_thread.isInterruptionRequested():
raise ThreadStopped()
path2file = Path(path).joinpath(file_name)
if not validate_file(path2file, file_name):
continue
self.read_data_file(path2file, file_name)
count += 1
if count % 10 == 0:
self.track_info(
f'Read {count} files/{total}', LogType.INFO)
return count
def read_data_file(self, path2file: str, file_name: str) -> None:
"""
Read data from path <path2ffile>, with name <file_name>
:param path2file: absolute path to data file
:param file_name: name of data file
"""
pass
def finalize_data(self):
"""
This function should be called after all folders finish reading to
Loading