From 9c03ef3a5b69ae047f478b999b73fc216491cffe Mon Sep 17 00:00:00 2001
From: kienle <kienle@passcal.nmt.edu>
Date: Sun, 23 Jul 2023 21:44:21 -0600
Subject: [PATCH] Implement reading SOH packets
---
.../reftek/rt130_experiment/eh_et_packet.py | 3 +-
.../model/reftek/rt130_experiment/reftek.py | 85 +++++++++----------
.../reftek/rt130_experiment/soh_packets.py | 47 ++++++++++
3 files changed, 89 insertions(+), 46 deletions(-)
create mode 100644 sohstationviewer/model/reftek/rt130_experiment/soh_packets.py
diff --git a/sohstationviewer/model/reftek/rt130_experiment/eh_et_packet.py b/sohstationviewer/model/reftek/rt130_experiment/eh_et_packet.py
index b0355262e..150ca33dc 100644
--- a/sohstationviewer/model/reftek/rt130_experiment/eh_et_packet.py
+++ b/sohstationviewer/model/reftek/rt130_experiment/eh_et_packet.py
@@ -5,6 +5,7 @@ from sohstationviewer.model.mseed.read_mseed_experiment.mseed_helper import \
from sohstationviewer.model.reftek.rt130_experiment.reftek_helper import \
PacketHeader
+eh_et_payload_end = 92
def read_eh_et_packet(packet: bytes, unpacker: Unpacker):
event_number = int(packet[16:18].hex())
@@ -14,7 +15,7 @@ def read_eh_et_packet(packet: bytes, unpacker: Unpacker):
extended_header = EHETExtendedHeader(event_number, data_stream_number,
flags, data_format)
- payload = packet[24:92]
+ payload = packet[24:eh_et_payload_end]
return extended_header, payload
diff --git a/sohstationviewer/model/reftek/rt130_experiment/reftek.py b/sohstationviewer/model/reftek/rt130_experiment/reftek.py
index cb2d045e0..201c6f72a 100644
--- a/sohstationviewer/model/reftek/rt130_experiment/reftek.py
+++ b/sohstationviewer/model/reftek/rt130_experiment/reftek.py
@@ -19,42 +19,17 @@ from sohstationviewer.model.reftek.rt130_experiment.dt_packet import \
)
from sohstationviewer.model.reftek.rt130_experiment.eh_et_packet import \
(
- EHETPacket, read_eh_et_packet,
+ EHETPacket, read_eh_et_packet, eh_et_payload_end,
)
from sohstationviewer.model.reftek.rt130_experiment.reftek_helper import (
print_error, packet_reader_placeholder, RT130ParseError, PacketHeader,
)
from sohstationviewer.model.reftek.from_rt2ms import core
-
-
-# This is the minimum size of the payload we can use while keeping the
-# necessary information from the EH/ET packets.
-payload_size = 68
-dt_payload_padding = [0] * (payload_size - 4)
-
-
-PACKET = [
- ("packet_type", "|S2", None, "S2"),
- ("experiment_number", np.uint8, bcd, np.uint8),
- ("year", np.uint8, bcd, np.uint8),
- ("unit_id", (np.uint8, 2), bcd_hex, "S4"),
- ("time", (np.uint8, 6), bcd_julian_day_string_to_nanoseconds_of_year,
- np.int64),
- ("byte_count", (np.uint8, 2), bcd_16bit_int, np.uint16),
- ("packet_sequence", (np.uint8, 2), bcd_16bit_int, np.uint16),
- ("event_number", (np.uint8, 2), bcd_16bit_int, np.uint16),
- ("data_stream_number", np.uint8, bcd, np.uint8),
- ("channel_number", np.uint8, bcd, np.uint8),
- ("number_of_samples", (np.uint8, 2), bcd_16bit_int, np.uint32),
- ("flags", np.uint8, None, np.uint8),
- ("data_format", np.uint8, bcd_8bit_hex, "S2"),
- # Temporarily store the payload here.
- ("payload", (np.uint8, payload_size), None, (np.uint8, payload_size))]
-
-PACKET_FINAL_DTYPE = np.dtype([
- (name, dtype_final)
- for name, dtype_initial, converter, dtype_final in PACKET])
+from sohstationviewer.model.reftek.rt130_experiment.soh_packets import \
+ (
+ read_soh_packet, SOHPacket,
+)
def parse_rt130_time(year: int, time_bytes: bytes) -> UTCDateTime:
@@ -133,14 +108,20 @@ def read_rt130_file(file_name: str, unpacker: Unpacker):
packet = rt130_file.read(1024)
packet_header = get_rt130_packet_header(packet, unpacker)
- packet_handlers: Dict[str, Callable] = {
- 'SC': packet_reader_placeholder,
- 'SH': packet_reader_placeholder,
+ waveform_handlers: Dict[str, Callable] = {
'EH': read_eh_et_packet,
'ET': read_eh_et_packet,
'DT': read_dt_packet,
}
+ soh_handlers: Dict[str, Callable] = dict.fromkeys(
+ ['AD', 'CD', 'DS', 'FD', 'OM', 'SC', 'SH'], read_soh_packet
+ )
+
+ packet_handlers = {
+ **waveform_handlers, **soh_handlers
+ }
+
packet_handler = packet_handlers.get(
packet_header.packet_type, packet_reader_placeholder
)
@@ -149,6 +130,8 @@ def read_rt130_file(file_name: str, unpacker: Unpacker):
packet_type = DTPacket
elif packet_header.packet_type in ['EH', 'ET']:
packet_type = EHETPacket
+ else:
+ packet_type = SOHPacket
extended_header, data = return_val
current_packet = packet_type(packet_header, extended_header, data)
@@ -157,7 +140,8 @@ def read_rt130_file(file_name: str, unpacker: Unpacker):
return packets
-def convert_packet_to_obspy_format(packet: Union[EHETPacket, DTPacket], unpacker: Unpacker):
+def convert_packet_to_obspy_format(packet: Union[EHETPacket, DTPacket],
+ unpacker: Unpacker):
# We want to convert the packet to a tuple. In order to make it easier to
# maintain, we first convert the packet to a dictionary. Then, we grab the
# values of the dictionary as tuple to get the final result.
@@ -171,9 +155,11 @@ def convert_packet_to_obspy_format(packet: Union[EHETPacket, DTPacket], unpacker
converted_packet['byte_count'] = packet.header.byte_count
converted_packet['packet_sequence'] = packet.header.packet_sequence
converted_packet['event_number'] = packet.extended_header.event_number
- converted_packet['data_stream_number'] = packet.extended_header.data_stream_number
+ converted_packet[
+ 'data_stream_number'] = packet.extended_header.data_stream_number
converted_packet['channel_number'] = packet.extended_header.channel_number
- converted_packet['number_of_samples'] = packet.extended_header.number_of_samples
+ converted_packet[
+ 'number_of_samples'] = packet.extended_header.number_of_samples
converted_packet['flags'] = packet.extended_header.flags
converted_packet['data_format'] = packet.extended_header.data_format
@@ -188,15 +174,19 @@ def convert_packet_to_obspy_format(packet: Union[EHETPacket, DTPacket], unpacker
# because of a peculiarity of the 2's complement encoding).
data_size = 4
format_char = 'B'
- converted_packet['payload'] = list(unpacker.unpack(
+ converted_packet['payload'] = numpy.empty(1000, np.uint8)
+ packet_data = list(unpacker.unpack(
f'{data_size}{format_char}',
packet.data.to_bytes(data_size, 'big', signed=True)
- )) + dt_payload_padding
- elif converted_packet['packet_type'] in ['EH', 'ET']:
- converted_packet['payload'] = list(unpacker.unpack(
- f'{payload_size}B', packet.data
))
-
+ converted_packet['payload'][:4] = packet_data
+ elif converted_packet['packet_type'] in ['EH', 'ET']:
+ eh_et_payload_size = eh_et_payload_end - 24
+ converted_packet['payload'] = numpy.empty(1000, np.uint8)
+ packet_data = numpy.frombuffer(packet.data, np.uint8)
+ converted_packet['payload'][:eh_et_payload_size] = packet_data
+ else:
+ converted_packet['payload'] = numpy.frombuffer(packet.data, np.uint8)
return tuple(converted_packet.values())
@@ -209,10 +199,15 @@ class Reftek130(core.Reftek130):
packets_in_file = read_rt130_file(filename, rt130_unpacker)
converted_packets = []
for packet in packets_in_file:
- converted_packets.append(convert_packet_to_obspy_format(packet, rt130_unpacker))
+ converted_packets.append(
+ convert_packet_to_obspy_format(packet, rt130_unpacker))
rt._data = numpy.array(converted_packets, PACKET_FINAL_DTYPE)
return rt
-
-
+waveform_file = '/Users/kle/PycharmProjects/sohstationviewer/tests/test_data/RT130-sample/2017149.92EB/2017150/92EB/1/000000015_0036EE80'
+soh_file = '/Users/kle/PycharmProjects/sohstationviewer/tests/test_data/RT130-sample/2017149.92EB/2017150/92EB/0/000000000_00000000'
+import time
+start = time.perf_counter()
+print((Reftek130.from_file(soh_file)._data))
+print('Time taken:', time.perf_counter() - start)
diff --git a/sohstationviewer/model/reftek/rt130_experiment/soh_packets.py b/sohstationviewer/model/reftek/rt130_experiment/soh_packets.py
new file mode 100644
index 000000000..4e83d6916
--- /dev/null
+++ b/sohstationviewer/model/reftek/rt130_experiment/soh_packets.py
@@ -0,0 +1,47 @@
+import dataclasses
+
+from obspy.io.reftek.util import bcd
+
+from sohstationviewer.model.mseed.read_mseed_experiment.mseed_helper import \
+ Unpacker
+from sohstationviewer.model.reftek.rt130_experiment.reftek_helper import \
+ PacketHeader
+
+import numpy
+
+
+@dataclasses.dataclass
+class SOHExtendedHeader:
+ event_number: int
+ data_stream_number: int
+ channel_number: int
+ number_of_samples: int
+ flags: int
+ data_format: str
+
+
+@dataclasses.dataclass
+class SOHPacket:
+ header: PacketHeader
+ extended_header: SOHExtendedHeader
+ data: bytes
+
+
+def bcd_16bit_int(_i):
+ _i = bcd(_i)
+ return _i[0] * 100 + _i[1]
+
+
+def read_soh_packet(packet: bytes, unpacker: Unpacker):
+ event_number = bcd_16bit_int(numpy.frombuffer(packet[16:18], numpy.uint8))
+ data_stream_number = bcd(numpy.frombuffer(packet[18:19], numpy.uint8))
+ channel_number = bcd(numpy.frombuffer(packet[19:20], numpy.uint8))
+ number_of_samples = bcd_16bit_int(numpy.frombuffer(packet[20:22], numpy.uint8))
+ flags = unpacker.unpack('B', packet[22:23])[0]
+ data_format = packet[23:24].hex().upper()
+
+ extended_header = SOHExtendedHeader(event_number, data_stream_number,
+ channel_number, number_of_samples,
+ flags, data_format)
+ payload = packet[24:]
+ return extended_header, payload
--
GitLab