Implement reading SOH packets

sohstationviewer/model/reftek/rt130_experiment/eh_et_packet.py

@@ -5,6 +5,7 @@ from sohstationviewer.model.mseed.read_mseed_experiment.mseed_helper import \
 from sohstationviewer.model.reftek.rt130_experiment.reftek_helper import \
     PacketHeader
 
+eh_et_payload_end = 92
 
 def read_eh_et_packet(packet: bytes, unpacker: Unpacker):
     event_number = int(packet[16:18].hex())
@@ -14,7 +15,7 @@ def read_eh_et_packet(packet: bytes, unpacker: Unpacker):
 
     extended_header = EHETExtendedHeader(event_number, data_stream_number,
                                          flags, data_format)
-    payload = packet[24:92]
+    payload = packet[24:eh_et_payload_end]
     return extended_header, payload
 
 

sohstationviewer/model/reftek/rt130_experiment/reftek.py

@@ -19,42 +19,17 @@ from sohstationviewer.model.reftek.rt130_experiment.dt_packet import \
 )
 from sohstationviewer.model.reftek.rt130_experiment.eh_et_packet import \
     (
-    EHETPacket, read_eh_et_packet,
+    EHETPacket, read_eh_et_packet, eh_et_payload_end,
 )
 from sohstationviewer.model.reftek.rt130_experiment.reftek_helper import (
     print_error, packet_reader_placeholder, RT130ParseError, PacketHeader,
 )
 
 from sohstationviewer.model.reftek.from_rt2ms import core
-
-
-# This is the minimum size of the payload we can use while keeping the
-# necessary information from the EH/ET packets.
-payload_size = 68
-dt_payload_padding = [0] * (payload_size - 4)
-
-
-PACKET = [
-    ("packet_type", "|S2", None, "S2"),
-    ("experiment_number", np.uint8, bcd, np.uint8),
-    ("year", np.uint8, bcd, np.uint8),
-    ("unit_id", (np.uint8, 2), bcd_hex, "S4"),
-    ("time", (np.uint8, 6), bcd_julian_day_string_to_nanoseconds_of_year,
-     np.int64),
-    ("byte_count", (np.uint8, 2), bcd_16bit_int, np.uint16),
-    ("packet_sequence", (np.uint8, 2), bcd_16bit_int, np.uint16),
-    ("event_number", (np.uint8, 2), bcd_16bit_int, np.uint16),
-    ("data_stream_number", np.uint8, bcd, np.uint8),
-    ("channel_number", np.uint8, bcd, np.uint8),
-    ("number_of_samples", (np.uint8, 2), bcd_16bit_int, np.uint32),
-    ("flags", np.uint8, None, np.uint8),
-    ("data_format", np.uint8, bcd_8bit_hex, "S2"),
-    # Temporarily store the payload here.
-    ("payload", (np.uint8, payload_size), None, (np.uint8, payload_size))]
-
-PACKET_FINAL_DTYPE = np.dtype([
-    (name, dtype_final)
-    for name, dtype_initial, converter, dtype_final in PACKET])
+from sohstationviewer.model.reftek.rt130_experiment.soh_packets import \
+    (
+    read_soh_packet, SOHPacket,
+)
 
 
 def parse_rt130_time(year: int, time_bytes: bytes) -> UTCDateTime:
@@ -133,14 +108,20 @@ def read_rt130_file(file_name: str, unpacker: Unpacker):
             packet = rt130_file.read(1024)
             packet_header = get_rt130_packet_header(packet, unpacker)
 
-            packet_handlers: Dict[str, Callable] = {
-                'SC': packet_reader_placeholder,
-                'SH': packet_reader_placeholder,
+            waveform_handlers: Dict[str, Callable] = {
                 'EH': read_eh_et_packet,
                 'ET': read_eh_et_packet,
                 'DT': read_dt_packet,
             }
 
+            soh_handlers: Dict[str, Callable] = dict.fromkeys(
+                ['AD', 'CD', 'DS', 'FD', 'OM', 'SC', 'SH'], read_soh_packet
+            )
+
+            packet_handlers = {
+                **waveform_handlers, **soh_handlers
+            }
+
             packet_handler = packet_handlers.get(
                 packet_header.packet_type, packet_reader_placeholder
             )
@@ -149,6 +130,8 @@ def read_rt130_file(file_name: str, unpacker: Unpacker):
                 packet_type = DTPacket
             elif packet_header.packet_type in ['EH', 'ET']:
                 packet_type = EHETPacket
+            else:
+                packet_type = SOHPacket
 
             extended_header, data = return_val
             current_packet = packet_type(packet_header, extended_header, data)
@@ -157,7 +140,8 @@ def read_rt130_file(file_name: str, unpacker: Unpacker):
     return packets
 
 
-def convert_packet_to_obspy_format(packet: Union[EHETPacket, DTPacket], unpacker: Unpacker):
+def convert_packet_to_obspy_format(packet: Union[EHETPacket, DTPacket],
+                                   unpacker: Unpacker):
     # We want to convert the packet to a tuple. In order to make it easier to
     # maintain, we first convert the packet to a dictionary. Then, we grab the
     # values of the dictionary as tuple to get the final result.
@@ -171,9 +155,11 @@ def convert_packet_to_obspy_format(packet: Union[EHETPacket, DTPacket], unpacker
     converted_packet['byte_count'] = packet.header.byte_count
     converted_packet['packet_sequence'] = packet.header.packet_sequence
     converted_packet['event_number'] = packet.extended_header.event_number
-    converted_packet['data_stream_number'] = packet.extended_header.data_stream_number
+    converted_packet[
+        'data_stream_number'] = packet.extended_header.data_stream_number
     converted_packet['channel_number'] = packet.extended_header.channel_number
-    converted_packet['number_of_samples'] = packet.extended_header.number_of_samples
+    converted_packet[
+        'number_of_samples'] = packet.extended_header.number_of_samples
     converted_packet['flags'] = packet.extended_header.flags
     converted_packet['data_format'] = packet.extended_header.data_format
 
@@ -188,15 +174,19 @@ def convert_packet_to_obspy_format(packet: Union[EHETPacket, DTPacket], unpacker
         # because of a peculiarity of the 2's complement encoding).
         data_size = 4
         format_char = 'B'
-        converted_packet['payload'] = list(unpacker.unpack(
+        converted_packet['payload'] = numpy.empty(1000, np.uint8)
+        packet_data = list(unpacker.unpack(
             f'{data_size}{format_char}',
             packet.data.to_bytes(data_size, 'big', signed=True)
-        )) + dt_payload_padding
-    elif converted_packet['packet_type'] in ['EH', 'ET']:
-        converted_packet['payload'] = list(unpacker.unpack(
-            f'{payload_size}B', packet.data
         ))
-
+        converted_packet['payload'][:4] = packet_data
+    elif converted_packet['packet_type'] in ['EH', 'ET']:
+        eh_et_payload_size = eh_et_payload_end - 24
+        converted_packet['payload'] = numpy.empty(1000, np.uint8)
+        packet_data = numpy.frombuffer(packet.data, np.uint8)
+        converted_packet['payload'][:eh_et_payload_size] = packet_data
+    else:
+        converted_packet['payload'] = numpy.frombuffer(packet.data, np.uint8)
     return tuple(converted_packet.values())
 
 
@@ -209,10 +199,15 @@ class Reftek130(core.Reftek130):
         packets_in_file = read_rt130_file(filename, rt130_unpacker)
         converted_packets = []
         for packet in packets_in_file:
-            converted_packets.append(convert_packet_to_obspy_format(packet, rt130_unpacker))
+            converted_packets.append(
+                convert_packet_to_obspy_format(packet, rt130_unpacker))
         rt._data = numpy.array(converted_packets, PACKET_FINAL_DTYPE)
         return rt
 
 
-
-
+waveform_file = '/Users/kle/PycharmProjects/sohstationviewer/tests/test_data/RT130-sample/2017149.92EB/2017150/92EB/1/000000015_0036EE80'
+soh_file = '/Users/kle/PycharmProjects/sohstationviewer/tests/test_data/RT130-sample/2017149.92EB/2017150/92EB/0/000000000_00000000'
+import time
+start = time.perf_counter()
+print((Reftek130.from_file(soh_file)._data))
+print('Time taken:', time.perf_counter() - start)

sohstationviewer/model/reftek/rt130_experiment/soh_packets.py

+import dataclasses
+
+from obspy.io.reftek.util import bcd
+
+from sohstationviewer.model.mseed.read_mseed_experiment.mseed_helper import \
+    Unpacker
+from sohstationviewer.model.reftek.rt130_experiment.reftek_helper import \
+    PacketHeader
+
+import numpy
+
+
+@dataclasses.dataclass
+class SOHExtendedHeader:
+    event_number: int
+    data_stream_number: int
+    channel_number: int
+    number_of_samples: int
+    flags: int
+    data_format: str
+
+
+@dataclasses.dataclass
+class SOHPacket:
+    header: PacketHeader
+    extended_header: SOHExtendedHeader
+    data: bytes
+
+
+def bcd_16bit_int(_i):
+    _i = bcd(_i)
+    return _i[0] * 100 + _i[1]
+
+
+def read_soh_packet(packet: bytes, unpacker: Unpacker):
+    event_number = bcd_16bit_int(numpy.frombuffer(packet[16:18], numpy.uint8))
+    data_stream_number = bcd(numpy.frombuffer(packet[18:19], numpy.uint8))
+    channel_number = bcd(numpy.frombuffer(packet[19:20], numpy.uint8))
+    number_of_samples = bcd_16bit_int(numpy.frombuffer(packet[20:22], numpy.uint8))
+    flags = unpacker.unpack('B', packet[22:23])[0]
+    data_format = packet[23:24].hex().upper()
+
+    extended_header = SOHExtendedHeader(event_number, data_stream_number,
+                                        channel_number, number_of_samples,
+                                        flags, data_format)
+    payload = packet[24:]
+    return extended_header, payload