From cd76e386eaa71cb9487f72c3633b5bcb6cce2982 Mon Sep 17 00:00:00 2001
From: Maeva Pourpoint <maeva@passcal.nmt.edu>
Date: Tue, 26 May 2020 09:21:11 -0600
Subject: [PATCH] Specify timeouts for unit tests; Clean up imports and
try/excepts in data2passcal.py; Recreate test.dat under
ftp://ftp.passcal.nmt.edu/download/public/
---
conda.recipe/meta.yaml | 1 +
data2passcal/data2passcal.py | 44 +++++++++++++++++++-----------------
setup.py | 1 +
tests/test_data2passcal.py | 4 ++++
tox.ini | 5 +++-
5 files changed, 33 insertions(+), 22 deletions(-)
diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml
index ecfa27d..3509037 100644
--- a/conda.recipe/meta.yaml
+++ b/conda.recipe/meta.yaml
@@ -20,6 +20,7 @@ requirements:
test:
requires:
+ - timeout-decorator
- mock # [py<33]
source_files:
- tests
diff --git a/data2passcal/data2passcal.py b/data2passcal/data2passcal.py
index 38be7fc..cb2b713 100644
--- a/data2passcal/data2passcal.py
+++ b/data2passcal/data2passcal.py
@@ -18,6 +18,11 @@ import struct
import sys
from time import sleep, time
+try:
+ from urllib.request import urlopen
+except:
+ from urllib2 import urlopen
+
VERSION = '2020.119'
@@ -150,7 +155,8 @@ def format_size(num):
def ismseed(file):
try:
ms = open(file, 'rb')
- except:
+ except Exception as e:
+ logger.debug(e)
return None
order = ByteOrder(ms)
ms.close()
@@ -161,7 +167,6 @@ def ismseed(file):
return False
#########################################################
-# needs import sys,struct
# Taken from Bruces LibTrace
@@ -198,7 +203,8 @@ def ByteOrder(infile, seekval=20):
0 <= Min <= 59 and \
0 <= Sec <= 59:
Order = "little"
- except Exception:
+ except Exception as e:
+ logger.debug(e)
pass
return Order
@@ -234,18 +240,13 @@ def get_FTP():
trys = 0
while trys < FTP_CONNECT_ATTEMPTS:
trys += 1
+ logger.info('Connecting to FTP host %s. Attempt %d of %d'
+ % (FTP_HOST,
+ trys,
+ FTP_CONNECT_ATTEMPTS))
try:
- import socket
- logger.info('Connecting to FTP host %s from %s. Attempt %d of %d'
- % (FTP_HOST,
- socket.gethostbyname(socket.gethostname()),
- trys,
- FTP_CONNECT_ATTEMPTS))
FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER,
passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
- FTP.set_debuglevel(FTP_DEBUG_LEVEL)
- FTP.cwd(FTP_DIR)
- FTP.set_pasv(True)
except ftplib.all_errors as e:
logger.error('Failed to open FTP connection to %s' % FTP_HOST)
logger.error(e)
@@ -255,6 +256,9 @@ def get_FTP():
sleep(FTP_RECONNECT_WAIT)
else:
logger.info('Success: Connected to PASSCAL FTP')
+ FTP.set_debuglevel(FTP_DEBUG_LEVEL)
+ FTP.cwd(FTP_DIR)
+ FTP.set_pasv(True)
return FTP
logger.error('Giving up.')
return None
@@ -273,7 +277,7 @@ def passcal_http_reachable():
def google_http_reachable():
- '''download and time passcal home page'''
+ '''download and time google home page'''
url = 'http://www.google.com/'
return url_reachable(url=url)
@@ -286,12 +290,9 @@ def passcal_ftp_reachable():
def url_reachable(url='http://www.passcal.nmt.edu/'):
'''fetches a url returns True or False'''
- import urllib.request
- import urllib.error
- import urllib.parse
start = time()
try:
- f = urllib.request.urlopen(url)
+ f = urlopen(url)
except Exception as e:
logger.error("Failed to open connection to %s" % url)
logger.error(e)
@@ -383,10 +384,10 @@ def send2passcal(mslist, sentlist=None):
while trys < FTP_SEND_ATTEMPTS:
trys += 1
update.bytes_sent = 0
+ fh = open(f, 'rb')
+ file_size = os.path.getsize(f)
+ update.file_size = float(file_size)
try:
- fh = open(f, 'rb')
- file_size = os.path.getsize(f)
- update.file_size = float(file_size)
FTP.storbinary('STOR %s' % current_file, fh,
blocksize=FTP_BLOCKSIZE, callback=update)
except ftplib.error_perm as e:
@@ -411,7 +412,8 @@ def send2passcal(mslist, sentlist=None):
if FTP:
FTP.abort()
FTP.quit()
- except:
+ except Exception as e:
+ logger.debug(e)
pass
FTP = get_FTP()
if FTP is None:
diff --git a/setup.py b/setup.py
index d04d19d..001e26b 100644
--- a/setup.py
+++ b/setup.py
@@ -38,6 +38,7 @@ setup(
'dev': [
'flake8',
'tox',
+ 'timeout-decorator',
"mock;python_version<'3.3'"
]
},
diff --git a/tests/test_data2passcal.py b/tests/test_data2passcal.py
index 056f6ff..bf52fd0 100644
--- a/tests/test_data2passcal.py
+++ b/tests/test_data2passcal.py
@@ -8,6 +8,7 @@ from __future__ import division, print_function
import ftplib
import os
import sys
+import timeout_decorator
import unittest
from data2passcal.data2passcal import (get_FTP, ismseed, scan_dir,
@@ -51,6 +52,7 @@ class TestData2passcal(unittest.TestCase):
self.assertTrue(ismseed(f), '{} is not a miniseed file'
.format(os.path.basename(f)))
+ @timeout_decorator.timeout(5)
@patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
def test_get_FTP_mock(self, mock_ftp_constructor):
"""Mock test creating ftp connection to PASSCAL"""
@@ -61,6 +63,7 @@ class TestData2passcal(unittest.TestCase):
.format(FTP_CONNECT_ATTEMPTS))
mock_ftp.quit()
+ @timeout_decorator.timeout(5)
@patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
def test_send_data_mock(self, mock_ftp_constructor):
"""Mock test sending MSEED files (test data) to PASSCAL's QC system"""
@@ -81,6 +84,7 @@ class TestData2passcal(unittest.TestCase):
'Attempted to send file {0} more than {1} times'
.format(f, FTP_SEND_ATTEMPTS))
+ @timeout_decorator.timeout(5)
@unittest.skipIf(MOCK_TEST == True, "skipping real send2passcal test")
def test_send_data(self):
"""Test sending MSEED files (test data) to PASSCAL's QC system"""
diff --git a/tox.ini b/tox.ini
index d540347..7270737 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,8 +9,11 @@ commands = flake8 --ignore=E722,E712 data2passcal
[testenv:py27]
changedir = tests
-deps = mock
+deps =
+ mock
+ timeout-decorator
commands = python -m unittest test_data2passcal
[testenv]
+deps = timeout-decorator
commands = python -m unittest
--
GitLab