diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml
index 35090374d6ad5d46790d5a1b17d50fd0b750fa9b..ecfa27df20eaba1e8fa3dc106867aef7357d34a8 100644
--- a/conda.recipe/meta.yaml
+++ b/conda.recipe/meta.yaml
@@ -20,7 +20,6 @@ requirements:
test:
requires:
- - timeout-decorator
- mock # [py<33]
source_files:
- tests
diff --git a/data2passcal/data2passcal.py b/data2passcal/data2passcal.py
index cb2b713ec7556f73732fdc84d25aa6f8d9aa11f8..7d6a5f7f1b732b25858df7168999339302eb25bd 100644
--- a/data2passcal/data2passcal.py
+++ b/data2passcal/data2passcal.py
@@ -14,13 +14,14 @@ import os
import pickle
import re
import signal
+import socket
import struct
import sys
from time import sleep, time
try:
from urllib.request import urlopen
-except:
+except ImportError:
from urllib2 import urlopen
VERSION = '2020.119'
@@ -111,7 +112,7 @@ def scan_dir(dir):
try:
filesize += os.path.getsize(f)
except OSError:
- logger.debug('Can not stat %s' % f)
+ logger.exception('Can not stat %s' % f)
else:
filelist.append(f)
logger.info('Total Size = %s' % format_size(filesize))
@@ -155,8 +156,9 @@ def format_size(num):
def ismseed(file):
try:
ms = open(file, 'rb')
- except Exception as e:
- logger.debug(e)
+ except Exception:
+ logger.exception('Failed to open file %s in binary mode'
+ % os.path.basename(file))
return None
order = ByteOrder(ms)
ms.close()
@@ -203,8 +205,8 @@ def ByteOrder(infile, seekval=20):
0 <= Min <= 59 and \
0 <= Sec <= 59:
Order = "little"
- except Exception as e:
- logger.debug(e)
+ except Exception:
+ logger.exception('Failed to read time info from fixed header')
pass
return Order
@@ -240,25 +242,30 @@ def get_FTP():
trys = 0
while trys < FTP_CONNECT_ATTEMPTS:
trys += 1
- logger.info('Connecting to FTP host %s. Attempt %d of %d'
+ try:
+ local_ip = socket.gethostbyname(socket.gethostname())
+ except socket.gaierror:
+ logger.exception('Valid address-to-host mapping does not exist')
+ local_ip = ''
+ logger.info('Connecting to FTP host %s from %s. Attempt %d of %d'
% (FTP_HOST,
+ local_ip,
trys,
FTP_CONNECT_ATTEMPTS))
try:
FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER,
passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
- except ftplib.all_errors as e:
- logger.error('Failed to open FTP connection to %s' % FTP_HOST)
- logger.error(e)
+ FTP.set_debuglevel(FTP_DEBUG_LEVEL)
+ FTP.cwd(FTP_DIR)
+ FTP.set_pasv(True)
+ except ftplib.all_errors:
+ logger.exception('Failed to open FTP connection to %s' % FTP_HOST)
test_network()
logger.info('Waiting %d seconds before trying to reconnect...'
% FTP_RECONNECT_WAIT)
sleep(FTP_RECONNECT_WAIT)
else:
logger.info('Success: Connected to PASSCAL FTP')
- FTP.set_debuglevel(FTP_DEBUG_LEVEL)
- FTP.cwd(FTP_DIR)
- FTP.set_pasv(True)
return FTP
logger.error('Giving up.')
return None
@@ -284,7 +291,7 @@ def google_http_reachable():
def passcal_ftp_reachable():
'''Download a small file from passcals general ftp'''
- url = 'ftp://ftp.passcal.nmt.edu/download/public/test.dat'
+ url = 'ftp://ftp.passcal.nmt.edu/download/public/DO_NOT_DELETE.dat'
return url_reachable(url=url)
@@ -293,9 +300,8 @@ def url_reachable(url='http://www.passcal.nmt.edu/'):
start = time()
try:
f = urlopen(url)
- except Exception as e:
- logger.error("Failed to open connection to %s" % url)
- logger.error(e)
+ except Exception:
+ logger.exception("Failed to open connection to %s" % url)
return False
logger.info('connection made to %s in %f sec' % (url, time() - start))
data = f.read()
@@ -309,11 +315,13 @@ def url_reachable(url='http://www.passcal.nmt.edu/'):
def test_FTP(FTP):
try:
FTP.voidcmd('NOOP')
- except ftplib.all_errors as e:
- logger.error(e)
+ except ftplib.all_errors:
+ logger.exception('Failed to send a simple command string to the server\
+ and handle the response')
return False
- except AssertionError as e:
- logger.error(e)
+ except AssertionError:
+ logger.exception('Failed to send a simple command string to the server\
+ and handle the response')
return False
else:
return True
@@ -339,8 +347,9 @@ def send2passcal(mslist, sentlist=None):
if FTP:
FTP.abort()
FTP.quit()
- except Exception as e:
- logger.debug(e)
+ except Exception:
+ logger.exception("Failed to abort file transfer and close"
+ "FTP connection")
os._exit(1)
def update(data):
@@ -390,30 +399,29 @@ def send2passcal(mslist, sentlist=None):
try:
FTP.storbinary('STOR %s' % current_file, fh,
blocksize=FTP_BLOCKSIZE, callback=update)
- except ftplib.error_perm as e:
+ except ftplib.error_perm:
# This is permission and the error when 550 for the .in file
# already exists so we should just continue with the next file
# todo create a list of failed files and resend those at the
# end instead of requiring a rerun
- logger.error('Failed to send file %s, permission error.'
- % current_file)
- logger.error(e)
+ logger.exception('Failed to send file %s, permission error.'
+ % current_file)
break
- except (ftplib.all_errors, AttributeError) as e:
+ except (ftplib.all_errors, AttributeError):
# since we can restore with how we have proftp setup.
# There is nothing more we can do with this file
# Until the server rms the .in.file
# , trys, FTP_SEND_ATTEMPTS)
- logger.error('Failed to send file %s.' % (current_file))
- logger.error(e)
+ logger.exception('Failed to send file %s.' % (current_file))
# if DEBUG:
# print "Waiting %d..." %FTP_TIMEOUT; sleep(FTP_TIMEOUT)
try:
if FTP:
FTP.abort()
FTP.quit()
- except Exception as e:
- logger.debug(e)
+ except Exception:
+ logger.exception("Failed to abort file transfer and close"
+ "FTP connection")
pass
FTP = get_FTP()
if FTP is None:
diff --git a/setup.py b/setup.py
index 001e26b88fee3276f442c74f71fdb32228ba52e0..d04d19d6d99b97b44644fccc6f2c55d640c6ee57 100644
--- a/setup.py
+++ b/setup.py
@@ -38,7 +38,6 @@ setup(
'dev': [
'flake8',
'tox',
- 'timeout-decorator',
"mock;python_version<'3.3'"
]
},
diff --git a/tests/test_data2passcal.py b/tests/test_data2passcal.py
index bf52fd077f452f7a72cd419b4cf7f9c510d3fdd0..ccb61338ac1f7b6cb582ad0a3becce0e57366a1e 100644
--- a/tests/test_data2passcal.py
+++ b/tests/test_data2passcal.py
@@ -8,12 +8,8 @@ from __future__ import division, print_function
import ftplib
import os
import sys
-import timeout_decorator
import unittest
-
-from data2passcal.data2passcal import (get_FTP, ismseed, scan_dir,
- send2passcal, FTP_CONNECT_ATTEMPTS,
- FTP_SEND_ATTEMPTS)
+import data2passcal.data2passcal as d2p
if sys.version_info < (3, 3):
from mock import patch
@@ -22,13 +18,19 @@ else:
VERSION = '2020.119'
-MOCK_TEST = True
+MOCK_TEST = os.environ.get('MOCK_TEST', 'True')
+print("MOCK_TEST = True by default. If one wants to test sending data to"
+ "PASSCAL for 'real', set MOCK_TEST=False as environment variable. "
+ "ex: MOCK_TEST=False python -m unittest test_data2passcal")
TEST_DIR = os.path.dirname(os.path.realpath(__file__)) + '/test_data'
MS_FILELIST = ['ST00.AB..BHZ.2007.160', 'ST00.AB..BHZ.2007.161',
'ST00.AB..BHZ.2007.162', 'ST00.AB..BHZ.2007.163',
'ST00.AB..BHZ.2007.164']
+d2p.FTP_TIMEOUT = 5
+d2p.FTP_RECONNECT_WAIT = 1
+
class TestData2passcal(unittest.TestCase):
"""Tests for `data2passcal` package."""
@@ -39,37 +41,36 @@ class TestData2passcal(unittest.TestCase):
"""
filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST]
if sys.version_info < (3, 2):
- self.assertItemsEqual(filelist, scan_dir(TEST_DIR),
+ self.assertItemsEqual(filelist, d2p.scan_dir(TEST_DIR),
'scan_dir did not find the correct file(s)')
else:
- self.assertCountEqual(filelist, scan_dir(TEST_DIR),
+ self.assertCountEqual(filelist, d2p.scan_dir(TEST_DIR),
'scan_dir did not find the correct file(s)')
def test_ismseed(self):
"""Test basic functionality of ismseed function"""
filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST]
for f in filelist:
- self.assertTrue(ismseed(f), '{} is not a miniseed file'
+ self.assertTrue(d2p.ismseed(f), '{} is not a miniseed file'
.format(os.path.basename(f)))
- @timeout_decorator.timeout(5)
@patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
def test_get_FTP_mock(self, mock_ftp_constructor):
"""Mock test creating ftp connection to PASSCAL"""
mock_ftp = mock_ftp_constructor.return_value
- get_FTP()
- self.assertLess(mock_ftp_constructor.call_count, FTP_CONNECT_ATTEMPTS,
+ d2p.get_FTP()
+ self.assertLess(mock_ftp_constructor.call_count,
+ d2p.FTP_CONNECT_ATTEMPTS,
'Number of ftp connection attempts exceeeds {}'
- .format(FTP_CONNECT_ATTEMPTS))
+ .format(d2p.FTP_CONNECT_ATTEMPTS))
mock_ftp.quit()
- @timeout_decorator.timeout(5)
@patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
def test_send_data_mock(self, mock_ftp_constructor):
"""Mock test sending MSEED files (test data) to PASSCAL's QC system"""
mock_ftp = mock_ftp_constructor.return_value
filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST]
- send2passcal(filelist)
+ d2p.send2passcal(filelist)
self.assertTrue(mock_ftp.storbinary.called, 'No data sent')
self.assertEqual(mock_ftp.storbinary.call_count, len(filelist),
'Failed to send all files - Sent {0} of {1}'
@@ -80,17 +81,16 @@ class TestData2passcal(unittest.TestCase):
args, kwargs = x
files_sent.append(args[0].split(' ')[1])
for f in MS_FILELIST:
- self.assertLess(files_sent.count(f), FTP_SEND_ATTEMPTS,
+ self.assertLess(files_sent.count(f), d2p.FTP_SEND_ATTEMPTS,
'Attempted to send file {0} more than {1} times'
- .format(f, FTP_SEND_ATTEMPTS))
+ .format(f, d2p.FTP_SEND_ATTEMPTS))
- @timeout_decorator.timeout(5)
- @unittest.skipIf(MOCK_TEST == True, "skipping real send2passcal test")
+ @unittest.skipIf(MOCK_TEST == 'True', "skipping real send2passcal test")
def test_send_data(self):
"""Test sending MSEED files (test data) to PASSCAL's QC system"""
- ftp = get_FTP()
+ ftp = d2p.get_FTP()
filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST]
- send2passcal(filelist)
+ d2p.send2passcal(filelist)
wdir = ftp.pwd()
try:
files_sent = [os.path.basename(x) for x in ftp.nlst(wdir)]