From a2fd4187fbab0e40e45bf978e4ffe814bcf9e352 Mon Sep 17 00:00:00 2001
From: Maeva Pourpoint <maeva@passcal.nmt.edu>
Date: Tue, 16 Jun 2020 13:00:25 -0600
Subject: [PATCH] Update test_data2passcal.py - reduce running time for tests
mocking FTP connection failure
---
data2passcal/data2passcal.py | 6 +++---
tests/test_data2passcal.py | 25 +++++++++++--------------
tox.ini | 8 ++------
3 files changed, 16 insertions(+), 23 deletions(-)
diff --git a/data2passcal/data2passcal.py b/data2passcal/data2passcal.py
index 2edc50b..6a136fc 100644
--- a/data2passcal/data2passcal.py
+++ b/data2passcal/data2passcal.py
@@ -258,10 +258,10 @@ def get_FTP():
FTP.set_debuglevel(FTP_DEBUG_LEVEL)
FTP.cwd(FTP_DIR)
FTP.set_pasv(True)
- except Exception:
+ except (ftplib.all_errors + (AttributeError,)):
logger.exception('Failed to open FTP connection to %s' % FTP_HOST)
test_network()
- logger.info('Waiting %d seconds before trying to reconnect...'
+ logger.info('Waiting %.2f seconds before trying to reconnect...'
% FTP_RECONNECT_WAIT)
sleep(FTP_RECONNECT_WAIT)
else:
@@ -407,7 +407,7 @@ def send2passcal(mslist, sentlist=None):
logger.exception('Failed to send file %s, permission error.'
% current_file)
break
- except Exception:
+ except (ftplib.all_errors + (AttributeError,)):
# since we can restore with how we have proftp setup.
# There is nothing more we can do with this file
# Until the server rms the .in.file
diff --git a/tests/test_data2passcal.py b/tests/test_data2passcal.py
index cb075bc..7c8c0ff 100644
--- a/tests/test_data2passcal.py
+++ b/tests/test_data2passcal.py
@@ -22,10 +22,6 @@ SEND4REAL = os.environ.get('SEND4REAL', 'False')
print("SEND4REAL=False by default. If one wants to test sending data to "
"PASSCAL for 'real', set SEND4REAL=True as environment variable. "
"ex: SEND4REAL=True python -m unittest test_data2passcal")
-FTP_FAILURE = os.environ.get('FTP_FAILURE', 'False')
-print("FTP_FAILURE=False by default. If one wants to mock test a failure to "
- "FTP connect to PASSCAL, set FTP_FAILURE=True as environment variable. "
- "ex: FTP_FAILURE=True python -m unittest test_data2passcal")
TEST_DIR = os.path.dirname(os.path.realpath(__file__)) + '/test_data'
MS_FILELIST = ['ST00.AB..BHZ.2007.160', 'ST00.AB..BHZ.2007.161',
@@ -33,7 +29,7 @@ MS_FILELIST = ['ST00.AB..BHZ.2007.160', 'ST00.AB..BHZ.2007.161',
'ST00.AB..BHZ.2007.164']
d2p.FTP_TIMEOUT = 1
-d2p.FTP_RECONNECT_WAIT = 1
+d2p.FTP_RECONNECT_WAIT = 0.01
d2p.FTP_CONNECT_ATTEMPTS = 2
d2p.FTP_SEND_ATTEMPTS = 2
@@ -71,9 +67,9 @@ class TestData2passcal(unittest.TestCase):
.format(d2p.FTP_CONNECT_ATTEMPTS))
mock_ftp.quit()
+ @patch('data2passcal.data2passcal.urlopen', autospec=True)
@patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
- @unittest.skipIf(FTP_FAILURE == 'False', "skipping mocking FTP failure")
- def test_get_FTP_failure_mock(self, mock_ftp_constructor):
+ def test_get_FTP_failure_mock(self, mock_ftp_constructor, mock_urlopen):
"""
Mock test failure to create ftp connection to PASSCAL and exercise
get_FTP()
@@ -88,7 +84,7 @@ class TestData2passcal(unittest.TestCase):
def test_send_data_mock(self, mock_ftp_constructor):
"""Mock test sending MSEED files (test data) to PASSCAL's QC system"""
mock_ftp = mock_ftp_constructor.return_value
- filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST]
+ filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST[0:2]]
d2p.send2passcal(filelist)
self.assertTrue(mock_ftp.storbinary.called, 'No data sent')
self.assertEqual(mock_ftp.storbinary.call_count, len(filelist),
@@ -99,21 +95,22 @@ class TestData2passcal(unittest.TestCase):
for x in mock_ftp.storbinary.call_args_list:
args, kwargs = x
files_sent.append(args[0].split(' ')[1])
- for f in MS_FILELIST:
+ for f in MS_FILELIST[0:2]:
self.assertLess(files_sent.count(f), d2p.FTP_SEND_ATTEMPTS,
'Attempted to send file {0} more than {1} times'
.format(f, d2p.FTP_SEND_ATTEMPTS))
@patch('data2passcal.data2passcal.os._exit', autospec=True)
+ @patch('data2passcal.data2passcal.urlopen', autospec=True)
@patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
- @unittest.skipIf(FTP_FAILURE == 'False', "skipping mocking FTP failure")
- def test_send_data_failure_mock(self, mock_ftp_constructor, mock_exit):
+ def test_send_data_failure_mock(self, mock_ftp_constructor,
+ mock_urlopen, mock_exit):
"""
Mock test failure to create ftp connection to PASSCAL and exercise
send_data()
"""
mock_ftp_constructor.return_value = None
- filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST]
+ filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST[0:2]]
d2p.send2passcal(filelist)
self.assertTrue(mock_exit.called, "os._exit(1) never called - "
"send_data() not fully exercised!")
@@ -122,7 +119,7 @@ class TestData2passcal(unittest.TestCase):
def test_send_data(self):
"""Test sending MSEED files (test data) to PASSCAL's QC system"""
ftp = d2p.get_FTP()
- filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST]
+ filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST[0:2]]
d2p.send2passcal(filelist)
wdir = ftp.pwd()
try:
@@ -132,7 +129,7 @@ class TestData2passcal(unittest.TestCase):
print("No files found in this directory")
else:
raise
- for f in MS_FILELIST:
+ for f in MS_FILELIST[0:2]:
self.assertIn(f, files_sent, 'File {} was not sent to PASSCAL'
.format(f))
ftp.quit()
diff --git a/tox.ini b/tox.ini
index 2c6aebb..954a1f0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,17 +9,13 @@ commands = flake8 --ignore=E722,E712 data2passcal
[testenv:py27]
changedir = tests
-passenv =
- SEND4REAL
- FTP_FAILURE
+passenv = SEND4REAL
deps =
mock
timeout-decorator
commands = python -m unittest test_data2passcal
[testenv]
-passenv =
- SEND4REAL
- FTP_FAILURE
+passenv = SEND4REAL
deps = timeout-decorator
commands = python -m unittest
--
GitLab