Update test_data2passcal.py - reduce running time for tests mocking FTP connection failure

data2passcal/data2passcal.py

@@ -258,10 +258,10 @@ def get_FTP():
             FTP.set_debuglevel(FTP_DEBUG_LEVEL)
             FTP.cwd(FTP_DIR)
             FTP.set_pasv(True)
-        except Exception:
+        except (ftplib.all_errors + (AttributeError,)):
             logger.exception('Failed to open FTP connection to %s' % FTP_HOST)
             test_network()
-            logger.info('Waiting %d seconds before trying to reconnect...'
+            logger.info('Waiting %.2f seconds before trying to reconnect...'
                         % FTP_RECONNECT_WAIT)
             sleep(FTP_RECONNECT_WAIT)
         else:
@@ -407,7 +407,7 @@ def send2passcal(mslist, sentlist=None):
                 logger.exception('Failed to send file %s, permission error.'
                                  % current_file)
                 break
-            except Exception:
+            except (ftplib.all_errors + (AttributeError,)):
                 # since we can restore with how we have proftp setup.
                 # There is nothing more we can do with this file
                 # Until the server rms the .in.file

tests/test_data2passcal.py

@@ -22,10 +22,6 @@ SEND4REAL = os.environ.get('SEND4REAL', 'False')
 print("SEND4REAL=False by default. If one wants to test sending data to "
       "PASSCAL for 'real', set SEND4REAL=True as environment variable. "
       "ex: SEND4REAL=True python -m unittest test_data2passcal")
-FTP_FAILURE = os.environ.get('FTP_FAILURE', 'False')
-print("FTP_FAILURE=False by default. If one wants to mock test a failure to "
-      "FTP connect to PASSCAL, set FTP_FAILURE=True as environment variable. "
-      "ex: FTP_FAILURE=True python -m unittest test_data2passcal")
 
 TEST_DIR = os.path.dirname(os.path.realpath(__file__)) + '/test_data'
 MS_FILELIST = ['ST00.AB..BHZ.2007.160', 'ST00.AB..BHZ.2007.161',
@@ -33,7 +29,7 @@ MS_FILELIST = ['ST00.AB..BHZ.2007.160', 'ST00.AB..BHZ.2007.161',
                'ST00.AB..BHZ.2007.164']
 
 d2p.FTP_TIMEOUT = 1
-d2p.FTP_RECONNECT_WAIT = 1
+d2p.FTP_RECONNECT_WAIT = 0.01
 d2p.FTP_CONNECT_ATTEMPTS = 2
 d2p.FTP_SEND_ATTEMPTS = 2
 
@@ -71,9 +67,9 @@ class TestData2passcal(unittest.TestCase):
                         .format(d2p.FTP_CONNECT_ATTEMPTS))
         mock_ftp.quit()
 
+    @patch('data2passcal.data2passcal.urlopen', autospec=True)
     @patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
-    @unittest.skipIf(FTP_FAILURE == 'False', "skipping mocking FTP failure")
-    def test_get_FTP_failure_mock(self, mock_ftp_constructor):
+    def test_get_FTP_failure_mock(self, mock_ftp_constructor, mock_urlopen):
         """
         Mock test failure to create ftp connection to PASSCAL and exercise
         get_FTP()
@@ -88,7 +84,7 @@ class TestData2passcal(unittest.TestCase):
     def test_send_data_mock(self, mock_ftp_constructor):
         """Mock test sending MSEED files (test data) to PASSCAL's QC system"""
         mock_ftp = mock_ftp_constructor.return_value
-        filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST]
+        filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST[0:2]]
         d2p.send2passcal(filelist)
         self.assertTrue(mock_ftp.storbinary.called, 'No data sent')
         self.assertEqual(mock_ftp.storbinary.call_count, len(filelist),
@@ -99,21 +95,22 @@ class TestData2passcal(unittest.TestCase):
         for x in mock_ftp.storbinary.call_args_list:
             args, kwargs = x
             files_sent.append(args[0].split(' ')[1])
-        for f in MS_FILELIST:
+        for f in MS_FILELIST[0:2]:
             self.assertLess(files_sent.count(f), d2p.FTP_SEND_ATTEMPTS,
                             'Attempted to send file {0} more than {1} times'
                             .format(f, d2p.FTP_SEND_ATTEMPTS))
 
     @patch('data2passcal.data2passcal.os._exit', autospec=True)
+    @patch('data2passcal.data2passcal.urlopen', autospec=True)
     @patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
-    @unittest.skipIf(FTP_FAILURE == 'False', "skipping mocking FTP failure")
-    def test_send_data_failure_mock(self, mock_ftp_constructor, mock_exit):
+    def test_send_data_failure_mock(self, mock_ftp_constructor,
+                                    mock_urlopen, mock_exit):
         """
         Mock test failure to create ftp connection to PASSCAL and exercise
         send_data()
         """
         mock_ftp_constructor.return_value = None
-        filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST]
+        filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST[0:2]]
         d2p.send2passcal(filelist)
         self.assertTrue(mock_exit.called, "os._exit(1) never called - "
                                           "send_data() not fully exercised!")
@@ -122,7 +119,7 @@ class TestData2passcal(unittest.TestCase):
     def test_send_data(self):
         """Test sending MSEED files (test data) to PASSCAL's QC system"""
         ftp = d2p.get_FTP()
-        filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST]
+        filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST[0:2]]
         d2p.send2passcal(filelist)
         wdir = ftp.pwd()
         try:
@@ -132,7 +129,7 @@ class TestData2passcal(unittest.TestCase):
                 print("No files found in this directory")
             else:
                 raise
-        for f in MS_FILELIST:
+        for f in MS_FILELIST[0:2]:
             self.assertIn(f, files_sent, 'File {} was not sent to PASSCAL'
                           .format(f))
         ftp.quit()

tox.ini

@@ -9,17 +9,13 @@ commands = flake8 --ignore=E722,E712 data2passcal
 
 [testenv:py27]
 changedir = tests
-passenv =
-         SEND4REAL
-         FTP_FAILURE
+passenv = SEND4REAL
 deps =
       mock
       timeout-decorator
 commands = python -m unittest test_data2passcal
 
 [testenv]
-passenv =
-         SEND4REAL
-         FTP_FAILURE
+passenv = SEND4REAL
 deps = timeout-decorator
 commands = python -m unittest