Update data2passcal.py - replace logger.error by logger.exception, add...

Update data2passcal.py - replace logger.error by logger.exception, add exception for socket.gethostname(), and rename test.dat; Update test_data2passcal.py - use data2passcal timeout constants, and set MOCK_TEST as an environment variable

conda.recipe/meta.yaml

@@ -20,7 +20,6 @@ requirements:
 
 test:
   requires:
-    - timeout-decorator
     - mock # [py<33]
   source_files:
     - tests

data2passcal/data2passcal.py

@@ -14,13 +14,14 @@ import os
 import pickle
 import re
 import signal
+import socket
 import struct
 import sys
 from time import sleep, time
 
 try:
     from urllib.request import urlopen
-except:
+except ImportError:
     from urllib2 import urlopen
 
 VERSION = '2020.119'
@@ -111,7 +112,7 @@ def scan_dir(dir):
             try:
                 filesize += os.path.getsize(f)
             except OSError:
-                logger.debug('Can not stat %s' % f)
+                logger.exception('Can not stat %s' % f)
             else:
                 filelist.append(f)
     logger.info('Total Size = %s' % format_size(filesize))
@@ -155,8 +156,9 @@ def format_size(num):
 def ismseed(file):
     try:
         ms = open(file, 'rb')
-    except Exception as e:
-        logger.debug(e)
+    except Exception:
+        logger.exception('Failed to open file %s in binary mode'
+                         % os.path.basename(file))
         return None
     order = ByteOrder(ms)
     ms.close()
@@ -203,8 +205,8 @@ def ByteOrder(infile, seekval=20):
                0 <= Min <= 59 and \
                0 <= Sec <= 59:
                 Order = "little"
-    except Exception as e:
-        logger.debug(e)
+    except Exception:
+        logger.exception('Failed to read time info from fixed header')
         pass
 
     return Order
@@ -240,25 +242,30 @@ def get_FTP():
     trys = 0
     while trys < FTP_CONNECT_ATTEMPTS:
         trys += 1
-        logger.info('Connecting to FTP host %s. Attempt %d of %d'
+        try:
+            local_ip = socket.gethostbyname(socket.gethostname())
+        except socket.gaierror:
+            logger.exception('Valid address-to-host mapping does not exist')
+            local_ip = ''
+        logger.info('Connecting to FTP host %s from %s. Attempt %d of %d'
                     % (FTP_HOST,
+                       local_ip,
                        trys,
                        FTP_CONNECT_ATTEMPTS))
         try:
             FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER,
                              passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
-        except ftplib.all_errors as e:
-            logger.error('Failed to open FTP connection to %s' % FTP_HOST)
-            logger.error(e)
+            FTP.set_debuglevel(FTP_DEBUG_LEVEL)
+            FTP.cwd(FTP_DIR)
+            FTP.set_pasv(True)
+        except ftplib.all_errors:
+            logger.exception('Failed to open FTP connection to %s' % FTP_HOST)
             test_network()
             logger.info('Waiting %d seconds before trying to reconnect...'
                         % FTP_RECONNECT_WAIT)
             sleep(FTP_RECONNECT_WAIT)
         else:
             logger.info('Success: Connected to PASSCAL FTP')
-            FTP.set_debuglevel(FTP_DEBUG_LEVEL)
-            FTP.cwd(FTP_DIR)
-            FTP.set_pasv(True)
             return FTP
     logger.error('Giving up.')
     return None
@@ -284,7 +291,7 @@ def google_http_reachable():
 
 def passcal_ftp_reachable():
     '''Download a small file from passcals general ftp'''
-    url = 'ftp://ftp.passcal.nmt.edu/download/public/test.dat'
+    url = 'ftp://ftp.passcal.nmt.edu/download/public/DO_NOT_DELETE.dat'
     return url_reachable(url=url)
 
 
@@ -293,9 +300,8 @@ def url_reachable(url='http://www.passcal.nmt.edu/'):
     start = time()
     try:
         f = urlopen(url)
-    except Exception as e:
-        logger.error("Failed to open connection to %s" % url)
-        logger.error(e)
+    except Exception:
+        logger.exception("Failed to open connection to %s" % url)
         return False
     logger.info('connection made to %s in %f sec' % (url, time() - start))
     data = f.read()
@@ -309,11 +315,13 @@ def url_reachable(url='http://www.passcal.nmt.edu/'):
 def test_FTP(FTP):
     try:
         FTP.voidcmd('NOOP')
-    except ftplib.all_errors as e:
-        logger.error(e)
+    except ftplib.all_errors:
+        logger.exception('Failed to send a simple command string to the server\
+                          and handle the response')
         return False
-    except AssertionError as e:
-        logger.error(e)
+    except AssertionError:
+        logger.exception('Failed to send a simple command string to the server\
+                          and handle the response')
         return False
     else:
         return True
@@ -339,8 +347,9 @@ def send2passcal(mslist, sentlist=None):
             if FTP:
                 FTP.abort()
                 FTP.quit()
-        except Exception as e:
-            logger.debug(e)
+        except Exception:
+            logger.exception("Failed to abort file transfer and close"
+                             "FTP connection")
         os._exit(1)
 
     def update(data):
@@ -390,30 +399,29 @@ def send2passcal(mslist, sentlist=None):
             try:
                 FTP.storbinary('STOR %s' % current_file, fh,
                                blocksize=FTP_BLOCKSIZE, callback=update)
-            except ftplib.error_perm as e:
+            except ftplib.error_perm:
                 # This is permission and the error when 550 for the .in file
                 # already exists so we should just continue with the next file
                 # todo create a list of failed files and resend those at the
                 # end instead of requiring a rerun
-                logger.error('Failed to send file %s, permission error.'
-                             % current_file)
-                logger.error(e)
+                logger.exception('Failed to send file %s, permission error.'
+                                 % current_file)
                 break
-            except (ftplib.all_errors, AttributeError) as e:
+            except (ftplib.all_errors, AttributeError):
                 # since we can restore with how we have proftp setup.
                 # There is nothing more we can do with this file
                 # Until the server rms the .in.file
                 # , trys, FTP_SEND_ATTEMPTS)
-                logger.error('Failed to send file %s.' % (current_file))
-                logger.error(e)
+                logger.exception('Failed to send file %s.' % (current_file))
                 # if DEBUG:
                 # print "Waiting %d..." %FTP_TIMEOUT; sleep(FTP_TIMEOUT)
                 try:
                     if FTP:
                         FTP.abort()
                         FTP.quit()
-                except Exception as e:
-                    logger.debug(e)
+                except Exception:
+                    logger.exception("Failed to abort file transfer and close"
+                                     "FTP connection")
                     pass
                 FTP = get_FTP()
                 if FTP is None:

setup.py

@@ -38,7 +38,6 @@ setup(
         'dev': [
             'flake8',
             'tox',
-            'timeout-decorator',
             "mock;python_version<'3.3'"
         ]
     },

tests/test_data2passcal.py

@@ -8,12 +8,8 @@ from __future__ import division, print_function
 import ftplib
 import os
 import sys
-import timeout_decorator
 import unittest
-
-from data2passcal.data2passcal import (get_FTP, ismseed, scan_dir,
-                                       send2passcal, FTP_CONNECT_ATTEMPTS,
-                                       FTP_SEND_ATTEMPTS)
+import data2passcal.data2passcal as d2p
 
 if sys.version_info < (3, 3):
     from mock import patch
@@ -22,13 +18,19 @@ else:
 
 VERSION = '2020.119'
 
-MOCK_TEST = True
+MOCK_TEST = os.environ.get('MOCK_TEST', 'True')
+print("MOCK_TEST = True by default. If one wants to test sending data to"
+      "PASSCAL for 'real', set MOCK_TEST=False as environment variable. "
+      "ex: MOCK_TEST=False python -m unittest test_data2passcal")
 
 TEST_DIR = os.path.dirname(os.path.realpath(__file__)) + '/test_data'
 MS_FILELIST = ['ST00.AB..BHZ.2007.160', 'ST00.AB..BHZ.2007.161',
                'ST00.AB..BHZ.2007.162', 'ST00.AB..BHZ.2007.163',
                'ST00.AB..BHZ.2007.164']
 
+d2p.FTP_TIMEOUT = 5
+d2p.FTP_RECONNECT_WAIT = 1
+
 
 class TestData2passcal(unittest.TestCase):
     """Tests for `data2passcal` package."""
@@ -39,37 +41,36 @@ class TestData2passcal(unittest.TestCase):
         """
         filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST]
         if sys.version_info < (3, 2):
-            self.assertItemsEqual(filelist, scan_dir(TEST_DIR),
+            self.assertItemsEqual(filelist, d2p.scan_dir(TEST_DIR),
                                   'scan_dir did not find the correct file(s)')
         else:
-            self.assertCountEqual(filelist, scan_dir(TEST_DIR),
+            self.assertCountEqual(filelist, d2p.scan_dir(TEST_DIR),
                                   'scan_dir did not find the correct file(s)')
 
     def test_ismseed(self):
         """Test basic functionality of ismseed function"""
         filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST]
         for f in filelist:
-            self.assertTrue(ismseed(f), '{} is not a miniseed file'
+            self.assertTrue(d2p.ismseed(f), '{} is not a miniseed file'
                             .format(os.path.basename(f)))
 
-    @timeout_decorator.timeout(5)
     @patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
     def test_get_FTP_mock(self, mock_ftp_constructor):
         """Mock test creating ftp connection to PASSCAL"""
         mock_ftp = mock_ftp_constructor.return_value
-        get_FTP()
-        self.assertLess(mock_ftp_constructor.call_count, FTP_CONNECT_ATTEMPTS,
+        d2p.get_FTP()
+        self.assertLess(mock_ftp_constructor.call_count,
+                        d2p.FTP_CONNECT_ATTEMPTS,
                         'Number of ftp connection attempts exceeeds {}'
-                        .format(FTP_CONNECT_ATTEMPTS))
+                        .format(d2p.FTP_CONNECT_ATTEMPTS))
         mock_ftp.quit()
 
-    @timeout_decorator.timeout(5)
     @patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
     def test_send_data_mock(self, mock_ftp_constructor):
         """Mock test sending MSEED files (test data) to PASSCAL's QC system"""
         mock_ftp = mock_ftp_constructor.return_value
         filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST]
-        send2passcal(filelist)
+        d2p.send2passcal(filelist)
         self.assertTrue(mock_ftp.storbinary.called, 'No data sent')
         self.assertEqual(mock_ftp.storbinary.call_count, len(filelist),
                          'Failed to send all files - Sent {0} of {1}'
@@ -80,17 +81,16 @@ class TestData2passcal(unittest.TestCase):
             args, kwargs = x
             files_sent.append(args[0].split(' ')[1])
         for f in MS_FILELIST:
-            self.assertLess(files_sent.count(f), FTP_SEND_ATTEMPTS,
+            self.assertLess(files_sent.count(f), d2p.FTP_SEND_ATTEMPTS,
                             'Attempted to send file {0} more than {1} times'
-                            .format(f, FTP_SEND_ATTEMPTS))
+                            .format(f, d2p.FTP_SEND_ATTEMPTS))
 
-    @timeout_decorator.timeout(5)
-    @unittest.skipIf(MOCK_TEST == True, "skipping real send2passcal test")
+    @unittest.skipIf(MOCK_TEST == 'True', "skipping real send2passcal test")
     def test_send_data(self):
         """Test sending MSEED files (test data) to PASSCAL's QC system"""
-        ftp = get_FTP()
+        ftp = d2p.get_FTP()
         filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST]
-        send2passcal(filelist)
+        d2p.send2passcal(filelist)
         wdir = ftp.pwd()
         try:
             files_sent = [os.path.basename(x) for x in ftp.nlst(wdir)]