Python 2/3 compatibility

Updated data2passcal to work with both Python 2 and 3.

• Resources used: Python-Future project and Futurize tool.
• Changes tested using unittest module on individual test methods(tests.test_data2passcal)
• Changes tested against real seismic dataset (AB.2002 - 2012)
• Added some unit tests to ensure basic functionality of data2passcal
• Updated list of platform specific dependencies to be installed when installing data2passcal in dev mode (see setup.py)
• Installed and tested data2passcal against Python2.7 and Python3.6 using tox
• Formatted Python code to conform to the PEP8 style guide (exception: E722, E712, E501)

data2passcal/data2passcal.py

@@ -21,13 +21,6 @@ from time import sleep, time
 VERSION = '2020.107'
 
 
-# Cache the ftplib.FTP class so it will be available to test_FTP(FTP) when calling isinstance assert
-FTPCache = ftplib.FTP
-
-
-# TEST related
-TESTMODE = False
-
 # FTP related
 FTP_IP = '129.138.26.29'
 FTP_HOST = 'qc.passcal.nmt.edu'
@@ -36,10 +29,6 @@ FTP_PASSWORD = 'data2passcal'
 FTP_DIR = 'AUTO/MSEED'
 FTP_TIMEOUT = 120
 FTP_RECONNECT_WAIT = 60
-if TESTMODE:
-    FTP_DIR = 'AUTO/TEST'
-    FTP_TIMEOUT = 5
-    FTP_RECONNECT_WAIT = 5
 FTP_BLOCKSIZE = 8192
 # number of time to try to open the ftp connection
 FTP_CONNECT_ATTEMPTS = 60 * 60 * 24 * 7 / FTP_RECONNECT_WAIT
@@ -52,19 +41,23 @@ FTP_DEBUG_LEVEL = 0
 LOGFILE = 'data2passcal.log'
 # store the files sent in ~/data2passcal.sent
 SENTFILE = os.path.join(os.path.expanduser('~'), '.data2passcal.sent')
-# If this file exists open it, incorporate, and save to new name, and delete old
+# If this file exists open it, incorporate and save to new name, and delete old
 SENTFILE_OLD = os.path.join(os.path.expanduser('~'), '.send2passcal.sent')
 
 HELP = '''
 data2passcal
 VERSION: %s
 Usage: data2passcal dir
-  data2passcal is a utility that sends day-long MSEED files ready for archival at the DMC to PASSCAL's QC system by:
-  Scanning all files below directory dir.
-  Filtering out non-miniseed files, by inspecting the first blockette of each file.
-  Sending the files to the automated system for ingestion into the QC system via ftp.
+  data2passcal is a utility that sends day-long MSEED files ready for archival
+  at the DMC to PASSCAL's QC system by:
+  - Scanning all files below directory dir.
+  - Filtering out non-miniseed files, by inspecting the first blockette of each
+    file.
+  - Sending the files to the automated system for ingestion into the QC system
+    via ftp.
   You can send a SIGTERM (ctl-c) to data2passcal and it will shutdown cleanly.
-  A list of sent files is kept in ~/.data2passcal.sent. Subsequent runs of send2passcal will not send files already sent.
+  A list of sent files is kept in ~/.data2passcal.sent.
+  Subsequent runs of send2passcal will not send files already sent.
   A log is stored in data2passcal.log in the current directory.
 ''' % VERSION
 
@@ -72,13 +65,16 @@ Usage: data2passcal dir
 DEBUG = logging.DEBUG
 
 # Configure logging file and stdout
-# todo if sending a file do we want to send all previous logs even if already sent
+# todo if sending a file do we want to send all previous logs even if already
+# sent
 logger = logging.getLogger('__name__')
 logger.setLevel(logging.DEBUG)
 # Log to a file with debug level
 logfh = logging.FileHandler(LOGFILE)
 logfh.setLevel(logging.DEBUG)
-logfh.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s -\t%(message)s', datefmt='%Y-%m-%d %H:%M:%S %z'))
+logfh.setFormatter(logging.Formatter(
+                   '%(asctime)s - %(levelname)s -\t%(message)s',
+                   datefmt='%Y-%m-%d %H:%M:%S %z'))
 logger.addHandler(logfh)
 # Log to stdout
 logconsole = logging.StreamHandler()
@@ -123,7 +119,7 @@ def scan_dir(dir):
 
 
 def sendable(file):
-    '''Filters files scanned returning a new list of files to send i.e. miniseed'''
+    '''Filters files scanned returning a new list of files to send'''
     if os.path.basename(file).startswith('.'):
         return False
     if ismseed(file):
@@ -133,7 +129,8 @@ def sendable(file):
 
 # Basic Miniseed file metadata extracted from filename
 # This includes .p files which we may accept in the future but won't h
-MseedRE = re.compile(r'\A(.*)\.([A-Z0-9][A-Z0-9])\.(.*)\.([A-Z][A-Z]\w)\.([0-9]{4})\.([0-9]{3})(?:\.p)*')
+MseedRE = re.compile(
+    r'\A(.*)\.([A-Z0-9][A-Z0-9])\.(.*)\.([A-Z][A-Z]\w)\.([0-9]{4})\.([0-9]{3})(?:\.p)*') # noqa
 
 
 def filename_qc_format(file):
@@ -230,14 +227,22 @@ def write_sent_file_list(sentlist, sentfile=SENTFILE):
 
 
 def get_FTP():
-    '''returns a FTP connection or None if a connection could not be made after attempts allowed'''
+    '''
+    returns a FTP connection or None if a connection could not be made
+    after number of attempts allowed
+    '''
     trys = 0
     while trys < FTP_CONNECT_ATTEMPTS:
         trys += 1
         try:
             import socket
-            logger.info('Connecting to FTP host %s from %s. Attempt %d of %d' % (FTP_HOST, socket.gethostbyname(socket.gethostname()), trys, FTP_CONNECT_ATTEMPTS))
-            FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
+            logger.info('Connecting to FTP host %s from %s. Attempt %d of %d'
+                        % (FTP_HOST,
+                           socket.gethostbyname(socket.gethostname()),
+                           trys,
+                           FTP_CONNECT_ATTEMPTS))
+            FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER,
+                             passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
             FTP.set_debuglevel(FTP_DEBUG_LEVEL)
             FTP.cwd(FTP_DIR)
             FTP.set_pasv(True)
@@ -245,7 +250,8 @@ def get_FTP():
             logger.error('Failed to open FTP connection to %s' % FTP_HOST)
             logger.error(e)
             test_network()
-            logger.info('Waiting %d seconds before trying to reconnect...' % FTP_RECONNECT_WAIT)
+            logger.info('Waiting %d seconds before trying to reconnect...'
+                        % FTP_RECONNECT_WAIT)
             sleep(FTP_RECONNECT_WAIT)
         else:
             logger.info('Success: Connected to PASSCAL FTP')
@@ -301,7 +307,6 @@ def url_reachable(url='http://www.passcal.nmt.edu/'):
 
 def test_FTP(FTP):
     try:
-        assert isinstance(FTP, FTPCache)
         FTP.voidcmd('NOOP')
     except ftplib.all_errors as e:
         logger.error(e)
@@ -316,13 +321,17 @@ def test_FTP(FTP):
 def send2passcal(mslist, sentlist=None):
     '''Send the list of files in mslist to passcal via FTP'''
 
-    # Handle SIGINT while in this function: to gracefully close connection and save sentlist to disk file
+    # Handle SIGINT while in this function: to gracefully close connection and
+    # save sentlist to disk file
     def signal_handler(signum, frame):
         print()
-        logger.info('Caught interrupt while FTPing. Aborting transfer %s' % current_file)
+        logger.info('Caught interrupt while FTPing. Aborting transfer %s'
+                    % current_file)
         logger.info('Sent %d of %d' % (num_sent, num_to_send))
-        logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)))
-        logger.info('%s /sec' % (format_size(size_sent / (time() - starttime))))
+        logger.info('Sent %s of %s'
+                    % (format_size(size_sent), format_size(size_to_send)))
+        logger.info('%s /sec'
+                    % (format_size(size_sent / (time() - starttime))))
         logger.info('Ran for %f sec' % (time() - starttime))
         write_sent_file_list(sentlist)
         try:
@@ -337,16 +346,19 @@ def send2passcal(mslist, sentlist=None):
         '''Updates the terminal display.'''
         signal.signal(signal.SIGINT, signal_handler)
         update.bytes_sent += len(data)
-        print('\r' + str(PB) + ' %s /sec ' % (format_size(size_sent / (time() - starttime))), end=' ')
+        print('\r' + str(PB) + ' %s /sec '
+              % (format_size(size_sent / (time() - starttime))), end=' ')
         '''
-        print '%s %0.2f%%. %0.10d / %0.10d.' % ( current_file.center(20),
-                                                (update.bytes_sent / update.file_size)*100,
-                                                update.bytes_sent,
-                                                file_size,
-                                                ) ,
+        print '%s %0.2f%%. %0.10d / %0.10d.'
+              % (current_file.center(20),
+                 (update.bytes_sent / update.file_size) * 100,
+                 update.bytes_sent,
+                 file_size)
         '''
-        ETA_sec = ((time() - starttime) / size_sent) * (size_to_send - size_sent)
-        print('ETA %s %s %s' % (str(datetime.timedelta(seconds=(ETA_sec))), current_file.center(20), ' ' * 20), end=' ')
+        ETA_sec = ((time() - starttime) / size_sent) * \
+            (size_to_send - size_sent)
+        print('ETA %s %s %s' % (str(datetime.timedelta(seconds=(ETA_sec))),
+                                current_file.center(20), ' ' * 20), end=' ')
         sys.stdout.flush()
 
     if sentlist is None:
@@ -359,7 +371,8 @@ def send2passcal(mslist, sentlist=None):
     size_sent = 1
     current_file = ''
     signal.signal(signal.SIGINT, signal_handler)
-    logger.info('Sending %d, %s files to PASSCAL' % (num_to_send, format_size(size_to_send)))
+    logger.info('Sending %d, %s files to PASSCAL' %
+                (num_to_send, format_size(size_to_send)))
     FTP = get_FTP()
     starttime = time()
     PB = ProgressBar(num_to_send)
@@ -374,19 +387,26 @@ def send2passcal(mslist, sentlist=None):
                 fh = open(f, 'rb')
                 file_size = os.path.getsize(f)
                 update.file_size = float(file_size)
-                FTP.storbinary('STOR %s' % current_file, fh, blocksize=FTP_BLOCKSIZE, callback=update)
+                FTP.storbinary('STOR %s' % current_file, fh,
+                               blocksize=FTP_BLOCKSIZE, callback=update)
             except ftplib.error_perm as e:
-                # This is permission and the error when 550 for the .in file already exists so we should just continue with the next file
-                # todo create a list of failed files and resend those at the end instead of requiring a rerun
-                logger.error('Failed to send file %s, permission error. Skipping...' % current_file)
+                # This is permission and the error when 550 for the .in file
+                # already exists so we should just continue with the next file
+                # todo create a list of failed files and resend those at the
+                # end instead of requiring a rerun
+                logger.error('Failed to send file %s, permission error.'
+                             % current_file)
                 logger.error(e)
                 break
             except (ftplib.all_errors, AttributeError) as e:
-                # since we can restore with how we have proftp setup. There is nothing more we can do with this file
+                # since we can restore with how we have proftp setup.
+                # There is nothing more we can do with this file
                 # Until the server rms the .in.file
-                logger.error('Failed to send file %s.' % (current_file))  # , trys, FTP_SEND_ATTEMPTS)
+                # , trys, FTP_SEND_ATTEMPTS)
+                logger.error('Failed to send file %s.' % (current_file))
                 logger.error(e)
-                # if DEBUG: print "Waiting %d..." % FTP_TIMEOUT; sleep(FTP_TIMEOUT)
+                # if DEBUG:
+                # print "Waiting %d..." %FTP_TIMEOUT; sleep(FTP_TIMEOUT)
                 try:
                     if FTP:
                         FTP.abort()
@@ -406,7 +426,8 @@ def send2passcal(mslist, sentlist=None):
 
     print()
     logger.info('Sent %d of %d' % (num_sent, num_to_send))
-    logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)))
+    logger.info('Sent %s of %s'
+                % (format_size(size_sent), format_size(size_to_send)))
     logger.info('%s /sec' % (format_size(size_sent / (time() - starttime))))
     logger.info('Ran for %f sec' % (time() - starttime))
     if FTP and test_FTP(FTP):
@@ -442,7 +463,8 @@ class ProgressBar(object):
         percent_done = int(round((new_amount / 100.0) * 100.0))
         all_full = self.width - 2
         num_hashes = int(round((percent_done / 100.0) * all_full))
-        self.prog_bar = '[' + self.fill_char * num_hashes + ' ' * (all_full - num_hashes) + ']'
+        self.prog_bar = '[' + self.fill_char * \
+            num_hashes + ' ' * (all_full - num_hashes) + ']'
         pct_place = int((len(self.prog_bar) / 2) - len(str(percent_done)))
         pct_string = '%d%%' % percent_done
         self.prog_bar = self.prog_bar[0:pct_place] + \
@@ -470,12 +492,14 @@ def main():
     logger.info('Removing files that are not Miniseed.')
     mslist = list(filter(sendable, msnamedlist))
     logger.info('MiniSEED files: %d' % len(mslist))
-    logger.info('Properly named but not miniseed files: %d' % (len(msnamedlist) - len(mslist)))
+    logger.info('Properly named but not miniseed files: %d'
+                % (len(msnamedlist) - len(mslist)))
     print()
     logger.info('Removing files already sent to PASSCAL')
     sentlist = get_sent_file_list()
     unsentms = [f for f in mslist if f not in sentlist]
-    logger.info('%d miniSEED files have already been sent, not resending.' % (len(mslist) - len(unsentms)))
+    logger.info('%d miniSEED files have already been sent, not resending.'
+                % (len(mslist) - len(unsentms)))
     send2passcal(unsentms, sentlist)
     write_sent_file_list(sentlist)