diff --git a/AUTHORS.rst b/AUTHORS.rst
index 8029b2489bd99e83141f2a6e55f9aeb1f9c6f30c..207d6dd429fa08cf357717a87a587314d3012b29 100644
--- a/AUTHORS.rst
+++ b/AUTHORS.rst
@@ -10,4 +10,4 @@ Development Lead
Contributors
------------
-None yet. Why not be the first?
+* Maeva Pourpoint <software-support@passcal.nmt.edu>
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index e8536cc925d7042cd527fed67034e6b6781936cf..b262255e1da62b498a111fdf7990ac7c80c4afdb 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -76,7 +76,7 @@ Ready to contribute? Here's how to set up `data2passcal` for local development.
5. When you're done making changes, check that your changes pass the
tests::
- $ python setup.py test
+ $ python -m pytest
6. Commit your changes and push your branch to GitHub::
$ git add .
@@ -111,4 +111,3 @@ Then run::
$ git push
$ git push --tags
-
diff --git a/HISTORY.rst b/HISTORY.rst
index db916b9f439e3d1737299b77f75377f8c715b676..d35c92df96d952c9a569f2e4cdf40715dd450ed9 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -17,3 +17,11 @@ History
* Resources used: Python-Future project and Futurize tool.
* Changes tested using unittest module on individual test methods(tests.test_data2passcal)
* Changes tested against real seismic dataset (AB.2002 - 2012)
+
+2020.107 (2020-04-16)
+------------------
+* Added some unit tests to ensure basic functionality of data2passcal
+* Created configuration file (.ini file) to store and access ftp and test related variables
+* Updated list of platform specific dependencies to be installed when installing data2passcal (see setup.py)
+* Installed and tested data2passcal against Python2.7 and Python3.6 using tox
+* Formatted Python code to conform to the PEP8 style guide (exception: E722, E712, E501)
diff --git a/README.rst b/README.rst
index efc03cf70b5aeeb77a45be7fff9dd765383a0534..a1160b99c3905a79cfa2cd8fa0e04228ddbc9968 100644
--- a/README.rst
+++ b/README.rst
@@ -3,23 +3,9 @@ data2passcal
============
-Prepare SEED data for shipment to PASSCAL.
-Only accept day-long MSEED files.
+Description: Send MSEED files ready for archival at the DMC to PASSCAL's QC system
+ Only accept day-long MSEED files.
+Usage: data2passcal dir
* Free software: GNU General Public License v3 (GPLv3)
-
-
-
-Features
---------
-
-* TODO
-
-Credits
--------
-
-This package was created with Cookiecutter_ and the `passoft/cookiecutter`_ project template.
-
-.. _Cookiecutter: https://github.com/audreyr/cookiecutter
-.. _`passoft/cookiecutter`: https://git.passcal.nmt.edu/passoft/cookiecutter
diff --git a/data2passcal/__init__.py b/data2passcal/__init__.py
index afe21e9225139b7e746aabc93df87e9991ff3b07..8f21cef6e0f1a9f038e49585d5d448812b2883df 100644
--- a/data2passcal/__init__.py
+++ b/data2passcal/__init__.py
@@ -4,4 +4,4 @@
__author__ = """IRIS PASSCAL"""
__email__ = 'software-support@passcal.nmt.edu'
-__version__ = '2020.093'
+__version__ = '2020.107'
diff --git a/data2passcal/config.ini b/data2passcal/config.ini
new file mode 100644
index 0000000000000000000000000000000000000000..6422e381c6501486395a1a03217b97da2d96db20
--- /dev/null
+++ b/data2passcal/config.ini
@@ -0,0 +1,22 @@
+[FTP_INFO]
+ftp_ip = 129.138.26.29
+ftp_user = ftp
+ftp_password = data2passcal
+ftp_send_attempts = 3
+ftp_blocksize = 8192
+ftp_timeout = 120
+ftp_dir = AUTO/MSEED
+ftp_reconnect_wait = 60
+ftp_debug_level = 0
+ftp_host = qc.passcal.nmt.edu
+
+[TEST]
+ftp_dir = AUTO/TEST
+ftp_reconnect_wait = 5
+mock_test = True
+testmode = False
+ftp_timeout = 5
+
+[FILES]
+logfile = data2passcal.log
+
diff --git a/data2passcal/config.py b/data2passcal/config.py
new file mode 100644
index 0000000000000000000000000000000000000000..6f3ac698871c815665f4efe475f7e8570bc08de4
--- /dev/null
+++ b/data2passcal/config.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+'''
+data2passcal
+
+Create configuration file
+'''
+
+import configparser
+
+VERSION = '2020.107'
+
+config = configparser.ConfigParser()
+config['FTP_INFO'] = {'FTP_IP': '129.138.26.29',
+ 'FTP_HOST': 'qc.passcal.nmt.edu',
+ 'FTP_USER': 'ftp',
+ 'FTP_PASSWORD': 'data2passcal',
+ 'FTP_DIR': 'AUTO/MSEED',
+ 'FTP_TIMEOUT': '120',
+ 'FTP_RECONNECT_WAIT': '60',
+ 'FTP_BLOCKSIZE': '8192',
+ 'FTP_SEND_ATTEMPTS': '3',
+ 'FTP_DEBUG_LEVEL': '0'}
+config['TEST'] = {'TESTMODE': 'False',
+ 'FTP_DIR': 'AUTO/TEST',
+ 'FTP_TIMEOUT': '5',
+ 'FTP_RECONNECT_WAIT': '5',
+ 'MOCK_TEST': 'True'}
+config['FILES'] = {'LOGFILE': 'data2passcal.log'}
+
+with open('config.ini', 'w') as configfile:
+ config.write(configfile)
diff --git a/data2passcal/data2passcal.py b/data2passcal/data2passcal.py
index a4afaafc0317720dab90d5c265387f05d369a197..7cb855196ee6f11a189b0478270d8815548fbd6d 100644
--- a/data2passcal/data2passcal.py
+++ b/data2passcal/data2passcal.py
@@ -5,62 +5,77 @@ data2passcal
Ship miniseed data to passcal via ftp for qc before archival at DMC DMS
Lloyd Carothers IRIS/PASSCAL
'''
-from __future__ import print_function, division
-from builtins import *
-VERSION = '2020.093'
+from __future__ import division, print_function
-import ftplib
-import sys, os, signal, struct, pickle, logging
-from time import time, sleep
+import configparser
import datetime
+import ftplib
+import logging
+import os
+import pickle
+import re
+import signal
+import struct
+import sys
+from time import sleep, time
-#TESTMODE = True
-TESTMODE = False
-# Verbosity of console log from logging module from DEBUG to CRITICAL
-DEBUG = logging.DEBUG
-# After transmission or on interrupt try to ftp logfile to PIC
-#SEND_LOGFILE = True
-
-#FTP related
-FTP_IP = '129.138.26.29'
-FTP_HOST = 'qc.passcal.nmt.edu'
-FTP_USER = 'ftp'
-FTP_PASSWORD = 'data2passcal'
-FTP_DIR = 'AUTO/MSEED'
-FTP_TIMEOUT = 120
-FTP_RECONNECT_WAIT = 60
+VERSION = '2020.107'
+
+
+# Cache the ftplib.FTP class so it will be available to test_FTP(FTP) when calling isinstance assert
+FTPCache = ftplib.FTP
+
+# Parse configuration file
+config = configparser.ConfigParser()
+print(config.read('data2passcal/config.ini'))
+
+
+# TEST related
+TESTMODE = config.getboolean('TEST', 'TESTMODE')
+
+# FTP related
+FTP_IP = config.get('FTP_INFO', 'FTP_IP')
+FTP_HOST = config.get('FTP_INFO', 'FTP_HOST')
+FTP_USER = config.get('FTP_INFO', 'FTP_USER')
+FTP_PASSWORD = config.get('FTP_INFO', 'FTP_PASSWORD')
+FTP_DIR = config.get('FTP_INFO', 'FTP_DIR')
+FTP_TIMEOUT = config.getint('FTP_INFO', 'FTP_TIMEOUT')
+FTP_RECONNECT_WAIT = config.getint('FTP_INFO', 'FTP_RECONNECT_WAIT')
if TESTMODE:
- FTP_DIR = 'AUTO/TEST'
- FTP_TIMEOUT = 5
- FTP_RECONNECT_WAIT = 5
-FTP_BLOCKSIZE = 8192
-#number of time to try to open the ftp connection
-#FTP_CONNECT_ATTEMPTS = 3
-# testing will retry for a week. Why not?
-FTP_CONNECT_ATTEMPTS = 60*60*24*7 / FTP_RECONNECT_WAIT
-#number of times a single file with try to be sent
-FTP_SEND_ATTEMPTS = 3
+ FTP_DIR = config.get('TEST', 'FTP_DIR')
+ FTP_TIMEOUT = config.getint('TEST', 'FTP_TIMEOUT')
+ FTP_RECONNECT_WAIT = config.getint('TEST', 'FTP_RECONNECT_WAIT')
+FTP_BLOCKSIZE = config.getint('FTP_INFO', 'FTP_BLOCKSIZE')
+# number of time to try to open the ftp connection - testing will retry for a week. Why not?
+FTP_CONNECT_ATTEMPTS = 60 * 60 * 24 * 7 / FTP_RECONNECT_WAIT
+# number of times a single file with try to be sent
+FTP_SEND_ATTEMPTS = config.getint('FTP_INFO', 'FTP_SEND_ATTEMPTS')
# debug level of ftplib, 0-3
-FTP_DEBUG_LEVEL = 0
-#store the files sent in ~/data2passcal.sent
-SENTFILE = os.path.join( os.path.expanduser('~'), '.data2passcal.sent')
+FTP_DEBUG_LEVEL = config.getint('FTP_INFO', 'FTP_DEBUG_LEVEL')
+
+# Files related
+LOGFILE = config.get('FILES', 'LOGFILE')
+# store the files sent in ~/data2passcal.sent
+SENTFILE = os.path.join(os.path.expanduser('~'), '.data2passcal.sent')
# If this file exists open it, incorporate, and save to new name, and delete old
-SENTFILE_OLD = os.path.join( os.path.expanduser('~'), '.send2passcal.sent')
-LOGFILE = 'data2passcal.log'
+SENTFILE_OLD = os.path.join(os.path.expanduser('~'), '.send2passcal.sent')
HELP = '''
data2passcal
VERSION: %s
Usage: data2passcal dir
- data2passcal is a utility that sends MSEED files ready for archival at the DMC to PASSCAL's QC system by:
- Scans all files below directory dir.
- Filters out non-miniseed files, by inspecting the first blockette of each file.
- Sends the files to the automated system for ingestion into the QC system via ftp.
+ data2passcal is a utility that sends day-long MSEED files ready for archival at the DMC to PASSCAL's QC system by:
+ Scanning all files below directory dir.
+ Filtering out non-miniseed files, by inspecting the first blockette of each file.
+ Sending the files to the automated system for ingestion into the QC system via ftp.
You can send a SIGTERM (ctl-c) to data2passcal and it will shutdown cleanly.
A list of sent files is kept in ~/.data2passcal.sent. Subsequent runs of send2passcal will not send files already sent.
A log is stored in data2passcal.log in the current directory.
''' % VERSION
+# Verbosity of console log from logging module from DEBUG to CRITICAL
+DEBUG = logging.DEBUG
+
# Configure logging file and stdout
# todo if sending a file do we want to send all previous logs even if already sent
logger = logging.getLogger('__name__')
@@ -73,16 +88,17 @@ logger.addHandler(logfh)
# Log to stdout
logconsole = logging.StreamHandler()
logconsole.setLevel(logging.INFO)
-logconsole.setFormatter(logging.Formatter('%(message)s' ))
+logconsole.setFormatter(logging.Formatter('%(message)s'))
logger.addHandler(logconsole)
logger.debug('Program starting.')
-logger.debug('%s - v%s - Python:%s' % (sys.argv, VERSION, sys.version) )
+logger.debug('%s - v%s - Python:%s' % (sys.argv, VERSION, sys.version))
logger.debug('CWD: %s' % os.getcwd())
-logger.info('Version: '+ VERSION)
+logger.info('Version: ' + VERSION)
logger.info('TIMEOUT: %d' % FTP_TIMEOUT)
logger.info('FTP RECONNECT WAIT: %d' % FTP_RECONNECT_WAIT)
# -- End logging
+
def scan_dir(dir):
'''Returns a list of absolute file names found below root dir'''
rootdir = dir
@@ -99,17 +115,18 @@ def scan_dir(dir):
try:
filesize += os.path.getsize(f)
except OSError:
- logger.debug('Can not stat %s'% f)
+ logger.debug('Can not stat %s' % f)
else:
filelist.append(f)
- logger.info('Total Size = %s'% format_size(filesize) )
- logger.info('Total Files = %s'% len(filelist) )
- logger.info('Total Dirs = %s'% foldercount)
- logger.info('Scan time = %0.2fs'% (time() - starttime))
+ logger.info('Total Size = %s' % format_size(filesize))
+ logger.info('Total Files = %s' % len(filelist))
+ logger.info('Total Dirs = %s' % foldercount)
+ logger.info('Scan time = %0.2fs' % (time() - starttime))
print()
return filelist
+
def sendable(file):
'''Filters files scanned returning a new list of files to send i.e. miniseed'''
if os.path.basename(file).startswith('.'):
@@ -118,11 +135,12 @@ def sendable(file):
return True
return False
-#Basic Miniseed file metadata extracted from filename
-import re
-#This includes .p files which we may accept in the future but won't h
+
+# Basic Miniseed file metadata extracted from filename
+# This includes .p files which we may accept in the future but won't h
MseedRE = re.compile(r'\A(.*)\.([A-Z0-9][A-Z0-9])\.(.*)\.([A-Z][A-Z]\w)\.([0-9]{4})\.([0-9]{3})(?:\.p)*')
+
def filename_qc_format(file):
'''Used to filter filename if correct for qc system'''
return MseedRE.match(os.path.basename(file))
@@ -130,15 +148,16 @@ def filename_qc_format(file):
def format_size(num):
'''Format bytes into human readble with suffix'''
- for suffix in [ 'bytes', 'KB', 'MB', 'GB' ]:
+ for suffix in ['bytes', 'KB', 'MB', 'GB']:
if num < 1024.0:
return '%3.3f %s' % (num, suffix)
num /= 1024.0
return '%3.3f %s' % (num, 'TB')
+
def ismseed(file):
try:
- ms = open(file,'rb')
+ ms = open(file, 'rb')
except:
return None
order = ByteOrder(ms)
@@ -150,24 +169,26 @@ def ismseed(file):
return False
#########################################################
-#needs import sys,struct
-#Taken from Bruces LibTrace
-def ByteOrder(infile, seekval=20) :
+# needs import sys,struct
+# Taken from Bruces LibTrace
+
+
+def ByteOrder(infile, seekval=20):
"""
read file as if it is mseed just pulling time info
from fixed header and determine if it makes sense unpacked
as big endian or little endian
"""
Order = "unknown"
- try :
- #seek to timeblock and read
+ try:
+ # seek to timeblock and read
infile.seek(seekval)
- timeblock=infile.read(10)
+ timeblock = infile.read(10)
- #assume big endian
- (Year, Day, Hour, Min, Sec, junk, Micro)=\
- struct.unpack('>HHBBBBH',timeblock)
- #test if big endian read makes sense
+ # assume big endian
+ (Year, Day, Hour, Min, Sec, junk, Micro) =\
+ struct.unpack('>HHBBBBH', timeblock)
+ # test if big endian read makes sense
if 1950 <= Year <= 2050 and \
1 <= Day <= 366 and \
0 <= Hour <= 23 and \
@@ -175,23 +196,24 @@ def ByteOrder(infile, seekval=20) :
0 <= Sec <= 59:
Order = "big"
else:
- #try little endian read
- (Year, Day, Hour, Min, Sec, junk, Micro)=\
- struct.unpack('<HHBBBBH',timeblock)
- #test if little endian read makes sense
+ # try little endian read
+ (Year, Day, Hour, Min, Sec, junk, Micro) =\
+ struct.unpack('<HHBBBBH', timeblock)
+ # test if little endian read makes sense
if 1950 <= Year <= 2050 and \
1 <= Day <= 366 and \
0 <= Hour <= 23 and \
0 <= Min <= 59 and \
0 <= Sec <= 59:
Order = "little"
- except Exception as e:
+ except Exception:
pass
return Order
#########################################################
-def get_sent_file_list(sentfile = SENTFILE):
+
+def get_sent_file_list(sentfile=SENTFILE):
sentlist = []
if os.path.isfile(sentfile):
logger.debug('Using sentfile %s' % sentfile)
@@ -205,7 +227,8 @@ def get_sent_file_list(sentfile = SENTFILE):
sentlist = pickle.load(f)
return sentlist
-def write_sent_file_list(sentlist , sentfile = SENTFILE ):
+
+def write_sent_file_list(sentlist, sentfile=SENTFILE):
logger.info('Saving list of files sent to passcal')
with open(sentfile, 'wb+') as f:
pickle.dump(sentlist, f, protocol=2)
@@ -218,8 +241,8 @@ def get_FTP():
trys += 1
try:
import socket
- logger.info('Connecting to FTP host %s from %s. Attempt %d of %d' % ( FTP_HOST, socket.gethostbyname(socket.gethostname()), trys, FTP_CONNECT_ATTEMPTS))
- FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
+ logger.info('Connecting to FTP host %s from %s. Attempt %d of %d' % (FTP_HOST, socket.gethostbyname(socket.gethostname()), trys, FTP_CONNECT_ATTEMPTS))
+ FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
FTP.set_debuglevel(FTP_DEBUG_LEVEL)
FTP.cwd(FTP_DIR)
FTP.set_pasv(True)
@@ -235,29 +258,36 @@ def get_FTP():
logger.error('Giving up.')
return None
+
def test_network():
passcal_http_reachable()
google_http_reachable()
passcal_ftp_reachable()
+
def passcal_http_reachable():
'''download and time passcal home page'''
url = 'http://www.passcal.nmt.edu/'
return url_reachable(url=url)
+
def google_http_reachable():
'''download and time passcal home page'''
url = 'http://www.google.com/'
return url_reachable(url=url)
+
def passcal_ftp_reachable():
'''Download a small file from passcals general ftp'''
url = 'ftp://ftp.passcal.nmt.edu/download/public/test.dat'
return url_reachable(url=url)
-def url_reachable(url = 'http://www.passcal.nmt.edu/'):
+
+def url_reachable(url='http://www.passcal.nmt.edu/'):
'''fetches a url returns True or False'''
- import urllib.request, urllib.error, urllib.parse
+ import urllib.request
+ import urllib.error
+ import urllib.parse
start = time()
try:
f = urllib.request.urlopen(url)
@@ -265,20 +295,20 @@ def url_reachable(url = 'http://www.passcal.nmt.edu/'):
logger.error("Failed to open connection to %s" % url)
logger.error(e)
return False
- logger.info('connection made to %s in %f sec' % ( url, time() - start) )
+ logger.info('connection made to %s in %f sec' % (url, time() - start))
data = f.read()
runtime = time() - start
size = len(data)
- rate = size/runtime
- logger.info('%d B/sec in %f sec' % (rate, runtime) )
+ rate = size / runtime
+ logger.info('%d B/sec in %f sec' % (rate, runtime))
return True
def test_FTP(FTP):
try:
- assert isinstance(FTP, ftplib.FTP)
+ assert isinstance(FTP, FTPCache)
FTP.voidcmd('NOOP')
- except ftplib.all_errors as e :
+ except ftplib.all_errors as e:
logger.error(e)
return False
except AssertionError as e:
@@ -294,11 +324,11 @@ def send2passcal(mslist, sentlist=None):
# Handle SIGINT while in this function: to gracefully close connection and save sentlist to disk file
def signal_handler(signum, frame):
print()
- logger.info('Caught interrupt while FTPing. Aborting transfer %s' % current_file )
- logger.info('Sent %d of %d' % (num_sent, num_to_send) )
- logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)) )
- logger.info('%s /sec' % ( format_size(size_sent / (time() - starttime )) ) )
- logger.info('Ran for %f sec' % (time()-starttime) )
+ logger.info('Caught interrupt while FTPing. Aborting transfer %s' % current_file)
+ logger.info('Sent %d of %d' % (num_sent, num_to_send))
+ logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)))
+ logger.info('%s /sec' % (format_size(size_sent / (time() - starttime))))
+ logger.info('Ran for %f sec' % (time() - starttime))
write_sent_file_list(sentlist)
try:
if FTP:
@@ -312,7 +342,7 @@ def send2passcal(mslist, sentlist=None):
'''Updates the terminal display.'''
signal.signal(signal.SIGINT, signal_handler)
update.bytes_sent += len(data)
- print('\r' + str(PB) + ' %s /sec ' % ( format_size(size_sent / (time() - starttime )) ), end=' ')
+ print('\r' + str(PB) + ' %s /sec ' % (format_size(size_sent / (time() - starttime))), end=' ')
'''
print '%s %0.2f%%. %0.10d / %0.10d.' % ( current_file.center(20),
(update.bytes_sent / update.file_size)*100,
@@ -320,8 +350,8 @@ def send2passcal(mslist, sentlist=None):
file_size,
) ,
'''
- ETA_sec = ((time() - starttime) / size_sent ) * (size_to_send - size_sent)
- print('ETA %s %s %s' % ( str(datetime.timedelta(seconds=(ETA_sec))), current_file.center(20), ' '*20 ), end=' ')
+ ETA_sec = ((time() - starttime) / size_sent) * (size_to_send - size_sent)
+ print('ETA %s %s %s' % (str(datetime.timedelta(seconds=(ETA_sec))), current_file.center(20), ' ' * 20), end=' ')
sys.stdout.flush()
if sentlist is None:
@@ -334,7 +364,7 @@ def send2passcal(mslist, sentlist=None):
size_sent = 1
current_file = ''
signal.signal(signal.SIGINT, signal_handler)
- logger.info('Sending %d, %s files to PASSCAL' % (num_to_send, format_size (size_to_send)))
+ logger.info('Sending %d, %s files to PASSCAL' % (num_to_send, format_size(size_to_send)))
FTP = get_FTP()
starttime = time()
PB = ProgressBar(num_to_send)
@@ -349,19 +379,19 @@ def send2passcal(mslist, sentlist=None):
fh = open(f, 'rb')
file_size = os.path.getsize(f)
update.file_size = float(file_size)
- FTP.storbinary('STOR %s' % current_file , fh, blocksize=FTP_BLOCKSIZE, callback=update)
+ FTP.storbinary('STOR %s' % current_file, fh, blocksize=FTP_BLOCKSIZE, callback=update)
except ftplib.error_perm as e:
# This is permission and the error when 550 for the .in file already exists so we should just continue with the next file
# todo create a list of failed files and resend those at the end instead of requiring a rerun
- logger.error( 'Failed to send file %s, permission error. Skipping...' % current_file)
+ logger.error('Failed to send file %s, permission error. Skipping...' % current_file)
logger.error(e)
break
- except (ftplib.all_errors, AttributeError) as e :
- #since we can restore with how we have proftp setup. There is nothing more we can do with this file
- #Until the server rms the .in.file
- logger.error('Failed to send file %s.' % ( current_file) ) #, trys, FTP_SEND_ATTEMPTS)
+ except (ftplib.all_errors, AttributeError) as e:
+ # since we can restore with how we have proftp setup. There is nothing more we can do with this file
+ # Until the server rms the .in.file
+ logger.error('Failed to send file %s.' % (current_file)) # , trys, FTP_SEND_ATTEMPTS)
logger.error(e)
- #if DEBUG: print "Waiting %d..." % FTP_TIMEOUT; sleep(FTP_TIMEOUT)
+ # if DEBUG: print "Waiting %d..." % FTP_TIMEOUT; sleep(FTP_TIMEOUT)
try:
if FTP:
FTP.abort()
@@ -380,15 +410,17 @@ def send2passcal(mslist, sentlist=None):
break
print()
- logger.info( 'Sent %d of %d' % (num_sent, num_to_send) )
- logger.info( 'Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)) )
- logger.info( '%s /sec' % ( format_size(size_sent / (time() - starttime )) ) )
- logger.info( 'Ran for %f sec' % (time()-starttime) )
+ logger.info('Sent %d of %d' % (num_sent, num_to_send))
+ logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)))
+ logger.info('%s /sec' % (format_size(size_sent / (time() - starttime))))
+ logger.info('Ran for %f sec' % (time() - starttime))
if FTP and test_FTP(FTP):
FTP.quit()
########################################################
# From progressbar 3rd party class will eventually dump not awesome
+
+
class ProgressBar(object):
def __init__(self, duration):
self.duration = duration
@@ -409,7 +441,7 @@ class ProgressBar(object):
def update_time(self, elapsed_secs):
self.__update_amount((elapsed_secs / float(self.duration)) * 100.0)
- #self.prog_bar += ' %d/%s files' % (elapsed_secs, self.duration)
+ # self.prog_bar += ' %d/%s files' % (elapsed_secs, self.duration)
def __update_amount(self, new_amount):
percent_done = int(round((new_amount / 100.0) * 100.0))
@@ -426,8 +458,10 @@ class ProgressBar(object):
# End progressbar
########################################################
+
+
def main():
- if len(sys.argv) < 2 or sys.argv[1] in [ '-h', '--help', '-?']:
+ if len(sys.argv) < 2 or sys.argv[1] in ['-h', '--help', '-?']:
print(HELP)
sys.exit()
# find all files below dir
@@ -436,19 +470,20 @@ def main():
logger.info('Removing improperly named files.')
msnamedlist = list(filter(filename_qc_format, filelist))
logger.info('Properly named files files: %d' % len(msnamedlist))
- logger.info('Other files: %d'% (len(filelist) - len(msnamedlist)))
+ logger.info('Other files: %d' % (len(filelist) - len(msnamedlist)))
print()
logger.info('Removing files that are not Miniseed.')
mslist = list(filter(sendable, msnamedlist))
logger.info('MiniSEED files: %d' % len(mslist))
- logger.info('Properly named but not miniseed files: %d'% (len(msnamedlist) - len(mslist)))
+ logger.info('Properly named but not miniseed files: %d' % (len(msnamedlist) - len(mslist)))
print()
logger.info('Removing files already sent to PASSCAL')
sentlist = get_sent_file_list()
unsentms = [f for f in mslist if f not in sentlist]
- logger.info('%d miniSEED files have already been sent, not resending.' % (len( mslist) - len(unsentms)))
+ logger.info('%d miniSEED files have already been sent, not resending.' % (len(mslist) - len(unsentms)))
send2passcal(unsentms, sentlist)
write_sent_file_list(sentlist)
+
if __name__ == '__main__':
main()
diff --git a/setup.cfg b/setup.cfg
index ac124ca411cea8f9dabc9e8c0af2752987f18625..a6afa20c057127edbc8738bc2805b6bb1c8c5fc7 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 2020.093
+current_version = 2020.107
commit = True
tag = True
diff --git a/setup.py b/setup.py
index 6afb3d2ddf200dcb6435ce2c75f77154b81a83e3..9f29d0f7b3af75df852fe0cc018c63e905075029 100644
--- a/setup.py
+++ b/setup.py
@@ -29,8 +29,11 @@ setup(
'data2passcal=data2passcal.data2passcal:main',
],
},
- install_requires=[],
- setup_requires = [],
+ install_requires=[
+ "mock;python_version<'3.3'",
+ "configparser;python_version<'3.2'"
+ ],
+ setup_requires=[],
extras_require={
'dev': [
'pip',
@@ -42,6 +45,7 @@ setup(
'coverage',
'Sphinx',
'twine',
+ 'pytest'
]
},
license="GNU General Public License v3",
@@ -52,6 +56,6 @@ setup(
packages=find_packages(include=['data2passcal']),
test_suite='tests',
url='https://git.passcal.nmt.edu/passoft/data2passcal',
- version='2020.093',
+ version='2020.107',
zip_safe=False,
)
diff --git a/tests/test_data/ST00.AB..BHZ.2007.160 b/tests/test_data/ST00.AB..BHZ.2007.160
new file mode 100644
index 0000000000000000000000000000000000000000..668ccee282506f7101f4f6de9e5175a3103d0043
Binary files /dev/null and b/tests/test_data/ST00.AB..BHZ.2007.160 differ
diff --git a/tests/test_data/ST00.AB..BHZ.2007.161 b/tests/test_data/ST00.AB..BHZ.2007.161
new file mode 100644
index 0000000000000000000000000000000000000000..f3fa1ece76f3655d59e123b5d92c235b4f5e9e4a
Binary files /dev/null and b/tests/test_data/ST00.AB..BHZ.2007.161 differ
diff --git a/tests/test_data/ST00.AB..BHZ.2007.162 b/tests/test_data/ST00.AB..BHZ.2007.162
new file mode 100644
index 0000000000000000000000000000000000000000..75c29514caed9cc89925cc00f5edca42c11a3950
Binary files /dev/null and b/tests/test_data/ST00.AB..BHZ.2007.162 differ
diff --git a/tests/test_data/ST00.AB..BHZ.2007.163 b/tests/test_data/ST00.AB..BHZ.2007.163
new file mode 100644
index 0000000000000000000000000000000000000000..907b07ca7097a791aa3c346e26df10f9c7788851
Binary files /dev/null and b/tests/test_data/ST00.AB..BHZ.2007.163 differ
diff --git a/tests/test_data/ST00.AB..BHZ.2007.164 b/tests/test_data/ST00.AB..BHZ.2007.164
new file mode 100644
index 0000000000000000000000000000000000000000..b925f35dace4e038f204d9b993113d9318f277e5
Binary files /dev/null and b/tests/test_data/ST00.AB..BHZ.2007.164 differ
diff --git a/tests/test_data2passcal.py b/tests/test_data2passcal.py
index a82fa89fd59939d3dfaf33a9cf4ba03f1c71c670..02d5c3319849c78c498560163d2000e8f4fd3a57 100644
--- a/tests/test_data2passcal.py
+++ b/tests/test_data2passcal.py
@@ -3,25 +3,102 @@
"""Tests for `data2passcal` package."""
+from __future__ import division, print_function
+
+import configparser
+import ftplib
+import os
import unittest
-import sys
+
+from data2passcal.data2passcal import get_FTP, ismseed, scan_dir, send2passcal
+
+VERSION = '2020.107'
+
try:
- import data2passcal
+ from unittest.mock import patch
except ImportError:
- pass
+ from mock import patch
+
+config = configparser.ConfigParser()
+config.read('data2passcal/config.ini')
+FTP_HOST = config.get('FTP_INFO', 'FTP_HOST')
+FTP_USER = config.get('FTP_INFO', 'FTP_USER')
+FTP_PASSWORD = config.get('FTP_INFO', 'FTP_PASSWORD')
+FTP_DIR = config.get('FTP_INFO', 'FTP_DIR')
+FTP_TIMEOUT = config.getint('FTP_INFO', 'FTP_TIMEOUT')
+FTP_RECONNECT_WAIT = config.getint('FTP_INFO', 'FTP_RECONNECT_WAIT')
+FTP_CONNECT_ATTEMPTS = 60 * 60 * 24 * 7 / FTP_RECONNECT_WAIT
+FTP_SEND_ATTEMPTS = config.getint('FTP_INFO', 'FTP_SEND_ATTEMPTS')
+MOCK_TEST = config.getboolean('TEST', 'MOCK_TEST')
+
class TestData2passcal(unittest.TestCase):
"""Tests for `data2passcal` package."""
def setUp(self):
- """Set up test fixtures, if any."""
+ """Set up test fixtures, if any"""
+ dir_testdata = os.path.dirname(os.path.realpath(__file__)) + '/test_data'
+ filelist = [x for x in scan_dir(dir_testdata) if not os.path.basename(x).startswith('.') and not os.path.basename(x).endswith('.log')]
+ self.dir_testdata = dir_testdata
+ self.filelist = filelist
+
+ def test_scan_dir(self):
+ """
+ Test basic functionality of scan_dir function
+ Only 5 files available under ./test_data directory
+ Not taking into account .DS_STORE file
+ """
+ self.assertEqual(len(self.filelist), 5, 'Incorrect number of files')
+
+ def test_ismseed(self):
+ """Test basic functionality of ismseed function"""
+ for f in self.filelist:
+ self.assertTrue(ismseed(f), '{} is not a miniseed file'.format(os.path.basename(f)))
+
+ @patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
+ def test_get_FTP_mock(self, mock_ftp_constructor):
+ """Mock test creating ftp connection to PASSCAL"""
+ mock_ftp = mock_ftp_constructor.return_value
+ get_FTP()
+ mock_ftp_constructor.assert_called_with(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
+ mock_ftp.cwd.assert_called_with(FTP_DIR)
+ self.assertLess(mock_ftp_constructor.call_count, FTP_CONNECT_ATTEMPTS, 'Number of ftp connection attempts exceeeds {}'.format(FTP_CONNECT_ATTEMPTS))
+ mock_ftp.quit()
+
+ @patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
+ def test_send_data_mock(self, mock_ftp_constructor):
+ """Mock test sending MSEED files (test data) to PASSCAL's QC system"""
+ mock_ftp = mock_ftp_constructor.return_value
+ send2passcal(self.filelist)
+ self.assertTrue(mock_ftp.storbinary.called, 'No data sent')
+ self.assertEqual(mock_ftp.storbinary.call_count, len(self.filelist), 'Failed to send all files - Sent {0} of {1}'.format(mock_ftp.storbinary.call_count, len(self.filelist)))
+ files_sent = []
+ for x in mock_ftp.storbinary.call_args_list:
+ args, kwargs = x
+ files_sent.append(args[0].split(' ')[1])
+ for f in self.filelist:
+ f = os.path.basename(f)
+ self.assertLess(files_sent.count(f), FTP_SEND_ATTEMPTS, 'Attempted to send file {0} more than {1} times'.format(f, FTP_SEND_ATTEMPTS))
+
+ @unittest.skipIf(MOCK_TEST == True, "skipping real send2passcal test")
+ def test_send_data(self):
+ """Test sending MSEED files (test data) to PASSCAL's QC system - Optional"""
+ ftp = get_FTP()
+ send2passcal(self.filelist)
+ wdir = ftp.pwd()
+ try:
+ files_sent = [os.path.basename(x) for x in ftp.nlst(wdir)]
+ except ftplib.error_perm() as resp:
+ if str(resp) == "550 No files found":
+ print("No files found in this directory")
+ else:
+ raise
+ for f in self.filelist:
+ f_ = os.path.basename(f)
+ self.assertIn(f_, files_sent, 'File {} was not sent to PASSCAL'.format(f_))
+ ftp.quit()
- def tearDown(self):
- """Tear down test fixtures, if any."""
- def test_import(self):
- if 'data2passcal' in sys.modules:
- self.assertTrue(True, "data2passcal loaded")
- else:
- self.fail()
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tox.ini b/tox.ini
index 60d2b1bb52fdc8539b7f39176385af7599226b5a..f6954a3b222f97b8819393542d29ed2b6ff99bc7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist = py27, py36 flake8
+envlist = py27, py36, flake8
[travis]
python =
@@ -9,11 +9,10 @@ python =
[testenv:flake8]
basepython = python
deps = flake8
-commands = flake8 data2passcal
+commands = flake8 --ignore=E722,E712,E501 data2passcal
+ flake8 --ignore=E722,E712,E501 tests
[testenv]
-setenv =
- PYTHONPATH = {toxinidir}
-commands=python setup.py test
-
-
+deps = pytest
+setenv = PYTHONPATH = {toxinidir}
+commands = pytest