From 2ba86aece7e65e008cb32749fa16cff8c0360cca Mon Sep 17 00:00:00 2001
From: Maeva Pourpoint <maeva@passcal.nmt.edu>
Date: Thu, 16 Apr 2020 15:01:33 -0600
Subject: [PATCH] Updated and tested data2passcal to work with Python 2 and 3
---
AUTHORS.rst | 2 +-
CONTRIBUTING.rst | 3 +-
HISTORY.rst | 8 +
README.rst | 20 +--
data2passcal/__init__.py | 2 +-
data2passcal/config.ini | 22 +++
data2passcal/config.py | 31 ++++
data2passcal/data2passcal.py | 243 +++++++++++++++-----------
setup.cfg | 2 +-
setup.py | 10 +-
tests/test_data/ST00.AB..BHZ.2007.160 | Bin 0 -> 1024 bytes
tests/test_data/ST00.AB..BHZ.2007.161 | Bin 0 -> 1024 bytes
tests/test_data/ST00.AB..BHZ.2007.162 | Bin 0 -> 1024 bytes
tests/test_data/ST00.AB..BHZ.2007.163 | Bin 0 -> 1024 bytes
tests/test_data/ST00.AB..BHZ.2007.164 | Bin 0 -> 1024 bytes
tests/test_data2passcal.py | 99 +++++++++--
tox.ini | 13 +-
17 files changed, 308 insertions(+), 147 deletions(-)
create mode 100644 data2passcal/config.ini
create mode 100644 data2passcal/config.py
create mode 100644 tests/test_data/ST00.AB..BHZ.2007.160
create mode 100644 tests/test_data/ST00.AB..BHZ.2007.161
create mode 100644 tests/test_data/ST00.AB..BHZ.2007.162
create mode 100644 tests/test_data/ST00.AB..BHZ.2007.163
create mode 100644 tests/test_data/ST00.AB..BHZ.2007.164
diff --git a/AUTHORS.rst b/AUTHORS.rst
index 8029b24..207d6dd 100644
--- a/AUTHORS.rst
+++ b/AUTHORS.rst
@@ -10,4 +10,4 @@ Development Lead
Contributors
------------
-None yet. Why not be the first?
+* Maeva Pourpoint <software-support@passcal.nmt.edu>
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index e8536cc..b262255 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -76,7 +76,7 @@ Ready to contribute? Here's how to set up `data2passcal` for local development.
5. When you're done making changes, check that your changes pass the
tests::
- $ python setup.py test
+ $ python -m pytest
6. Commit your changes and push your branch to GitHub::
$ git add .
@@ -111,4 +111,3 @@ Then run::
$ git push
$ git push --tags
-
diff --git a/HISTORY.rst b/HISTORY.rst
index db916b9..d35c92d 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -17,3 +17,11 @@ History
* Resources used: Python-Future project and Futurize tool.
* Changes tested using unittest module on individual test methods(tests.test_data2passcal)
* Changes tested against real seismic dataset (AB.2002 - 2012)
+
+2020.107 (2020-04-16)
+------------------
+* Added some unit tests to ensure basic functionality of data2passcal
+* Created configuration file (.ini file) to store and access ftp and test related variables
+* Updated list of platform specific dependencies to be installed when installing data2passcal (see setup.py)
+* Installed and tested data2passcal against Python2.7 and Python3.6 using tox
+* Formatted Python code to conform to the PEP8 style guide (exception: E722, E712, E501)
diff --git a/README.rst b/README.rst
index efc03cf..a1160b9 100644
--- a/README.rst
+++ b/README.rst
@@ -3,23 +3,9 @@ data2passcal
============
-Prepare SEED data for shipment to PASSCAL.
-Only accept day-long MSEED files.
+Description: Send MSEED files ready for archival at the DMC to PASSCAL's QC system
+ Only accept day-long MSEED files.
+Usage: data2passcal dir
* Free software: GNU General Public License v3 (GPLv3)
-
-
-
-Features
---------
-
-* TODO
-
-Credits
--------
-
-This package was created with Cookiecutter_ and the `passoft/cookiecutter`_ project template.
-
-.. _Cookiecutter: https://github.com/audreyr/cookiecutter
-.. _`passoft/cookiecutter`: https://git.passcal.nmt.edu/passoft/cookiecutter
diff --git a/data2passcal/__init__.py b/data2passcal/__init__.py
index afe21e9..8f21cef 100644
--- a/data2passcal/__init__.py
+++ b/data2passcal/__init__.py
@@ -4,4 +4,4 @@
__author__ = """IRIS PASSCAL"""
__email__ = 'software-support@passcal.nmt.edu'
-__version__ = '2020.093'
+__version__ = '2020.107'
diff --git a/data2passcal/config.ini b/data2passcal/config.ini
new file mode 100644
index 0000000..6422e38
--- /dev/null
+++ b/data2passcal/config.ini
@@ -0,0 +1,22 @@
+[FTP_INFO]
+ftp_ip = 129.138.26.29
+ftp_user = ftp
+ftp_password = data2passcal
+ftp_send_attempts = 3
+ftp_blocksize = 8192
+ftp_timeout = 120
+ftp_dir = AUTO/MSEED
+ftp_reconnect_wait = 60
+ftp_debug_level = 0
+ftp_host = qc.passcal.nmt.edu
+
+[TEST]
+ftp_dir = AUTO/TEST
+ftp_reconnect_wait = 5
+mock_test = True
+testmode = False
+ftp_timeout = 5
+
+[FILES]
+logfile = data2passcal.log
+
diff --git a/data2passcal/config.py b/data2passcal/config.py
new file mode 100644
index 0000000..6f3ac69
--- /dev/null
+++ b/data2passcal/config.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+'''
+data2passcal
+
+Create configuration file
+'''
+
+import configparser
+
+VERSION = '2020.107'
+
+config = configparser.ConfigParser()
+config['FTP_INFO'] = {'FTP_IP': '129.138.26.29',
+ 'FTP_HOST': 'qc.passcal.nmt.edu',
+ 'FTP_USER': 'ftp',
+ 'FTP_PASSWORD': 'data2passcal',
+ 'FTP_DIR': 'AUTO/MSEED',
+ 'FTP_TIMEOUT': '120',
+ 'FTP_RECONNECT_WAIT': '60',
+ 'FTP_BLOCKSIZE': '8192',
+ 'FTP_SEND_ATTEMPTS': '3',
+ 'FTP_DEBUG_LEVEL': '0'}
+config['TEST'] = {'TESTMODE': 'False',
+ 'FTP_DIR': 'AUTO/TEST',
+ 'FTP_TIMEOUT': '5',
+ 'FTP_RECONNECT_WAIT': '5',
+ 'MOCK_TEST': 'True'}
+config['FILES'] = {'LOGFILE': 'data2passcal.log'}
+
+with open('config.ini', 'w') as configfile:
+ config.write(configfile)
diff --git a/data2passcal/data2passcal.py b/data2passcal/data2passcal.py
index a4afaaf..7cb8551 100644
--- a/data2passcal/data2passcal.py
+++ b/data2passcal/data2passcal.py
@@ -5,62 +5,77 @@ data2passcal
Ship miniseed data to passcal via ftp for qc before archival at DMC DMS
Lloyd Carothers IRIS/PASSCAL
'''
-from __future__ import print_function, division
-from builtins import *
-VERSION = '2020.093'
+from __future__ import division, print_function
-import ftplib
-import sys, os, signal, struct, pickle, logging
-from time import time, sleep
+import configparser
import datetime
+import ftplib
+import logging
+import os
+import pickle
+import re
+import signal
+import struct
+import sys
+from time import sleep, time
-#TESTMODE = True
-TESTMODE = False
-# Verbosity of console log from logging module from DEBUG to CRITICAL
-DEBUG = logging.DEBUG
-# After transmission or on interrupt try to ftp logfile to PIC
-#SEND_LOGFILE = True
-
-#FTP related
-FTP_IP = '129.138.26.29'
-FTP_HOST = 'qc.passcal.nmt.edu'
-FTP_USER = 'ftp'
-FTP_PASSWORD = 'data2passcal'
-FTP_DIR = 'AUTO/MSEED'
-FTP_TIMEOUT = 120
-FTP_RECONNECT_WAIT = 60
+VERSION = '2020.107'
+
+
+# Cache the ftplib.FTP class so it will be available to test_FTP(FTP) when calling isinstance assert
+FTPCache = ftplib.FTP
+
+# Parse configuration file
+config = configparser.ConfigParser()
+print(config.read('data2passcal/config.ini'))
+
+
+# TEST related
+TESTMODE = config.getboolean('TEST', 'TESTMODE')
+
+# FTP related
+FTP_IP = config.get('FTP_INFO', 'FTP_IP')
+FTP_HOST = config.get('FTP_INFO', 'FTP_HOST')
+FTP_USER = config.get('FTP_INFO', 'FTP_USER')
+FTP_PASSWORD = config.get('FTP_INFO', 'FTP_PASSWORD')
+FTP_DIR = config.get('FTP_INFO', 'FTP_DIR')
+FTP_TIMEOUT = config.getint('FTP_INFO', 'FTP_TIMEOUT')
+FTP_RECONNECT_WAIT = config.getint('FTP_INFO', 'FTP_RECONNECT_WAIT')
if TESTMODE:
- FTP_DIR = 'AUTO/TEST'
- FTP_TIMEOUT = 5
- FTP_RECONNECT_WAIT = 5
-FTP_BLOCKSIZE = 8192
-#number of time to try to open the ftp connection
-#FTP_CONNECT_ATTEMPTS = 3
-# testing will retry for a week. Why not?
-FTP_CONNECT_ATTEMPTS = 60*60*24*7 / FTP_RECONNECT_WAIT
-#number of times a single file with try to be sent
-FTP_SEND_ATTEMPTS = 3
+ FTP_DIR = config.get('TEST', 'FTP_DIR')
+ FTP_TIMEOUT = config.getint('TEST', 'FTP_TIMEOUT')
+ FTP_RECONNECT_WAIT = config.getint('TEST', 'FTP_RECONNECT_WAIT')
+FTP_BLOCKSIZE = config.getint('FTP_INFO', 'FTP_BLOCKSIZE')
+# number of time to try to open the ftp connection - testing will retry for a week. Why not?
+FTP_CONNECT_ATTEMPTS = 60 * 60 * 24 * 7 / FTP_RECONNECT_WAIT
+# number of times a single file with try to be sent
+FTP_SEND_ATTEMPTS = config.getint('FTP_INFO', 'FTP_SEND_ATTEMPTS')
# debug level of ftplib, 0-3
-FTP_DEBUG_LEVEL = 0
-#store the files sent in ~/data2passcal.sent
-SENTFILE = os.path.join( os.path.expanduser('~'), '.data2passcal.sent')
+FTP_DEBUG_LEVEL = config.getint('FTP_INFO', 'FTP_DEBUG_LEVEL')
+
+# Files related
+LOGFILE = config.get('FILES', 'LOGFILE')
+# store the files sent in ~/data2passcal.sent
+SENTFILE = os.path.join(os.path.expanduser('~'), '.data2passcal.sent')
# If this file exists open it, incorporate, and save to new name, and delete old
-SENTFILE_OLD = os.path.join( os.path.expanduser('~'), '.send2passcal.sent')
-LOGFILE = 'data2passcal.log'
+SENTFILE_OLD = os.path.join(os.path.expanduser('~'), '.send2passcal.sent')
HELP = '''
data2passcal
VERSION: %s
Usage: data2passcal dir
- data2passcal is a utility that sends MSEED files ready for archival at the DMC to PASSCAL's QC system by:
- Scans all files below directory dir.
- Filters out non-miniseed files, by inspecting the first blockette of each file.
- Sends the files to the automated system for ingestion into the QC system via ftp.
+ data2passcal is a utility that sends day-long MSEED files ready for archival at the DMC to PASSCAL's QC system by:
+ Scanning all files below directory dir.
+ Filtering out non-miniseed files, by inspecting the first blockette of each file.
+ Sending the files to the automated system for ingestion into the QC system via ftp.
You can send a SIGTERM (ctl-c) to data2passcal and it will shutdown cleanly.
A list of sent files is kept in ~/.data2passcal.sent. Subsequent runs of send2passcal will not send files already sent.
A log is stored in data2passcal.log in the current directory.
''' % VERSION
+# Verbosity of console log from logging module from DEBUG to CRITICAL
+DEBUG = logging.DEBUG
+
# Configure logging file and stdout
# todo if sending a file do we want to send all previous logs even if already sent
logger = logging.getLogger('__name__')
@@ -73,16 +88,17 @@ logger.addHandler(logfh)
# Log to stdout
logconsole = logging.StreamHandler()
logconsole.setLevel(logging.INFO)
-logconsole.setFormatter(logging.Formatter('%(message)s' ))
+logconsole.setFormatter(logging.Formatter('%(message)s'))
logger.addHandler(logconsole)
logger.debug('Program starting.')
-logger.debug('%s - v%s - Python:%s' % (sys.argv, VERSION, sys.version) )
+logger.debug('%s - v%s - Python:%s' % (sys.argv, VERSION, sys.version))
logger.debug('CWD: %s' % os.getcwd())
-logger.info('Version: '+ VERSION)
+logger.info('Version: ' + VERSION)
logger.info('TIMEOUT: %d' % FTP_TIMEOUT)
logger.info('FTP RECONNECT WAIT: %d' % FTP_RECONNECT_WAIT)
# -- End logging
+
def scan_dir(dir):
'''Returns a list of absolute file names found below root dir'''
rootdir = dir
@@ -99,17 +115,18 @@ def scan_dir(dir):
try:
filesize += os.path.getsize(f)
except OSError:
- logger.debug('Can not stat %s'% f)
+ logger.debug('Can not stat %s' % f)
else:
filelist.append(f)
- logger.info('Total Size = %s'% format_size(filesize) )
- logger.info('Total Files = %s'% len(filelist) )
- logger.info('Total Dirs = %s'% foldercount)
- logger.info('Scan time = %0.2fs'% (time() - starttime))
+ logger.info('Total Size = %s' % format_size(filesize))
+ logger.info('Total Files = %s' % len(filelist))
+ logger.info('Total Dirs = %s' % foldercount)
+ logger.info('Scan time = %0.2fs' % (time() - starttime))
print()
return filelist
+
def sendable(file):
'''Filters files scanned returning a new list of files to send i.e. miniseed'''
if os.path.basename(file).startswith('.'):
@@ -118,11 +135,12 @@ def sendable(file):
return True
return False
-#Basic Miniseed file metadata extracted from filename
-import re
-#This includes .p files which we may accept in the future but won't h
+
+# Basic Miniseed file metadata extracted from filename
+# This includes .p files which we may accept in the future but won't h
MseedRE = re.compile(r'\A(.*)\.([A-Z0-9][A-Z0-9])\.(.*)\.([A-Z][A-Z]\w)\.([0-9]{4})\.([0-9]{3})(?:\.p)*')
+
def filename_qc_format(file):
'''Used to filter filename if correct for qc system'''
return MseedRE.match(os.path.basename(file))
@@ -130,15 +148,16 @@ def filename_qc_format(file):
def format_size(num):
'''Format bytes into human readble with suffix'''
- for suffix in [ 'bytes', 'KB', 'MB', 'GB' ]:
+ for suffix in ['bytes', 'KB', 'MB', 'GB']:
if num < 1024.0:
return '%3.3f %s' % (num, suffix)
num /= 1024.0
return '%3.3f %s' % (num, 'TB')
+
def ismseed(file):
try:
- ms = open(file,'rb')
+ ms = open(file, 'rb')
except:
return None
order = ByteOrder(ms)
@@ -150,24 +169,26 @@ def ismseed(file):
return False
#########################################################
-#needs import sys,struct
-#Taken from Bruces LibTrace
-def ByteOrder(infile, seekval=20) :
+# needs import sys,struct
+# Taken from Bruces LibTrace
+
+
+def ByteOrder(infile, seekval=20):
"""
read file as if it is mseed just pulling time info
from fixed header and determine if it makes sense unpacked
as big endian or little endian
"""
Order = "unknown"
- try :
- #seek to timeblock and read
+ try:
+ # seek to timeblock and read
infile.seek(seekval)
- timeblock=infile.read(10)
+ timeblock = infile.read(10)
- #assume big endian
- (Year, Day, Hour, Min, Sec, junk, Micro)=\
- struct.unpack('>HHBBBBH',timeblock)
- #test if big endian read makes sense
+ # assume big endian
+ (Year, Day, Hour, Min, Sec, junk, Micro) =\
+ struct.unpack('>HHBBBBH', timeblock)
+ # test if big endian read makes sense
if 1950 <= Year <= 2050 and \
1 <= Day <= 366 and \
0 <= Hour <= 23 and \
@@ -175,23 +196,24 @@ def ByteOrder(infile, seekval=20) :
0 <= Sec <= 59:
Order = "big"
else:
- #try little endian read
- (Year, Day, Hour, Min, Sec, junk, Micro)=\
- struct.unpack('<HHBBBBH',timeblock)
- #test if little endian read makes sense
+ # try little endian read
+ (Year, Day, Hour, Min, Sec, junk, Micro) =\
+ struct.unpack('<HHBBBBH', timeblock)
+ # test if little endian read makes sense
if 1950 <= Year <= 2050 and \
1 <= Day <= 366 and \
0 <= Hour <= 23 and \
0 <= Min <= 59 and \
0 <= Sec <= 59:
Order = "little"
- except Exception as e:
+ except Exception:
pass
return Order
#########################################################
-def get_sent_file_list(sentfile = SENTFILE):
+
+def get_sent_file_list(sentfile=SENTFILE):
sentlist = []
if os.path.isfile(sentfile):
logger.debug('Using sentfile %s' % sentfile)
@@ -205,7 +227,8 @@ def get_sent_file_list(sentfile = SENTFILE):
sentlist = pickle.load(f)
return sentlist
-def write_sent_file_list(sentlist , sentfile = SENTFILE ):
+
+def write_sent_file_list(sentlist, sentfile=SENTFILE):
logger.info('Saving list of files sent to passcal')
with open(sentfile, 'wb+') as f:
pickle.dump(sentlist, f, protocol=2)
@@ -218,8 +241,8 @@ def get_FTP():
trys += 1
try:
import socket
- logger.info('Connecting to FTP host %s from %s. Attempt %d of %d' % ( FTP_HOST, socket.gethostbyname(socket.gethostname()), trys, FTP_CONNECT_ATTEMPTS))
- FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
+ logger.info('Connecting to FTP host %s from %s. Attempt %d of %d' % (FTP_HOST, socket.gethostbyname(socket.gethostname()), trys, FTP_CONNECT_ATTEMPTS))
+ FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
FTP.set_debuglevel(FTP_DEBUG_LEVEL)
FTP.cwd(FTP_DIR)
FTP.set_pasv(True)
@@ -235,29 +258,36 @@ def get_FTP():
logger.error('Giving up.')
return None
+
def test_network():
passcal_http_reachable()
google_http_reachable()
passcal_ftp_reachable()
+
def passcal_http_reachable():
'''download and time passcal home page'''
url = 'http://www.passcal.nmt.edu/'
return url_reachable(url=url)
+
def google_http_reachable():
'''download and time passcal home page'''
url = 'http://www.google.com/'
return url_reachable(url=url)
+
def passcal_ftp_reachable():
'''Download a small file from passcals general ftp'''
url = 'ftp://ftp.passcal.nmt.edu/download/public/test.dat'
return url_reachable(url=url)
-def url_reachable(url = 'http://www.passcal.nmt.edu/'):
+
+def url_reachable(url='http://www.passcal.nmt.edu/'):
'''fetches a url returns True or False'''
- import urllib.request, urllib.error, urllib.parse
+ import urllib.request
+ import urllib.error
+ import urllib.parse
start = time()
try:
f = urllib.request.urlopen(url)
@@ -265,20 +295,20 @@ def url_reachable(url = 'http://www.passcal.nmt.edu/'):
logger.error("Failed to open connection to %s" % url)
logger.error(e)
return False
- logger.info('connection made to %s in %f sec' % ( url, time() - start) )
+ logger.info('connection made to %s in %f sec' % (url, time() - start))
data = f.read()
runtime = time() - start
size = len(data)
- rate = size/runtime
- logger.info('%d B/sec in %f sec' % (rate, runtime) )
+ rate = size / runtime
+ logger.info('%d B/sec in %f sec' % (rate, runtime))
return True
def test_FTP(FTP):
try:
- assert isinstance(FTP, ftplib.FTP)
+ assert isinstance(FTP, FTPCache)
FTP.voidcmd('NOOP')
- except ftplib.all_errors as e :
+ except ftplib.all_errors as e:
logger.error(e)
return False
except AssertionError as e:
@@ -294,11 +324,11 @@ def send2passcal(mslist, sentlist=None):
# Handle SIGINT while in this function: to gracefully close connection and save sentlist to disk file
def signal_handler(signum, frame):
print()
- logger.info('Caught interrupt while FTPing. Aborting transfer %s' % current_file )
- logger.info('Sent %d of %d' % (num_sent, num_to_send) )
- logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)) )
- logger.info('%s /sec' % ( format_size(size_sent / (time() - starttime )) ) )
- logger.info('Ran for %f sec' % (time()-starttime) )
+ logger.info('Caught interrupt while FTPing. Aborting transfer %s' % current_file)
+ logger.info('Sent %d of %d' % (num_sent, num_to_send))
+ logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)))
+ logger.info('%s /sec' % (format_size(size_sent / (time() - starttime))))
+ logger.info('Ran for %f sec' % (time() - starttime))
write_sent_file_list(sentlist)
try:
if FTP:
@@ -312,7 +342,7 @@ def send2passcal(mslist, sentlist=None):
'''Updates the terminal display.'''
signal.signal(signal.SIGINT, signal_handler)
update.bytes_sent += len(data)
- print('\r' + str(PB) + ' %s /sec ' % ( format_size(size_sent / (time() - starttime )) ), end=' ')
+ print('\r' + str(PB) + ' %s /sec ' % (format_size(size_sent / (time() - starttime))), end=' ')
'''
print '%s %0.2f%%. %0.10d / %0.10d.' % ( current_file.center(20),
(update.bytes_sent / update.file_size)*100,
@@ -320,8 +350,8 @@ def send2passcal(mslist, sentlist=None):
file_size,
) ,
'''
- ETA_sec = ((time() - starttime) / size_sent ) * (size_to_send - size_sent)
- print('ETA %s %s %s' % ( str(datetime.timedelta(seconds=(ETA_sec))), current_file.center(20), ' '*20 ), end=' ')
+ ETA_sec = ((time() - starttime) / size_sent) * (size_to_send - size_sent)
+ print('ETA %s %s %s' % (str(datetime.timedelta(seconds=(ETA_sec))), current_file.center(20), ' ' * 20), end=' ')
sys.stdout.flush()
if sentlist is None:
@@ -334,7 +364,7 @@ def send2passcal(mslist, sentlist=None):
size_sent = 1
current_file = ''
signal.signal(signal.SIGINT, signal_handler)
- logger.info('Sending %d, %s files to PASSCAL' % (num_to_send, format_size (size_to_send)))
+ logger.info('Sending %d, %s files to PASSCAL' % (num_to_send, format_size(size_to_send)))
FTP = get_FTP()
starttime = time()
PB = ProgressBar(num_to_send)
@@ -349,19 +379,19 @@ def send2passcal(mslist, sentlist=None):
fh = open(f, 'rb')
file_size = os.path.getsize(f)
update.file_size = float(file_size)
- FTP.storbinary('STOR %s' % current_file , fh, blocksize=FTP_BLOCKSIZE, callback=update)
+ FTP.storbinary('STOR %s' % current_file, fh, blocksize=FTP_BLOCKSIZE, callback=update)
except ftplib.error_perm as e:
# This is permission and the error when 550 for the .in file already exists so we should just continue with the next file
# todo create a list of failed files and resend those at the end instead of requiring a rerun
- logger.error( 'Failed to send file %s, permission error. Skipping...' % current_file)
+ logger.error('Failed to send file %s, permission error. Skipping...' % current_file)
logger.error(e)
break
- except (ftplib.all_errors, AttributeError) as e :
- #since we can restore with how we have proftp setup. There is nothing more we can do with this file
- #Until the server rms the .in.file
- logger.error('Failed to send file %s.' % ( current_file) ) #, trys, FTP_SEND_ATTEMPTS)
+ except (ftplib.all_errors, AttributeError) as e:
+ # since we can restore with how we have proftp setup. There is nothing more we can do with this file
+ # Until the server rms the .in.file
+ logger.error('Failed to send file %s.' % (current_file)) # , trys, FTP_SEND_ATTEMPTS)
logger.error(e)
- #if DEBUG: print "Waiting %d..." % FTP_TIMEOUT; sleep(FTP_TIMEOUT)
+ # if DEBUG: print "Waiting %d..." % FTP_TIMEOUT; sleep(FTP_TIMEOUT)
try:
if FTP:
FTP.abort()
@@ -380,15 +410,17 @@ def send2passcal(mslist, sentlist=None):
break
print()
- logger.info( 'Sent %d of %d' % (num_sent, num_to_send) )
- logger.info( 'Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)) )
- logger.info( '%s /sec' % ( format_size(size_sent / (time() - starttime )) ) )
- logger.info( 'Ran for %f sec' % (time()-starttime) )
+ logger.info('Sent %d of %d' % (num_sent, num_to_send))
+ logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)))
+ logger.info('%s /sec' % (format_size(size_sent / (time() - starttime))))
+ logger.info('Ran for %f sec' % (time() - starttime))
if FTP and test_FTP(FTP):
FTP.quit()
########################################################
# From progressbar 3rd party class will eventually dump not awesome
+
+
class ProgressBar(object):
def __init__(self, duration):
self.duration = duration
@@ -409,7 +441,7 @@ class ProgressBar(object):
def update_time(self, elapsed_secs):
self.__update_amount((elapsed_secs / float(self.duration)) * 100.0)
- #self.prog_bar += ' %d/%s files' % (elapsed_secs, self.duration)
+ # self.prog_bar += ' %d/%s files' % (elapsed_secs, self.duration)
def __update_amount(self, new_amount):
percent_done = int(round((new_amount / 100.0) * 100.0))
@@ -426,8 +458,10 @@ class ProgressBar(object):
# End progressbar
########################################################
+
+
def main():
- if len(sys.argv) < 2 or sys.argv[1] in [ '-h', '--help', '-?']:
+ if len(sys.argv) < 2 or sys.argv[1] in ['-h', '--help', '-?']:
print(HELP)
sys.exit()
# find all files below dir
@@ -436,19 +470,20 @@ def main():
logger.info('Removing improperly named files.')
msnamedlist = list(filter(filename_qc_format, filelist))
logger.info('Properly named files files: %d' % len(msnamedlist))
- logger.info('Other files: %d'% (len(filelist) - len(msnamedlist)))
+ logger.info('Other files: %d' % (len(filelist) - len(msnamedlist)))
print()
logger.info('Removing files that are not Miniseed.')
mslist = list(filter(sendable, msnamedlist))
logger.info('MiniSEED files: %d' % len(mslist))
- logger.info('Properly named but not miniseed files: %d'% (len(msnamedlist) - len(mslist)))
+ logger.info('Properly named but not miniseed files: %d' % (len(msnamedlist) - len(mslist)))
print()
logger.info('Removing files already sent to PASSCAL')
sentlist = get_sent_file_list()
unsentms = [f for f in mslist if f not in sentlist]
- logger.info('%d miniSEED files have already been sent, not resending.' % (len( mslist) - len(unsentms)))
+ logger.info('%d miniSEED files have already been sent, not resending.' % (len(mslist) - len(unsentms)))
send2passcal(unsentms, sentlist)
write_sent_file_list(sentlist)
+
if __name__ == '__main__':
main()
diff --git a/setup.cfg b/setup.cfg
index ac124ca..a6afa20 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 2020.093
+current_version = 2020.107
commit = True
tag = True
diff --git a/setup.py b/setup.py
index 6afb3d2..9f29d0f 100644
--- a/setup.py
+++ b/setup.py
@@ -29,8 +29,11 @@ setup(
'data2passcal=data2passcal.data2passcal:main',
],
},
- install_requires=[],
- setup_requires = [],
+ install_requires=[
+ "mock;python_version<'3.3'",
+ "configparser;python_version<'3.2'"
+ ],
+ setup_requires=[],
extras_require={
'dev': [
'pip',
@@ -42,6 +45,7 @@ setup(
'coverage',
'Sphinx',
'twine',
+ 'pytest'
]
},
license="GNU General Public License v3",
@@ -52,6 +56,6 @@ setup(
packages=find_packages(include=['data2passcal']),
test_suite='tests',
url='https://git.passcal.nmt.edu/passoft/data2passcal',
- version='2020.093',
+ version='2020.107',
zip_safe=False,
)
diff --git a/tests/test_data/ST00.AB..BHZ.2007.160 b/tests/test_data/ST00.AB..BHZ.2007.160
new file mode 100644
index 0000000000000000000000000000000000000000..668ccee282506f7101f4f6de9e5175a3103d0043
GIT binary patch
literal 1024
zcmW;KOK;p%6bJC@YtQ&OevRkp$t0OH2_=LksFgre6e=W;*a1-`wruziAhBf2o&^g8
z9{`Y8bkS7=qKig^ypuNb$ari|?6Ez5+qZF!bk1LA@!K4ot*x7S=kuj6zTDnkT3UMi
z=~o{;CJ({S05kyiJ^pR=2lNvF0J~`bJ_K911^|OGw`(!5ot>SV_yr;K4;Zy~&DE#U
zdw*58o>n&xtMpNKz9$Ug)xOX?XNdtv+YFqAuR^4yiHvJ(IjP-voGk}|kG*`=f{`$@
zK^*&dgT>Hddmqq$>(?fKVXoQ!f|_^_;Qu6H<l>pDj=&l^iVA;EJFWSaCu~Rg9j;WB
z$TUP|$PT%>*Tl<NV()!CHy#Y$-QE4nNwZJXKi2MC20b^gm~U%^KY7UOr8oQ&y3EC>
z9mH~(AZ!Bshhrmx$c6&jS7Jf*7>DFody18$XO=RNk$_w(DE!LAUKvHC46?i8MB^Mn
zKixRq3j(NxB|H$bi+ddl>DhSHnNFmfuRBmp_*E_I%z<2Z*7TG}<^oP*8#W`gWGsqm
zQ}_sK?WWNxo9`dH^<STazkT<+`{>7S(4SwQ#~;`+(g8Z43NLB`^2F0ib)qdeK9g+X
zC(D^rw@LFvC3S~e7+5IWrcIuqQrI^n?ImELsHASb7#4oNf?p{$H}xho=_f%`=~wun
z7+fHtwtsOyyxvr?dKgp>EO=0k&RTWH>8hr(?lEshc;OdJr>p2@trHrh0CbJ(O-83k
zmY$|kl5$w!Q2Dr{=xig6t5L3)EOx{z@hlSZfGIqMyM#2WK(9UY;3XfDim|d_B;AIY
zv&UD&@FHMUMp(yEHMvd@IhZ0Nw&>Ql%ssaXPn_Uh*c{e)Ll_O6NboVzYThl2#0($r
z5&W2t2{>2DF`HMZ%lhm&mknj=Fld|MTlK;RS;<VBwnb)s2}8?07IM@q0xlw_X)6i3
z3|x+}WkB{Mhvan=Ph*^}P+dIRz~aJ7c(+nn*S(GL3p&%PTl45)<S$O(rM~>kuKe4O
zKu=K7tSw@}JfF<d7ryE;O2kzX|Mq{qG#o(xti%w&Vgx4|PWd!VQtPNcoChUlTwgK?
zs5Rj^G3uvNCvN~3&ax<QM&;Z4A)Jrg7MSwnoGp3e%#(}>oyHwiy1xiT49(5lp?Z0x
Tr^qbM2#^SU30`H$uF2eMGq>Ji
literal 0
HcmV?d00001
diff --git a/tests/test_data/ST00.AB..BHZ.2007.161 b/tests/test_data/ST00.AB..BHZ.2007.161
new file mode 100644
index 0000000000000000000000000000000000000000..f3fa1ece76f3655d59e123b5d92c235b4f5e9e4a
GIT binary patch
literal 1024
zcmW;I&2QUu7zgm5?fC6yJBgDxiSxdsO<!PbItG^sRw{u&;xy=C)BX@Z;>3ju{{a$Q
zIe_580ae<V*wA30uHD)~k~S|(^KLti?bvaC32`6kdGz}oKKkl;N~LAJaVPijCp$a2
zTyF1!d++Yir{FsPHUPCtd>;O4@eKd~TQ&f@phR>4fEc{`dIQ_v-(M^i?d3C@k)nB-
zq-<weC3pw=APHARJiwYmb=ZZxmi5h2CT8%w9z_w>q(`&%SXz3c!T=B-<H<e|?a+27
zIchQ0LmV$N@{kiJ4nq$dx89)96O!qL8)mX&NHaYt@hPk6dBVMz!PZQi#Mc-?fI~%Z
z*oz6zPsV^HNC9U;M37Z@)S|{c0|^!sP#$a3&S>h)J&z#nEF28kk(iJ5ha)r$lE9*<
z=VW{$Ax^EWAU5k0F(E(%&;CWHDq+xxd_c}bX~px<>ZXad(eF0w3}RAvM)pxN_wurj
z0*d8ob50_-1R0`DKk6;87lUxUzc6AxLMEl4=R<;N9F_5=xz3Xb4A0UhmhiHoTcnDI
z$s|2QqF@5pza+Hbtd4IkOm5s86WV|Qqq^sL39O2dO&gQ@nrjMfJ6iA&d7^S1o@L?k
z;G#T8r|p{zwnAOBi+^|Wr#yzIE(88^db;#?YXP6-I4MjK2xob*0M%KcPSOjVD-T^|
zJPP_vfhnWJv+i4VILQ?IrN{Q2#eMM6==(3e_VV8e*FXKKbgi5>B-K*?bE>1HyIuU_
zHmqejv4e;3!W=SI@HLkYggf)X@eK9kv#Drol5_oP|Lt=YRYU3o_HYiEx?uFAXs0QR
zn{wnqtxzS_`hFSlVQos5d7vZ>q>Qe;Qj#pF+deP95+{tvu|#Xz7f>=H4SC+$);VYw
zt7b5By#m-*e)oI}n-(A3u(8s&pUr;%@aR45uuyRmSH77OjqprREO{canxCWF)N0pq
zbZH&;?<)N2WBO)8iS(+*TrC!>dAvTSURTTlb&O|QUOtEEMr@V91BoAD{>fgwT6$Pd
zg@2Rof9dQVyq~<p8vLQ1JLoVin7F9HkuQ#A*;<x6ktTi5i@5b@ZT-(YvJ&aobAGP!
zzk>8%W0(`Qjd;)CT(sVXTmM*5<)u>!6<F<7TVMGhj!w28#XO%JZ4Kn~EEZxgZ6Fh&
t!n1h3dqdSxO;$wyNaWNF=Jf4x@Agx2{B-A+=B=OMo7%c1W?u-y{{d2SzWM+F
literal 0
HcmV?d00001
diff --git a/tests/test_data/ST00.AB..BHZ.2007.162 b/tests/test_data/ST00.AB..BHZ.2007.162
new file mode 100644
index 0000000000000000000000000000000000000000..75c29514caed9cc89925cc00f5edca42c11a3950
GIT binary patch
literal 1024
zcmW;HYiL__6aesZpSgM6<R(p<q^3<%`!EfC*jd`?>N-u4+RZV|m{no>FouGPAC!R)
zhJwxy>tI-msM`=K(y46dD06;sv&@dIUAu+8QqxBsZS&5}y*JPMH}SyvAI{<Y_;Wft
z%dzK({pDA>y6krQffq&w4k#AkGQeJFIz#ju&!X=D04pcJe&{5&0aTdk|F-gA9*?J7
zD1^{YA^l9s5sfO_i_vyINQG0%93A&^Vo7<cRyEa=^tR2{U=N<#ia)<X^o6q3R6>v}
zM}M45S?)W)i9SB1|Kh{%(eT*Ye`uZK$45)B7hQle$cwJSSLal3T!s_qCJ^u?ItV90
zfsVo~`rJ9`87a>u&`B6YKcG0gk50f8I*Zof)AHUs>99@5EO-C1Kf3wiy@r^pxID73
zcKpUNTtRLag3p|j6CQa7bD`VlF!mZ6L{~5loks6rRp?c84pSi`48kX<<Ac_5XP+sc
zIsCiJeSTnNYG{E_Oj|LHseFHLyh-;LtC<%XmYVv1ZSE?xs4bGb@qqLxn|wy+Vx4KG
zK__T;YG4x;Y^#22%v76-S8*$PS*rN%WT40!jH*n@NE>Ml8IZ;Fzgw+BSSu<SHHT8r
zObaDnB)cIOWX(p7iU<S=e302*wN|`DIs{JIu8VBSl4d=`h7tzZXF`ACF+U>^d978F
zcpWRMaa}<ml65$f2ps%5<Gy+%ml&$}--fxl-XN3DaN-u_nm4`V<y>O=eox}kTe<0$
z+m4X$qY2xa*M{#hgXfP$=bxR%V>&+E{;+eK>#hFAGxm@BkyYUz`5KM^jxNE)s~yc_
zZ~=L9!``mD*_JyhP9mSgdirj@?j_+6I(_I)V&CWhkLR(W*~<UV$h5-$?l`;W+uj?d
z=;q+XQ^30)?z_%eXX=Cas>!xQ)X>xUv|R0N6pD@4G_k3gM@c)X;wYiwEqL0nC~k$c
zGf;}u8`?F6+Fd~@)3_jE$?ZTgAEPv4*wHV_EKB%{xhzX0*Nav%t5fSMo{egi5jwe+
zmect<Ro2>WQP*N?yUC4UK`7}NbHn2niKT)hRRROCYcBff)nHvNrD$x^nk$~t;0rlA
zwUnv%b5GakQ}z0}Y&@Dv6I4l|=aG(dkyLjeZx7;+*|0gxm?FfOPMI-4++3)5zg+OI
zieic(46VwnT8uL>?|MPJm#yN>P8*9G^N7f#cMOS$Vu5Pn8I~tiq|PGZ)xLPeiz{MK
zd<sFE+{WxU#ZWQxcdTKBRu($+cC%QWQ@P_arrld5Kc)*lk#G)7H?y-O!H12>c-Md5
CAKgU&
literal 0
HcmV?d00001
diff --git a/tests/test_data/ST00.AB..BHZ.2007.163 b/tests/test_data/ST00.AB..BHZ.2007.163
new file mode 100644
index 0000000000000000000000000000000000000000..907b07ca7097a791aa3c346e26df10f9c7788851
GIT binary patch
literal 1024
zcmW;K$!_Cx00!{C?Koay$Bwf(PBL{nNw;B|l!2Z)Q)Z-*kU$zyrWX!80gu1~@DLn0
zbLIdi1XMzz8cJtcwUcIFn$6DQ*s=5XMm?YO9e?^RU%7mrPmkrVU+?eBvi#!nZ=Sy(
zuE9?L;Xty9eINh9{}BLy+!uh)Kp7hVKqEAGDZe~EeswG(CnqPq?|*mSFHe={?Z}aU
zEfVGHwzuMc{)entZTAV5%)zJmuF$Gg{|?{5ZH)BTmHv>8(G-%r_t$o(Wf?m@U9{s8
zs1r#nx@1O%vZVfah0MP_`$U@TXx>$-3rlAQ0>2KAE|MGXo(w4bufxE@%44KXQ}4(f
zR5(P<hrhmG{^p~ZH1<FjO?dcbnQo-VZtCsKTCx2;Bd|<Lw)NdiCyL$#p7ZW_kB&|b
zq~-$xh9{nFsudS@L}6CUVUj+Q+=?-{&?w(pvPA|Bn*>s=E$z;GYY}*zw^7+6O5|eI
zn~lfw^S;uNJ*~t*6qUM)c&lNRMYBs7OrA<nHZj$V-eS~P0v{Ix|4dd32XFyRLZA8O
zWJEx(fIWJq3fXI&TUHyoZcyuaF`bvK^!5Z}3{}?~6lW(g9vXNlluY5#kU%ha&qYvF
zvk4N*meYJ<V&wWg<a`Et<ZKuXN3+p<!VFDL-&!qogYW^C2|OD9h<614>c*V6J346j
z8+iJwZC?`1f6uF@T|4B*zSs#B)<f2~el_$IN3z|m1e8T9<^=w-HdF?qVMubhkM}oY
z4DM=|=k01^<#X{kj_zjXHtYj!$n@C~mY$=<scC4Akz;REDe%+IRs!F`wWvL0y)KIl
zVEv6fZ#g6OFr$%=P&73*=+Q41oK1;Bnl3QYXl!lO5X_z%4!jEsq!9;`zRoAY>1b6M
zuh4ncL2_g)o<w=VEi%)=T=@^<mkC4V@a&#I5%ufaPXEtX;LFp^-Gx%~C@qY+P=!Ee
z)-cLw`eSF0ab@hbrz})00vMXhj?ircqXxIlwx1#a<^|po*NJ3Sa8%23m8vI-F^`Ko
zD?b-8$wkvHN=uWYO<N5;W`!L%H`~>&A1iESHaDyf|Bu{xu<K!r#UI&C%L-c#&k^$o
z8F>mX@vu^vsv{KAt0v**_xi~a2|9>#W13S5*4BdkM6!g3=vJkBWyE-^jjQ!AyDlX`
iZ*g`O(yDZPkdSo2^o2##(1yS3YK=|JS^yD5uJk{l&E=f{
literal 0
HcmV?d00001
diff --git a/tests/test_data/ST00.AB..BHZ.2007.164 b/tests/test_data/ST00.AB..BHZ.2007.164
new file mode 100644
index 0000000000000000000000000000000000000000..b925f35dace4e038f204d9b993113d9318f277e5
GIT binary patch
literal 1024
zcmW;J%W~Uf00rP*_lqUly83=iC?upa2~Y-DkupFxumG~*S$F^*f<>Q!CBw2)rYvBl
zohHG#Dvl*vvSnF!$@-;6oSC!y<}A+P;X^+CVff=uM@Pfq@aeOkkDpR!;5k4>p!OX5
zMf$b)4FG^V6o4<mAvOnqMOg5~@cZw+K7RZya&mG~6vbB$wWB$^@ZF+VFcz`bfU8QS
z=bb9v>1+~+{pRS|QG-~SGu*pgod;3<^IE-8MU~Ps9!gf4bs<i{hOP?S>h=Qtc*Kh4
zjmRs9T)pjN#bttPalf6SO>SFN{3vm3;(elB`hMQfEq;OOFF5iXNZqGmG}8XKdSh?H
z(a>MwJVBu#y3h(y3~QY^Y<YBbiWN&^Vy{cjc83a;y$bvB)W|)HSGePEGJ${Tu7CHp
zXX8hCN0nKuk<+a1-2J@{?<>vNi7OHSj|QbDr}zBE;ZlcW>?RlP5BNNTx(h)>Kp!<D
zg_QT26r1CiZW_r5hpp7?3W4mb2TAD{5NwQ$Ri&7r=1_`mmT#Ag(G%h)Ld<6OIj=7X
zbOl=7ZX$!f{KP}Ks-7`dgHK$g^i*D*OK|Eh4+LG^m@L_D?e-lvPh(dP@nz*bR+JGz
zfKBv;4&AucUYp3cV{%8W+ga&9f-i;iDkmP(A-7=%6&=Xui)}Q<-g@YQ`H=4_=zzm6
z-c+w|e0*py_3b{FZ6kO&BdO98h@9QwMy#HaN0ZZmVqs#IF;N)YRHYEk5XI-QtPV))
zHRk*oP;+5lB>K&arc=u=eRjQKQ=h9VLLy2gLQbH1;Sra#09Aq65(=cT|JKMoT1|KN
zL^SUg7UFIo$A&oKmc9a$(FhIESi9VG3Jr41zs3`e>Zx_M6NeSp0~&?f9$8l;g3g^x
zl(#92Ol@6A|NqxhdK{r7xpz%kf#i<%Q@SIguIFj3o!969D~zbt;SfGdRd14dZr}-F
zlG&|bJ}An4ZHvrqAr%IjHI{i+EENbAxw<*M-Jq0*s`$-jSJ%{(!g5kW>JEa3DV632
z5@-^oN8NZkvGt-g702$lOVixYM)FAchwNX9DqxeS!gbYEt7D$s4kF{TZXgVebmxvx
z{c-7wSGC<~dv{jpXNHWeb^@>Jhelwz2N-#fHd+@BO~lsRCx8_nHiZvjQnpdi3>v}A
Feg_Dm+`#|<
literal 0
HcmV?d00001
diff --git a/tests/test_data2passcal.py b/tests/test_data2passcal.py
index a82fa89..02d5c33 100644
--- a/tests/test_data2passcal.py
+++ b/tests/test_data2passcal.py
@@ -3,25 +3,102 @@
"""Tests for `data2passcal` package."""
+from __future__ import division, print_function
+
+import configparser
+import ftplib
+import os
import unittest
-import sys
+
+from data2passcal.data2passcal import get_FTP, ismseed, scan_dir, send2passcal
+
+VERSION = '2020.107'
+
try:
- import data2passcal
+ from unittest.mock import patch
except ImportError:
- pass
+ from mock import patch
+
+config = configparser.ConfigParser()
+config.read('data2passcal/config.ini')
+FTP_HOST = config.get('FTP_INFO', 'FTP_HOST')
+FTP_USER = config.get('FTP_INFO', 'FTP_USER')
+FTP_PASSWORD = config.get('FTP_INFO', 'FTP_PASSWORD')
+FTP_DIR = config.get('FTP_INFO', 'FTP_DIR')
+FTP_TIMEOUT = config.getint('FTP_INFO', 'FTP_TIMEOUT')
+FTP_RECONNECT_WAIT = config.getint('FTP_INFO', 'FTP_RECONNECT_WAIT')
+FTP_CONNECT_ATTEMPTS = 60 * 60 * 24 * 7 / FTP_RECONNECT_WAIT
+FTP_SEND_ATTEMPTS = config.getint('FTP_INFO', 'FTP_SEND_ATTEMPTS')
+MOCK_TEST = config.getboolean('TEST', 'MOCK_TEST')
+
class TestData2passcal(unittest.TestCase):
"""Tests for `data2passcal` package."""
def setUp(self):
- """Set up test fixtures, if any."""
+ """Set up test fixtures, if any"""
+ dir_testdata = os.path.dirname(os.path.realpath(__file__)) + '/test_data'
+ filelist = [x for x in scan_dir(dir_testdata) if not os.path.basename(x).startswith('.') and not os.path.basename(x).endswith('.log')]
+ self.dir_testdata = dir_testdata
+ self.filelist = filelist
+
+ def test_scan_dir(self):
+ """
+ Test basic functionality of scan_dir function
+ Only 5 files available under ./test_data directory
+ Not taking into account .DS_STORE file
+ """
+ self.assertEqual(len(self.filelist), 5, 'Incorrect number of files')
+
+ def test_ismseed(self):
+ """Test basic functionality of ismseed function"""
+ for f in self.filelist:
+ self.assertTrue(ismseed(f), '{} is not a miniseed file'.format(os.path.basename(f)))
+
+ @patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
+ def test_get_FTP_mock(self, mock_ftp_constructor):
+ """Mock test creating ftp connection to PASSCAL"""
+ mock_ftp = mock_ftp_constructor.return_value
+ get_FTP()
+ mock_ftp_constructor.assert_called_with(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
+ mock_ftp.cwd.assert_called_with(FTP_DIR)
+ self.assertLess(mock_ftp_constructor.call_count, FTP_CONNECT_ATTEMPTS, 'Number of ftp connection attempts exceeeds {}'.format(FTP_CONNECT_ATTEMPTS))
+ mock_ftp.quit()
+
+ @patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
+ def test_send_data_mock(self, mock_ftp_constructor):
+ """Mock test sending MSEED files (test data) to PASSCAL's QC system"""
+ mock_ftp = mock_ftp_constructor.return_value
+ send2passcal(self.filelist)
+ self.assertTrue(mock_ftp.storbinary.called, 'No data sent')
+ self.assertEqual(mock_ftp.storbinary.call_count, len(self.filelist), 'Failed to send all files - Sent {0} of {1}'.format(mock_ftp.storbinary.call_count, len(self.filelist)))
+ files_sent = []
+ for x in mock_ftp.storbinary.call_args_list:
+ args, kwargs = x
+ files_sent.append(args[0].split(' ')[1])
+ for f in self.filelist:
+ f = os.path.basename(f)
+ self.assertLess(files_sent.count(f), FTP_SEND_ATTEMPTS, 'Attempted to send file {0} more than {1} times'.format(f, FTP_SEND_ATTEMPTS))
+
+ @unittest.skipIf(MOCK_TEST == True, "skipping real send2passcal test")
+ def test_send_data(self):
+ """Test sending MSEED files (test data) to PASSCAL's QC system - Optional"""
+ ftp = get_FTP()
+ send2passcal(self.filelist)
+ wdir = ftp.pwd()
+ try:
+ files_sent = [os.path.basename(x) for x in ftp.nlst(wdir)]
+ except ftplib.error_perm() as resp:
+ if str(resp) == "550 No files found":
+ print("No files found in this directory")
+ else:
+ raise
+ for f in self.filelist:
+ f_ = os.path.basename(f)
+ self.assertIn(f_, files_sent, 'File {} was not sent to PASSCAL'.format(f_))
+ ftp.quit()
- def tearDown(self):
- """Tear down test fixtures, if any."""
- def test_import(self):
- if 'data2passcal' in sys.modules:
- self.assertTrue(True, "data2passcal loaded")
- else:
- self.fail()
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tox.ini b/tox.ini
index 60d2b1b..f6954a3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist = py27, py36 flake8
+envlist = py27, py36, flake8
[travis]
python =
@@ -9,11 +9,10 @@ python =
[testenv:flake8]
basepython = python
deps = flake8
-commands = flake8 data2passcal
+commands = flake8 --ignore=E722,E712,E501 data2passcal
+ flake8 --ignore=E722,E712,E501 tests
[testenv]
-setenv =
- PYTHONPATH = {toxinidir}
-commands=python setup.py test
-
-
+deps = pytest
+setenv = PYTHONPATH = {toxinidir}
+commands = pytest
--
GitLab