From b4d6022e9730c4c4331e473ebb365ec3c2b5b697 Mon Sep 17 00:00:00 2001
From: Maeva Pourpoint <maeva@passcal.nmt.edu>
Date: Fri, 24 Apr 2020 18:20:27 -0600
Subject: [PATCH] Created conda packages for data2passcal that can run on
Python 2 and 3
---
AUTHORS.rst | 2 +-
CONTRIBUTING.rst | 4 +-
HISTORY.rst | 30 ++++
MANIFEST.in | 4 +-
README.rst | 19 +--
conda.recipe/bld.bat | 2 +
conda.recipe/build.sh | 1 +
conda.recipe/conda_build_config.yaml | 4 +
conda.recipe/meta.yaml | 27 ++--
data2passcal/__init__.py | 2 +-
data2passcal/data2passcal.py | 219 +++++++++++++++-----------
setup.cfg | 3 +-
setup.py | 6 +-
tests/test_data/ST00.AB..BHZ.2007.160 | Bin 0 -> 1024 bytes
tests/test_data/ST00.AB..BHZ.2007.161 | Bin 0 -> 1024 bytes
tests/test_data/ST00.AB..BHZ.2007.162 | Bin 0 -> 1024 bytes
tests/test_data/ST00.AB..BHZ.2007.163 | Bin 0 -> 1024 bytes
tests/test_data/ST00.AB..BHZ.2007.164 | Bin 0 -> 1024 bytes
tests/test_data2passcal.py | 94 +++++++++--
tox.ini | 16 +-
20 files changed, 284 insertions(+), 149 deletions(-)
create mode 100644 conda.recipe/bld.bat
create mode 100644 conda.recipe/build.sh
create mode 100644 conda.recipe/conda_build_config.yaml
create mode 100644 tests/test_data/ST00.AB..BHZ.2007.160
create mode 100644 tests/test_data/ST00.AB..BHZ.2007.161
create mode 100644 tests/test_data/ST00.AB..BHZ.2007.162
create mode 100644 tests/test_data/ST00.AB..BHZ.2007.163
create mode 100644 tests/test_data/ST00.AB..BHZ.2007.164
diff --git a/AUTHORS.rst b/AUTHORS.rst
index 8029b24..207d6dd 100644
--- a/AUTHORS.rst
+++ b/AUTHORS.rst
@@ -10,4 +10,4 @@ Development Lead
Contributors
------------
-None yet. Why not be the first?
+* Maeva Pourpoint <software-support@passcal.nmt.edu>
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index e8536cc..0866d54 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -76,7 +76,8 @@ Ready to contribute? Here's how to set up `data2passcal` for local development.
5. When you're done making changes, check that your changes pass the
tests::
- $ python setup.py test
+ $ python -m unittest (or python -m unittest tests.test_data2passcal under Python2)
+
6. Commit your changes and push your branch to GitHub::
$ git add .
@@ -111,4 +112,3 @@ Then run::
$ git push
$ git push --tags
-
diff --git a/HISTORY.rst b/HISTORY.rst
index a94f390..5c319c5 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -10,3 +10,33 @@ History
2018.228 (2018-08-16)
------------------
* Updated to work with python 2 and 3
+
+2020.093 (2020-04-02)
+------------------
+* Updated to work with python 2 and 3.
+* Resources used: Python-Future project and Futurize tool.
+* Changes tested using unittest module on individual test methods(tests.test_data2passcal)
+* Changes tested against real seismic dataset (AB.2002 - 2012)
+
+2020.107 (2020-04-16)
+------------------
+* Added some unit tests to ensure basic functionality of data2passcal
+* Created configuration file (.ini file) to store and access ftp and test related variables
+* Updated list of platform specific dependencies to be installed when installing data2passcal in dev mode (see setup.py)
+* Installed and tested data2passcal against Python2.7 and Python3.6 using tox
+* Formatted Python code to conform to the PEP8 style guide (exception: E722, E712, E501)
+
+2020.114 (2020-04-23)
+------------------
+* Created conda packages for "data2passcal" that can run on Python2.7 and Python3.6
+=> To create package:
+ - clone the "data2passcal" repo
+ - "cd" in "data2passcal" directory
+ - run "conda-build ."
+=> To install locally with dependencies:
+ - run "conda install -c ${CONDA_PREFIX}/conda-bld/ data2passcal"
+=> To install on user's computer from tarball with all dependencies:
+ - download tarball and "cd" to directory where tarball was downloaded
+ - run "conda install ./data2passcal-2020.114-py*_0.tar.bz2"
+ (choose appropriate python version for your platform)
+ - run "conda update data2passcal"
diff --git a/MANIFEST.in b/MANIFEST.in
index 965b2dd..ced9d12 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -5,7 +5,5 @@ include LICENSE
include README.rst
recursive-include tests *
-recursive-exclude * __pycache__
-recursive-exclude * *.py[co]
-recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif
+recursive-include docs *.rst conf.py Makefile make.bat
diff --git a/README.rst b/README.rst
index 1d6dee0..a1160b9 100644
--- a/README.rst
+++ b/README.rst
@@ -3,22 +3,9 @@ data2passcal
============
-Prepare SEED data for shipment to PASSCAL.
+Description: Send MSEED files ready for archival at the DMC to PASSCAL's QC system
+ Only accept day-long MSEED files.
+Usage: data2passcal dir
* Free software: GNU General Public License v3 (GPLv3)
-
-
-
-Features
---------
-
-* TODO
-
-Credits
--------
-
-This package was created with Cookiecutter_ and the `passoft/cookiecutter`_ project template.
-
-.. _Cookiecutter: https://github.com/audreyr/cookiecutter
-.. _`passoft/cookiecutter`: https://git.passcal.nmt.edu/passoft/cookiecutter
diff --git a/conda.recipe/bld.bat b/conda.recipe/bld.bat
new file mode 100644
index 0000000..b9cd616
--- /dev/null
+++ b/conda.recipe/bld.bat
@@ -0,0 +1,2 @@
+"%PYTHON%" setup.py install --single-version-externally-managed --record=record.txt
+if errorlevel 1 exit 1
diff --git a/conda.recipe/build.sh b/conda.recipe/build.sh
new file mode 100644
index 0000000..a660906
--- /dev/null
+++ b/conda.recipe/build.sh
@@ -0,0 +1 @@
+$PYTHON setup.py install --single-version-externally-managed --record=record.txt
diff --git a/conda.recipe/conda_build_config.yaml b/conda.recipe/conda_build_config.yaml
new file mode 100644
index 0000000..b0e3e77
--- /dev/null
+++ b/conda.recipe/conda_build_config.yaml
@@ -0,0 +1,4 @@
+python:
+ - 2.7
+ - 3.6
+target_platform: osx-64
diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml
index 2604327..43ddd10 100644
--- a/conda.recipe/meta.yaml
+++ b/conda.recipe/meta.yaml
@@ -1,24 +1,33 @@
package:
name: data2passcal
- version: 2014.125
+ version: 2020.114
source:
- path: ..
+ path: ../
build:
- # If the installation is complex, or different between Unix and Windows, use
- # separate bld.bat and build.sh files instead of this key. Add the line
- # "skip: True # [py<35]" (for example) to limit to Python 3.5 and newer, or
- # "skip: True # [not win]" to limit to Windows.
- script: python setup.py install --single-version-externally-managed --record=record.txt
+ number: 0
+ entry_points:
+ - data2passcal = data2passcal.data2passcal:main
requirements:
- build:
+ host:
- python
- setuptools
run:
- python
+test:
+ requires:
+ - mock # [py<33]
+ source_files:
+ - tests
+ commands:
+ - python -c "import data2passcal.data2passcal; print(data2passcal.__version__)"
+ - python -m unittest tests.test_data2passcal
+
about:
home: https://git.passcal.nmt.edu/passoft/data2passcal
- summary: Prepare SEED data for shipment to PASSCAL.
+ license: GPLv3
+ license_file: LICENSE
+ summary: Send MSEED files ready for archival at the DMC to PASSCAL's QC system
diff --git a/data2passcal/__init__.py b/data2passcal/__init__.py
index 1636f68..3a08473 100644
--- a/data2passcal/__init__.py
+++ b/data2passcal/__init__.py
@@ -4,4 +4,4 @@
__author__ = """IRIS PASSCAL"""
__email__ = 'software-support@passcal.nmt.edu'
-__version__ = '2018.228'
+__version__ = '2020.114'
diff --git a/data2passcal/data2passcal.py b/data2passcal/data2passcal.py
index 7632678..554aad9 100644
--- a/data2passcal/data2passcal.py
+++ b/data2passcal/data2passcal.py
@@ -5,22 +5,30 @@ data2passcal
Ship miniseed data to passcal via ftp for qc before archival at DMC DMS
Lloyd Carothers IRIS/PASSCAL
'''
-from __future__ import print_function
-VERSION = '2018.228'
+from __future__ import division, print_function
-import ftplib
-import sys, os, signal, struct, pickle, logging
-from time import time, sleep
import datetime
+import ftplib
+import logging
+import os
+import pickle
+import re
+import signal
+import struct
+import sys
+from time import sleep, time
+
+VERSION = '2020.114'
+
-#TESTMODE = True
+# Cache the ftplib.FTP class so it will be available to test_FTP(FTP) when calling isinstance assert
+FTPCache = ftplib.FTP
+
+
+# TEST related
TESTMODE = False
-# Verbosity of console log from logging module from DEBUG to CRITICAL
-DEBUG = logging.DEBUG
-# After transmission or on interrupt try to ftp logfile to PIC
-#SEND_LOGFILE = True
-#FTP related
+# FTP related
FTP_IP = '129.138.26.29'
FTP_HOST = 'qc.passcal.nmt.edu'
FTP_USER = 'ftp'
@@ -33,33 +41,36 @@ if TESTMODE:
FTP_TIMEOUT = 5
FTP_RECONNECT_WAIT = 5
FTP_BLOCKSIZE = 8192
-#number of time to try to open the ftp connection
-#FTP_CONNECT_ATTEMPTS = 3
-# testing will retry for a week. Why not?
-FTP_CONNECT_ATTEMPTS = 60*60*24*7 / FTP_RECONNECT_WAIT
-#number of times a single file with try to be sent
+# number of time to try to open the ftp connection
+FTP_CONNECT_ATTEMPTS = 60 * 60 * 24 * 7 / FTP_RECONNECT_WAIT
+# number of times a single file with try to be sent
FTP_SEND_ATTEMPTS = 3
# debug level of ftplib, 0-3
FTP_DEBUG_LEVEL = 0
-#store the files sent in ~/data2passcal.sent
-SENTFILE = os.path.join( os.path.expanduser('~'), '.data2passcal.sent')
-# If this file exists open it, incorporate, and save to new name, and delete old
-SENTFILE_OLD = os.path.join( os.path.expanduser('~'), '.send2passcal.sent')
+
+# Files related
LOGFILE = 'data2passcal.log'
+# store the files sent in ~/data2passcal.sent
+SENTFILE = os.path.join(os.path.expanduser('~'), '.data2passcal.sent')
+# If this file exists open it, incorporate, and save to new name, and delete old
+SENTFILE_OLD = os.path.join(os.path.expanduser('~'), '.send2passcal.sent')
HELP = '''
data2passcal
VERSION: %s
Usage: data2passcal dir
- data2passcal is a utility that sends MSEED files ready for archival at the DMC to PASSCAL's QC system by:
- Scans all files below directory dir.
- Filters out non-miniseed files, by inspecting the first blockette of each file.
- Sends the files to the automated system for ingestion into the QC system via ftp.
+ data2passcal is a utility that sends day-long MSEED files ready for archival at the DMC to PASSCAL's QC system by:
+ Scanning all files below directory dir.
+ Filtering out non-miniseed files, by inspecting the first blockette of each file.
+ Sending the files to the automated system for ingestion into the QC system via ftp.
You can send a SIGTERM (ctl-c) to data2passcal and it will shutdown cleanly.
A list of sent files is kept in ~/.data2passcal.sent. Subsequent runs of send2passcal will not send files already sent.
A log is stored in data2passcal.log in the current directory.
''' % VERSION
+# Verbosity of console log from logging module from DEBUG to CRITICAL
+DEBUG = logging.DEBUG
+
# Configure logging file and stdout
# todo if sending a file do we want to send all previous logs even if already sent
logger = logging.getLogger('__name__')
@@ -72,16 +83,17 @@ logger.addHandler(logfh)
# Log to stdout
logconsole = logging.StreamHandler()
logconsole.setLevel(logging.INFO)
-logconsole.setFormatter(logging.Formatter('%(message)s' ))
+logconsole.setFormatter(logging.Formatter('%(message)s'))
logger.addHandler(logconsole)
logger.debug('Program starting.')
-logger.debug('%s - v%s - Python:%s' % (sys.argv, VERSION, sys.version) )
+logger.debug('%s - v%s - Python:%s' % (sys.argv, VERSION, sys.version))
logger.debug('CWD: %s' % os.getcwd())
-logger.info('Version: '+ VERSION)
+logger.info('Version: ' + VERSION)
logger.info('TIMEOUT: %d' % FTP_TIMEOUT)
logger.info('FTP RECONNECT WAIT: %d' % FTP_RECONNECT_WAIT)
# -- End logging
+
def scan_dir(dir):
'''Returns a list of absolute file names found below root dir'''
rootdir = dir
@@ -98,17 +110,18 @@ def scan_dir(dir):
try:
filesize += os.path.getsize(f)
except OSError:
- logger.debug('Can not stat %s'% f)
+ logger.debug('Can not stat %s' % f)
else:
filelist.append(f)
- logger.info('Total Size = %s'% format_size(filesize) )
- logger.info('Total Files = %s'% len(filelist) )
- logger.info('Total Dirs = %s'% foldercount)
- logger.info('Scan time = %0.2fs'% (time() - starttime))
+ logger.info('Total Size = %s' % format_size(filesize))
+ logger.info('Total Files = %s' % len(filelist))
+ logger.info('Total Dirs = %s' % foldercount)
+ logger.info('Scan time = %0.2fs' % (time() - starttime))
print()
return filelist
+
def sendable(file):
'''Filters files scanned returning a new list of files to send i.e. miniseed'''
if os.path.basename(file).startswith('.'):
@@ -117,11 +130,12 @@ def sendable(file):
return True
return False
-#Basic Miniseed file metadata extracted from filename
-import re
-#This includes .p files which we may accept in the future but won't h
+
+# Basic Miniseed file metadata extracted from filename
+# This includes .p files which we may accept in the future but won't h
MseedRE = re.compile(r'\A(.*)\.([A-Z0-9][A-Z0-9])\.(.*)\.([A-Z][A-Z]\w)\.([0-9]{4})\.([0-9]{3})(?:\.p)*')
+
def filename_qc_format(file):
'''Used to filter filename if correct for qc system'''
return MseedRE.match(os.path.basename(file))
@@ -129,15 +143,16 @@ def filename_qc_format(file):
def format_size(num):
'''Format bytes into human readble with suffix'''
- for suffix in [ 'bytes', 'KB', 'MB', 'GB' ]:
+ for suffix in ['bytes', 'KB', 'MB', 'GB']:
if num < 1024.0:
return '%3.3f %s' % (num, suffix)
num /= 1024.0
return '%3.3f %s' % (num, 'TB')
+
def ismseed(file):
try:
- ms = open(file,'rb')
+ ms = open(file, 'rb')
except:
return None
order = ByteOrder(ms)
@@ -149,24 +164,26 @@ def ismseed(file):
return False
#########################################################
-#needs import sys,struct
-#Taken from Bruces LibTrace
-def ByteOrder(infile, seekval=20) :
+# needs import sys,struct
+# Taken from Bruces LibTrace
+
+
+def ByteOrder(infile, seekval=20):
"""
read file as if it is mseed just pulling time info
from fixed header and determine if it makes sense unpacked
as big endian or little endian
"""
Order = "unknown"
- try :
- #seek to timeblock and read
+ try:
+ # seek to timeblock and read
infile.seek(seekval)
- timeblock=infile.read(10)
+ timeblock = infile.read(10)
- #assume big endian
- (Year, Day, Hour, Min, Sec, junk, Micro)=\
- struct.unpack('>HHBBBBH',timeblock)
- #test if big endian read makes sense
+ # assume big endian
+ (Year, Day, Hour, Min, Sec, junk, Micro) =\
+ struct.unpack('>HHBBBBH', timeblock)
+ # test if big endian read makes sense
if 1950 <= Year <= 2050 and \
1 <= Day <= 366 and \
0 <= Hour <= 23 and \
@@ -174,23 +191,24 @@ def ByteOrder(infile, seekval=20) :
0 <= Sec <= 59:
Order = "big"
else:
- #try little endian read
- (Year, Day, Hour, Min, Sec, junk, Micro)=\
- struct.unpack('<HHBBBBH',timeblock)
- #test if little endian read makes sense
+ # try little endian read
+ (Year, Day, Hour, Min, Sec, junk, Micro) =\
+ struct.unpack('<HHBBBBH', timeblock)
+ # test if little endian read makes sense
if 1950 <= Year <= 2050 and \
1 <= Day <= 366 and \
0 <= Hour <= 23 and \
0 <= Min <= 59 and \
0 <= Sec <= 59:
Order = "little"
- except Exception as e:
+ except Exception:
pass
return Order
#########################################################
-def get_sent_file_list(sentfile = SENTFILE):
+
+def get_sent_file_list(sentfile=SENTFILE):
sentlist = []
if os.path.isfile(sentfile):
logger.debug('Using sentfile %s' % sentfile)
@@ -204,7 +222,8 @@ def get_sent_file_list(sentfile = SENTFILE):
sentlist = pickle.load(f)
return sentlist
-def write_sent_file_list(sentlist , sentfile = SENTFILE ):
+
+def write_sent_file_list(sentlist, sentfile=SENTFILE):
logger.info('Saving list of files sent to passcal')
with open(sentfile, 'wb+') as f:
pickle.dump(sentlist, f, protocol=2)
@@ -217,8 +236,8 @@ def get_FTP():
trys += 1
try:
import socket
- logger.info('Connecting to FTP host %s from %s. Attempt %d of %d' % ( FTP_HOST, socket.gethostbyname(socket.gethostname()), trys, FTP_CONNECT_ATTEMPTS))
- FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
+ logger.info('Connecting to FTP host %s from %s. Attempt %d of %d' % (FTP_HOST, socket.gethostbyname(socket.gethostname()), trys, FTP_CONNECT_ATTEMPTS))
+ FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
FTP.set_debuglevel(FTP_DEBUG_LEVEL)
FTP.cwd(FTP_DIR)
FTP.set_pasv(True)
@@ -234,29 +253,36 @@ def get_FTP():
logger.error('Giving up.')
return None
+
def test_network():
passcal_http_reachable()
google_http_reachable()
passcal_ftp_reachable()
+
def passcal_http_reachable():
'''download and time passcal home page'''
url = 'http://www.passcal.nmt.edu/'
return url_reachable(url=url)
+
def google_http_reachable():
'''download and time passcal home page'''
url = 'http://www.google.com/'
return url_reachable(url=url)
+
def passcal_ftp_reachable():
'''Download a small file from passcals general ftp'''
url = 'ftp://ftp.passcal.nmt.edu/download/public/test.dat'
return url_reachable(url=url)
-def url_reachable(url = 'http://www.passcal.nmt.edu/'):
+
+def url_reachable(url='http://www.passcal.nmt.edu/'):
'''fetches a url returns True or False'''
- import urllib.request, urllib.error, urllib.parse
+ import urllib.request
+ import urllib.error
+ import urllib.parse
start = time()
try:
f = urllib.request.urlopen(url)
@@ -264,20 +290,20 @@ def url_reachable(url = 'http://www.passcal.nmt.edu/'):
logger.error("Failed to open connection to %s" % url)
logger.error(e)
return False
- logger.info('connection made to %s in %f sec' % ( url, time() - start) )
+ logger.info('connection made to %s in %f sec' % (url, time() - start))
data = f.read()
runtime = time() - start
size = len(data)
- rate = size/runtime
- logger.info('%d B/sec in %f sec' % (rate, runtime) )
+ rate = size / runtime
+ logger.info('%d B/sec in %f sec' % (rate, runtime))
return True
def test_FTP(FTP):
try:
- assert isinstance(FTP, ftplib.FTP)
+ assert isinstance(FTP, FTPCache)
FTP.voidcmd('NOOP')
- except ftplib.all_errors as e :
+ except ftplib.all_errors as e:
logger.error(e)
return False
except AssertionError as e:
@@ -293,11 +319,11 @@ def send2passcal(mslist, sentlist=None):
# Handle SIGINT while in this function: to gracefully close connection and save sentlist to disk file
def signal_handler(signum, frame):
print()
- logger.info('Caught interrupt while FTPing. Aborting transfer %s' % current_file )
- logger.info('Sent %d of %d' % (num_sent, num_to_send) )
- logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)) )
- logger.info('%s /sec' % ( format_size(size_sent / (time() - starttime )) ) )
- logger.info('Ran for %f sec' % (time()-starttime) )
+ logger.info('Caught interrupt while FTPing. Aborting transfer %s' % current_file)
+ logger.info('Sent %d of %d' % (num_sent, num_to_send))
+ logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)))
+ logger.info('%s /sec' % (format_size(size_sent / (time() - starttime))))
+ logger.info('Ran for %f sec' % (time() - starttime))
write_sent_file_list(sentlist)
try:
if FTP:
@@ -311,16 +337,16 @@ def send2passcal(mslist, sentlist=None):
'''Updates the terminal display.'''
signal.signal(signal.SIGINT, signal_handler)
update.bytes_sent += len(data)
- print('\r' + str(PB) + ' %s /sec ' % ( format_size(size_sent / (time() - starttime )) ), end=' ')
+ print('\r' + str(PB) + ' %s /sec ' % (format_size(size_sent / (time() - starttime))), end=' ')
'''
- print '%s %0.2f%%. %0.10d / %0.10d.' % ( current_file.center(20),
- (update.bytes_sent / update.file_size)*100,
- update.bytes_sent,
- file_size,
- ) ,
+ print '%s %0.2f%%. %0.10d / %0.10d.' % ( current_file.center(20),
+ (update.bytes_sent / update.file_size)*100,
+ update.bytes_sent,
+ file_size,
+ ) ,
'''
- ETA_sec = ((time() - starttime) / size_sent ) * (size_to_send - size_sent)
- print('ETA %s %s %s' % ( str(datetime.timedelta(seconds=(ETA_sec))), current_file.center(20), ' '*20 ), end=' ')
+ ETA_sec = ((time() - starttime) / size_sent) * (size_to_send - size_sent)
+ print('ETA %s %s %s' % (str(datetime.timedelta(seconds=(ETA_sec))), current_file.center(20), ' ' * 20), end=' ')
sys.stdout.flush()
if sentlist is None:
@@ -333,7 +359,7 @@ def send2passcal(mslist, sentlist=None):
size_sent = 1
current_file = ''
signal.signal(signal.SIGINT, signal_handler)
- logger.info('Sending %d, %s files to PASSCAL' % (num_to_send, format_size (size_to_send)))
+ logger.info('Sending %d, %s files to PASSCAL' % (num_to_send, format_size(size_to_send)))
FTP = get_FTP()
starttime = time()
PB = ProgressBar(num_to_send)
@@ -348,19 +374,19 @@ def send2passcal(mslist, sentlist=None):
fh = open(f, 'rb')
file_size = os.path.getsize(f)
update.file_size = float(file_size)
- FTP.storbinary('STOR %s' % current_file , fh, blocksize=FTP_BLOCKSIZE, callback=update)
+ FTP.storbinary('STOR %s' % current_file, fh, blocksize=FTP_BLOCKSIZE, callback=update)
except ftplib.error_perm as e:
# This is permission and the error when 550 for the .in file already exists so we should just continue with the next file
# todo create a list of failed files and resend those at the end instead of requiring a rerun
- logger.error( 'Failed to send file %s, permission error. Skipping...' % current_file)
+ logger.error('Failed to send file %s, permission error. Skipping...' % current_file)
logger.error(e)
break
- except (ftplib.all_errors, AttributeError) as e :
- #since we can restore with how we have proftp setup. There is nothing more we can do with this file
- #Until the server rms the .in.file
- logger.error('Failed to send file %s.' % ( current_file) ) #, trys, FTP_SEND_ATTEMPTS)
+ except (ftplib.all_errors, AttributeError) as e:
+ # since we can restore with how we have proftp setup. There is nothing more we can do with this file
+ # Until the server rms the .in.file
+ logger.error('Failed to send file %s.' % (current_file)) # , trys, FTP_SEND_ATTEMPTS)
logger.error(e)
- #if DEBUG: print "Waiting %d..." % FTP_TIMEOUT; sleep(FTP_TIMEOUT)
+ # if DEBUG: print "Waiting %d..." % FTP_TIMEOUT; sleep(FTP_TIMEOUT)
try:
if FTP:
FTP.abort()
@@ -379,16 +405,18 @@ def send2passcal(mslist, sentlist=None):
break
print()
- logger.info( 'Sent %d of %d' % (num_sent, num_to_send) )
- logger.info( 'Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)) )
- logger.info( '%s /sec' % ( format_size(size_sent / (time() - starttime )) ) )
- logger.info( 'Ran for %f sec' % (time()-starttime) )
+ logger.info('Sent %d of %d' % (num_sent, num_to_send))
+ logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)))
+ logger.info('%s /sec' % (format_size(size_sent / (time() - starttime))))
+ logger.info('Ran for %f sec' % (time() - starttime))
if FTP and test_FTP(FTP):
FTP.quit()
########################################################
# From progressbar 3rd party class will eventually dump not awesome
-class ProgressBar:
+
+
+class ProgressBar(object):
def __init__(self, duration):
self.duration = duration
self.prog_bar = '[]'
@@ -408,7 +436,7 @@ class ProgressBar:
def update_time(self, elapsed_secs):
self.__update_amount((elapsed_secs / float(self.duration)) * 100.0)
- #self.prog_bar += ' %d/%s files' % (elapsed_secs, self.duration)
+ # self.prog_bar += ' %d/%s files' % (elapsed_secs, self.duration)
def __update_amount(self, new_amount):
percent_done = int(round((new_amount / 100.0) * 100.0))
@@ -425,8 +453,10 @@ class ProgressBar:
# End progressbar
########################################################
+
+
def main():
- if len(sys.argv) < 2 or sys.argv[1] in [ '-h', '--help', '-?']:
+ if len(sys.argv) < 2 or sys.argv[1] in ['-h', '--help', '-?']:
print(HELP)
sys.exit()
# find all files below dir
@@ -435,19 +465,20 @@ def main():
logger.info('Removing improperly named files.')
msnamedlist = list(filter(filename_qc_format, filelist))
logger.info('Properly named files files: %d' % len(msnamedlist))
- logger.info('Other files: %d'% (len(filelist) - len(msnamedlist)))
+ logger.info('Other files: %d' % (len(filelist) - len(msnamedlist)))
print()
logger.info('Removing files that are not Miniseed.')
mslist = list(filter(sendable, msnamedlist))
logger.info('MiniSEED files: %d' % len(mslist))
- logger.info('Properly named but not miniseed files: %d'% (len(msnamedlist) - len(mslist)))
+ logger.info('Properly named but not miniseed files: %d' % (len(msnamedlist) - len(mslist)))
print()
logger.info('Removing files already sent to PASSCAL')
sentlist = get_sent_file_list()
unsentms = [f for f in mslist if f not in sentlist]
- logger.info('%d miniSEED files have already been sent, not resending.' % (len( mslist) - len(unsentms)))
+ logger.info('%d miniSEED files have already been sent, not resending.' % (len(mslist) - len(unsentms)))
send2passcal(unsentms, sentlist)
write_sent_file_list(sentlist)
+
if __name__ == '__main__':
main()
diff --git a/setup.cfg b/setup.cfg
index bd5e695..379997a 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 2018.228
+current_version = 2020.114
commit = True
tag = True
@@ -19,4 +19,3 @@ exclude = docs
[aliases]
# Define setup.py command aliases here
-
diff --git a/setup.py b/setup.py
index 3c7b133..40dc85e 100644
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,7 @@ setup(
],
},
install_requires=[],
- setup_requires = [],
+ setup_requires=[],
extras_require={
'dev': [
'pip',
@@ -42,6 +42,7 @@ setup(
'coverage',
'Sphinx',
'twine',
+ "mock;python_version<'3.3'"
]
},
license="GNU General Public License v3",
@@ -50,8 +51,7 @@ setup(
keywords='data2passcal',
name='data2passcal',
packages=find_packages(include=['data2passcal']),
- test_suite='tests',
url='https://git.passcal.nmt.edu/passoft/data2passcal',
- version='2018.228',
+ version='2020.114',
zip_safe=False,
)
diff --git a/tests/test_data/ST00.AB..BHZ.2007.160 b/tests/test_data/ST00.AB..BHZ.2007.160
new file mode 100644
index 0000000000000000000000000000000000000000..668ccee282506f7101f4f6de9e5175a3103d0043
GIT binary patch
literal 1024
zcmW;KOK;p%6bJC@YtQ&OevRkp$t0OH2_=LksFgre6e=W;*a1-`wruziAhBf2o&^g8
z9{`Y8bkS7=qKig^ypuNb$ari|?6Ez5+qZF!bk1LA@!K4ot*x7S=kuj6zTDnkT3UMi
z=~o{;CJ({S05kyiJ^pR=2lNvF0J~`bJ_K911^|OGw`(!5ot>SV_yr;K4;Zy~&DE#U
zdw*58o>n&xtMpNKz9$Ug)xOX?XNdtv+YFqAuR^4yiHvJ(IjP-voGk}|kG*`=f{`$@
zK^*&dgT>Hddmqq$>(?fKVXoQ!f|_^_;Qu6H<l>pDj=&l^iVA;EJFWSaCu~Rg9j;WB
z$TUP|$PT%>*Tl<NV()!CHy#Y$-QE4nNwZJXKi2MC20b^gm~U%^KY7UOr8oQ&y3EC>
z9mH~(AZ!Bshhrmx$c6&jS7Jf*7>DFody18$XO=RNk$_w(DE!LAUKvHC46?i8MB^Mn
zKixRq3j(NxB|H$bi+ddl>DhSHnNFmfuRBmp_*E_I%z<2Z*7TG}<^oP*8#W`gWGsqm
zQ}_sK?WWNxo9`dH^<STazkT<+`{>7S(4SwQ#~;`+(g8Z43NLB`^2F0ib)qdeK9g+X
zC(D^rw@LFvC3S~e7+5IWrcIuqQrI^n?ImELsHASb7#4oNf?p{$H}xho=_f%`=~wun
z7+fHtwtsOyyxvr?dKgp>EO=0k&RTWH>8hr(?lEshc;OdJr>p2@trHrh0CbJ(O-83k
zmY$|kl5$w!Q2Dr{=xig6t5L3)EOx{z@hlSZfGIqMyM#2WK(9UY;3XfDim|d_B;AIY
zv&UD&@FHMUMp(yEHMvd@IhZ0Nw&>Ql%ssaXPn_Uh*c{e)Ll_O6NboVzYThl2#0($r
z5&W2t2{>2DF`HMZ%lhm&mknj=Fld|MTlK;RS;<VBwnb)s2}8?07IM@q0xlw_X)6i3
z3|x+}WkB{Mhvan=Ph*^}P+dIRz~aJ7c(+nn*S(GL3p&%PTl45)<S$O(rM~>kuKe4O
zKu=K7tSw@}JfF<d7ryE;O2kzX|Mq{qG#o(xti%w&Vgx4|PWd!VQtPNcoChUlTwgK?
zs5Rj^G3uvNCvN~3&ax<QM&;Z4A)Jrg7MSwnoGp3e%#(}>oyHwiy1xiT49(5lp?Z0x
Tr^qbM2#^SU30`H$uF2eMGq>Ji
literal 0
HcmV?d00001
diff --git a/tests/test_data/ST00.AB..BHZ.2007.161 b/tests/test_data/ST00.AB..BHZ.2007.161
new file mode 100644
index 0000000000000000000000000000000000000000..f3fa1ece76f3655d59e123b5d92c235b4f5e9e4a
GIT binary patch
literal 1024
zcmW;I&2QUu7zgm5?fC6yJBgDxiSxdsO<!PbItG^sRw{u&;xy=C)BX@Z;>3ju{{a$Q
zIe_580ae<V*wA30uHD)~k~S|(^KLti?bvaC32`6kdGz}oKKkl;N~LAJaVPijCp$a2
zTyF1!d++Yir{FsPHUPCtd>;O4@eKd~TQ&f@phR>4fEc{`dIQ_v-(M^i?d3C@k)nB-
zq-<weC3pw=APHARJiwYmb=ZZxmi5h2CT8%w9z_w>q(`&%SXz3c!T=B-<H<e|?a+27
zIchQ0LmV$N@{kiJ4nq$dx89)96O!qL8)mX&NHaYt@hPk6dBVMz!PZQi#Mc-?fI~%Z
z*oz6zPsV^HNC9U;M37Z@)S|{c0|^!sP#$a3&S>h)J&z#nEF28kk(iJ5ha)r$lE9*<
z=VW{$Ax^EWAU5k0F(E(%&;CWHDq+xxd_c}bX~px<>ZXad(eF0w3}RAvM)pxN_wurj
z0*d8ob50_-1R0`DKk6;87lUxUzc6AxLMEl4=R<;N9F_5=xz3Xb4A0UhmhiHoTcnDI
z$s|2QqF@5pza+Hbtd4IkOm5s86WV|Qqq^sL39O2dO&gQ@nrjMfJ6iA&d7^S1o@L?k
z;G#T8r|p{zwnAOBi+^|Wr#yzIE(88^db;#?YXP6-I4MjK2xob*0M%KcPSOjVD-T^|
zJPP_vfhnWJv+i4VILQ?IrN{Q2#eMM6==(3e_VV8e*FXKKbgi5>B-K*?bE>1HyIuU_
zHmqejv4e;3!W=SI@HLkYggf)X@eK9kv#Drol5_oP|Lt=YRYU3o_HYiEx?uFAXs0QR
zn{wnqtxzS_`hFSlVQos5d7vZ>q>Qe;Qj#pF+deP95+{tvu|#Xz7f>=H4SC+$);VYw
zt7b5By#m-*e)oI}n-(A3u(8s&pUr;%@aR45uuyRmSH77OjqprREO{canxCWF)N0pq
zbZH&;?<)N2WBO)8iS(+*TrC!>dAvTSURTTlb&O|QUOtEEMr@V91BoAD{>fgwT6$Pd
zg@2Rof9dQVyq~<p8vLQ1JLoVin7F9HkuQ#A*;<x6ktTi5i@5b@ZT-(YvJ&aobAGP!
zzk>8%W0(`Qjd;)CT(sVXTmM*5<)u>!6<F<7TVMGhj!w28#XO%JZ4Kn~EEZxgZ6Fh&
t!n1h3dqdSxO;$wyNaWNF=Jf4x@Agx2{B-A+=B=OMo7%c1W?u-y{{d2SzWM+F
literal 0
HcmV?d00001
diff --git a/tests/test_data/ST00.AB..BHZ.2007.162 b/tests/test_data/ST00.AB..BHZ.2007.162
new file mode 100644
index 0000000000000000000000000000000000000000..75c29514caed9cc89925cc00f5edca42c11a3950
GIT binary patch
literal 1024
zcmW;HYiL__6aesZpSgM6<R(p<q^3<%`!EfC*jd`?>N-u4+RZV|m{no>FouGPAC!R)
zhJwxy>tI-msM`=K(y46dD06;sv&@dIUAu+8QqxBsZS&5}y*JPMH}SyvAI{<Y_;Wft
z%dzK({pDA>y6krQffq&w4k#AkGQeJFIz#ju&!X=D04pcJe&{5&0aTdk|F-gA9*?J7
zD1^{YA^l9s5sfO_i_vyINQG0%93A&^Vo7<cRyEa=^tR2{U=N<#ia)<X^o6q3R6>v}
zM}M45S?)W)i9SB1|Kh{%(eT*Ye`uZK$45)B7hQle$cwJSSLal3T!s_qCJ^u?ItV90
zfsVo~`rJ9`87a>u&`B6YKcG0gk50f8I*Zof)AHUs>99@5EO-C1Kf3wiy@r^pxID73
zcKpUNTtRLag3p|j6CQa7bD`VlF!mZ6L{~5loks6rRp?c84pSi`48kX<<Ac_5XP+sc
zIsCiJeSTnNYG{E_Oj|LHseFHLyh-;LtC<%XmYVv1ZSE?xs4bGb@qqLxn|wy+Vx4KG
zK__T;YG4x;Y^#22%v76-S8*$PS*rN%WT40!jH*n@NE>Ml8IZ;Fzgw+BSSu<SHHT8r
zObaDnB)cIOWX(p7iU<S=e302*wN|`DIs{JIu8VBSl4d=`h7tzZXF`ACF+U>^d978F
zcpWRMaa}<ml65$f2ps%5<Gy+%ml&$}--fxl-XN3DaN-u_nm4`V<y>O=eox}kTe<0$
z+m4X$qY2xa*M{#hgXfP$=bxR%V>&+E{;+eK>#hFAGxm@BkyYUz`5KM^jxNE)s~yc_
zZ~=L9!``mD*_JyhP9mSgdirj@?j_+6I(_I)V&CWhkLR(W*~<UV$h5-$?l`;W+uj?d
z=;q+XQ^30)?z_%eXX=Cas>!xQ)X>xUv|R0N6pD@4G_k3gM@c)X;wYiwEqL0nC~k$c
zGf;}u8`?F6+Fd~@)3_jE$?ZTgAEPv4*wHV_EKB%{xhzX0*Nav%t5fSMo{egi5jwe+
zmect<Ro2>WQP*N?yUC4UK`7}NbHn2niKT)hRRROCYcBff)nHvNrD$x^nk$~t;0rlA
zwUnv%b5GakQ}z0}Y&@Dv6I4l|=aG(dkyLjeZx7;+*|0gxm?FfOPMI-4++3)5zg+OI
zieic(46VwnT8uL>?|MPJm#yN>P8*9G^N7f#cMOS$Vu5Pn8I~tiq|PGZ)xLPeiz{MK
zd<sFE+{WxU#ZWQxcdTKBRu($+cC%QWQ@P_arrld5Kc)*lk#G)7H?y-O!H12>c-Md5
CAKgU&
literal 0
HcmV?d00001
diff --git a/tests/test_data/ST00.AB..BHZ.2007.163 b/tests/test_data/ST00.AB..BHZ.2007.163
new file mode 100644
index 0000000000000000000000000000000000000000..907b07ca7097a791aa3c346e26df10f9c7788851
GIT binary patch
literal 1024
zcmW;K$!_Cx00!{C?Koay$Bwf(PBL{nNw;B|l!2Z)Q)Z-*kU$zyrWX!80gu1~@DLn0
zbLIdi1XMzz8cJtcwUcIFn$6DQ*s=5XMm?YO9e?^RU%7mrPmkrVU+?eBvi#!nZ=Sy(
zuE9?L;Xty9eINh9{}BLy+!uh)Kp7hVKqEAGDZe~EeswG(CnqPq?|*mSFHe={?Z}aU
zEfVGHwzuMc{)entZTAV5%)zJmuF$Gg{|?{5ZH)BTmHv>8(G-%r_t$o(Wf?m@U9{s8
zs1r#nx@1O%vZVfah0MP_`$U@TXx>$-3rlAQ0>2KAE|MGXo(w4bufxE@%44KXQ}4(f
zR5(P<hrhmG{^p~ZH1<FjO?dcbnQo-VZtCsKTCx2;Bd|<Lw)NdiCyL$#p7ZW_kB&|b
zq~-$xh9{nFsudS@L}6CUVUj+Q+=?-{&?w(pvPA|Bn*>s=E$z;GYY}*zw^7+6O5|eI
zn~lfw^S;uNJ*~t*6qUM)c&lNRMYBs7OrA<nHZj$V-eS~P0v{Ix|4dd32XFyRLZA8O
zWJEx(fIWJq3fXI&TUHyoZcyuaF`bvK^!5Z}3{}?~6lW(g9vXNlluY5#kU%ha&qYvF
zvk4N*meYJ<V&wWg<a`Et<ZKuXN3+p<!VFDL-&!qogYW^C2|OD9h<614>c*V6J346j
z8+iJwZC?`1f6uF@T|4B*zSs#B)<f2~el_$IN3z|m1e8T9<^=w-HdF?qVMubhkM}oY
z4DM=|=k01^<#X{kj_zjXHtYj!$n@C~mY$=<scC4Akz;REDe%+IRs!F`wWvL0y)KIl
zVEv6fZ#g6OFr$%=P&73*=+Q41oK1;Bnl3QYXl!lO5X_z%4!jEsq!9;`zRoAY>1b6M
zuh4ncL2_g)o<w=VEi%)=T=@^<mkC4V@a&#I5%ufaPXEtX;LFp^-Gx%~C@qY+P=!Ee
z)-cLw`eSF0ab@hbrz})00vMXhj?ircqXxIlwx1#a<^|po*NJ3Sa8%23m8vI-F^`Ko
zD?b-8$wkvHN=uWYO<N5;W`!L%H`~>&A1iESHaDyf|Bu{xu<K!r#UI&C%L-c#&k^$o
z8F>mX@vu^vsv{KAt0v**_xi~a2|9>#W13S5*4BdkM6!g3=vJkBWyE-^jjQ!AyDlX`
iZ*g`O(yDZPkdSo2^o2##(1yS3YK=|JS^yD5uJk{l&E=f{
literal 0
HcmV?d00001
diff --git a/tests/test_data/ST00.AB..BHZ.2007.164 b/tests/test_data/ST00.AB..BHZ.2007.164
new file mode 100644
index 0000000000000000000000000000000000000000..b925f35dace4e038f204d9b993113d9318f277e5
GIT binary patch
literal 1024
zcmW;J%W~Uf00rP*_lqUly83=iC?upa2~Y-DkupFxumG~*S$F^*f<>Q!CBw2)rYvBl
zohHG#Dvl*vvSnF!$@-;6oSC!y<}A+P;X^+CVff=uM@Pfq@aeOkkDpR!;5k4>p!OX5
zMf$b)4FG^V6o4<mAvOnqMOg5~@cZw+K7RZya&mG~6vbB$wWB$^@ZF+VFcz`bfU8QS
z=bb9v>1+~+{pRS|QG-~SGu*pgod;3<^IE-8MU~Ps9!gf4bs<i{hOP?S>h=Qtc*Kh4
zjmRs9T)pjN#bttPalf6SO>SFN{3vm3;(elB`hMQfEq;OOFF5iXNZqGmG}8XKdSh?H
z(a>MwJVBu#y3h(y3~QY^Y<YBbiWN&^Vy{cjc83a;y$bvB)W|)HSGePEGJ${Tu7CHp
zXX8hCN0nKuk<+a1-2J@{?<>vNi7OHSj|QbDr}zBE;ZlcW>?RlP5BNNTx(h)>Kp!<D
zg_QT26r1CiZW_r5hpp7?3W4mb2TAD{5NwQ$Ri&7r=1_`mmT#Ag(G%h)Ld<6OIj=7X
zbOl=7ZX$!f{KP}Ks-7`dgHK$g^i*D*OK|Eh4+LG^m@L_D?e-lvPh(dP@nz*bR+JGz
zfKBv;4&AucUYp3cV{%8W+ga&9f-i;iDkmP(A-7=%6&=Xui)}Q<-g@YQ`H=4_=zzm6
z-c+w|e0*py_3b{FZ6kO&BdO98h@9QwMy#HaN0ZZmVqs#IF;N)YRHYEk5XI-QtPV))
zHRk*oP;+5lB>K&arc=u=eRjQKQ=h9VLLy2gLQbH1;Sra#09Aq65(=cT|JKMoT1|KN
zL^SUg7UFIo$A&oKmc9a$(FhIESi9VG3Jr41zs3`e>Zx_M6NeSp0~&?f9$8l;g3g^x
zl(#92Ol@6A|NqxhdK{r7xpz%kf#i<%Q@SIguIFj3o!969D~zbt;SfGdRd14dZr}-F
zlG&|bJ}An4ZHvrqAr%IjHI{i+EENbAxw<*M-Jq0*s`$-jSJ%{(!g5kW>JEa3DV632
z5@-^oN8NZkvGt-g702$lOVixYM)FAchwNX9DqxeS!gbYEt7D$s4kF{TZXgVebmxvx
z{c-7wSGC<~dv{jpXNHWeb^@>Jhelwz2N-#fHd+@BO~lsRCx8_nHiZvjQnpdi3>v}A
Feg_Dm+`#|<
literal 0
HcmV?d00001
diff --git a/tests/test_data2passcal.py b/tests/test_data2passcal.py
index e102d31..4888784 100644
--- a/tests/test_data2passcal.py
+++ b/tests/test_data2passcal.py
@@ -3,26 +3,98 @@
"""Tests for `data2passcal` package."""
+from __future__ import division, print_function
+
+import ftplib
+import os
import unittest
-import sys
+
+from data2passcal.data2passcal import get_FTP, ismseed, scan_dir, send2passcal
try:
- import data2passcal
+ from unittest.mock import patch
except ImportError:
- pass
+ from mock import patch
+
+VERSION = '2020.114'
+
+FTP_HOST = 'qc.passcal.nmt.edu'
+FTP_USER = 'ftp'
+FTP_PASSWORD = 'data2passcal'
+FTP_DIR = 'AUTO/MSEED'
+FTP_TIMEOUT = 120
+FTP_RECONNECT_WAIT = 60
+FTP_CONNECT_ATTEMPTS = 60 * 60 * 24 * 7 / FTP_RECONNECT_WAIT
+FTP_SEND_ATTEMPTS = 3
+MOCK_TEST = True
+
class TestData2passcal(unittest.TestCase):
"""Tests for `data2passcal` package."""
def setUp(self):
- """Set up test fixtures, if any."""
+ """Set up test fixtures, if any"""
+ dir_testdata = os.path.dirname(os.path.realpath(__file__)) + '/test_data'
+ filelist = [x for x in scan_dir(dir_testdata) if not os.path.basename(x).startswith('.') and not os.path.basename(x).endswith('.log')]
+ self.dir_testdata = dir_testdata
+ self.filelist = filelist
+
+ def test_scan_dir(self):
+ """
+ Test basic functionality of scan_dir function
+ Only 5 files available under ./test_data directory
+ Not taking into account .DS_STORE file
+ """
+ self.assertEqual(len(self.filelist), 5, 'Incorrect number of files')
+
+ def test_ismseed(self):
+ """Test basic functionality of ismseed function"""
+ for f in self.filelist:
+ self.assertTrue(ismseed(f), '{} is not a miniseed file'.format(os.path.basename(f)))
+
+ @patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
+ def test_get_FTP_mock(self, mock_ftp_constructor):
+ """Mock test creating ftp connection to PASSCAL"""
+ mock_ftp = mock_ftp_constructor.return_value
+ get_FTP()
+ mock_ftp_constructor.assert_called_with(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
+ mock_ftp.cwd.assert_called_with(FTP_DIR)
+ self.assertLess(mock_ftp_constructor.call_count, FTP_CONNECT_ATTEMPTS, 'Number of ftp connection attempts exceeeds {}'.format(FTP_CONNECT_ATTEMPTS))
+ mock_ftp.quit()
+
+ @patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
+ def test_send_data_mock(self, mock_ftp_constructor):
+ """Mock test sending MSEED files (test data) to PASSCAL's QC system"""
+ mock_ftp = mock_ftp_constructor.return_value
+ send2passcal(self.filelist)
+ self.assertTrue(mock_ftp.storbinary.called, 'No data sent')
+ self.assertEqual(mock_ftp.storbinary.call_count, len(self.filelist), 'Failed to send all files - Sent {0} of {1}'.format(mock_ftp.storbinary.call_count, len(self.filelist)))
+ files_sent = []
+ for x in mock_ftp.storbinary.call_args_list:
+ args, kwargs = x
+ files_sent.append(args[0].split(' ')[1])
+ for f in self.filelist:
+ f = os.path.basename(f)
+ self.assertLess(files_sent.count(f), FTP_SEND_ATTEMPTS, 'Attempted to send file {0} more than {1} times'.format(f, FTP_SEND_ATTEMPTS))
- def tearDown(self):
- """Tear down test fixtures, if any."""
+ @unittest.skipIf(MOCK_TEST == True, "skipping real send2passcal test")
+ def test_send_data(self):
+ """Test sending MSEED files (test data) to PASSCAL's QC system - Optional"""
+ ftp = get_FTP()
+ send2passcal(self.filelist)
+ wdir = ftp.pwd()
+ try:
+ files_sent = [os.path.basename(x) for x in ftp.nlst(wdir)]
+ except ftplib.error_perm() as resp:
+ if str(resp) == "550 No files found":
+ print("No files found in this directory")
+ else:
+ raise
+ for f in self.filelist:
+ f_ = os.path.basename(f)
+ self.assertIn(f_, files_sent, 'File {} was not sent to PASSCAL'.format(f_))
+ ftp.quit()
- def test_import(self):
- if 'data2passcal' in sys.modules:
- self.assert_(True, "data2passcal loaded")
- else:
- self.fail()
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tox.ini b/tox.ini
index 60d2b1b..d1ee218 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist = py27, py36 flake8
+envlist = py27, py36, flake8
[travis]
python =
@@ -9,11 +9,13 @@ python =
[testenv:flake8]
basepython = python
deps = flake8
-commands = flake8 data2passcal
-
-[testenv]
-setenv =
- PYTHONPATH = {toxinidir}
-commands=python setup.py test
+commands = flake8 --ignore=E722,E712,E501 data2passcal
+ flake8 --ignore=E722,E712,E501 tests
+[testenv:py27]
+changedir = tests
+deps = mock
+commands = python -m unittest test_data2passcal
+[testenv:py36]
+commands = python -m unittest
--
GitLab