diff --git a/AUTHORS.rst b/AUTHORS.rst
index 8029b2489bd99e83141f2a6e55f9aeb1f9c6f30c..207d6dd429fa08cf357717a87a587314d3012b29 100644
--- a/AUTHORS.rst
+++ b/AUTHORS.rst
@@ -10,4 +10,4 @@ Development Lead
Contributors
------------
-None yet. Why not be the first?
+* Maeva Pourpoint <software-support@passcal.nmt.edu>
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index e8536cc925d7042cd527fed67034e6b6781936cf..0866d5424cab8c69c25d6c43a6d3359a67816bc4 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -76,7 +76,8 @@ Ready to contribute? Here's how to set up `data2passcal` for local development.
5. When you're done making changes, check that your changes pass the
tests::
- $ python setup.py test
+ $ python -m unittest (or python -m unittest tests.test_data2passcal under Python2)
+
6. Commit your changes and push your branch to GitHub::
$ git add .
@@ -111,4 +112,3 @@ Then run::
$ git push
$ git push --tags
-
diff --git a/HISTORY.rst b/HISTORY.rst
index a94f3900674ff45112c0d5377c4db9759781243d..5c319c5fcb7d091c5ddb103d6dc878b81376f73b 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -10,3 +10,33 @@ History
2018.228 (2018-08-16)
------------------
* Updated to work with python 2 and 3
+
+2020.093 (2020-04-02)
+------------------
+* Updated to work with python 2 and 3.
+* Resources used: Python-Future project and Futurize tool.
+* Changes tested using unittest module on individual test methods(tests.test_data2passcal)
+* Changes tested against real seismic dataset (AB.2002 - 2012)
+
+2020.107 (2020-04-16)
+------------------
+* Added some unit tests to ensure basic functionality of data2passcal
+* Created configuration file (.ini file) to store and access ftp and test related variables
+* Updated list of platform specific dependencies to be installed when installing data2passcal in dev mode (see setup.py)
+* Installed and tested data2passcal against Python2.7 and Python3.6 using tox
+* Formatted Python code to conform to the PEP8 style guide (exception: E722, E712, E501)
+
+2020.114 (2020-04-23)
+------------------
+* Created conda packages for "data2passcal" that can run on Python2.7 and Python3.6
+=> To create package:
+ - clone the "data2passcal" repo
+ - "cd" in "data2passcal" directory
+ - run "conda-build ."
+=> To install locally with dependencies:
+ - run "conda install -c ${CONDA_PREFIX}/conda-bld/ data2passcal"
+=> To install on user's computer from tarball with all dependencies:
+ - download tarball and "cd" to directory where tarball was downloaded
+ - run "conda install ./data2passcal-2020.114-py*_0.tar.bz2"
+ (choose appropriate python version for your platform)
+ - run "conda update data2passcal"
diff --git a/MANIFEST.in b/MANIFEST.in
index 965b2dda7db7c49f68857dc3aea9af37e30a745e..ced9d127ece3b0481c36a07ff0a66b3dbd705cfe 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -5,7 +5,5 @@ include LICENSE
include README.rst
recursive-include tests *
-recursive-exclude * __pycache__
-recursive-exclude * *.py[co]
-recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif
+recursive-include docs *.rst conf.py Makefile make.bat
diff --git a/README.rst b/README.rst
index 1d6dee089e41341f1618e01f5a91e7ee72a5c3e0..a1160b99c3905a79cfa2cd8fa0e04228ddbc9968 100644
--- a/README.rst
+++ b/README.rst
@@ -3,22 +3,9 @@ data2passcal
============
-Prepare SEED data for shipment to PASSCAL.
+Description: Send MSEED files ready for archival at the DMC to PASSCAL's QC system
+ Only accept day-long MSEED files.
+Usage: data2passcal dir
* Free software: GNU General Public License v3 (GPLv3)
-
-
-
-Features
---------
-
-* TODO
-
-Credits
--------
-
-This package was created with Cookiecutter_ and the `passoft/cookiecutter`_ project template.
-
-.. _Cookiecutter: https://github.com/audreyr/cookiecutter
-.. _`passoft/cookiecutter`: https://git.passcal.nmt.edu/passoft/cookiecutter
diff --git a/conda.recipe/bld.bat b/conda.recipe/bld.bat
new file mode 100644
index 0000000000000000000000000000000000000000..b9cd616ce4d9273557e30ece062955645916aeb2
--- /dev/null
+++ b/conda.recipe/bld.bat
@@ -0,0 +1,2 @@
+"%PYTHON%" setup.py install --single-version-externally-managed --record=record.txt
+if errorlevel 1 exit 1
diff --git a/conda.recipe/build.sh b/conda.recipe/build.sh
new file mode 100644
index 0000000000000000000000000000000000000000..a6609066d90c9754bd83da5f77a7c017e19608d7
--- /dev/null
+++ b/conda.recipe/build.sh
@@ -0,0 +1 @@
+$PYTHON setup.py install --single-version-externally-managed --record=record.txt
diff --git a/conda.recipe/conda_build_config.yaml b/conda.recipe/conda_build_config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b0e3e7797a0e50b86ac0203fdbd89c201eb3a12a
--- /dev/null
+++ b/conda.recipe/conda_build_config.yaml
@@ -0,0 +1,4 @@
+python:
+ - 2.7
+ - 3.6
+target_platform: osx-64
diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml
index 260432777941989967c8ee02178c86241ee4d03c..43ddd10b0e25696c53e8499b5bb58314b1bf26bf 100644
--- a/conda.recipe/meta.yaml
+++ b/conda.recipe/meta.yaml
@@ -1,24 +1,33 @@
package:
name: data2passcal
- version: 2014.125
+ version: 2020.114
source:
- path: ..
+ path: ../
build:
- # If the installation is complex, or different between Unix and Windows, use
- # separate bld.bat and build.sh files instead of this key. Add the line
- # "skip: True # [py<35]" (for example) to limit to Python 3.5 and newer, or
- # "skip: True # [not win]" to limit to Windows.
- script: python setup.py install --single-version-externally-managed --record=record.txt
+ number: 0
+ entry_points:
+ - data2passcal = data2passcal.data2passcal:main
requirements:
- build:
+ host:
- python
- setuptools
run:
- python
+test:
+ requires:
+ - mock # [py<33]
+ source_files:
+ - tests
+ commands:
+ - python -c "import data2passcal.data2passcal; print(data2passcal.__version__)"
+ - python -m unittest tests.test_data2passcal
+
about:
home: https://git.passcal.nmt.edu/passoft/data2passcal
- summary: Prepare SEED data for shipment to PASSCAL.
+ license: GPLv3
+ license_file: LICENSE
+ summary: Send MSEED files ready for archival at the DMC to PASSCAL's QC system
diff --git a/data2passcal/__init__.py b/data2passcal/__init__.py
index 1636f688fe53d37abbe1eb0fb8f667675ef3b664..3a08473ed7ec7352ef5ec141b7749c18bb91002b 100644
--- a/data2passcal/__init__.py
+++ b/data2passcal/__init__.py
@@ -4,4 +4,4 @@
__author__ = """IRIS PASSCAL"""
__email__ = 'software-support@passcal.nmt.edu'
-__version__ = '2018.228'
+__version__ = '2020.114'
diff --git a/data2passcal/data2passcal.py b/data2passcal/data2passcal.py
index 7632678c2c3cef869cf1a7490bd7f69fbbd63b97..554aad9f9186fc289640daa9f3a374739e8768b8 100644
--- a/data2passcal/data2passcal.py
+++ b/data2passcal/data2passcal.py
@@ -5,22 +5,30 @@ data2passcal
Ship miniseed data to passcal via ftp for qc before archival at DMC DMS
Lloyd Carothers IRIS/PASSCAL
'''
-from __future__ import print_function
-VERSION = '2018.228'
+from __future__ import division, print_function
-import ftplib
-import sys, os, signal, struct, pickle, logging
-from time import time, sleep
import datetime
+import ftplib
+import logging
+import os
+import pickle
+import re
+import signal
+import struct
+import sys
+from time import sleep, time
+
+VERSION = '2020.114'
+
-#TESTMODE = True
+# Cache the ftplib.FTP class so it will be available to test_FTP(FTP) when calling isinstance assert
+FTPCache = ftplib.FTP
+
+
+# TEST related
TESTMODE = False
-# Verbosity of console log from logging module from DEBUG to CRITICAL
-DEBUG = logging.DEBUG
-# After transmission or on interrupt try to ftp logfile to PIC
-#SEND_LOGFILE = True
-#FTP related
+# FTP related
FTP_IP = '129.138.26.29'
FTP_HOST = 'qc.passcal.nmt.edu'
FTP_USER = 'ftp'
@@ -33,33 +41,36 @@ if TESTMODE:
FTP_TIMEOUT = 5
FTP_RECONNECT_WAIT = 5
FTP_BLOCKSIZE = 8192
-#number of time to try to open the ftp connection
-#FTP_CONNECT_ATTEMPTS = 3
-# testing will retry for a week. Why not?
-FTP_CONNECT_ATTEMPTS = 60*60*24*7 / FTP_RECONNECT_WAIT
-#number of times a single file with try to be sent
+# number of time to try to open the ftp connection
+FTP_CONNECT_ATTEMPTS = 60 * 60 * 24 * 7 / FTP_RECONNECT_WAIT
+# number of times a single file with try to be sent
FTP_SEND_ATTEMPTS = 3
# debug level of ftplib, 0-3
FTP_DEBUG_LEVEL = 0
-#store the files sent in ~/data2passcal.sent
-SENTFILE = os.path.join( os.path.expanduser('~'), '.data2passcal.sent')
-# If this file exists open it, incorporate, and save to new name, and delete old
-SENTFILE_OLD = os.path.join( os.path.expanduser('~'), '.send2passcal.sent')
+
+# Files related
LOGFILE = 'data2passcal.log'
+# store the files sent in ~/data2passcal.sent
+SENTFILE = os.path.join(os.path.expanduser('~'), '.data2passcal.sent')
+# If this file exists open it, incorporate, and save to new name, and delete old
+SENTFILE_OLD = os.path.join(os.path.expanduser('~'), '.send2passcal.sent')
HELP = '''
data2passcal
VERSION: %s
Usage: data2passcal dir
- data2passcal is a utility that sends MSEED files ready for archival at the DMC to PASSCAL's QC system by:
- Scans all files below directory dir.
- Filters out non-miniseed files, by inspecting the first blockette of each file.
- Sends the files to the automated system for ingestion into the QC system via ftp.
+ data2passcal is a utility that sends day-long MSEED files ready for archival at the DMC to PASSCAL's QC system by:
+ Scanning all files below directory dir.
+ Filtering out non-miniseed files, by inspecting the first blockette of each file.
+ Sending the files to the automated system for ingestion into the QC system via ftp.
You can send a SIGTERM (ctl-c) to data2passcal and it will shutdown cleanly.
A list of sent files is kept in ~/.data2passcal.sent. Subsequent runs of send2passcal will not send files already sent.
A log is stored in data2passcal.log in the current directory.
''' % VERSION
+# Verbosity of console log from logging module from DEBUG to CRITICAL
+DEBUG = logging.DEBUG
+
# Configure logging file and stdout
# todo if sending a file do we want to send all previous logs even if already sent
logger = logging.getLogger('__name__')
@@ -72,16 +83,17 @@ logger.addHandler(logfh)
# Log to stdout
logconsole = logging.StreamHandler()
logconsole.setLevel(logging.INFO)
-logconsole.setFormatter(logging.Formatter('%(message)s' ))
+logconsole.setFormatter(logging.Formatter('%(message)s'))
logger.addHandler(logconsole)
logger.debug('Program starting.')
-logger.debug('%s - v%s - Python:%s' % (sys.argv, VERSION, sys.version) )
+logger.debug('%s - v%s - Python:%s' % (sys.argv, VERSION, sys.version))
logger.debug('CWD: %s' % os.getcwd())
-logger.info('Version: '+ VERSION)
+logger.info('Version: ' + VERSION)
logger.info('TIMEOUT: %d' % FTP_TIMEOUT)
logger.info('FTP RECONNECT WAIT: %d' % FTP_RECONNECT_WAIT)
# -- End logging
+
def scan_dir(dir):
'''Returns a list of absolute file names found below root dir'''
rootdir = dir
@@ -98,17 +110,18 @@ def scan_dir(dir):
try:
filesize += os.path.getsize(f)
except OSError:
- logger.debug('Can not stat %s'% f)
+ logger.debug('Can not stat %s' % f)
else:
filelist.append(f)
- logger.info('Total Size = %s'% format_size(filesize) )
- logger.info('Total Files = %s'% len(filelist) )
- logger.info('Total Dirs = %s'% foldercount)
- logger.info('Scan time = %0.2fs'% (time() - starttime))
+ logger.info('Total Size = %s' % format_size(filesize))
+ logger.info('Total Files = %s' % len(filelist))
+ logger.info('Total Dirs = %s' % foldercount)
+ logger.info('Scan time = %0.2fs' % (time() - starttime))
print()
return filelist
+
def sendable(file):
'''Filters files scanned returning a new list of files to send i.e. miniseed'''
if os.path.basename(file).startswith('.'):
@@ -117,11 +130,12 @@ def sendable(file):
return True
return False
-#Basic Miniseed file metadata extracted from filename
-import re
-#This includes .p files which we may accept in the future but won't h
+
+# Basic Miniseed file metadata extracted from filename
+# This includes .p files which we may accept in the future but won't h
MseedRE = re.compile(r'\A(.*)\.([A-Z0-9][A-Z0-9])\.(.*)\.([A-Z][A-Z]\w)\.([0-9]{4})\.([0-9]{3})(?:\.p)*')
+
def filename_qc_format(file):
'''Used to filter filename if correct for qc system'''
return MseedRE.match(os.path.basename(file))
@@ -129,15 +143,16 @@ def filename_qc_format(file):
def format_size(num):
'''Format bytes into human readble with suffix'''
- for suffix in [ 'bytes', 'KB', 'MB', 'GB' ]:
+ for suffix in ['bytes', 'KB', 'MB', 'GB']:
if num < 1024.0:
return '%3.3f %s' % (num, suffix)
num /= 1024.0
return '%3.3f %s' % (num, 'TB')
+
def ismseed(file):
try:
- ms = open(file,'rb')
+ ms = open(file, 'rb')
except:
return None
order = ByteOrder(ms)
@@ -149,24 +164,26 @@ def ismseed(file):
return False
#########################################################
-#needs import sys,struct
-#Taken from Bruces LibTrace
-def ByteOrder(infile, seekval=20) :
+# needs import sys,struct
+# Taken from Bruces LibTrace
+
+
+def ByteOrder(infile, seekval=20):
"""
read file as if it is mseed just pulling time info
from fixed header and determine if it makes sense unpacked
as big endian or little endian
"""
Order = "unknown"
- try :
- #seek to timeblock and read
+ try:
+ # seek to timeblock and read
infile.seek(seekval)
- timeblock=infile.read(10)
+ timeblock = infile.read(10)
- #assume big endian
- (Year, Day, Hour, Min, Sec, junk, Micro)=\
- struct.unpack('>HHBBBBH',timeblock)
- #test if big endian read makes sense
+ # assume big endian
+ (Year, Day, Hour, Min, Sec, junk, Micro) =\
+ struct.unpack('>HHBBBBH', timeblock)
+ # test if big endian read makes sense
if 1950 <= Year <= 2050 and \
1 <= Day <= 366 and \
0 <= Hour <= 23 and \
@@ -174,23 +191,24 @@ def ByteOrder(infile, seekval=20) :
0 <= Sec <= 59:
Order = "big"
else:
- #try little endian read
- (Year, Day, Hour, Min, Sec, junk, Micro)=\
- struct.unpack('<HHBBBBH',timeblock)
- #test if little endian read makes sense
+ # try little endian read
+ (Year, Day, Hour, Min, Sec, junk, Micro) =\
+ struct.unpack('<HHBBBBH', timeblock)
+ # test if little endian read makes sense
if 1950 <= Year <= 2050 and \
1 <= Day <= 366 and \
0 <= Hour <= 23 and \
0 <= Min <= 59 and \
0 <= Sec <= 59:
Order = "little"
- except Exception as e:
+ except Exception:
pass
return Order
#########################################################
-def get_sent_file_list(sentfile = SENTFILE):
+
+def get_sent_file_list(sentfile=SENTFILE):
sentlist = []
if os.path.isfile(sentfile):
logger.debug('Using sentfile %s' % sentfile)
@@ -204,7 +222,8 @@ def get_sent_file_list(sentfile = SENTFILE):
sentlist = pickle.load(f)
return sentlist
-def write_sent_file_list(sentlist , sentfile = SENTFILE ):
+
+def write_sent_file_list(sentlist, sentfile=SENTFILE):
logger.info('Saving list of files sent to passcal')
with open(sentfile, 'wb+') as f:
pickle.dump(sentlist, f, protocol=2)
@@ -217,8 +236,8 @@ def get_FTP():
trys += 1
try:
import socket
- logger.info('Connecting to FTP host %s from %s. Attempt %d of %d' % ( FTP_HOST, socket.gethostbyname(socket.gethostname()), trys, FTP_CONNECT_ATTEMPTS))
- FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
+ logger.info('Connecting to FTP host %s from %s. Attempt %d of %d' % (FTP_HOST, socket.gethostbyname(socket.gethostname()), trys, FTP_CONNECT_ATTEMPTS))
+ FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
FTP.set_debuglevel(FTP_DEBUG_LEVEL)
FTP.cwd(FTP_DIR)
FTP.set_pasv(True)
@@ -234,29 +253,36 @@ def get_FTP():
logger.error('Giving up.')
return None
+
def test_network():
passcal_http_reachable()
google_http_reachable()
passcal_ftp_reachable()
+
def passcal_http_reachable():
'''download and time passcal home page'''
url = 'http://www.passcal.nmt.edu/'
return url_reachable(url=url)
+
def google_http_reachable():
'''download and time passcal home page'''
url = 'http://www.google.com/'
return url_reachable(url=url)
+
def passcal_ftp_reachable():
'''Download a small file from passcals general ftp'''
url = 'ftp://ftp.passcal.nmt.edu/download/public/test.dat'
return url_reachable(url=url)
-def url_reachable(url = 'http://www.passcal.nmt.edu/'):
+
+def url_reachable(url='http://www.passcal.nmt.edu/'):
'''fetches a url returns True or False'''
- import urllib.request, urllib.error, urllib.parse
+ import urllib.request
+ import urllib.error
+ import urllib.parse
start = time()
try:
f = urllib.request.urlopen(url)
@@ -264,20 +290,20 @@ def url_reachable(url = 'http://www.passcal.nmt.edu/'):
logger.error("Failed to open connection to %s" % url)
logger.error(e)
return False
- logger.info('connection made to %s in %f sec' % ( url, time() - start) )
+ logger.info('connection made to %s in %f sec' % (url, time() - start))
data = f.read()
runtime = time() - start
size = len(data)
- rate = size/runtime
- logger.info('%d B/sec in %f sec' % (rate, runtime) )
+ rate = size / runtime
+ logger.info('%d B/sec in %f sec' % (rate, runtime))
return True
def test_FTP(FTP):
try:
- assert isinstance(FTP, ftplib.FTP)
+ assert isinstance(FTP, FTPCache)
FTP.voidcmd('NOOP')
- except ftplib.all_errors as e :
+ except ftplib.all_errors as e:
logger.error(e)
return False
except AssertionError as e:
@@ -293,11 +319,11 @@ def send2passcal(mslist, sentlist=None):
# Handle SIGINT while in this function: to gracefully close connection and save sentlist to disk file
def signal_handler(signum, frame):
print()
- logger.info('Caught interrupt while FTPing. Aborting transfer %s' % current_file )
- logger.info('Sent %d of %d' % (num_sent, num_to_send) )
- logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)) )
- logger.info('%s /sec' % ( format_size(size_sent / (time() - starttime )) ) )
- logger.info('Ran for %f sec' % (time()-starttime) )
+ logger.info('Caught interrupt while FTPing. Aborting transfer %s' % current_file)
+ logger.info('Sent %d of %d' % (num_sent, num_to_send))
+ logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)))
+ logger.info('%s /sec' % (format_size(size_sent / (time() - starttime))))
+ logger.info('Ran for %f sec' % (time() - starttime))
write_sent_file_list(sentlist)
try:
if FTP:
@@ -311,16 +337,16 @@ def send2passcal(mslist, sentlist=None):
'''Updates the terminal display.'''
signal.signal(signal.SIGINT, signal_handler)
update.bytes_sent += len(data)
- print('\r' + str(PB) + ' %s /sec ' % ( format_size(size_sent / (time() - starttime )) ), end=' ')
+ print('\r' + str(PB) + ' %s /sec ' % (format_size(size_sent / (time() - starttime))), end=' ')
'''
- print '%s %0.2f%%. %0.10d / %0.10d.' % ( current_file.center(20),
- (update.bytes_sent / update.file_size)*100,
- update.bytes_sent,
- file_size,
- ) ,
+ print '%s %0.2f%%. %0.10d / %0.10d.' % ( current_file.center(20),
+ (update.bytes_sent / update.file_size)*100,
+ update.bytes_sent,
+ file_size,
+ ) ,
'''
- ETA_sec = ((time() - starttime) / size_sent ) * (size_to_send - size_sent)
- print('ETA %s %s %s' % ( str(datetime.timedelta(seconds=(ETA_sec))), current_file.center(20), ' '*20 ), end=' ')
+ ETA_sec = ((time() - starttime) / size_sent) * (size_to_send - size_sent)
+ print('ETA %s %s %s' % (str(datetime.timedelta(seconds=(ETA_sec))), current_file.center(20), ' ' * 20), end=' ')
sys.stdout.flush()
if sentlist is None:
@@ -333,7 +359,7 @@ def send2passcal(mslist, sentlist=None):
size_sent = 1
current_file = ''
signal.signal(signal.SIGINT, signal_handler)
- logger.info('Sending %d, %s files to PASSCAL' % (num_to_send, format_size (size_to_send)))
+ logger.info('Sending %d, %s files to PASSCAL' % (num_to_send, format_size(size_to_send)))
FTP = get_FTP()
starttime = time()
PB = ProgressBar(num_to_send)
@@ -348,19 +374,19 @@ def send2passcal(mslist, sentlist=None):
fh = open(f, 'rb')
file_size = os.path.getsize(f)
update.file_size = float(file_size)
- FTP.storbinary('STOR %s' % current_file , fh, blocksize=FTP_BLOCKSIZE, callback=update)
+ FTP.storbinary('STOR %s' % current_file, fh, blocksize=FTP_BLOCKSIZE, callback=update)
except ftplib.error_perm as e:
# This is permission and the error when 550 for the .in file already exists so we should just continue with the next file
# todo create a list of failed files and resend those at the end instead of requiring a rerun
- logger.error( 'Failed to send file %s, permission error. Skipping...' % current_file)
+ logger.error('Failed to send file %s, permission error. Skipping...' % current_file)
logger.error(e)
break
- except (ftplib.all_errors, AttributeError) as e :
- #since we can restore with how we have proftp setup. There is nothing more we can do with this file
- #Until the server rms the .in.file
- logger.error('Failed to send file %s.' % ( current_file) ) #, trys, FTP_SEND_ATTEMPTS)
+ except (ftplib.all_errors, AttributeError) as e:
+ # since we can restore with how we have proftp setup. There is nothing more we can do with this file
+ # Until the server rms the .in.file
+ logger.error('Failed to send file %s.' % (current_file)) # , trys, FTP_SEND_ATTEMPTS)
logger.error(e)
- #if DEBUG: print "Waiting %d..." % FTP_TIMEOUT; sleep(FTP_TIMEOUT)
+ # if DEBUG: print "Waiting %d..." % FTP_TIMEOUT; sleep(FTP_TIMEOUT)
try:
if FTP:
FTP.abort()
@@ -379,16 +405,18 @@ def send2passcal(mslist, sentlist=None):
break
print()
- logger.info( 'Sent %d of %d' % (num_sent, num_to_send) )
- logger.info( 'Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)) )
- logger.info( '%s /sec' % ( format_size(size_sent / (time() - starttime )) ) )
- logger.info( 'Ran for %f sec' % (time()-starttime) )
+ logger.info('Sent %d of %d' % (num_sent, num_to_send))
+ logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)))
+ logger.info('%s /sec' % (format_size(size_sent / (time() - starttime))))
+ logger.info('Ran for %f sec' % (time() - starttime))
if FTP and test_FTP(FTP):
FTP.quit()
########################################################
# From progressbar 3rd party class will eventually dump not awesome
-class ProgressBar:
+
+
+class ProgressBar(object):
def __init__(self, duration):
self.duration = duration
self.prog_bar = '[]'
@@ -408,7 +436,7 @@ class ProgressBar:
def update_time(self, elapsed_secs):
self.__update_amount((elapsed_secs / float(self.duration)) * 100.0)
- #self.prog_bar += ' %d/%s files' % (elapsed_secs, self.duration)
+ # self.prog_bar += ' %d/%s files' % (elapsed_secs, self.duration)
def __update_amount(self, new_amount):
percent_done = int(round((new_amount / 100.0) * 100.0))
@@ -425,8 +453,10 @@ class ProgressBar:
# End progressbar
########################################################
+
+
def main():
- if len(sys.argv) < 2 or sys.argv[1] in [ '-h', '--help', '-?']:
+ if len(sys.argv) < 2 or sys.argv[1] in ['-h', '--help', '-?']:
print(HELP)
sys.exit()
# find all files below dir
@@ -435,19 +465,20 @@ def main():
logger.info('Removing improperly named files.')
msnamedlist = list(filter(filename_qc_format, filelist))
logger.info('Properly named files files: %d' % len(msnamedlist))
- logger.info('Other files: %d'% (len(filelist) - len(msnamedlist)))
+ logger.info('Other files: %d' % (len(filelist) - len(msnamedlist)))
print()
logger.info('Removing files that are not Miniseed.')
mslist = list(filter(sendable, msnamedlist))
logger.info('MiniSEED files: %d' % len(mslist))
- logger.info('Properly named but not miniseed files: %d'% (len(msnamedlist) - len(mslist)))
+ logger.info('Properly named but not miniseed files: %d' % (len(msnamedlist) - len(mslist)))
print()
logger.info('Removing files already sent to PASSCAL')
sentlist = get_sent_file_list()
unsentms = [f for f in mslist if f not in sentlist]
- logger.info('%d miniSEED files have already been sent, not resending.' % (len( mslist) - len(unsentms)))
+ logger.info('%d miniSEED files have already been sent, not resending.' % (len(mslist) - len(unsentms)))
send2passcal(unsentms, sentlist)
write_sent_file_list(sentlist)
+
if __name__ == '__main__':
main()
diff --git a/setup.cfg b/setup.cfg
index bd5e695214d2220c9df4bb5f53418783cc3037b2..379997ac164d680ab7b7db737261a1d12e2e1456 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 2018.228
+current_version = 2020.114
commit = True
tag = True
@@ -19,4 +19,3 @@ exclude = docs
[aliases]
# Define setup.py command aliases here
-
diff --git a/setup.py b/setup.py
index 3c7b133727cf7718141b7ce8e55399c30d670526..40dc85e039b2f3de2a3840f96862de35314d2833 100644
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,7 @@ setup(
],
},
install_requires=[],
- setup_requires = [],
+ setup_requires=[],
extras_require={
'dev': [
'pip',
@@ -42,6 +42,7 @@ setup(
'coverage',
'Sphinx',
'twine',
+ "mock;python_version<'3.3'"
]
},
license="GNU General Public License v3",
@@ -50,8 +51,7 @@ setup(
keywords='data2passcal',
name='data2passcal',
packages=find_packages(include=['data2passcal']),
- test_suite='tests',
url='https://git.passcal.nmt.edu/passoft/data2passcal',
- version='2018.228',
+ version='2020.114',
zip_safe=False,
)
diff --git a/tests/test_data/ST00.AB..BHZ.2007.160 b/tests/test_data/ST00.AB..BHZ.2007.160
new file mode 100644
index 0000000000000000000000000000000000000000..668ccee282506f7101f4f6de9e5175a3103d0043
Binary files /dev/null and b/tests/test_data/ST00.AB..BHZ.2007.160 differ
diff --git a/tests/test_data/ST00.AB..BHZ.2007.161 b/tests/test_data/ST00.AB..BHZ.2007.161
new file mode 100644
index 0000000000000000000000000000000000000000..f3fa1ece76f3655d59e123b5d92c235b4f5e9e4a
Binary files /dev/null and b/tests/test_data/ST00.AB..BHZ.2007.161 differ
diff --git a/tests/test_data/ST00.AB..BHZ.2007.162 b/tests/test_data/ST00.AB..BHZ.2007.162
new file mode 100644
index 0000000000000000000000000000000000000000..75c29514caed9cc89925cc00f5edca42c11a3950
Binary files /dev/null and b/tests/test_data/ST00.AB..BHZ.2007.162 differ
diff --git a/tests/test_data/ST00.AB..BHZ.2007.163 b/tests/test_data/ST00.AB..BHZ.2007.163
new file mode 100644
index 0000000000000000000000000000000000000000..907b07ca7097a791aa3c346e26df10f9c7788851
Binary files /dev/null and b/tests/test_data/ST00.AB..BHZ.2007.163 differ
diff --git a/tests/test_data/ST00.AB..BHZ.2007.164 b/tests/test_data/ST00.AB..BHZ.2007.164
new file mode 100644
index 0000000000000000000000000000000000000000..b925f35dace4e038f204d9b993113d9318f277e5
Binary files /dev/null and b/tests/test_data/ST00.AB..BHZ.2007.164 differ
diff --git a/tests/test_data2passcal.py b/tests/test_data2passcal.py
index e102d31f6b7762155c53b59bb37b4039d921533f..488878408c69d8af205c6078e4c48dbb111b9f4a 100644
--- a/tests/test_data2passcal.py
+++ b/tests/test_data2passcal.py
@@ -3,26 +3,98 @@
"""Tests for `data2passcal` package."""
+from __future__ import division, print_function
+
+import ftplib
+import os
import unittest
-import sys
+
+from data2passcal.data2passcal import get_FTP, ismseed, scan_dir, send2passcal
try:
- import data2passcal
+ from unittest.mock import patch
except ImportError:
- pass
+ from mock import patch
+
+VERSION = '2020.114'
+
+FTP_HOST = 'qc.passcal.nmt.edu'
+FTP_USER = 'ftp'
+FTP_PASSWORD = 'data2passcal'
+FTP_DIR = 'AUTO/MSEED'
+FTP_TIMEOUT = 120
+FTP_RECONNECT_WAIT = 60
+FTP_CONNECT_ATTEMPTS = 60 * 60 * 24 * 7 / FTP_RECONNECT_WAIT
+FTP_SEND_ATTEMPTS = 3
+MOCK_TEST = True
+
class TestData2passcal(unittest.TestCase):
"""Tests for `data2passcal` package."""
def setUp(self):
- """Set up test fixtures, if any."""
+ """Set up test fixtures, if any"""
+ dir_testdata = os.path.dirname(os.path.realpath(__file__)) + '/test_data'
+ filelist = [x for x in scan_dir(dir_testdata) if not os.path.basename(x).startswith('.') and not os.path.basename(x).endswith('.log')]
+ self.dir_testdata = dir_testdata
+ self.filelist = filelist
+
+ def test_scan_dir(self):
+ """
+ Test basic functionality of scan_dir function
+ Only 5 files available under ./test_data directory
+ Not taking into account .DS_STORE file
+ """
+ self.assertEqual(len(self.filelist), 5, 'Incorrect number of files')
+
+ def test_ismseed(self):
+ """Test basic functionality of ismseed function"""
+ for f in self.filelist:
+ self.assertTrue(ismseed(f), '{} is not a miniseed file'.format(os.path.basename(f)))
+
+ @patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
+ def test_get_FTP_mock(self, mock_ftp_constructor):
+ """Mock test creating ftp connection to PASSCAL"""
+ mock_ftp = mock_ftp_constructor.return_value
+ get_FTP()
+ mock_ftp_constructor.assert_called_with(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
+ mock_ftp.cwd.assert_called_with(FTP_DIR)
+ self.assertLess(mock_ftp_constructor.call_count, FTP_CONNECT_ATTEMPTS, 'Number of ftp connection attempts exceeeds {}'.format(FTP_CONNECT_ATTEMPTS))
+ mock_ftp.quit()
+
+ @patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
+ def test_send_data_mock(self, mock_ftp_constructor):
+ """Mock test sending MSEED files (test data) to PASSCAL's QC system"""
+ mock_ftp = mock_ftp_constructor.return_value
+ send2passcal(self.filelist)
+ self.assertTrue(mock_ftp.storbinary.called, 'No data sent')
+ self.assertEqual(mock_ftp.storbinary.call_count, len(self.filelist), 'Failed to send all files - Sent {0} of {1}'.format(mock_ftp.storbinary.call_count, len(self.filelist)))
+ files_sent = []
+ for x in mock_ftp.storbinary.call_args_list:
+ args, kwargs = x
+ files_sent.append(args[0].split(' ')[1])
+ for f in self.filelist:
+ f = os.path.basename(f)
+ self.assertLess(files_sent.count(f), FTP_SEND_ATTEMPTS, 'Attempted to send file {0} more than {1} times'.format(f, FTP_SEND_ATTEMPTS))
- def tearDown(self):
- """Tear down test fixtures, if any."""
+ @unittest.skipIf(MOCK_TEST == True, "skipping real send2passcal test")
+ def test_send_data(self):
+ """Test sending MSEED files (test data) to PASSCAL's QC system - Optional"""
+ ftp = get_FTP()
+ send2passcal(self.filelist)
+ wdir = ftp.pwd()
+ try:
+ files_sent = [os.path.basename(x) for x in ftp.nlst(wdir)]
+ except ftplib.error_perm() as resp:
+ if str(resp) == "550 No files found":
+ print("No files found in this directory")
+ else:
+ raise
+ for f in self.filelist:
+ f_ = os.path.basename(f)
+ self.assertIn(f_, files_sent, 'File {} was not sent to PASSCAL'.format(f_))
+ ftp.quit()
- def test_import(self):
- if 'data2passcal' in sys.modules:
- self.assert_(True, "data2passcal loaded")
- else:
- self.fail()
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tox.ini b/tox.ini
index 60d2b1bb52fdc8539b7f39176385af7599226b5a..d1ee218c28cea282e06e2caa6de96076edaf5e98 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist = py27, py36 flake8
+envlist = py27, py36, flake8
[travis]
python =
@@ -9,11 +9,13 @@ python =
[testenv:flake8]
basepython = python
deps = flake8
-commands = flake8 data2passcal
-
-[testenv]
-setenv =
- PYTHONPATH = {toxinidir}
-commands=python setup.py test
+commands = flake8 --ignore=E722,E712,E501 data2passcal
+ flake8 --ignore=E722,E712,E501 tests
+[testenv:py27]
+changedir = tests
+deps = mock
+commands = python -m unittest test_data2passcal
+[testenv:py36]
+commands = python -m unittest