Skip to content
Snippets Groups Projects
Commit a717a608 authored by Maeva Pourpoint's avatar Maeva Pourpoint
Browse files

Remove FTPCache and TESTMODE; Clean up test_data2passcal; Remove call to... 

Remove FTPCache and TESTMODE; Clean up test_data2passcal; Remove call to unused liraries (e.g. bumpversion); Check for E501 when running flake8.
parent 6c03bb86
No related branches found
No related tags found
1 merge request!3Python 2/3 compatibility
Pipeline #698 passed with stage
in 37 seconds
Hide whitespace changes
Inline Side-by-side
Makefile
+ 16
− 36
View file @ a717a608
......@@ -29,61 +29,41 @@ BROWSER := python -c "$$BROWSER_PYSCRIPT"
help:
@python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
## remove all build, test, coverage and Python artifacts
clean: clean-build clean-pyc clean-test
clean-build: ## remove build artifacts
## remove build artifacts
clean-build:
rm -fr build/
rm -fr dist/
rm -fr .eggs/
find . -name '*.egg-info' -exec rm -fr {} +
find . -name '*.egg' -exec rm -f {} +
clean-pyc: ## remove Python file artifacts
## remove Python file artifacts
clean-pyc:
find . -name '*.pyc' -exec rm -f {} +
find . -name '*.pyo' -exec rm -f {} +
find . -name '*~' -exec rm -f {} +
find . -name '__pycache__' -exec rm -fr {} +
clean-test: ## remove test and coverage artifacts
## remove test artifacts
clean-test:
rm -fr .tox/
rm -f .coverage
rm -fr htmlcov/
rm -fr .pytest_cache
lint: ## check style with flake8
## check style with flake8
lint:
flake8 data2passcal tests
test: ## run tests quickly with the default Python
## run tests quickly with the default Python - deprecated
test:
python setup.py test
test-all: ## run tests on every Python version with tox
## run tests on every Python version with tox
test-all:
tox
coverage: ## check code coverage quickly with the default Python
coverage run --source data2passcal setup.py test
coverage report -m
coverage html
$(BROWSER) htmlcov/index.html
docs: ## generate Sphinx HTML documentation, including API docs
rm -f docs/data2passcal.rst
rm -f docs/modules.rst
sphinx-apidoc -o docs/ data2passcal
$(MAKE) -C docs clean
$(MAKE) -C docs html
$(BROWSER) docs/_build/html/index.html
servedocs: docs ## compile the docs watching for changes
watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
release: dist ## package and upload a release
twine upload dist/*
dist: clean ## builds source and wheel package
python setup.py sdist
python setup.py bdist_wheel
ls -l dist
install: clean ## install the package to the active Python's site-packages
## install the package to the active Python's site-packages
install: clean
python setup.py install
data2passcal/data2passcal.py
+ 74
− 50
View file @ a717a608
......@@ -21,13 +21,6 @@ from time import sleep, time
VERSION = '2020.107'
# Cache the ftplib.FTP class so it will be available to test_FTP(FTP) when calling isinstance assert
FTPCache = ftplib.FTP
# TEST related
TESTMODE = False
# FTP related
FTP_IP = '129.138.26.29'
FTP_HOST = 'qc.passcal.nmt.edu'
......@@ -36,10 +29,6 @@ FTP_PASSWORD = 'data2passcal'
FTP_DIR = 'AUTO/MSEED'
FTP_TIMEOUT = 120
FTP_RECONNECT_WAIT = 60
if TESTMODE:
FTP_DIR = 'AUTO/TEST'
FTP_TIMEOUT = 5
FTP_RECONNECT_WAIT = 5
FTP_BLOCKSIZE = 8192
# number of time to try to open the ftp connection
FTP_CONNECT_ATTEMPTS = 60 * 60 * 24 * 7 / FTP_RECONNECT_WAIT
......@@ -52,19 +41,23 @@ FTP_DEBUG_LEVEL = 0
LOGFILE = 'data2passcal.log'
# store the files sent in ~/data2passcal.sent
SENTFILE = os.path.join(os.path.expanduser('~'), '.data2passcal.sent')
# If this file exists open it, incorporate, and save to new name, and delete old
# If this file exists open it, incorporate and save to new name, and delete old
SENTFILE_OLD = os.path.join(os.path.expanduser('~'), '.send2passcal.sent')
HELP = '''
data2passcal
VERSION: %s
Usage: data2passcal dir
data2passcal is a utility that sends day-long MSEED files ready for archival at the DMC to PASSCAL's QC system by:
Scanning all files below directory dir.
Filtering out non-miniseed files, by inspecting the first blockette of each file.
Sending the files to the automated system for ingestion into the QC system via ftp.
data2passcal is a utility that sends day-long MSEED files ready for archival
at the DMC to PASSCAL's QC system by:
- Scanning all files below directory dir.
- Filtering out non-miniseed files, by inspecting the first blockette of each
file.
- Sending the files to the automated system for ingestion into the QC system
via ftp.
You can send a SIGTERM (ctl-c) to data2passcal and it will shutdown cleanly.
A list of sent files is kept in ~/.data2passcal.sent. Subsequent runs of send2passcal will not send files already sent.
A list of sent files is kept in ~/.data2passcal.sent.
Subsequent runs of send2passcal will not send files already sent.
A log is stored in data2passcal.log in the current directory.
''' % VERSION
......@@ -72,13 +65,16 @@ Usage: data2passcal dir
DEBUG = logging.DEBUG
# Configure logging file and stdout
# todo if sending a file do we want to send all previous logs even if already sent
# todo if sending a file do we want to send all previous logs even if already
# sent
logger = logging.getLogger('__name__')
logger.setLevel(logging.DEBUG)
# Log to a file with debug level
logfh = logging.FileHandler(LOGFILE)
logfh.setLevel(logging.DEBUG)
logfh.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s -\t%(message)s', datefmt='%Y-%m-%d %H:%M:%S %z'))
logfh.setFormatter(logging.Formatter(
'%(asctime)s - %(levelname)s -\t%(message)s',
datefmt='%Y-%m-%d %H:%M:%S %z'))
logger.addHandler(logfh)
# Log to stdout
logconsole = logging.StreamHandler()
......@@ -123,7 +119,7 @@ def scan_dir(dir):
def sendable(file):
'''Filters files scanned returning a new list of files to send i.e. miniseed'''
'''Filters files scanned returning a new list of files to send'''
if os.path.basename(file).startswith('.'):
return False
if ismseed(file):
......@@ -133,7 +129,8 @@ def sendable(file):
# Basic Miniseed file metadata extracted from filename
# This includes .p files which we may accept in the future but won't h
MseedRE = re.compile(r'\A(.*)\.([A-Z0-9][A-Z0-9])\.(.*)\.([A-Z][A-Z]\w)\.([0-9]{4})\.([0-9]{3})(?:\.p)*')
MseedRE = re.compile(
r'\A(.*)\.([A-Z0-9][A-Z0-9])\.(.*)\.([A-Z][A-Z]\w)\.([0-9]{4})\.([0-9]{3})(?:\.p)*') # noqa
def filename_qc_format(file):
......@@ -230,14 +227,22 @@ def write_sent_file_list(sentlist, sentfile=SENTFILE):
def get_FTP():
'''returns a FTP connection or None if a connection could not be made after attempts allowed'''
'''
returns a FTP connection or None if a connection could not be made
after number of attempts allowed
'''
trys = 0
while trys < FTP_CONNECT_ATTEMPTS:
trys += 1
try:
import socket
logger.info('Connecting to FTP host %s from %s. Attempt %d of %d' % (FTP_HOST, socket.gethostbyname(socket.gethostname()), trys, FTP_CONNECT_ATTEMPTS))
FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
logger.info('Connecting to FTP host %s from %s. Attempt %d of %d'
% (FTP_HOST,
socket.gethostbyname(socket.gethostname()),
trys,
FTP_CONNECT_ATTEMPTS))
FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER,
passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
FTP.set_debuglevel(FTP_DEBUG_LEVEL)
FTP.cwd(FTP_DIR)
FTP.set_pasv(True)
......@@ -245,7 +250,8 @@ def get_FTP():
logger.error('Failed to open FTP connection to %s' % FTP_HOST)
logger.error(e)
test_network()
logger.info('Waiting %d seconds before trying to reconnect...' % FTP_RECONNECT_WAIT)
logger.info('Waiting %d seconds before trying to reconnect...'
% FTP_RECONNECT_WAIT)
sleep(FTP_RECONNECT_WAIT)
else:
logger.info('Success: Connected to PASSCAL FTP')
......@@ -301,7 +307,6 @@ def url_reachable(url='http://www.passcal.nmt.edu/'):
def test_FTP(FTP):
try:
assert isinstance(FTP, FTPCache)
FTP.voidcmd('NOOP')
except ftplib.all_errors as e:
logger.error(e)
......@@ -316,13 +321,17 @@ def test_FTP(FTP):
def send2passcal(mslist, sentlist=None):
'''Send the list of files in mslist to passcal via FTP'''
# Handle SIGINT while in this function: to gracefully close connection and save sentlist to disk file
# Handle SIGINT while in this function: to gracefully close connection and
# save sentlist to disk file
def signal_handler(signum, frame):
print()
logger.info('Caught interrupt while FTPing. Aborting transfer %s' % current_file)
logger.info('Caught interrupt while FTPing. Aborting transfer %s'
% current_file)
logger.info('Sent %d of %d' % (num_sent, num_to_send))
logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)))
logger.info('%s /sec' % (format_size(size_sent / (time() - starttime))))
logger.info('Sent %s of %s'
% (format_size(size_sent), format_size(size_to_send)))
logger.info('%s /sec'
% (format_size(size_sent / (time() - starttime))))
logger.info('Ran for %f sec' % (time() - starttime))
write_sent_file_list(sentlist)
try:
......@@ -337,16 +346,19 @@ def send2passcal(mslist, sentlist=None):
'''Updates the terminal display.'''
signal.signal(signal.SIGINT, signal_handler)
update.bytes_sent += len(data)
print('\r' + str(PB) + ' %s /sec ' % (format_size(size_sent / (time() - starttime))), end=' ')
print('\r' + str(PB) + ' %s /sec '
% (format_size(size_sent / (time() - starttime))), end=' ')
'''
print '%s %0.2f%%. %0.10d / %0.10d.' % ( current_file.center(20),
(update.bytes_sent / update.file_size)*100,
update.bytes_sent,
file_size,
) ,
print '%s %0.2f%%. %0.10d / %0.10d.'
% (current_file.center(20),
(update.bytes_sent / update.file_size) * 100,
update.bytes_sent,
file_size)
'''
ETA_sec = ((time() - starttime) / size_sent) * (size_to_send - size_sent)
print('ETA %s %s %s' % (str(datetime.timedelta(seconds=(ETA_sec))), current_file.center(20), ' ' * 20), end=' ')
ETA_sec = ((time() - starttime) / size_sent) * \
(size_to_send - size_sent)
print('ETA %s %s %s' % (str(datetime.timedelta(seconds=(ETA_sec))),
current_file.center(20), ' ' * 20), end=' ')
sys.stdout.flush()
if sentlist is None:
......@@ -359,7 +371,8 @@ def send2passcal(mslist, sentlist=None):
size_sent = 1
current_file = ''
signal.signal(signal.SIGINT, signal_handler)
logger.info('Sending %d, %s files to PASSCAL' % (num_to_send, format_size(size_to_send)))
logger.info('Sending %d, %s files to PASSCAL' %
(num_to_send, format_size(size_to_send)))
FTP = get_FTP()
starttime = time()
PB = ProgressBar(num_to_send)
......@@ -374,19 +387,26 @@ def send2passcal(mslist, sentlist=None):
fh = open(f, 'rb')
file_size = os.path.getsize(f)
update.file_size = float(file_size)
FTP.storbinary('STOR %s' % current_file, fh, blocksize=FTP_BLOCKSIZE, callback=update)
FTP.storbinary('STOR %s' % current_file, fh,
blocksize=FTP_BLOCKSIZE, callback=update)
except ftplib.error_perm as e:
# This is permission and the error when 550 for the .in file already exists so we should just continue with the next file
# todo create a list of failed files and resend those at the end instead of requiring a rerun
logger.error('Failed to send file %s, permission error. Skipping...' % current_file)
# This is permission and the error when 550 for the .in file
# already exists so we should just continue with the next file
# todo create a list of failed files and resend those at the
# end instead of requiring a rerun
logger.error('Failed to send file %s, permission error.'
% current_file)
logger.error(e)
break
except (ftplib.all_errors, AttributeError) as e:
# since we can restore with how we have proftp setup. There is nothing more we can do with this file
# since we can restore with how we have proftp setup.
# There is nothing more we can do with this file
# Until the server rms the .in.file
logger.error('Failed to send file %s.' % (current_file)) # , trys, FTP_SEND_ATTEMPTS)
# , trys, FTP_SEND_ATTEMPTS)
logger.error('Failed to send file %s.' % (current_file))
logger.error(e)
# if DEBUG: print "Waiting %d..." % FTP_TIMEOUT; sleep(FTP_TIMEOUT)
# if DEBUG:
# print "Waiting %d..." %FTP_TIMEOUT; sleep(FTP_TIMEOUT)
try:
if FTP:
FTP.abort()
......@@ -406,7 +426,8 @@ def send2passcal(mslist, sentlist=None):
print()
logger.info('Sent %d of %d' % (num_sent, num_to_send))
logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)))
logger.info('Sent %s of %s'
% (format_size(size_sent), format_size(size_to_send)))
logger.info('%s /sec' % (format_size(size_sent / (time() - starttime))))
logger.info('Ran for %f sec' % (time() - starttime))
if FTP and test_FTP(FTP):
......@@ -442,7 +463,8 @@ class ProgressBar(object):
percent_done = int(round((new_amount / 100.0) * 100.0))
all_full = self.width - 2
num_hashes = int(round((percent_done / 100.0) * all_full))
self.prog_bar = '[' + self.fill_char * num_hashes + ' ' * (all_full - num_hashes) + ']'
self.prog_bar = '[' + self.fill_char * \
num_hashes + ' ' * (all_full - num_hashes) + ']'
pct_place = int((len(self.prog_bar) / 2) - len(str(percent_done)))
pct_string = '%d%%' % percent_done
self.prog_bar = self.prog_bar[0:pct_place] + \
......@@ -470,12 +492,14 @@ def main():
logger.info('Removing files that are not Miniseed.')
mslist = list(filter(sendable, msnamedlist))
logger.info('MiniSEED files: %d' % len(mslist))
logger.info('Properly named but not miniseed files: %d' % (len(msnamedlist) - len(mslist)))
logger.info('Properly named but not miniseed files: %d'
% (len(msnamedlist) - len(mslist)))
print()
logger.info('Removing files already sent to PASSCAL')
sentlist = get_sent_file_list()
unsentms = [f for f in mslist if f not in sentlist]
logger.info('%d miniSEED files have already been sent, not resending.' % (len(mslist) - len(unsentms)))
logger.info('%d miniSEED files have already been sent, not resending.'
% (len(mslist) - len(unsentms)))
send2passcal(unsentms, sentlist)
write_sent_file_list(sentlist)
......
setup.cfg
+ 0
− 16
View file @ a717a608
[bumpversion]
current_version = 2020.107
commit = True
tag = True
[bumpversion:file:setup.py]
search = version='{current_version}'
replace = version='{new_version}'
[bumpversion:file:data2passcal/__init__.py]
search = __version__ = '{current_version}'
replace = __version__ = '{new_version}'
[bdist_wheel]
universal = 1
[flake8]
exclude = docs
......
setup.py
+ 0
− 7
View file @ a717a608
......@@ -33,15 +33,8 @@ setup(
setup_requires=[],
extras_require={
'dev': [
'pip',
'bumpversion',
'wheel',
'watchdog',
'flake8',
'tox',
'coverage',
'Sphinx',
'twine',
"mock;python_version<'3.3'"
]
},
......
tests/test_data2passcal.py
+ 28
− 22
View file @ a717a608
......@@ -7,25 +7,19 @@ from __future__ import division, print_function
import ftplib
import os
import sys
import unittest
from data2passcal.data2passcal import get_FTP, ismseed, scan_dir, send2passcal
from data2passcal.data2passcal import FTP_CONNECT_ATTEMPTS, FTP_SEND_ATTEMPTS
try:
from unittest.mock import patch
except ImportError:
if sys.version_info < (3, 3):
from mock import patch
else:
from unittest.mock import patch
VERSION = '2020.107'
FTP_HOST = 'qc.passcal.nmt.edu'
FTP_USER = 'ftp'
FTP_PASSWORD = 'data2passcal'
FTP_DIR = 'AUTO/MSEED'
FTP_TIMEOUT = 120
FTP_RECONNECT_WAIT = 60
FTP_CONNECT_ATTEMPTS = 60 * 60 * 24 * 7 / FTP_RECONNECT_WAIT
FTP_SEND_ATTEMPTS = 3
MOCK_TEST = True
......@@ -34,8 +28,10 @@ class TestData2passcal(unittest.TestCase):
def setUp(self):
"""Set up test fixtures, if any"""
dir_testdata = os.path.dirname(os.path.realpath(__file__)) + '/test_data'
filelist = [x for x in scan_dir(dir_testdata) if not os.path.basename(x).startswith('.') and not os.path.basename(x).endswith('.log')]
dir_testdata = os.path.dirname(os.path.realpath(__file__)) + '/test_data' # noqa
filelist = [x for x in scan_dir(dir_testdata)
if not os.path.basename(x).startswith('.') and not
os.path.basename(x).endswith('.log')]
self.dir_testdata = dir_testdata
self.filelist = filelist
......@@ -45,21 +41,25 @@ class TestData2passcal(unittest.TestCase):
Only 5 files available under ./test_data directory
Not taking into account .DS_STORE file
"""
self.assertEqual(len(self.filelist), 5, 'Incorrect number of files')
filelist = [x for x in scan_dir(self.dir_testdata)
if not os.path.basename(x).startswith('.') and not
os.path.basename(x).endswith('.log')]
self.assertEqual(len(filelist), 5, 'Incorrect number of files')
def test_ismseed(self):
"""Test basic functionality of ismseed function"""
for f in self.filelist:
self.assertTrue(ismseed(f), '{} is not a miniseed file'.format(os.path.basename(f)))
self.assertTrue(ismseed(f), '{} is not a miniseed file'
.format(os.path.basename(f)))
@patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
def test_get_FTP_mock(self, mock_ftp_constructor):
"""Mock test creating ftp connection to PASSCAL"""
mock_ftp = mock_ftp_constructor.return_value
get_FTP()
mock_ftp_constructor.assert_called_with(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
mock_ftp.cwd.assert_called_with(FTP_DIR)
self.assertLess(mock_ftp_constructor.call_count, FTP_CONNECT_ATTEMPTS, 'Number of ftp connection attempts exceeeds {}'.format(FTP_CONNECT_ATTEMPTS))
self.assertLess(mock_ftp_constructor.call_count, FTP_CONNECT_ATTEMPTS,
'Number of ftp connection attempts exceeeds {}'
.format(FTP_CONNECT_ATTEMPTS))
mock_ftp.quit()
@patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
......@@ -68,18 +68,23 @@ class TestData2passcal(unittest.TestCase):
mock_ftp = mock_ftp_constructor.return_value
send2passcal(self.filelist)
self.assertTrue(mock_ftp.storbinary.called, 'No data sent')
self.assertEqual(mock_ftp.storbinary.call_count, len(self.filelist), 'Failed to send all files - Sent {0} of {1}'.format(mock_ftp.storbinary.call_count, len(self.filelist)))
self.assertEqual(mock_ftp.storbinary.call_count, len(self.filelist),
'Failed to send all files - Sent {0} of {1}'
.format(mock_ftp.storbinary.call_count,
len(self.filelist)))
files_sent = []
for x in mock_ftp.storbinary.call_args_list:
args, kwargs = x
files_sent.append(args[0].split(' ')[1])
for f in self.filelist:
f = os.path.basename(f)
self.assertLess(files_sent.count(f), FTP_SEND_ATTEMPTS, 'Attempted to send file {0} more than {1} times'.format(f, FTP_SEND_ATTEMPTS))
self.assertLess(files_sent.count(f), FTP_SEND_ATTEMPTS,
'Attempted to send file {0} more than {1} times'
.format(f, FTP_SEND_ATTEMPTS))
@unittest.skipIf(MOCK_TEST == True, "skipping real send2passcal test")
def test_send_data(self):
"""Test sending MSEED files (test data) to PASSCAL's QC system - Optional"""
"""Test sending MSEED files (test data) to PASSCAL's QC system"""
ftp = get_FTP()
send2passcal(self.filelist)
wdir = ftp.pwd()
......@@ -92,7 +97,8 @@ class TestData2passcal(unittest.TestCase):
raise
for f in self.filelist:
f_ = os.path.basename(f)
self.assertIn(f_, files_sent, 'File {} was not sent to PASSCAL'.format(f_))
self.assertIn(f_, files_sent, 'File {} was not sent to PASSCAL'
.format(f_))
ftp.quit()
......
tox.ini
+ 2
− 2
View file @ a717a608
......@@ -9,8 +9,8 @@ python =
[testenv:flake8]
basepython = python
deps = flake8
commands = flake8 --ignore=E722,E712,E501 data2passcal
flake8 --ignore=E722,E712,E501 tests
commands = flake8 --ignore=E722,E712 data2passcal
flake8 --ignore=E722,E712 tests
[testenv:py27]
changedir = tests
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment