From a717a6089ec78596415d59e74b506942b8670cb5 Mon Sep 17 00:00:00 2001
From: Maeva Pourpoint <maeva@passcal.nmt.edu>
Date: Mon, 11 May 2020 08:49:58 -0600
Subject: [PATCH] Remove FTPCache and TESTMODE; Clean up test_data2passcal;
Remove call to unused liraries (e.g. bumpversion); Check for E501 when
running flake8.
---
Makefile | 52 +++++----------
data2passcal/data2passcal.py | 124 +++++++++++++++++++++--------------
setup.cfg | 16 -----
setup.py | 7 --
tests/test_data2passcal.py | 50 +++++++-------
tox.ini | 4 +-
6 files changed, 120 insertions(+), 133 deletions(-)
diff --git a/Makefile b/Makefile
index 11b99c4..4e77fd3 100644
--- a/Makefile
+++ b/Makefile
@@ -29,61 +29,41 @@ BROWSER := python -c "$$BROWSER_PYSCRIPT"
help:
@python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
-clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
+## remove all build, test, coverage and Python artifacts
+clean: clean-build clean-pyc clean-test
-clean-build: ## remove build artifacts
+## remove build artifacts
+clean-build:
rm -fr build/
rm -fr dist/
rm -fr .eggs/
find . -name '*.egg-info' -exec rm -fr {} +
find . -name '*.egg' -exec rm -f {} +
-clean-pyc: ## remove Python file artifacts
+## remove Python file artifacts
+clean-pyc:
find . -name '*.pyc' -exec rm -f {} +
find . -name '*.pyo' -exec rm -f {} +
find . -name '*~' -exec rm -f {} +
find . -name '__pycache__' -exec rm -fr {} +
-clean-test: ## remove test and coverage artifacts
+## remove test artifacts
+clean-test:
rm -fr .tox/
- rm -f .coverage
- rm -fr htmlcov/
rm -fr .pytest_cache
-lint: ## check style with flake8
+## check style with flake8
+lint:
flake8 data2passcal tests
-test: ## run tests quickly with the default Python
+## run tests quickly with the default Python - deprecated
+test:
python setup.py test
-
-test-all: ## run tests on every Python version with tox
+## run tests on every Python version with tox
+test-all:
tox
-coverage: ## check code coverage quickly with the default Python
- coverage run --source data2passcal setup.py test
- coverage report -m
- coverage html
- $(BROWSER) htmlcov/index.html
-
-docs: ## generate Sphinx HTML documentation, including API docs
- rm -f docs/data2passcal.rst
- rm -f docs/modules.rst
- sphinx-apidoc -o docs/ data2passcal
- $(MAKE) -C docs clean
- $(MAKE) -C docs html
- $(BROWSER) docs/_build/html/index.html
-
-servedocs: docs ## compile the docs watching for changes
- watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
-
-release: dist ## package and upload a release
- twine upload dist/*
-
-dist: clean ## builds source and wheel package
- python setup.py sdist
- python setup.py bdist_wheel
- ls -l dist
-
-install: clean ## install the package to the active Python's site-packages
+## install the package to the active Python's site-packages
+install: clean
python setup.py install
diff --git a/data2passcal/data2passcal.py b/data2passcal/data2passcal.py
index ebdae01..6cdba1f 100644
--- a/data2passcal/data2passcal.py
+++ b/data2passcal/data2passcal.py
@@ -21,13 +21,6 @@ from time import sleep, time
VERSION = '2020.107'
-# Cache the ftplib.FTP class so it will be available to test_FTP(FTP) when calling isinstance assert
-FTPCache = ftplib.FTP
-
-
-# TEST related
-TESTMODE = False
-
# FTP related
FTP_IP = '129.138.26.29'
FTP_HOST = 'qc.passcal.nmt.edu'
@@ -36,10 +29,6 @@ FTP_PASSWORD = 'data2passcal'
FTP_DIR = 'AUTO/MSEED'
FTP_TIMEOUT = 120
FTP_RECONNECT_WAIT = 60
-if TESTMODE:
- FTP_DIR = 'AUTO/TEST'
- FTP_TIMEOUT = 5
- FTP_RECONNECT_WAIT = 5
FTP_BLOCKSIZE = 8192
# number of time to try to open the ftp connection
FTP_CONNECT_ATTEMPTS = 60 * 60 * 24 * 7 / FTP_RECONNECT_WAIT
@@ -52,19 +41,23 @@ FTP_DEBUG_LEVEL = 0
LOGFILE = 'data2passcal.log'
# store the files sent in ~/data2passcal.sent
SENTFILE = os.path.join(os.path.expanduser('~'), '.data2passcal.sent')
-# If this file exists open it, incorporate, and save to new name, and delete old
+# If this file exists open it, incorporate and save to new name, and delete old
SENTFILE_OLD = os.path.join(os.path.expanduser('~'), '.send2passcal.sent')
HELP = '''
data2passcal
VERSION: %s
Usage: data2passcal dir
- data2passcal is a utility that sends day-long MSEED files ready for archival at the DMC to PASSCAL's QC system by:
- Scanning all files below directory dir.
- Filtering out non-miniseed files, by inspecting the first blockette of each file.
- Sending the files to the automated system for ingestion into the QC system via ftp.
+ data2passcal is a utility that sends day-long MSEED files ready for archival
+ at the DMC to PASSCAL's QC system by:
+ - Scanning all files below directory dir.
+ - Filtering out non-miniseed files, by inspecting the first blockette of each
+ file.
+ - Sending the files to the automated system for ingestion into the QC system
+ via ftp.
You can send a SIGTERM (ctl-c) to data2passcal and it will shutdown cleanly.
- A list of sent files is kept in ~/.data2passcal.sent. Subsequent runs of send2passcal will not send files already sent.
+ A list of sent files is kept in ~/.data2passcal.sent.
+ Subsequent runs of send2passcal will not send files already sent.
A log is stored in data2passcal.log in the current directory.
''' % VERSION
@@ -72,13 +65,16 @@ Usage: data2passcal dir
DEBUG = logging.DEBUG
# Configure logging file and stdout
-# todo if sending a file do we want to send all previous logs even if already sent
+# todo if sending a file do we want to send all previous logs even if already
+# sent
logger = logging.getLogger('__name__')
logger.setLevel(logging.DEBUG)
# Log to a file with debug level
logfh = logging.FileHandler(LOGFILE)
logfh.setLevel(logging.DEBUG)
-logfh.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s -\t%(message)s', datefmt='%Y-%m-%d %H:%M:%S %z'))
+logfh.setFormatter(logging.Formatter(
+ '%(asctime)s - %(levelname)s -\t%(message)s',
+ datefmt='%Y-%m-%d %H:%M:%S %z'))
logger.addHandler(logfh)
# Log to stdout
logconsole = logging.StreamHandler()
@@ -123,7 +119,7 @@ def scan_dir(dir):
def sendable(file):
- '''Filters files scanned returning a new list of files to send i.e. miniseed'''
+ '''Filters files scanned returning a new list of files to send'''
if os.path.basename(file).startswith('.'):
return False
if ismseed(file):
@@ -133,7 +129,8 @@ def sendable(file):
# Basic Miniseed file metadata extracted from filename
# This includes .p files which we may accept in the future but won't h
-MseedRE = re.compile(r'\A(.*)\.([A-Z0-9][A-Z0-9])\.(.*)\.([A-Z][A-Z]\w)\.([0-9]{4})\.([0-9]{3})(?:\.p)*')
+MseedRE = re.compile(
+ r'\A(.*)\.([A-Z0-9][A-Z0-9])\.(.*)\.([A-Z][A-Z]\w)\.([0-9]{4})\.([0-9]{3})(?:\.p)*') # noqa
def filename_qc_format(file):
@@ -230,14 +227,22 @@ def write_sent_file_list(sentlist, sentfile=SENTFILE):
def get_FTP():
- '''returns a FTP connection or None if a connection could not be made after attempts allowed'''
+ '''
+ returns a FTP connection or None if a connection could not be made
+ after number of attempts allowed
+ '''
trys = 0
while trys < FTP_CONNECT_ATTEMPTS:
trys += 1
try:
import socket
- logger.info('Connecting to FTP host %s from %s. Attempt %d of %d' % (FTP_HOST, socket.gethostbyname(socket.gethostname()), trys, FTP_CONNECT_ATTEMPTS))
- FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
+ logger.info('Connecting to FTP host %s from %s. Attempt %d of %d'
+ % (FTP_HOST,
+ socket.gethostbyname(socket.gethostname()),
+ trys,
+ FTP_CONNECT_ATTEMPTS))
+ FTP = ftplib.FTP(host=FTP_HOST, user=FTP_USER,
+ passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
FTP.set_debuglevel(FTP_DEBUG_LEVEL)
FTP.cwd(FTP_DIR)
FTP.set_pasv(True)
@@ -245,7 +250,8 @@ def get_FTP():
logger.error('Failed to open FTP connection to %s' % FTP_HOST)
logger.error(e)
test_network()
- logger.info('Waiting %d seconds before trying to reconnect...' % FTP_RECONNECT_WAIT)
+ logger.info('Waiting %d seconds before trying to reconnect...'
+ % FTP_RECONNECT_WAIT)
sleep(FTP_RECONNECT_WAIT)
else:
logger.info('Success: Connected to PASSCAL FTP')
@@ -301,7 +307,6 @@ def url_reachable(url='http://www.passcal.nmt.edu/'):
def test_FTP(FTP):
try:
- assert isinstance(FTP, FTPCache)
FTP.voidcmd('NOOP')
except ftplib.all_errors as e:
logger.error(e)
@@ -316,13 +321,17 @@ def test_FTP(FTP):
def send2passcal(mslist, sentlist=None):
'''Send the list of files in mslist to passcal via FTP'''
- # Handle SIGINT while in this function: to gracefully close connection and save sentlist to disk file
+ # Handle SIGINT while in this function: to gracefully close connection and
+ # save sentlist to disk file
def signal_handler(signum, frame):
print()
- logger.info('Caught interrupt while FTPing. Aborting transfer %s' % current_file)
+ logger.info('Caught interrupt while FTPing. Aborting transfer %s'
+ % current_file)
logger.info('Sent %d of %d' % (num_sent, num_to_send))
- logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)))
- logger.info('%s /sec' % (format_size(size_sent / (time() - starttime))))
+ logger.info('Sent %s of %s'
+ % (format_size(size_sent), format_size(size_to_send)))
+ logger.info('%s /sec'
+ % (format_size(size_sent / (time() - starttime))))
logger.info('Ran for %f sec' % (time() - starttime))
write_sent_file_list(sentlist)
try:
@@ -337,16 +346,19 @@ def send2passcal(mslist, sentlist=None):
'''Updates the terminal display.'''
signal.signal(signal.SIGINT, signal_handler)
update.bytes_sent += len(data)
- print('\r' + str(PB) + ' %s /sec ' % (format_size(size_sent / (time() - starttime))), end=' ')
+ print('\r' + str(PB) + ' %s /sec '
+ % (format_size(size_sent / (time() - starttime))), end=' ')
'''
- print '%s %0.2f%%. %0.10d / %0.10d.' % ( current_file.center(20),
- (update.bytes_sent / update.file_size)*100,
- update.bytes_sent,
- file_size,
- ) ,
+ print '%s %0.2f%%. %0.10d / %0.10d.'
+ % (current_file.center(20),
+ (update.bytes_sent / update.file_size) * 100,
+ update.bytes_sent,
+ file_size)
'''
- ETA_sec = ((time() - starttime) / size_sent) * (size_to_send - size_sent)
- print('ETA %s %s %s' % (str(datetime.timedelta(seconds=(ETA_sec))), current_file.center(20), ' ' * 20), end=' ')
+ ETA_sec = ((time() - starttime) / size_sent) * \
+ (size_to_send - size_sent)
+ print('ETA %s %s %s' % (str(datetime.timedelta(seconds=(ETA_sec))),
+ current_file.center(20), ' ' * 20), end=' ')
sys.stdout.flush()
if sentlist is None:
@@ -359,7 +371,8 @@ def send2passcal(mslist, sentlist=None):
size_sent = 1
current_file = ''
signal.signal(signal.SIGINT, signal_handler)
- logger.info('Sending %d, %s files to PASSCAL' % (num_to_send, format_size(size_to_send)))
+ logger.info('Sending %d, %s files to PASSCAL' %
+ (num_to_send, format_size(size_to_send)))
FTP = get_FTP()
starttime = time()
PB = ProgressBar(num_to_send)
@@ -374,19 +387,26 @@ def send2passcal(mslist, sentlist=None):
fh = open(f, 'rb')
file_size = os.path.getsize(f)
update.file_size = float(file_size)
- FTP.storbinary('STOR %s' % current_file, fh, blocksize=FTP_BLOCKSIZE, callback=update)
+ FTP.storbinary('STOR %s' % current_file, fh,
+ blocksize=FTP_BLOCKSIZE, callback=update)
except ftplib.error_perm as e:
- # This is permission and the error when 550 for the .in file already exists so we should just continue with the next file
- # todo create a list of failed files and resend those at the end instead of requiring a rerun
- logger.error('Failed to send file %s, permission error. Skipping...' % current_file)
+ # This is permission and the error when 550 for the .in file
+ # already exists so we should just continue with the next file
+ # todo create a list of failed files and resend those at the
+ # end instead of requiring a rerun
+ logger.error('Failed to send file %s, permission error.'
+ % current_file)
logger.error(e)
break
except (ftplib.all_errors, AttributeError) as e:
- # since we can restore with how we have proftp setup. There is nothing more we can do with this file
+ # since we can restore with how we have proftp setup.
+ # There is nothing more we can do with this file
# Until the server rms the .in.file
- logger.error('Failed to send file %s.' % (current_file)) # , trys, FTP_SEND_ATTEMPTS)
+ # , trys, FTP_SEND_ATTEMPTS)
+ logger.error('Failed to send file %s.' % (current_file))
logger.error(e)
- # if DEBUG: print "Waiting %d..." % FTP_TIMEOUT; sleep(FTP_TIMEOUT)
+ # if DEBUG:
+ # print "Waiting %d..." %FTP_TIMEOUT; sleep(FTP_TIMEOUT)
try:
if FTP:
FTP.abort()
@@ -406,7 +426,8 @@ def send2passcal(mslist, sentlist=None):
print()
logger.info('Sent %d of %d' % (num_sent, num_to_send))
- logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send)))
+ logger.info('Sent %s of %s'
+ % (format_size(size_sent), format_size(size_to_send)))
logger.info('%s /sec' % (format_size(size_sent / (time() - starttime))))
logger.info('Ran for %f sec' % (time() - starttime))
if FTP and test_FTP(FTP):
@@ -442,7 +463,8 @@ class ProgressBar(object):
percent_done = int(round((new_amount / 100.0) * 100.0))
all_full = self.width - 2
num_hashes = int(round((percent_done / 100.0) * all_full))
- self.prog_bar = '[' + self.fill_char * num_hashes + ' ' * (all_full - num_hashes) + ']'
+ self.prog_bar = '[' + self.fill_char * \
+ num_hashes + ' ' * (all_full - num_hashes) + ']'
pct_place = int((len(self.prog_bar) / 2) - len(str(percent_done)))
pct_string = '%d%%' % percent_done
self.prog_bar = self.prog_bar[0:pct_place] + \
@@ -470,12 +492,14 @@ def main():
logger.info('Removing files that are not Miniseed.')
mslist = list(filter(sendable, msnamedlist))
logger.info('MiniSEED files: %d' % len(mslist))
- logger.info('Properly named but not miniseed files: %d' % (len(msnamedlist) - len(mslist)))
+ logger.info('Properly named but not miniseed files: %d'
+ % (len(msnamedlist) - len(mslist)))
print()
logger.info('Removing files already sent to PASSCAL')
sentlist = get_sent_file_list()
unsentms = [f for f in mslist if f not in sentlist]
- logger.info('%d miniSEED files have already been sent, not resending.' % (len(mslist) - len(unsentms)))
+ logger.info('%d miniSEED files have already been sent, not resending.'
+ % (len(mslist) - len(unsentms)))
send2passcal(unsentms, sentlist)
write_sent_file_list(sentlist)
diff --git a/setup.cfg b/setup.cfg
index a6afa20..3e16c9a 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,19 +1,3 @@
-[bumpversion]
-current_version = 2020.107
-commit = True
-tag = True
-
-[bumpversion:file:setup.py]
-search = version='{current_version}'
-replace = version='{new_version}'
-
-[bumpversion:file:data2passcal/__init__.py]
-search = __version__ = '{current_version}'
-replace = __version__ = '{new_version}'
-
-[bdist_wheel]
-universal = 1
-
[flake8]
exclude = docs
diff --git a/setup.py b/setup.py
index 11504a8..01c7af5 100644
--- a/setup.py
+++ b/setup.py
@@ -33,15 +33,8 @@ setup(
setup_requires=[],
extras_require={
'dev': [
- 'pip',
- 'bumpversion',
- 'wheel',
- 'watchdog',
'flake8',
'tox',
- 'coverage',
- 'Sphinx',
- 'twine',
"mock;python_version<'3.3'"
]
},
diff --git a/tests/test_data2passcal.py b/tests/test_data2passcal.py
index fb90024..cce51db 100644
--- a/tests/test_data2passcal.py
+++ b/tests/test_data2passcal.py
@@ -7,25 +7,19 @@ from __future__ import division, print_function
import ftplib
import os
+import sys
import unittest
from data2passcal.data2passcal import get_FTP, ismseed, scan_dir, send2passcal
+from data2passcal.data2passcal import FTP_CONNECT_ATTEMPTS, FTP_SEND_ATTEMPTS
-try:
- from unittest.mock import patch
-except ImportError:
+if sys.version_info < (3, 3):
from mock import patch
+else:
+ from unittest.mock import patch
VERSION = '2020.107'
-FTP_HOST = 'qc.passcal.nmt.edu'
-FTP_USER = 'ftp'
-FTP_PASSWORD = 'data2passcal'
-FTP_DIR = 'AUTO/MSEED'
-FTP_TIMEOUT = 120
-FTP_RECONNECT_WAIT = 60
-FTP_CONNECT_ATTEMPTS = 60 * 60 * 24 * 7 / FTP_RECONNECT_WAIT
-FTP_SEND_ATTEMPTS = 3
MOCK_TEST = True
@@ -34,8 +28,10 @@ class TestData2passcal(unittest.TestCase):
def setUp(self):
"""Set up test fixtures, if any"""
- dir_testdata = os.path.dirname(os.path.realpath(__file__)) + '/test_data'
- filelist = [x for x in scan_dir(dir_testdata) if not os.path.basename(x).startswith('.') and not os.path.basename(x).endswith('.log')]
+ dir_testdata = os.path.dirname(os.path.realpath(__file__)) + '/test_data' # noqa
+ filelist = [x for x in scan_dir(dir_testdata)
+ if not os.path.basename(x).startswith('.') and not
+ os.path.basename(x).endswith('.log')]
self.dir_testdata = dir_testdata
self.filelist = filelist
@@ -45,21 +41,25 @@ class TestData2passcal(unittest.TestCase):
Only 5 files available under ./test_data directory
Not taking into account .DS_STORE file
"""
- self.assertEqual(len(self.filelist), 5, 'Incorrect number of files')
+ filelist = [x for x in scan_dir(self.dir_testdata)
+ if not os.path.basename(x).startswith('.') and not
+ os.path.basename(x).endswith('.log')]
+ self.assertEqual(len(filelist), 5, 'Incorrect number of files')
def test_ismseed(self):
"""Test basic functionality of ismseed function"""
for f in self.filelist:
- self.assertTrue(ismseed(f), '{} is not a miniseed file'.format(os.path.basename(f)))
+ self.assertTrue(ismseed(f), '{} is not a miniseed file'
+ .format(os.path.basename(f)))
@patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
def test_get_FTP_mock(self, mock_ftp_constructor):
"""Mock test creating ftp connection to PASSCAL"""
mock_ftp = mock_ftp_constructor.return_value
get_FTP()
- mock_ftp_constructor.assert_called_with(host=FTP_HOST, user=FTP_USER, passwd=FTP_PASSWORD, timeout=FTP_TIMEOUT)
- mock_ftp.cwd.assert_called_with(FTP_DIR)
- self.assertLess(mock_ftp_constructor.call_count, FTP_CONNECT_ATTEMPTS, 'Number of ftp connection attempts exceeeds {}'.format(FTP_CONNECT_ATTEMPTS))
+ self.assertLess(mock_ftp_constructor.call_count, FTP_CONNECT_ATTEMPTS,
+ 'Number of ftp connection attempts exceeeds {}'
+ .format(FTP_CONNECT_ATTEMPTS))
mock_ftp.quit()
@patch('data2passcal.data2passcal.ftplib.FTP', autospec=True)
@@ -68,18 +68,23 @@ class TestData2passcal(unittest.TestCase):
mock_ftp = mock_ftp_constructor.return_value
send2passcal(self.filelist)
self.assertTrue(mock_ftp.storbinary.called, 'No data sent')
- self.assertEqual(mock_ftp.storbinary.call_count, len(self.filelist), 'Failed to send all files - Sent {0} of {1}'.format(mock_ftp.storbinary.call_count, len(self.filelist)))
+ self.assertEqual(mock_ftp.storbinary.call_count, len(self.filelist),
+ 'Failed to send all files - Sent {0} of {1}'
+ .format(mock_ftp.storbinary.call_count,
+ len(self.filelist)))
files_sent = []
for x in mock_ftp.storbinary.call_args_list:
args, kwargs = x
files_sent.append(args[0].split(' ')[1])
for f in self.filelist:
f = os.path.basename(f)
- self.assertLess(files_sent.count(f), FTP_SEND_ATTEMPTS, 'Attempted to send file {0} more than {1} times'.format(f, FTP_SEND_ATTEMPTS))
+ self.assertLess(files_sent.count(f), FTP_SEND_ATTEMPTS,
+ 'Attempted to send file {0} more than {1} times'
+ .format(f, FTP_SEND_ATTEMPTS))
@unittest.skipIf(MOCK_TEST == True, "skipping real send2passcal test")
def test_send_data(self):
- """Test sending MSEED files (test data) to PASSCAL's QC system - Optional"""
+ """Test sending MSEED files (test data) to PASSCAL's QC system"""
ftp = get_FTP()
send2passcal(self.filelist)
wdir = ftp.pwd()
@@ -92,7 +97,8 @@ class TestData2passcal(unittest.TestCase):
raise
for f in self.filelist:
f_ = os.path.basename(f)
- self.assertIn(f_, files_sent, 'File {} was not sent to PASSCAL'.format(f_))
+ self.assertIn(f_, files_sent, 'File {} was not sent to PASSCAL'
+ .format(f_))
ftp.quit()
diff --git a/tox.ini b/tox.ini
index d1ee218..bd1c8ff 100644
--- a/tox.ini
+++ b/tox.ini
@@ -9,8 +9,8 @@ python =
[testenv:flake8]
basepython = python
deps = flake8
-commands = flake8 --ignore=E722,E712,E501 data2passcal
- flake8 --ignore=E722,E712,E501 tests
+commands = flake8 --ignore=E722,E712 data2passcal
+ flake8 --ignore=E722,E712 tests
[testenv:py27]
changedir = tests
--
GitLab