diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 61ce95b258c58babcea93e2da604bba56c85d598..d74e7c03214c69162c040c133e954dd3f759604c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,7 +15,6 @@ cache: - .cache/pip stages: -- static analysis - test before_script: @@ -25,7 +24,7 @@ linting_python2: image: python:2.7 tags: - passoft - stage: static analysis + stage: test script: - flake8 --ignore=E722,E712 data2passcal - flake8 --ignore=E722,E712 tests @@ -34,7 +33,7 @@ linting_python3: image: python:3.6 tags: - passoft - stage: static analysis + stage: test script: - flake8 --ignore=E722,E712 data2passcal - flake8 --ignore=E722,E712 tests diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 0866d5424cab8c69c25d6c43a6d3359a67816bc4..69b8d731421c58d03df0b213f114b991d723a89d 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -45,7 +45,8 @@ articles, and such. Submit Feedback ~~~~~~~~~~~~~~~ -The best way to send feedback is to file an issue at https://git.passcal.nmt.edu/passoft/data2passcal/issues. +The best way to send feedback is to file an issue at +https://git.passcal.nmt.edu/passoft/data2passcal/issues. If you are proposing a feature: @@ -59,32 +60,32 @@ Get Started! Ready to contribute? Here's how to set up `data2passcal` for local development. -1. Cone the `data2passcal` repo:: +1. Clone the `data2passcal` repo: $ git clone https://git.passcal.nmt.edu/passoft/data2passcal.git -3. Install your local copy:: +2. Install your local copy: $ pip install -e .[dev] -4. Create a branch for local development:: +3. Create a branch for local development: $ git checkout -b name-of-your-bugfix-or-feature Now you can make your changes locally. -5. When you're done making changes, check that your changes pass the - tests:: +4. When you're done making changes, check that your changes pass the + tests: $ python -m unittest (or python -m unittest tests.test_data2passcal under Python2) - -6. Commit your changes and push your branch to GitHub:: + +5. Commit your changes and push your branch to GitHub: $ git add . $ git commit -m "Your detailed description of your changes." $ git push origin name-of-your-bugfix-or-feature -7. Submit a merge request through the Gitlab website. +6. Submit a merge request through the Gitlab website. Pull Request Guidelines ----------------------- @@ -95,7 +96,7 @@ Before you submit a merge request, check that it meets these guidelines: 2. If the merge request adds functionality, the docs should be updated. Put your new functionality into a function with a docstring, and add the feature to the list in README.rst. -3. The pull request should work for Python 2.7 +3. The pull request should work for Python 2.7 and Python 3.[5,6,7,8] Tips ---- diff --git a/HISTORY.rst b/HISTORY.rst index 3c3e83bc085689e587f07c39f940ef83342eb088..bc694a413f909aeeab250acdc5351d3444f184ed 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -41,3 +41,8 @@ History * Created conda packages for "data2passcal" that can run on Python3.[5,7,8] * Tested data2passcal against Python3.[5,7,8] using tox * Updated .gitlab-ci.yml to run a linter and unit tests for Python2 and Python3.[5,6,7,8] in GitLab CI pipeline + +2020.213 (2020-07-31) +------------------ +* Updated meta.yaml to build architecture and platform independent package +* Updated mock unit tests diff --git a/MANIFEST.in b/MANIFEST.in index ced9d127ece3b0481c36a07ff0a66b3dbd705cfe..2387d6e279b924229fa0c8c485eeeebf49784996 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,4 +6,4 @@ include README.rst recursive-include tests * -recursive-include docs *.rst conf.py Makefile make.bat +recursive-include docs *.rst conf.py diff --git a/README.rst b/README.rst index a1160b99c3905a79cfa2cd8fa0e04228ddbc9968..0b4fbe637614c42f65ae7fa28b8969d817cc5251 100644 --- a/README.rst +++ b/README.rst @@ -2,10 +2,9 @@ data2passcal ============ +* Description: Send MSEED files ready for archival at the DMC to PASSCAL's QC system + Only accept day-long MSEED files. -Description: Send MSEED files ready for archival at the DMC to PASSCAL's QC system - Only accept day-long MSEED files. -Usage: data2passcal dir - +* Usage: data2passcal dir * Free software: GNU General Public License v3 (GPLv3) diff --git a/conda.recipe/conda_build_config.yaml b/conda.recipe/conda_build_config.yaml deleted file mode 100644 index e1f365adb0f3f454a3958bd00907f34862452d6e..0000000000000000000000000000000000000000 --- a/conda.recipe/conda_build_config.yaml +++ /dev/null @@ -1,6 +0,0 @@ -python: - - 3.5 - - 3.6 - - 3.7 - - 3.8 -target_platform: osx-64 diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index ecfa27df20eaba1e8fa3dc106867aef7357d34a8..e107f5f964ecc6c4748eaa4196af614ce0cf8c59 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,22 +1,22 @@ package: name: data2passcal - version: 2020.119 + version: 2020.213 source: path: ../ build: + noarch: python number: 0 - script: python setup.py install --single-version-externally-managed --record=record.txt - entry_points: - - data2passcal = data2passcal.data2passcal:main + script: {{ PYTHON }} -m pip install . --no-deps -vv + string: py_2_3 requirements: build: - - python - - setuptools + - python >=2.7 + - pip run: - - python + - python >=2.7 test: requires: diff --git a/data2passcal/__init__.py b/data2passcal/__init__.py index fdf64fa1bbddd3c7a070eb155b0a5f81839cc2b1..a49f05ebad00ac286528b632fcd8b03962ed1726 100644 --- a/data2passcal/__init__.py +++ b/data2passcal/__init__.py @@ -4,4 +4,4 @@ __author__ = """IRIS PASSCAL""" __email__ = 'software-support@passcal.nmt.edu' -__version__ = '2020.119' +__version__ = '2020.213' diff --git a/data2passcal/data2passcal.py b/data2passcal/data2passcal.py index 6a136fcc9e9d930783593f13a122d8fb3fccbc0c..4b97b5c2f5dddde2916c4efd92e8755847d4e8a5 100644 --- a/data2passcal/data2passcal.py +++ b/data2passcal/data2passcal.py @@ -4,6 +4,14 @@ data2passcal Ship miniseed data to passcal via ftp for qc before archival at DMC DMS Lloyd Carothers IRIS/PASSCAL + +Maeva Pourpoint +July 2020 +Updates to work under Python 2 and 3. +Unit tests to ensure basic functionality. +Code cleanup to conform to the PEP8 style guide. +Directory cleanup (remove unused files introduced by Cookiecutter). +Packaged with conda. ''' from __future__ import division, print_function @@ -17,6 +25,8 @@ import signal import socket import struct import sys + +from io import open from time import sleep, time try: @@ -24,7 +34,7 @@ try: except ImportError: from urllib2 import urlopen -VERSION = '2020.119' +VERSION = '2020.213' # FTP related @@ -99,7 +109,6 @@ logger.info('FTP RECONNECT WAIT: %d' % FTP_RECONNECT_WAIT) def scan_dir(dir): '''Returns a list of absolute file names found below root dir''' rootdir = dir - print() logger.info('Scanning: %s' % os.path.abspath(dir)) filelist = [] filesize = 0 @@ -119,8 +128,6 @@ def scan_dir(dir): logger.info('Total Files = %s' % len(filelist)) logger.info('Total Dirs = %s' % foldercount) logger.info('Scan time = %0.2fs' % (time() - starttime)) - print() - return filelist @@ -259,7 +266,7 @@ def get_FTP(): FTP.cwd(FTP_DIR) FTP.set_pasv(True) except (ftplib.all_errors + (AttributeError,)): - logger.exception('Failed to open FTP connection to %s' % FTP_HOST) + logger.error('Failed to open FTP connection to %s' % FTP_HOST) test_network() logger.info('Waiting %.2f seconds before trying to reconnect...' % FTP_RECONNECT_WAIT) @@ -316,12 +323,12 @@ def test_FTP(FTP): try: FTP.voidcmd('NOOP') except ftplib.all_errors: - logger.exception('Failed to send a simple command string to the server\ - and handle the response') + logger.error("Failed to send a simple command string to the server" + "and handle the response") return False except AssertionError: - logger.exception('Failed to send a simple command string to the server\ - and handle the response') + logger.error("Failed to send a simple command string to the server" + "and handle the response") return False else: return True @@ -333,7 +340,6 @@ def send2passcal(mslist, sentlist=None): # Handle SIGINT while in this function: to gracefully close connection and # save sentlist to disk file def signal_handler(signum, frame): - print() logger.info('Caught interrupt while FTPing. Aborting transfer %s' % current_file) logger.info('Sent %d of %d' % (num_sent, num_to_send)) @@ -348,8 +354,8 @@ def send2passcal(mslist, sentlist=None): FTP.abort() FTP.quit() except Exception: - logger.exception("Failed to abort file transfer and close" - "FTP connection") + logger.error("Failed to abort file transfer and close FTP " + "connection") os._exit(1) def update(data): @@ -373,7 +379,6 @@ def send2passcal(mslist, sentlist=None): if sentlist is None: sentlist = [] - print() logger.info('Sending MSEED files to PASSCAL') num_to_send = len(mslist) num_sent = 0 @@ -404,15 +409,16 @@ def send2passcal(mslist, sentlist=None): # already exists so we should just continue with the next file # todo create a list of failed files and resend those at the # end instead of requiring a rerun - logger.exception('Failed to send file %s, permission error.' - % current_file) + logger.error('Failed to send file %s, permission error.' + % current_file) + fh.close() break except (ftplib.all_errors + (AttributeError,)): # since we can restore with how we have proftp setup. # There is nothing more we can do with this file # Until the server rms the .in.file # , trys, FTP_SEND_ATTEMPTS) - logger.exception('Failed to send file %s.' % (current_file)) + logger.error('Failed to send file %s.' % (current_file)) # if DEBUG: # print "Waiting %d..." %FTP_TIMEOUT; sleep(FTP_TIMEOUT) try: @@ -424,6 +430,7 @@ def send2passcal(mslist, sentlist=None): "FTP connection") pass FTP = get_FTP() + fh.close() if FTP is None: signal_handler(None, None) else: @@ -434,7 +441,6 @@ def send2passcal(mslist, sentlist=None): sentlist.append(f) break - print() logger.info('Sent %d of %d' % (num_sent, num_to_send)) logger.info('Sent %s of %s' % (format_size(size_sent), format_size(size_to_send))) @@ -498,13 +504,11 @@ def main(): msnamedlist = list(filter(filename_qc_format, filelist)) logger.info('Properly named files files: %d' % len(msnamedlist)) logger.info('Other files: %d' % (len(filelist) - len(msnamedlist))) - print() logger.info('Removing files that are not Miniseed.') mslist = list(filter(sendable, msnamedlist)) logger.info('MiniSEED files: %d' % len(mslist)) logger.info('Properly named but not miniseed files: %d' % (len(msnamedlist) - len(mslist))) - print() logger.info('Removing files already sent to PASSCAL') sentlist = get_sent_file_list() unsentms = [f for f in mslist if f not in sentlist] diff --git a/setup.py b/setup.py index d04d19d6d99b97b44644fccc6f2c55d640c6ee57..6aff95297d65f8016b95522f1a5c4e3b9930ef8f 100644 --- a/setup.py +++ b/setup.py @@ -3,12 +3,13 @@ """The setup script.""" +from io import open from setuptools import setup, find_packages -with open('README.rst') as readme_file: +with open('README.rst', 'rt') as readme_file: readme = readme_file.read() -with open('HISTORY.rst') as history_file: +with open('HISTORY.rst', 'rt') as history_file: history = history_file.read() @@ -48,6 +49,6 @@ setup( name='data2passcal', packages=find_packages(include=['data2passcal']), url='https://git.passcal.nmt.edu/passoft/data2passcal', - version='2020.119', + version='2020.213', zip_safe=False, ) diff --git a/tests/test_data2passcal.py b/tests/test_data2passcal.py index 7c8c0ff0d64805a487f15c2a833cf5532acf5d05..6515358cf0f180243f86bfc0c3c28b8b0fa2d25b 100644 --- a/tests/test_data2passcal.py +++ b/tests/test_data2passcal.py @@ -16,7 +16,7 @@ if sys.version_info < (3, 3): else: from unittest.mock import patch -VERSION = '2020.119' +VERSION = '2020.213' SEND4REAL = os.environ.get('SEND4REAL', 'False') print("SEND4REAL=False by default. If one wants to test sending data to " @@ -74,7 +74,7 @@ class TestData2passcal(unittest.TestCase): Mock test failure to create ftp connection to PASSCAL and exercise get_FTP() """ - mock_ftp_constructor.return_value = None + mock_ftp_constructor.return_value = ftplib.error_temp d2p.get_FTP() self.assertGreater(mock_ftp_constructor.call_count, 1, "ftplib.FTP() called only once - get_FTP() not " @@ -109,7 +109,7 @@ class TestData2passcal(unittest.TestCase): Mock test failure to create ftp connection to PASSCAL and exercise send_data() """ - mock_ftp_constructor.return_value = None + mock_ftp_constructor.return_value = ftplib.error_temp filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST[0:2]] d2p.send2passcal(filelist) self.assertTrue(mock_exit.called, "os._exit(1) never called - "