From 5b381bc97a73e390d32a72edc97f0b06bd89e887 Mon Sep 17 00:00:00 2001
From: Maeva Pourpoint <maeva@passcal.nmt.edu>
Date: Fri, 31 Jul 2020 15:53:58 -0600
Subject: [PATCH] Update conda recipe to build python and platform independent
package + Update mock unit tests
---
.gitlab-ci.yml | 5 ++--
CONTRIBUTING.rst | 21 +++++++-------
HISTORY.rst | 5 ++++
MANIFEST.in | 2 +-
README.rst | 7 ++---
conda.recipe/conda_build_config.yaml | 6 ----
conda.recipe/meta.yaml | 14 +++++-----
data2passcal/__init__.py | 2 +-
data2passcal/data2passcal.py | 42 +++++++++++++++-------------
setup.py | 7 +++--
tests/test_data2passcal.py | 6 ++--
11 files changed, 60 insertions(+), 57 deletions(-)
delete mode 100644 conda.recipe/conda_build_config.yaml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 61ce95b..d74e7c0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -15,7 +15,6 @@ cache:
- .cache/pip
stages:
-- static analysis
- test
before_script:
@@ -25,7 +24,7 @@ linting_python2:
image: python:2.7
tags:
- passoft
- stage: static analysis
+ stage: test
script:
- flake8 --ignore=E722,E712 data2passcal
- flake8 --ignore=E722,E712 tests
@@ -34,7 +33,7 @@ linting_python3:
image: python:3.6
tags:
- passoft
- stage: static analysis
+ stage: test
script:
- flake8 --ignore=E722,E712 data2passcal
- flake8 --ignore=E722,E712 tests
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 0866d54..69b8d73 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -45,7 +45,8 @@ articles, and such.
Submit Feedback
~~~~~~~~~~~~~~~
-The best way to send feedback is to file an issue at https://git.passcal.nmt.edu/passoft/data2passcal/issues.
+The best way to send feedback is to file an issue at
+https://git.passcal.nmt.edu/passoft/data2passcal/issues.
If you are proposing a feature:
@@ -59,32 +60,32 @@ Get Started!
Ready to contribute? Here's how to set up `data2passcal` for local development.
-1. Cone the `data2passcal` repo::
+1. Clone the `data2passcal` repo:
$ git clone https://git.passcal.nmt.edu/passoft/data2passcal.git
-3. Install your local copy::
+2. Install your local copy:
$ pip install -e .[dev]
-4. Create a branch for local development::
+3. Create a branch for local development:
$ git checkout -b name-of-your-bugfix-or-feature
Now you can make your changes locally.
-5. When you're done making changes, check that your changes pass the
- tests::
+4. When you're done making changes, check that your changes pass the
+ tests:
$ python -m unittest (or python -m unittest tests.test_data2passcal under Python2)
-
-6. Commit your changes and push your branch to GitHub::
+
+5. Commit your changes and push your branch to GitHub:
$ git add .
$ git commit -m "Your detailed description of your changes."
$ git push origin name-of-your-bugfix-or-feature
-7. Submit a merge request through the Gitlab website.
+6. Submit a merge request through the Gitlab website.
Pull Request Guidelines
-----------------------
@@ -95,7 +96,7 @@ Before you submit a merge request, check that it meets these guidelines:
2. If the merge request adds functionality, the docs should be updated. Put
your new functionality into a function with a docstring, and add the
feature to the list in README.rst.
-3. The pull request should work for Python 2.7
+3. The pull request should work for Python 2.7 and Python 3.[5,6,7,8]
Tips
----
diff --git a/HISTORY.rst b/HISTORY.rst
index 3c3e83b..bc694a4 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -41,3 +41,8 @@ History
* Created conda packages for "data2passcal" that can run on Python3.[5,7,8]
* Tested data2passcal against Python3.[5,7,8] using tox
* Updated .gitlab-ci.yml to run a linter and unit tests for Python2 and Python3.[5,6,7,8] in GitLab CI pipeline
+
+2020.213 (2020-07-31)
+------------------
+* Updated meta.yaml to build architecture and platform independent package
+* Updated mock unit tests
diff --git a/MANIFEST.in b/MANIFEST.in
index ced9d12..2387d6e 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -6,4 +6,4 @@ include README.rst
recursive-include tests *
-recursive-include docs *.rst conf.py Makefile make.bat
+recursive-include docs *.rst conf.py
diff --git a/README.rst b/README.rst
index a1160b9..0b4fbe6 100644
--- a/README.rst
+++ b/README.rst
@@ -2,10 +2,9 @@
data2passcal
============
+* Description: Send MSEED files ready for archival at the DMC to PASSCAL's QC system
+ Only accept day-long MSEED files.
-Description: Send MSEED files ready for archival at the DMC to PASSCAL's QC system
- Only accept day-long MSEED files.
-Usage: data2passcal dir
-
+* Usage: data2passcal dir
* Free software: GNU General Public License v3 (GPLv3)
diff --git a/conda.recipe/conda_build_config.yaml b/conda.recipe/conda_build_config.yaml
deleted file mode 100644
index e1f365a..0000000
--- a/conda.recipe/conda_build_config.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-python:
- - 3.5
- - 3.6
- - 3.7
- - 3.8
-target_platform: osx-64
diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml
index ecfa27d..e107f5f 100644
--- a/conda.recipe/meta.yaml
+++ b/conda.recipe/meta.yaml
@@ -1,22 +1,22 @@
package:
name: data2passcal
- version: 2020.119
+ version: 2020.213
source:
path: ../
build:
+ noarch: python
number: 0
- script: python setup.py install --single-version-externally-managed --record=record.txt
- entry_points:
- - data2passcal = data2passcal.data2passcal:main
+ script: {{ PYTHON }} -m pip install . --no-deps -vv
+ string: py_2_3
requirements:
build:
- - python
- - setuptools
+ - python >=2.7
+ - pip
run:
- - python
+ - python >=2.7
test:
requires:
diff --git a/data2passcal/__init__.py b/data2passcal/__init__.py
index fdf64fa..a49f05e 100644
--- a/data2passcal/__init__.py
+++ b/data2passcal/__init__.py
@@ -4,4 +4,4 @@
__author__ = """IRIS PASSCAL"""
__email__ = 'software-support@passcal.nmt.edu'
-__version__ = '2020.119'
+__version__ = '2020.213'
diff --git a/data2passcal/data2passcal.py b/data2passcal/data2passcal.py
index 6a136fc..4b97b5c 100644
--- a/data2passcal/data2passcal.py
+++ b/data2passcal/data2passcal.py
@@ -4,6 +4,14 @@ data2passcal
Ship miniseed data to passcal via ftp for qc before archival at DMC DMS
Lloyd Carothers IRIS/PASSCAL
+
+Maeva Pourpoint
+July 2020
+Updates to work under Python 2 and 3.
+Unit tests to ensure basic functionality.
+Code cleanup to conform to the PEP8 style guide.
+Directory cleanup (remove unused files introduced by Cookiecutter).
+Packaged with conda.
'''
from __future__ import division, print_function
@@ -17,6 +25,8 @@ import signal
import socket
import struct
import sys
+
+from io import open
from time import sleep, time
try:
@@ -24,7 +34,7 @@ try:
except ImportError:
from urllib2 import urlopen
-VERSION = '2020.119'
+VERSION = '2020.213'
# FTP related
@@ -99,7 +109,6 @@ logger.info('FTP RECONNECT WAIT: %d' % FTP_RECONNECT_WAIT)
def scan_dir(dir):
'''Returns a list of absolute file names found below root dir'''
rootdir = dir
- print()
logger.info('Scanning: %s' % os.path.abspath(dir))
filelist = []
filesize = 0
@@ -119,8 +128,6 @@ def scan_dir(dir):
logger.info('Total Files = %s' % len(filelist))
logger.info('Total Dirs = %s' % foldercount)
logger.info('Scan time = %0.2fs' % (time() - starttime))
- print()
-
return filelist
@@ -259,7 +266,7 @@ def get_FTP():
FTP.cwd(FTP_DIR)
FTP.set_pasv(True)
except (ftplib.all_errors + (AttributeError,)):
- logger.exception('Failed to open FTP connection to %s' % FTP_HOST)
+ logger.error('Failed to open FTP connection to %s' % FTP_HOST)
test_network()
logger.info('Waiting %.2f seconds before trying to reconnect...'
% FTP_RECONNECT_WAIT)
@@ -316,12 +323,12 @@ def test_FTP(FTP):
try:
FTP.voidcmd('NOOP')
except ftplib.all_errors:
- logger.exception('Failed to send a simple command string to the server\
- and handle the response')
+ logger.error("Failed to send a simple command string to the server"
+ "and handle the response")
return False
except AssertionError:
- logger.exception('Failed to send a simple command string to the server\
- and handle the response')
+ logger.error("Failed to send a simple command string to the server"
+ "and handle the response")
return False
else:
return True
@@ -333,7 +340,6 @@ def send2passcal(mslist, sentlist=None):
# Handle SIGINT while in this function: to gracefully close connection and
# save sentlist to disk file
def signal_handler(signum, frame):
- print()
logger.info('Caught interrupt while FTPing. Aborting transfer %s'
% current_file)
logger.info('Sent %d of %d' % (num_sent, num_to_send))
@@ -348,8 +354,8 @@ def send2passcal(mslist, sentlist=None):
FTP.abort()
FTP.quit()
except Exception:
- logger.exception("Failed to abort file transfer and close"
- "FTP connection")
+ logger.error("Failed to abort file transfer and close FTP "
+ "connection")
os._exit(1)
def update(data):
@@ -373,7 +379,6 @@ def send2passcal(mslist, sentlist=None):
if sentlist is None:
sentlist = []
- print()
logger.info('Sending MSEED files to PASSCAL')
num_to_send = len(mslist)
num_sent = 0
@@ -404,15 +409,16 @@ def send2passcal(mslist, sentlist=None):
# already exists so we should just continue with the next file
# todo create a list of failed files and resend those at the
# end instead of requiring a rerun
- logger.exception('Failed to send file %s, permission error.'
- % current_file)
+ logger.error('Failed to send file %s, permission error.'
+ % current_file)
+ fh.close()
break
except (ftplib.all_errors + (AttributeError,)):
# since we can restore with how we have proftp setup.
# There is nothing more we can do with this file
# Until the server rms the .in.file
# , trys, FTP_SEND_ATTEMPTS)
- logger.exception('Failed to send file %s.' % (current_file))
+ logger.error('Failed to send file %s.' % (current_file))
# if DEBUG:
# print "Waiting %d..." %FTP_TIMEOUT; sleep(FTP_TIMEOUT)
try:
@@ -424,6 +430,7 @@ def send2passcal(mslist, sentlist=None):
"FTP connection")
pass
FTP = get_FTP()
+ fh.close()
if FTP is None:
signal_handler(None, None)
else:
@@ -434,7 +441,6 @@ def send2passcal(mslist, sentlist=None):
sentlist.append(f)
break
- print()
logger.info('Sent %d of %d' % (num_sent, num_to_send))
logger.info('Sent %s of %s'
% (format_size(size_sent), format_size(size_to_send)))
@@ -498,13 +504,11 @@ def main():
msnamedlist = list(filter(filename_qc_format, filelist))
logger.info('Properly named files files: %d' % len(msnamedlist))
logger.info('Other files: %d' % (len(filelist) - len(msnamedlist)))
- print()
logger.info('Removing files that are not Miniseed.')
mslist = list(filter(sendable, msnamedlist))
logger.info('MiniSEED files: %d' % len(mslist))
logger.info('Properly named but not miniseed files: %d'
% (len(msnamedlist) - len(mslist)))
- print()
logger.info('Removing files already sent to PASSCAL')
sentlist = get_sent_file_list()
unsentms = [f for f in mslist if f not in sentlist]
diff --git a/setup.py b/setup.py
index d04d19d..6aff952 100644
--- a/setup.py
+++ b/setup.py
@@ -3,12 +3,13 @@
"""The setup script."""
+from io import open
from setuptools import setup, find_packages
-with open('README.rst') as readme_file:
+with open('README.rst', 'rt') as readme_file:
readme = readme_file.read()
-with open('HISTORY.rst') as history_file:
+with open('HISTORY.rst', 'rt') as history_file:
history = history_file.read()
@@ -48,6 +49,6 @@ setup(
name='data2passcal',
packages=find_packages(include=['data2passcal']),
url='https://git.passcal.nmt.edu/passoft/data2passcal',
- version='2020.119',
+ version='2020.213',
zip_safe=False,
)
diff --git a/tests/test_data2passcal.py b/tests/test_data2passcal.py
index 7c8c0ff..6515358 100644
--- a/tests/test_data2passcal.py
+++ b/tests/test_data2passcal.py
@@ -16,7 +16,7 @@ if sys.version_info < (3, 3):
else:
from unittest.mock import patch
-VERSION = '2020.119'
+VERSION = '2020.213'
SEND4REAL = os.environ.get('SEND4REAL', 'False')
print("SEND4REAL=False by default. If one wants to test sending data to "
@@ -74,7 +74,7 @@ class TestData2passcal(unittest.TestCase):
Mock test failure to create ftp connection to PASSCAL and exercise
get_FTP()
"""
- mock_ftp_constructor.return_value = None
+ mock_ftp_constructor.return_value = ftplib.error_temp
d2p.get_FTP()
self.assertGreater(mock_ftp_constructor.call_count, 1,
"ftplib.FTP() called only once - get_FTP() not "
@@ -109,7 +109,7 @@ class TestData2passcal(unittest.TestCase):
Mock test failure to create ftp connection to PASSCAL and exercise
send_data()
"""
- mock_ftp_constructor.return_value = None
+ mock_ftp_constructor.return_value = ftplib.error_temp
filelist = [os.path.join(TEST_DIR, f) for f in MS_FILELIST[0:2]]
d2p.send2passcal(filelist)
self.assertTrue(mock_exit.called, "os._exit(1) never called - "
--
GitLab