Repository: climate Updated Branches: refs/heads/master f9a9517cb -> fc520781d
CLIMATE-874 - Remove Easy-OCW Project: http://git-wip-us.apache.org/repos/asf/climate/repo Commit: http://git-wip-us.apache.org/repos/asf/climate/commit/fc520781 Tree: http://git-wip-us.apache.org/repos/asf/climate/tree/fc520781 Diff: http://git-wip-us.apache.org/repos/asf/climate/diff/fc520781 Branch: refs/heads/master Commit: fc520781d218ad2ab6bade79cd3779d1fd6e0dee Parents: f9a9517 Author: Alex Goodman <ago...@users.noreply.github.com> Authored: Wed Oct 19 20:49:18 2016 -0700 Committer: Alex Goodman <ago...@users.noreply.github.com> Committed: Thu Oct 20 13:58:33 2016 -0700 ---------------------------------------------------------------------- .travis.yml | 36 ++++- deps_py2.txt | 14 ++ deps_py3.txt | 11 ++ easy-ocw/install-osx.sh | 242 ------------------------------- easy-ocw/install-ubuntu.sh | 157 -------------------- easy-ocw/ocw-conda-dependencies.txt | 23 --- easy-ocw/ocw-pip-dependencies.txt | 12 -- ocw/dataset_loader.py | 16 +- test.sh | 6 +- test_smoke.py | 14 +- 10 files changed, 80 insertions(+), 451 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 10d9d17..1a50d8b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,23 +17,49 @@ language: python python: - "2.7" - - "3.2" - - "3.3" - "3.4" - "3.5" notifications: email: false install: - sudo apt-get update - - source easy-ocw/install-ubuntu.sh - - pip install python-coveralls + # We do this conditionally because it saves us some downloading if the + # version is the same. + - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then + wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh; + DEPS=deps_py2.txt; + else + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; + DEPS=deps_py3.txt; + fi + - bash miniconda.sh -b -p $HOME/miniconda + - export PATH="$HOME/miniconda/bin:$PATH" + - hash -r + - conda config --set always_yes yes --set changeps1 no + - conda update -q conda + + # Useful for debugging any issues with conda + - conda info -a + + # Add conda-forge to search so all dependencies can be resolved + - conda config --add channels conda-forge + - conda create -q -n test-environment nose python=$TRAVIS_PYTHON_VERSION --file $DEPS + + # Need this to exclude pydap test for python 3 + - pip install nose-exclude + + # Install ocw + - source activate test-environment + - python setup.py install + # Workaround to get travis build working with matplotlib before_script: - "export DISPLAY=:99.0" - "sh -e /etc/init.d/xvfb start" - sleep 3 # give xvfb some time to start + script: - chmod a+x test.sh - - ./test.sh + - ./test.sh $TRAVIS_PYTHON_VERSION after_success: - coveralls http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/deps_py2.txt ---------------------------------------------------------------------- diff --git a/deps_py2.txt b/deps_py2.txt new file mode 100644 index 0000000..c5cd8b6 --- /dev/null +++ b/deps_py2.txt @@ -0,0 +1,14 @@ +numpy +scipy +matplotlib +basemap +netcdf4 +h5py +bottle +pydap +python-dateutil +mock +webtest +myproxyclient +esgf-pyclient +podaacpy http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/deps_py3.txt ---------------------------------------------------------------------- diff --git a/deps_py3.txt b/deps_py3.txt new file mode 100644 index 0000000..b5fbfa8 --- /dev/null +++ b/deps_py3.txt @@ -0,0 +1,11 @@ +numpy +scipy +matplotlib +basemap +netcdf4 +h5py +bottle +python-dateutil +mock +webtest +podaacpy http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/easy-ocw/install-osx.sh ---------------------------------------------------------------------- diff --git a/easy-ocw/install-osx.sh b/easy-ocw/install-osx.sh deleted file mode 100755 index aeaeae6..0000000 --- a/easy-ocw/install-osx.sh +++ /dev/null @@ -1,242 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -export ARCHFLAGS=-Wno-error=unused-command-line-argument-hard-error-in-future - -help() -{ -cat << ENDHELP - -Easy OCW assists with the building of the Apache Open Climate Workbench and its dependencies - -Flags: - -h Display this help message. - -e Install and configure a virtualenv environment before installation. - -q Quiet install. User prompts are removed (when possible). - -It is recommended that you pass -e when running this script. If you don't, parts -of this installation will pollute your global Python install. If you're unsure, -pass -e just to be safe! - -ENDHELP -} - -header() -{ - echo - echo $1 -} - -task() -{ - echo " - " $1 -} - -subtask() -{ - echo " " $1 -} - -echo -echo "---------------------------------------------------------------------------" -echo " Welcome to Easy OCW" -echo "---------------------------------------------------------------------------" -echo - -WITH_VIRTUAL_ENV=0 -WITH_HOMEBREW=0 -WITH_INTERACT=1 -INIT_PWD=$PWD - -while getopts ":h :e :q" FLAG -do - case $FLAG in - h) - help - exit 1 - ;; - e) - WITH_VIRTUAL_ENV=1 - ;; - q) - WITH_INTERACT=0 - ;; - ?) - help - exit 1 - ;; - esac -done - -if [ $WITH_INTERACT == 1 ]; then -cat << ENDINTRO -A number of dependencies for OCW will now be installed. Please check the wiki -for a complete list of dependencies. Additionally, please read the wiki for -useful installation guidelines and information that may be pertinent to your -situation. All of this can be found at http://s.apache.org/3p2 - -ENDINTRO - -if [ $WITH_VIRTUAL_ENV != 1 ]; then -cat << VIRTUALENV_WARNING -$(tput setaf 1)<-----------------------------[WARNING!]----------------------------------->$(tput sgr 0) -It is highly recommended that you allow Easy OCW to install the dependencies -into a virtualenv environment to ensure that your global Python install is -not affected. If you're UNSURE, you should pass the -e flag -to this script. If you aren't concerned, or you want to create your own -virtualenv environment, then feel free to ignore this message. - -VIRTUALENV_WARNING - -read -p "Press [Yy] to begin installation with the flag -e $(tput setaf 2)[RECOMMENDED]$(tput sgr 0) -[OR] -Press [Nn] to continue with the normal installation..." yn -case $yn in - [Yy]* ) - WITH_VIRTUAL_ENV=1 - ;; - [Nn]* ) - WITH_VIRTUAL_ENV=0 - ;; - * ) echo "Please answer yes or no.." ;; -esac -fi -fi - -header "Checking for pip ..." -command -v pip >/dev/null 2>&1 || { - task "Unable to locate pip." - task "Installing Distribute" - curl http://python-distribute.org/distribute_setup.py | python >> install_log - subtask "done" - - task "Installing Pip" - curl -O http://pypi.python.org/packages/source/p/pip/pip-1.2.1.tar.gz >> install_log - tar xzf pip-1.2.1.tar.gz >> install_log - cd pip-1.2.1/ - python setup.py install >> install_log - subtask "done" -} - -if [ $WITH_VIRTUAL_ENV == 1 ]; then - header "Setting up a virtualenv ..." - - # Check if virtualenv is installed. If it's not, we'll install it for the user. - command -v virtualenv >/dev/null 2>&1 || { - task "Installing virtualenv ..." - pip install virtualenv >> install_log - subtask "done" - } - - #Read the path for installation from the user. - read -e -p "Enter the path where you want to install ocw..." ocw_path - #check if the file path exists. - while [ ! -d "$ocw_path" ]; - do - echo "Path not found..." - read -e -p "Please enter a valid path..." ocw_path - done - - header "Checking for previously installed ocw virtual environment..." - if [ -e $ocw_path/ocw/bin/python ]; then - echo "We found an existing 'ocw' virtualenv on your system in $ocw_path/ocw." - read -n1 -p "Do you want to replace it with a clean install? y/n :" replace - if [ "$replace" == "y" ]; then - echo "" - echo "$(tput setaf 1)[WARNING!] this will delete all file and data in $ocw_path/ocw on your system.$(tput sgr 0)" - read -n1 -p "To confirm and proceed type y or n to quit:" confirm - if [ "$confirm" == "y" ]; then - echo "" - echo "Deleting contents of $ocw_path/ocw" >> install_log - rm -rf $ocw_path/ocw - else - echo "" - echo "Stopping Open Climate Workbench Installation" - exit - fi - else - echo "" - echo "Stopping Open Climate Workbench Installation because an existing 'ocw'" - echo "virtual environment already exists." - exit - fi - fi - - cd $ocw_path - # Create a new environment for OCW work - task "Creating a new environment in ~/ocw..." - virtualenv ocw >> install_log - source $ocw_path/ocw/bin/activate >> install_log - cd $INIT_PWD - subtask "done" -fi - -header "Installing conda for package management ..." - -pip install conda >> install_log -conda init >> install_log - - -header "Installing dependencies with conda ..." -echo | conda install --file ocw-conda-dependencies.txt - -# We only use conda for the annoying dependencies like numpy, -# scipy, matplotlib, and basemap. For everything else, we stick -# with pip. -header "Installing additional Python packages" -pip install -r ocw-pip-dependencies.txt >> install_log - -if [ $WITH_VIRTUAL_ENV == 1 ]; then - echo "***POST INSTALLATION NOTE*** - -If you are familiar with virtualenv you should know that activating -the new 'ocw' environment is different because we use conda to install -packages. An example of the command you want to run is listed in the -alias below. - -To make it easier to change into the 'ocw' virtualenv add the -following alias to your ~/.bash_profile - - alias ocw='source $ocw_path/ocw/bin/activate $ocw_path/ocw/' - -When you want to use ocw in the future, you just have to type 'ocw' -in your terminal." -else - echo "***POST INSTALLATION NOTE*** - -If you have run this script within your own virtualenv you need to know -a couple of caveats/side effects that are caused by using conda to install -packages within the virtualenv. - -- Virtualenv wrapper will throw errors like those outlined here: -https://issues.apache.org/jira/browse/CLIMATE-451 - -- You will not be able to 'activate' the environment using the normal -virtualenv command, you must instead use the conda activate command as follows: - -source path/to/your_env/bin/activate path/to/your_env - -Example: (assuming your env is in ~/.virtualenv/ocw) - -source ~/.virtualenv/ocw/bin/activate ~/.virtualenv/ocw - -" - -fi - http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/easy-ocw/install-ubuntu.sh ---------------------------------------------------------------------- diff --git a/easy-ocw/install-ubuntu.sh b/easy-ocw/install-ubuntu.sh deleted file mode 100755 index c9db70c..0000000 --- a/easy-ocw/install-ubuntu.sh +++ /dev/null @@ -1,157 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -help() -{ -cat << ENDHELP - -Easy OCW assists with the building of the Apache Open Climate Workbench and -its dependencies. - -Flags: - -h Display this help message. - -N.B. This install script has been tested against Ubuntu 12.04 and 14.04. -Please report problems with this script to d...@climate.apache.org -ENDHELP -} - -header() -{ - echo - echo $1 -} - -task() -{ - echo " - " $1 -} - -subtask() -{ - echo " " $1 -} - -echo -echo "---------------------------------------------------------------------------" -echo " Welcome to Easy OCW" -echo "---------------------------------------------------------------------------" -echo - -# Find absolute path to the easy-ocw directory -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -cd $DIR -ocw_path="$DIR/../climate" - -while getopts ":h" FLAG -do - case $FLAG in - h) - help - exit 1 - ;; - ?) - help - exit 1 - ;; - esac -done - -cat << ENDINTRO -A number of dependencies for OCW will now be installed. Please check the wiki -for a complete list of dependencies. Additionally, please read the wiki for -useful installation guidelines and information that may be pertinent to your -situation. All of this can be found at http://s.apache.org/3p2 -ENDINTRO - -cat << VIRTUALENV_WARNING -$(tput setaf 1)<-----------------------------[WARNING!]----------------------------------->$(tput sgr 0) -Easy OCW will automatically create a conda environment "venv-ocw". -All the dependencies will be installed in the virtual environment -to ensure that your global Python install is not affected. - -Please activate the virtual environment venv-ocw after the script has completed to use OCW. - -To activate the virtual environment execute -> source active venv-ocw - -To deactivate the virtual environment execute -> source deactivate - -VIRTUALENV_WARNING - -echo "Easy-OCW script logs" > install_log - -# Install Continuum Analytics Miniconda Python distribution. This gives -# almost all the dependencies that OCW needs in a single, easy to -# install package. -header "Installing Miniconda Python distribution ..." - -MACHINE_TYPE=`uname -m` -if [ ${MACHINE_TYPE} == 'x86_64' ]; then - link="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh"; -else - link="https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86.sh"; -fi - -header "Checking for conda ..." -command -v conda >/dev/null 2>&1 || { - task "Unable to locate conda." - if ! [ -f Miniconda-latest-linux.sh ]; then - task "Downloading Miniconda ..." - wget -O Miniconda-latest-linux.sh $link >> install_log 2>&1 - subtask "done" - fi - task "Installing ..." - bash Miniconda-latest-linux.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin/:$PATH" - subtask "done" -} - -header "Creating venv-ocw environment and installing dependencies" -conda create --name venv-ocw -y --file ocw-conda-dependencies.txt -task "Activating venv-ocw virtual environment" -source activate venv-ocw - -conda config --set always_yes yes -conda update -q conda - -header "Conda Information" -echo "------------------------------------------------------------------" -conda info -a - -echo -task "Installing python-dev" -sudo apt-get -y install python-dev >> install_log 2>&1 -subtask "done" - -cd $DIR - -header "Installing remaining dependencies via pip" -pip install -r ocw-pip-dependencies.txt -subtask "done" - -header "Installing ocw module" - -cd .. -python setup.py install -subtask "Finished installing OCW module" - -header "For any issues with installation please contact d...@climate.apache.org" http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/easy-ocw/ocw-conda-dependencies.txt ---------------------------------------------------------------------- diff --git a/easy-ocw/ocw-conda-dependencies.txt b/easy-ocw/ocw-conda-dependencies.txt deleted file mode 100644 index 665be4d..0000000 --- a/easy-ocw/ocw-conda-dependencies.txt +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -numpy>=1.10.4 -scipy>=0.17.0 -matplotlib>=1.5.1 -basemap>=1.0.7 -netcdf4>=1.2.2 -h5py>=2.6.0 http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/easy-ocw/ocw-pip-dependencies.txt ---------------------------------------------------------------------- diff --git a/easy-ocw/ocw-pip-dependencies.txt b/easy-ocw/ocw-pip-dependencies.txt deleted file mode 100644 index 867c801..0000000 --- a/easy-ocw/ocw-pip-dependencies.txt +++ /dev/null @@ -1,12 +0,0 @@ -requests>=2.10.0 -bottle>=0.12.9 -pydap>=3.1.1 -webtest>=2.0.21 -nose>=1.3.7 -sphinx>=1.4.4 -sphinxcontrib-httpdomain>=1.5.0 -esgf-pyclient>=0.1.6 -python-dateutil>=2.5.3 -mock>=2.0.0 -myproxyclient>=1.4.3 -podaacpy>=1.0.2 http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/ocw/dataset_loader.py ---------------------------------------------------------------------- diff --git a/ocw/dataset_loader.py b/ocw/dataset_loader.py index ed8fdb0..fdcbb1b 100644 --- a/ocw/dataset_loader.py +++ b/ocw/dataset_loader.py @@ -21,11 +21,9 @@ Classes: ''' import ocw.data_source.local as local -import ocw.data_source.esgf as esgf import ocw.data_source.rcmed as rcmed -import ocw.data_source.dap as dap import ocw.data_source.podaac_datasource as podaac - +import warnings class DatasetLoader: '''Generate a list of OCW Dataset objects from a variety of sources.''' @@ -95,11 +93,19 @@ class DatasetLoader: self._source_loaders = { 'local': local.load_multiple_files, 'local_split': local.load_dataset_from_multiple_netcdf_files, - 'esgf': esgf.load_dataset, 'rcmed': rcmed.parameter_dataset, - 'dap': dap.load, 'podaac': podaac.load_dataset } + + # Exclude esgf and dap for python 3 until they are compatible + try: + import ocw.data_source.esgf as esgf + import ocw.data_source.dap as dap + self._source_loaders['dap'] = dap.load + self._source_loaders['esgf'] = esgf.load_dataset + except ImportError: + warnings.warn('dap and esgf loaders missing. If these are needed, ' + 'fallback to python 2.7.x.') def add_source_loader(self, loader_name, loader_func): ''' http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/test.sh ---------------------------------------------------------------------- diff --git a/test.sh b/test.sh index 0ea488d..cc73d04 100755 --- a/test.sh +++ b/test.sh @@ -25,7 +25,11 @@ echo "" # nosetests config file should be in home directory cp .noserc $HOME/.noserc +# Exclude dap tests if using python 3 +if [[ "$1" != "2.7" ]]; then + export NOSE_EXCLUDE_TESTS=ocw.tests.test_dap.TestDap +fi + echo "---------------- Running Unit Tests ---------------" nosetests echo "---------------- All Tests successfully completed ---------------" - http://git-wip-us.apache.org/repos/asf/climate/blob/fc520781/test_smoke.py ---------------------------------------------------------------------- diff --git a/test_smoke.py b/test_smoke.py index b1599df..f08d72a 100644 --- a/test_smoke.py +++ b/test_smoke.py @@ -21,9 +21,13 @@ from ocw.tests.test_local import create_netcdf_object from ocw.data_source import local from ocw import dataset_processor as dsp import os +import six + +if six.PY3: + DEPENDENCIES_FILE = 'deps_py3.txt' +else: + DEPENDENCIES_FILE = 'deps_py2.txt' -PIP_DEPENDENCIES_FILE = 'easy-ocw/ocw-pip-dependencies.txt' -CONDA_DEPENDENCIES_FILE = 'easy-ocw/ocw-conda-dependencies.txt' SUCCESS_MARK = '\033[92m' + u'\u2713' + '\033[0m' FAILURE_MARK = '\033[91m' + u'\u274C' + '\033[0m' @@ -123,11 +127,9 @@ def end(dataset=None): def main(): - pip_file = open(PIP_DEPENDENCIES_FILE, 'r') - conda_file = open(CONDA_DEPENDENCIES_FILE, 'r') + dep_file = open(DEPENDENCIES_FILE, 'r') print("Checking installed dependencies\n") - check_dependencies(conda_file) - check_dependencies(pip_file) + check_dependencies(dep_file) success("\nDependencies") dataset = check_dataset_loading() check_some_dataset_functions(dataset)
