This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 0374adb ARROW-2613: [Docs] Update the gen_apidocs docker script
0374adb is described below
commit 0374adb3546738271fdeb4e55926430b3dcbb0de
Author: Korn, Uwe <uwe.k...@blue-yonder.com>
AuthorDate: Mon May 21 14:33:10 2018 -0400
ARROW-2613: [Docs] Update the gen_apidocs docker script
Author: Korn, Uwe <uwe.k...@blue-yonder.com>
Closes #2068 from xhochy/ARROW-2613 and squashes the following commits:
ec955c4a <Korn, Uwe> ARROW-2613: Update the gen_apidocs docker script
---
dev/gen_apidocs/Dockerfile | 49 ++++++++++++-----------
dev/gen_apidocs/create_documents.sh | 79 +++++++++++++++----------------------
dev/run_docker_compose.sh | 1 +
site/README.md | 29 +-------------
4 files changed, 59 insertions(+), 99 deletions(-)
diff --git a/dev/gen_apidocs/Dockerfile b/dev/gen_apidocs/Dockerfile
index da740ee..13a6ff4 100644
--- a/dev/gen_apidocs/Dockerfile
+++ b/dev/gen_apidocs/Dockerfile
@@ -14,40 +14,38 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-FROM ubuntu:16.04
+FROM ubuntu:18.04
# Basic OS dependencies
-RUN apt-get update && apt-get install -y \
- wget \
+# First install JDK 8, then do a second apt call to install maven.
+# Otherwise this will install openjdk 8 and openjdk 11 at once.
+RUN apt-get update && \
+ apt-get install -y openjdk-8-jdk-headless && \
+ apt-get install -y \
+ gcc-8 \
+ g++-8 \
+ vim \
+ git \
+ wget \
+ make \
+ ninja-build \
rsync \
- git \
- gcc-4.9 \
- g++-4.9 \
- build-essential \
- software-properties-common
-
-# Java build fails with default JDK8
-RUN add-apt-repository ppa:openjdk-r/ppa &&\
- apt-get update &&\
- apt-get install -y openjdk-7-jdk &&\
- update-java-alternatives -s java-1.7.0-openjdk-amd64
+ maven \
+ pkg-config \
+ gobject-introspection \
+ libgirepository1.0-dev \
+ gtk-doc-tools \
+ libtool \
+ autoconf-archive \
+ npm
# This will install conda in /home/ubuntu/miniconda
RUN wget -O /tmp/miniconda.sh \
https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
bash /tmp/miniconda.sh -b -p /home/ubuntu/miniconda && \
rm /tmp/miniconda.sh
-# C_Glib dependencies
-RUN apt-get install -y \
- libgtk2.0-dev \
- gtk-doc-tools \
- gobject-introspection \
- libgirepository1.0-dev \
- autogen \
- autoconf-archive
+
# Python dependencies
-RUN apt-get install -y \
- pkg-config
# Create Conda environment
RUN /home/ubuntu/miniconda/bin/conda create -y -q -n pyarrow-dev \
# Python
@@ -79,6 +77,9 @@ RUN /home/ubuntu/miniconda/bin/conda create -y -q -n
pyarrow-dev \
maven \
-c conda-forge
+ENV CC=gcc-8
+ENV CXX=g++-8
+
ADD . /apache-arrow
WORKDIR /apache-arrow
CMD arrow/dev/gen_apidocs/create_documents.sh
diff --git a/dev/gen_apidocs/create_documents.sh
b/dev/gen_apidocs/create_documents.sh
index 6332ac9..f7d5ed3 100755
--- a/dev/gen_apidocs/create_documents.sh
+++ b/dev/gen_apidocs/create_documents.sh
@@ -20,66 +20,53 @@ set -ex
# Set up environment and output directory for C++ libraries
cd /apache-arrow
-rm -rf dist
-mkdir dist
+
+mkdir -p apidocs-dist
export ARROW_BUILD_TYPE=release
-export ARROW_HOME=$(pwd)/dist
-export PARQUET_HOME=$(pwd)/dist
+export ARROW_HOME=$(pwd)/apidocs-dist
+export PARQUET_HOME=$(pwd)/apidocs-dist
CONDA_BASE=/home/ubuntu/miniconda
-export LD_LIBRARY_PATH=$(pwd)/dist/lib:${CONDA_BASE}/lib:${LD_LIBRARY_PATH}
-export PKG_CONFIG_PATH=$(pwd)/dist/lib/pkgconfig:${PKG_CONFIG_PATH}
+export PKG_CONFIG_PATH=$(pwd)/apidocs-dist/lib/pkgconfig:${PKG_CONFIG_PATH}
export PATH=${CONDA_BASE}/bin:${PATH}
-
-# Prepare the asf-site before copying api docs
-pushd arrow/site
-rm -rf asf-site
-export GIT_COMMITTER_NAME="Nobody"
-export GIT_COMMITTER_EMAIL="nob...@nowhere.com"
-git clone --branch=asf-site \
- https://git-wip-us.apache.org/repos/asf/arrow-site.git asf-site
-popd
+# For newer GCC per
https://arrow.apache.org/docs/python/development.html#known-issues
+export CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0"
# Make Java documentation
-export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
-wget
http://mirrors.gigenet.com/apache/maven/maven-3/3.5.2/binaries/apache-maven-3.5.2-bin.tar.gz
-tar xvf apache-maven-3.5.2-bin.tar.gz
-export PATH=$(pwd)/apache-maven-3.5.2/bin:$PATH
-
+# Override user.home to cache dependencies outside the Docker container
pushd arrow/java
-rm -rf target/site/apidocs/*
-mvn -Drat.skip=true clean
-mvn -Drat.skip=true install
-mvn -Drat.skip=true site
-mkdir -p ../site/asf-site/docs/java/
-rsync -r target/site/apidocs/ ../site/asf-site/docs/java/
+#mvn -Duser.home=`pwd`/.apidocs-m2 -Drat.skip=true -Dcheckstyle.skip=true
install site
+#mkdir -p ../site/asf-site/docs/java/
+#rsync -r target/site/apidocs/ ../site/asf-site/docs/java/
+popd
+
+# Make Javascript documentation
+pushd arrow/js
+npm install
+npm run doc
+rsync -r doc/ ../site/asf-site/docs/js
popd
# Make Python documentation (Depends on C++ )
# Build Arrow C++
source activate pyarrow-dev
-
export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX
-export BOOST_ROOT=$CONDA_PREFIX
export PARQUET_BUILD_TOOLCHAIN=$CONDA_PREFIX
-export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:${LD_LIBRARY_PATH}
-export PKG_CONFIG_PATH=$CONDA_PREFIX/lib/pkgconfig:${PKG_CONFIG_PATH}
-
-export CC=gcc-4.9
-export CXX=g++-4.9
+export
LD_LIBRARY_PATH=$(pwd)/apidocs-dist/lib:${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH}
+export PKG_CONFIG_PATH=${CONDA_PREFIX}/lib/pkgconfig:${PKG_CONFIG_PATH}
+export PKG_CONFIG_PATH=$(pwd)/apidocs-dist/lib/pkgconfig:${PKG_CONFIG_PATH}
-CPP_BUILD_DIR=$(pwd)/arrow/cpp/build_docs
+CPP_BUILD_DIR=$(pwd)/arrow/cpp/build_apidocs
-rm -rf $CPP_BUILD_DIR
-mkdir $CPP_BUILD_DIR
+mkdir -p $CPP_BUILD_DIR
pushd $CPP_BUILD_DIR
cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
-DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
-DARROW_PYTHON=on \
-DARROW_PLASMA=on \
-DARROW_BUILD_TESTS=OFF \
+ -GNinja \
..
-make -j4
-make install
+ninja install
popd
# Build c_glib documentation
@@ -90,9 +77,8 @@ if [ -f Makefile ]; then
make distclean
fi
./autogen.sh
-rm -rf build_docs
-mkdir build_docs
-pushd build_docs
+mkdir -p build_apidocs
+pushd build_apidocs
../configure \
--prefix=${AROW_HOME} \
--enable-gtk-doc
@@ -103,23 +89,20 @@ popd
popd
# Build Parquet C++
-rm -rf parquet-cpp/build_docs
-mkdir parquet-cpp/build_docs
-pushd parquet-cpp/build_docs
+mkdir -p parquet-cpp/build_apidocs
+pushd parquet-cpp/build_apidocs
cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
-DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
-DPARQUET_BUILD_BENCHMARKS=off \
-DPARQUET_BUILD_EXECUTABLES=off \
-DPARQUET_BUILD_TESTS=off \
+ -GNinja \
..
-make -j4
-make install
+ninja install
popd
# Now Python documentation can be built
pushd arrow/python
-rm -rf build/*
-rm -rf doc/_build
python setup.py build_ext --build-type=$ARROW_BUILD_TYPE \
--with-plasma --with-parquet --inplace
python setup.py build_sphinx -s doc/source
diff --git a/dev/run_docker_compose.sh b/dev/run_docker_compose.sh
index bb353b5..503efd5 100755
--- a/dev/run_docker_compose.sh
+++ b/dev/run_docker_compose.sh
@@ -35,4 +35,5 @@ if [ ! -d parquet-cpp ]; then
exit 1
fi
+docker-compose -f arrow/dev/docker-compose.yml build "${@}"
docker-compose -f arrow/dev/docker-compose.yml run --rm "${@}"
diff --git a/site/README.md b/site/README.md
index 3da7c97..62d060a 100644
--- a/site/README.md
+++ b/site/README.md
@@ -59,33 +59,8 @@ git push
### Updating Code Documentation
-#### Java
-
-```
-cd ../java
-mvn install
-mvn site
-rsync -r target/site/apidocs/ ../site/asf-site/docs/java/
-```
-
-#### C++
-
-```
-cd ../cpp/apidoc
-doxygen Doxyfile
-rsync -r html/ ../../site/asf-site/docs/cpp
-```
-
-#### Python
-
-First, build PyArrow with all optional extensions (Apache Parquet).
-
-```
-cd ../python
-python setup.py build_ext --inplace --with-parquet --with-plasma
-python setup.py build_sphinx -s doc/source
-rsync -r doc/_build/html/ ../site/asf-site/docs/python/
-```
+To update the documentation, run the script `./dev/gen_apidocs.sh`. This script
+will run the code documentation tools in a fixed environment.
#### C (GLib)
--
To stop receiving notification emails like this one, please contact
w...@apache.org.
