Repository: bahir-website Updated Branches: refs/heads/master 0d53ccdd4 -> 4142e1e34
[BAHIR-92] Add Flink extensions in update-doc.sh Refactoring update-doc.sh script to include Bahir-Flink extensions and make it easier to add future extensions. Closes #8 Project: http://git-wip-us.apache.org/repos/asf/bahir-website/repo Commit: http://git-wip-us.apache.org/repos/asf/bahir-website/commit/4142e1e3 Tree: http://git-wip-us.apache.org/repos/asf/bahir-website/tree/4142e1e3 Diff: http://git-wip-us.apache.org/repos/asf/bahir-website/diff/4142e1e3 Branch: refs/heads/master Commit: 4142e1e34462c3037e0db5366bee28f27b00ff37 Parents: 0d53ccd Author: Tzu-Li (Gordon) Tai <[email protected]> Authored: Mon Mar 6 01:20:35 2017 -0800 Committer: Christian Kadner <[email protected]> Committed: Mon Mar 6 01:20:35 2017 -0800 ---------------------------------------------------------------------- .../templates/flink-streaming-activemq.template | 27 +++++ .../templates/flink-streaming-akka.template | 27 +++++ .../templates/flink-streaming-flume.template | 27 +++++ .../templates/flink-streaming-netty.template | 27 +++++ .../templates/flink-streaming-redis.template | 27 +++++ update-doc.sh | 117 ++++++++++++------- 6 files changed, 212 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bahir-website/blob/4142e1e3/site/docs/flink/templates/flink-streaming-activemq.template ---------------------------------------------------------------------- diff --git a/site/docs/flink/templates/flink-streaming-activemq.template b/site/docs/flink/templates/flink-streaming-activemq.template new file mode 100644 index 0000000..0116e96 --- /dev/null +++ b/site/docs/flink/templates/flink-streaming-activemq.template @@ -0,0 +1,27 @@ +--- +layout: page +title: Apache Flink Streaming Connector for ActiveMQ +description: Apache Flink Streaming Connector for ActiveMQ +group: nav-right +--- +<!-- +{% comment %} +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to you under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% endcomment %} +--> + +{% include JB/setup %} + http://git-wip-us.apache.org/repos/asf/bahir-website/blob/4142e1e3/site/docs/flink/templates/flink-streaming-akka.template ---------------------------------------------------------------------- diff --git a/site/docs/flink/templates/flink-streaming-akka.template b/site/docs/flink/templates/flink-streaming-akka.template new file mode 100644 index 0000000..c0d6790 --- /dev/null +++ b/site/docs/flink/templates/flink-streaming-akka.template @@ -0,0 +1,27 @@ +--- +layout: page +title: Apache Flink Streaming Connector for Akka +description: Apache Flink Streaming Connector for Akka +group: nav-right +--- +<!-- +{% comment %} +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to you under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% endcomment %} +--> + +{% include JB/setup %} + http://git-wip-us.apache.org/repos/asf/bahir-website/blob/4142e1e3/site/docs/flink/templates/flink-streaming-flume.template ---------------------------------------------------------------------- diff --git a/site/docs/flink/templates/flink-streaming-flume.template b/site/docs/flink/templates/flink-streaming-flume.template new file mode 100644 index 0000000..699ee05 --- /dev/null +++ b/site/docs/flink/templates/flink-streaming-flume.template @@ -0,0 +1,27 @@ +--- +layout: page +title: Apache Flink Streaming Connector for Apache Flume +description: Apache Flink Streaming Connector for Apache Flume +group: nav-right +--- +<!-- +{% comment %} +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to you under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% endcomment %} +--> + +{% include JB/setup %} + http://git-wip-us.apache.org/repos/asf/bahir-website/blob/4142e1e3/site/docs/flink/templates/flink-streaming-netty.template ---------------------------------------------------------------------- diff --git a/site/docs/flink/templates/flink-streaming-netty.template b/site/docs/flink/templates/flink-streaming-netty.template new file mode 100644 index 0000000..fc57b6d --- /dev/null +++ b/site/docs/flink/templates/flink-streaming-netty.template @@ -0,0 +1,27 @@ +--- +layout: page +title: Apache Flink Streaming Connector for Netty +description: Apache Flink Streaming Connector for Netty +group: nav-right +--- +<!-- +{% comment %} +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to you under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% endcomment %} +--> + +{% include JB/setup %} + http://git-wip-us.apache.org/repos/asf/bahir-website/blob/4142e1e3/site/docs/flink/templates/flink-streaming-redis.template ---------------------------------------------------------------------- diff --git a/site/docs/flink/templates/flink-streaming-redis.template b/site/docs/flink/templates/flink-streaming-redis.template new file mode 100644 index 0000000..8d3c7dc --- /dev/null +++ b/site/docs/flink/templates/flink-streaming-redis.template @@ -0,0 +1,27 @@ +--- +layout: page +title: Apache Flink Streaming Connector for Redis +description: Apache Flink Streaming Connector for Redis +group: nav-right +--- +<!-- +{% comment %} +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to you under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% endcomment %} +--> + +{% include JB/setup %} + http://git-wip-us.apache.org/repos/asf/bahir-website/blob/4142e1e3/update-doc.sh ---------------------------------------------------------------------- diff --git a/update-doc.sh b/update-doc.sh index 81e4a84..d5615a5 100755 --- a/update-doc.sh +++ b/update-doc.sh @@ -43,62 +43,99 @@ set -e BASE_DIR=$(pwd) -WEBSITE_TEMPLATES_DIR=$BASE_DIR/site/docs/spark/templates -WEBSITE_DOC_DIR=$BASE_DIR/site/docs/spark/current -BAHIR_SOURCE_DIR=$BASE_DIR/target/bahir + +SPARK_WEBSITE_TEMPLATES_DIR=$BASE_DIR/site/docs/spark/templates +SPARK_WEBSITE_DOC_DIR=$BASE_DIR/site/docs/spark/current +SPARK_REPO_NAME=bahir +SPARK_BAHIR_SOURCE_DIR=$BASE_DIR/target/$SPARK_REPO_NAME + +FLINK_WEBSITE_TEMPLATES_DIR=$BASE_DIR/site/docs/flink/templates +FLINK_WEBSITE_DOC_DIR=$BASE_DIR/site/docs/flink/current +FLINK_REPO_NAME=bahir-flink +FLINK_BAHIR_SOURCE_DIR=$BASE_DIR/target/$FLINK_REPO_NAME function checkout_code { # Checkout code + cd "$BASE_DIR" # make sure we're in the base dir + rm -rf target mkdir target cd target - git clone https://git-wip-us.apache.org/repos/asf/bahir.git --quiet - cd bahir - git checkout $GIT_REF - git_hash=`git rev-parse --short HEAD` - echo "Checked out Bahir git hash $git_hash" + git clone "https://git-wip-us.apache.org/repos/asf/$1.git"; --quiet + cd "$1" + git checkout "$GIT_REF" + git_hash=$(git rev-parse --short HEAD) + echo "Checked out $1 git hash $git_hash" git clean -d -f -x - cd "$BASE_DIR" #return to base dir + cd "$BASE_DIR" # return to base dir } -checkout_code +function update_docs { + WEBSITE_TEMPLATES_DIR=$1 + WEBSITE_DOC_DIR=$2 + SOURCE_DIR=$3 + shift 3 + + [ ! -d "$WEBSITE_DOC_DIR" ] && mkdir "$WEBSITE_DOC_DIR" + + while [ $# -ne 0 ]; do + echo "Syncing document $WEBSITE_DOC_DIR/$1.md with source $SOURCE_DIR/$2/README.md" + rm -f "$WEBSITE_DOC_DIR/$1.md" + cp "$WEBSITE_TEMPLATES_DIR/$1.template" "$WEBSITE_DOC_DIR/$1.md" + cat "$SOURCE_DIR/$2/README.md" >> "$WEBSITE_DOC_DIR/$1.md" + shift 2 + done +} -rm -rf $WEBSITE_DOC_DIR/spark*.md +function check_version_strings { + if grep -q -r "[0-9]-SNAPSHOT" "$1"/*.md ; then + echo + echo "TODO: Replace '...-SNAPSHOT' version strings:" + echo + grep -r -n "[0-9]-SNAPSHOT" "$1"/*.md | sed -e 's|'"$(pwd)"/'||g' | grep --color "[0-9.]*-SNAPSHOT" + echo + echo "i.e. to replace '2.1.0-SNAPSHOT' with '2.0.2' run the following command:" + echo + echo " perl -i -pe 's/2.1.0-SNAPSHOT/2.0.2/g' $1/*.md" + echo + else + echo + echo "Generated files:" + echo + ls "$1"/*.md | xargs -n1 | sed -e 's|'"$(pwd -P)"/'||g' + fi +} -cp $WEBSITE_TEMPLATES_DIR/spark-sql-streaming-mqtt.template $WEBSITE_DOC_DIR/spark-sql-streaming-mqtt.md -cat $BAHIR_SOURCE_DIR/sql-streaming-mqtt/README.md >> $WEBSITE_DOC_DIR/spark-sql-streaming-mqtt.md +echo +echo "================= Updating Apache Spark Extension documents =================" +echo -cp $WEBSITE_TEMPLATES_DIR/spark-streaming-akka.template $WEBSITE_DOC_DIR/spark-streaming-akka.md -cat $BAHIR_SOURCE_DIR/streaming-akka/README.md >> $WEBSITE_DOC_DIR/spark-streaming-akka.md +checkout_code $SPARK_REPO_NAME -cp $WEBSITE_TEMPLATES_DIR/spark-streaming-mqtt.template $WEBSITE_DOC_DIR/spark-streaming-mqtt.md -cat $BAHIR_SOURCE_DIR/streaming-mqtt/README.md >> $WEBSITE_DOC_DIR/spark-streaming-mqtt.md +update_docs "$SPARK_WEBSITE_TEMPLATES_DIR" "$SPARK_WEBSITE_DOC_DIR" "$SPARK_BAHIR_SOURCE_DIR" \ + spark-sql-streaming-mqtt sql-streaming-mqtt \ + spark-streaming-akka streaming-akka \ + spark-streaming-mqtt streaming-mqtt \ + spark-streaming-twitter streaming-twitter \ + spark-streaming-zeromq streaming-zeromq -cp $WEBSITE_TEMPLATES_DIR/spark-streaming-twitter.template $WEBSITE_DOC_DIR/spark-streaming-twitter.md -cat $BAHIR_SOURCE_DIR/streaming-twitter/README.md >> $WEBSITE_DOC_DIR/spark-streaming-twitter.md +check_version_strings "$SPARK_WEBSITE_DOC_DIR" -cp $WEBSITE_TEMPLATES_DIR/spark-streaming-zeromq.template $WEBSITE_DOC_DIR/spark-streaming-zeromq.md -cat $BAHIR_SOURCE_DIR/streaming-zeromq/README.md >> $WEBSITE_DOC_DIR/spark-streaming-zeromq.md +echo +echo "================= Updating Apache Flink Extension documents =================" +echo -set +e +checkout_code $FLINK_REPO_NAME + +update_docs "$FLINK_WEBSITE_TEMPLATES_DIR" "$FLINK_WEBSITE_DOC_DIR" "$FLINK_BAHIR_SOURCE_DIR" \ + flink-streaming-activemq flink-connector-activemq \ + flink-streaming-akka flink-connector-akka \ + flink-streaming-flume flink-connector-flume \ + flink-streaming-netty flink-connector-netty \ + flink-streaming-redis flink-connector-redis -echo "Done." - -if grep -q -r "[0-9]-SNAPSHOT" $WEBSITE_DOC_DIR/spark*.md ; then - echo - echo "TODO: Replace '...-SNAPSHOT' version strings:" - echo - grep -r -n "[0-9]-SNAPSHOT" $WEBSITE_DOC_DIR/spark*.md | sed -e 's|'$(pwd)/'||g' | grep --color "[0-9.]*-SNAPSHOT" - echo - echo "i.e. to replace '2.1.0-SNAPSHOT' with '2.0.2' run the following command:" - echo - echo " perl -i -pe 's/2.1.0-SNAPSHOT/2.0.2/g' $WEBSITE_DOC_DIR/spark*.md" - echo -else - echo - echo "Generated files:" - echo - ls $WEBSITE_DOC_DIR/spark*.md | xargs -n1 | sed -e 's|'$(pwd -P)/'||g' -fi +check_version_strings "$FLINK_WEBSITE_DOC_DIR" + +set +e
