This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/main by this push:
new b5a0939 YETUS-1065. Add support for codespell (#191)
b5a0939 is described below
commit b5a093982e5aae31ce1a7f4fe6ef24031edc6c12
Author: Allen Wittenauer <[email protected]>
AuthorDate: Tue Nov 17 12:33:27 2020 -0800
YETUS-1065. Add support for codespell (#191)
Signed-off-by: Roman Shaposhnik <[email protected]>
---
.codespellrc => .codespellignorelines | 7 +-
.codespellrc | 4 +-
asf-site-src/config.rb | 2 +-
.../precommit/plugins/codespell.html.md | 52 +++++++
precommit/src/main/shell/core.d/change-analysis.sh | 59 +++++++-
.../src/main/shell/test-patch-docker/Dockerfile | 2 +-
precommit/src/main/shell/test-patch.d/codespell.sh | 159 +++++++++++++++++++++
website-tester.sh | 17 +++
8 files changed, 295 insertions(+), 7 deletions(-)
diff --git a/.codespellrc b/.codespellignorelines
similarity index 85%
copy from .codespellrc
copy to .codespellignorelines
index f34e714..fa991af 100644
--- a/.codespellrc
+++ b/.codespellignorelines
@@ -12,6 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
-[codespell]
-skip =
./*.css.*,./*.js,./.git/*,./.asf.yaml,./asf-site-src/config.rb,./asf-site-src/Gemfile.lock
\ No newline at end of file
+ rouge (~> 3.2)
+ rouge (3.23.0)
+ unless FileUtils.uptodate?(output, docs) &&
+ FileUtils.uptodate?(output, [SHELLDOCS])
diff --git a/.codespellrc b/.codespellrc
index f34e714..cb9f120 100644
--- a/.codespellrc
+++ b/.codespellrc
@@ -14,4 +14,6 @@
# limitations under the License.
[codespell]
-skip =
./*.css.*,./*.js,./.git/*,./.asf.yaml,./asf-site-src/config.rb,./asf-site-src/Gemfile.lock
\ No newline at end of file
+skip = *.css.*,*.js,./.git/*,./asf.yaml
+ignore-regex = class\=\"nd\"
+
diff --git a/asf-site-src/config.rb b/asf-site-src/config.rb
index b163495..033e532 100644
--- a/asf-site-src/config.rb
+++ b/asf-site-src/config.rb
@@ -181,7 +181,7 @@ after_configuration do # rubocop:disable Metrics/BlockLength
# For Precommit we regenerate source files so they can be rendered.
# we rely on a symlink. to avoid an error from the file watcher, our target
- # has to be outside of hte asf-site-src directory.
+ # has to be outside of the asf-site-src directory.
# TODO when we can, update to middleman 4 so we can use multiple source dirs
# instead of symlinks
FileUtils.mkdir_p 'target/in-progress/precommit/apidocs/'
diff --git
a/asf-site-src/source/documentation/in-progress/precommit/plugins/codespell.html.md
b/asf-site-src/source/documentation/in-progress/precommit/plugins/codespell.html.md
new file mode 100644
index 0000000..fba303a
--- /dev/null
+++
b/asf-site-src/source/documentation/in-progress/precommit/plugins/codespell.html.md
@@ -0,0 +1,52 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Name
+
+codespell
+
+# Category
+
+Test
+
+# Description
+
+Runs [codespell](https://github.com/codespell-project/codespell) on the
repository.
+
+Due to how `codespell` executes, `./` is prefixed onto paths at runtime. This
prefixing is
+to be a bit more consistent with how one would run it from the command line
such
+that `.codespellrc` is easier to manage.
+
+# Environment Variables
+
+None
+
+# Options
+
+| Option | Notes |
+|:---------|:------|
+| `--codespell-exclude-lines=<file>` | File of lines that codespell should
ignore (defaults to `.codespellignorelines`) |
+
+# Docker Notes
+
+None
+
+# Developer Notes
+
+None
diff --git a/precommit/src/main/shell/core.d/change-analysis.sh
b/precommit/src/main/shell/core.d/change-analysis.sh
index f9dfc08..4456751 100755
--- a/precommit/src/main/shell/core.d/change-analysis.sh
+++ b/precommit/src/main/shell/core.d/change-analysis.sh
@@ -81,6 +81,62 @@ function find_changed_files
popd >/dev/null || return 1
}
+
+## @description Determine directories with
+## @description changed content. Should be used with
+## @description static linters that don't care about
+## @description the build system.
+## @audience private
+## @stability evolving
+## @replaceable no
+## @return None; sets ${CHANGED_DIRS}
+function find_changed_dirs
+{
+ declare f
+ declare -a newarray
+ declare dir
+
+ CHANGED_DIRS=()
+ for f in "${CHANGED_FILES[@]}"; do
+ dir=$(dirname "./${f}")
+ if [[ "${dir}" = . ]]; then
+ CHANGED_DIRS=('.')
+ continue
+ fi
+ yetus_add_array_element CHANGED_DIRS "${dir}"
+ done
+
+ if [[ "${#CHANGED_DIRS[@]}" -eq 1 ]]; then
+ return
+ fi
+
+ echo "${#CHANGED_DIRS[@]}"
+
+ newarray=()
+
+ for f in "${CHANGED_DIRS[@]}"; do
+ dir=${f%/*}
+ found=false
+ while [[ ${dir} != "." ]] && [[ ${found} = false ]]; do
+ if yetus_array_contains "${dir}" "${newarray[@]}"; then
+ found=true
+ continue
+ fi
+ if yetus_array_contains "${dir}" "${CHANGED_DIRS[@]}"; then
+ found=true
+ continue
+ fi
+ dir=${dir%/*}
+ done
+
+ if [[ "${found}" == false ]]; then
+ newarray+=("${f}")
+ fi
+ done
+
+ CHANGED_DIRS=("${newarray[@]}")
+}
+
## @description Apply the EXCLUDE_PATHS to CHANGED_FILES
## @audience private
## @stability stable
@@ -93,7 +149,6 @@ function exclude_paths_from_changed_files
declare strip
declare -a a
-
# empty the existing list
EXCLUDE_PATHS=()
@@ -148,6 +203,8 @@ function exclude_paths_from_changed_files
done
CHANGED_FILES=("${a[@]}")
+
+ find_changed_dirs
}
## @description Check for directories to skip during
diff --git a/precommit/src/main/shell/test-patch-docker/Dockerfile
b/precommit/src/main/shell/test-patch-docker/Dockerfile
index ae0ccb8..9014851 100644
--- a/precommit/src/main/shell/test-patch-docker/Dockerfile
+++ b/precommit/src/main/shell/test-patch-docker/Dockerfile
@@ -319,6 +319,7 @@ RUN apt-get -q update && apt-get -q install
--no-install-recommends -y \
&& rm /usr/local/bin/pip /tmp/get-pip.py \
&& pip3 install -v \
astroid==2.4.2 \
+
git+https://github.com/codespell-project/codespell.git@8d8d4ae2bbd24a91d1a9bd8aec25c9f86724c280
\
docker-compose==1.26.2 \
pylint==2.5.3 \
yamllint==1.24.2 \
@@ -345,7 +346,6 @@ RUN echo 'gem: --no-rdoc --no-ri' >> /root/.gemrc \
ENV PATH ${PATH}:/var/tmp/.bundler-gems/bin
ENV BUNDLE_PATH /var/tmp/.bundler-gems
-
###
# Install npm and JSHint
###
diff --git a/precommit/src/main/shell/test-patch.d/codespell.sh
b/precommit/src/main/shell/test-patch.d/codespell.sh
new file mode 100755
index 0000000..c4325f8
--- /dev/null
+++ b/precommit/src/main/shell/test-patch.d/codespell.sh
@@ -0,0 +1,159 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# no public APIs here
+# SHELLDOC-IGNORE
+
+add_test_type codespell
+
+CODESPELL_TIMER=0
+CODESPELL=${CODESPELL:-$(command -v codespell 2>/dev/null)}
+CODESPELL_X_FILE=".codespellignorelines"
+
+function github_usage
+{
+ yetus_add_option "--codespell-exclude-lines=<file>" "Lines to ignore via
codespell -x (default: '${CODESPELL_X_FILE}')"
+}
+
+function github_parse_args
+{
+ declare i
+
+ for i in "$@"; do
+ case ${i} in
+ --codespell-exclude-lines=*)
+ delete_parameter "${i}"
+ CODESPELL_X_FILE=${i#*=}
+ ;;
+ esac
+ done
+
+ CODESPELL_X_FILE=$(yetus_abs "${CODESPELL_X_FILE}")
+}
+
+function codespell_filefilter
+{
+ add_test codespell
+}
+
+function codespell_precheck
+{
+ if ! verify_command "codespell" "${CODESPELL}"; then
+ add_vote_table_v2 0 codespell "" "codespell was not available."
+ delete_test codespell
+ fi
+}
+
+function codespell_logic
+{
+ declare repostatus=$1
+ declare -a codespellargs
+ declare i
+
+ pushd "${BASEDIR}" >/dev/null || return 1
+
+ # codespell will ignore skip directives if you give it
+ # a specific file name. so best we can do is
+ # use CHANGED_DIRS[@]. Will still need to filter out
+ # files, but this should at least cut back on the runtime
+
+ if [[ -f "${CODESPELL_X_FILE}" ]]; then
+ codespellargs=(-x "${CODESPELL_X_FILE}")
+ fi
+
+ for i in "${CHANGED_DIRS[@]}"; do
+ if [[ -f "${i}" ]]; then
+ # specifically add ./ because otherwise the .codespellrc file gets weird
+ "${CODESPELL}" \
+ --disable-colors \
+ --interactive 0 \
+ --quiet-level 2 \
+ "${codespellargs[@]}" \
+ "./${i}" \
+ | "${SED}" -e 's,^./,,g' \
+ >> "${PATCH_DIR}/${repostatus}-codespell-tmp.txt" \
+ 2>/dev/null
+ fi
+ done
+
+ for i in "${CHANGED_FILES[@]}"; do
+ "${GREP}" -E "^${i}:" \
+ "${PATCH_DIR}/${repostatus}-codespell-tmp.txt" \
+ >> "${PATCH_DIR}/${repostatus}-codespell-result.txt"
+ done
+
+ popd > /dev/null || return 1
+}
+
+function codespell_preapply
+{
+ if ! verify_needed_test codespell; then
+ return 0
+ fi
+
+ big_console_header "codespell plugin: ${PATCH_BRANCH}"
+
+ start_clock
+
+ codespell_logic branch
+
+ # keep track of how much as elapsed for us already
+ CODESPELL_TIMER=$(stop_clock)
+ return 0
+}
+
+function codespell_calcdiffs
+{
+ error_calcdiffs "$@"
+}
+
+function codespell_postapply
+{
+ declare version
+
+ if ! verify_needed_test codespell; then
+ return 0
+ fi
+
+ big_console_header "codespell plugin: ${BUILDMODE}"
+
+ start_clock
+
+ # add our previous elapsed to our new timer
+ # by setting the clock back
+ offset_clock "${CODESPELL_TIMER}"
+
+ codespell_logic patch
+
+ version=$("${CODESPELL}" --version)
+ add_version_data codespell "${version##* v}"
+
+ root_postlog_compare \
+ codespell \
+ "${PATCH_DIR}/branch-codespell-result.txt" \
+ "${PATCH_DIR}/patch-codespell-result.txt"
+}
+
+function codespell_postcompile
+{
+ declare repostatus=$1
+
+ if [[ "${repostatus}" = branch ]]; then
+ codespell_preapply
+ else
+ codespell_postapply
+ fi
+}
diff --git a/website-tester.sh b/website-tester.sh
index 8e4367f..3139c91 100755
--- a/website-tester.sh
+++ b/website-tester.sh
@@ -43,6 +43,23 @@ linkchecker \
result=$?
echo "::endgroup::"
+echo "::group::codespell releasenotes"
+codespell \
+ --disable-colors \
+ --interactive 0 \
+ --quiet-level 2 \
+ ./asf-site-src/source/downloads/releasenotes \
+| sed -e 's,^./,,g' \
+ > /tmp/codespell.txt
+
+while read -r; do
+ filename=$(echo "${REPLY}" | cut -f1 -d:)
+ line=$(echo "${REPLY}" | cut -f2 -d:)
+ msg=$(echo "${REPLY}" | cut -f3 -d:)
+ echo "::error file=${filename},line=${line}:: codespell: ${msg}"
+done < /tmp/codespell.txt
+echo "::endgroup::"
+
#
#
urlname;parentname;base;result;warningstring;infostring;valid;url;line;column;name;dltime;size;checktime;cached;level;modified
# in-page reference: $1
