This is an automated email from the ASF dual-hosted git repository.
lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 0713adc59 NUTCH-3224 crawl --wait hour suffix waits 120 seconds
instead of one hour (#988)
0713adc59 is described below
commit 0713adc5965aa0a0f8f192c71aaa995309787c2f
Author: Lewis John McGibbney <[email protected]>
AuthorDate: Thu Nov 5 08:11:15 2026 -0800
NUTCH-3224 crawl --wait hour suffix waits 120 seconds instead of one hour
(#988)
---
.github/workflows/master-build.yml | 30 +++++++++++++++++++++
build.xml | 34 ++++++++++++++++++++++++
default.properties | 4 +++
src/bin/crawl | 9 +++++--
src/test/bin/crawl_test.sh | 53 ++++++++++++++++++++++++++++++++++++++
5 files changed, 128 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/master-build.yml
b/.github/workflows/master-build.yml
index c4e53ffce..a6a40fc6f 100644
--- a/.github/workflows/master-build.yml
+++ b/.github/workflows/master-build.yml
@@ -139,6 +139,36 @@ jobs:
if: steps.filter.outputs.jenkinsfile == 'true'
run: jenkinsfilelint --local --include 'Jenkinsfile*'
Jenkinsfile.smoke-test-single-node-hadoop
+ # Shell tests. Path-filtered: runs when any bash file changes, including
+ # src/bin/crawl, src/bin/nutch, and src/test/bin.
+ bashunit:
+ strategy:
+ matrix:
+ java: ['17']
+ os: [ubuntu-24.04]
+ runs-on: ${{ matrix.os }}
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@v7
+ - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d
+ id: filter
+ with:
+ filters: |
+ bashtests:
+ - 'src/bin/**'
+ - 'src/test/bin/**'
+ - '**/*.sh'
+ - 'src/plugin/parse-ext/command'
+ - name: Set up JDK ${{ matrix.java }}
+ if: steps.filter.outputs.bashtests == 'true'
+ uses: actions/setup-java@v6
+ with:
+ java-version: ${{ matrix.java }}
+ distribution: 'temurin'
+ - name: Run bashunit tests
+ if: steps.filter.outputs.bashtests == 'true'
+ run: ant test-bash -buildfile build.xml
+
# Build verification on JDK 17 and 21 (bytecode target Java 17)
build:
strategy:
diff --git a/build.xml b/build.xml
index a85175909..5aca3717e 100644
--- a/build.xml
+++ b/build.xml
@@ -465,6 +465,40 @@
</antcall>
</target>
+ <!-- bashunit is a test-only download. It is not part of the JUnit suite. -->
+ <target name="bashunit-check">
+ <mkdir dir="${build.dir}"/>
+ <available file="${build.dir}/bashunit-${bashunit.version}"
property="bashunit.present"/>
+ </target>
+
+ <target name="bashunit-download" depends="bashunit-check"
unless="bashunit.present"
+ description="--> download the pinned bashunit release">
+ <get
src="https://github.com/TypedDevs/bashunit/releases/download/${bashunit.version}/bashunit"
+ dest="${build.dir}/bashunit-${bashunit.version}"
usetimestamp="false"/>
+ </target>
+
+ <target name="bashunit-verify" depends="bashunit-download">
+ <checksum file="${build.dir}/bashunit-${bashunit.version}"
algorithm="SHA-256"
+ property="bashunit.sha256.actual"/>
+ <condition property="bashunit.checksum.ok">
+ <equals arg1="${bashunit.sha256.actual}" arg2="${bashunit.sha256}"
casesensitive="false"/>
+ </condition>
+ <!-- Drop a bad download so the next run fetches it again. -->
+ <antcall target="bashunit-reject"/>
+ <chmod file="${build.dir}/bashunit-${bashunit.version}" perm="755"/>
+ </target>
+
+ <target name="bashunit-reject" unless="bashunit.checksum.ok">
+ <delete file="${build.dir}/bashunit-${bashunit.version}"/>
+ <fail message="bashunit ${bashunit.version} checksum mismatch (expected
${bashunit.sha256}, got ${bashunit.sha256.actual})"/>
+ </target>
+
+ <target name="test-bash" depends="bashunit-verify" description="--> run
bashunit tests for src/bin">
+ <exec executable="${build.dir}/bashunit-${bashunit.version}"
failonerror="true" dir="${basedir}">
+ <arg value="src/test/bin"/>
+ </exec>
+ </target>
+
<target name="test-core"
depends="compile-core-test, job, jacoco-download"
description="--> run core JUnit tests">
diff --git a/default.properties b/default.properties
index da97dae37..493d3dfa7 100644
--- a/default.properties
+++ b/default.properties
@@ -52,6 +52,10 @@ spotbugs.version=4.10.5
# Apache RAT software distribution audit (https://creadur.apache.org/rat/)
apache-rat.version=0.18
+# bashunit (https://bashunit.com/) — shell tests for src/bin
+bashunit.version=0.52.0
+bashunit.sha256=6193370a2164abcc1e64601af461ea811505c6cefdc8fa33e579a07e2d05ffe3
+
# Forbidden APIs (https://github.com/policeman-tools/forbidden-apis)
forbiddenapis.version=3.11
diff --git a/src/bin/crawl b/src/bin/crawl
index 7f60ab120..ae8f4f6ad 100755
--- a/src/bin/crawl
+++ b/src/bin/crawl
@@ -64,8 +64,7 @@ function __to_seconds() {
SECONDS=$((10#$NUMBER * 60))
;;
h|H)
- # h means 120 seconds here, not 3600. A follow-up issue tracks the
correction.
- SECONDS=$((10#$NUMBER * 120))
+ SECONDS=$((10#$NUMBER * 3600))
;;
d|D)
SECONDS=$((10#$NUMBER * 86400))
@@ -121,6 +120,12 @@ function __print_usage {
exit 1
}
+# Functions above are sourced by the bashunit tests. The crawl itself runs
+# only when this file is executed.
+if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then
+ return 0
+fi
+
# default values
INDEXFLAG=false
HOSTDBUPDATE=false
diff --git a/src/test/bin/crawl_test.sh b/src/test/bin/crawl_test.sh
new file mode 100755
index 000000000..9a6af45d5
--- /dev/null
+++ b/src/test/bin/crawl_test.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+## @audience private
+## @stability stable
+function set_up() {
+ local root
+ root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
+ # Path is computed from this file; ShellCheck cannot follow it.
+ # shellcheck disable=SC1091
+ source "${root}/src/bin/crawl"
+}
+
+# Rows are: wait argument, expected seconds, description shown in the report.
+# Lowercase suffixes follow the usage text. Uppercase letters are dropped by
+# the suffix filter, so the remaining digits are counted as seconds.
+## @audience private
+## @stability stable
+function provider_wait_suffixes() {
+ bashunit::data_set "1h" "3600" "lowercase h is one hour, so --wait 1h is
3600 seconds"
+ bashunit::data_set "2h" "7200" "lowercase h is one hour, so --wait 2h is
7200 seconds"
+ bashunit::data_set "10h" "36000" "lowercase h is one hour, so --wait 10h is
36000 seconds"
+ bashunit::data_set "1m" "60" "lowercase m is one minute, so --wait 1m is 60
seconds"
+ bashunit::data_set "1d" "86400" "lowercase d is one day, so --wait 1d is
86400 seconds"
+ bashunit::data_set "1s" "1" "lowercase s is seconds, so --wait 1s is 1
second"
+ bashunit::data_set "15" "15" "no suffix means seconds, so --wait 15 is 15
seconds"
+ bashunit::data_set "1H" "1" "uppercase H is dropped by the suffix filter, so
--wait 1H is 1 second"
+ bashunit::data_set "1M" "1" "uppercase M is dropped by the suffix filter, so
--wait 1M is 1 second"
+ bashunit::data_set "1D" "1" "uppercase D is dropped by the suffix filter, so
--wait 1D is 1 second"
+}
+
+## @audience private
+## @stability stable
+# @data_provider provider_wait_suffixes
+function test_wait_suffix_::1::_is_::2::_seconds() {
+ bashunit::set_test_title "$3"
+ assert_same "$2" "$(__to_seconds "$1")"
+}