This is an automated email from the ASF dual-hosted git repository.
ostinru pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
The following commit(s) were added to refs/heads/main by this push:
new c8e39e19 Automation: run HBASE in standalone mode (#95)
c8e39e19 is described below
commit c8e39e19aacc0a5a5deb4a8065927afd7d82f725
Author: Nikolay Antonov <[email protected]>
AuthorDate: Wed Apr 15 08:37:23 2026 +0500
Automation: run HBASE in standalone mode (#95)
Start HBase only for test groups that require it (hbase, proxy) instead of
launching it as part of the default Hadoop stack. This frees RAM for all other
test groups.
Additionally, run HBase in standalone mode:
* No separate RegionServer process
* No separate ZooKeeper process
* No dependency on HDFS
---
.../pxf/automation/components/hbase/HBase.java | 20 ++++-
.../pxf/automation/features/hbase/HBaseTest.java | 48 +++++------
.../pxf/automation/smoke/HBaseSmokeTest.java | 99 ----------------------
automation/src/test/resources/sut/default.xml | 1 +
ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh | 18 +---
ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh | 2 +-
ci/docker/pxf-cbdb-dev/common/script/run_tests.sh | 21 +++++
ci/docker/pxf-cbdb-dev/common/script/utils.sh | 11 +--
ci/singlecluster/Dockerfile | 10 ---
ci/singlecluster/README.HDP3.md | 10 +--
ci/singlecluster/bin/gphd-env.sh | 17 ----
ci/singlecluster/bin/hbase-regionserver.sh | 79 -----------------
ci/singlecluster/bin/init-gphd.sh | 3 +-
ci/singlecluster/bin/start-gphd.sh | 1 -
ci/singlecluster/bin/start-hbase.sh | 32 +------
ci/singlecluster/bin/start-zookeeper.sh | 24 ------
ci/singlecluster/bin/stop-gphd.sh | 3 -
ci/singlecluster/bin/stop-hbase.sh | 31 +------
ci/singlecluster/bin/stop-zookeeper.sh | 10 ---
ci/singlecluster/conf/gphd-conf.sh | 3 +-
ci/singlecluster/templates/hbase/conf/hbase-env.sh | 2 +-
.../templates/hbase/conf/hbase-site.xml | 31 ++++---
ci/singlecluster/templates/zookeeper/conf/zoo.cfg | 26 ------
23 files changed, 97 insertions(+), 405 deletions(-)
diff --git
a/automation/src/main/java/org/apache/cloudberry/pxf/automation/components/hbase/HBase.java
b/automation/src/main/java/org/apache/cloudberry/pxf/automation/components/hbase/HBase.java
index c881cc07..26ac3471 100755
---
a/automation/src/main/java/org/apache/cloudberry/pxf/automation/components/hbase/HBase.java
+++
b/automation/src/main/java/org/apache/cloudberry/pxf/automation/components/hbase/HBase.java
@@ -16,6 +16,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
@@ -74,10 +75,10 @@ public class HBase extends BaseSystemObject implements
IDbFunctionality {
if (StringUtils.isNotEmpty(hbaseRoot)) {
config.addResource(new Path(getHbaseRoot() +
"/conf/hbase-site.xml"));
} else {
- config.set("hbase.rootdir", "hdfs://" + host + ":8020/hbase");
+ throw new RuntimeException("No path to hbase-site specified.
Please configure hbaseRoot property.");
}
- HBaseAdmin.checkHBaseAvailable(config);
+ waitHBaseAvailable(config);
connection = ConnectionFactory.createConnection(config);
admin = connection.getAdmin();
if (admin.getClusterStatus().getServersSize() == 0) {
@@ -90,6 +91,21 @@ public class HBase extends BaseSystemObject implements
IDbFunctionality {
ReportUtils.stopLevel(report);
}
+ private void waitHBaseAvailable(Configuration config) throws Exception {
+ int attemptsLeft = 50;
+ while (attemptsLeft > 0) {
+ try {
+ HBaseAdmin.checkHBaseAvailable(config);
+ return;
+ } catch (IOException e) {
+ Thread.sleep(500);
+ attemptsLeft--;
+ ReportUtils.report(report, getClass(), "HBase availability
probe failed: " + e.getMessage(), Reporter.WARNING);
+ }
+ }
+ throw new RuntimeException("HBase did not become available after 50
attempts (25s)");
+ }
+
@Override
public void close() {
if (admin != null) {
diff --git
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hbase/HBaseTest.java
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hbase/HBaseTest.java
index b8058a77..c27055d8 100755
---
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hbase/HBaseTest.java
+++
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/hbase/HBaseTest.java
@@ -152,7 +152,7 @@ public class HBaseTest extends BaseFeature {
*
* @throws Exception if test fails to run
*/
- @Test(groups = { "hbase", "features", "sanity", "gpdb" })
+ @Test(groups = { "hbase" })
public void sanity() throws Exception {
verifyFilterResults(hbaseTable, exTable, "", NO_FILTER, "sanity",
false);
@@ -164,7 +164,7 @@ public class HBaseTest extends BaseFeature {
* @throws Exception if test fails to run
*/
// TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = { "hbase", "features", "gpdb" })
+ @Test(enabled = false, groups = { "hbase" })
public void lowerFilter() throws Exception {
String whereClause = " WHERE \"cf1:q3\" < '00000030'";
@@ -178,7 +178,7 @@ public class HBaseTest extends BaseFeature {
* @throws Exception if test fails to run
*/
// TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = { "hbase", "features", "gpdb" })
+ @Test(enabled = false, groups = { "hbase" })
public void rangeFilter() throws Exception {
String whereClause = " WHERE \"cf1:q3\" > '00000090' AND \"cf1:q3\" <=
'00000103'";
@@ -192,7 +192,7 @@ public class HBaseTest extends BaseFeature {
* @throws Exception if test fails to run
*/
// TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = { "hbase", "features", "gpdb" })
+ @Test(enabled = false, groups = { "hbase" })
public void specificRowFilter() throws Exception {
String whereClause = " WHERE \"cf1:q3\" = 4";
@@ -206,7 +206,7 @@ public class HBaseTest extends BaseFeature {
* @throws Exception if test fails to run
*/
// TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = { "hbase", "features", "gpdb" })
+ @Test(enabled = false, groups = { "hbase" })
public void notEqualsFilter() throws Exception {
String whereClause = " WHERE \"cf1:q3\" != 30";
@@ -220,7 +220,7 @@ public class HBaseTest extends BaseFeature {
* @throws Exception if test fails to run
*/
// TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = { "hbase", "features", "gpdb" })
+ @Test(enabled = false, groups = { "hbase" })
public void rowkeyEqualsFilter() throws Exception {
String whereClause = " WHERE recordkey = '00000090'";
@@ -234,7 +234,7 @@ public class HBaseTest extends BaseFeature {
* @throws Exception if test fails to run
*/
// TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = { "hbase", "features", "gpdb" })
+ @Test(enabled = false, groups = { "hbase" })
public void rowkeyRangeFilter() throws Exception {
String whereClause = " WHERE recordkey > '00000090' AND recordkey <=
'00000103'";
@@ -248,7 +248,7 @@ public class HBaseTest extends BaseFeature {
* @throws Exception if test fails to run
*/
// TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = { "hbase", "features", "gpdb" })
+ @Test(enabled = false, groups = { "hbase" })
public void multipleQualifiersPushdownFilter() throws Exception {
String whereClause = " WHERE recordkey != '00000002' AND \"cf1:q3\" >
6 AND \"cf1:q8\" < 10 AND \"cf1:q9\" > 0";
@@ -263,7 +263,7 @@ public class HBaseTest extends BaseFeature {
*
* @throws Exception if test fails to run
*/
- @Test(groups = { "hbase", "features", "gpdb" })
+ @Test(groups = { "hbase" })
public void partialFilterPushdown() throws Exception {
String whereClause = " WHERE \"cf1:q3\" > 6 AND \"cf1:q7\" = '42'";
@@ -279,7 +279,7 @@ public class HBaseTest extends BaseFeature {
* @throws Exception if test fails to run
*/
// TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = { "hbase", "features", "gpdb" })
+ @Test(enabled = false, groups = { "hbase" })
public void textFilter() throws Exception {
String whereClause = " WHERE \"cf1:q2\" = 'UTF8_計算機用語_00000024'";
@@ -292,7 +292,7 @@ public class HBaseTest extends BaseFeature {
*
* @throws Exception if test fails to run
*/
- @Test(groups = { "hbase", "features", "gpdb" })
+ @Test(groups = { "hbase" })
public void doubleFilter() throws Exception {
String whereClause = " WHERE \"cf1:q5\" > 91.92 AND \"cf1:q6\" <=
99999999.99";
@@ -306,7 +306,7 @@ public class HBaseTest extends BaseFeature {
*
* @throws Exception if test fails to run
*/
- @Test(groups = { "hbase", "features", "gpdb" })
+ @Test(groups = { "hbase" })
public void orFilter() throws Exception {
String whereClause = " WHERE \"cf1:q3\" < 10 OR \"cf1:q5\" > 90";
@@ -323,7 +323,7 @@ public class HBaseTest extends BaseFeature {
*
* @throws Exception if test fails to run
*/
- @Test(groups = { "hbase", "features", "gpdb" })
+ @Test(groups = { "hbase" })
public void mixedFilterPushdownOrAnd() throws Exception {
String whereClause = " WHERE (\"cf1:q3\" < 10 OR \"cf1:q5\" > 90) AND
(\"cf1:q3\" > 5 AND \"cf1:q8\" < 30)";
@@ -344,7 +344,7 @@ public class HBaseTest extends BaseFeature {
*
* @throws Exception if test fails to run
*/
- @Test(groups = { "hbase", "features", "gpdb" })
+ @Test(groups = { "hbase" })
public void isNullFilter() throws Exception {
String whereClause = " WHERE \"cf1:q3\" is null";
@@ -358,7 +358,7 @@ public class HBaseTest extends BaseFeature {
*
* @throws Exception if test fails to run
*/
- @Test(groups = { "hbase", "features", "gpdb" })
+ @Test(groups = { "hbase" })
public void differentColumnNames() throws Exception {
exTableNullHBase = TableFactory.getPxfHBaseReadableTable(
@@ -377,7 +377,7 @@ public class HBaseTest extends BaseFeature {
*
* @throws Exception if test fails to run
*/
- @Test(groups = { "hbase", "features", "gpdb" })
+ @Test(groups = { "hbase" })
public void disableLookupTable() throws Exception {
try {
@@ -396,7 +396,7 @@ public class HBaseTest extends BaseFeature {
*
* @throws Exception if test fails to run
*/
- @Test(groups = { "hbase", "features", "gpdb" })
+ @Test(groups = { "hbase" })
public void noLookupTable() throws Exception {
try {
@@ -415,7 +415,7 @@ public class HBaseTest extends BaseFeature {
*
* @throws Exception if test fails to run
*/
- @Test(groups = { "hbase", "features", "gpdb" })
+ @Test(groups = { "hbase" })
public void removeColumnFromLookupTable() throws Exception {
try {
@@ -435,7 +435,7 @@ public class HBaseTest extends BaseFeature {
* @throws Exception if test fails to run
*/
// TODO: pxf_regress shows diff for this test. Should be fixed.
- @Test(enabled = false, groups = { "hbase", "features", "gpdb" })
+ @Test(enabled = false, groups = { "hbase" })
public void recordkeyAsInteger() throws Exception {
// create external table with record key as INTEGER
@@ -457,7 +457,7 @@ public class HBaseTest extends BaseFeature {
*
* @throws Exception if test fails to run
*/
- @Test(groups = { "hbase", "features", "gpdb" })
+ @Test(groups = { "hbase" })
public void notExistingHBaseTable() throws Exception {
ReadableExternalTable notExistsHBaseTableExtTable =
TableFactory.getPxfHBaseReadableTable(
@@ -474,7 +474,7 @@ public class HBaseTest extends BaseFeature {
*
* @throws Exception if test fails to run
*/
- @Test(groups = { "hbase", "features", "gpdb" })
+ @Test(groups = { "hbase" })
public void multiRegionsData() throws Exception {
HBaseTable multiDataHBaseTable = new HBaseTable(
@@ -513,7 +513,7 @@ public class HBaseTest extends BaseFeature {
*
* @throws Exception if test fails to run
*/
- @Test(groups = { "hbase", "features", "gpdb" })
+ @Test(groups = { "hbase" })
public void longHBaseQualifierNameNoLookupTable() throws Exception {
HBaseTable longQualifiersNamesHBaseTable = new HBaseTable(
@@ -559,7 +559,7 @@ public class HBaseTest extends BaseFeature {
*
* @throws Exception if test fails to run
*/
- @Test(groups = { "hbase", "features", "gpdb" })
+ @Test(groups = { "hbase" })
public void longHBaseQualifierNameUsingLookupTable() throws Exception {
HBaseTable longQualifiersNamesHBaseTable = new HBaseTable(
@@ -600,7 +600,7 @@ public class HBaseTest extends BaseFeature {
*
* @throws Exception if test fails to run
*/
- @Test(groups = { "hbase", "features", "gpdb" })
+ @Test(groups = { "hbase" })
public void emptyHBaseTable() throws Exception {
HBaseTable emptyTable = new HBaseTable("empty_table", new String[] {
"cf1" });
diff --git
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/smoke/HBaseSmokeTest.java
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/smoke/HBaseSmokeTest.java
deleted file mode 100755
index 7f3c2e94..00000000
---
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/smoke/HBaseSmokeTest.java
+++ /dev/null
@@ -1,99 +0,0 @@
-package org.apache.cloudberry.pxf.automation.smoke;
-
-import annotations.WorksWithFDW;
-import jsystem.framework.system.SystemManagerImpl;
-
-import
org.apache.cloudberry.pxf.automation.components.common.ShellSystemObject;
-import org.apache.cloudberry.pxf.automation.components.hbase.HBase;
-import org.apache.cloudberry.pxf.automation.structures.tables.basic.Table;
-import org.apache.cloudberry.pxf.automation.structures.tables.hbase.HBaseTable;
-import
org.apache.cloudberry.pxf.automation.structures.tables.hbase.LookupTable;
-import
org.apache.cloudberry.pxf.automation.structures.tables.utils.TableFactory;
-import org.testng.annotations.Test;
-
-import
org.apache.cloudberry.pxf.automation.datapreparer.hbase.HBaseSmokeDataPreparer;
-
-/** Basic PXF on HBase table filled with small data. */
-public class HBaseSmokeTest extends BaseSmoke {
- HBase hbase;
- Table dataTable;
- HBaseTable hbaseTable;
- LookupTable lookupTable;
-
- @Override
- public void beforeClass() throws Exception {
- // Initialize HBase System Object
- hbase = (HBase)
SystemManagerImpl.getInstance().getSystemObject("hbase");
-
- // if hbase authorization is not enabled then grant will not be
- // performed
- hbase.grantGlobalForUser("pxf");
- }
-
- @Override
- protected void prepareData() throws Exception {
- // Create HBase Table
- hbaseTable = new HBaseTable("hbase_table", new String[] { "col" });
- hbaseTable.setNumberOfSplits(0);
- hbaseTable.setRowsPerSplit(100);
- hbaseTable.setRowKeyPrefix("row");
- hbaseTable.setQualifiers(new String[] {
- "name",
- "number",
- "doub",
- "longnum",
- "bool" });
-
- hbase.createTableAndVerify(hbaseTable);
-
- // Prepare data for HBase table
- HBaseSmokeDataPreparer dataPreparer = new HBaseSmokeDataPreparer();
- dataPreparer.setColumnFamilyName(hbaseTable.getFields()[0]);
- dataPreparer.prepareData(hbaseTable.getRowsPerSplit(), hbaseTable);
-
- hbase.put(hbaseTable);
-
- Thread.sleep(ShellSystemObject._5_SECONDS);
- }
-
- @Override
- protected void createTables() throws Exception {
- // Create GPDB external table
- exTable = TableFactory.getPxfHBaseReadableTable("pxf_smoke_small_data",
- new String[] {
- "name text",
- "num integer",
- "dub double precision",
- "longnum bigint",
- "bool boolean" }, hbaseTable);
- exTable.setHost(pxfHost);
- exTable.setPort(pxfPort);
- gpdb.createTableAndVerify(exTable);
-
- // Create PXF Lookup HBase table
- lookupTable = new LookupTable();
- lookupTable.addMapping(hbaseTable.getName(), "name", "col:name");
- lookupTable.addMapping(hbaseTable.getName(), "num", "col:number");
- lookupTable.addMapping(hbaseTable.getName(), "dub", "col:doub");
- lookupTable.addMapping(hbaseTable.getName(), "longnum", "col:longnum");
- lookupTable.addMapping(hbaseTable.getName(), "bool", "col:bool");
-
- hbase.createTableAndVerify(lookupTable);
- hbase.put(lookupTable);
-
- Thread.sleep(ShellSystemObject._5_SECONDS);
- }
-
- @Override
- protected void queryResults() throws Exception {
- // Give HBase a moment to settle after table creation and data load
- Thread.sleep(ShellSystemObject._5_SECONDS);
- runSqlTest("smoke/small_data");
- }
-
- @Test(groups = { "smoke", "gpdb", "hbase" })
- @WorksWithFDW
- public void test() throws Exception {
- runTest();
- }
-}
diff --git a/automation/src/test/resources/sut/default.xml
b/automation/src/test/resources/sut/default.xml
index 7c9c4b68..31113e69 100644
--- a/automation/src/test/resources/sut/default.xml
+++ b/automation/src/test/resources/sut/default.xml
@@ -72,6 +72,7 @@
<hbase>
<class>org.apache.cloudberry.pxf.automation.components.hbase.HBase</class>
<host>localhost</host>
+ <hbaseRoot>/home/gpadmin/workspace/singlecluster/hbase</hbaseRoot>
</hbase>
<hive>
diff --git a/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh
b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh
index 832e5067..08e2c531 100755
--- a/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh
+++ b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh
@@ -32,7 +32,6 @@ source "${COMMON_SCRIPTS}/utils.sh"
HADOOP_ROOT=${GPHD_ROOT}/hadoop
HIVE_ROOT=${GPHD_ROOT}/hive
HBASE_ROOT=${GPHD_ROOT}/hbase
-ZOOKEEPER_ROOT=${GPHD_ROOT}/zookeeper
# --------------------------------------------------------------------
# OS detection: "deb" (Ubuntu/Debian) or "rpm" (Rocky/RHEL/CentOS)
@@ -447,12 +446,6 @@ prepare_hadoop_stack() {
source "${GPHD_ROOT}/bin/gphd-env.sh"
cd "${REPO_DIR}/automation"
make symlink_pxf_jars
- cp /home/gpadmin/automation_tmp_lib/pxf-hbase.jar "$GPHD_ROOT/hbase/lib/" ||
true
- # Ensure HBase sees PXF comparator classes even if automation_tmp_lib was
empty
- if [ ! -f "${GPHD_ROOT}/hbase/lib/pxf-hbase.jar" ]; then
- pxf_app=$(ls -1v /usr/local/pxf/application/pxf-app-*.jar | grep -v
'plain' | tail -n 1)
- unzip -qq -j "${pxf_app}" 'BOOT-INF/lib/pxf-hbase-*.jar' -d
"${GPHD_ROOT}/hbase/lib/"
- fi
# clean stale Hive locks and stop any leftover services to avoid start
failures
rm -f "${GPHD_ROOT}/storage/hive/metastore_db/"*.lck 2>/dev/null || true
rm -f "${GPHD_ROOT}/storage/pids"/hive-*.pid 2>/dev/null || true
@@ -468,20 +461,15 @@ prepare_hadoop_stack() {
log "initializing HDFS namenode..."
${GPHD_ROOT}/bin/init-gphd.sh 2>&1 || log "init-gphd.sh failed with exit
code $?"
fi
- log "starting HDFS/YARN/HBase via start-gphd.sh..."
+ # HBase is started on demand per test group, not as part of the default stack
+ export START_HBASE=false
+ log "starting HDFS/YARN via start-gphd.sh..."
if ! ${GPHD_ROOT}/bin/start-gphd.sh 2>&1; then
log "start-gphd.sh returned non-zero (services may already be running),
continue"
fi
# Wait for HDFS DataNode to be ready before proceeding; Tez upload in
# start_hive_services will fail if no DataNode is accepting writes.
wait_for_datanode
- if ! ${GPHD_ROOT}/bin/start-zookeeper.sh; then
- log "start-zookeeper.sh returned non-zero (may already be running)"
- fi
- # ensure HBase is up
- if ! ${GPHD_ROOT}/bin/start-hbase.sh; then
- log "start-hbase.sh returned non-zero (services may already be running),
continue"
- fi
start_hive_services
}
diff --git a/ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh
b/ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh
index 81b23556..e25f7bb2 100755
--- a/ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh
+++ b/ci/docker/pxf-cbdb-dev/common/script/pxf-env.sh
@@ -49,7 +49,7 @@ export PXF_HOME=${PXF_HOME:-/usr/local/pxf}
export PXF_BASE=${PXF_BASE:-/home/gpadmin/pxf-base}
export GPHD_ROOT=${GPHD_ROOT:-/home/gpadmin/workspace/singlecluster}
export GOPATH=${GOPATH:-/home/gpadmin/go}
-export
PATH="$GPHD_ROOT/bin:$GPHD_ROOT/hadoop/bin:$GPHD_ROOT/hive/bin:$GPHD_ROOT/hbase/bin:$GPHD_ROOT/zookeeper/bin:$JAVA_BUILD/bin:/usr/local/go/bin:$GOPATH/bin:$GPHOME/bin:$PXF_HOME/bin:$PATH"
+export
PATH="$GPHD_ROOT/bin:$GPHD_ROOT/hadoop/bin:$GPHD_ROOT/hive/bin:$GPHD_ROOT/hbase/bin:$JAVA_BUILD/bin:/usr/local/go/bin:$GOPATH/bin:$GPHOME/bin:$PXF_HOME/bin:$PATH"
export COMMON_JAVA_OPTS=${COMMON_JAVA_OPTS:-}
# --------------------------------------------------------------------
diff --git a/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
b/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
index 63b99352..40bd2bd0 100755
--- a/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
+++ b/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
@@ -111,6 +111,24 @@ cleanup_hive_state() {
hdfs dfs -rm -r -f /hive/warehouse/hive_small_data_orc >/dev/null 2>&1 ||
true
}
+start_hbase() {
+ echo "[run_tests] copying pxf-hbase.jar to HBase lib..."
+ cp /home/gpadmin/automation_tmp_lib/pxf-hbase.jar "${GPHD_ROOT}/hbase/lib/"
2>/dev/null || true
+ if [ ! -f "${GPHD_ROOT}/hbase/lib/pxf-hbase.jar" ]; then
+ pxf_app=$(ls -1v /usr/local/pxf/application/pxf-app-*.jar 2>/dev/null |
grep -v 'plain' | tail -n 1)
+ [ -n "${pxf_app}" ] && unzip -qq -j "${pxf_app}"
'BOOT-INF/lib/pxf-hbase-*.jar' -d "${GPHD_ROOT}/hbase/lib/" || true
+ fi
+ if pgrep -f HMaster >/dev/null 2>&1; then
+ echo "[run_tests] HBase HMaster already running, skipping start"
+ else
+ echo "[run_tests] starting HBase..."
+ "${GPHD_ROOT}/bin/start-hbase.sh"
+ fi
+ echo "[run_tests] waiting for HBase ZooKeeper on 127.0.0.1:2181..."
+ wait_port 127.0.0.1 2181 30 2 || { echo "[run_tests] ERROR: HBase ZooKeeper
did not become ready on 127.0.0.1:2181"; return 1; }
+ echo "[run_tests] HBase ZooKeeper is ready"
+}
+
cleanup_hbase_state() {
echo "disable 'pxflookup'; drop 'pxflookup';
disable 'hbase_table'; drop 'hbase_table';
@@ -445,6 +463,7 @@ base_test(){
save_test_reports "hive"
echo "[run_tests] GROUP=hive finished"
+ start_hbase
cleanup_hbase_state
make GROUP="hbase" || true
save_test_reports "hbase"
@@ -891,6 +910,7 @@ run_single_group() {
save_test_reports "hive"
;;
hbase)
+ start_hbase
cleanup_hbase_state
export PROTOCOL=
make GROUP="hbase"
@@ -924,6 +944,7 @@ run_single_group() {
performance_test
;;
proxy)
+ start_hbase
export PROTOCOL=
make GROUP="proxy"
save_test_reports "proxy"
diff --git a/ci/docker/pxf-cbdb-dev/common/script/utils.sh
b/ci/docker/pxf-cbdb-dev/common/script/utils.sh
index c055dd25..b6936ef5 100755
--- a/ci/docker/pxf-cbdb-dev/common/script/utils.sh
+++ b/ci/docker/pxf-cbdb-dev/common/script/utils.sh
@@ -52,14 +52,12 @@ check_hbase() {
local hbase_host="${HBASE_HOST:-$(hostname -I | awk '{print $1}')}"
hbase_host=${hbase_host:-127.0.0.1}
+ # In standalone mode HMaster embeds RegionServer and ZooKeeper in a single
JVM,
+ # so only the HMaster process exists — no separate HRegionServer process.
if ! echo "$jps_out" | grep -q HMaster && ! pgrep -f HMaster >/dev/null
2>&1; then
die "HBase HMaster not running"
fi
- if ! echo "$jps_out" | grep -q HRegionServer && ! pgrep -f HRegionServer
>/dev/null 2>&1; then
- die "HBase RegionServer not running"
- fi
-
local hbase_ok=true
if ! printf "status 'simple'\n" | "${HBASE_ROOT}/bin/hbase" shell -n
>/tmp/hbase_status.log 2>&1; then
hbase_ok=false
@@ -121,7 +119,7 @@ _detect_java8_default() {
}
health_check() {
- log "sanity check Hadoop/Hive/HBase/PXF"
+ log "sanity check HDFS/Hive/PXF"
GPHD_ROOT=${GPHD_ROOT:-/home/gpadmin/workspace/singlecluster}
HADOOP_ROOT=${HADOOP_ROOT:-${GPHD_ROOT}/hadoop}
HBASE_ROOT=${HBASE_ROOT:-${GPHD_ROOT}/hbase}
@@ -133,9 +131,8 @@ health_check() {
[ -f "${GPHD_ROOT}/bin/gphd-env.sh" ] && source
"${GPHD_ROOT}/bin/gphd-env.sh"
check_jvm_procs
- check_hbase
check_hdfs
check_hive
check_pxf
- log "all components healthy: HDFS/HBase/Hive/PXF"
+ log "all components healthy: HDFS/Hive/PXF"
}
diff --git a/ci/singlecluster/Dockerfile b/ci/singlecluster/Dockerfile
index a16c3e9f..11e561ad 100644
--- a/ci/singlecluster/Dockerfile
+++ b/ci/singlecluster/Dockerfile
@@ -39,7 +39,6 @@ RUN if command -v apt-get >/dev/null 2>&1; then \
# TODO: update hive to support java 11+
ENV HADOOP_VERSION=3.1.2
ENV HIVE_VERSION=3.1.3
-ENV ZOOKEEPER_VERSION=3.5.9
ENV HBASE_VERSION=2.3.7
ENV TEZ_VERSION=0.9.2
ENV GO_VERSION=1.24.0
@@ -47,7 +46,6 @@ ENV GO_VERSION=1.24.0
# checksums from archive.apache.org
ENV
HADOOP_SHA512="0e0ee817c89b3c4eb761eca7f16640742a83b0e99b6fda26c1bee2baabedad93aab86e252bf5f1e2381c6d464bc4003d10c7cc0f61b2062f4c59732ca24d1bd9"
ENV
HIVE_SHA256="0c9b6a6359a7341b6029cc9347435ee7b379f93846f779d710b13f795b54bb16"
-ENV
ZOOKEEPER_SHA512="0e5a64713abc6f36d961dd61a06f681868171a9d9228366e512a01324806d263e05508029c94d8e18307811867cdc39d848e736c252bf56c461273ef74c66a45"
ENV
HBASE_SHA512="1032521025660daa70260cdc931f52a26c87596be444451fe1fa88b526ede55e9d6b4220e91ff6f7422bec11f30d64fa6745e95a9c36971fdb1a264a2c745693"
ENV
TEZ_SHA512="a2d94bd9fa778d42a8bac9d9da8e263e469ddfef93968b06434716554995f490231de5607541ac236e770aa0158b64250c38bc1cd57dbfa629fea705f2ffa2f5"
@@ -58,7 +56,6 @@ ENV APACHE_MIRROR="repo.huaweicloud.com/apache"
ENV
HADOOP_URL="https://$APACHE_MIRROR/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz";
ENV
HIVE_URL="https://$APACHE_MIRROR/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz";
-ENV
ZOOKEEPER_URL="https://$APACHE_MIRROR/zookeeper/zookeeper-$ZOOKEEPER_VERSION/apache-zookeeper-$ZOOKEEPER_VERSION-bin.tar.gz";
ENV
HBASE_URL="https://$APACHE_MIRROR/hbase/$HBASE_VERSION/hbase-$HBASE_VERSION-bin.tar.gz";
ENV
TEZ_URL="https://$APACHE_MIRROR/tez/$TEZ_VERSION/apache-tez-$TEZ_VERSION-bin.tar.gz";
@@ -66,7 +63,6 @@ ENV GPHD_ROOT=/home/gpadmin/workspace/singlecluster
ENV HADOOP_ROOT=$GPHD_ROOT/hadoop
ENV HBASE_ROOT=$GPHD_ROOT/hbase
ENV HIVE_ROOT=$GPHD_ROOT/hive
-ENV ZOOKEEPER_ROOT=$GPHD_ROOT/zookeeper
ENV TEZ_ROOT=$GPHD_ROOT/tez
RUN mkdir -p $HADOOP_ROOT && \
@@ -83,12 +79,6 @@ RUN mkdir -p $HIVE_ROOT && \
tar xvf /tmp/hive.tar.gz -C $HIVE_ROOT --strip-components 1 && \
rm /tmp/hive.tar.gz
-RUN mkdir -p $ZOOKEEPER_ROOT && \
- curl -fSL $ZOOKEEPER_URL -o /tmp/zookeeper.tar.gz && \
- echo "$ZOOKEEPER_SHA512 /tmp/zookeeper.tar.gz" | sha512sum -c && \
- tar xvf /tmp/zookeeper.tar.gz -C $ZOOKEEPER_ROOT --strip-components 1
--exclude="docs/*" && \
- rm /tmp/zookeeper.tar.gz
-
RUN mkdir -p $HBASE_ROOT && \
curl -fSL "$HBASE_URL" -o /tmp/hbase.tar.gz && \
echo "$HBASE_SHA512 /tmp/hbase.tar.gz" | sha512sum -c && \
diff --git a/ci/singlecluster/README.HDP3.md b/ci/singlecluster/README.HDP3.md
index 3a906e50..4afd6cea 100644
--- a/ci/singlecluster/README.HDP3.md
+++ b/ci/singlecluster/README.HDP3.md
@@ -6,7 +6,6 @@ It contains the following versions:
- Hadoop 3.3.6
- Hive 3.1.3
-- Zookeeper 3.5.9
- HBase 2.3.7
- Tez 0.9.2
@@ -40,8 +39,7 @@ Initialization
export HADOOP_ROOT=$GPHD_ROOT/hadoop
export HBASE_ROOT=$GPHD_ROOT/hbase
export HIVE_ROOT=$GPHD_ROOT/hive
- export ZOOKEEPER_ROOT=$GPHD_ROOT/zookeeper
- export
PATH=$PATH:$GPHD_ROOT/bin:$HADOOP_ROOT/bin:$HBASE_ROOT/bin:$HIVE_ROOT/bin:$ZOOKEEPER_ROOT/bin
+ export
PATH=$PATH:$GPHD_ROOT/bin:$HADOOP_ROOT/bin:$HBASE_ROOT/bin:$HIVE_ROOT/bin
```
Usage
@@ -53,10 +51,8 @@ Usage
- `$GPHD_ROOT/bin/start-hdfs.sh`
- Start PXF only (Install pxf first to make this work. [See Install PXF
session
here](https://cwiki.apache.org/confluence/display/HAWQ/PXF+Build+and+Install))
- `$GPHD_ROOT/bin/start-pxf.sh`
-- Start HBase only (requires hdfs and zookeeper)
+- Start HBase only (standalone mode)
- `$GPHD_ROOT/bin/start-hbase.sh`
-- Start ZooKeeper only
- - `$GPHD_ROOT/bin/start-zookeeper.sh`
- Start YARN only
- `$GPHD_ROOT/bin/start-yarn.sh`
- Start Hive (MetaStore)
@@ -64,7 +60,7 @@ Usage
- Stop all PHD services
- `$GPHD_ROOT/bin/stop-gphd.sh`
- Stop an individual component
- - `$GPHD_ROOT/bin/stop-[hdfs|pxf|hbase|zookeeper|yarn|hive].sh`
+ - `$GPHD_ROOT/bin/stop-[hdfs|pxf|hbase|yarn|hive].sh`
- Start/stop HiveServer2
- `$GPHD_ROOT/bin/hive-service.sh hiveserver2 start`
- `$GPHD_ROOT/bin/hive-service.sh hiveserver2 stop`
diff --git a/ci/singlecluster/bin/gphd-env.sh b/ci/singlecluster/bin/gphd-env.sh
index 418f3cdc..ea528361 100755
--- a/ci/singlecluster/bin/gphd-env.sh
+++ b/ci/singlecluster/bin/gphd-env.sh
@@ -27,7 +27,6 @@ fi
# Some basic definitions
export HADOOP_ROOT=${GPHD_ROOT}/hadoop
export HBASE_ROOT=${GPHD_ROOT}/hbase
-export ZOOKEEPER_ROOT=${GPHD_ROOT}/zookeeper
export HIVE_ROOT=${GPHD_ROOT}/hive
export TEZ_ROOT=${GPHD_ROOT}/tez
export RANGER_ROOT=${GPHD_ROOT}/ranger
@@ -37,12 +36,10 @@ export PIDS_ROOT=${STORAGE_ROOT}/pids
export HADOOP_BIN=${HADOOP_ROOT}/bin
export HADOOP_SBIN=${HADOOP_ROOT}/sbin
-export ZOOKEEPER_BIN=${ZOOKEEPER_ROOT}/bin
export HBASE_BIN=${HBASE_ROOT}/bin
export HIVE_BIN=${HIVE_ROOT}/bin
export HADOOP_CONF=${HADOOP_ROOT}/etc/hadoop
-export ZOOKEEPER_CONF=${ZOOKEEPER_ROOT}/conf
export HBASE_HOME=${HBASE_ROOT}
export HBASE_CONF=${HBASE_ROOT}/conf
export HIVE_CONF=${HIVE_ROOT}/conf
@@ -80,17 +77,3 @@ function hdfs_running()
`${bin}/hdfs dfsadmin -Dipc.client.connect.max.retries.on.timeouts=0
-safemode get 2>&1 | grep -q "Safe mode is OFF"`
return $?
}
-
-function zookeeper_running()
-{
- local retval=1
- for i in {1..10}; do
- sleep 5s
- ${ZOOKEEPER_BIN}/zkServer.sh status > /dev/null 2>&1
- retval=$?
- if [ ${retval} -eq 0 ]; then
- return 0
- fi
- done
- return ${retval}
-}
diff --git a/ci/singlecluster/bin/hbase-regionserver.sh
b/ci/singlecluster/bin/hbase-regionserver.sh
deleted file mode 100755
index 10ce3c5d..00000000
--- a/ci/singlecluster/bin/hbase-regionserver.sh
+++ /dev/null
@@ -1,79 +0,0 @@
-#!/usr/bin/env bash
-
-usage="Usage: `basename $0` <start|stop> <node_id>"
-
-if [ $# -ne 2 ]; then
- echo ${usage}
- exit 1
-fi
-
-command=$1
-nodeid=$2
-
-# Load settings
-root=`cd \`dirname $0\`/..;pwd`
-bin=${root}/bin
-. ${bin}/gphd-env.sh
-
-hbase_root=${HBASE_ROOT}
-hbase_conf=${HBASE_CONF}
-hbase_bin=${HBASE_BIN}
-
-regionserver_root=${HBASE_STORAGE_ROOT}/regionserver$nodeid
-regionserver_conf=${regionserver_root}/conf
-
-export HBASE_REGIONSERVER_OPTS="-Dhbase.tmp.dir=$regionserver_root/tmp"
-export HBASE_CONF_DIR=${regionserver_conf}
-export HBASE_IDENT_STRING=${USER}-node${nodeid}
-
-function clear_conf_directory()
-{
- if [ -d ${regionserver_conf} ]; then
- rm -rf ${regionserver_conf}
- fi
- mkdir -p ${regionserver_conf}
-}
-
-function setup_hbasesite()
-{
- cat ${hbase_conf}/hbase-site.xml | \
- sed "/^<configuration>$/ a\\
- <property>\\
- <name>hbase.regionserver.port</name>\\
- <value>6002$nodeid</value>\\
- </property>\\
- <property>\\
- <name>hbase.regionserver.info.port</name>\\
- <value>6003$nodeid</value>\\
- </property>\\
- " > ${regionserver_conf}/hbase-site.xml
-}
-
-function copy_conf_files()
-{
- for file in $(find ${hbase_conf} -type f -not -iname "*~"); do
- cp ${file} ${regionserver_conf}/$(basename ${file})
- done
-}
-
-function handle_start()
-{
- clear_conf_directory
- copy_conf_files
- setup_hbasesite
-}
-
-case "$command" in
- "start" )
- handle_start
- ;;
- "stop" )
- ;;
- * )
- echo unknown command "$command"
- echo ${usage}
- exit 1
- ;;
-esac
-
-${hbase_bin}/hbase-daemon.sh --config ${regionserver_conf} ${command}
regionserver
diff --git a/ci/singlecluster/bin/init-gphd.sh
b/ci/singlecluster/bin/init-gphd.sh
index 2d7c4f0d..d41f93a0 100755
--- a/ci/singlecluster/bin/init-gphd.sh
+++ b/ci/singlecluster/bin/init-gphd.sh
@@ -42,6 +42,5 @@ echo export GPHD_ROOT=${GPHD_ROOT}
echo export HADOOP_ROOT=\$GPHD_ROOT/hadoop
echo export HBASE_ROOT=\$GPHD_ROOT/hbase
echo export HIVE_ROOT=\$GPHD_ROOT/hive
-echo export ZOOKEEPER_ROOT=\$GPHD_ROOT/zookeeper
-echo export
PATH=\$PATH:\$GPHD_ROOT/bin:\$HADOOP_ROOT/bin:\$HBASE_ROOT/bin:\$HIVE_ROOT/bin:\$ZOOKEEPER_ROOT/bin
+echo export
PATH=\$PATH:\$GPHD_ROOT/bin:\$HADOOP_ROOT/bin:\$HBASE_ROOT/bin:\$HIVE_ROOT/bin
echo -------------------------------------------------------------
diff --git a/ci/singlecluster/bin/start-gphd.sh
b/ci/singlecluster/bin/start-gphd.sh
index 3fe07e85..460b851e 100755
--- a/ci/singlecluster/bin/start-gphd.sh
+++ b/ci/singlecluster/bin/start-gphd.sh
@@ -30,5 +30,4 @@ if [ "$START_HBASE" != "true" ]; then
exit 0
fi
-${bin}/start-zookeeper.sh || exit 1
${bin}/start-hbase.sh || exit 1
diff --git a/ci/singlecluster/bin/start-hbase.sh
b/ci/singlecluster/bin/start-hbase.sh
index 7d238f95..6b8d14ec 100755
--- a/ci/singlecluster/bin/start-hbase.sh
+++ b/ci/singlecluster/bin/start-hbase.sh
@@ -5,33 +5,5 @@ root=`cd \`dirname $0\`/..;pwd`
bin=${root}/bin
. ${bin}/gphd-env.sh
-# Check to see HDFS is up
-hdfs_running
-if [ $? != 0 ]; then
- echo HDFS is not ready, HBase cannot start
- echo Please see HDFS is up and out of safemode
- exit 1
-fi
-
-# Check to see Zookeeper is up
-zookeeper_running
-if [ $? != 0 ]; then
- echo Zookeeper is not running, HBase cannot start
- echo Have you start-zookeeper.sh ?
- exit 1
-fi
-
-echo Starting HBase...
-# Start master
-${HBASE_BIN}/hbase-daemon.sh --config ${HBASE_CONF} start master
-
-# Start regions
-for (( i=0; i < $SLAVES; i++ )); do
- ${bin}/hbase-regionserver.sh start ${i} | sed "s/^/node $i: /"
-done
-
-# Start Stargate
-if [ "$START_STARGATE" == "true" ]; then
- echo Starting Stargate...
- ${HBASE_BIN}/hbase-daemon.sh --config ${HBASE_CONF} start rest -p
${STARGATE_PORT}
-fi
+echo Starting HBase standalone...
+${HBASE_ROOT}/bin/start-hbase.sh
\ No newline at end of file
diff --git a/ci/singlecluster/bin/start-zookeeper.sh
b/ci/singlecluster/bin/start-zookeeper.sh
deleted file mode 100755
index 673b90b9..00000000
--- a/ci/singlecluster/bin/start-zookeeper.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-
-# Load settings
-root=`cd \`dirname $0\`/..;pwd`
-bin=${root}/bin
-. ${bin}/gphd-env.sh
-
-zookeeper_cfg=$ZOOKEEPER_CONF/zoo.cfg
-zookeeper_cfg_tmp=$zookeeper_cfg.preped
-
-sudo chown -R gpadmin:gpadmin "${ZOOKEEPER_CONF}"
-
-sed "s|dataDir.*$|dataDir=$ZOOKEEPER_STORAGE_ROOT|" ${zookeeper_cfg} >
${zookeeper_cfg_tmp}
-rm -f ${zookeeper_cfg}
-mv ${zookeeper_cfg_tmp} ${zookeeper_cfg}
-
-mkdir -p ${ZOOKEEPER_STORAGE_ROOT}
-
-export ZOO_LOG_DIR=${LOGS_ROOT}
-
-echo Starting Zookeeper...
-pushd ${ZOOKEEPER_ROOT} > /dev/null
-bin/zkServer.sh start
-popd > /dev/null
diff --git a/ci/singlecluster/bin/stop-gphd.sh
b/ci/singlecluster/bin/stop-gphd.sh
index 0adb04db..02934632 100755
--- a/ci/singlecluster/bin/stop-gphd.sh
+++ b/ci/singlecluster/bin/stop-gphd.sh
@@ -8,9 +8,6 @@ bin=${root}/bin
if [ "$START_HBASE" == "true" ]; then
echo Stopping HBase...
${bin}/stop-hbase.sh
-
- echo Stopping Zookeeper...
- ${bin}/stop-zookeeper.sh
fi
if [ "$START_PXF" == "true" ]; then
diff --git a/ci/singlecluster/bin/stop-hbase.sh
b/ci/singlecluster/bin/stop-hbase.sh
index ad12c144..97204e33 100755
--- a/ci/singlecluster/bin/stop-hbase.sh
+++ b/ci/singlecluster/bin/stop-hbase.sh
@@ -7,32 +7,5 @@ bin=${root}/bin
# TODO cleanup after hbase?
-if [ "$START_STARGATE" == "true" ]; then
- echo Stopping Stargate...
- ${HBASE_BIN}/hbase-daemon.sh --config ${HBASE_CONF} stop rest
-fi
-
-# stop region servers
-for (( i=0; i < ${SLAVES}; i++ )); do
- ${bin}/hbase-regionserver.sh stop ${i} | sed "s/^/node $i: /"
-done
-
-# stop master
-pid=`echo ${PIDS_ROOT}/hbase-*-master.pid`
-hbase_pid=`cat ${pid}`
-command=master
-hbase_stop_timeout=9
-
-${bin}/hbase ${command} stop > /dev/null 2>&1 &
-
-if kill -0 ${hbase_pid} > /dev/null 2>&1; then
- echo stopping ${command}
- kill ${hbase_pid}
- sleep ${hbase_stop_timeout}
- if kill -0 ${hbase_pid} > /dev/null 2>&1; then
- echo "$command did not stop gracefully after
$hbase_stop_timeout seconds: killing with kill -9"
- kill -9 ${hbase_pid}
- fi
-else
- echo no ${command} to stop
-fi
+echo Stopping HBase standalone...
+${HBASE_ROOT}/bin/stop-hbase.sh
\ No newline at end of file
diff --git a/ci/singlecluster/bin/stop-zookeeper.sh
b/ci/singlecluster/bin/stop-zookeeper.sh
deleted file mode 100755
index 6970438b..00000000
--- a/ci/singlecluster/bin/stop-zookeeper.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-# Load settings
-root=`cd \`dirname $0\`/..;pwd`
-bin=${root}/bin
-. ${bin}/gphd-env.sh
-
-pushd ${ZOOKEEPER_ROOT} > /dev/null
-bin/zkServer.sh stop
-popd > /dev/null
diff --git a/ci/singlecluster/conf/gphd-conf.sh
b/ci/singlecluster/conf/gphd-conf.sh
index fc35cb8c..178728c9 100755
--- a/ci/singlecluster/conf/gphd-conf.sh
+++ b/ci/singlecluster/conf/gphd-conf.sh
@@ -16,7 +16,6 @@ if [ -z "${JAVA_HOME:-}" ]; then
fi
export STORAGE_ROOT=$GPHD_ROOT/storage
export HADOOP_STORAGE_ROOT=$STORAGE_ROOT/hadoop
-export ZOOKEEPER_STORAGE_ROOT=$STORAGE_ROOT/zookeeper
export HBASE_STORAGE_ROOT=$STORAGE_ROOT/hbase
export HIVE_STORAGE_ROOT=$STORAGE_ROOT/hive
export PXF_STORAGE_ROOT=$STORAGE_ROOT/pxf
@@ -26,7 +25,7 @@ export RANGER_STORAGE_ROOT=$STORAGE_ROOT/ranger
export SLAVES=${SLAVES:-1}
# Automatically start HBase during GPHD startup
-export START_HBASE=true
+export START_HBASE=false
# Automatically start Stargate during HBase startup
export START_STARGATE=false
diff --git a/ci/singlecluster/templates/hbase/conf/hbase-env.sh
b/ci/singlecluster/templates/hbase/conf/hbase-env.sh
index 36f3aadf..f6b292dd 100755
--- a/ci/singlecluster/templates/hbase/conf/hbase-env.sh
+++ b/ci/singlecluster/templates/hbase/conf/hbase-env.sh
@@ -91,7 +91,7 @@ export HBASE_PID_DIR=$PIDS_ROOT
# export HBASE_SLAVE_SLEEP=0.1
# Tell HBase whether it should manage it's own instance of Zookeeper or not.
-export HBASE_MANAGES_ZK=false
+export HBASE_MANAGES_ZK=true
# Prefer JAVA_HOME from gphd-env.sh; fail fast if missing to avoid divergent
per-service detection.
if [ -z "${JAVA_HOME:-}" ]; then
diff --git a/ci/singlecluster/templates/hbase/conf/hbase-site.xml
b/ci/singlecluster/templates/hbase/conf/hbase-site.xml
index 00841290..dad64bb3 100755
--- a/ci/singlecluster/templates/hbase/conf/hbase-site.xml
+++ b/ci/singlecluster/templates/hbase/conf/hbase-site.xml
@@ -24,28 +24,27 @@
<configuration>
<property>
<name>hbase.rootdir</name>
- <value>hdfs://0.0.0.0:8020/hbase</value>
+
<value>file:///home/gpadmin/workspace/singlecluster/storage/hbase</value>
</property>
<property>
- <name>dfs.replication</name>
- <value>3</value>
+ <name>hbase.zookeeper.property.dataDir</name>
+ <value>/home/gpadmin/workspace/singlecluster/storage/hbase-zk</value>
</property>
<property>
- <name>dfs.support.append</name>
- <value>true</value>
+ <name>hbase.cluster.distributed</name>
+ <value>false</value>
</property>
<property>
- <name>hbase.cluster.distributed</name>
- <value>true</value>
+ <!-- Explicit use IPv4 address instead of "localhost" to prevent Java
from
+ resolving it to ::1 (IPv6) first, which causes ConnectionLoss
errors
+ when the embedded ZooKeeper only listens on 127.0.0.1. -->
+ <name>hbase.zookeeper.quorum</name>
+ <value>127.0.0.1</value>
+ </property>
+ <property>
+ <name>hbase.zookeeper.property.clientPort</name>
+ <value>2181</value>
</property>
- <property>
- <name>hbase.zookeeper.quorum</name>
- <value>127.0.0.1</value>
- </property>
- <property>
- <name>hbase.zookeeper.property.clientPort</name>
- <value>2181</value>
- </property>
<property>
<name>hadoop.proxyuser.gpadmin.hosts</name>
<value>*</value>
@@ -89,4 +88,4 @@
<name>hbase.unsafe.stream.capability.enforce</name>
<value>false</value>
</property>
-</configuration>
+</configuration>
\ No newline at end of file
diff --git a/ci/singlecluster/templates/zookeeper/conf/zoo.cfg
b/ci/singlecluster/templates/zookeeper/conf/zoo.cfg
deleted file mode 100755
index bd8dc90a..00000000
--- a/ci/singlecluster/templates/zookeeper/conf/zoo.cfg
+++ /dev/null
@@ -1,26 +0,0 @@
-# The number of milliseconds of each tick
-tickTime=2000
-# The number of ticks that the initial
-# synchronization phase can take
-initLimit=10
-# The number of ticks that can pass between
-# sending a request and getting an acknowledgement
-syncLimit=5
-# the directory where the snapshot is stored.
-# do not use /tmp for storage, /tmp here is just
-# example sakes.
-# automatically updated by GPHD-sc scripts
-dataDir=/tmp/zookeeper
-# the port at which the clients will connect
-clientPort=2181
-#
-# Be sure to read the maintenance section of the
-# administrator guide before turning on autopurge.
-#
-# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
-#
-# The number of snapshots to retain in dataDir
-#autopurge.snapRetainCount=3
-# Purge task interval in hours
-# Set to "0" to disable auto purge feature
-#autopurge.purgeInterval=1
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
